Gitlab Community Edition Instance

Commit d2840109 authored by mhellka's avatar mhellka
Browse files

Faster digest calculation for whole files.

By increasing (virtaul) buffer size to 2^31, IO overhead can be reduced by ~20% in some tests.
parent f3fcb40b
package de.gwdg.cdstar.pool.nio;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.nio.channels.ClosedChannelException;
import java.nio.channels.FileChannel;
import java.nio.channels.ReadableByteChannel;
import java.nio.channels.WritableByteChannel;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.Collection;
......@@ -141,9 +139,7 @@ class ResouceWriteChannel implements WritableByteChannel {
missing.removeAll(digestCache.keySet());
if(!missing.isEmpty()) {
NioResource.log.debug("Calculating partial digest after clone ({} bytes)", getSize());
try (InputStream src = Files.newInputStream(path)) {
digestCache.putAll(new MultiDigest(missing).update(src).digest());
}
digestCache.putAll(new MultiDigest(missing).update(path).digest());
}
}
......@@ -166,9 +162,7 @@ class ResouceWriteChannel implements WritableByteChannel {
if (rollingDigest == null) {
NioResource.log.debug("Calculating full digest ({} bytes, non-rolling)", getSize());
rollingDigest = new MultiDigest(digestAlgos);
try (InputStream src = Files.newInputStream(path)) {
rollingDigest.update(src);
}
rollingDigest.update(path);
}
digestCache = rollingDigest.digest();
rollingDigest = null;
......
......@@ -4,6 +4,10 @@ import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.FileChannel.MapMode;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
......@@ -95,6 +99,15 @@ public class MultiDigest {
return this;
}
public MultiDigest update(Path path) throws IOException {
try (FileChannel fc = FileChannel.open(path, StandardOpenOption.READ)) {
for(long i=0; i<fc.size(); i+=Integer.MAX_VALUE) {
update(fc.map(MapMode.READ_ONLY, i, Math.min(fc.size()-i, Integer.MAX_VALUE)));
}
}
return this;
}
public InputStream wrap(InputStream stream) {
return new FilterInputStream(stream) {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment