Gitlab Community Edition Instance

Commit 883feeba authored by mhellka's avatar mhellka
Browse files

tus: Improved tus.io protocoll support.

- Advertise creation-defer-length support
- Store upload metadata in separate file
- Pick up old uploads on restart
- Expire uploads 24h after creation
- Implement tus.io protocol more strictly
parent a01ae50d
Pipeline #291696 passed with stages
in 9 minutes and 31 seconds
package de.gwdg.cdstar.rest.ext.tus; package de.gwdg.cdstar.rest.ext.tus;
import java.io.IOException; import java.io.IOException;
import java.nio.channels.FileChannel;
import java.nio.file.StandardOpenOption;
import java.time.Instant;
import java.util.Date; import java.util.Date;
import de.gwdg.cdstar.Promise;
import de.gwdg.cdstar.Utils; import de.gwdg.cdstar.Utils;
import de.gwdg.cdstar.rest.api.Blueprint; import de.gwdg.cdstar.rest.api.Blueprint;
import de.gwdg.cdstar.rest.api.RestBlueprint; import de.gwdg.cdstar.rest.api.RestBlueprint;
...@@ -23,15 +19,15 @@ public class TusBlueprint implements RestBlueprint { ...@@ -23,15 +19,15 @@ public class TusBlueprint implements RestBlueprint {
private static final String TUS_UPLOAD_METADATA = "Upload-Metadata"; private static final String TUS_UPLOAD_METADATA = "Upload-Metadata";
private static final String TUS_UPLOAD_EXPIRES = "Upload-Expires"; private static final String TUS_UPLOAD_EXPIRES = "Upload-Expires";
private static final String TUS_UPLOAD_LENGTH = "Upload-Length"; private static final String TUS_UPLOAD_LENGTH = "Upload-Length";
private static final String TUS_UPLOAD_DEFER_LENGTH = "Upload-Defer-Length";
private static final String TUS_UPLOAD_OFFSET = "Upload-Offset"; private static final String TUS_UPLOAD_OFFSET = "Upload-Offset";
private static final String TUS_EXTENSION = "Tus-Extension"; private static final String TUS_EXTENSION = "Tus-Extension";
private static final String TUS_RESUMABLE = "Tus-Resumable"; private static final String TUS_RESUMABLE = "Tus-Resumable";
private static final String TUS_EXTENSIONS = "creation,expiration,termination"; private static final String TUS_EXTENSIONS = "creation,creation-defer-length,expiration,termination";
private static final String TUS_VERSION = "1.0.0"; private static final String TUS_VERSION = "1.0.0";
private final TusService chunkService; private final TusService chunkService;
public TusBlueprint(TusService service) { public TusBlueprint(TusService service) {
chunkService = service; chunkService = service;
} }
...@@ -48,28 +44,6 @@ public class TusBlueprint implements RestBlueprint { ...@@ -48,28 +44,6 @@ public class TusBlueprint implements RestBlueprint {
.HEAD(this::handleHead); .HEAD(this::handleHead);
} }
public Void handleCreate(RestContext ctx) {
checkTusHeader(ctx);
// TODO: Access control, DDOS protection
// TODO: Handle Upload-Defer-Length, creation-defer-length extension
final long length = getUploadLength(ctx);
final String meta = getUploadMetadata(ctx);
final TusFile chunk = chunkService.createChunk();
chunk.setLength(length);
chunk.setMeta(meta);
ctx.status(201);
ctx.header(TUS_RESUMABLE, TUS_VERSION);
ctx.header("Location", ctx.resolvePath(chunk.getName(), true));
if (chunk.getExpireMillis() > 0)
ctx.header(TUS_UPLOAD_EXPIRES, Date.from(Instant.ofEpochMilli(chunk.getExpireMillis())));
return null;
}
public Void handleOptions(RestContext ctx) throws IOException { public Void handleOptions(RestContext ctx) throws IOException {
// The Client SHOULD NOT include the Tus-Resumable header in the request and the // The Client SHOULD NOT include the Tus-Resumable header in the request and the
// Server MUST ignore the header. // Server MUST ignore the header.
...@@ -81,130 +55,145 @@ public class TusBlueprint implements RestBlueprint { ...@@ -81,130 +55,145 @@ public class TusBlueprint implements RestBlueprint {
return null; return null;
} }
public Void handleHead(RestContext ctx) throws IOException { public Void handleCreate(RestContext ctx) throws IOException {
checkTusHeader(ctx); checkTusRequestVersion(ctx);
final TusFile chunk = getChunk(ctx.getPathParam("chunk"));
ctx.status(200); // TODO: Access control, DDOS protection
ctx.header("Cache-Control", "no-store");
long length = "1".equals(ctx.getHeader(TUS_UPLOAD_DEFER_LENGTH))
? -1
: requireLongHeader(ctx, TUS_UPLOAD_LENGTH);
var meta = ctx.getHeader(TUS_UPLOAD_METADATA);
TusUpload upload = chunkService.createNewUpload(length, meta);
ctx.status(201);
ctx.header(TUS_RESUMABLE, TUS_VERSION); ctx.header(TUS_RESUMABLE, TUS_VERSION);
ctx.header(TUS_UPLOAD_OFFSET, chunk.getActualSize()); ctx.header(TUS_UPLOAD_EXPIRES, Date.from(upload.getExpiresAt()));
if (chunk.getLength() > -1) ctx.header("Location", ctx.resolvePath(upload.getName(), true));
ctx.header(TUS_UPLOAD_LENGTH, chunk.getLength());
if (chunk.getExpireMillis() > 0)
ctx.header(TUS_UPLOAD_EXPIRES, Date.from(Instant.ofEpochMilli(chunk.getExpireMillis())));
return null; return null;
} }
public Void handleDelete(RestContext ctx) { public Void handleHead(RestContext ctx) throws IOException {
checkTusHeader(ctx); checkTusRequestVersion(ctx);
final TusFile chunk = getChunk(ctx.getPathParam("chunk"));
if (!chunk.tryLock())
throw new ErrorResponse(409, "Conflict", "Failed to open chunk: Resource locked.");
chunkService.removeChunk(chunk); final TusUpload upload = getUpload(ctx.getPathParam("chunk"));
ctx.status(204); ctx.status(200);
ctx.header(TUS_RESUMABLE, TUS_VERSION); addResponseHeaders(ctx, upload);
return null; return null;
} }
public Void handlePatch(RestContext ctx) throws IOException { public Void handlePatch(RestContext ctx) throws IOException {
checkTusHeader(ctx); checkTusRequestVersion(ctx);
final TusFile chunk = getChunk(ctx.getPathParam("chunk"));
ctx.status(204); final TusUpload upload = getUpload(ctx.getPathParam("chunk"));
ctx.header(TUS_RESUMABLE, TUS_VERSION);
if (chunk.getLength() > -1)
ctx.header(TUS_UPLOAD_LENGTH, chunk.getLength());
if (chunk.getExpireMillis() > 0)
ctx.header(TUS_UPLOAD_EXPIRES, Date.from(Instant.ofEpochMilli(chunk.getExpireMillis())));
if (!Utils.equal(ctx.getHeader("Content-Type"), "application/offset+octet-stream")) if (!Utils.equal(ctx.getHeader("Content-Type"), "application/offset+octet-stream"))
throw new ErrorResponse(406, "TusError", "Content type MUST be 'application/offset+octet-stream'."); throw new ErrorResponse(406, "TusError", "Content type MUST be 'application/offset+octet-stream'.");
final long offset = getUploadOffset(ctx); if (!upload.tryLock())
final long length = getUploadLength(ctx); throw new ErrorResponse(409, "Conflict", "Failed to open upload: Resource locked.");
if (!chunk.tryLock()) try {
throw new ErrorResponse(409, "Conflict", "Failed to open chunk: Resource locked."); if (requireLongHeader(ctx, TUS_UPLOAD_OFFSET) != upload.getCurrentOffset())
throw new ErrorResponse(409, "TusError", "Upload-Offset header does not match current upload size.");
if (upload.isComplete())
throw new ErrorResponse(409, "TusError", "Upload complete.");
if (ctx.getHeader(TUS_UPLOAD_LENGTH) != null) {
long length = requireLongHeader(ctx, TUS_UPLOAD_LENGTH);
if (!upload.hasLength()) {
upload.setLength(length);
upload.persistInfo();
} else if (length != upload.getLength()) {
throw new ErrorResponse(400, "TusError", "Cannot change Upload-Length once it has been set.");
}
}
if (offset != chunk.getActualSize()) { // TODO: Limit upload size if target length is known
chunk.unlock(); var channel = upload.getWriteChannel();
throw new ErrorResponse(409, "TusError", "Upload-Offset header does not match current file size."); new AsyncUpload(ctx, channel).dispatch().whenComplete((totalUpload, err) -> {
} Utils.closeQuietly(channel);
upload.unlock();
if (length > -1) { if (err != null) {
if (chunk.getLength() == -1) { ctx.abort(err);
chunk.setLength(length); return;
ctx.header(TUS_UPLOAD_LENGTH, chunk.getLength()); }
} else if (length != chunk.getLength()) {
chunk.unlock();
throw new ErrorResponse(400, "TusError", "Cannot change Upload-Length once it has a value.");
}
}
try { ctx.status(204);
final FileChannel ch = FileChannel.open(chunk.getPath(), StandardOpenOption.CREATE, addResponseHeaders(ctx, upload);
StandardOpenOption.WRITE, StandardOpenOption.APPEND); ctx.close();
});
Promise.wrap(new AsyncUpload(ctx, ch).dispatch())
.then(r -> {
Utils.closeQuietly(ch);
chunk.unlock();
ctx.header(TUS_UPLOAD_OFFSET, chunk.getActualSize());
ctx.close();
}, err -> {
Utils.closeQuietly(ch);
chunk.unlock();
ctx.header(TUS_UPLOAD_OFFSET, chunk.getActualSize());
ctx.abort(err);
});
return null; return null;
} catch (final IOException e) { } catch (ErrorResponse | IOException e) {
chunk.unlock(); upload.unlock();
throw e; throw e;
} }
} }
private TusFile getChunk(String id) { public Void handleDelete(RestContext ctx) {
checkTusRequestVersion(ctx);
final TusUpload chunk = getUpload(ctx.getPathParam("chunk"));
if (!chunk.tryLock())
throw new ErrorResponse(409, "Conflict", "Failed to open upload: Resource locked.");
chunkService.removeUpload(chunk);
ctx.status(204);
ctx.header(TUS_RESUMABLE, TUS_VERSION);
return null;
}
private TusUpload getUpload(String id) {
return chunkService return chunkService
.getChunk(id) .getUpload(id)
.orElseThrow(() -> new ErrorResponse(404, "TusError", "Chunk not found or expired")); .orElseThrow(() -> new ErrorResponse(404, "TusError", "Upload not found or expired"));
}
/**
* Add Cache-Control, {@value #TUS_RESUMABLE}, {@value #TUS_UPLOAD_OFFSET},
* {@value #TUS_UPLOAD_LENGTH}/{@value #TUS_UPLOAD_DEFER_LENGTH} and
* {@value #TUS_UPLOAD_EXPIRES} headers as required for HEAD and PATCH requests.
*
* @param ctx
* @param chunk
*/
private void addResponseHeaders(RestContext ctx, final TusUpload upload) {
ctx.header("Cache-Control", "no-store");
ctx.header(TUS_RESUMABLE, TUS_VERSION);
ctx.header(TUS_UPLOAD_OFFSET, upload.getCurrentOffset());
ctx.header(TUS_UPLOAD_EXPIRES, Date.from(upload.getExpiresAt()));
if (upload.hasLength())
ctx.header(TUS_UPLOAD_LENGTH, upload.getLength());
else
ctx.header(TUS_UPLOAD_DEFER_LENGTH, "1");
if (upload.getMeta() != null) // Not required for PATCH
ctx.header(TUS_UPLOAD_METADATA, upload.getMeta());
} }
private void checkTusHeader(RestContext ctx) { private void checkTusRequestVersion(RestContext ctx) {
if (!Utils.equal(ctx.getHeader(TUS_RESUMABLE), TUS_VERSION)) if (!Utils.equal(ctx.getHeader(TUS_RESUMABLE), TUS_VERSION))
throw new ErrorResponse(400, "TusError", "Tus-Resumable header missing or wrong version.") throw new ErrorResponse(400, "TusError", "Tus-Resumable header missing or wrong version.")
.detail("expected", TUS_VERSION); .detail("expected", TUS_VERSION);
} }
private long getUploadLength(RestContext ctx) { private long requireLongHeader(RestContext ctx, String name) {
String lengthHeader = ctx.getHeader(TUS_UPLOAD_LENGTH);
if (lengthHeader == null)
return -1;
try { try {
return Long.parseUnsignedLong(ctx.getHeader(TUS_UPLOAD_LENGTH)); return Long.parseUnsignedLong(ctx.getHeader(name));
} catch (final NumberFormatException e) { } catch (final NumberFormatException e) {
throw new ErrorResponse(400, "TusError", "Upload-Length header present, but invalid."); throw new ErrorResponse(400, "TusError", name + " header missing or invalid.");
} }
} }
private long getUploadOffset(RestContext ctx) {
try {
return Long.parseUnsignedLong(ctx.getHeader(TUS_UPLOAD_OFFSET));
} catch (final NumberFormatException e) {
throw new ErrorResponse(400, "TusError", "Upload-Offset header missing or invalid.");
}
}
private String getUploadMetadata(RestContext ctx) {
return ctx.getHeader(TUS_UPLOAD_METADATA);
}
} }
\ No newline at end of file
package de.gwdg.cdstar.rest.ext.tus;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.time.Instant;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
class TusFile {
private static final Logger log = LoggerFactory.getLogger(TusService.class);
private long length;
private long expire = -1;
private String meta;
private final String name;
private final Path path;
private boolean locked;
TusFile(String name, Path path) {
this(name, path, -1);
}
TusFile(String name, Path path, long length) {
this.name = name;
this.path = path;
this.length = length;
log.info("New chunk: {} -> {}", name, path);
}
public synchronized boolean tryLock() {
if (locked)
return false;
locked = true;
return locked;
}
public synchronized void unlock() {
locked = false;
}
Path getPath() {
return path;
}
public String getMeta() {
return meta;
}
public void setMeta(String meta) {
this.meta = meta;
}
public long getLength() {
return length;
}
public void setLength(long length) {
this.length = length;
}
public long getActualSize() {
try {
return Files.size(path);
} catch (final IOException e) {
return 0;
}
}
public void expireIn(Duration timeout) {
expire = Instant.now().plus(timeout).toEpochMilli();
}
public void expire() {
expire = 0;
}
public long getExpireMillis() {
return expire;
}
boolean isExpired(Instant now) {
return expire > -1 && expire < now.toEpochMilli();
}
boolean isExpired() {
return isExpired(Instant.now());
}
public String getName() {
return name;
}
}
\ No newline at end of file
package de.gwdg.cdstar.rest.ext.tus; package de.gwdg.cdstar.rest.ext.tus;
import java.util.concurrent.TimeUnit;
import com.codahale.metrics.Gauge; import com.codahale.metrics.Gauge;
import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.MetricRegistry;
...@@ -7,6 +9,7 @@ import de.gwdg.cdstar.rest.v3.async.ArchiveUpdater; ...@@ -7,6 +9,7 @@ import de.gwdg.cdstar.rest.v3.async.ArchiveUpdater;
import de.gwdg.cdstar.runtime.Plugin; import de.gwdg.cdstar.runtime.Plugin;
import de.gwdg.cdstar.runtime.RuntimeContext; import de.gwdg.cdstar.runtime.RuntimeContext;
import de.gwdg.cdstar.runtime.listener.RuntimeListener; import de.gwdg.cdstar.runtime.listener.RuntimeListener;
import de.gwdg.cdstar.runtime.services.CronService;
/** /**
* This plugin installs a TUS (tus.io) REST endpoint to create and write to * This plugin installs a TUS (tus.io) REST endpoint to create and write to
...@@ -33,24 +36,21 @@ public class TusPlugin implements RuntimeListener { ...@@ -33,24 +36,21 @@ public class TusPlugin implements RuntimeListener {
metrics.register(MetricRegistry.name(pluginName, "count"), new Gauge<Integer>() { metrics.register(MetricRegistry.name(pluginName, "count"), new Gauge<Integer>() {
@Override @Override
public Integer getValue() { public Integer getValue() {
return Integer.valueOf(service.getChunkCount()); return Integer.valueOf(service.getUploadCount());
} }
}); });
metrics.register(MetricRegistry.name(pluginName, "bytes"), new Gauge<Long>() { metrics.register(MetricRegistry.name(pluginName, "bytes"), new Gauge<Long>() {
@Override @Override
public Long getValue() { public Long getValue() {
return Long.valueOf(service.getChunkTotalSize()); return Long.valueOf(service.getUploadTotalSize());
} }
}); });
}); });
service.start(); service.start();
} ctx.lookupRequired(CronService.class)
.scheduleWithFixedDelay(service::cleanupExpiredUploads, 60, 60, TimeUnit.SECONDS);
@Override
public void onShutdown(RuntimeContext ctx) {
service.removeExpiredChunks();
} }
} }
...@@ -10,7 +10,6 @@ import java.util.List; ...@@ -10,7 +10,6 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Optional; import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.LongAdder;
import java.util.stream.Stream; import java.util.stream.Stream;
import org.slf4j.Logger; import org.slf4j.Logger;
...@@ -18,106 +17,91 @@ import org.slf4j.LoggerFactory; ...@@ -18,106 +17,91 @@ import org.slf4j.LoggerFactory;
import de.gwdg.cdstar.Utils; import de.gwdg.cdstar.Utils;
import de.gwdg.cdstar.rest.v3.async.ArchiveUpdater; import de.gwdg.cdstar.rest.v3.async.ArchiveUpdater;
import de.gwdg.cdstar.runtime.Plugin;
/** /**
* Implements a TUD (tus.io) REST endpoint to create and write to anonymous * Implements a TUS (tus.io) REST endpoint to create and write to anonymous
* temporary files, and a way for {@link ArchiveUpdater} to import from these * temporary files, and a way for {@link ArchiveUpdater} to import from these
* files. All bundled as an optional plugin. * files. All bundled as an optional plugin.
*/ */
@Plugin(name = "tus")
public class TusService { public class TusService {
public static final String pluginName = "tus-upload"; public static final String pluginName = "tus-upload";
static final Logger log = LoggerFactory.getLogger(TusService.class); static final Logger log = LoggerFactory.getLogger(TusService.class);
private Duration defaultExpire = Duration.ofHours(24); Duration defaultExpire = Duration.ofHours(24);
Path basePath; Path basePath;
Map<String, TusFile> chunks = new ConcurrentHashMap<>(); Map<String, TusUpload> uploads = new ConcurrentHashMap<>();
LongAdder totalSize = new LongAdder();
public TusService(Path basePath) { public TusService(Path basePath) {
this.basePath = basePath; this.basePath = basePath;
} }
public void start() throws IOException { public void start() throws IOException {
Files.createDirectories(basePath);