Gitlab Community Edition Instance

Commit 9e494620 authored by Marcel Hellkamp's avatar Marcel Hellkamp
Browse files

Work on 'put' command

parent 04d6e57a
Upload one or more files to an existing archive.
Upload files to an archive.
You can also create new archives and set ACL entries or metadata attributes with this command.
It works a bit as a swiss army knife for simple archive creation or manipulation. For everything not
covered here, there are more specialized commands available.
File upload is save by default: Existing remote files are not overwritten. Use -f to force re-upload or -u to
re-upload newer files only. Note that -u relies on a working clock on both local and remote side, as it compares the
file modification times only.
All uploads and changes are wrapped in a transaction. If something goes wrong, nothing is committed on remote side and
you can simply re-run the same command.
import os
from pycdstar3 import FormUpdate
from pycdstar3.cli import printer, CliError
from pycdstar3.cli._utils import compile_glob, hbytes
from pycdstar3.cli._utils import compile_glob, hbytes, kvtype, globtype
def register(subparsers):
parser = subparsers.add_parser("put", help=__doc__.strip().splitlines()[0], description=__doc__)
parser.add_argument("-a", "--all", action="store_true", help="Include hidden files (skipped by default)")
parser.add_argument("-n", "--dry-run", action="store_true",
help="Do not upload anything, just print what would have been uploaded")
parser.add_argument("--flat", action="store_true",
help="Strip all directory names from the local path and store"
" all files into the root path (e.g. ./path/to/file.txt"
" would be uploaded as /file.txt)")
parser.add_argument("-x", "--exclude", metavar="GLOB", action="append",
help="Exclude files by glob pattern")
parser.add_argument("-i", "--include", metavar="GLOB", action="append",
# parser.add_argument("-u", "--update", action="store_true",
# help="Update outdated remote files. (default: skip)")
# parser.add_argument("-f", "--force", action="store_true",
# help="Always overwrite remote files, even if they are newer. (default: skip)")
# parser.add_argument("--delete", action="store_true",
# help="Delete remote files not present locally, if they match a PATH parameter.")
parser.add_argument("-i", "--include", metavar="GLOB", type=globtype, action="append",
help="Include files by glob pattern (default: all)")
parser.add_argument("-%", "-p", "--progress", action="store_true",
help="Show progress bar for large files or slow uploads")
parser.add_argument("-x", "--exclude", metavar="GLOB", type=globtype, action="append",
help="Exclude files by glob pattern")
parser.add_argument("--include-hidden", action="store_true", help="Include hidden files (default: skip)")
parser.add_argument("--prefix", metavar="PREFIX", help="Upload files to this remote directory. (default: `/`)")
parser.add_argument("--flat", action="store_true",
help="Strip local directory names and only use the basename when uploading files."
" (e.g. ./path/to/file.txt would be uploaded as /file.txt)")
# parser.add_argument("--tus", action="store_true",
# help="Upload large files via and retry on connection errors (needs server support)")
parser.add_argument("--meta", metavar="KEY=VAL", type=kvtype, action="append",
help="Set archive metadata attributes. An empty value removes the attribute. Can be repeated to"
" set multiple values for the same attribute.")
parser.add_argument("--acl", metavar="SUBJECT=PERM", type=kvtype, action="append",
help="Set ACL entries. PERM can be a comma separated list of permissions or permission sets. "
" An empty PERM value removes all permissions for that SUBJECT.")
parser.add_argument("-n", "--dry-run", action="store_true",
help="Do not upload or change anything, just print what would have been done")
parser.add_argument("-%", "--progress", action="store_true", help="Show progress bar.")
parser.add_argument("TARGET", help="Archive ID, or 'new' to create a new archive")
parser.add_argument("PATH", nargs='+', help="Files or directories to upload")
def findfiles(flist, include_hidden=False):
def keep(name):
return include_hidden or not name.startswith('.')
def collect_files(path, hidden=False):
if os.path.isfile(path):
yield path
elif os.path.isdir(path):
files = []
for (root, dlist, flist) in os.walk(path):
if not hidden:
dlist[:] = [name for name in dlist if not name.startswith('.')]
flist[:] = [name for name in flist if not name.startswith('.')]
yield from (os.path.join(root, name) for name in flist)
raise CliError("Not a file: " + path)
def filter_files(files, inc_rules, exc_rules):
for file in files:
if inc_rules and not any(rule.match(file) for rule in inc_rules):
printer.vv("Skipping: {} (not included)", file)
if any(rule.match(file) for rule in exc_rules):
printer.vv("Skipping: {} (excluded)", file)
yield file
for entry in flist:
if os.path.isfile(entry):
yield entry
elif os.path.isdir(entry):
for (root, dirs, files) in os.walk(entry):
dirs[:] = filter(keep, dirs)
files[:] = filter(keep, files)
for file in files:
yield os.path.join(root, file)
raise CliError("Not a file: " + entry)
def remote_name(base, file, prefix="", flatten=False):
target = os.path.relpath(file, base)
if flatten:
target = os.path.basename(target)
if prefix:
target = os.path.join(prefix, target)
if target != os.path.normpath(target):
raise ValueError("Unable to upload relative path: {!r}".format(target))
return target
def command(ctx, args): # noqa: C901
client, vault, archive, rfile = ctx.resolve(args.TARGET)
inc = [compile_glob(rule).match for rule in args.include or []]
exc = [compile_glob(rule).match for rule in args.exclude or []]
progress = args.progress
if not rfile:
rfile = '/'
if rfile and args.prefix:
raise CliError("Either provide a file reference, or set --prefix. Not both.")
prefix = rfile or args.prefix or ''
prefix = prefix.lstrip('/')
inc_rules = args.include or []
exc_rules = args.exclude or []
uploads = {}
total = 0
files = findfiles(args.PATH, args.all)
for file in files:
if inc and not any(rule(file) for rule in inc):
printer.vv("Skipping: {} (not included)", file)
if any(rule(file) for rule in exc):
printer.vv("Skipping: {} (excluded)", file)
target = os.path.relpath(file)
if args.flat:
target = os.path.basename(target)
if rfile.endswith("/"):
target = os.path.join(rfile, target)
target = rfile
if target != os.path.normpath(target):
raise ValueError("Unable to upload relative path: {!r}".format(target))
if target in uploads and uploads[target][0] != file:
raise ValueError("File included twice: {} and {} both map to {}".format(file, uploads[target][0], target))
stat = os.stat(file)
total += stat.st_size
uploads[target] = (file, stat)
for path in args.PATH:
files = collect_files(path, args.include_hidden)
files = filter_files(files, inc_rules, exc_rules)
for file in files:
fstat = os.stat(file)
uploads[remote_name(".", file, prefix, args.flat)] = file, fstat
total += fstat.st_size
meta = {}
for key, val in args.meta or []:
meta.setdefault(key, []).append(val)
acl = {}
for key, val in args.acl or []:
acl.setdefault(key, []).append(val)
if args.dry_run:
for target in sorted(uploads):
printer("\nWould upload {} files ({}) to {}",
len(uploads), hbytes(total), "new archive" if archive == 'new' else "archive: " + vault + "/" + archive)
form = FormUpdate()
for key, values in meta.items():
form.meta(key, *values)
for key, values in acl.items():
form.acl(key, *values)
with client.begin(autocommit=True):
if archive == 'new':
archive = client.create_archive(vault)['id']
printer("Uploading {} files ({}) to new archive: /{}/{}", len(uploads), hbytes(total), vault, archive)
printer("Uploading {} files ({}) to archive: /{}/{}", len(uploads), hbytes(total), vault, archive)
archive = client.create_archive(vault, form=form)['id']
printer("Created new archive: /{}/{} ", vault, archive)
elif form.fields:
client.update_archive(vault, archive, form=form)
printer("Updated archive: /{}/{} ", vault, archive)
elif not client.exists(vault, archive):
raise CliError("Archive /{}/{} does not exist.".format(vault, archive))
printer("Uploading {} files ({}) to archive: /{}/{}", len(uploads), hbytes(total), vault, archive)
pbar = None
if progress and not printer.quiet:
if args.progress and not printer.quiet:
from tqdm import tqdm
total = sum(stat.st_size for (file, stat) in uploads.values())
pbar = tqdm(total=total, unit='b', unit_scale=True, unit_divisor=1024, dynamic_ncols=True,
......@@ -106,11 +162,11 @@ def command(ctx, args): # noqa: C901
for i, target in enumerate(sorted(uploads)):
file, stat = uploads[target]
with open(file, 'rb') as fp:
line = "[{}/{}] {} ({})".format(i + 1, len(uploads), target[1:], hbytes(stat.st_size))
line = "[{}/{}] {} ({})".format(i + 1, len(uploads), target, hbytes(stat.st_size))
if pbar:
read =
chunks = iter(lambda: read(1024*8), b'')
chunks = iter(lambda: read(1024 * 64), b'')
chunks = (chunk for chunk in chunks if not pbar.update(len(chunk)))
fp = chunks
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment