Gitlab Community Edition Instance

Commit 3a587531 authored by mhellka's avatar mhellka
Browse files

Bite the bullet and enforce 'black' formatting.

parent ce9ab39c
......@@ -38,8 +38,9 @@ test-pyRC:
linters:
stage: test
script:
- pip install flake8
- pip install flake8 black
- flake8 src/
- black --check src/
allow_failure: true
build:
......
......@@ -16,6 +16,10 @@ dist: venv $(SRC_ALL)
lint: venv
venv/bin/flake8 src/
venv/bin/black --check src/
format: venv
venv/bin/black src/
test: venv
venv/bin/tox
......@@ -30,7 +34,7 @@ push: test lint
git diff-index --exit-code HEAD -- # Ensure workdir and index are both clean
git push
_PHONY: venv docs dist lint test push clean
_PHONY: venv docs dist lint test push clean format
clean:
rm -rf venv .tox
......@@ -42,7 +42,7 @@ setup(
'iso8601 >= 0.1.12',
],
extras_require={
'dev': ['flake8', 'wheel', 'twine', 'tox'],
'dev': ['flake8', 'wheel', 'twine', 'tox', 'black'],
'test': [
'mock',
'pytest>=3.6',
......
......@@ -6,5 +6,5 @@ def main(): # pragma: no cover
return pycdstar3.cli.main(*sys.argv[1:])
if __name__ == '__main__': # pragma: no cover
if __name__ == "__main__": # pragma: no cover
sys.exit(main() or 0)
......@@ -9,7 +9,7 @@ if hasattr(os, "PathLike"):
class cached_property(object):
def __init__(self, func):
self.__doc__ = getattr(func, '__doc__')
self.__doc__ = getattr(func, "__doc__")
self.func = func
def __get__(self, obj, cls):
......
......@@ -57,22 +57,23 @@ class CDStar:
"""
if self.auth:
options['auth'] = self.auth
options["auth"] = self.auth
if self.tx:
options.setdefault("headers", {})['X-Transaction'] = self.tx['id']
options.setdefault("headers", {})["X-Transaction"] = self.tx["id"]
path = [requests.utils.quote(p) for p in path]
rs = self._session.request(
method, self.url + '/'.join(path), **options)
rs = self._session.request(method, self.url + "/".join(path), **options)
if rs.ok or (expect_status and rs.status_code in expect_status):
return rs
raise ApiError(rs)
def rest(self, method, *path, expect_status=None, **options) -> typing.Optional[JsonObject]:
def rest(
self, method, *path, expect_status=None, **options
) -> typing.Optional[JsonObject]:
""" Just like `raw()`, but expects the response to be JSON and returns
the parsed result instead of the raw response. Non-JSON responses
are errors. Empty (204) responses return None.
......@@ -154,7 +155,7 @@ class CDStar:
def _auto_keepalive_start(self):
self._auto_keepalive_stop()
# leeway: 10% or 2 seconds, but sleep for at least a second
interval = max(1, min(self.tx['ttl'] * 0.9, self.tx['ttl'] - 2))
interval = max(1, min(self.tx["ttl"] * 0.9, self.tx["ttl"] - 2))
self._keepalive_timer = IntervalTimer(interval, self._auto_keepalive, self.tx)
self._keepalive_timer.daemon = True
self._keepalive_timer.start()
......@@ -169,7 +170,7 @@ class CDStar:
self.keepalive()
# leeway: 10% or 2 seconds, but sleep for at least a second
timer.set_interval(max(1, min(self.tx['ttl'] * 0.9, self.tx['ttl'] - 2)))
timer.set_interval(max(1, min(self.tx["ttl"] * 0.9, self.tx["ttl"] - 2)))
def _auto_keepalive_stop(self):
""" Stop the current keepalive timer, if any. """
......@@ -201,24 +202,33 @@ class CDStar:
def service_info(self) -> JsonObject:
""" Get information about the cdstar service instance """
return self.rest('GET')
return self.rest("GET")
def vault_info(self, vault: str) -> JsonObject:
""" Get information about a vault """
return self.rest('GET', vault)
return self.rest("GET", vault)
def create_archive(self, vault, form: FormUpdate = None) -> JsonObject:
""" Create a new archive. """
if form:
return self.rest("POST", vault, data=form.body,
headers={'Content-Type': form.content_type})
return self.rest(
"POST",
vault,
data=form.body,
headers={"Content-Type": form.content_type},
)
else:
return self.rest("POST", vault)
def update_archive(self, vault, archive, form: FormUpdate) -> JsonObject:
""" Update an existing archive """
return self.rest("POST", vault, archive, data=form.body,
headers={'Content-Type': form.content_type})
return self.rest(
"POST",
vault,
archive,
data=form.body,
headers={"Content-Type": form.content_type},
)
def archive_info(self, vault, archive, meta=False, files=False) -> JsonObject:
""" Get information about an archive """
......@@ -233,7 +243,9 @@ class CDStar:
""" Remove an archive. This cannot be undone. """
return self.raw("DELETE", vault, archive).ok
def put_file(self, vault, archive, name, source, type=None, replace=True) -> JsonObject:
def put_file(
self, vault, archive, name, source, type=None, replace=True
) -> JsonObject:
""" Create or replace a single file on an existing archive.
If the file exists remotely and `replace=True` is set (default), the file content is overridden but everything
......@@ -249,14 +261,18 @@ class CDStar:
"""
if isinstance(source, PATH_TYPES):
with open(source, 'rb') as source:
return self.put_file(vault, archive, name, source, type=None, replace=True)
with open(source, "rb") as source:
return self.put_file(
vault, archive, name, source, type=None, replace=True
)
headers = {'Content-Type': type or "application/x-autodetect"}
headers = {"Content-Type": type or "application/x-autodetect"}
if not replace:
headers['If-None-Match'] = "*"
headers["If-None-Match"] = "*"
return self.rest("PUT", vault, archive, _fix_filename(name), data=source, headers=headers)
return self.rest(
"PUT", vault, archive, _fix_filename(name), data=source, headers=headers
)
def get_file(self, vault, archive, name, offset=0) -> FileDownload:
""" Request a file and return a stream-able :class:`FileDownload`.
......@@ -268,7 +284,7 @@ class CDStar:
dl.save_to("~/Downloads/")
"""
headers = {'Range': "bytes={}-".format(offset)} if offset > 0 else {}
headers = {"Range": "bytes={}-".format(offset)} if offset > 0 else {}
name = _fix_filename(name)
rs = self.raw("GET", vault, archive, name, stream=True, headers=headers)
return FileDownload(vault, archive, name, rs)
......@@ -277,11 +293,21 @@ class CDStar:
""" Get information about a file """
query = {"info": "true"}
if meta:
query['with'] = "meta"
query["with"] = "meta"
return self.rest("GET", vault, archive, _fix_filename(name), params=query)
def list_files(self, vault, archive, offset=0, limit=100, meta=False, order=None, reverse=False,
include_glob=None, exclude_glob=None) -> JsonObject:
def list_files(
self,
vault,
archive,
offset=0,
limit=100,
meta=False,
order=None,
reverse=False,
include_glob=None,
exclude_glob=None,
) -> JsonObject:
""" Request a FileList for an archive.
The FileList may be incomplete of more than `limit` files are in an archive. See iter_files() for a
......@@ -302,7 +328,9 @@ class CDStar:
return self.rest("GET", vault, archive, params=query)
def iter_files(self, vault, archive, offset=0, **args) -> typing.Iterator[JsonObject]:
def iter_files(
self, vault, archive, offset=0, **args
) -> typing.Iterator[JsonObject]:
""" Yield all FileInfo entries of an archive.
This method may (lazily) issue more than one request if an archive contains more than `limit` files.
......@@ -310,9 +338,9 @@ class CDStar:
while True:
files = self.list_files(vault, archive, offset, **args)
if files['files'] and offset + files['count'] <= files['total']:
yield from files['files']
offset += files['count']
if files["files"] and offset + files["count"] <= files["total"]:
yield from files["files"]
offset += files["count"]
else:
break
......@@ -320,20 +348,22 @@ class CDStar:
""" Remove a archive file. This cannot be undone. """
return self.raw("DELETE", vault, archive, file).ok
def search(self, vault, q, order=None, limit=0, scroll=None, groups=None) -> JsonObject:
def search(
self, vault, q, order=None, limit=0, scroll=None, groups=None
) -> JsonObject:
""" Perform a search and return a single page of search results.
See iter_search() for a convenient way to fetch more than `limit` results.
"""
params = {"q": q}
if order:
params['order'] = order
params["order"] = order
if limit:
params['limit'] = limit
params["limit"] = limit
if scroll:
params['scroll'] = scroll
params["scroll"] = scroll
if groups:
params['groups'] = groups
params["groups"] = groups
return self.rest("GET", vault, params=params)
def iter_search(self, vault, q, scroll=None, **args) -> typing.Iterator[JsonObject]:
......@@ -343,9 +373,9 @@ class CDStar:
"""
while True:
page = self.search(vault, q, scroll=scroll or "", **args)
if page['hits']:
yield from page['hits']
scroll = page['scroll']
if page["hits"]:
yield from page["hits"]
scroll = page["scroll"]
else:
break
......@@ -360,7 +390,11 @@ def _fix_filename(name):
# Fail hard on relative filenames
if name != os.path.normpath(name):
raise ValueError("Archive file name not in a normalized form: {} != {}".format(name, os.path.normpath(name)))
raise ValueError(
"Archive file name not in a normalized form: {} != {}".format(
name, os.path.normpath(name)
)
)
return name
......@@ -377,6 +411,7 @@ class CDStarVault:
This handle, as well als other handles returned by it, are just lightweight pointers to remote resources.
No remote state is cached locally and most method calls will trigger REST requests.
"""
__slots__ = "api", "name"
def __init__(self, api: CDStar, vault: str):
......@@ -418,6 +453,7 @@ class CDStarArchive:
See :class:`CDStarVault` for details on how handles work.
"""
__slots__ = "api", "vault", "id"
def __init__(self, vault: CDStarVault, archive_id):
......@@ -450,6 +486,7 @@ class CDStarFile:
See :class:`CDStarVault` for details on how handles work.
"""
__slots__ = "api", "archive", "name"
def __init__(self, archive: CDStarArchive, name: str):
......@@ -469,10 +506,14 @@ class CDStarFile:
def put(self, **ka) -> JsonObject:
""" Create or overwrite file content """
return self.api.put_file(self.archive.vault.name, self.archive.id, self.name, **ka)
return self.api.put_file(
self.archive.vault.name, self.archive.id, self.name, **ka
)
def stream(self, **ka) -> FileDownload:
""" Request file content as a stream-able :class:`FileDownload`. """
return self.api.get_file(self.archive.vault.name, self.archive.id, self.name, **ka)
return self.api.get_file(
self.archive.vault.name, self.archive.id, self.name, **ka
)
# TODO: Implement meee
......@@ -11,18 +11,33 @@ from pycdstar3 import __version__ as VERSION, ApiError
__ALL__ = ["main", "printer"]
parser = argparse.ArgumentParser(prog="pycdstar3")
parser.add_argument("--server", metavar="URI",
help="CDSTAR server URI. Defaults to CDSTAR_SERVER environment variable or workspace settings.")
parser.add_argument("--vault", metavar="NAME",
help="Vault to work with. Defaults to CDSTAR_VAULT environment variable or workspace settings.")
parser.add_argument(
"--server",
metavar="URI",
help="CDSTAR server URI. Defaults to CDSTAR_SERVER environment variable or workspace settings.",
)
parser.add_argument(
"--vault",
metavar="NAME",
help="Vault to work with. Defaults to CDSTAR_VAULT environment variable or workspace settings.",
)
parser.add_argument("--version", action="store_true", help="Print version and exit.")
_grp = parser.add_mutually_exclusive_group()
_grp.add_argument("-v", "--verbose", action="count", default=0,
help="Print more info. Repeat to increase verbosity.")
_grp.add_argument("-q", "--quiet", action="store_true", help="Be quiet. Only print errors.")
subparsers = parser.add_subparsers(title="available commands",
description='Run "COMAMND -h" to get help for a specific command.',
metavar="COMMAND")
_grp.add_argument(
"-v",
"--verbose",
action="count",
default=0,
help="Print more info. Repeat to increase verbosity.",
)
_grp.add_argument(
"-q", "--quiet", action="store_true", help="Be quiet. Only print errors."
)
subparsers = parser.add_subparsers(
title="available commands",
description='Run "COMAMND -h" to get help for a specific command.',
metavar="COMMAND",
)
def _autodiscover_commands():
......@@ -33,6 +48,7 @@ def _autodiscover_commands():
"""
import pkgutil
import pycdstar3.cli.commands
for _, name, ispkg in pkgutil.iter_modules(pycdstar3.cli.commands.__path__):
if ispkg:
continue
......@@ -58,6 +74,7 @@ def main(*args): # noqa: C901
return 1
from pycdstar3.cli.context import CliContext
ctx = CliContext(opts, workdir=".")
try:
......
......@@ -16,7 +16,7 @@ def walk_up(start):
def hbytes(n, units=None):
for unit in units or ('B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'):
for unit in units or ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"):
if abs(n) < 1024.0:
return "{:.1f} {}".format(n, unit)
n /= 1024.0
......@@ -27,7 +27,7 @@ def kvtype(val):
""" Argparse type that parses a KEY=VALUE parameter into a tuple.
The value may be empty, but the '=' is required. """
k, _, v = val.partition('=')
k, _, v = val.partition("=")
if not _:
raise argparse.ArgumentTypeError("Expected KAY=VALUE argument.")
return k, v
......@@ -39,24 +39,25 @@ def globtype(str):
def compile_glob(pattern):
parts = re.split(r'(\*\*|\*|\?)', pattern)
parts = re.split(r"(\*\*|\*|\?)", pattern)
res = ["^" if pattern.startswith("/") else ".*"]
for i, part in enumerate(parts):
if i % 2 == 0:
res.append(re.escape(part))
elif part == '*':
res.append(r'[^/]+')
elif part == '**':
res.append(r'.+')
elif part == '?':
res.append(r'[^/]')
return re.compile(''.join(res) + "$")
elif part == "*":
res.append(r"[^/]+")
elif part == "**":
res.append(r".+")
elif part == "?":
res.append(r"[^/]")
return re.compile("".join(res) + "$")
class FileProgress:
def __init__(self, fp, chunksize=1024 * 8, **baropts):
from requests.utils import super_len
self.opts = baropts
self.fp = fp
self.len = super_len(fp)
......@@ -64,8 +65,15 @@ class FileProgress:
def __iter__(self):
from tqdm import tqdm
with tqdm(total=self.len, unit='b', unit_scale=True, unit_divisor=1024, dynamic_ncols=True,
**self.opts) as pbar:
with tqdm(
total=self.len,
unit="b",
unit_scale=True,
unit_divisor=1024,
dynamic_ncols=True,
**self.opts
) as pbar:
read = self.fp.read
update = pbar.update
while True:
......@@ -79,6 +87,7 @@ class FileProgress:
class Printer:
""" Helper class to print to stderr based on verbosity levels."""
__slots__ = ("verbosity", "quiet", "file")
def __init__(self, level=0, file=sys.stderr):
......@@ -94,14 +103,21 @@ class Printer:
if args:
msg = msg.format(*args)
if indent:
msg = ''.join(' ' * indent + line for line in msg.splitlines(True))
msg = "".join(" " * indent + line for line in msg.splitlines(True))
if hr:
ncols = self._ncols()
msg = (hr * ncols) + '\n' + msg + ('' if msg.endswith('\n') else '\n') + (hr * ncols)
msg = (
(hr * ncols)
+ "\n"
+ msg
+ ("" if msg.endswith("\n") else "\n")
+ (hr * ncols)
)
print(msg, file=self.file, **kwargs)
def _ncols(self):
import shutil
return shutil.get_terminal_size((40, 20))[0]
def __call__(self, msg="", *args, **kwargs):
......@@ -133,6 +149,7 @@ class Printer:
self._print("ERROR: " + msg, *args, **kwargs)
if self.verbosity >= 2:
import traceback
self._print(traceback.format_exc(), highlight="=")
def fatal(self, msg, *args, **kwargs):
......@@ -140,6 +157,7 @@ class Printer:
self._print("FATAL: " + msg, *args, **kwargs)
if self.verbosity >= 2:
import traceback
self._print(traceback.format_exc(), highlight="=")
else:
self("Stacktrace not shown. Add -vv to print a full stacktrace.")
......@@ -5,13 +5,17 @@ from pycdstar3 import FormUpdate
def register(subparsers):
parser = subparsers.add_parser("acl", help=__doc__.strip().splitlines()[0], description=__doc__)
parser = subparsers.add_parser(
"acl", help=__doc__.strip().splitlines()[0], description=__doc__
)
sub = parser.add_subparsers()
pset = sub.add_parser("set")
pset.add_argument("ARCHIVE", help="Archive ID")
pset.add_argument("SUBJECT", help="Subject do modify")
pset.add_argument("PERMISSIONS", nargs="*", help="Permissions to set for this subject.")
pset.add_argument(
"PERMISSIONS", nargs="*", help="Permissions to set for this subject."
)
pset.set_defaults(main=acl_set)
......@@ -22,7 +26,13 @@ def acl_set(ctx, args):
subject = args.SUBJECT
permissions = args.PERMISSIONS
ctx.print("Set ACL for {!r} on /{}/{} -> [{}]", subject, vault, archive, ', '.join(permissions))
ctx.print(
"Set ACL for {!r} on /{}/{} -> [{}]",
subject,
vault,
archive,
", ".join(permissions),
)
with archive.api.begin(autocommit=True):
update = FormUpdate()
update.acl(subject, *permissions)
......
......@@ -9,18 +9,36 @@ from pycdstar3.cli._utils import hbytes
def register(subparsers):
parser = subparsers.add_parser("get", help=__doc__.strip().splitlines()[0], description=__doc__)
parser.add_argument("-%", "-p", "--progress", action="store_true",
help="Show progress bar for large files or slow downloads")
parser.add_argument("--resume", action="store_true",
help="If a <DST>.part file exists, try to resume an"
" interrupted download. Also, keep the *.part file"
" on any errors.")
parser.add_argument("-f", "--force", action="store_true",
help="Overwrite local files or print binary data to a terminal.")
parser = subparsers.add_parser(
"get", help=__doc__.strip().splitlines()[0], description=__doc__
)
parser.add_argument(
"-%",
"-p",
"--progress",
action="store_true",
help="Show progress bar for large files or slow downloads",
)
parser.add_argument(
"--resume",
action="store_true",
help="If a <DST>.part file exists, try to resume an"
" interrupted download. Also, keep the *.part file"
" on any errors.",
)
parser.add_argument(
"-f",
"--force",
action="store_true",
help="Overwrite local files or print binary data to a terminal.",
)
parser.add_argument("ARCHIVE", help="Archive ID")
parser.add_argument("FILE", help="File Name")
parser.add_argument("DST", nargs="?", help="Destination filename, directory or '-' for stdout. (default: '-')")
parser.add_argument(
"DST",
nargs="?",
help="Destination filename, directory or '-' for stdout. (default: '-')",
)
parser.set_defaults(main=get)
......@@ -32,7 +50,7 @@ def get(ctx, args): # noqa: C901
resume = args.resume
progress = args.progress
dst = args.DST or '-'
dst = args.DST or "-"
force = args.force
if not file:
......@@ -41,7 +59,7 @@ def get(ctx, args): # noqa: C901
if dst.endswith("/") or os.path.isdir(dst):
dst = os.path.join(dst, os.path.basename(file))
ispipe = dst == '-'
ispipe = dst == "-"
partfile = dst + ".part" if not ispipe else None
offset = 0
......@@ -53,12 +71,14 @@ def get(ctx, args): # noqa: C901
offset = os.path.getsize(partfile)
ctx.print("Resuming download at: {}", hbytes(offset))
else:
raise CliError("Found partial download, but --resume is not set: " + partfile)
raise CliError(
"Found partial download, but --resume is not set: " + partfile
)