Gitlab Community Edition Instance

Commit 3a587531 authored by mhellka's avatar mhellka
Browse files

Bite the bullet and enforce 'black' formatting.

parent ce9ab39c
...@@ -38,8 +38,9 @@ test-pyRC: ...@@ -38,8 +38,9 @@ test-pyRC:
linters: linters:
stage: test stage: test
script: script:
- pip install flake8 - pip install flake8 black
- flake8 src/ - flake8 src/
- black --check src/
allow_failure: true allow_failure: true
build: build:
......
...@@ -16,6 +16,10 @@ dist: venv $(SRC_ALL) ...@@ -16,6 +16,10 @@ dist: venv $(SRC_ALL)
lint: venv lint: venv
venv/bin/flake8 src/ venv/bin/flake8 src/
venv/bin/black --check src/
format: venv
venv/bin/black src/
test: venv test: venv
venv/bin/tox venv/bin/tox
...@@ -30,7 +34,7 @@ push: test lint ...@@ -30,7 +34,7 @@ push: test lint
git diff-index --exit-code HEAD -- # Ensure workdir and index are both clean git diff-index --exit-code HEAD -- # Ensure workdir and index are both clean
git push git push
_PHONY: venv docs dist lint test push clean _PHONY: venv docs dist lint test push clean format
clean: clean:
rm -rf venv .tox rm -rf venv .tox
...@@ -42,7 +42,7 @@ setup( ...@@ -42,7 +42,7 @@ setup(
'iso8601 >= 0.1.12', 'iso8601 >= 0.1.12',
], ],
extras_require={ extras_require={
'dev': ['flake8', 'wheel', 'twine', 'tox'], 'dev': ['flake8', 'wheel', 'twine', 'tox', 'black'],
'test': [ 'test': [
'mock', 'mock',
'pytest>=3.6', 'pytest>=3.6',
......
...@@ -6,5 +6,5 @@ def main(): # pragma: no cover ...@@ -6,5 +6,5 @@ def main(): # pragma: no cover
return pycdstar3.cli.main(*sys.argv[1:]) return pycdstar3.cli.main(*sys.argv[1:])
if __name__ == '__main__': # pragma: no cover if __name__ == "__main__": # pragma: no cover
sys.exit(main() or 0) sys.exit(main() or 0)
...@@ -9,7 +9,7 @@ if hasattr(os, "PathLike"): ...@@ -9,7 +9,7 @@ if hasattr(os, "PathLike"):
class cached_property(object): class cached_property(object):
def __init__(self, func): def __init__(self, func):
self.__doc__ = getattr(func, '__doc__') self.__doc__ = getattr(func, "__doc__")
self.func = func self.func = func
def __get__(self, obj, cls): def __get__(self, obj, cls):
......
...@@ -57,22 +57,23 @@ class CDStar: ...@@ -57,22 +57,23 @@ class CDStar:
""" """
if self.auth: if self.auth:
options['auth'] = self.auth options["auth"] = self.auth
if self.tx: if self.tx:
options.setdefault("headers", {})['X-Transaction'] = self.tx['id'] options.setdefault("headers", {})["X-Transaction"] = self.tx["id"]
path = [requests.utils.quote(p) for p in path] path = [requests.utils.quote(p) for p in path]
rs = self._session.request( rs = self._session.request(method, self.url + "/".join(path), **options)
method, self.url + '/'.join(path), **options)
if rs.ok or (expect_status and rs.status_code in expect_status): if rs.ok or (expect_status and rs.status_code in expect_status):
return rs return rs
raise ApiError(rs) raise ApiError(rs)
def rest(self, method, *path, expect_status=None, **options) -> typing.Optional[JsonObject]: def rest(
self, method, *path, expect_status=None, **options
) -> typing.Optional[JsonObject]:
""" Just like `raw()`, but expects the response to be JSON and returns """ Just like `raw()`, but expects the response to be JSON and returns
the parsed result instead of the raw response. Non-JSON responses the parsed result instead of the raw response. Non-JSON responses
are errors. Empty (204) responses return None. are errors. Empty (204) responses return None.
...@@ -154,7 +155,7 @@ class CDStar: ...@@ -154,7 +155,7 @@ class CDStar:
def _auto_keepalive_start(self): def _auto_keepalive_start(self):
self._auto_keepalive_stop() self._auto_keepalive_stop()
# leeway: 10% or 2 seconds, but sleep for at least a second # leeway: 10% or 2 seconds, but sleep for at least a second
interval = max(1, min(self.tx['ttl'] * 0.9, self.tx['ttl'] - 2)) interval = max(1, min(self.tx["ttl"] * 0.9, self.tx["ttl"] - 2))
self._keepalive_timer = IntervalTimer(interval, self._auto_keepalive, self.tx) self._keepalive_timer = IntervalTimer(interval, self._auto_keepalive, self.tx)
self._keepalive_timer.daemon = True self._keepalive_timer.daemon = True
self._keepalive_timer.start() self._keepalive_timer.start()
...@@ -169,7 +170,7 @@ class CDStar: ...@@ -169,7 +170,7 @@ class CDStar:
self.keepalive() self.keepalive()
# leeway: 10% or 2 seconds, but sleep for at least a second # leeway: 10% or 2 seconds, but sleep for at least a second
timer.set_interval(max(1, min(self.tx['ttl'] * 0.9, self.tx['ttl'] - 2))) timer.set_interval(max(1, min(self.tx["ttl"] * 0.9, self.tx["ttl"] - 2)))
def _auto_keepalive_stop(self): def _auto_keepalive_stop(self):
""" Stop the current keepalive timer, if any. """ """ Stop the current keepalive timer, if any. """
...@@ -201,24 +202,33 @@ class CDStar: ...@@ -201,24 +202,33 @@ class CDStar:
def service_info(self) -> JsonObject: def service_info(self) -> JsonObject:
""" Get information about the cdstar service instance """ """ Get information about the cdstar service instance """
return self.rest('GET') return self.rest("GET")
def vault_info(self, vault: str) -> JsonObject: def vault_info(self, vault: str) -> JsonObject:
""" Get information about a vault """ """ Get information about a vault """
return self.rest('GET', vault) return self.rest("GET", vault)
def create_archive(self, vault, form: FormUpdate = None) -> JsonObject: def create_archive(self, vault, form: FormUpdate = None) -> JsonObject:
""" Create a new archive. """ """ Create a new archive. """
if form: if form:
return self.rest("POST", vault, data=form.body, return self.rest(
headers={'Content-Type': form.content_type}) "POST",
vault,
data=form.body,
headers={"Content-Type": form.content_type},
)
else: else:
return self.rest("POST", vault) return self.rest("POST", vault)
def update_archive(self, vault, archive, form: FormUpdate) -> JsonObject: def update_archive(self, vault, archive, form: FormUpdate) -> JsonObject:
""" Update an existing archive """ """ Update an existing archive """
return self.rest("POST", vault, archive, data=form.body, return self.rest(
headers={'Content-Type': form.content_type}) "POST",
vault,
archive,
data=form.body,
headers={"Content-Type": form.content_type},
)
def archive_info(self, vault, archive, meta=False, files=False) -> JsonObject: def archive_info(self, vault, archive, meta=False, files=False) -> JsonObject:
""" Get information about an archive """ """ Get information about an archive """
...@@ -233,7 +243,9 @@ class CDStar: ...@@ -233,7 +243,9 @@ class CDStar:
""" Remove an archive. This cannot be undone. """ """ Remove an archive. This cannot be undone. """
return self.raw("DELETE", vault, archive).ok return self.raw("DELETE", vault, archive).ok
def put_file(self, vault, archive, name, source, type=None, replace=True) -> JsonObject: def put_file(
self, vault, archive, name, source, type=None, replace=True
) -> JsonObject:
""" Create or replace a single file on an existing archive. """ Create or replace a single file on an existing archive.
If the file exists remotely and `replace=True` is set (default), the file content is overridden but everything If the file exists remotely and `replace=True` is set (default), the file content is overridden but everything
...@@ -249,14 +261,18 @@ class CDStar: ...@@ -249,14 +261,18 @@ class CDStar:
""" """
if isinstance(source, PATH_TYPES): if isinstance(source, PATH_TYPES):
with open(source, 'rb') as source: with open(source, "rb") as source:
return self.put_file(vault, archive, name, source, type=None, replace=True) return self.put_file(
vault, archive, name, source, type=None, replace=True
)
headers = {'Content-Type': type or "application/x-autodetect"} headers = {"Content-Type": type or "application/x-autodetect"}
if not replace: if not replace:
headers['If-None-Match'] = "*" headers["If-None-Match"] = "*"
return self.rest("PUT", vault, archive, _fix_filename(name), data=source, headers=headers) return self.rest(
"PUT", vault, archive, _fix_filename(name), data=source, headers=headers
)
def get_file(self, vault, archive, name, offset=0) -> FileDownload: def get_file(self, vault, archive, name, offset=0) -> FileDownload:
""" Request a file and return a stream-able :class:`FileDownload`. """ Request a file and return a stream-able :class:`FileDownload`.
...@@ -268,7 +284,7 @@ class CDStar: ...@@ -268,7 +284,7 @@ class CDStar:
dl.save_to("~/Downloads/") dl.save_to("~/Downloads/")
""" """
headers = {'Range': "bytes={}-".format(offset)} if offset > 0 else {} headers = {"Range": "bytes={}-".format(offset)} if offset > 0 else {}
name = _fix_filename(name) name = _fix_filename(name)
rs = self.raw("GET", vault, archive, name, stream=True, headers=headers) rs = self.raw("GET", vault, archive, name, stream=True, headers=headers)
return FileDownload(vault, archive, name, rs) return FileDownload(vault, archive, name, rs)
...@@ -277,11 +293,21 @@ class CDStar: ...@@ -277,11 +293,21 @@ class CDStar:
""" Get information about a file """ """ Get information about a file """
query = {"info": "true"} query = {"info": "true"}
if meta: if meta:
query['with'] = "meta" query["with"] = "meta"
return self.rest("GET", vault, archive, _fix_filename(name), params=query) return self.rest("GET", vault, archive, _fix_filename(name), params=query)
def list_files(self, vault, archive, offset=0, limit=100, meta=False, order=None, reverse=False, def list_files(
include_glob=None, exclude_glob=None) -> JsonObject: self,
vault,
archive,
offset=0,
limit=100,
meta=False,
order=None,
reverse=False,
include_glob=None,
exclude_glob=None,
) -> JsonObject:
""" Request a FileList for an archive. """ Request a FileList for an archive.
The FileList may be incomplete of more than `limit` files are in an archive. See iter_files() for a The FileList may be incomplete of more than `limit` files are in an archive. See iter_files() for a
...@@ -302,7 +328,9 @@ class CDStar: ...@@ -302,7 +328,9 @@ class CDStar:
return self.rest("GET", vault, archive, params=query) return self.rest("GET", vault, archive, params=query)
def iter_files(self, vault, archive, offset=0, **args) -> typing.Iterator[JsonObject]: def iter_files(
self, vault, archive, offset=0, **args
) -> typing.Iterator[JsonObject]:
""" Yield all FileInfo entries of an archive. """ Yield all FileInfo entries of an archive.
This method may (lazily) issue more than one request if an archive contains more than `limit` files. This method may (lazily) issue more than one request if an archive contains more than `limit` files.
...@@ -310,9 +338,9 @@ class CDStar: ...@@ -310,9 +338,9 @@ class CDStar:
while True: while True:
files = self.list_files(vault, archive, offset, **args) files = self.list_files(vault, archive, offset, **args)
if files['files'] and offset + files['count'] <= files['total']: if files["files"] and offset + files["count"] <= files["total"]:
yield from files['files'] yield from files["files"]
offset += files['count'] offset += files["count"]
else: else:
break break
...@@ -320,20 +348,22 @@ class CDStar: ...@@ -320,20 +348,22 @@ class CDStar:
""" Remove a archive file. This cannot be undone. """ """ Remove a archive file. This cannot be undone. """
return self.raw("DELETE", vault, archive, file).ok return self.raw("DELETE", vault, archive, file).ok
def search(self, vault, q, order=None, limit=0, scroll=None, groups=None) -> JsonObject: def search(
self, vault, q, order=None, limit=0, scroll=None, groups=None
) -> JsonObject:
""" Perform a search and return a single page of search results. """ Perform a search and return a single page of search results.
See iter_search() for a convenient way to fetch more than `limit` results. See iter_search() for a convenient way to fetch more than `limit` results.
""" """
params = {"q": q} params = {"q": q}
if order: if order:
params['order'] = order params["order"] = order
if limit: if limit:
params['limit'] = limit params["limit"] = limit
if scroll: if scroll:
params['scroll'] = scroll params["scroll"] = scroll
if groups: if groups:
params['groups'] = groups params["groups"] = groups
return self.rest("GET", vault, params=params) return self.rest("GET", vault, params=params)
def iter_search(self, vault, q, scroll=None, **args) -> typing.Iterator[JsonObject]: def iter_search(self, vault, q, scroll=None, **args) -> typing.Iterator[JsonObject]:
...@@ -343,9 +373,9 @@ class CDStar: ...@@ -343,9 +373,9 @@ class CDStar:
""" """
while True: while True:
page = self.search(vault, q, scroll=scroll or "", **args) page = self.search(vault, q, scroll=scroll or "", **args)
if page['hits']: if page["hits"]:
yield from page['hits'] yield from page["hits"]
scroll = page['scroll'] scroll = page["scroll"]
else: else:
break break
...@@ -360,7 +390,11 @@ def _fix_filename(name): ...@@ -360,7 +390,11 @@ def _fix_filename(name):
# Fail hard on relative filenames # Fail hard on relative filenames
if name != os.path.normpath(name): if name != os.path.normpath(name):
raise ValueError("Archive file name not in a normalized form: {} != {}".format(name, os.path.normpath(name))) raise ValueError(
"Archive file name not in a normalized form: {} != {}".format(
name, os.path.normpath(name)
)
)
return name return name
...@@ -377,6 +411,7 @@ class CDStarVault: ...@@ -377,6 +411,7 @@ class CDStarVault:
This handle, as well als other handles returned by it, are just lightweight pointers to remote resources. This handle, as well als other handles returned by it, are just lightweight pointers to remote resources.
No remote state is cached locally and most method calls will trigger REST requests. No remote state is cached locally and most method calls will trigger REST requests.
""" """
__slots__ = "api", "name" __slots__ = "api", "name"
def __init__(self, api: CDStar, vault: str): def __init__(self, api: CDStar, vault: str):
...@@ -418,6 +453,7 @@ class CDStarArchive: ...@@ -418,6 +453,7 @@ class CDStarArchive:
See :class:`CDStarVault` for details on how handles work. See :class:`CDStarVault` for details on how handles work.
""" """
__slots__ = "api", "vault", "id" __slots__ = "api", "vault", "id"
def __init__(self, vault: CDStarVault, archive_id): def __init__(self, vault: CDStarVault, archive_id):
...@@ -450,6 +486,7 @@ class CDStarFile: ...@@ -450,6 +486,7 @@ class CDStarFile:
See :class:`CDStarVault` for details on how handles work. See :class:`CDStarVault` for details on how handles work.
""" """
__slots__ = "api", "archive", "name" __slots__ = "api", "archive", "name"
def __init__(self, archive: CDStarArchive, name: str): def __init__(self, archive: CDStarArchive, name: str):
...@@ -469,10 +506,14 @@ class CDStarFile: ...@@ -469,10 +506,14 @@ class CDStarFile:
def put(self, **ka) -> JsonObject: def put(self, **ka) -> JsonObject:
""" Create or overwrite file content """ """ Create or overwrite file content """
return self.api.put_file(self.archive.vault.name, self.archive.id, self.name, **ka) return self.api.put_file(
self.archive.vault.name, self.archive.id, self.name, **ka
)
def stream(self, **ka) -> FileDownload: def stream(self, **ka) -> FileDownload:
""" Request file content as a stream-able :class:`FileDownload`. """ """ Request file content as a stream-able :class:`FileDownload`. """
return self.api.get_file(self.archive.vault.name, self.archive.id, self.name, **ka) return self.api.get_file(
self.archive.vault.name, self.archive.id, self.name, **ka
)
# TODO: Implement meee # TODO: Implement meee
...@@ -11,18 +11,33 @@ from pycdstar3 import __version__ as VERSION, ApiError ...@@ -11,18 +11,33 @@ from pycdstar3 import __version__ as VERSION, ApiError
__ALL__ = ["main", "printer"] __ALL__ = ["main", "printer"]
parser = argparse.ArgumentParser(prog="pycdstar3") parser = argparse.ArgumentParser(prog="pycdstar3")
parser.add_argument("--server", metavar="URI", parser.add_argument(
help="CDSTAR server URI. Defaults to CDSTAR_SERVER environment variable or workspace settings.") "--server",
parser.add_argument("--vault", metavar="NAME", metavar="URI",
help="Vault to work with. Defaults to CDSTAR_VAULT environment variable or workspace settings.") help="CDSTAR server URI. Defaults to CDSTAR_SERVER environment variable or workspace settings.",
)
parser.add_argument(
"--vault",
metavar="NAME",
help="Vault to work with. Defaults to CDSTAR_VAULT environment variable or workspace settings.",
)
parser.add_argument("--version", action="store_true", help="Print version and exit.") parser.add_argument("--version", action="store_true", help="Print version and exit.")
_grp = parser.add_mutually_exclusive_group() _grp = parser.add_mutually_exclusive_group()
_grp.add_argument("-v", "--verbose", action="count", default=0, _grp.add_argument(
help="Print more info. Repeat to increase verbosity.") "-v",
_grp.add_argument("-q", "--quiet", action="store_true", help="Be quiet. Only print errors.") "--verbose",
subparsers = parser.add_subparsers(title="available commands", action="count",
description='Run "COMAMND -h" to get help for a specific command.', default=0,
metavar="COMMAND") help="Print more info. Repeat to increase verbosity.",
)
_grp.add_argument(
"-q", "--quiet", action="store_true", help="Be quiet. Only print errors."
)
subparsers = parser.add_subparsers(
title="available commands",
description='Run "COMAMND -h" to get help for a specific command.',
metavar="COMMAND",
)
def _autodiscover_commands(): def _autodiscover_commands():
...@@ -33,6 +48,7 @@ def _autodiscover_commands(): ...@@ -33,6 +48,7 @@ def _autodiscover_commands():
""" """
import pkgutil import pkgutil
import pycdstar3.cli.commands import pycdstar3.cli.commands
for _, name, ispkg in pkgutil.iter_modules(pycdstar3.cli.commands.__path__): for _, name, ispkg in pkgutil.iter_modules(pycdstar3.cli.commands.__path__):
if ispkg: if ispkg:
continue continue
...@@ -58,6 +74,7 @@ def main(*args): # noqa: C901 ...@@ -58,6 +74,7 @@ def main(*args): # noqa: C901
return 1 return 1
from pycdstar3.cli.context import CliContext from pycdstar3.cli.context import CliContext
ctx = CliContext(opts, workdir=".") ctx = CliContext(opts, workdir=".")
try: try:
......
...@@ -16,7 +16,7 @@ def walk_up(start): ...@@ -16,7 +16,7 @@ def walk_up(start):
def hbytes(n, units=None): def hbytes(n, units=None):
for unit in units or ('B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'): for unit in units or ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"):
if abs(n) < 1024.0: if abs(n) < 1024.0:
return "{:.1f} {}".format(n, unit) return "{:.1f} {}".format(n, unit)
n /= 1024.0 n /= 1024.0
...@@ -27,7 +27,7 @@ def kvtype(val): ...@@ -27,7 +27,7 @@ def kvtype(val):
""" Argparse type that parses a KEY=VALUE parameter into a tuple. """ Argparse type that parses a KEY=VALUE parameter into a tuple.
The value may be empty, but the '=' is required. """ The value may be empty, but the '=' is required. """
k, _, v = val.partition('=') k, _, v = val.partition("=")
if not _: if not _:
raise argparse.ArgumentTypeError("Expected KAY=VALUE argument.") raise argparse.ArgumentTypeError("Expected KAY=VALUE argument.")
return k, v return k, v
...@@ -39,24 +39,25 @@ def globtype(str): ...@@ -39,24 +39,25 @@ def globtype(str):
def compile_glob(pattern): def compile_glob(pattern):
parts = re.split(r'(\*\*|\*|\?)', pattern) parts = re.split(r"(\*\*|\*|\?)", pattern)
res = ["^" if pattern.startswith("/") else ".*"] res = ["^" if pattern.startswith("/") else ".*"]
for i, part in enumerate(parts): for i, part in enumerate(parts):
if i % 2 == 0: if i % 2 == 0:
res.append(re.escape(part)) res.append(re.escape(part))
elif part == '*': elif part == "*":
res.append(r'[^/]+') res.append(r"[^/]+")
elif part == '**': elif part == "**":
res.append(r'.+') res.append(r".+")
elif part == '?': elif part == "?":
res.append(r'[^/]') res.append(r"[^/]")
return re.compile(''.join(res) + "$") return re.compile("".join(res) + "$")
class FileProgress: class FileProgress:
def __init__(self, fp, chunksize=1024 * 8, **baropts): def __init__(self, fp, chunksize=1024 * 8, **baropts):
from requests.utils import super_len from requests.utils import super_len
self.opts = baropts self.opts = baropts
self.fp = fp self.fp = fp
self.len = super_len(fp) self.len = super_len(fp)
...@@ -64,8 +65,15 @@ class FileProgress: ...@@ -64,8 +65,15 @@ class FileProgress: