Gitlab Community Edition Instance

Skip to content

GitLab

Commit 0958afbe authored by mhellka's avatar mhellka
Browse files

New maximum line length: 88

parent 3a587531
Hide whitespace changes
Inline Side-by-side
src/pycdstar3/_utils.py
View file @ 0958afbe
......@@ -20,10 +20,11 @@ class cached_property(object):
def url_split_auth(url: str):
""" Extract and remove auth information from an URL. Return a (cleaned-url, username, password) tuple.
""" Extract and remove auth information from an URL.
Return a (cleaned-url, username, password) tuple.
The cleaned-url will have any auth information removed from the netloc part.
Username and password may be None if not present.
The cleaned-url will have any auth information removed from the netloc part.
Username and password may be None if not present.
"""
split = urllib.parse.urlsplit(url)
username, password = split.username, split.password
......@@ -38,6 +39,7 @@ def url_split_auth(url: str):
class IntervalTimer(Thread):
""" A thread that runs a function over and over until stopped.
Example::
t = IntervalTimer(30.0, f, args=None, kwargs=None)
t.start()
t.cancel()
......
src/pycdstar3/api.py
View file @ 0958afbe
"""
Client api implementation. Usually imported directly from :mod:`pycdstar` and not from here.
Client api implementation. Usually imported directly from :mod:`pycdstar` and not from
here.
"""
import os
......@@ -17,9 +18,10 @@ __all__ = "CDStar", "CDStarVault", "FormUpdate", "ApiError"
class CDStar:
""" Provide low-level methods for corresponding server-side REST endpoints.
If not documented otherwise, each method call triggers exactly one REST request and return
a :class:`pycdstar3.model.JsonObject`, which offers dict-like and attribute access to json fields.
There is no internal caching. The only state that is tracked by this class is the running transaction, if any.
If not documented otherwise, each method call triggers exactly one REST request
and return a :class:`pycdstar3.model.JsonObject`, which offers dict-like and
attribute access to json fields. There is no internal caching. The only state
that is tracked by this class is the running transaction, if any.
:param url: CDSTAR API URL, with or without auth information
:param auth: A (username, password) tuple, or None.
......@@ -51,9 +53,9 @@ class CDStar:
Error responses are thrown as `ApiError`, unless the status code is
explicitly accepted as valid via `expect_status`.
Disclaimer: Avoid using this method if there is a more specific implementation available.
If you find a feature missing from this class, please submit a feature request instead of
over-using this method.
Disclaimer: Avoid using this method if there is a more specific
implementation available. If you find a feature missing from this class,
please submit a feature request instead of over-using this method.
"""
if self.auth:
......@@ -78,9 +80,9 @@ class CDStar:
the parsed result instead of the raw response. Non-JSON responses
are errors. Empty (204) responses return None.
Disclaimer: Avoid using this method if there is a more specific implementation available.
If you find a feature missing from this class, please submit a feature request instead of
over-using this method.
Disclaimer: Avoid using this method if there is a more specific
implementation available. If you find a feature missing from this class,
please submit a feature request instead of over-using this method.
"""
# TODO: Expect json errors or non-json responses and
......@@ -93,23 +95,27 @@ class CDStar:
def begin(self, autocommit=False, readonly=False, keepalive=False):
""" Start a new transaction and return self.
Transactions are used to group multiple operations into a single atomic action. After begin(), you have to
call commit() or rollback() to apply or undo all operations made while the transaction was active.
Transactions are used to group multiple operations into a single atomic
action. After begin(), you have to call commit() or rollback() to apply
or undo all operations made while the transaction was active.
It is strongly recommended to only always start transactions as with-statements::
It is strongly recommended to always wrap transactions in with-blocks::
with cdstar.begin():
# do stuff
You can commit() or rollback() early, or even begin() a new transaction while still in the with-block, but
there can only be a single transaction per client active at a time. On exit, the current transaction is
closed automatically. If autocommit is true and no exception was raised, it is committed. Otherwise, it is
rolled back.
You can commit() or rollback() early, or even begin() a new transaction
while still in the with-block, but there can only be a single transaction
per client active at a time. On exit, the current transaction is closed
automatically. If autocommit is true and no exception was raised,
it is committed. Otherwise, it is rolled back.
:param autocommit: Commit this transaction if the with-block ended without errors. (default: False)
:param autocommit: Commit this transaction if the with-block ended without
errors. (default: False)
:param readonly: Create a (cheaper) read-only transaction. (default: False)
:param keepalive: Automatically call :meth:`keepalive` from a separate thread. This is only
required when waiting for user input for a long time. (default: False)
:param keepalive: Automatically call :meth:`keepalive` from a separate
thread. This is only required when waiting for user input
for a long time. (default: False)
"""
self.rollback()
self._autocommit = autocommit
......@@ -184,7 +190,7 @@ class CDStar:
raise RuntimeError("No transaction running. Call begin() frist.")
def __exit__(self, exc_type, exc_value, traceback):
""" Commit or roll-back the current transaction, depending on its autocommit setting. """
""" Commit or roll-back the current transaction (see autocommit) """
if self._tx:
if exc_type is None and self._autocommit:
self.commit()
......@@ -248,15 +254,17 @@ class CDStar:
) -> JsonObject:
""" Create or replace a single file on an existing archive.
If the file exists remotely and `replace=True` is set (default), the file content is overridden but everything
else (metadata, type, file id) stays the same. If `replace` is `False` then a file name conflict is an error.
If the file exists remotely and `replace=True` is set (default), the file
content is overridden but everything else (metadata, type, file id) stays the
same. If `replace` is `False` then a file name conflict is an error.
:param vault: Vault name
:param archive: Archive ID
:param name: Target file name. May start with `/` (optional) but must not end with `/`.
:param source: Readable file, byte buffer or iterator, or a file path that will then be opened in 'rb' mode.
:param type: Mime-type to set on the uploaded file. (default: guess based on filename)
:param replace: If the remote file already exists, replace its content. (default: True)
:param name: Target file name. May start with `/` (optional).
:param source: Readable file, byte buffer or iterator, or a file path that will
then be opened in 'rb' mode.
:param type: Mime-type to set on the uploaded file. (default: guess)
:param replace: Replace existing remote files (default: True)
:return:
"""
......@@ -277,8 +285,9 @@ class CDStar:
def get_file(self, vault, archive, name, offset=0) -> FileDownload:
""" Request a file and return a stream-able :class:`FileDownload`.
The request is issued with `stream=True`, which means it is still open and not fully read when this
method returns. The returned wrapper MUST be `close()`d after use, or wrapped in a `with` statement::
The request is issued with `stream=True`, which means it is still open and
not fully read when this method returns. The returned wrapper MUST be
`close()`d after use, or wrapped in a `with` statement::
with cdstar.get_file(vault, id, "/file/name.txt") as dl:
dl.save_to("~/Downloads/")
......@@ -310,8 +319,8 @@ class CDStar:
) -> JsonObject:
""" Request a FileList for an archive.
The FileList may be incomplete of more than `limit` files are in an archive. See iter_files() for a
convenient way to get all files as an iterator.
The FileList may be incomplete of more than `limit` files are in an archive.
See iter_files() for a convenient way to get all files as an iterator.
"""
query = {"files": "true", "offset": offset, "limit": limit}
......@@ -333,7 +342,8 @@ class CDStar:
) -> typing.Iterator[JsonObject]:
""" Yield all FileInfo entries of an archive.
This method may (lazily) issue more than one request if an archive contains more than `limit` files.
This method may (lazily) issue more than one request if an archive contains
more than `limit` files.
"""
while True:
......@@ -369,7 +379,8 @@ class CDStar:
def iter_search(self, vault, q, scroll=None, **args) -> typing.Iterator[JsonObject]:
""" Yield all search hits of a search.
This method may (lazily) issue more than one request if a search returns more than `limit` results.
This method may (lazily) issue more than one request if a search returns
more than `limit` results.
"""
while True:
page = self.search(vault, q, scroll=scroll or "", **args)
......@@ -399,17 +410,19 @@ def _fix_filename(name):
# Design notes for the following resource handles:
# - The handle instances are really just a slim handle for a remote resource, NOT a wrapper, local copy or cache.
# They should not cache or store anything that might change remotely.
# - The handle instances are really just a slim handle for a remote resource, NOT a
# wrapper, local copy or cache. They should not cache or store anything that might
# change remotely.
# - Only methods are allowed to trigger requests, preferably only one request per call.
# - Handles MUST implement exists()->bool and info()->JsonObject.
class CDStarVault:
""" Handle for a CDSTAR vault, providing a more fluent and object-oriented API on top of :class:`CDStar`.
""" Fluent API handle for a CDSTAR vault.
This handle, as well als other handles returned by it, are just lightweight pointers to remote resources.
No remote state is cached locally and most method calls will trigger REST requests.
This is just a thin wrapper to provide a more fluent and object-oriented API on
top of :class:`CDStar`. No remote state is cached locally and most method calls
will trigger REST requests.
"""
__slots__ = "api", "name"
......@@ -444,7 +457,9 @@ class CDStarVault:
return self.api.search(self.name, *a, **ka)
def iter_search(self, *a, **ka) -> typing.Iterator[JsonObject]:
""" Search in this vault. Return a result iterator, which lazily issues more requests on demand. """
""" Search in this vault.
Return a result iterator, which lazily issues more requests on demand. """
return self.api.iter_search(self.name, *a, **ka)
......
src/pycdstar3/cli/__init__.py
View file @ 0958afbe
......@@ -14,12 +14,14 @@ parser = argparse.ArgumentParser(prog="pycdstar3")
parser.add_argument(
"--server",
metavar="URI",
help="CDSTAR server URI. Defaults to CDSTAR_SERVER environment variable or workspace settings.",
help="CDSTAR server URI. Defaults to CDSTAR_SERVER environment variable or"
" workspace settings.",
)
parser.add_argument(
"--vault",
metavar="NAME",
help="Vault to work with. Defaults to CDSTAR_VAULT environment variable or workspace settings.",
help="Vault to work with. Defaults to CDSTAR_VAULT environment variable or"
" workspace settings.",
)
parser.add_argument("--version", action="store_true", help="Print version and exit.")
_grp = parser.add_mutually_exclusive_group()
......@@ -43,8 +45,8 @@ subparsers = parser.add_subparsers(
def _autodiscover_commands():
""" Autodiscover and import all modules in the pycdstar3.cli.commands namespace.
This also works for namespace packages. Another approach would be to auto-discover
all top-level modules named `pycdstar3_*`.
This also works for namespace packages. Another approach would be to
auto-discover all top-level modules named `pycdstar3_*`.
"""
import pkgutil
import pycdstar3.cli.commands
......
src/pycdstar3/cli/_utils.py
View file @ 0958afbe
......@@ -145,7 +145,7 @@ class Printer:
self._print("WARN: " + msg, *args, **kwargs)
def error(self, msg, *args, **kwargs):
""" Print an error message (if not quiet) and optionally (-vv or higher) a stacktrace."""
""" Print an error message (if not quiet) and optionally (-vv) a stacktrace."""
self._print("ERROR: " + msg, *args, **kwargs)
if self.verbosity >= 2:
import traceback
......@@ -153,7 +153,7 @@ class Printer:
self._print(traceback.format_exc(), highlight="=")
def fatal(self, msg, *args, **kwargs):
""" Print an error message (even if quiet) and optionally (-vv or higher) a stacktrace."""
""" Print an error message (even if quiet) and optionally (-vv) a stacktrace."""
self._print("FATAL: " + msg, *args, **kwargs)
if self.verbosity >= 2:
import traceback
......
src/pycdstar3/cli/commands/get.py
View file @ 0958afbe
......@@ -87,9 +87,8 @@ def get(ctx, args): # noqa: C901
if ispipe and out.isatty() and not dl.type.startswith("text/") and not force:
raise CliError(
"Not printing binary data ({}) to a terminal. Use --force to override.".format(
dl.type
)
"Not printing binary data ({}) to a terminal."
" Use --force to override.".format(dl.type)
)
if progress and not ctx.print.quiet:
......
src/pycdstar3/cli/commands/init.py
View file @ 0958afbe
"""
Initialize a cdstar working directory.
Create a config file in the current directory, so it can be found by future invocations of cdstar-cli commands.
Settings not provided as command line arguments are asked for interactively.
Create a config file in the current directory, so it can be found by future invocations
of cdstar-cli commands. Settings not provided as command line arguments are asked for
interactively.
If the main --config parameter is set, the configuration is saved at the specified location
instead of the current working directory.
If the main --config parameter is set, the configuration is saved at the specified
location instead of the current working directory.
"""
import os
......
src/pycdstar3/cli/commands/ls.py
View file @ 0958afbe
......@@ -26,7 +26,8 @@ def register(subparsers):
"--order",
default="name",
choices=["name", "type", "size", "created", "modified", "hash", "id"],
help="Order by name, type, size, created, modified, hash or id. (default: name)",
help="Order by name, type, size, created, modified, hash or id. "
"(default: name)",
)
parser.add_argument("--reverse", action="store_true", help="Reverse list order")
parser.add_argument(
......
src/pycdstar3/cli/commands/put.py
View file @ 0958afbe
"""
Upload files to an archive.
You can also create new archives and set ACL entries or metadata attributes with this command.
It works a bit as a swiss army knife for simple archive creation or manipulation. For everything not covered here,
there are more specialized commands available.
You can also create new archives and set ACL entries or metadata attributes with this
command. It works a bit as a swiss army knife for simple archive creation or
manipulation. For everything not covered here, there are more specialized commands
available.
File upload is save by default: Existing remote files are not overwritten. You can --force re-upload or just --update
newer files. Note that --update relies on a working clock on both local and remote side, as it compares the file
modification times only.
File upload is save by default: Existing remote files are not overwritten.
You can --force re-upload or just --update newer files. Note that --update relies on a
working clock on both local and remote side, as it compares the file modification times
only.
All uploads and changes are wrapped in a transaction. If something goes wrong, nothing is committed on remote side and
you can simply re-run the same command.
All uploads and changes are wrapped in a transaction. If something goes wrong,
nothing is committed on remote side and you can simply re-run the same command.
"""
import os
......@@ -37,7 +39,8 @@ def register(subparsers):
)
# parser.add_argument("--delete", action="store_true",
# help="Delete remote files not present locally, if they match a PATH parameter.")
# help="Delete remote files not present locally,"
# " if they match a PATH parameter.")
parser.add_argument(
"-i",
"--include",
......@@ -69,28 +72,30 @@ def register(subparsers):
parser.add_argument(
"--flat",
action="store_true",
help="Strip local directory names and only use the basename when uploading files."
" (e.g. ./path/to/file.txt would be uploaded as /file.txt)",
help="Strip local directory names and only use the basename when uploading"
" files. (e.g. ./path/to/file.txt would be uploaded as /file.txt)",
)
# parser.add_argument("--tus", action="store_true",
# help="Upload large files via tus.io and retry on connection errors (needs server support)")
# help="Upload large files via tus.io and retry on connection"
# " errors (needs server support)")
parser.add_argument(
"--meta",
metavar="KEY=VAL",
type=kvtype,
action="append",
help="Set archive metadata attributes. An empty value removes the attribute. Can be repeated to"
" set multiple values for the same attribute.",
help="Set archive metadata attributes. An empty value removes the attribute."
" Can be repeated to set multiple values for the same attribute.",
)
parser.add_argument(
"--acl",
metavar="SUBJECT=PERM",
type=kvtype,
action="append",
help="Set ACL entries. PERM can be a comma separated list of permissions or permission sets. "
" An empty PERM value removes all permissions for that SUBJECT.",
help="Set ACL entries. PERM can be a comma separated list of permissions or"
" permission sets. An empty PERM value removes all permissions for that"
" SUBJECT.",
)
parser.add_argument(
......@@ -238,7 +243,8 @@ def command(ctx, args): # noqa: C901
client.put_file(vault, archive, target, fp, replace=force)
except ApiError as e:
if e.status == 412:
# TODO: Make more specific as soon as CDSTAR returns a proper error code.
# TODO: Make more specific as soon as CDSTAR returns a proper
# error code.
ctx.print.warn("Upload failed (file exists): {}", target)
continue
raise
......
src/pycdstar3/cli/commands/rma.py
View file @ 0958afbe
......@@ -36,8 +36,9 @@ def remove_files(ctx, args):
if len(archives) > 1:
stack.enter_context(client.begin(autocommit=True))
prompt = "Do you really want to delete {} archives? This cannot be undone!".format(
len(archives)
prompt = (
"Do you really want to delete {} archives? "
"This cannot be undone!".format(len(archives))
)
if not (yes or force or ctx.ask_yes(prompt)):
return
......
src/pycdstar3/cli/commands/search.py
View file @ 0958afbe
......@@ -25,7 +25,8 @@ def register(subparsers):
parser.add_argument(
"--no-scroll",
action="store_true",
help="Disables auto-fetching more results if less than --limit hits were returned.",
help="Disables auto-fetching more results scrolling if less than --limit hits "
"were returned.",
)
parser.add_argument(
"QUERY", help="Search query. Syntax depends on back-end configuration."
......
src/pycdstar3/cli/context.py
View file @ 0958afbe
......@@ -23,8 +23,9 @@ ENV_PREFIX = "CDSTAR_"
class CliContext:
""" Provide context and tools to CLI commands.
This class may provide anything that is used by more than one command. For example a ready to use CDSTAR client,
workspace config settings, default vault or other global settings. Some properties may raise CliError ask for
This class may provide anything that is used by more than one command.
For example a ready to use CDSTAR client, workspace config settings, default
vault or other global settings. Some properties may raise CliError ask for
user input.
"""
......@@ -34,8 +35,9 @@ class CliContext:
@cached_property
def print(self) -> Printer:
""" An instance of :class:`pycdstar3.cli._utils.Printer` to print optional messages to the user.
Messages are printed to stderr, so only use it for complementary information, not for the primary results.
""" An instance of :class:`pycdstar3.cli._utils.Printer` to print optional
messages to the user. Messages are printed to stderr, so only use it for
complementary information, not for the primary results.
"""
printer = Printer(level=0, file=sys.stderr)
......@@ -67,7 +69,8 @@ class CliContext:
return ws
def _get_setting(self, name) -> typing.Any:
""" Look for a setting in command-line arguments, environment variables or workspace configuration. """
""" Look for a setting in command-line arguments, environment variables or
workspace configuration. """
return (
getattr(self.args, name, None)
or os.environ.get(ENV_PREFIX + name.upper())
......@@ -80,13 +83,12 @@ class CliContext:
return result
raise CliError(
"Missing --{} parameter.\n\n"
" Alternatively, set the {} environment variable or create a workspace.".format(
name, ENV_PREFIX + name.upper()
)
" Alternatively, set the {} environment variable or create a "
"workspace.".format(name, ENV_PREFIX + name.upper())
)
def _ask_pass(self, url):
# Enter password for {url.scheme}://{url.netloc}/{url.path} (user={url.username})
# Enter password for {url.scheme}://{url.netloc}/{url.path}
raise RuntimeError("Asking for password not implemented yet")
def ask_yes(self, prompt, default="yes") -> bool:
......@@ -120,7 +122,8 @@ def _find_workspace(start="."):
class Workspace:
""" Workspace directory with configuration and more.
Currently a workspace directory is simply a folder which contains a `.cdstar/workspace.conf` file.
Currently a workspace directory is simply a folder which contains a
`.cdstar/workspace.conf` file.
"""
def __init__(self, root, config_dir=CONFDIR_NAME, config_file=CONFIG_NAME):
......
src/pycdstar3/model.py
View file @ 0958afbe
......@@ -89,12 +89,14 @@ class FileDownload:
)
def readall(self):
""" Read the entire download into memory and return a single large byte object. """
""" Read the entire download into memory and return a single large byte object.
"""
return self.response.content
class FormUpdate:
""" Builder for CDSTAR POST multipart/form-data requests to upload multiple files or change aspects of an archive.
""" Builder for CDSTAR POST multipart/form-data requests to upload multiple files or
change aspects of an archive.
"""
def __init__(self):
......@@ -125,7 +127,8 @@ class FormUpdate:
if not name.startswith("/"):
name = "/" + name
if isinstance(src, PATH_TYPES):
# TODO: Check what types are accepted as file-like and buold a lazily opened wrapper.
# TODO: Check what types are accepted as file-like and build a lazily opened
# wrapper.
self.fields.append((name, (os.path.basename(src), open(src, "rb"), type)))
elif hasattr(src, "fileno") or hasattr(src, "getvalue"):
self.fields.append((name, (os.path.basename(src), src, type)))
......@@ -136,7 +139,8 @@ class FormUpdate:
return self
def acl(self, subject, *permissions):
""" Set permissions for a subject. Existing permissions for the same subject are replaced.
""" Set permissions for a subject. Existing permissions for the same subject are
replaced.
:param subject: A subject-name, @groupname or one of `$any`, `$user`, `$user`
:param permissions:
......@@ -153,9 +157,11 @@ class FormUpdate:
"""
Set metadata for the archive, or a file within the archive.
:param field: Meta-attribute field name. Should start with a schema prefix (e.g. `dc:` for DublinCore)
:param field: Meta-attribute field name. Should start with a schema prefix
(e.g. `dc:` for DublinCore)
:param values: Values for this meta attribute.
:param file: File name to attach the metadata to. If not set, it is assigned to the entire archive.
:param file: File name to attach the metadata to. If not set, it is assigned to
the entire archive.
:return: self
"""
......@@ -186,9 +192,8 @@ class ApiError(Exception):
except JSONDecodeError:
raise ValueError(
"Failed to decode server response (invalid JSON):"
" {0.method} {0.url} -> {1.status_code} ({1.headers[content-type]})".format(
rs.request, rs
)
" {0.method} {0.url} ->"
" {1.status_code} ({1.headers[content-type]})".format(rs.request, rs)
)
@property
......
tox.ini
View file @ 0958afbe
......@@ -9,5 +9,5 @@ passenv = TEST_CDSTAR TEST_VAULT
[flake8]
ignore = E121,E123,E126,E127,E133,E226,E241,E242,E704,W503,W504,W505,
max-line-length = 160
max-line-length = 88
max-complexity = 10
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment