From 09221c57e1b82e9f4ea6f23745cf59838508965c Mon Sep 17 00:00:00 2001 From: Marcel Parciak <marcel.parciak@gmail.com> Date: Thu, 16 Jul 2020 08:19:39 +0200 Subject: [PATCH 1/8] Added SQL storage of upload procedures. Signed-off-by: Marcel Parciak <marcel.parciak@gmail.com> --- .gitignore | 141 +++++++- Pipfile | 4 + Pipfile.lock | 302 +++++++++++++++--- alembic.ini | 85 +++++ alembic/README | 1 + alembic/env.py | 77 +++++ alembic/script.py.mako | 24 ++ .../535aa0f40ba9_initial_table_layout.py | 45 +++ prestart.sh | 6 + requirements.txt | 16 +- setup.py | 129 ++++++++ uploader/__version__.py | 4 + uploader/cdstar.py | 95 ++++++ uploader/config.py | 6 +- uploader/database.py | 10 + uploader/main.py | 181 +++++++---- uploader/models_activeworkflow.py | 8 +- uploader/models_internal.py | 21 ++ uploader/store.py | 122 +++---- 19 files changed, 1070 insertions(+), 207 deletions(-) create mode 100644 alembic.ini create mode 100644 alembic/README create mode 100644 alembic/env.py create mode 100644 alembic/script.py.mako create mode 100644 alembic/versions/535aa0f40ba9_initial_table_layout.py create mode 100644 prestart.sh create mode 100644 setup.py create mode 100644 uploader/__version__.py create mode 100644 uploader/cdstar.py create mode 100644 uploader/database.py create mode 100644 uploader/models_internal.py diff --git a/.gitignore b/.gitignore index 05c6168..836cf32 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,141 @@ # Do not upload the test directory, this is for local testing only -test \ No newline at end of file +test + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ \ No newline at end of file diff --git a/Pipfile b/Pipfile index afb855e..102bc15 100644 --- a/Pipfile +++ b/Pipfile @@ -7,9 +7,13 @@ verify_ssl = true fastapi = "*" uvicorn = "*" black = "*" +alembic = "*" [packages] 3-0-dev0 = {git = "https://gitlab.gwdg.de/cdstar/pycdstar3.git"} +psycopg2-binary = "*" +sqlalchemy = "*" +umg-datalake-upload-agent = {editable = true,path = "."} [requires] python_version = "3.8" diff --git a/Pipfile.lock b/Pipfile.lock index 95ffa36..d39471f 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "70748a47ce3ee97754bd53f9fad84aa0ab98b8ea6a50df2ae04073de9e6f681c" + "sha256": "30e12061e05ef407521b674b77f6abb9f5dcb8b9ecfde9450bc00cc413b89090" }, "pipfile-spec": 6, "requires": { @@ -19,9 +19,126 @@ "3-0-dev0": { "git": "https://gitlab.gwdg.de/cdstar/pycdstar3.git", "ref": "a0e0fd5ee137d09988756ab19c9b68f428ff7d38" + }, + "fastapi": { + "hashes": [ + "sha256:50b58aa3e7d5bcb4a4404ac7e550cc53f0cf7ca0fd13c7fd515693dc23c9caef", + "sha256:c04dacd3deed0fd0ffdcdb116b5a04ffa257656885be7fae7234f4f62ec4a0a9" + ], + "version": "==0.59.0" + }, + "psycopg2-binary": { + "hashes": [ + "sha256:008da3ab51adc70a5f1cfbbe5db3a22607ab030eb44bcecf517ad11a0c2b3cac", + "sha256:07cf82c870ec2d2ce94d18e70c13323c89f2f2a2628cbf1feee700630be2519a", + "sha256:08507efbe532029adee21b8d4c999170a83760d38249936038bd0602327029b5", + "sha256:107d9be3b614e52a192719c6bf32e8813030020ea1d1215daa86ded9a24d8b04", + "sha256:17a0ea0b0eabf07035e5e0d520dabc7950aeb15a17c6d36128ba99b2721b25b1", + "sha256:3286541b9d85a340ee4ed42732d15fc1bb441dc500c97243a768154ab8505bb5", + "sha256:3939cf75fc89c5e9ed836e228c4a63604dff95ad19aed2bbf71d5d04c15ed5ce", + "sha256:40abc319f7f26c042a11658bf3dd3b0b3bceccf883ec1c565d5c909a90204434", + "sha256:51f7823f1b087d2020d8e8c9e6687473d3d239ba9afc162d9b2ab6e80b53f9f9", + "sha256:6bb2dd006a46a4a4ce95201f836194eb6a1e863f69ee5bab506673e0ca767057", + "sha256:702f09d8f77dc4794651f650828791af82f7c2efd8c91ae79e3d9fe4bb7d4c98", + "sha256:7036ccf715925251fac969f4da9ad37e4b7e211b1e920860148a10c0de963522", + "sha256:7b832d76cc65c092abd9505cc670c4e3421fd136fb6ea5b94efbe4c146572505", + "sha256:8f74e631b67482d504d7e9cf364071fc5d54c28e79a093ff402d5f8f81e23bfa", + "sha256:930315ac53dc65cbf52ab6b6d27422611f5fb461d763c531db229c7e1af6c0b3", + "sha256:96d3038f5bd061401996614f65d27a4ecb62d843eb4f48e212e6d129171a721f", + "sha256:a20299ee0ea2f9cca494396ac472d6e636745652a64a418b39522c120fd0a0a4", + "sha256:a34826d6465c2e2bbe9d0605f944f19d2480589f89863ed5f091943be27c9de4", + "sha256:a69970ee896e21db4c57e398646af9edc71c003bc52a3cc77fb150240fefd266", + "sha256:b9a8b391c2b0321e0cd7ec6b4cfcc3dd6349347bd1207d48bcb752aa6c553a66", + "sha256:ba13346ff6d3eb2dca0b6fa0d8a9d999eff3dcd9b55f3a890f12b0b6362b2b38", + "sha256:bb0608694a91db1e230b4a314e8ed00ad07ed0c518f9a69b83af2717e31291a3", + "sha256:c8830b7d5f16fd79d39b21e3d94f247219036b29b30c8270314c46bf8b732389", + "sha256:cac918cd7c4c498a60f5d2a61d4f0a6091c2c9490d81bc805c963444032d0dab", + "sha256:cc30cb900f42c8a246e2cb76539d9726f407330bc244ca7729c41a44e8d807fb", + "sha256:ccdc6a87f32b491129ada4b87a43b1895cf2c20fdb7f98ad979647506ffc41b6", + "sha256:d1a8b01f6a964fec702d6b6dac1f91f2b9f9fe41b310cbb16c7ef1fac82df06d", + "sha256:e004db88e5a75e5fdab1620fb9f90c9598c2a195a594225ac4ed2a6f1c23e162", + "sha256:eb2f43ae3037f1ef5e19339c41cf56947021ac892f668765cd65f8ab9814192e", + "sha256:fa466306fcf6b39b8a61d003123d442b23707d635a5cb05ac4e1b62cc79105cd" + ], + "index": "pypi", + "version": "==2.8.5" + }, + "pydantic": { + "hashes": [ + "sha256:1783c1d927f9e1366e0e0609ae324039b2479a1a282a98ed6a6836c9ed02002c", + "sha256:2dc946b07cf24bee4737ced0ae77e2ea6bc97489ba5a035b603bd1b40ad81f7e", + "sha256:2de562a456c4ecdc80cf1a8c3e70c666625f7d02d89a6174ecf63754c734592e", + "sha256:36dbf6f1be212ab37b5fda07667461a9219c956181aa5570a00edfb0acdfe4a1", + "sha256:3fa799f3cfff3e5f536cbd389368fc96a44bb30308f258c94ee76b73bd60531d", + "sha256:40d765fa2d31d5be8e29c1794657ad46f5ee583a565c83cea56630d3ae5878b9", + "sha256:418b84654b60e44c0cdd5384294b0e4bc1ebf42d6e873819424f3b78b8690614", + "sha256:4900b8820b687c9a3ed753684337979574df20e6ebe4227381d04b3c3c628f99", + "sha256:530d7222a2786a97bc59ee0e0ebbe23728f82974b1f1ad9a11cd966143410633", + "sha256:54122a8ed6b75fe1dd80797f8251ad2063ea348a03b77218d73ea9fe19bd4e73", + "sha256:6c3f162ba175678218629f446a947e3356415b6b09122dcb364e58c442c645a7", + "sha256:b49c86aecde15cde33835d5d6360e55f5e0067bb7143a8303bf03b872935c75b", + "sha256:b5b3489cb303d0f41ad4a7390cf606a5f2c7a94dcba20c051cd1c653694cb14d", + "sha256:cf3933c98cb5e808b62fae509f74f209730b180b1e3c3954ee3f7949e083a7df", + "sha256:eb75dc1809875d5738df14b6566ccf9fd9c0bcde4f36b72870f318f16b9f5c20", + "sha256:f769141ab0abfadf3305d4fcf36660e5cf568a666dd3efab7c3d4782f70946b1", + "sha256:f8af9b840a9074e08c0e6dc93101de84ba95df89b267bf7151d74c553d66833b" + ], + "version": "==1.6.1" + }, + "sqlalchemy": { + "hashes": [ + "sha256:0942a3a0df3f6131580eddd26d99071b48cfe5aaf3eab2783076fbc5a1c1882e", + "sha256:0ec575db1b54909750332c2e335c2bb11257883914a03bc5a3306a4488ecc772", + "sha256:109581ccc8915001e8037b73c29590e78ce74be49ca0a3630a23831f9e3ed6c7", + "sha256:16593fd748944726540cd20f7e83afec816c2ac96b082e26ae226e8f7e9688cf", + "sha256:427273b08efc16a85aa2b39892817e78e3ed074fcb89b2a51c4979bae7e7ba98", + "sha256:50c4ee32f0e1581828843267d8de35c3298e86ceecd5e9017dc45788be70a864", + "sha256:512a85c3c8c3995cc91af3e90f38f460da5d3cade8dc3a229c8e0879037547c9", + "sha256:57aa843b783179ab72e863512e14bdcba186641daf69e4e3a5761d705dcc35b1", + "sha256:621f58cd921cd71ba6215c42954ffaa8a918eecd8c535d97befa1a8acad986dd", + "sha256:6ac2558631a81b85e7fb7a44e5035347938b0a73f5fdc27a8566777d0792a6a4", + "sha256:716754d0b5490bdcf68e1e4925edc02ac07209883314ad01a137642ddb2056f1", + "sha256:736d41cfebedecc6f159fc4ac0769dc89528a989471dc1d378ba07d29a60ba1c", + "sha256:8619b86cb68b185a778635be5b3e6018623c0761dde4df2f112896424aa27bd8", + "sha256:87fad64529cde4f1914a5b9c383628e1a8f9e3930304c09cf22c2ae118a1280e", + "sha256:89494df7f93b1836cae210c42864b292f9b31eeabca4810193761990dc689cce", + "sha256:8cac7bb373a5f1423e28de3fd5fc8063b9c8ffe8957dc1b1a59cb90453db6da1", + "sha256:8fd452dc3d49b3cc54483e033de6c006c304432e6f84b74d7b2c68afa2569ae5", + "sha256:adad60eea2c4c2a1875eb6305a0b6e61a83163f8e233586a4d6a55221ef984fe", + "sha256:c26f95e7609b821b5f08a72dab929baa0d685406b953efd7c89423a511d5c413", + "sha256:cbe1324ef52ff26ccde2cb84b8593c8bf930069dfc06c1e616f1bfd4e47f48a3", + "sha256:d05c4adae06bd0c7f696ae3ec8d993ed8ffcc4e11a76b1b35a5af8a099bd2284", + "sha256:d98bc827a1293ae767c8f2f18be3bb5151fd37ddcd7da2a5f9581baeeb7a3fa1", + "sha256:da2fb75f64792c1fc64c82313a00c728a7c301efe6a60b7a9fe35b16b4368ce7", + "sha256:e4624d7edb2576cd72bb83636cd71c8ce544d8e272f308bd80885056972ca299", + "sha256:e89e0d9e106f8a9180a4ca92a6adde60c58b1b0299e1b43bd5e0312f535fbf33", + "sha256:f11c2437fb5f812d020932119ba02d9e2bc29a6eca01a055233a8b449e3e1e7d", + "sha256:f57be5673e12763dd400fea568608700a63ce1c6bd5bdbc3cc3a2c5fdb045274", + "sha256:fc728ece3d5c772c196fd338a99798e7efac7a04f9cb6416299a3638ee9a94cd" + ], + "index": "pypi", + "version": "==1.3.18" + }, + "starlette": { + "hashes": [ + "sha256:04fe51d86fd9a594d9b71356ed322ccde5c9b448fc716ac74155e5821a922f8d", + "sha256:0fb4b38d22945b46acb880fedee7ee143fd6c0542992501be8c45c0ed737dd1a" + ], + "version": "==0.13.4" + }, + "umg-datalake-upload-agent": { + "editable": true, + "path": "." } }, "develop": { + "alembic": { + "hashes": [ + "sha256:035ab00497217628bf5d0be82d664d8713ab13d37b630084da8e1f98facf4dbf" + ], + "index": "pypi", + "version": "==1.4.2" + }, "appdirs": { "hashes": [ "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41", @@ -44,6 +161,14 @@ "index": "pypi", "version": "==19.10b0" }, + "bump2version": { + "hashes": [ + "sha256:477f0e18a0d58e50bb3dbc9af7fcda464fd0ebfc7a6151d8888602d7153171a0", + "sha256:cd4f3a231305e405ed8944d8ff35bd742d9bc740ad62f483bd0ca21ce7131984" + ], + "index": "pypi", + "version": "==1.0.0" + }, "click": { "hashes": [ "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a", @@ -56,7 +181,6 @@ "sha256:50b58aa3e7d5bcb4a4404ac7e550cc53f0cf7ca0fd13c7fd515693dc23c9caef", "sha256:c04dacd3deed0fd0ffdcdb116b5a04ffa257656885be7fae7234f4f62ec4a0a9" ], - "index": "pypi", "version": "==0.59.0" }, "h11": { @@ -84,6 +208,40 @@ "markers": "sys_platform != 'win32' and sys_platform != 'cygwin' and platform_python_implementation != 'PyPy'", "version": "==0.1.1" }, + "mako": { + "hashes": [ + "sha256:8195c8c1400ceb53496064314c6736719c6f25e7479cd24c77be3d9361cddc27", + "sha256:93729a258e4ff0747c876bd9e20df1b9758028946e976324ccd2d68245c7b6a9" + ], + "version": "==1.1.3" + }, + "markupsafe": { + "hashes": [ + "sha256:06358015a4dee8ee23ae426bf885616ab3963622defd829eb45b44e3dee3515f", + "sha256:0b0c4fc852c5f02c6277ef3b33d23fcbe89b1b227460423e3335374da046b6db", + "sha256:267677fc42afed5094fc5ea1c4236bbe4b6a00fe4b08e93451e65ae9048139c7", + "sha256:303cb70893e2c345588fb5d5b86e0ca369f9bb56942f03064c5e3e75fa7a238a", + "sha256:3c9b624a0d9ed5a5093ac4edc4e823e6b125441e60ef35d36e6f4a6fdacd5054", + "sha256:42033e14cae1f6c86fc0c3e90d04d08ce73ac8e46ba420a0d22d545c2abd4977", + "sha256:4e4a99b6af7bdc0856b50020c095848ec050356a001e1f751510aef6ab14d0e0", + "sha256:4eb07faad54bb07427d848f31030a65a49ebb0cec0b30674f91cf1ddd456bfe4", + "sha256:63a7161cd8c2bc563feeda45df62f42c860dd0675e2b8da2667f25bb3c95eaba", + "sha256:68e0fd039b68d2945b4beb947d4023ca7f8e95b708031c345762efba214ea761", + "sha256:8092a63397025c2f655acd42784b2a1528339b90b987beb9253f22e8cdbb36c3", + "sha256:841218860683c0f2223e24756843d84cc49cccdae6765e04962607754a52d3e0", + "sha256:94076b2314bd2f6cfae508ad65b4d493e3a58a50112b7a2cbb6287bdbc404ae8", + "sha256:9d22aff1c5322e402adfb3ce40839a5056c353e711c033798cf4f02eb9f5124d", + "sha256:b0e4584f62b3e5f5c1a7bcefd2b52f236505e6ef032cc508caa4f4c8dc8d3af1", + "sha256:b1163ffc1384d242964426a8164da12dbcdbc0de18ea36e2c34b898ed38c3b45", + "sha256:beac28ed60c8e838301226a7a85841d0af2068eba2dcb1a58c2d32d6c05e440e", + "sha256:c29f096ce79c03054a1101d6e5fe6bf04b0bb489165d5e0e9653fb4fe8048ee1", + "sha256:c58779966d53e5f14ba393d64e2402a7926601d1ac8adeb4e83893def79d0428", + "sha256:cfe14b37908eaf7d5506302987228bff69e1b8e7071ccd4e70fd0283b1b47f0b", + "sha256:e834249c45aa9837d0753351cdca61a4b8b383cc9ad0ff2325c97ff7b69e72a6", + "sha256:eed1b234c4499811ee85bcefa22ef5e466e75d132502226ed29740d593316c1f" + ], + "version": "==2.0.0a1" + }, "pathspec": { "hashes": [ "sha256:7d91249d21749788d07a2d0f94147accd8f845507400749ea19c1ec9054a12b0", @@ -93,51 +251,107 @@ }, "pydantic": { "hashes": [ - "sha256:1998e5f783c37853c6dfc1e6ba3a0cc486798b26920822b569ea883b38fd39eb", - "sha256:1ab625f56534edd1ecec5871e0777c9eee1c7b38f1963d97c4a78b09daf89526", - "sha256:1c97f90056c2811d58a6343f6352820c531e822941303a16ccebb77bbdcd4a98", - "sha256:2b49f9ecefcdee47f6b70fd475160eeda01c28507137d9c1ed41a758d231c060", - "sha256:390844ede21e29e762c0017e46b4105edee9dcdc0119ce1aa8ab6fe58448bd2f", - "sha256:4aa11a8a65fa891489d9e4e8f05299d80b000c554ce18b03bb1b5f0afe5b73f4", - "sha256:61e3cde8e7b8517615da5341e0b28cc01de507e7ac9174c3c0e95069482dcbeb", - "sha256:6ea91ec880de48699c4a01aa92ac8c3a71363bb6833bf031cb6aa2b99567d5ab", - "sha256:8154df22783601d9693712d1cee284c596cdb5d6817f57c1624bcb54e4611eba", - "sha256:82242b0458b0a5bad0c15c36d461b2bd99eec2d807c0c5263f802ba30cb856f0", - "sha256:a12e2fc2f5529b6aeb956f30a2b5efe46283b69e965888cd683cd1a04d75a1b8", - "sha256:aedd4df265600889907d2c74dc0432709b0ac91712f85f3ffa605b40a00f2577", - "sha256:b1c229e595b53d1397435fb7433c841c5bc4032ba81f901156124b9d077b5f6a", - "sha256:d88e55dc77241e2b78139dac33ad5edc3fd78b0c6e635824e4eeba6679371707", - "sha256:dd78ccb5cfe26ae6bc3d2a1b47b18a5267f88be43cb768aa6bea470c4ac17099", - "sha256:de093dcf6a8c6a2f92f8ac28b713b9f4ad80f1e664bdc9232ad06ac912c559fc", - "sha256:df5c511b8af11834b3061b33211e0aefb4fb8e2f94b939aa51d844cae22bad5c" - ], - "version": "==1.6" + "sha256:1783c1d927f9e1366e0e0609ae324039b2479a1a282a98ed6a6836c9ed02002c", + "sha256:2dc946b07cf24bee4737ced0ae77e2ea6bc97489ba5a035b603bd1b40ad81f7e", + "sha256:2de562a456c4ecdc80cf1a8c3e70c666625f7d02d89a6174ecf63754c734592e", + "sha256:36dbf6f1be212ab37b5fda07667461a9219c956181aa5570a00edfb0acdfe4a1", + "sha256:3fa799f3cfff3e5f536cbd389368fc96a44bb30308f258c94ee76b73bd60531d", + "sha256:40d765fa2d31d5be8e29c1794657ad46f5ee583a565c83cea56630d3ae5878b9", + "sha256:418b84654b60e44c0cdd5384294b0e4bc1ebf42d6e873819424f3b78b8690614", + "sha256:4900b8820b687c9a3ed753684337979574df20e6ebe4227381d04b3c3c628f99", + "sha256:530d7222a2786a97bc59ee0e0ebbe23728f82974b1f1ad9a11cd966143410633", + "sha256:54122a8ed6b75fe1dd80797f8251ad2063ea348a03b77218d73ea9fe19bd4e73", + "sha256:6c3f162ba175678218629f446a947e3356415b6b09122dcb364e58c442c645a7", + "sha256:b49c86aecde15cde33835d5d6360e55f5e0067bb7143a8303bf03b872935c75b", + "sha256:b5b3489cb303d0f41ad4a7390cf606a5f2c7a94dcba20c051cd1c653694cb14d", + "sha256:cf3933c98cb5e808b62fae509f74f209730b180b1e3c3954ee3f7949e083a7df", + "sha256:eb75dc1809875d5738df14b6566ccf9fd9c0bcde4f36b72870f318f16b9f5c20", + "sha256:f769141ab0abfadf3305d4fcf36660e5cf568a666dd3efab7c3d4782f70946b1", + "sha256:f8af9b840a9074e08c0e6dc93101de84ba95df89b267bf7151d74c553d66833b" + ], + "version": "==1.6.1" + }, + "python-dateutil": { + "hashes": [ + "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c", + "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a" + ], + "version": "==2.8.1" + }, + "python-editor": { + "hashes": [ + "sha256:1bf6e860a8ad52a14c3ee1252d5dc25b2030618ed80c022598f00176adc8367d", + "sha256:51fda6bcc5ddbbb7063b2af7509e43bd84bfc32a4ff71349ec7847713882327b", + "sha256:5f98b069316ea1c2ed3f67e7f5df6c0d8f10b689964a4a811ff64f0106819ec8" + ], + "version": "==1.0.4" }, "regex": { "hashes": [ - "sha256:08997a37b221a3e27d68ffb601e45abfb0093d39ee770e4257bd2f5115e8cb0a", - "sha256:112e34adf95e45158c597feea65d06a8124898bdeac975c9087fe71b572bd938", - "sha256:1700419d8a18c26ff396b3b06ace315b5f2a6e780dad387e4c48717a12a22c29", - "sha256:2f6f211633ee8d3f7706953e9d3edc7ce63a1d6aad0be5dcee1ece127eea13ae", - "sha256:52e1b4bef02f4040b2fd547357a170fc1146e60ab310cdbdd098db86e929b387", - "sha256:55b4c25cbb3b29f8d5e63aeed27b49fa0f8476b0d4e1b3171d85db891938cc3a", - "sha256:5aaa5928b039ae440d775acea11d01e42ff26e1561c0ffcd3d805750973c6baf", - "sha256:654cb773b2792e50151f0e22be0f2b6e1c3a04c5328ff1d9d59c0398d37ef610", - "sha256:690f858d9a94d903cf5cada62ce069b5d93b313d7d05456dbcd99420856562d9", - "sha256:6ad8663c17db4c5ef438141f99e291c4d4edfeaacc0ce28b5bba2b0bf273d9b5", - "sha256:89cda1a5d3e33ec9e231ece7307afc101b5217523d55ef4dc7fb2abd6de71ba3", - "sha256:92d8a043a4241a710c1cf7593f5577fbb832cf6c3a00ff3fc1ff2052aff5dd89", - "sha256:95fa7726d073c87141f7bbfb04c284901f8328e2d430eeb71b8ffdd5742a5ded", - "sha256:97712e0d0af05febd8ab63d2ef0ab2d0cd9deddf4476f7aa153f76feef4b2754", - "sha256:b2ba0f78b3ef375114856cbdaa30559914d081c416b431f2437f83ce4f8b7f2f", - "sha256:bae83f2a56ab30d5353b47f9b2a33e4aac4de9401fb582b55c42b132a8ac3868", - "sha256:c78e66a922de1c95a208e4ec02e2e5cf0bb83a36ceececc10a72841e53fbf2bd", - "sha256:cf59bbf282b627130f5ba68b7fa3abdb96372b24b66bdf72a4920e8153fc7910", - "sha256:e3cdc9423808f7e1bb9c2e0bdb1c9dc37b0607b30d646ff6faf0d4e41ee8fee3", - "sha256:e9b64e609d37438f7d6e68c2546d2cb8062f3adb27e6336bc129b51be20773ac", - "sha256:fbff901c54c22425a5b809b914a3bfaf4b9570eee0e5ce8186ac71eb2025191c" - ], - "version": "==2020.6.8" + "sha256:0dc64ee3f33cd7899f79a8d788abfbec168410be356ed9bd30bbd3f0a23a7204", + "sha256:1269fef3167bb52631ad4fa7dd27bf635d5a0790b8e6222065d42e91bede4162", + "sha256:14a53646369157baa0499513f96091eb70382eb50b2c82393d17d7ec81b7b85f", + "sha256:3a3af27a8d23143c49a3420efe5b3f8cf1a48c6fc8bc6856b03f638abc1833bb", + "sha256:46bac5ca10fb748d6c55843a931855e2727a7a22584f302dd9bb1506e69f83f6", + "sha256:4c037fd14c5f4e308b8370b447b469ca10e69427966527edcab07f52d88388f7", + "sha256:51178c738d559a2d1071ce0b0f56e57eb315bcf8f7d4cf127674b533e3101f88", + "sha256:5ea81ea3dbd6767873c611687141ec7b06ed8bab43f68fad5b7be184a920dc99", + "sha256:6961548bba529cac7c07af2fd4d527c5b91bb8fe18995fed6044ac22b3d14644", + "sha256:75aaa27aa521a182824d89e5ab0a1d16ca207318a6b65042b046053cfc8ed07a", + "sha256:7a2dd66d2d4df34fa82c9dc85657c5e019b87932019947faece7983f2089a840", + "sha256:8a51f2c6d1f884e98846a0a9021ff6861bdb98457879f412fdc2b42d14494067", + "sha256:9c568495e35599625f7b999774e29e8d6b01a6fb684d77dee1f56d41b11b40cd", + "sha256:9eddaafb3c48e0900690c1727fba226c4804b8e6127ea409689c3bb492d06de4", + "sha256:bbb332d45b32df41200380fff14712cb6093b61bd142272a10b16778c418e98e", + "sha256:bc3d98f621898b4a9bc7fecc00513eec8f40b5b83913d74ccb445f037d58cd89", + "sha256:c11d6033115dc4887c456565303f540c44197f4fc1a2bfb192224a301534888e", + "sha256:c50a724d136ec10d920661f1442e4a8b010a4fe5aebd65e0c2241ea41dbe93dc", + "sha256:d0a5095d52b90ff38592bbdc2644f17c6d495762edf47d876049cfd2968fbccf", + "sha256:d6cff2276e502b86a25fd10c2a96973fdb45c7a977dca2138d661417f3728341", + "sha256:e46d13f38cfcbb79bfdb2964b0fe12561fe633caf964a77a5f8d4e45fe5d2ef7" + ], + "version": "==2020.7.14" + }, + "six": { + "hashes": [ + "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", + "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" + ], + "version": "==1.15.0" + }, + "sqlalchemy": { + "hashes": [ + "sha256:0942a3a0df3f6131580eddd26d99071b48cfe5aaf3eab2783076fbc5a1c1882e", + "sha256:0ec575db1b54909750332c2e335c2bb11257883914a03bc5a3306a4488ecc772", + "sha256:109581ccc8915001e8037b73c29590e78ce74be49ca0a3630a23831f9e3ed6c7", + "sha256:16593fd748944726540cd20f7e83afec816c2ac96b082e26ae226e8f7e9688cf", + "sha256:427273b08efc16a85aa2b39892817e78e3ed074fcb89b2a51c4979bae7e7ba98", + "sha256:50c4ee32f0e1581828843267d8de35c3298e86ceecd5e9017dc45788be70a864", + "sha256:512a85c3c8c3995cc91af3e90f38f460da5d3cade8dc3a229c8e0879037547c9", + "sha256:57aa843b783179ab72e863512e14bdcba186641daf69e4e3a5761d705dcc35b1", + "sha256:621f58cd921cd71ba6215c42954ffaa8a918eecd8c535d97befa1a8acad986dd", + "sha256:6ac2558631a81b85e7fb7a44e5035347938b0a73f5fdc27a8566777d0792a6a4", + "sha256:716754d0b5490bdcf68e1e4925edc02ac07209883314ad01a137642ddb2056f1", + "sha256:736d41cfebedecc6f159fc4ac0769dc89528a989471dc1d378ba07d29a60ba1c", + "sha256:8619b86cb68b185a778635be5b3e6018623c0761dde4df2f112896424aa27bd8", + "sha256:87fad64529cde4f1914a5b9c383628e1a8f9e3930304c09cf22c2ae118a1280e", + "sha256:89494df7f93b1836cae210c42864b292f9b31eeabca4810193761990dc689cce", + "sha256:8cac7bb373a5f1423e28de3fd5fc8063b9c8ffe8957dc1b1a59cb90453db6da1", + "sha256:8fd452dc3d49b3cc54483e033de6c006c304432e6f84b74d7b2c68afa2569ae5", + "sha256:adad60eea2c4c2a1875eb6305a0b6e61a83163f8e233586a4d6a55221ef984fe", + "sha256:c26f95e7609b821b5f08a72dab929baa0d685406b953efd7c89423a511d5c413", + "sha256:cbe1324ef52ff26ccde2cb84b8593c8bf930069dfc06c1e616f1bfd4e47f48a3", + "sha256:d05c4adae06bd0c7f696ae3ec8d993ed8ffcc4e11a76b1b35a5af8a099bd2284", + "sha256:d98bc827a1293ae767c8f2f18be3bb5151fd37ddcd7da2a5f9581baeeb7a3fa1", + "sha256:da2fb75f64792c1fc64c82313a00c728a7c301efe6a60b7a9fe35b16b4368ce7", + "sha256:e4624d7edb2576cd72bb83636cd71c8ce544d8e272f308bd80885056972ca299", + "sha256:e89e0d9e106f8a9180a4ca92a6adde60c58b1b0299e1b43bd5e0312f535fbf33", + "sha256:f11c2437fb5f812d020932119ba02d9e2bc29a6eca01a055233a8b449e3e1e7d", + "sha256:f57be5673e12763dd400fea568608700a63ce1c6bd5bdbc3cc3a2c5fdb045274", + "sha256:fc728ece3d5c772c196fd338a99798e7efac7a04f9cb6416299a3638ee9a94cd" + ], + "index": "pypi", + "version": "==1.3.18" }, "starlette": { "hashes": [ diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..ed37faf --- /dev/null +++ b/alembic.ini @@ -0,0 +1,85 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = alembic + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# timezone to use when rendering the date +# within the migration file as well as the filename. +# string value is passed to dateutil.tz.gettz() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the +# "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; this defaults +# to alembic/versions. When using multiple version +# directories, initial revisions must be specified with --version-path +# version_locations = %(here)s/bar %(here)s/bat alembic/versions + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = postgresql://uploader:test@localhost:5432/uploader + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +hooks=black +black.type=console_scripts +black.entrypoint=black +black.options=-l 79 + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/alembic/README b/alembic/README new file mode 100644 index 0000000..98e4f9c --- /dev/null +++ b/alembic/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/alembic/env.py b/alembic/env.py new file mode 100644 index 0000000..379c754 --- /dev/null +++ b/alembic/env.py @@ -0,0 +1,77 @@ +from logging.config import fileConfig + +from sqlalchemy import engine_from_config +from sqlalchemy import pool + +from alembic import context + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +from uploader import models_internal + +target_metadata = models_internal.Base.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure(connection=connection, target_metadata=target_metadata) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/alembic/script.py.mako b/alembic/script.py.mako new file mode 100644 index 0000000..2c01563 --- /dev/null +++ b/alembic/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/alembic/versions/535aa0f40ba9_initial_table_layout.py b/alembic/versions/535aa0f40ba9_initial_table_layout.py new file mode 100644 index 0000000..9923b64 --- /dev/null +++ b/alembic/versions/535aa0f40ba9_initial_table_layout.py @@ -0,0 +1,45 @@ +"""Initial table layout + +Revision ID: 535aa0f40ba9 +Revises: +Create Date: 2020-07-15 22:59:57.498926 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "535aa0f40ba9" +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "upload", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column( + "state", + sa.Enum("started", "completed", "retrieved", name="uploadstate"), + nullable=True, + ), + sa.Column("started_at", sa.DateTime(), nullable=True), + sa.Column("completed_at", sa.DateTime(), nullable=True), + sa.Column("retrieved_at", sa.DateTime(), nullable=True), + sa.Column("response", sa.JSON(), nullable=True), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index(op.f("ix_upload_id"), "upload", ["id"], unique=False) + op.create_index(op.f("ix_upload_state"), "upload", ["state"], unique=False) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f("ix_upload_state"), table_name="upload") + op.drop_index(op.f("ix_upload_id"), table_name="upload") + op.drop_table("upload") + # ### end Alembic commands ### diff --git a/prestart.sh b/prestart.sh new file mode 100644 index 0000000..65c28a1 --- /dev/null +++ b/prestart.sh @@ -0,0 +1,6 @@ +#! /usr/bin/env sh + +echo "prestart.sh file for uvicorn-gunicorn docker image. Runs alembic migrations." +echo "Sleeping for 5 seconds to let the database start." +sleep 5 +alembic upgrade head \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 6520fd7..e7f7e7a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,26 +1,34 @@ +alembic==1.4.2 appdirs==1.4.4 attrs==19.3.0 black==19.10b0 certifi==2020.6.20 chardet==3.0.4 click==7.1.2 -fastapi==0.58.1 +fastapi==0.59.0 h11==0.9.0 httptools==0.1.1 idna==2.10 iso8601==0.1.12 -MarkupSafe==1.1.1 +Mako==1.1.3 +MarkupSafe==2.0.0a1 pathspec==0.8.0 +psycopg2-binary==2.8.5 pycdstar3 @ git+https://gitlab.gwdg.de/cdstar/pycdstar3.git@a0e0fd5ee137d09988756ab19c9b68f428ff7d38 -pydantic==1.5.1 -regex==2020.6.8 +pydantic==1.6.1 +python-dateutil==2.8.1 +python-editor==1.0.4 +regex==2020.7.14 requests==2.24.0 requests-toolbelt==0.9.1 +six==1.15.0 +SQLAlchemy==1.3.18 starlette==0.13.4 tabulate==0.8.7 toml==0.10.1 tqdm==4.46.1 typed-ast==1.4.1 +-e git+git@gitlab.gwdg.de:medinf/umgmedic/active-workflow-datalake-agent.git@0cb6c82dd2d3e5fc887ce1b24b8b0b9fd82766e3#egg=UMG_Datalake_Upload_Agent urllib3==1.25.9 uvicorn==0.11.5 uvloop==0.14.0 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..faeeab2 --- /dev/null +++ b/setup.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Note: To use the 'upload' functionality of this file, you must: +# $ pipenv install twine --dev + +import io +import os +import sys +from shutil import rmtree + +from setuptools import find_packages, setup, Command + +# Package meta-data. +NAME = "uploader" +DESCRIPTION = "An Active Workflow agent to upload files to CDSTAR." +URL = "https://gitlab.gwdg.de/medinf/umgmedic/active-workflow-datalake-agent" +EMAIL = "marcel.parciak@med.uni-goettingen.de" +AUTHOR = "Marcel Parciak" +REQUIRES_PYTHON = ">=3.8.0" +VERSION = None + +# What packages are required for this module to be executed? +REQUIRED = ["sqlalchemy", "psycopg2-binary", "fastapi"] + +# What packages are optional? +EXTRAS = { + # 'fancy feature': ['django'], +} + +# The rest you shouldn't have to touch too much :) +# ------------------------------------------------ +# Except, perhaps the License and Trove Classifiers! +# If you do change the License, remember to change the Trove Classifier for that! + +here = os.path.abspath(os.path.dirname(__file__)) + +# Import the README and use it as the long-description. +# Note: this will only work if 'README.md' is present in your MANIFEST.in file! +try: + with io.open(os.path.join(here, "README.md"), encoding="utf-8") as f: + long_description = "\n" + f.read() +except FileNotFoundError: + long_description = DESCRIPTION + +# Load the package's __version__.py module as a dictionary. +about = {} +if not VERSION: + project_slug = NAME.lower().replace("-", "_").replace(" ", "_") + with open(os.path.join(here, project_slug, "__version__.py")) as f: + exec(f.read(), about) +else: + about["__version__"] = VERSION + + +class UploadCommand(Command): + """Support setup.py upload.""" + + description = "Build and publish the package." + user_options = [] + + @staticmethod + def status(s): + """Prints things in bold.""" + print("\033[1m{0}\033[0m".format(s)) + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + try: + self.status("Removing previous builds…") + rmtree(os.path.join(here, "dist")) + except OSError: + pass + + self.status("Building Source and Wheel (universal) distribution…") + os.system("{0} setup.py sdist bdist_wheel --universal".format(sys.executable)) + + self.status("Uploading the package to PyPI via Twine…") + os.system("twine upload dist/*") + + self.status("Pushing git tags…") + os.system("git tag v{0}".format(about["__version__"])) + os.system("git push --tags") + + sys.exit() + + +# Where the magic happens: +setup( + name=NAME, + version=about["__version__"], + description=DESCRIPTION, + long_description=long_description, + long_description_content_type="text/markdown", + author=AUTHOR, + author_email=EMAIL, + python_requires=REQUIRES_PYTHON, + url=URL, + # packages=find_packages(exclude=["tests", "*.tests", "*.tests.*", "tests.*"]), + # If your package is a single module, use this instead of 'packages': + py_modules=["uploader"], + # entry_points={ + # 'console_scripts': ['mycli=mymodule:cli'], + # }, + install_requires=REQUIRED, + extras_require=EXTRAS, + include_package_data=True, + dependency_links=[ + "https://gitlab.gwdg.de/cdstar/pycdstar3.git@a0e0fd5ee137d09988756ab19c9b68f428ff7d38" + ], + license="GPLv2", + classifiers=[ + # Trove classifiers + # Full list: https://pypi.python.org/pypi?%3Aaction=list_classifiers + "License :: OSI Approved :: GNU General Public License v2 (GPLv2)", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + ], + # $ setup.py publish support. + cmdclass={"upload": UploadCommand,}, +) diff --git a/uploader/__version__.py b/uploader/__version__.py new file mode 100644 index 0000000..fb8ccf0 --- /dev/null +++ b/uploader/__version__.py @@ -0,0 +1,4 @@ +VERSION = (0, 9, 0) + +__version__ = ".".join(map(str, VERSION)) + diff --git a/uploader/cdstar.py b/uploader/cdstar.py new file mode 100644 index 0000000..fc8235f --- /dev/null +++ b/uploader/cdstar.py @@ -0,0 +1,95 @@ +import datetime +import os +from typing import List, Optional + +from pycdstar3 import CDStar, CDStarVault +from pycdstar3.api import CDStarArchive + +from uploader import config +from uploader import models_cdstar as cdmodels + + +def upload_files(filelist: List[str], mime_type: Optional[str]) -> cdmodels.ArchiveInfo: + """ Uploads a list of files to CDSTAR. + + For each filepath in filelist, a call to `upload_file_to_archive` is made. + + Parameters + ---------- + filelist : List[str] + A list of strings, where each sting is a full path to the file which shall be uploaded to CDSTAR. + mime_type: str + String representation of a MIME type that shall be assigned to all uploaded files. If `None`, it + will use the auto-detection method of CDSTAR to determine the MIME type. + + Returns + ------- + List[models.ArchiveInfo] + A list of Archives modelled after [CDSTARs ArchiveInfo data structure](https://cdstar.gwdg.de/docs/dev/#ArchiveInfo). + """ + + cdstar = CDStar( + config.BasicSettings().cdstar_uri, + auth=(config.BasicSettings().cdstar_user, config.BasicSettings().cdstar_pass), + ) + + with cdstar.begin(autocommit=True): + vault = CDStarVault(cdstar, config.BasicSettings().cdstar_vault_target) + archive = vault.new_archive() + response = cdmodels.ArchiveInfo( + id=archive.id, + vault=vault.name, + created=datetime.datetime.now().isoformat(), + modified=datetime.datetime.now().isoformat(), + file_count=0, + ) + + for file in filelist: + uploaded = upload_file_to_archive(file, archive, mime_type) + response.file_count += 1 + response.files.append(uploaded) + response.modified = datetime.datetime.now() + + return response + + +def upload_file_to_archive( + file: str, archive: CDStarArchive, mime_type: Optional[str] +) -> cdmodels.FileInfo: + """ Uploads a file to a CDStarArchive. + + If not CDSTAR archive is supplied, a new archive will be created prior to uploading a file. + + Parameters + ---------- + file : str + A full path to a file that shall be uploaded to the CDSTAR archive. + archive : CDStarArchive + An instance of a CDStarArchive. If None, it will be created prior to uploading file. + mime_type : str + The MIME-Type of the to-be-uploaded file. If None, the automatic detection of CDSTAR will be used. + + Returns + ------- + models.FileInfo + The result of the upload procedure modelled after [CDSTARs FileInfo data structure](https://cdstar.gwdg.de/docs/dev/#FileInfo). + On error, the ID of the annotation will be `None`. + """ + + if os.path.isfile(file) and os.path.exists(file) and os.access(file, os.R_OK): + filename = os.path.basename(file) + cdstar_file = archive.file(filename) + kargs = {"type": None, "source": file} + if mime_type is not None: + kargs["type"] = mime_type + response = cdstar_file.put(**kargs) + if ( + "id" not in response.keys() + or "name" not in response.keys() + or "type" not in response.keys() + ): + return cdmodels.FileInfo(id=None, name=filename) + return cdmodels.FileInfo.parse_obj(response) + else: + return cdmodels.FileInfo(id=None, name=file) + diff --git a/uploader/config.py b/uploader/config.py index 793b4ae..7abeb70 100644 --- a/uploader/config.py +++ b/uploader/config.py @@ -1,10 +1,13 @@ +from importlib import metadata from pydantic import BaseSettings import os +from uploader import __version__ + class BasicSettings(BaseSettings): application_name: str = "UMG Datalake Upload Agent" - application_version: str = "0.8.4" + application_version: str = __version__.__version__ base_directory: str = "/tmp/source" source_directory: str = "." source_filemask: str = ".*.csv" @@ -14,6 +17,7 @@ class BasicSettings(BaseSettings): cdstar_user: str = "source" cdstar_pass: str = "something" cdstar_vault_target: str = "sources" + db_uri: str = "postgresql://uploader:test@localhost:5432/uploader" def get_version(): diff --git a/uploader/database.py b/uploader/database.py new file mode 100644 index 0000000..2bd1fee --- /dev/null +++ b/uploader/database.py @@ -0,0 +1,10 @@ +from sqlalchemy import create_engine +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker + +from uploader import config + +engine = create_engine(config.BasicSettings().db_uri) +SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) +Base = declarative_base() + diff --git a/uploader/main.py b/uploader/main.py index d0a9138..2a0f1eb 100644 --- a/uploader/main.py +++ b/uploader/main.py @@ -1,33 +1,32 @@ -from fastapi import BackgroundTasks, FastAPI, HTTPException, Request import fastapi import fastapi.responses +from fastapi import BackgroundTasks, FastAPI, HTTPException, Request, Depends +from sqlalchemy.orm import Session import datetime import logging import os -import random -import string -import traceback -from typing import List, Dict +from typing import List from uploader.config import BasicSettings from uploader.errors import ActiveWorkflowException, InternalException from uploader import models_activeworkflow as awmodels - +from uploader import models_internal as intmodels +from uploader import store from uploader.utils import ( retrieve_files_by_mask, retrieve_files_by_name, retrieve_options_from_parameters, ) -from uploader import store +from uploader import cdstar app = FastAPI() access_log = logging.getLogger("gunicorn.access") error_log = logging.getLogger("gunicorn.error") # TODO: this is state, remove it!! -running_uploads: Dict[str, dict] = {} -completed_uploads: Dict[str, awmodels.ResponseCheck] = {} +# running_uploads: Dict[str, dict] = {} +# completed_uploads: Dict[str, awmodels.ResponseCheck] = {} @app.get("/appinfo") @@ -36,7 +35,7 @@ def appinfo(): """ access_log.info(BasicSettings()) - return BasicSettings().dict(exclude={"cdstar_pass", "cdstar_user"}) + return BasicSettings().dict(exclude={"cdstar_pass", "cdstar_user", "db_uri"}) @app.post( @@ -44,7 +43,11 @@ def appinfo(): response_model=awmodels.ResponseCommon, response_model_exclude_none=True, ) -def datalake(payload: awmodels.RequestCommon, background_tasks: BackgroundTasks): +def datalake( + payload: awmodels.RequestCommon, + background_tasks: BackgroundTasks, + db: Session = Depends(store.get_db), +): """ Implements an remote agent API endpoint as specified by active_workflow, see [active_workflow docs](https://github.com/automaticmode/active_workflow/blob/master/docs/remote_agent_api.md) for more details. See the explicit API endpoints for each method to find more information on them. @@ -56,10 +59,12 @@ def datalake(payload: awmodels.RequestCommon, background_tasks: BackgroundTasks) ) elif payload.method is not None and payload.method == "receive": return receive( - awmodels.RequestReceive.parse_obj(payload.dict()), background_tasks + awmodels.RequestReceive.parse_obj(payload.dict()), background_tasks, db ) elif payload.method is not None and payload.method == "check": - return check(awmodels.RequestCheck.parse_obj(payload.dict()), background_tasks) + return check( + awmodels.RequestCheck.parse_obj(payload.dict()), background_tasks, db + ) raise HTTPException( status_code=400, @@ -91,7 +96,11 @@ def register(payload: awmodels.RequestRegister, background_tasks: BackgroundTask response_model=awmodels.ResponseReceive, response_model_exclude_none=True, ) -def receive(payload: awmodels.RequestReceive, background_tasks: BackgroundTasks): +def receive( + payload: awmodels.RequestReceive, + background_tasks: BackgroundTasks, + db: Session = Depends(store.get_db), +): """ Starts and upload and annotation of a list of files. Handle the receive method of ActiveWorkflow, see @@ -111,30 +120,37 @@ def receive(payload: awmodels.RequestReceive, background_tasks: BackgroundTasks) response = awmodels.ResponseReceive() response.result.memory = payload.params.memory + opts = retrieve_options_from_parameters(payload.params) + msg_payload = payload.params.message.payload if msg_payload.files_moved is None or len(msg_payload.files_moved) < 1: - return fastapi.responses.JSONResponse( - status_code=fastapi.status.HTTP_202_ACCEPTED, - content=response.dict(exclude_none=True), + error_log.debug( + f"Received message specifies no files moved. Using filemask to find files." + ) + filelist = retrieve_files_by_mask( + subdir=opts["source_directory"], filemask=opts["source_filemask"] + ) + else: + error_log.debug( + f"Received message specifies files. Using namelist to find files." + ) + filelist = retrieve_files_by_name( + subdir=opts["source_directory"], namelist=msg_payload.files_moved ) - opts = retrieve_options_from_parameters(payload.params) - - filelist = retrieve_files_by_name( - subdir=opts["source_directory"], namelist=msg_payload.files_moved - ) - - upload_id = "".join(random.choices(string.ascii_letters, k=12)) - while upload_id in list(running_uploads.keys()) + list(completed_uploads.keys()): - upload_id = "".join(random.choices(string.ascii_letters, k=12)) - - running_uploads[upload_id] = {"started": datetime.datetime.now()} + upload_procedure = store.start_upload_procedure(db) + upload_id = upload_procedure.id background_tasks.add_task( - perform_datalake_upload, filelist=filelist, options=opts, upload_id=upload_id, + perform_datalake_upload, + filelist=filelist, + options=opts, + upload_id=upload_id, + db=db, ) response.result.logs.append(f"The upload has been started with id {upload_id}") response.result.memory.uploads.append(upload_id) + error_log.debug(f"The upload has been started with id {upload_id}") return fastapi.responses.JSONResponse( status_code=fastapi.status.HTTP_202_ACCEPTED, content=response.dict(exclude_none=True), @@ -147,7 +163,11 @@ def receive(payload: awmodels.RequestReceive, background_tasks: BackgroundTasks) response_model_exclude_none=True, status_code=fastapi.status.HTTP_200_OK, ) -def check(payload: awmodels.RequestCheck, background_tasks: BackgroundTasks): +def check( + payload: awmodels.RequestCheck, + background_tasks: BackgroundTasks, + db: Session = Depends(store.get_db), +): """ Check on data uploads as supplied by `$.params.memory.uploads`. Handles the check method of ActiveWorkflow, see @@ -160,26 +180,44 @@ def check(payload: awmodels.RequestCheck, background_tasks: BackgroundTasks): response = awmodels.ResponseCheck() - for upl in payload.params.memory.uploads: - if upl in running_uploads.keys(): - response.result.memory.uploads.append(upl) + running_uploads = { + upl.id: upl + for upl in store.get_uploads_by_state(db, intmodels.UploadState.started) + } + error_log.debug(f"Running Uploads: {running_uploads.keys()}") + + completed_uploads = { + upl.id: upl + for upl in store.get_uploads_by_state(db, intmodels.UploadState.completed) + } + error_log.debug(f"Completed Uploads: {completed_uploads.keys()}") + + for upload_id in payload.params.memory.uploads: + if upload_id in running_uploads.keys(): + response.result.memory.uploads.append(upload_id) response.result.logs.append( - f"Upload {upl} is running since {running_uploads[upl]['started'].isoformat()}" + f"Upload {upload_id} is running since {running_uploads[upload_id].started_at.isoformat()}" ) - elif upl in completed_uploads.copy().keys(): - finished_upl = completed_uploads.pop(upl) - response.result.logs = response.result.logs + finished_upl.result.logs - response.result.errors = response.result.errors + finished_upl.result.errors + elif upload_id in completed_uploads.keys(): + upl = completed_uploads[upload_id] + upl.state = intmodels.UploadState.retrieved + upl.retrieved_at = datetime.datetime.now() + upl = store.store_upload(db, upl) + upl_response = awmodels.ResponseCheck.parse_obj(upl.response) + response.result.logs = response.result.logs + upl_response.result.logs + response.result.errors = response.result.errors + upl_response.result.errors response.result.messages = ( - response.result.messages + finished_upl.result.messages + response.result.messages + upl_response.result.messages ) else: - error_log.warning(f"Found orphan upload id: {upl}") + error_log.warning(f"Found orphan upload ID: {upload_id}") return response -def perform_datalake_upload(filelist: List[str], options: dict, upload_id: str) -> None: +def perform_datalake_upload( + filelist: List[str], options: dict, upload_id: int, db: Session +) -> None: """ Perform the upload and annotation within the datalake. From the options and annotations supplied by the caller or configured in the application, find all files using `source_directory` and `source_filemask` from the `options` parameter. Upload each file to a new @@ -194,41 +232,38 @@ def perform_datalake_upload(filelist: List[str], options: dict, upload_id: str) annotations : dict A dictionary of annotations that are added to the CouchDB metadata after the files have been uploaded to CDSTAR. """ - resp = awmodels.ResponseCheck() + response = awmodels.ResponseCheck() if filelist is None or len(filelist) < 1: - resp.result.logs.append(f"No files found to upload for id {upload_id}.") - access_log.debug(f"No files found to upload for id {upload_id}.") - upload_completed(upload_id, resp) + response.result.logs.append(f"No files found to upload for id {upload_id}.") + error_log.debug(f"No files found to upload for id {upload_id}.") + upload_completed(upload_id, response, db) return # upload all files to a new cdstar archive try: - uploaded = store.upload_files(filelist, mime_type=options["mime_type"]) - resp.result.messages.append( + uploaded = cdstar.upload_files(filelist, mime_type=options["mime_type"]) + response.result.messages.append( awmodels.MessageOutput(upload_id=upload_id, archive_id=uploaded.id) ) - access_log.debug( - f"Message output: upload_id: {upload_id} | archive_id: {uploaded.id}" - ) - resp.result.logs.append( - f"Upload to CDSTAR for id {upload_id} finished at {datetime.datetime.now().isoformat()}." + response.result.logs.append( + f"Upload to CDSTAR for ID {upload_id} finished at {datetime.datetime.now().isoformat()}." ) - access_log.debug( - f"Upload to CDSTAR for id {upload_id} finished at {datetime.datetime.now().isoformat()}." + error_log.debug( + f"Upload to CDSTAR for ID {upload_id} (CDSTAR Archive: {uploaded.id} finished at {datetime.datetime.now().isoformat()}." ) except: - error_log.error(traceback.format_exc()) - resp.result.errors.append( - f"I could not establish a connection to CDSTAR in some way. Upload with id {upload_id} failed." + response.result.errors.append( + f"There was an error while uploading to CDSTAR. Check the server error log for more information. Upload with ID {upload_id} failed." ) - error_log.debug( - f"Error: I could not establish a connection to CDSTAR in some way. Upload with id {upload_id} failed." + # TODO: debug error_log.debug(traceback.format_exc()) + error_log.error( + f"There was an error while uploading to CDSTAR. Check the server error log for more information. Upload with ID {upload_id} failed." ) - upload_completed(upload_id, resp) + upload_completed(upload_id, response, db) return - resp.result.logs.append( + response.result.logs.append( uploaded.json(exclude_none=True, exclude_defaults=True, exclude_unset=True) ) for file in uploaded.files: @@ -242,25 +277,29 @@ def perform_datalake_upload(filelist: List[str], options: dict, upload_id: str) ) ) else: - resp.result.errors.append(f"Error: {file.name} could not be uploaded.") + response.result.errors.append(f"Error: {file.name} could not be uploaded.") error_log.debug(f"Error: {file.name} could not be uploaded.") - upload_completed(upload_id, resp) + upload_completed(upload_id, response, db) -def upload_completed(upload_id: str, response: awmodels.ResponseCheck): - upl = running_uploads.pop(upload_id, None) - if upl is None: - response.result.errors.append( - f"Upload ID {upload_id} is unknown or was not started properly before it has been finished." +def upload_completed(upload_id: int, response: awmodels.ResponseCheck, db: Session): + upload = store.get_upload_by_id(db, upload_id) + if upload == None: + error_log.error( + f"I was not able to find ID {upload_id} which has response {response.dict(exclude_none=True)}" ) - completed_uploads[upload_id] = response - access_log.info(f"Completed upload of {upload_id}.") + return + upload.state = intmodels.UploadState.completed + upload.completed_at = datetime.datetime.now() + upload.response = response.dict(exclude_none=None) + store.store_upload(db, upload) + error_log.info(f"Completed upload of {upload_id}.") @app.exception_handler(ActiveWorkflowException) async def aw_exception_handler(request: Request, exc: ActiveWorkflowException): - access_log.error(f"{exc.name}: {exc.message}") + error_log.error(f"{exc.name}: {exc.message}") resp = awmodels.ResponseCheck() resp.result.errors.append(f"{exc.name}: {exc.message}") diff --git a/uploader/models_activeworkflow.py b/uploader/models_activeworkflow.py index 0f74ee4..d7cbba3 100644 --- a/uploader/models_activeworkflow.py +++ b/uploader/models_activeworkflow.py @@ -87,12 +87,12 @@ class MessageOutput(BaseModel): This model represents the produced output schema that will be provided in the messages. """ - upload_id: str = Field(..., example="DoamRsPwHzrl") + upload_id: int = Field(..., example=4) archive_id: str = Field(..., example="iw7s2px9") @staticmethod def example() -> dict: - return {"upload_id": "DoamRsPwHzrl", "archive_id": "iw7s2px9"} + return {"upload_id": 4, "archive_id": "iw7s2px9"} class MemoryCommon(BaseModel): @@ -100,11 +100,11 @@ class MemoryCommon(BaseModel): This model represents the expected memory content to communicate state of the agent. """ - uploads: List[str] = Field([], example=["DoamRsPwHzrl", "lsZHWpxwMqaj"]) + uploads: List[int] = Field([], example=[4, 5]) @staticmethod def example() -> dict: - return {"uploads": ["DoamRsPwHzrl", "lsZHWpxwMqaj"]} + return {"uploads": [4, 5]} # diff --git a/uploader/models_internal.py b/uploader/models_internal.py new file mode 100644 index 0000000..05f44a8 --- /dev/null +++ b/uploader/models_internal.py @@ -0,0 +1,21 @@ +import enum + +from sqlalchemy import Integer, Column, Enum, DateTime, JSON +from uploader.database import Base + + +class UploadState(enum.Enum): + started = 1 + completed = 2 + retrieved = 3 + + +class UploadProcedure(Base): + __tablename__ = "upload" + + id = Column(Integer, primary_key=True, index=True) + state = Column(Enum(UploadState), index=True) + started_at = Column(DateTime) + completed_at = Column(DateTime) + retrieved_at = Column(DateTime) + response = Column(JSON) diff --git a/uploader/store.py b/uploader/store.py index fc8235f..fc91f4a 100644 --- a/uploader/store.py +++ b/uploader/store.py @@ -1,95 +1,53 @@ import datetime -import os -from typing import List, Optional - -from pycdstar3 import CDStar, CDStarVault -from pycdstar3.api import CDStarArchive -from uploader import config -from uploader import models_cdstar as cdmodels +from typing import List, Optional +from sqlalchemy.orm import Session -def upload_files(filelist: List[str], mime_type: Optional[str]) -> cdmodels.ArchiveInfo: - """ Uploads a list of files to CDSTAR. +from uploader import database +from uploader import models_internal as intmodel - For each filepath in filelist, a call to `upload_file_to_archive` is made. - Parameters - ---------- - filelist : List[str] - A list of strings, where each sting is a full path to the file which shall be uploaded to CDSTAR. - mime_type: str - String representation of a MIME type that shall be assigned to all uploaded files. If `None`, it - will use the auto-detection method of CDSTAR to determine the MIME type. +def get_db(): + db = database.SessionLocal() + try: + yield db + finally: + db.close() - Returns - ------- - List[models.ArchiveInfo] - A list of Archives modelled after [CDSTARs ArchiveInfo data structure](https://cdstar.gwdg.de/docs/dev/#ArchiveInfo). - """ - cdstar = CDStar( - config.BasicSettings().cdstar_uri, - auth=(config.BasicSettings().cdstar_user, config.BasicSettings().cdstar_pass), +def start_upload_procedure(db: Session) -> intmodel.UploadProcedure: + upload_procedure = intmodel.UploadProcedure( + state=intmodel.UploadState.started, started_at=datetime.datetime.now() + ) + db.add(upload_procedure) + db.commit() + db.refresh(upload_procedure) + return upload_procedure + + +def get_uploads_by_state( + db: Session, state: intmodel.UploadState +) -> List[intmodel.UploadProcedure]: + return ( + db.query(intmodel.UploadProcedure) + .filter(intmodel.UploadProcedure.state == state) + .all() ) - with cdstar.begin(autocommit=True): - vault = CDStarVault(cdstar, config.BasicSettings().cdstar_vault_target) - archive = vault.new_archive() - response = cdmodels.ArchiveInfo( - id=archive.id, - vault=vault.name, - created=datetime.datetime.now().isoformat(), - modified=datetime.datetime.now().isoformat(), - file_count=0, - ) - - for file in filelist: - uploaded = upload_file_to_archive(file, archive, mime_type) - response.file_count += 1 - response.files.append(uploaded) - response.modified = datetime.datetime.now() - - return response - - -def upload_file_to_archive( - file: str, archive: CDStarArchive, mime_type: Optional[str] -) -> cdmodels.FileInfo: - """ Uploads a file to a CDStarArchive. - - If not CDSTAR archive is supplied, a new archive will be created prior to uploading a file. - Parameters - ---------- - file : str - A full path to a file that shall be uploaded to the CDSTAR archive. - archive : CDStarArchive - An instance of a CDStarArchive. If None, it will be created prior to uploading file. - mime_type : str - The MIME-Type of the to-be-uploaded file. If None, the automatic detection of CDSTAR will be used. - - Returns - ------- - models.FileInfo - The result of the upload procedure modelled after [CDSTARs FileInfo data structure](https://cdstar.gwdg.de/docs/dev/#FileInfo). - On error, the ID of the annotation will be `None`. - """ +def store_upload( + db: Session, upload: intmodel.UploadProcedure +) -> intmodel.UploadProcedure: + db.add(upload) + db.commit() + db.refresh(upload) + return upload - if os.path.isfile(file) and os.path.exists(file) and os.access(file, os.R_OK): - filename = os.path.basename(file) - cdstar_file = archive.file(filename) - kargs = {"type": None, "source": file} - if mime_type is not None: - kargs["type"] = mime_type - response = cdstar_file.put(**kargs) - if ( - "id" not in response.keys() - or "name" not in response.keys() - or "type" not in response.keys() - ): - return cdmodels.FileInfo(id=None, name=filename) - return cdmodels.FileInfo.parse_obj(response) - else: - return cdmodels.FileInfo(id=None, name=file) +def get_upload_by_id(db: Session, id: int) -> Optional[intmodel.UploadProcedure]: + return ( + db.query(intmodel.UploadProcedure) + .filter(intmodel.UploadProcedure.id == id) + .first() + ) -- GitLab From 238e1565aa236d4881e625ef45388353c2315970 Mon Sep 17 00:00:00 2001 From: Marcel Parciak <marcel.parciak@gmail.com> Date: Thu, 16 Jul 2020 08:20:22 +0200 Subject: [PATCH 2/8] Added new handling of verisoning in __version__.py to gitlab-ci. Documented SQL Backend change. Signed-off-by: Marcel Parciak <marcel.parciak@gmail.com> --- .gitlab-ci.yml | 3 +-- Readme.md | 7 +++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 32b417e..7d3f3ae 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -28,8 +28,7 @@ publish_image: before_script: - docker login --username $CI_REGISTRY_USER --password $CI_REGISTRY_PASSWORD $CI_REGISTRY - - pip3 install --user pydantic - - AGENTVERSION=`python3 -c 'import uploader.config; print(uploader.config.get_version())'` + - AGENTVERSION=`python3 -c "from uploader import __version__; print(__version__.__version__)"` script: - docker build -t $CI_REGISTRY_IMAGE:latest . diff --git a/Readme.md b/Readme.md index ba7e638..6de1e66 100644 --- a/Readme.md +++ b/Readme.md @@ -13,7 +13,7 @@ A caller of this agent has to comply to the remote agent API by ActiveWorkflow. Registering this agent necessitates setting some options needed to work with Retrieve and Check. * `source_directory` specifies a sub-path under the configured `base_directory` in which the agent will search for all supplied files when retrieving. -* `source_filemask` specifies a mask for files that will be applied when automatically searching for files under `source_directory`. Currently, this option is **unused**. +* `source_filemask` specifies a mask for files that will be applied when automatically searching for files under `source_directory`. * `delete_after_upload` specifies whether uploaded files should be deleted. * `mime_type` allows to set a specific mime_type to set in CDSTAR. If `None`, the autodetection method of CSTAR will be used. @@ -75,7 +75,10 @@ Configuration can be done using environment variables. It does not matter whethe * `delete_after_upload`: Boolean to indicate whether uploaded files shall be deleted. It may be overridden by the caller. Default: `True` * `mime_type`: MIME Type that shall be used for uploads to CDSTAR. It may be overridden by the caller. Default: `"applciation/csv"` * `base_directory`: Default base directory which cannot be overridden with options or messages. Useful to limit the access of the application to certain folders. Default: `"/tmp/source"` +* `db_uri`: Path to a database (tested with postgres-12) in the [sqlalchemy notation](https://docs.sqlalchemy.org/en/13/core/engines.html#database-urls): `postgresql://<user>:<pass>@<host>:<port>/<db>`. Default: `"postgresql://uploader:test@localhost:5432/uploader"` * `cdstar_uri`: The CDSTAR URL where the files shall be uploaded to including the version of the API. This will be passed directly to [pycdstar3](https://gitlab.gwdg.de/cdstar/pycdstar3) to establish a connection. Default: `"http://localhost:8082/v3"` * `cdstar_user`: The user to authenticate with CDSTAR. Default: `"source"` * `cdstar_pass`: The password to authenticate with CDSTAR. Default: `"something"` -* `cdstar_vault_target` The vault that will be used to create new archives. Default: `"sources"` +* `cdstar_vault_target`: The vault that will be used to create new archives. Default: `"sources"` + +To initialize the database, this agent makes use of [alembic](https://alembic.sqlalchemy.org). In its docker container, this is run automatically after 5 seconds using the `prestart.sh` shell script. \ No newline at end of file -- GitLab From 8d27c2ee4f5f9c734c59967f1a164c432daeac76 Mon Sep 17 00:00:00 2001 From: Marcel Parciak <marcel.parciak@gmail.com> Date: Thu, 16 Jul 2020 08:22:16 +0200 Subject: [PATCH 3/8] Updated harbor publishing to the new version retrieval. Signed-off-by: Marcel Parciak <marcel.parciak@gmail.com> --- .gitlab-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ffd2477..38f992e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -51,8 +51,7 @@ publish_to_harbor: before_script: - docker login --username marcelparciak --password $HARBOR_PUSH_TOKEN harbor.umg.eu - - pip3 install --user pydantic - - AGENTVERSION=`python3 -c 'import uploader.config; print(uploader.config.get_version())'` + - AGENTVERSION=`python3 -c "from uploader import __version__; print(__version__.__version__)"` script: - docker build -t harbor.umg.eu/medic/$CI_PROJECT_NAME:latest . -- GitLab From b93c904588aafbde64ff263efec0ce68df71ceb6 Mon Sep 17 00:00:00 2001 From: Marcel Parciak <marcel.parciak@gmail.com> Date: Thu, 16 Jul 2020 08:56:22 +0200 Subject: [PATCH 4/8] Cleanup and recreation of requirements. Added necessary files for alembic to Dockerfile. Signed-off-by: Marcel Parciak <marcel.parciak@gmail.com> --- Dockerfile | 4 + Pipfile | 7 +- Pipfile.lock | 308 ++++++++++++++++++++++++++++++------------- requirements-dev.txt | 40 ++++++ requirements.txt | 41 +++--- 5 files changed, 281 insertions(+), 119 deletions(-) create mode 100644 requirements-dev.txt diff --git a/Dockerfile b/Dockerfile index 276ca81..ccdd017 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,6 +3,10 @@ FROM docker.io/tiangolo/uvicorn-gunicorn-fastapi:python3.8 COPY ./requirements.txt /app/requirements.txt RUN pip3.8 install -r requirements.txt +COPY ./alembic /app/alembic +COPY ./alembic.ini /app/alembic.ini +COPY ./prestart.sh /app/prestart.sh +COPY ./setup.py /app/setup.py COPY ./uploader /app/uploader COPY ./Readme.md /app/Readme.md diff --git a/Pipfile b/Pipfile index 102bc15..8a2c126 100644 --- a/Pipfile +++ b/Pipfile @@ -7,13 +7,14 @@ verify_ssl = true fastapi = "*" uvicorn = "*" black = "*" -alembic = "*" +umg-datalake-upload-agent = {editable = true,path = "."} +pipenv-to-requirements = "*" [packages] -3-0-dev0 = {git = "https://gitlab.gwdg.de/cdstar/pycdstar3.git"} psycopg2-binary = "*" sqlalchemy = "*" -umg-datalake-upload-agent = {editable = true,path = "."} +alembic = "*" +pycdstar3 = {editable = true,git = "https://gitlab.gwdg.de/cdstar/pycdstar3"} [requires] python_version = "3.8" diff --git a/Pipfile.lock b/Pipfile.lock index d39471f..0d02328 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "30e12061e05ef407521b674b77f6abb9f5dcb8b9ecfde9450bc00cc413b89090" + "sha256": "8007750e2b275e5cdfd5eee1604ef2430b7c201162c4a5da2eca2d2bafbea66c" }, "pipfile-spec": 6, "requires": { @@ -16,16 +16,75 @@ ] }, "default": { - "3-0-dev0": { - "git": "https://gitlab.gwdg.de/cdstar/pycdstar3.git", - "ref": "a0e0fd5ee137d09988756ab19c9b68f428ff7d38" + "alembic": { + "hashes": [ + "sha256:035ab00497217628bf5d0be82d664d8713ab13d37b630084da8e1f98facf4dbf" + ], + "index": "pypi", + "version": "==1.4.2" }, - "fastapi": { + "certifi": { "hashes": [ - "sha256:50b58aa3e7d5bcb4a4404ac7e550cc53f0cf7ca0fd13c7fd515693dc23c9caef", - "sha256:c04dacd3deed0fd0ffdcdb116b5a04ffa257656885be7fae7234f4f62ec4a0a9" + "sha256:5930595817496dd21bb8dc35dad090f1c2cd0adfaf21204bf6732ca5d8ee34d3", + "sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41" ], - "version": "==0.59.0" + "version": "==2020.6.20" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "version": "==3.0.4" + }, + "idna": { + "hashes": [ + "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", + "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" + ], + "version": "==2.10" + }, + "iso8601": { + "hashes": [ + "sha256:210e0134677cc0d02f6028087fee1df1e1d76d372ee1db0bf30bf66c5c1c89a3", + "sha256:49c4b20e1f38aa5cf109ddcd39647ac419f928512c869dc01d5c7098eddede82", + "sha256:bbbae5fb4a7abfe71d4688fd64bff70b91bbd74ef6a99d964bab18f7fdf286dd" + ], + "version": "==0.1.12" + }, + "mako": { + "hashes": [ + "sha256:8195c8c1400ceb53496064314c6736719c6f25e7479cd24c77be3d9361cddc27", + "sha256:93729a258e4ff0747c876bd9e20df1b9758028946e976324ccd2d68245c7b6a9" + ], + "version": "==1.1.3" + }, + "markupsafe": { + "hashes": [ + "sha256:06358015a4dee8ee23ae426bf885616ab3963622defd829eb45b44e3dee3515f", + "sha256:0b0c4fc852c5f02c6277ef3b33d23fcbe89b1b227460423e3335374da046b6db", + "sha256:267677fc42afed5094fc5ea1c4236bbe4b6a00fe4b08e93451e65ae9048139c7", + "sha256:303cb70893e2c345588fb5d5b86e0ca369f9bb56942f03064c5e3e75fa7a238a", + "sha256:3c9b624a0d9ed5a5093ac4edc4e823e6b125441e60ef35d36e6f4a6fdacd5054", + "sha256:42033e14cae1f6c86fc0c3e90d04d08ce73ac8e46ba420a0d22d545c2abd4977", + "sha256:4e4a99b6af7bdc0856b50020c095848ec050356a001e1f751510aef6ab14d0e0", + "sha256:4eb07faad54bb07427d848f31030a65a49ebb0cec0b30674f91cf1ddd456bfe4", + "sha256:63a7161cd8c2bc563feeda45df62f42c860dd0675e2b8da2667f25bb3c95eaba", + "sha256:68e0fd039b68d2945b4beb947d4023ca7f8e95b708031c345762efba214ea761", + "sha256:8092a63397025c2f655acd42784b2a1528339b90b987beb9253f22e8cdbb36c3", + "sha256:841218860683c0f2223e24756843d84cc49cccdae6765e04962607754a52d3e0", + "sha256:94076b2314bd2f6cfae508ad65b4d493e3a58a50112b7a2cbb6287bdbc404ae8", + "sha256:9d22aff1c5322e402adfb3ce40839a5056c353e711c033798cf4f02eb9f5124d", + "sha256:b0e4584f62b3e5f5c1a7bcefd2b52f236505e6ef032cc508caa4f4c8dc8d3af1", + "sha256:b1163ffc1384d242964426a8164da12dbcdbc0de18ea36e2c34b898ed38c3b45", + "sha256:beac28ed60c8e838301226a7a85841d0af2068eba2dcb1a58c2d32d6c05e440e", + "sha256:c29f096ce79c03054a1101d6e5fe6bf04b0bb489165d5e0e9653fb4fe8048ee1", + "sha256:c58779966d53e5f14ba393d64e2402a7926601d1ac8adeb4e83893def79d0428", + "sha256:cfe14b37908eaf7d5506302987228bff69e1b8e7071ccd4e70fd0283b1b47f0b", + "sha256:e834249c45aa9837d0753351cdca61a4b8b383cc9ad0ff2325c97ff7b69e72a6", + "sha256:eed1b234c4499811ee85bcefa22ef5e466e75d132502226ed29740d593316c1f" + ], + "version": "==2.0.0a1" }, "psycopg2-binary": { "hashes": [ @@ -63,27 +122,46 @@ "index": "pypi", "version": "==2.8.5" }, - "pydantic": { + "pycdstar3": { + "editable": true, + "git": "https://gitlab.gwdg.de/cdstar/pycdstar3", + "ref": "a0e0fd5ee137d09988756ab19c9b68f428ff7d38" + }, + "python-dateutil": { "hashes": [ - "sha256:1783c1d927f9e1366e0e0609ae324039b2479a1a282a98ed6a6836c9ed02002c", - "sha256:2dc946b07cf24bee4737ced0ae77e2ea6bc97489ba5a035b603bd1b40ad81f7e", - "sha256:2de562a456c4ecdc80cf1a8c3e70c666625f7d02d89a6174ecf63754c734592e", - "sha256:36dbf6f1be212ab37b5fda07667461a9219c956181aa5570a00edfb0acdfe4a1", - "sha256:3fa799f3cfff3e5f536cbd389368fc96a44bb30308f258c94ee76b73bd60531d", - "sha256:40d765fa2d31d5be8e29c1794657ad46f5ee583a565c83cea56630d3ae5878b9", - "sha256:418b84654b60e44c0cdd5384294b0e4bc1ebf42d6e873819424f3b78b8690614", - "sha256:4900b8820b687c9a3ed753684337979574df20e6ebe4227381d04b3c3c628f99", - "sha256:530d7222a2786a97bc59ee0e0ebbe23728f82974b1f1ad9a11cd966143410633", - "sha256:54122a8ed6b75fe1dd80797f8251ad2063ea348a03b77218d73ea9fe19bd4e73", - "sha256:6c3f162ba175678218629f446a947e3356415b6b09122dcb364e58c442c645a7", - "sha256:b49c86aecde15cde33835d5d6360e55f5e0067bb7143a8303bf03b872935c75b", - "sha256:b5b3489cb303d0f41ad4a7390cf606a5f2c7a94dcba20c051cd1c653694cb14d", - "sha256:cf3933c98cb5e808b62fae509f74f209730b180b1e3c3954ee3f7949e083a7df", - "sha256:eb75dc1809875d5738df14b6566ccf9fd9c0bcde4f36b72870f318f16b9f5c20", - "sha256:f769141ab0abfadf3305d4fcf36660e5cf568a666dd3efab7c3d4782f70946b1", - "sha256:f8af9b840a9074e08c0e6dc93101de84ba95df89b267bf7151d74c553d66833b" + "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c", + "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a" ], - "version": "==1.6.1" + "version": "==2.8.1" + }, + "python-editor": { + "hashes": [ + "sha256:1bf6e860a8ad52a14c3ee1252d5dc25b2030618ed80c022598f00176adc8367d", + "sha256:51fda6bcc5ddbbb7063b2af7509e43bd84bfc32a4ff71349ec7847713882327b", + "sha256:5f98b069316ea1c2ed3f67e7f5df6c0d8f10b689964a4a811ff64f0106819ec8" + ], + "version": "==1.0.4" + }, + "requests": { + "hashes": [ + "sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b", + "sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898" + ], + "version": "==2.24.0" + }, + "requests-toolbelt": { + "hashes": [ + "sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f", + "sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0" + ], + "version": "==0.9.1" + }, + "six": { + "hashes": [ + "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", + "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" + ], + "version": "==1.15.0" }, "sqlalchemy": { "hashes": [ @@ -119,26 +197,29 @@ "index": "pypi", "version": "==1.3.18" }, - "starlette": { + "tabulate": { "hashes": [ - "sha256:04fe51d86fd9a594d9b71356ed322ccde5c9b448fc716ac74155e5821a922f8d", - "sha256:0fb4b38d22945b46acb880fedee7ee143fd6c0542992501be8c45c0ed737dd1a" + "sha256:ac64cb76d53b1231d364babcd72abbb16855adac7de6665122f97b593f1eb2ba", + "sha256:db2723a20d04bcda8522165c73eea7c300eda74e0ce852d9022e0159d7895007" ], - "version": "==0.13.4" + "version": "==0.8.7" }, - "umg-datalake-upload-agent": { - "editable": true, - "path": "." - } - }, - "develop": { - "alembic": { + "tqdm": { "hashes": [ - "sha256:035ab00497217628bf5d0be82d664d8713ab13d37b630084da8e1f98facf4dbf" + "sha256:63ef7a6d3eb39f80d6b36e4867566b3d8e5f1fe3d6cb50c5e9ede2b3198ba7b7", + "sha256:7810e627bcf9d983a99d9ff8a0c09674400fd2927eddabeadf153c14a2ec8656" ], - "index": "pypi", - "version": "==1.4.2" + "version": "==4.47.0" }, + "urllib3": { + "hashes": [ + "sha256:3018294ebefce6572a474f0604c2021e33b3fd8006ecd11d62107a5d2a963527", + "sha256:88206b0eb87e6d677d424843ac5209e3fb9d0190d0ee169599165ec25e9d9115" + ], + "version": "==1.25.9" + } + }, + "develop": { "appdirs": { "hashes": [ "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41", @@ -161,13 +242,12 @@ "index": "pypi", "version": "==19.10b0" }, - "bump2version": { + "certifi": { "hashes": [ - "sha256:477f0e18a0d58e50bb3dbc9af7fcda464fd0ebfc7a6151d8888602d7153171a0", - "sha256:cd4f3a231305e405ed8944d8ff35bd742d9bc740ad62f483bd0ca21ce7131984" + "sha256:5930595817496dd21bb8dc35dad090f1c2cd0adfaf21204bf6732ca5d8ee34d3", + "sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41" ], - "index": "pypi", - "version": "==1.0.0" + "version": "==2020.6.20" }, "click": { "hashes": [ @@ -176,13 +256,28 @@ ], "version": "==7.1.2" }, + "distlib": { + "hashes": [ + "sha256:8c09de2c67b3e7deef7184574fc060ab8a793e7adbb183d942c389c8b13c52fb", + "sha256:edf6116872c863e1aa9d5bb7cb5e05a022c519a4594dc703843343a9ddd9bff1" + ], + "version": "==0.3.1" + }, "fastapi": { "hashes": [ "sha256:50b58aa3e7d5bcb4a4404ac7e550cc53f0cf7ca0fd13c7fd515693dc23c9caef", "sha256:c04dacd3deed0fd0ffdcdb116b5a04ffa257656885be7fae7234f4f62ec4a0a9" ], + "index": "pypi", "version": "==0.59.0" }, + "filelock": { + "hashes": [ + "sha256:18d82244ee114f543149c66a6e0c14e9c4f8a1044b5cdaadd0f82159d6a6ff59", + "sha256:929b7d63ec5b7d6b71b0fa5ac14e030b3f70b75747cef1b10da9b879fef15836" + ], + "version": "==3.0.12" + }, "h11": { "hashes": [ "sha256:33d4bca7be0fa039f4e84d50ab00531047e53d6ee8ffbc83501ea602c169cae1", @@ -208,46 +303,70 @@ "markers": "sys_platform != 'win32' and sys_platform != 'cygwin' and platform_python_implementation != 'PyPy'", "version": "==0.1.1" }, - "mako": { + "pathspec": { "hashes": [ - "sha256:8195c8c1400ceb53496064314c6736719c6f25e7479cd24c77be3d9361cddc27", - "sha256:93729a258e4ff0747c876bd9e20df1b9758028946e976324ccd2d68245c7b6a9" + "sha256:7d91249d21749788d07a2d0f94147accd8f845507400749ea19c1ec9054a12b0", + "sha256:da45173eb3a6f2a5a487efba21f050af2b41948be6ab52b6a1e3ff22bb8b7061" ], - "version": "==1.1.3" + "version": "==0.8.0" }, - "markupsafe": { + "pbr": { "hashes": [ - "sha256:06358015a4dee8ee23ae426bf885616ab3963622defd829eb45b44e3dee3515f", - "sha256:0b0c4fc852c5f02c6277ef3b33d23fcbe89b1b227460423e3335374da046b6db", - "sha256:267677fc42afed5094fc5ea1c4236bbe4b6a00fe4b08e93451e65ae9048139c7", - "sha256:303cb70893e2c345588fb5d5b86e0ca369f9bb56942f03064c5e3e75fa7a238a", - "sha256:3c9b624a0d9ed5a5093ac4edc4e823e6b125441e60ef35d36e6f4a6fdacd5054", - "sha256:42033e14cae1f6c86fc0c3e90d04d08ce73ac8e46ba420a0d22d545c2abd4977", - "sha256:4e4a99b6af7bdc0856b50020c095848ec050356a001e1f751510aef6ab14d0e0", - "sha256:4eb07faad54bb07427d848f31030a65a49ebb0cec0b30674f91cf1ddd456bfe4", - "sha256:63a7161cd8c2bc563feeda45df62f42c860dd0675e2b8da2667f25bb3c95eaba", - "sha256:68e0fd039b68d2945b4beb947d4023ca7f8e95b708031c345762efba214ea761", - "sha256:8092a63397025c2f655acd42784b2a1528339b90b987beb9253f22e8cdbb36c3", - "sha256:841218860683c0f2223e24756843d84cc49cccdae6765e04962607754a52d3e0", - "sha256:94076b2314bd2f6cfae508ad65b4d493e3a58a50112b7a2cbb6287bdbc404ae8", - "sha256:9d22aff1c5322e402adfb3ce40839a5056c353e711c033798cf4f02eb9f5124d", - "sha256:b0e4584f62b3e5f5c1a7bcefd2b52f236505e6ef032cc508caa4f4c8dc8d3af1", - "sha256:b1163ffc1384d242964426a8164da12dbcdbc0de18ea36e2c34b898ed38c3b45", - "sha256:beac28ed60c8e838301226a7a85841d0af2068eba2dcb1a58c2d32d6c05e440e", - "sha256:c29f096ce79c03054a1101d6e5fe6bf04b0bb489165d5e0e9653fb4fe8048ee1", - "sha256:c58779966d53e5f14ba393d64e2402a7926601d1ac8adeb4e83893def79d0428", - "sha256:cfe14b37908eaf7d5506302987228bff69e1b8e7071ccd4e70fd0283b1b47f0b", - "sha256:e834249c45aa9837d0753351cdca61a4b8b383cc9ad0ff2325c97ff7b69e72a6", - "sha256:eed1b234c4499811ee85bcefa22ef5e466e75d132502226ed29740d593316c1f" + "sha256:07f558fece33b05caf857474a366dfcc00562bca13dd8b47b2b3e22d9f9bf55c", + "sha256:579170e23f8e0c2f24b0de612f71f648eccb79fb1322c814ae6b3c07b5ba23e8" ], - "version": "==2.0.0a1" + "version": "==5.4.5" }, - "pathspec": { + "pipenv": { "hashes": [ - "sha256:7d91249d21749788d07a2d0f94147accd8f845507400749ea19c1ec9054a12b0", - "sha256:da45173eb3a6f2a5a487efba21f050af2b41948be6ab52b6a1e3ff22bb8b7061" + "sha256:7dd49a7345fa5da7843907bab42a996dece474e45dd309dbd7619739dc60478b", + "sha256:cc289dc78feaa14def4e1723b0b4d85ff628c8e5f740eeeb68197b90dafa8dff" ], - "version": "==0.8.0" + "version": "==2020.6.2" + }, + "pipenv-to-requirements": { + "hashes": [ + "sha256:1c18682a4ec70eb07261d2b558df3ee22ea00192663a1b98fd1e45e22946c163", + "sha256:cb70471a17a7d4658caffe989539413313d51df1b3a54838bcd7e7d3ab3fcc18" + ], + "index": "pypi", + "version": "==0.9.0" + }, + "psycopg2-binary": { + "hashes": [ + "sha256:008da3ab51adc70a5f1cfbbe5db3a22607ab030eb44bcecf517ad11a0c2b3cac", + "sha256:07cf82c870ec2d2ce94d18e70c13323c89f2f2a2628cbf1feee700630be2519a", + "sha256:08507efbe532029adee21b8d4c999170a83760d38249936038bd0602327029b5", + "sha256:107d9be3b614e52a192719c6bf32e8813030020ea1d1215daa86ded9a24d8b04", + "sha256:17a0ea0b0eabf07035e5e0d520dabc7950aeb15a17c6d36128ba99b2721b25b1", + "sha256:3286541b9d85a340ee4ed42732d15fc1bb441dc500c97243a768154ab8505bb5", + "sha256:3939cf75fc89c5e9ed836e228c4a63604dff95ad19aed2bbf71d5d04c15ed5ce", + "sha256:40abc319f7f26c042a11658bf3dd3b0b3bceccf883ec1c565d5c909a90204434", + "sha256:51f7823f1b087d2020d8e8c9e6687473d3d239ba9afc162d9b2ab6e80b53f9f9", + "sha256:6bb2dd006a46a4a4ce95201f836194eb6a1e863f69ee5bab506673e0ca767057", + "sha256:702f09d8f77dc4794651f650828791af82f7c2efd8c91ae79e3d9fe4bb7d4c98", + "sha256:7036ccf715925251fac969f4da9ad37e4b7e211b1e920860148a10c0de963522", + "sha256:7b832d76cc65c092abd9505cc670c4e3421fd136fb6ea5b94efbe4c146572505", + "sha256:8f74e631b67482d504d7e9cf364071fc5d54c28e79a093ff402d5f8f81e23bfa", + "sha256:930315ac53dc65cbf52ab6b6d27422611f5fb461d763c531db229c7e1af6c0b3", + "sha256:96d3038f5bd061401996614f65d27a4ecb62d843eb4f48e212e6d129171a721f", + "sha256:a20299ee0ea2f9cca494396ac472d6e636745652a64a418b39522c120fd0a0a4", + "sha256:a34826d6465c2e2bbe9d0605f944f19d2480589f89863ed5f091943be27c9de4", + "sha256:a69970ee896e21db4c57e398646af9edc71c003bc52a3cc77fb150240fefd266", + "sha256:b9a8b391c2b0321e0cd7ec6b4cfcc3dd6349347bd1207d48bcb752aa6c553a66", + "sha256:ba13346ff6d3eb2dca0b6fa0d8a9d999eff3dcd9b55f3a890f12b0b6362b2b38", + "sha256:bb0608694a91db1e230b4a314e8ed00ad07ed0c518f9a69b83af2717e31291a3", + "sha256:c8830b7d5f16fd79d39b21e3d94f247219036b29b30c8270314c46bf8b732389", + "sha256:cac918cd7c4c498a60f5d2a61d4f0a6091c2c9490d81bc805c963444032d0dab", + "sha256:cc30cb900f42c8a246e2cb76539d9726f407330bc244ca7729c41a44e8d807fb", + "sha256:ccdc6a87f32b491129ada4b87a43b1895cf2c20fdb7f98ad979647506ffc41b6", + "sha256:d1a8b01f6a964fec702d6b6dac1f91f2b9f9fe41b310cbb16c7ef1fac82df06d", + "sha256:e004db88e5a75e5fdab1620fb9f90c9598c2a195a594225ac4ed2a6f1c23e162", + "sha256:eb2f43ae3037f1ef5e19339c41cf56947021ac892f668765cd65f8ab9814192e", + "sha256:fa466306fcf6b39b8a61d003123d442b23707d635a5cb05ac4e1b62cc79105cd" + ], + "index": "pypi", + "version": "==2.8.5" }, "pydantic": { "hashes": [ @@ -271,21 +390,6 @@ ], "version": "==1.6.1" }, - "python-dateutil": { - "hashes": [ - "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c", - "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a" - ], - "version": "==2.8.1" - }, - "python-editor": { - "hashes": [ - "sha256:1bf6e860a8ad52a14c3ee1252d5dc25b2030618ed80c022598f00176adc8367d", - "sha256:51fda6bcc5ddbbb7063b2af7509e43bd84bfc32a4ff71349ec7847713882327b", - "sha256:5f98b069316ea1c2ed3f67e7f5df6c0d8f10b689964a4a811ff64f0106819ec8" - ], - "version": "==1.0.4" - }, "regex": { "hashes": [ "sha256:0dc64ee3f33cd7899f79a8d788abfbec168410be356ed9bd30bbd3f0a23a7204", @@ -393,6 +497,10 @@ ], "version": "==1.4.1" }, + "umg-datalake-upload-agent": { + "editable": true, + "path": "." + }, "uvicorn": { "hashes": [ "sha256:50577d599775dac2301bac8bd5b540d19a9560144143c5bdab13cba92783b6e7", @@ -416,6 +524,20 @@ "markers": "sys_platform != 'win32' and sys_platform != 'cygwin' and platform_python_implementation != 'PyPy'", "version": "==0.14.0" }, + "virtualenv": { + "hashes": [ + "sha256:26cdd725a57fef4c7c22060dba4647ebd8ca377e30d1c1cf547b30a0b79c43b4", + "sha256:c51f1ba727d1614ce8fd62457748b469fbedfdab2c7e5dd480c9ae3fbe1233f1" + ], + "version": "==20.0.27" + }, + "virtualenv-clone": { + "hashes": [ + "sha256:07e74418b7cc64f4fda987bf5bc71ebd59af27a7bc9e8a8ee9fd54b1f2390a27", + "sha256:665e48dd54c84b98b71a657acb49104c54e7652bce9c1c4f6c6976ed4c827a29" + ], + "version": "==0.5.4" + }, "websockets": { "hashes": [ "sha256:0e4fb4de42701340bd2353bb2eee45314651caa6ccee80dbd5f5d5978888fed5", diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..1f7c4b8 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,40 @@ +################################################################################ +# This requirements file has been automatically generated from `Pipfile` with +# `pipenv-to-requirements` +# +# +# This has been done to maintain backward compatibility with tools and services +# that do not support `Pipfile` yet. +# +# Do NOT edit it directly, use `pipenv install [-d]` to modify `Pipfile` and +# `Pipfile.lock` and then regenerate `requirements*.txt`. +################################################################################ + +-e . +appdirs==1.4.4 +attrs==19.3.0 +black==19.10b0 +certifi==2020.6.20 +click==7.1.2 +distlib==0.3.1 +fastapi==0.59.0 +filelock==3.0.12 +h11==0.9.0 +httptools==0.1.1 ; sys_platform != 'win32' and sys_platform != 'cygwin' and platform_python_implementation != 'PyPy' +pathspec==0.8.0 +pbr==5.4.5 +pipenv-to-requirements==0.9.0 +pipenv==2020.6.2 +psycopg2-binary==2.8.5 +pydantic==1.6.1 +regex==2020.7.14 +six==1.15.0 +sqlalchemy==1.3.18 +starlette==0.13.4 +toml==0.10.1 +typed-ast==1.4.1 +uvicorn==0.11.5 +uvloop==0.14.0 ; sys_platform != 'win32' and sys_platform != 'cygwin' and platform_python_implementation != 'PyPy' +virtualenv-clone==0.5.4 +virtualenv==20.0.27 +websockets==8.1 diff --git a/requirements.txt b/requirements.txt index e7f7e7a..39f0725 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,35 +1,30 @@ +################################################################################ +# This requirements file has been automatically generated from `Pipfile` with +# `pipenv-to-requirements` +# +# +# This has been done to maintain backward compatibility with tools and services +# that do not support `Pipfile` yet. +# +# Do NOT edit it directly, use `pipenv install [-d]` to modify `Pipfile` and +# `Pipfile.lock` and then regenerate `requirements*.txt`. +################################################################################ + +pycdstar3 @ git+https://gitlab.gwdg.de/cdstar/pycdstar3@a0e0fd5ee137d09988756ab19c9b68f428ff7d38#egg=pycdstar3 alembic==1.4.2 -appdirs==1.4.4 -attrs==19.3.0 -black==19.10b0 certifi==2020.6.20 chardet==3.0.4 -click==7.1.2 -fastapi==0.59.0 -h11==0.9.0 -httptools==0.1.1 idna==2.10 iso8601==0.1.12 -Mako==1.1.3 -MarkupSafe==2.0.0a1 -pathspec==0.8.0 +mako==1.1.3 +markupsafe==2.0.0a1 psycopg2-binary==2.8.5 -pycdstar3 @ git+https://gitlab.gwdg.de/cdstar/pycdstar3.git@a0e0fd5ee137d09988756ab19c9b68f428ff7d38 -pydantic==1.6.1 python-dateutil==2.8.1 python-editor==1.0.4 -regex==2020.7.14 -requests==2.24.0 requests-toolbelt==0.9.1 +requests==2.24.0 six==1.15.0 -SQLAlchemy==1.3.18 -starlette==0.13.4 +sqlalchemy==1.3.18 tabulate==0.8.7 -toml==0.10.1 -tqdm==4.46.1 -typed-ast==1.4.1 --e git+git@gitlab.gwdg.de:medinf/umgmedic/active-workflow-datalake-agent.git@0cb6c82dd2d3e5fc887ce1b24b8b0b9fd82766e3#egg=UMG_Datalake_Upload_Agent +tqdm==4.47.0 urllib3==1.25.9 -uvicorn==0.11.5 -uvloop==0.14.0 -websockets==8.1 -- GitLab From 66e70b7e5b8872f91123a039b652984dfbdc7041 Mon Sep 17 00:00:00 2001 From: Marcel Parciak <marcel.parciak@gmail.com> Date: Fri, 17 Jul 2020 08:22:45 +0200 Subject: [PATCH 5/8] Change to alembic env to use the database config from the app. Signed-off-by: Marcel Parciak <marcel.parciak@gmail.com> --- alembic/env.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/alembic/env.py b/alembic/env.py index 379c754..7967f13 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -1,6 +1,6 @@ from logging.config import fileConfig -from sqlalchemy import engine_from_config +from sqlalchemy import create_engine from sqlalchemy import pool from alembic import context @@ -18,6 +18,7 @@ fileConfig(config.config_file_name) # from myapp import mymodel # target_metadata = mymodel.Base.metadata from uploader import models_internal +from uploader import config as upl_cfg target_metadata = models_internal.Base.metadata @@ -39,7 +40,7 @@ def run_migrations_offline(): script output. """ - url = config.get_main_option("sqlalchemy.url") + url = upl_cfg.BasicSettings().db_uri # config.get_main_option("sqlalchemy.url") context.configure( url=url, target_metadata=target_metadata, @@ -58,11 +59,7 @@ def run_migrations_online(): and associate a connection with the context. """ - connectable = engine_from_config( - config.get_section(config.config_ini_section), - prefix="sqlalchemy.", - poolclass=pool.NullPool, - ) + connectable = create_engine(upl_cfg.BasicSettings().db_uri) with connectable.connect() as connection: context.configure(connection=connection, target_metadata=target_metadata) -- GitLab From ed03f7ef5b072bc24503c94342c7c2379839b4eb Mon Sep 17 00:00:00 2001 From: Marcel Parciak <marcel.parciak@gmail.com> Date: Fri, 17 Jul 2020 08:33:07 +0200 Subject: [PATCH 6/8] Another change to alembic env.py script. Using os environment variables directly now (fastapi will not be initialized when starting alembic Signed-off-by: Marcel Parciak <marcel.parciak@gmail.com> --- alembic/env.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/alembic/env.py b/alembic/env.py index 7967f13..8993aed 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -1,3 +1,4 @@ +import os from logging.config import fileConfig from sqlalchemy import create_engine @@ -18,7 +19,6 @@ fileConfig(config.config_file_name) # from myapp import mymodel # target_metadata = mymodel.Base.metadata from uploader import models_internal -from uploader import config as upl_cfg target_metadata = models_internal.Base.metadata @@ -40,7 +40,7 @@ def run_migrations_offline(): script output. """ - url = upl_cfg.BasicSettings().db_uri # config.get_main_option("sqlalchemy.url") + url = os.getenv("DB_URI") # config.get_main_option("sqlalchemy.url") context.configure( url=url, target_metadata=target_metadata, @@ -59,7 +59,7 @@ def run_migrations_online(): and associate a connection with the context. """ - connectable = create_engine(upl_cfg.BasicSettings().db_uri) + connectable = create_engine(os.getenv("DB_URI")) with connectable.connect() as connection: context.configure(connection=connection, target_metadata=target_metadata) -- GitLab From 39773e4e4c67f1ba6b78bd5ef964dadd1c4297e7 Mon Sep 17 00:00:00 2001 From: Marcel Parciak <marcel.parciak@gmail.com> Date: Fri, 17 Jul 2020 08:35:20 +0200 Subject: [PATCH 7/8] Version bump Signed-off-by: Marcel Parciak <marcel.parciak@gmail.com> --- uploader/__version__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uploader/__version__.py b/uploader/__version__.py index fb8ccf0..4a1b1b0 100644 --- a/uploader/__version__.py +++ b/uploader/__version__.py @@ -1,4 +1,4 @@ -VERSION = (0, 9, 0) +VERSION = (0, 9, 1) __version__ = ".".join(map(str, VERSION)) -- GitLab From a802e213ce99a829361cfa4a730ae9012a672f65 Mon Sep 17 00:00:00 2001 From: Marcel Parciak <marcel.parciak@gmail.com> Date: Tue, 1 Sep 2020 07:36:15 +0200 Subject: [PATCH 8/8] Added LICENSE with reuse.software Signed-off-by: Marcel Parciak <marcel.parciak@gmail.com> --- .gitignore | 5 + .gitlab-ci.yml | 6 +- .reuse/dep5 | 10 + .vscode/settings.json.license | 3 + Dockerfile | 4 + LICENSES/CC0-1.0.txt | 119 ++++ LICENSES/GPL-3.0-or-later.txt | 625 ++++++++++++++++++ LICENSES/MIT.txt | 19 + Pipfile | 6 +- Pipfile.lock.license | 3 + Readme.md | 6 + alembic.ini | 6 +- alembic/README | 1 - alembic/env.py | 6 +- alembic/script.py.mako | 6 +- .../535aa0f40ba9_initial_table_layout.py | 6 +- prestart.sh | 4 + requirements-dev.txt | 6 +- requirements.txt | 6 +- setup.py | 7 +- uploader/.gitignore | 5 + uploader/__init__.py | 4 + uploader/__version__.py | 7 +- uploader/cdstar.py | 7 +- uploader/config.py | 6 +- uploader/database.py | 7 +- uploader/errors.py | 6 +- uploader/main.py | 9 +- uploader/models_activeworkflow.py | 6 +- uploader/models_cdstar.py | 7 +- uploader/models_internal.py | 6 +- uploader/store.py | 6 +- uploader/utils.py | 6 +- 33 files changed, 909 insertions(+), 27 deletions(-) create mode 100644 .reuse/dep5 create mode 100644 .vscode/settings.json.license create mode 100644 LICENSES/CC0-1.0.txt create mode 100644 LICENSES/GPL-3.0-or-later.txt create mode 100644 LICENSES/MIT.txt create mode 100644 Pipfile.lock.license delete mode 100644 alembic/README diff --git a/.gitignore b/.gitignore index 836cf32..aedd090 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,8 @@ +# SPDX-FileCopyrightText: 2020 2020 https://github.com/github/gitignore +# SPDX-FileCopyrightText: 2020 https://github.com/github/gitignore +# +# SPDX-License-Identifier: CC0-1.0 + # Do not upload the test directory, this is for local testing only test diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 38f992e..57c1026 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2020 2020 UMG MeDIC <marcel.parciak@med.uni-goettingen.de> +# +# SPDX-License-Identifier: GPL-3.0-or-later + stages: - test - publish @@ -57,4 +61,4 @@ publish_to_harbor: - docker build -t harbor.umg.eu/medic/$CI_PROJECT_NAME:latest . - docker tag harbor.umg.eu/medic/$CI_PROJECT_NAME:latest harbor.umg.eu/medic/$CI_PROJECT_NAME:$AGENTVERSION - docker push harbor.umg.eu/medic/$CI_PROJECT_NAME:latest - - docker push harbor.umg.eu/medic/$CI_PROJECT_NAME:$AGENTVERSION + - docker push harbor.umg.eu/medic/$CI_PROJECT_NAME:$AGENTVERSION \ No newline at end of file diff --git a/.reuse/dep5 b/.reuse/dep5 new file mode 100644 index 0000000..1785a92 --- /dev/null +++ b/.reuse/dep5 @@ -0,0 +1,10 @@ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: uploader +Upstream-Contact: UMG MeDIC <marcel.parciak@med.uni-goettingen.de> +Source: https://gitlab.gwdg.de/medinfpub/umg-medic/aw-agents/cdstar-upload-agent + +# Sample paragraph, commented out: +# +# Files: src/* +# Copyright: $YEAR $NAME <$CONTACT> +# License: ... diff --git a/.vscode/settings.json.license b/.vscode/settings.json.license new file mode 100644 index 0000000..8a0c58d --- /dev/null +++ b/.vscode/settings.json.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2020 UMG MeDIC <marcel.parciak@med.uni-goettingen.de> + +SPDX-License-Identifier: GPL-3.0-or-later \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index ccdd017..130d4fc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2020 2020 UMG MeDIC <marcel.parciak@med.uni-goettingen.de> +# +# SPDX-License-Identifier: GPL-3.0-or-later + FROM docker.io/tiangolo/uvicorn-gunicorn-fastapi:python3.8 COPY ./requirements.txt /app/requirements.txt diff --git a/LICENSES/CC0-1.0.txt b/LICENSES/CC0-1.0.txt new file mode 100644 index 0000000..a343ccd --- /dev/null +++ b/LICENSES/CC0-1.0.txt @@ -0,0 +1,119 @@ +Creative Commons Legal Code + +CC0 1.0 Universal CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES +NOT PROVIDE LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE +AN ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS INFORMATION +ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES REGARDING THE USE +OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED HEREUNDER, AND DISCLAIMS +LIABILITY FOR DAMAGES RESULTING FROM THE USE OF THIS DOCUMENT OR THE INFORMATION +OR WORKS PROVIDED HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer exclusive +Copyright and Related Rights (defined below) upon the creator and subsequent +owner(s) (each and all, an "owner") of an original work of authorship and/or +a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for the +purpose of contributing to a commons of creative, cultural and scientific +works ("Commons") that the public can reliably and without fear of later claims +of infringement build upon, modify, incorporate in other works, reuse and +redistribute as freely as possible in any form whatsoever and for any purposes, +including without limitation commercial purposes. These owners may contribute +to the Commons to promote the ideal of a free culture and the further production +of creative, cultural and scientific works, or to gain reputation or greater +distribution for their Work in part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any expectation +of additional consideration or compensation, the person associating CC0 with +a Work (the "Affirmer"), to the extent that he or she is an owner of Copyright +and Related Rights in the Work, voluntarily elects to apply CC0 to the Work +and publicly distribute the Work under its terms, with knowledge of his or +her Copyright and Related Rights in the Work and the meaning and intended +legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be protected +by copyright and related or neighboring rights ("Copyright and Related Rights"). +Copyright and Related Rights include, but are not limited to, the following: + +i. the right to reproduce, adapt, distribute, perform, display, communicate, +and translate a Work; + + ii. moral rights retained by the original author(s) and/or performer(s); + +iii. publicity and privacy rights pertaining to a person's image or likeness +depicted in a Work; + +iv. rights protecting against unfair competition in regards to a Work, subject +to the limitations in paragraph 4(a), below; + +v. rights protecting the extraction, dissemination, use and reuse of data +in a Work; + +vi. database rights (such as those arising under Directive 96/9/EC of the +European Parliament and of the Council of 11 March 1996 on the legal protection +of databases, and under any national implementation thereof, including any +amended or successor version of such directive); and + +vii. other similar, equivalent or corresponding rights throughout the world +based on applicable law or treaty, and any national implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention of, +applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and +unconditionally waives, abandons, and surrenders all of Affirmer's Copyright +and Related Rights and associated claims and causes of action, whether now +known or unknown (including existing as well as future claims and causes of +action), in the Work (i) in all territories worldwide, (ii) for the maximum +duration provided by applicable law or treaty (including future time extensions), +(iii) in any current or future medium and for any number of copies, and (iv) +for any purpose whatsoever, including without limitation commercial, advertising +or promotional purposes (the "Waiver"). Affirmer makes the Waiver for the +benefit of each member of the public at large and to the detriment of Affirmer's +heirs and successors, fully intending that such Waiver shall not be subject +to revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason be +judged legally invalid or ineffective under applicable law, then the Waiver +shall be preserved to the maximum extent permitted taking into account Affirmer's +express Statement of Purpose. In addition, to the extent the Waiver is so +judged Affirmer hereby grants to each affected person a royalty-free, non +transferable, non sublicensable, non exclusive, irrevocable and unconditional +license to exercise Affirmer's Copyright and Related Rights in the Work (i) +in all territories worldwide, (ii) for the maximum duration provided by applicable +law or treaty (including future time extensions), (iii) in any current or +future medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional purposes +(the "License"). The License shall be deemed effective as of the date CC0 +was applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder of +the License, and in such case Affirmer hereby affirms that he or she will +not (i) exercise any of his or her remaining Copyright and Related Rights +in the Work or (ii) assert any associated claims and causes of action with +respect to the Work, in either case contrary to Affirmer's express Statement +of Purpose. + + 4. Limitations and Disclaimers. + +a. No trademark or patent rights held by Affirmer are waived, abandoned, surrendered, +licensed or otherwise affected by this document. + +b. Affirmer offers the Work as-is and makes no representations or warranties +of any kind concerning the Work, express, implied, statutory or otherwise, +including without limitation warranties of title, merchantability, fitness +for a particular purpose, non infringement, or the absence of latent or other +defects, accuracy, or the present or absence of errors, whether or not discoverable, +all to the greatest extent permissible under applicable law. + +c. Affirmer disclaims responsibility for clearing rights of other persons +that may apply to the Work or any use thereof, including without limitation +any person's Copyright and Related Rights in the Work. Further, Affirmer disclaims +responsibility for obtaining any necessary consents, permissions or other +rights required for any use of the Work. + +d. Affirmer understands and acknowledges that Creative Commons is not a party +to this document and has no duty or obligation with respect to this CC0 or +use of the Work. diff --git a/LICENSES/GPL-3.0-or-later.txt b/LICENSES/GPL-3.0-or-later.txt new file mode 100644 index 0000000..e142a52 --- /dev/null +++ b/LICENSES/GPL-3.0-or-later.txt @@ -0,0 +1,625 @@ +GNU GENERAL PUBLIC LICENSE + +Version 3, 29 June 2007 + +Copyright © 2007 Free Software Foundation, Inc. <https://fsf.org/> + +Everyone is permitted to copy and distribute verbatim copies of this license +document, but changing it is not allowed. + +Preamble + +The GNU General Public License is a free, copyleft license for software and +other kinds of works. + +The licenses for most software and other practical works are designed to take +away your freedom to share and change the works. By contrast, the GNU General +Public License is intended to guarantee your freedom to share and change all +versions of a program--to make sure it remains free software for all its users. +We, the Free Software Foundation, use the GNU General Public License for most +of our software; it applies also to any other work released this way by its +authors. You can apply it to your programs, too. + +When we speak of free software, we are referring to freedom, not price. Our +General Public Licenses are designed to make sure that you have the freedom +to distribute copies of free software (and charge for them if you wish), that +you receive source code or can get it if you want it, that you can change +the software or use pieces of it in new free programs, and that you know you +can do these things. + +To protect your rights, we need to prevent others from denying you these rights +or asking you to surrender the rights. Therefore, you have certain responsibilities +if you distribute copies of the software, or if you modify it: responsibilities +to respect the freedom of others. + +For example, if you distribute copies of such a program, whether gratis or +for a fee, you must pass on to the recipients the same freedoms that you received. +You must make sure that they, too, receive or can get the source code. And +you must show them these terms so they know their rights. + +Developers that use the GNU GPL protect your rights with two steps: (1) assert +copyright on the software, and (2) offer you this License giving you legal +permission to copy, distribute and/or modify it. + +For the developers' and authors' protection, the GPL clearly explains that +there is no warranty for this free software. For both users' and authors' +sake, the GPL requires that modified versions be marked as changed, so that +their problems will not be attributed erroneously to authors of previous versions. + +Some devices are designed to deny users access to install or run modified +versions of the software inside them, although the manufacturer can do so. +This is fundamentally incompatible with the aim of protecting users' freedom +to change the software. The systematic pattern of such abuse occurs in the +area of products for individuals to use, which is precisely where it is most +unacceptable. Therefore, we have designed this version of the GPL to prohibit +the practice for those products. If such problems arise substantially in other +domains, we stand ready to extend this provision to those domains in future +versions of the GPL, as needed to protect the freedom of users. + +Finally, every program is threatened constantly by software patents. States +should not allow patents to restrict development and use of software on general-purpose +computers, but in those that do, we wish to avoid the special danger that +patents applied to a free program could make it effectively proprietary. To +prevent this, the GPL assures that patents cannot be used to render the program +non-free. + +The precise terms and conditions for copying, distribution and modification +follow. + +TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + +"Copyright" also means copyright-like laws that apply to other kinds of works, +such as semiconductor masks. + +"The Program" refers to any copyrightable work licensed under this License. +Each licensee is addressed as "you". "Licensees" and "recipients" may be individuals +or organizations. + +To "modify" a work means to copy from or adapt all or part of the work in +a fashion requiring copyright permission, other than the making of an exact +copy. The resulting work is called a "modified version" of the earlier work +or a work "based on" the earlier work. + +A "covered work" means either the unmodified Program or a work based on the +Program. + +To "propagate" a work means to do anything with it that, without permission, +would make you directly or secondarily liable for infringement under applicable +copyright law, except executing it on a computer or modifying a private copy. +Propagation includes copying, distribution (with or without modification), +making available to the public, and in some countries other activities as +well. + +To "convey" a work means any kind of propagation that enables other parties +to make or receive copies. Mere interaction with a user through a computer +network, with no transfer of a copy, is not conveying. + +An interactive user interface displays "Appropriate Legal Notices" to the +extent that it includes a convenient and prominently visible feature that +(1) displays an appropriate copyright notice, and (2) tells the user that +there is no warranty for the work (except to the extent that warranties are +provided), that licensees may convey the work under this License, and how +to view a copy of this License. If the interface presents a list of user commands +or options, such as a menu, a prominent item in the list meets this criterion. + + 1. Source Code. + +The "source code" for a work means the preferred form of the work for making +modifications to it. "Object code" means any non-source form of a work. + +A "Standard Interface" means an interface that either is an official standard +defined by a recognized standards body, or, in the case of interfaces specified +for a particular programming language, one that is widely used among developers +working in that language. + +The "System Libraries" of an executable work include anything, other than +the work as a whole, that (a) is included in the normal form of packaging +a Major Component, but which is not part of that Major Component, and (b) +serves only to enable use of the work with that Major Component, or to implement +a Standard Interface for which an implementation is available to the public +in source code form. A "Major Component", in this context, means a major essential +component (kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to produce +the work, or an object code interpreter used to run it. + +The "Corresponding Source" for a work in object code form means all the source +code needed to generate, install, and (for an executable work) run the object +code and to modify the work, including scripts to control those activities. +However, it does not include the work's System Libraries, or general-purpose +tools or generally available free programs which are used unmodified in performing +those activities but which are not part of the work. For example, Corresponding +Source includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically linked +subprograms that the work is specifically designed to require, such as by +intimate data communication or control flow between those subprograms and +other parts of the work. + +The Corresponding Source need not include anything that users can regenerate +automatically from other parts of the Corresponding Source. + + The Corresponding Source for a work in source code form is that same work. + + 2. Basic Permissions. + +All rights granted under this License are granted for the term of copyright +on the Program, and are irrevocable provided the stated conditions are met. +This License explicitly affirms your unlimited permission to run the unmodified +Program. The output from running a covered work is covered by this License +only if the output, given its content, constitutes a covered work. This License +acknowledges your rights of fair use or other equivalent, as provided by copyright +law. + +You may make, run and propagate covered works that you do not convey, without +conditions so long as your license otherwise remains in force. You may convey +covered works to others for the sole purpose of having them make modifications +exclusively for you, or provide you with facilities for running those works, +provided that you comply with the terms of this License in conveying all material +for which you do not control copyright. Those thus making or running the covered +works for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of your copyrighted +material outside their relationship with you. + +Conveying under any other circumstances is permitted solely under the conditions +stated below. Sublicensing is not allowed; section 10 makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + +No covered work shall be deemed part of an effective technological measure +under any applicable law fulfilling obligations under article 11 of the WIPO +copyright treaty adopted on 20 December 1996, or similar laws prohibiting +or restricting circumvention of such measures. + +When you convey a covered work, you waive any legal power to forbid circumvention +of technological measures to the extent such circumvention is effected by +exercising rights under this License with respect to the covered work, and +you disclaim any intention to limit operation or modification of the work +as a means of enforcing, against the work's users, your or third parties' +legal rights to forbid circumvention of technological measures. + + 4. Conveying Verbatim Copies. + +You may convey verbatim copies of the Program's source code as you receive +it, in any medium, provided that you conspicuously and appropriately publish +on each copy an appropriate copyright notice; keep intact all notices stating +that this License and any non-permissive terms added in accord with section +7 apply to the code; keep intact all notices of the absence of any warranty; +and give all recipients a copy of this License along with the Program. + +You may charge any price or no price for each copy that you convey, and you +may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + +You may convey a work based on the Program, or the modifications to produce +it from the Program, in the form of source code under the terms of section +4, provided that you also meet all of these conditions: + +a) The work must carry prominent notices stating that you modified it, and +giving a relevant date. + +b) The work must carry prominent notices stating that it is released under +this License and any conditions added under section 7. This requirement modifies +the requirement in section 4 to "keep intact all notices". + +c) You must license the entire work, as a whole, under this License to anyone +who comes into possession of a copy. This License will therefore apply, along +with any applicable section 7 additional terms, to the whole of the work, +and all its parts, regardless of how they are packaged. This License gives +no permission to license the work in any other way, but it does not invalidate +such permission if you have separately received it. + +d) If the work has interactive user interfaces, each must display Appropriate +Legal Notices; however, if the Program has interactive interfaces that do +not display Appropriate Legal Notices, your work need not make them do so. + +A compilation of a covered work with other separate and independent works, +which are not by their nature extensions of the covered work, and which are +not combined with it such as to form a larger program, in or on a volume of +a storage or distribution medium, is called an "aggregate" if the compilation +and its resulting copyright are not used to limit the access or legal rights +of the compilation's users beyond what the individual works permit. Inclusion +of a covered work in an aggregate does not cause this License to apply to +the other parts of the aggregate. + + 6. Conveying Non-Source Forms. + +You may convey a covered work in object code form under the terms of sections +4 and 5, provided that you also convey the machine-readable Corresponding +Source under the terms of this License, in one of these ways: + +a) Convey the object code in, or embodied in, a physical product (including +a physical distribution medium), accompanied by the Corresponding Source fixed +on a durable physical medium customarily used for software interchange. + +b) Convey the object code in, or embodied in, a physical product (including +a physical distribution medium), accompanied by a written offer, valid for +at least three years and valid for as long as you offer spare parts or customer +support for that product model, to give anyone who possesses the object code +either (1) a copy of the Corresponding Source for all the software in the +product that is covered by this License, on a durable physical medium customarily +used for software interchange, for a price no more than your reasonable cost +of physically performing this conveying of source, or (2) access to copy the +Corresponding Source from a network server at no charge. + +c) Convey individual copies of the object code with a copy of the written +offer to provide the Corresponding Source. This alternative is allowed only +occasionally and noncommercially, and only if you received the object code +with such an offer, in accord with subsection 6b. + +d) Convey the object code by offering access from a designated place (gratis +or for a charge), and offer equivalent access to the Corresponding Source +in the same way through the same place at no further charge. You need not +require recipients to copy the Corresponding Source along with the object +code. If the place to copy the object code is a network server, the Corresponding +Source may be on a different server (operated by you or a third party) that +supports equivalent copying facilities, provided you maintain clear directions +next to the object code saying where to find the Corresponding Source. Regardless +of what server hosts the Corresponding Source, you remain obligated to ensure +that it is available for as long as needed to satisfy these requirements. + +e) Convey the object code using peer-to-peer transmission, provided you inform +other peers where the object code and Corresponding Source of the work are +being offered to the general public at no charge under subsection 6d. + +A separable portion of the object code, whose source code is excluded from +the Corresponding Source as a System Library, need not be included in conveying +the object code work. + +A "User Product" is either (1) a "consumer product", which means any tangible +personal property which is normally used for personal, family, or household +purposes, or (2) anything designed or sold for incorporation into a dwelling. +In determining whether a product is a consumer product, doubtful cases shall +be resolved in favor of coverage. For a particular product received by a particular +user, "normally used" refers to a typical or common use of that class of product, +regardless of the status of the particular user or of the way in which the +particular user actually uses, or expects or is expected to use, the product. +A product is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent the +only significant mode of use of the product. + +"Installation Information" for a User Product means any methods, procedures, +authorization keys, or other information required to install and execute modified +versions of a covered work in that User Product from a modified version of +its Corresponding Source. The information must suffice to ensure that the +continued functioning of the modified object code is in no case prevented +or interfered with solely because modification has been made. + +If you convey an object code work under this section in, or with, or specifically +for use in, a User Product, and the conveying occurs as part of a transaction +in which the right of possession and use of the User Product is transferred +to the recipient in perpetuity or for a fixed term (regardless of how the +transaction is characterized), the Corresponding Source conveyed under this +section must be accompanied by the Installation Information. But this requirement +does not apply if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has been installed +in ROM). + +The requirement to provide Installation Information does not include a requirement +to continue to provide support service, warranty, or updates for a work that +has been modified or installed by the recipient, or for the User Product in +which it has been modified or installed. Access to a network may be denied +when the modification itself materially and adversely affects the operation +of the network or violates the rules and protocols for communication across +the network. + +Corresponding Source conveyed, and Installation Information provided, in accord +with this section must be in a format that is publicly documented (and with +an implementation available to the public in source code form), and must require +no special password or key for unpacking, reading or copying. + + 7. Additional Terms. + +"Additional permissions" are terms that supplement the terms of this License +by making exceptions from one or more of its conditions. Additional permissions +that are applicable to the entire Program shall be treated as though they +were included in this License, to the extent that they are valid under applicable +law. If additional permissions apply only to part of the Program, that part +may be used separately under those permissions, but the entire Program remains +governed by this License without regard to the additional permissions. + +When you convey a copy of a covered work, you may at your option remove any +additional permissions from that copy, or from any part of it. (Additional +permissions may be written to require their own removal in certain cases when +you modify the work.) You may place additional permissions on material, added +by you to a covered work, for which you have or can give appropriate copyright +permission. + +Notwithstanding any other provision of this License, for material you add +to a covered work, you may (if authorized by the copyright holders of that +material) supplement the terms of this License with terms: + +a) Disclaiming warranty or limiting liability differently from the terms of +sections 15 and 16 of this License; or + +b) Requiring preservation of specified reasonable legal notices or author +attributions in that material or in the Appropriate Legal Notices displayed +by works containing it; or + +c) Prohibiting misrepresentation of the origin of that material, or requiring +that modified versions of such material be marked in reasonable ways as different +from the original version; or + +d) Limiting the use for publicity purposes of names of licensors or authors +of the material; or + +e) Declining to grant rights under trademark law for use of some trade names, +trademarks, or service marks; or + +f) Requiring indemnification of licensors and authors of that material by +anyone who conveys the material (or modified versions of it) with contractual +assumptions of liability to the recipient, for any liability that these contractual +assumptions directly impose on those licensors and authors. + +All other non-permissive additional terms are considered "further restrictions" +within the meaning of section 10. If the Program as you received it, or any +part of it, contains a notice stating that it is governed by this License +along with a term that is a further restriction, you may remove that term. +If a license document contains a further restriction but permits relicensing +or conveying under this License, you may add to a covered work material governed +by the terms of that license document, provided that the further restriction +does not survive such relicensing or conveying. + +If you add terms to a covered work in accord with this section, you must place, +in the relevant source files, a statement of the additional terms that apply +to those files, or a notice indicating where to find the applicable terms. + +Additional terms, permissive or non-permissive, may be stated in the form +of a separately written license, or stated as exceptions; the above requirements +apply either way. + + 8. Termination. + +You may not propagate or modify a covered work except as expressly provided +under this License. Any attempt otherwise to propagate or modify it is void, +and will automatically terminate your rights under this License (including +any patent licenses granted under the third paragraph of section 11). + +However, if you cease all violation of this License, then your license from +a particular copyright holder is reinstated (a) provisionally, unless and +until the copyright holder explicitly and finally terminates your license, +and (b) permanently, if the copyright holder fails to notify you of the violation +by some reasonable means prior to 60 days after the cessation. + +Moreover, your license from a particular copyright holder is reinstated permanently +if the copyright holder notifies you of the violation by some reasonable means, +this is the first time you have received notice of violation of this License +(for any work) from that copyright holder, and you cure the violation prior +to 30 days after your receipt of the notice. + +Termination of your rights under this section does not terminate the licenses +of parties who have received copies or rights from you under this License. +If your rights have been terminated and not permanently reinstated, you do +not qualify to receive new licenses for the same material under section 10. + + 9. Acceptance Not Required for Having Copies. + +You are not required to accept this License in order to receive or run a copy +of the Program. Ancillary propagation of a covered work occurring solely as +a consequence of using peer-to-peer transmission to receive a copy likewise +does not require acceptance. However, nothing other than this License grants +you permission to propagate or modify any covered work. These actions infringe +copyright if you do not accept this License. Therefore, by modifying or propagating +a covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + +Each time you convey a covered work, the recipient automatically receives +a license from the original licensors, to run, modify and propagate that work, +subject to this License. You are not responsible for enforcing compliance +by third parties with this License. + +An "entity transaction" is a transaction transferring control of an organization, +or substantially all assets of one, or subdividing an organization, or merging +organizations. If propagation of a covered work results from an entity transaction, +each party to that transaction who receives a copy of the work also receives +whatever licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the Corresponding +Source of the work from the predecessor in interest, if the predecessor has +it or can get it with reasonable efforts. + +You may not impose any further restrictions on the exercise of the rights +granted or affirmed under this License. For example, you may not impose a +license fee, royalty, or other charge for exercise of rights granted under +this License, and you may not initiate litigation (including a cross-claim +or counterclaim in a lawsuit) alleging that any patent claim is infringed +by making, using, selling, offering for sale, or importing the Program or +any portion of it. + + 11. Patents. + +A "contributor" is a copyright holder who authorizes use under this License +of the Program or a work on which the Program is based. The work thus licensed +is called the contributor's "contributor version". + +A contributor's "essential patent claims" are all patent claims owned or controlled +by the contributor, whether already acquired or hereafter acquired, that would +be infringed by some manner, permitted by this License, of making, using, +or selling its contributor version, but do not include claims that would be +infringed only as a consequence of further modification of the contributor +version. For purposes of this definition, "control" includes the right to +grant patent sublicenses in a manner consistent with the requirements of this +License. + +Each contributor grants you a non-exclusive, worldwide, royalty-free patent +license under the contributor's essential patent claims, to make, use, sell, +offer for sale, import and otherwise run, modify and propagate the contents +of its contributor version. + +In the following three paragraphs, a "patent license" is any express agreement +or commitment, however denominated, not to enforce a patent (such as an express +permission to practice a patent or covenant not to sue for patent infringement). +To "grant" such a patent license to a party means to make such an agreement +or commitment not to enforce a patent against the party. + +If you convey a covered work, knowingly relying on a patent license, and the +Corresponding Source of the work is not available for anyone to copy, free +of charge and under the terms of this License, through a publicly available +network server or other readily accessible means, then you must either (1) +cause the Corresponding Source to be so available, or (2) arrange to deprive +yourself of the benefit of the patent license for this particular work, or +(3) arrange, in a manner consistent with the requirements of this License, +to extend the patent license to downstream recipients. "Knowingly relying" +means you have actual knowledge that, but for the patent license, your conveying +the covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that country +that you have reason to believe are valid. + +If, pursuant to or in connection with a single transaction or arrangement, +you convey, or propagate by procuring conveyance of, a covered work, and grant +a patent license to some of the parties receiving the covered work authorizing +them to use, propagate, modify or convey a specific copy of the covered work, +then the patent license you grant is automatically extended to all recipients +of the covered work and works based on it. + +A patent license is "discriminatory" if it does not include within the scope +of its coverage, prohibits the exercise of, or is conditioned on the non-exercise +of one or more of the rights that are specifically granted under this License. +You may not convey a covered work if you are a party to an arrangement with +a third party that is in the business of distributing software, under which +you make payment to the third party based on the extent of your activity of +conveying the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory patent +license (a) in connection with copies of the covered work conveyed by you +(or copies made from those copies), or (b) primarily for and in connection +with specific products or compilations that contain the covered work, unless +you entered into that arrangement, or that patent license was granted, prior +to 28 March 2007. + +Nothing in this License shall be construed as excluding or limiting any implied +license or other defenses to infringement that may otherwise be available +to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + +If conditions are imposed on you (whether by court order, agreement or otherwise) +that contradict the conditions of this License, they do not excuse you from +the conditions of this License. If you cannot convey a covered work so as +to satisfy simultaneously your obligations under this License and any other +pertinent obligations, then as a consequence you may not convey it at all. +For example, if you agree to terms that obligate you to collect a royalty +for further conveying from those to whom you convey the Program, the only +way you could satisfy both those terms and this License would be to refrain +entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + +Notwithstanding any other provision of this License, you have permission to +link or combine any covered work with a work licensed under version 3 of the +GNU Affero General Public License into a single combined work, and to convey +the resulting work. The terms of this License will continue to apply to the +part which is the covered work, but the special requirements of the GNU Affero +General Public License, section 13, concerning interaction through a network +will apply to the combination as such. + + 14. Revised Versions of this License. + +The Free Software Foundation may publish revised and/or new versions of the +GNU General Public License from time to time. Such new versions will be similar +in spirit to the present version, but may differ in detail to address new +problems or concerns. + +Each version is given a distinguishing version number. If the Program specifies +that a certain numbered version of the GNU General Public License "or any +later version" applies to it, you have the option of following the terms and +conditions either of that numbered version or of any later version published +by the Free Software Foundation. If the Program does not specify a version +number of the GNU General Public License, you may choose any version ever +published by the Free Software Foundation. + +If the Program specifies that a proxy can decide which future versions of +the GNU General Public License can be used, that proxy's public statement +of acceptance of a version permanently authorizes you to choose that version +for the Program. + +Later license versions may give you additional or different permissions. However, +no additional obligations are imposed on any author or copyright holder as +a result of your choosing to follow a later version. + + 15. Disclaimer of Warranty. + +THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE +LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER +EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM +PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR +CORRECTION. + + 16. Limitation of Liability. + +IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL +ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM +AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, +INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO +USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED +INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE +PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER +PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + +If the disclaimer of warranty and limitation of liability provided above cannot +be given local legal effect according to their terms, reviewing courts shall +apply local law that most closely approximates an absolute waiver of all civil +liability in connection with the Program, unless a warranty or assumption +of liability accompanies a copy of the Program in return for a fee. END OF +TERMS AND CONDITIONS + +How to Apply These Terms to Your New Programs + +If you develop a new program, and you want it to be of the greatest possible +use to the public, the best way to achieve this is to make it free software +which everyone can redistribute and change under these terms. + +To do so, attach the following notices to the program. It is safest to attach +them to the start of each source file to most effectively state the exclusion +of warranty; and each file should have at least the "copyright" line and a +pointer to where the full notice is found. + +<one line to give the program's name and a brief idea of what it does.> + +Copyright (C) <year> <name of author> + +This program is free software: you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation, either version 3 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program. If not, see <https://www.gnu.org/licenses/>. + +Also add information on how to contact you by electronic and paper mail. + +If the program does terminal interaction, make it output a short notice like +this when it starts in an interactive mode: + +<program> Copyright (C) <year> <name of author> + +This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + +This is free software, and you are welcome to redistribute it under certain +conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands might +be different; for a GUI interface, you would use an "about box". + +You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. For +more information on this, and how to apply and follow the GNU GPL, see <https://www.gnu.org/licenses/>. + +The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General Public +License instead of this License. But first, please read <https://www.gnu.org/ +licenses /why-not-lgpl.html>. diff --git a/LICENSES/MIT.txt b/LICENSES/MIT.txt new file mode 100644 index 0000000..204b93d --- /dev/null +++ b/LICENSES/MIT.txt @@ -0,0 +1,19 @@ +MIT License Copyright (c) <year> <copyright holders> + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is furnished +to do so, subject to the following conditions: + +The above copyright notice and this permission notice (including the next +paragraph) shall be included in all copies or substantial portions of the +Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF +OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/Pipfile b/Pipfile index 8a2c126..e98dda3 100644 --- a/Pipfile +++ b/Pipfile @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2020 2020 UMG MeDIC <marcel.parciak@med.uni-goettingen.de> +# +# SPDX-License-Identifier: GPL-3.0-or-later + [[source]] name = "pypi" url = "https://pypi.org/simple" @@ -20,4 +24,4 @@ pycdstar3 = {editable = true,git = "https://gitlab.gwdg.de/cdstar/pycdstar3"} python_version = "3.8" [pipenv] -allow_prereleases = true +allow_prereleases = true \ No newline at end of file diff --git a/Pipfile.lock.license b/Pipfile.lock.license new file mode 100644 index 0000000..8a0c58d --- /dev/null +++ b/Pipfile.lock.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2020 UMG MeDIC <marcel.parciak@med.uni-goettingen.de> + +SPDX-License-Identifier: GPL-3.0-or-later \ No newline at end of file diff --git a/Readme.md b/Readme.md index 6de1e66..620b0fc 100644 --- a/Readme.md +++ b/Readme.md @@ -1,3 +1,9 @@ +<!-- +SPDX-FileCopyrightText: 2020 2020 UMG MeDIC <marcel.parciak@med.uni-goettingen.de> + +SPDX-License-Identifier: GPL-3.0-or-later +--> + Uploads a list of files to CDSTAR. # CDSTAR Upload Agent diff --git a/alembic.ini b/alembic.ini index ed37faf..1d1bb35 100644 --- a/alembic.ini +++ b/alembic.ini @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2020 2020 UMG MeDIC <marcel.parciak@med.uni-goettingen.de> +# +# SPDX-License-Identifier: GPL-3.0-or-later + # A generic, single database configuration. [alembic] @@ -82,4 +86,4 @@ formatter = generic [formatter_generic] format = %(levelname)-5.5s [%(name)s] %(message)s -datefmt = %H:%M:%S +datefmt = %H:%M:%S \ No newline at end of file diff --git a/alembic/README b/alembic/README deleted file mode 100644 index 98e4f9c..0000000 --- a/alembic/README +++ /dev/null @@ -1 +0,0 @@ -Generic single-database configuration. \ No newline at end of file diff --git a/alembic/env.py b/alembic/env.py index 76e9130..0717a3b 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2020 2020 UMG MeDIC <marcel.parciak@med.uni-goettingen.de> +# +# SPDX-License-Identifier: GPL-3.0-or-later + import os from logging.config import fileConfig @@ -71,4 +75,4 @@ def run_migrations_online(): if context.is_offline_mode(): run_migrations_offline() else: - run_migrations_online() + run_migrations_online() \ No newline at end of file diff --git a/alembic/script.py.mako b/alembic/script.py.mako index 2c01563..accb3ab 100644 --- a/alembic/script.py.mako +++ b/alembic/script.py.mako @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2020 2020 UMG MeDIC <marcel.parciak@med.uni-goettingen.de> +# +# SPDX-License-Identifier: GPL-3.0-or-later + """${message} Revision ID: ${up_revision} @@ -21,4 +25,4 @@ def upgrade(): def downgrade(): - ${downgrades if downgrades else "pass"} + ${downgrades if downgrades else "pass"} \ No newline at end of file diff --git a/alembic/versions/535aa0f40ba9_initial_table_layout.py b/alembic/versions/535aa0f40ba9_initial_table_layout.py index 9923b64..343e1be 100644 --- a/alembic/versions/535aa0f40ba9_initial_table_layout.py +++ b/alembic/versions/535aa0f40ba9_initial_table_layout.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2020 2020 UMG MeDIC <marcel.parciak@med.uni-goettingen.de> +# +# SPDX-License-Identifier: GPL-3.0-or-later + """Initial table layout Revision ID: 535aa0f40ba9 @@ -42,4 +46,4 @@ def downgrade(): op.drop_index(op.f("ix_upload_state"), table_name="upload") op.drop_index(op.f("ix_upload_id"), table_name="upload") op.drop_table("upload") - # ### end Alembic commands ### + # ### end Alembic commands ### \ No newline at end of file diff --git a/prestart.sh b/prestart.sh index 65c28a1..7073221 100644 --- a/prestart.sh +++ b/prestart.sh @@ -1,5 +1,9 @@ #! /usr/bin/env sh +# SPDX-FileCopyrightText: 2020 2020 UMG MeDIC <marcel.parciak@med.uni-goettingen.de> +# +# SPDX-License-Identifier: GPL-3.0-or-later + echo "prestart.sh file for uvicorn-gunicorn docker image. Runs alembic migrations." echo "Sleeping for 5 seconds to let the database start." sleep 5 diff --git a/requirements-dev.txt b/requirements-dev.txt index 1f7c4b8..947210f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2020 UMG MeDIC <marcel.parciak@med.uni-goettingen.de> +# +# SPDX-License-Identifier: GPL-3.0-or-later + ################################################################################ # This requirements file has been automatically generated from `Pipfile` with # `pipenv-to-requirements` @@ -37,4 +41,4 @@ uvicorn==0.11.5 uvloop==0.14.0 ; sys_platform != 'win32' and sys_platform != 'cygwin' and platform_python_implementation != 'PyPy' virtualenv-clone==0.5.4 virtualenv==20.0.27 -websockets==8.1 +websockets==8.1 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 39f0725..ae9396a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2020 UMG MeDIC <marcel.parciak@med.uni-goettingen.de> +# +# SPDX-License-Identifier: GPL-3.0-or-later + ################################################################################ # This requirements file has been automatically generated from `Pipfile` with # `pipenv-to-requirements` @@ -27,4 +31,4 @@ six==1.15.0 sqlalchemy==1.3.18 tabulate==0.8.7 tqdm==4.47.0 -urllib3==1.25.9 +urllib3==1.25.9 \ No newline at end of file diff --git a/setup.py b/setup.py index faeeab2..816700c 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,9 @@ #!/usr/bin/env python + +# SPDX-FileCopyrightText: 2020 https://github.com/navdeep-G/setup.py +# +# SPDX-License-Identifier: MIT + # -*- coding: utf-8 -*- # Note: To use the 'upload' functionality of this file, you must: @@ -126,4 +131,4 @@ setup( ], # $ setup.py publish support. cmdclass={"upload": UploadCommand,}, -) +) \ No newline at end of file diff --git a/uploader/.gitignore b/uploader/.gitignore index 5391d87..1f99e0b 100644 --- a/uploader/.gitignore +++ b/uploader/.gitignore @@ -1,3 +1,8 @@ +# SPDX-FileCopyrightText: 2020 2020 https://github.com/github/gitignore +# SPDX-FileCopyrightText: 2020 https://github.com/github/gitignore +# +# SPDX-License-Identifier: CC0-1.0 + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/uploader/__init__.py b/uploader/__init__.py index 2ecbf65..940c462 100644 --- a/uploader/__init__.py +++ b/uploader/__init__.py @@ -1 +1,5 @@ +# SPDX-FileCopyrightText: 2020 2020 UMG MeDIC <marcel.parciak@med.uni-goettingen.de> +# +# SPDX-License-Identifier: GPL-3.0-or-later + # TODO: Do a config check here! \ No newline at end of file diff --git a/uploader/__version__.py b/uploader/__version__.py index 4a1b1b0..c215aad 100644 --- a/uploader/__version__.py +++ b/uploader/__version__.py @@ -1,4 +1,7 @@ -VERSION = (0, 9, 1) +# SPDX-FileCopyrightText: 2020 2020 UMG MeDIC <marcel.parciak@med.uni-goettingen.de> +# +# SPDX-License-Identifier: GPL-3.0-or-later -__version__ = ".".join(map(str, VERSION)) +VERSION = (0, 9, 1) +__version__ = ".".join(map(str, VERSION)) \ No newline at end of file diff --git a/uploader/cdstar.py b/uploader/cdstar.py index fc8235f..e415427 100644 --- a/uploader/cdstar.py +++ b/uploader/cdstar.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2020 2020 UMG MeDIC <marcel.parciak@med.uni-goettingen.de> +# +# SPDX-License-Identifier: GPL-3.0-or-later + import datetime import os from typing import List, Optional @@ -91,5 +95,4 @@ def upload_file_to_archive( return cdmodels.FileInfo(id=None, name=filename) return cdmodels.FileInfo.parse_obj(response) else: - return cdmodels.FileInfo(id=None, name=file) - + return cdmodels.FileInfo(id=None, name=file) \ No newline at end of file diff --git a/uploader/config.py b/uploader/config.py index 7abeb70..b5adde7 100644 --- a/uploader/config.py +++ b/uploader/config.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2020 2020 UMG MeDIC <marcel.parciak@med.uni-goettingen.de> +# +# SPDX-License-Identifier: GPL-3.0-or-later + from importlib import metadata from pydantic import BaseSettings import os @@ -36,4 +40,4 @@ with open( os.path.join(os.path.abspath(os.path.dirname(__file__)), os.pardir, "Readme.md"), "r", ) as readme: - agent_description = readme.read() + agent_description = readme.read() \ No newline at end of file diff --git a/uploader/database.py b/uploader/database.py index 2bd1fee..eb307a1 100644 --- a/uploader/database.py +++ b/uploader/database.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2020 2020 UMG MeDIC <marcel.parciak@med.uni-goettingen.de> +# +# SPDX-License-Identifier: GPL-3.0-or-later + from sqlalchemy import create_engine from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import sessionmaker @@ -6,5 +10,4 @@ from uploader import config engine = create_engine(config.BasicSettings().db_uri) SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) -Base = declarative_base() - +Base = declarative_base() \ No newline at end of file diff --git a/uploader/errors.py b/uploader/errors.py index b83e770..7ae93a5 100644 --- a/uploader/errors.py +++ b/uploader/errors.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2020 2020 UMG MeDIC <marcel.parciak@med.uni-goettingen.de> +# +# SPDX-License-Identifier: GPL-3.0-or-later + class NetworkException(Exception): def __init__(self, name: str, message: str): self.name = name @@ -13,4 +17,4 @@ class ActiveWorkflowException(Exception): class InternalException(Exception): def __init__(self, name: str, message: str): self.name = name - self.message = message + self.message = message \ No newline at end of file diff --git a/uploader/main.py b/uploader/main.py index 2a0f1eb..e19b67d 100644 --- a/uploader/main.py +++ b/uploader/main.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2020 2020 UMG MeDIC <marcel.parciak@med.uni-goettingen.de> +# +# SPDX-License-Identifier: GPL-3.0-or-later + import fastapi import fastapi.responses from fastapi import BackgroundTasks, FastAPI, HTTPException, Request, Depends @@ -292,7 +296,7 @@ def upload_completed(upload_id: int, response: awmodels.ResponseCheck, db: Sessi return upload.state = intmodels.UploadState.completed upload.completed_at = datetime.datetime.now() - upload.response = response.dict(exclude_none=None) + upload.response = response.dict(exclude_none=True) store.store_upload(db, upload) error_log.info(f"Completed upload of {upload_id}.") @@ -318,5 +322,4 @@ async def int_exception_handler(request: Request, exc: InternalException): return fastapi.responses.JSONResponse( status_code=fastapi.status.HTTP_500_INTERNAL_SERVER_ERROR, content=resp.dict(exclude_none=True), - ) - + ) \ No newline at end of file diff --git a/uploader/models_activeworkflow.py b/uploader/models_activeworkflow.py index d7cbba3..e741b5b 100644 --- a/uploader/models_activeworkflow.py +++ b/uploader/models_activeworkflow.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2020 2020 UMG MeDIC <marcel.parciak@med.uni-goettingen.de> +# +# SPDX-License-Identifier: GPL-3.0-or-later + import abc from typing import Any, Dict, List, Optional @@ -222,4 +226,4 @@ class ResultCheck(ResultCommon): class ResponseCheck(ResponseCommon): - result: ResultCheck = Field(ResultCheck(), example=ResultCheck()) + result: ResultCheck = Field(ResultCheck(), example=ResultCheck()) \ No newline at end of file diff --git a/uploader/models_cdstar.py b/uploader/models_cdstar.py index b04de01..4f0c19e 100644 --- a/uploader/models_cdstar.py +++ b/uploader/models_cdstar.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2020 2020 UMG MeDIC <marcel.parciak@med.uni-goettingen.de> +# +# SPDX-License-Identifier: GPL-3.0-or-later + import datetime from pydantic import BaseModel, Field @@ -67,5 +71,4 @@ class ArchiveInfo(BaseModel): [], example=[FileInfo(id="abcdef123456"), FileInfo(id="fedcba654321")], description="List of files in this archive. May be incomplete or missing based on query parameters, permissions and server configuration. See Get Archive Info for details.", - ) - + ) \ No newline at end of file diff --git a/uploader/models_internal.py b/uploader/models_internal.py index 05f44a8..4a1a72c 100644 --- a/uploader/models_internal.py +++ b/uploader/models_internal.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2020 2020 UMG MeDIC <marcel.parciak@med.uni-goettingen.de> +# +# SPDX-License-Identifier: GPL-3.0-or-later + import enum from sqlalchemy import Integer, Column, Enum, DateTime, JSON @@ -18,4 +22,4 @@ class UploadProcedure(Base): started_at = Column(DateTime) completed_at = Column(DateTime) retrieved_at = Column(DateTime) - response = Column(JSON) + response = Column(JSON) \ No newline at end of file diff --git a/uploader/store.py b/uploader/store.py index fc91f4a..6e550da 100644 --- a/uploader/store.py +++ b/uploader/store.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2020 2020 UMG MeDIC <marcel.parciak@med.uni-goettingen.de> +# +# SPDX-License-Identifier: GPL-3.0-or-later + import datetime from typing import List, Optional @@ -50,4 +54,4 @@ def get_upload_by_id(db: Session, id: int) -> Optional[intmodel.UploadProcedure] db.query(intmodel.UploadProcedure) .filter(intmodel.UploadProcedure.id == id) .first() - ) + ) \ No newline at end of file diff --git a/uploader/utils.py b/uploader/utils.py index 96c02ae..3fd1495 100644 --- a/uploader/utils.py +++ b/uploader/utils.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2020 2020 UMG MeDIC <marcel.parciak@med.uni-goettingen.de> +# +# SPDX-License-Identifier: GPL-3.0-or-later + import os import re from typing import List @@ -96,4 +100,4 @@ def retrieve_options_from_parameters(params: awmodels.ParamsReceive) -> dict: if params.message.payload.options[k] is not None: opts[k] = params.message.payload.options[k] - return opts + return opts \ No newline at end of file -- GitLab