grzctl 0.1.0__tar.gz

grzctl-0.1.0/.gitignore

@@ -0,0 +1,195 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+#poetry.toml
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Abstra
+# Abstra is an AI-powered process automation framework.
+# Ignore directories containing user credentials, local state, and settings.
+# Learn more at https://abstra.io/docs
+.abstra/
+
+# Visual Studio Code
+#  Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore 
+#  that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#  and can be added to the global gitignore or merged into this file. However, if you prefer, 
+#  you could uncomment the following to ignore the entire vscode folder
+# .vscode/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Cursor
+#  Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to
+#  exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
+#  refer to https://docs.cursor.com/context/ignore-files
+.cursorignore
+.cursorindexingignore

grzctl-0.1.0/CHANGELOG.md

@@ -0,0 +1,8 @@
+# Changelog
+
+## 0.1.0 (2025-06-11)
+
+
+### Features
+
+* migrate to monorepo configuration ([36c7360](https://github.com/BfArM-MVH/grz-tools/commit/36c736044ce09473cc664b4471117465c5cab9a3))

grzctl-0.1.0/PKG-INFO

@@ -0,0 +1,32 @@
+Metadata-Version: 2.4
+Name: grzctl
+Version: 0.1.0
+Summary: Control CLI for GRZ administrators.
+Project-URL: Homepage, https://github.com/BfArM-MVH/grz-tools
+Project-URL: Repository, https://github.com/BfArM-MVH/grz-tools
+Project-URL: Documentation, https://github.com/BfArM-MVH/grz-tools/tree/main/packages/grzctl
+Project-URL: Issues, https://github.com/BfArM-MVH/grz-tools/issues
+Author-email: Koray Kirli <koraykirli@gmail.com>, Mathias Lesche <mathias.lesche@tu-dresden.de>, "Florian R. Hölzlwimmer" <git.ich@frhoelzlwimmer.de>, Till Hartmann <till.hartmann@bih-charite.de>, Thomas Sell <thomas.sell@bih-charite.de>, Travis Wrightsman <travis.wrightsman@uni-tuebingen.de>
+License-Expression: MIT
+Keywords: GDC,GRZ,S3
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.12
+Requires-Dist: boto3<2,>=1.36
+Requires-Dist: click<9,>=8.1.7
+Requires-Dist: crypt4gh<2,>=1.7
+Requires-Dist: grz-common
+Requires-Dist: grz-db
+Requires-Dist: grz-pydantic-models>=1.3.0
+Requires-Dist: jsonschema<5,>=4.23.0
+Requires-Dist: platformdirs<5,>=4.3.6
+Requires-Dist: pydantic-settings<2.10,>=2.9.0
+Requires-Dist: pydantic<2.10,>=2.9.2
+Requires-Dist: pysam==0.23.*
+Requires-Dist: pyyaml<7,>=6.0.2
+Requires-Dist: requests<3,>=2.32.3
+Requires-Dist: rich==13.*
+Requires-Dist: tqdm<5,>=4.66.5
+Description-Content-Type: text/markdown
+
+GRZ internal tooling.

grzctl-0.1.0/README.md

@@ -0,0 +1 @@
+GRZ internal tooling.

grzctl-0.1.0/pyproject.toml

@@ -0,0 +1,59 @@
+[project]
+name = "grzctl"
+description = "Control CLI for GRZ administrators."
+readme = "README.md"
+license = "MIT"
+authors = [
+    { name = "Koray Kirli", email = "koraykirli@gmail.com" },
+    { name = "Mathias Lesche", email = "mathias.lesche@tu-dresden.de" },
+    { name = "Florian R. Hölzlwimmer", email = "git.ich@frhoelzlwimmer.de" },
+    { name = "Till Hartmann", email = "till.hartmann@bih-charite.de" },
+    { name = "Thomas Sell", email = "thomas.sell@bih-charite.de" },
+    { name = "Travis Wrightsman", email = "travis.wrightsman@uni-tuebingen.de" }
+]
+dynamic = ["version"]
+dependencies = [
+    "boto3 >=1.36,<2",
+    "click >=8.1.7,<9",
+    "crypt4gh >=1.7,<2",
+    "jsonschema >=4.23.0,<5",
+    "pyyaml >=6.0.2,<7",
+    "tqdm >=4.66.5,<5",
+    "pydantic >=2.9.2,<2.10",
+    "pydantic-settings >=2.9.0,<2.10",
+    "platformdirs >=4.3.6,<5",
+    "grz-pydantic-models >=1.3.0",
+    "pysam ==0.23.*",
+    "rich ==13.*",
+    "requests >=2.32.3,<3",
+    "grz-db",
+    "grz-common",
+]
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Operating System :: OS Independent"
+]
+requires-python = ">=3.12"
+keywords = ["GRZ", "GDC", "S3"]
+
+[tool.uv.sources]
+grz-common = { workspace = true }
+
+[project.urls]
+Homepage = "https://github.com/BfArM-MVH/grz-tools"
+Repository = "https://github.com/BfArM-MVH/grz-tools"
+Documentation = "https://github.com/BfArM-MVH/grz-tools/tree/main/packages/grzctl"
+Issues = "https://github.com/BfArM-MVH/grz-tools/issues"
+
+[project.scripts]
+grzctl = "grzctl.cli:main"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.version]
+path = "src/grzctl/__init__.py"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/grzctl"]

grzctl-0.1.0/src/grzctl/__init__.py

@@ -0,0 +1,5 @@
+"""
+GRZ Control CLI for GRZ administrators.
+"""
+
+__version__ = "0.1.0"

grzctl-0.1.0/src/grzctl/cli.py

@@ -0,0 +1,113 @@
+"""
+CLI module for handling command-line interface operations for GRZ administrators.
+"""
+
+PACKAGE_ROOT = "grzctl"
+
+import logging
+import logging.config
+from importlib.metadata import version
+
+import click
+import grz_pydantic_models.submission.metadata
+from grz_cli.commands.encrypt import encrypt
+from grz_cli.commands.submit import submit
+from grz_cli.commands.upload import upload
+from grz_cli.commands.validate import validate
+from grz_common.logging_setup import add_filelogger
+
+from .commands.archive import archive
+from .commands.clean import clean
+from .commands.consent import consent
+from .commands.db import db
+from .commands.decrypt import decrypt
+from .commands.download import download
+from .commands.list_submissions import list_submissions
+from .commands.pruefbericht import pruefbericht
+
+log = logging.getLogger(PACKAGE_ROOT + ".cli")
+
+
+class OrderedGroup(click.Group):
+    """
+    A click Group that keeps track of the order in which commands are added.
+    """
+
+    def list_commands(self, ctx):
+        """Return the list of commands in the order they were added."""
+        return list(self.commands.keys())
+
+
+def build_cli():
+    """
+    Factory for building the CLI application.
+    """
+
+    @click.group(
+        cls=OrderedGroup,
+        help="GRZ Control CLI for GRZ administrators.",
+    )
+    @click.version_option(
+        version=version("grzctl"),
+        prog_name="grzctl",
+        message=f"%(prog)s v%(version)s (metadata schema versions: {', '.join(grz_pydantic_models.submission.metadata.get_supported_versions())})",
+    )
+    @click.option("--log-file", metavar="FILE", type=str, help="Path to log file")
+    @click.option(
+        "--log-level",
+        default="INFO",
+        type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]),
+        help="Set the log level (default: INFO)",
+    )
+    def cli(log_file: str | None = None, log_level: str = "INFO"):
+        """
+        Command-line interface function for setting up logging.
+
+        :param log_file: Path to the log file. If provided, a file logger will be added.
+        :param log_level: Log level for the logger. It should be one of the following:
+                           DEBUG, INFO, WARNING, ERROR, CRITICAL.
+        """
+        if log_file:
+            add_filelogger(
+                log_file,
+                log_level.upper(),
+            )  # Add file logger
+
+        # show only time and log level in STDOUT
+        logging.basicConfig(
+            format="%(asctime)s - %(levelname)s - %(message)s",
+        )
+
+        # set the log level for this package
+        logging.getLogger(PACKAGE_ROOT).setLevel(log_level.upper())
+
+        log.debug("Logging setup complete.")
+
+    # For convenience, include grz-cli commands as well.
+    cli.add_command(validate)
+    cli.add_command(encrypt)
+    cli.add_command(upload)
+    cli.add_command(submit)
+
+    cli.add_command(list_submissions, name="list")
+    cli.add_command(download)
+    cli.add_command(decrypt)
+    cli.add_command(archive)
+    cli.add_command(clean)
+    cli.add_command(consent)
+    cli.add_command(pruefbericht)
+    cli.add_command(db)
+
+    return cli
+
+
+def main():
+    """
+    Main entry point for the CLI application.
+    """
+    cli = build_cli()
+    cli()
+
+
+if __name__ == "__main__":
+    main()

grzctl-0.1.0/src/grzctl/commands/__init__.py

@@ -0,0 +1,3 @@
+"""
+Command modules for the grzctl package.
+"""

grzctl-0.1.0/src/grzctl/commands/archive.py

@@ -0,0 +1,43 @@
+"""Command for archiving a submission."""
+
+import logging
+from pathlib import Path
+
+import click
+from grz_common.workers.worker import Worker
+
+from ..models.config import ArchiveConfig
+
+log = logging.getLogger(__name__)
+
+from grz_common.cli import config_file, submission_dir, threads
+
+
+@click.command()
+@submission_dir
+@config_file
+@threads
+def archive(
+    submission_dir,
+    config_file,
+    threads,
+):
+    """
+    Archive a submission within a GRZ/GDC.
+    """
+    config = ArchiveConfig.from_path(config_file)
+
+    log.info("Starting archival...")
+
+    submission_dir = Path(submission_dir)
+
+    worker_inst = Worker(
+        metadata_dir=submission_dir / "metadata",
+        files_dir=submission_dir / "files",
+        log_dir=submission_dir / "logs",
+        encrypted_files_dir=submission_dir / "encrypted_files",
+        threads=threads,
+    )
+    worker_inst.archive(config.s3)
+
+    log.info("Archival finished!")

grzctl-0.1.0/src/grzctl/commands/clean.py

@@ -0,0 +1,77 @@
+"""Command for cleaning a submission from the S3 inbox."""
+
+import logging
+import sys
+
+import click
+from grz_common.cli import config_file, submission_id
+from grz_common.transfer import init_s3_resource
+
+from ..models.config import CleanConfig
+
+log = logging.getLogger(__name__)
+
+
+@click.command()
+@submission_id
+@config_file
+@click.option("--yes-i-really-mean-it", is_flag=True)
+def clean(submission_id, config_file, yes_i_really_mean_it: bool):
+    """
+    Remove all files of a submission from the S3 inbox.
+    """
+    config = CleanConfig.from_path(config_file)
+    bucket_name = config.s3.bucket
+
+    if yes_i_really_mean_it or click.confirm(
+        f"Are you SURE you want to delete the submission from the inbox bucket ({bucket_name})?",
+        default=False,
+        show_default=True,
+    ):
+        prefix = submission_id
+        prefix = prefix + "/" if not prefix.endswith("/") else prefix
+
+        resource = init_s3_resource(config.s3)
+        bucket = resource.Bucket(bucket_name)
+        log.info(f"Deleting {prefix} from {bucket_name} …")
+
+        responses = bucket.objects.filter(Prefix=prefix).delete()
+        if not responses:
+            sys.exit(f"No objects with prefix '{prefix}' in bucket '{bucket_name}' found for deletion.")
+
+        successfully_deleted_keys = []
+        errors_encountered = []
+        objects_found = 0
+
+        # responses is a list of dicts reporting the result of the deletion API call
+        # because the API does things in batches
+        for response in responses:
+            deleted_batch = response.get("Deleted", [])
+            for deleted_obj in deleted_batch:
+                key = deleted_obj.get("Key")
+                if key:
+                    successfully_deleted_keys.append(key)
+
+            errors_batch = response.get("Errors", [])
+            for error_obj in errors_batch:
+                errors_encountered.append(
+                    {
+                        "Key": error_obj.get("Key", "N/A"),
+                        "Code": error_obj.get("Code", "N/A"),
+                        "Message": error_obj.get("Message", "N/A"),
+                    }
+                )
+
+            objects_found += len(deleted_batch) + len(errors_batch)
+
+        log.info(f"Total objects attempted to delete: {objects_found}")
+        log.info(f"Successfully deleted: {len(successfully_deleted_keys)} objects.")
+        log.info(f"Failed to delete: {len(errors_encountered)} objects.")
+
+        for error in errors_encountered:
+            log.error(f"  - Key: {error['Key']}, Code: {error['Code']}, Message: {error['Message']}")
+
+        if errors_encountered:
+            sys.exit(f"Errors encountered while deleting objects from bucket '{bucket_name}'. See log for details.")
+
+        log.info(f"Deleted '{prefix}' from '{bucket_name}'.")

grzctl-0.1.0/src/grzctl/commands/consent.py

@@ -0,0 +1,116 @@
+"""Command for determining whether a submission is consented for research."""
+
+import enum
+import json
+import logging
+import sys
+import typing
+from pathlib import Path
+from typing import Any
+
+import click
+import rich.console
+import rich.table
+import rich.text
+from grz_common.cli import output_json, show_details, submission_dir
+from grz_common.workers.submission import GrzSubmissionMetadata, SubmissionMetadata
+
+log = logging.getLogger(__name__)
+
+MDAT_WISSENSCHAFTLICH_NUTZEN_EU_DSGVO_NIVEAU = "2.16.840.1.113883.3.1937.777.24.5.3.8"
+
+
+class FhirProvision(enum.StrEnum):
+    """Possible FHIR Provision options."""
+
+    PERMIT = "permit"
+    DENY = "deny"
+
+
+@click.command()
+@submission_dir
+@output_json
+@show_details
+def consent(submission_dir, output_json, show_details):
+    """
+    Check if a submission is consented for research.
+
+    Returns 'true' if consented, 'false' if not.
+    A submission is considered consented if all donors have consented for research, that is
+    the FHIR MII IG Consent profiles all have a "permit" provision for code 2.16.840.1.113883.3.1937.777.24.5.3.8
+    """
+    metadata = SubmissionMetadata(Path(submission_dir) / "metadata" / "metadata.json").content
+
+    consents = _gather_consent_information(metadata)
+    overall_consent = _submission_has_research_consent(consents)
+
+    match output_json, show_details:
+        case True, True:
+            json.dump(consents, sys.stdout)
+        case True, False:
+            json.dump(overall_consent, sys.stdout)
+        case False, True:
+            _print_rich_table(consents)
+        case False, False:
+            click.echo(str(overall_consent).lower())
+
+
+def _submission_has_research_consent(consents):
+    return all(consents.values())
+
+
+def _print_rich_table(consents: dict[str, bool]):
+    console = rich.console.Console()
+    table = rich.table.Table()
+    table.add_column("Donor", no_wrap=True)
+    table.add_column("Research Consent", no_wrap=True)
+    for donor_pseudonym, consent_value in consents.items():
+        research_consent = rich.text.Text(
+            "True" if consent_value else "False",
+            style="green" if consent_value else "red",
+        )
+        table.add_row(
+            donor_pseudonym,
+            research_consent,
+        )
+    console.print(table)
+
+
+def _gather_consent_information(metadata: GrzSubmissionMetadata) -> dict[str, bool]:
+    consents = {donor.donor_pseudonym: False for donor in metadata.donors}
+    for donor in metadata.donors:
+        for research_consent in donor.research_consents:
+            mii_consent = research_consent.scope
+            if isinstance(mii_consent, str):
+                mii_consent = json.loads(mii_consent)
+            mii_consent = typing.cast(dict[str, Any], mii_consent)
+
+            if top_level_provision := mii_consent.get("provision"):
+                if top_level_provision.get("type") != FhirProvision.DENY:
+                    sys.exit(
+                        f"The root provision type must be deny, not {top_level_provision.get('type')}, "
+                        f"since the profile follows an opt-in consent scheme. "
+                        f"Explicit opt-in consents must be made via nested provisions."
+                    )
+                else:
+                    nested_provisions = top_level_provision.get("provision")
+                    consents[donor.donor_pseudonym] = _check_nested_provisions(nested_provisions)
+
+    return consents
+
+
+def _check_nested_provisions(provisions: list[dict[str, Any]]) -> bool:
+    for provision in provisions:
+        if provision.get("type") == FhirProvision.PERMIT:
+            for codeable_concept in provision.get("code", []):
+                for coding in codeable_concept.get("coding", []):
+                    code = coding.get("code")
+                    if isinstance(code, str):
+                        if code == MDAT_WISSENSCHAFTLICH_NUTZEN_EU_DSGVO_NIVEAU:
+                            return True
+                    elif isinstance(code, dict):
+                        if (value := code.get("value")) and value == MDAT_WISSENSCHAFTLICH_NUTZEN_EU_DSGVO_NIVEAU:
+                            return True
+                    else:
+                        raise ValueError(code, f"Expected str or dict, got {type(code)}")
+    return False