grz-cli 0.5.0__tar.gz → 0.6.0__tar.gz

{grz_cli-0.5.0/src/grz_cli.egg-info → grz_cli-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: grz-cli
-Version: 0.5.0
+Version: 0.6.0
 Summary: Tool for validation, encryption and upload of MV submissions to GDCs.
 Author-email: Koray Kirli <koraykirli@gmail.com>, Mathias Lesche <mathias.lesche@tu-dresden.de>, "Florian R. Hölzlwimmer" <git.ich@frhoelzlwimmer.de>, Till Hartmann <till.hartmann@bih-charite.de>, Thomas Sell <thomas.sell@bih-charite.de>
 License-Expression: MIT
@@ -18,10 +18,12 @@ Requires-Dist: jsonschema<5,>=4.23.0
 Requires-Dist: pyyaml<7,>=6.0.2
 Requires-Dist: tqdm<5,>=4.66.5
 Requires-Dist: pydantic<2.10,>=2.9.2
+Requires-Dist: pydantic-settings<2.10,>=2.9.0
 Requires-Dist: platformdirs<5,>=4.3.6
-Requires-Dist: grz-pydantic-models>=1.2.1
+Requires-Dist: grz-pydantic-models>=1.3.0
 Requires-Dist: pysam==0.23.*
 Requires-Dist: rich==13.*
+Requires-Dist: requests<3,>=2.32.3
 Provides-Extra: test
 Requires-Dist: pytest<9,>=8.3.3; extra == "test"
 Requires-Dist: pytest-cov<6,>=5.0.0; extra == "test"
@@ -30,6 +32,7 @@ Requires-Dist: moto[s3]<6,>=5.0.16; extra == "test"
 Requires-Dist: pytest-sugar<2,>=1.0.0; extra == "test"
 Requires-Dist: ruff<1,>=0.9.6; extra == "test"
 Requires-Dist: numpy<3,>=2.1.2; extra == "test"
+Requires-Dist: responses==0.25.*; extra == "test"
 Dynamic: license-file
 
 # GRZ CLI
@@ -140,6 +143,10 @@ pip upgrade grz-cli
 ```
 
 
+#### Docker
+Docker images are available via biocontainers at [https://biocontainers.pro/tools/grz-cli](https://biocontainers.pro/tools/grz-cli).
+
+
 ### Development setup
 For development purposes, you can clone the repository and install the package in editable mode:
 

{grz_cli-0.5.0 → grz_cli-0.6.0}/README.md

@@ -106,6 +106,10 @@ pip upgrade grz-cli
 ```
 
 
+#### Docker
+Docker images are available via biocontainers at [https://biocontainers.pro/tools/grz-cli](https://biocontainers.pro/tools/grz-cli).
+
+
 ### Development setup
 For development purposes, you can clone the repository and install the package in editable mode:
 

{grz_cli-0.5.0 → grz_cli-0.6.0}/pyproject.toml

@@ -23,10 +23,12 @@ dependencies = [
     "pyyaml >=6.0.2,<7",
     "tqdm >=4.66.5,<5",
     "pydantic >=2.9.2,<2.10",
+    "pydantic-settings >=2.9.0,<2.10",
     "platformdirs >=4.3.6,<5",
-    "grz-pydantic-models >=1.2.1",
+    "grz-pydantic-models >=1.3.0",
     "pysam ==0.23.*",
-    "rich ==13.*"
+    "rich ==13.*",
+    "requests >=2.32.3,<3"
 ]
 classifiers = [
     "Programming Language :: Python :: 3",
@@ -53,7 +55,8 @@ test = [
     "moto[s3] >=5.0.16,<6",
     "pytest-sugar >=1.0.0,<2",
     "ruff >=0.9.6,<1",
-    "numpy >=2.1.2,<3"
+    "numpy >=2.1.2,<3",
+    "responses ==0.25.*"
 ]
 
 [dependency-groups]
@@ -67,10 +70,10 @@ test = [
 ]
 lint = [
     "ruff",
-    "mypy",
     "types-pyyaml",
     "types-tqdm",
     "pydantic",
+    "pyrefly",
 ]
 
 [tool.ruff]
@@ -155,7 +158,7 @@ convention = "pep257"
 
 [tool.tox]
 requires = ["tox>=4.23"]
-env_list = ["format-check", "lints", "mypy", "3.12", "3.13"]
+env_list = ["format-check", "lints", "pyrefly", "3.12", "3.13"]
 allowlist_externals = ["pytest", "ruff"]
 isolated_build = true
 
@@ -177,16 +180,16 @@ skip_install = true
 dependency_groups = ["lint"]
 commands = [["ruff", "check"]]
 
-[tool.tox.env."mypy"]
+[tool.tox.env."pyrefly"]
 runner = "uv-venv-lock-runner"
 skip_install = false
 dependency_groups = ["lint"]
-commands = [["mypy"]]
+commands = [["pyrefly", "check"]]
 
 [tool.uv]
 dev-dependencies = [
     "datamodel-code-generator>=0.26.2",
-    "mypy>=1.13.0",
+    "pyrefly>=0.16.0,<0.17.0",
     "ruff>=0.9.6",
     "tox>=4.23.2",
     "tox-uv>=1.20.1",
@@ -194,14 +197,11 @@ dev-dependencies = [
     "types-tqdm>=4.66.0.20240417",
 ]
 
-[tool.mypy]
-python_version = "3.12"
-plugins = "pydantic.mypy"
-packages = "src"
-
-[[tool.mypy.overrides]]
-module = ["grz_pydantic_models.*"]
-follow_untyped_imports = true
+[tool.pyrefly]
+project_includes = ["src"]
+python_version = "3.12.0"
+use_untyped_imports = false
+ignore_missing_source = false
 
 [tool.pytest.ini_options]
 addopts = [

grz_cli-0.6.0/src/grz_cli/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.6.0"

grz_cli-0.6.0/src/grz_cli/cli.py

@@ -0,0 +1,113 @@
+"""
+CLI module for handling command-line interface operations.
+"""
+
+import logging
+import logging.config
+import os
+from importlib.metadata import version
+
+import click
+import grz_pydantic_models.submission.metadata
+
+from .commands.archive import archive
+from .commands.clean import clean
+from .commands.consent import consent
+from .commands.decrypt import decrypt
+from .commands.download import download
+from .commands.encrypt import encrypt
+from .commands.list_submissions import list_submissions
+from .commands.pruefbericht import pruefbericht
+from .commands.submit import submit
+from .commands.upload import upload
+from .commands.validate import validate
+from .constants import PACKAGE_ROOT
+from .logging_setup import add_filelogger
+
+log = logging.getLogger(PACKAGE_ROOT + ".cli")
+
+
+class OrderedGroup(click.Group):
+    """
+    A click Group that keeps track of the order in which commands are added.
+    """
+
+    def list_commands(self, ctx):
+        """Return the list of commands in the order they were added."""
+        return list(self.commands.keys())
+
+
+def build_cli(grz_mode=False):
+    """
+    Factory for building the CLI application.
+    """
+
+    @click.group(
+        cls=OrderedGroup,
+        help="Validate, encrypt, decrypt and upload submissions to a GRZ/GDC.",
+    )
+    @click.version_option(
+        version=version("grz-cli"),
+        prog_name="grz-cli",
+        message=f"%(prog)s v%(version)s (metadata schema versions: {', '.join(grz_pydantic_models.submission.metadata.get_supported_versions())})",
+    )
+    @click.option("--log-file", metavar="FILE", type=str, help="Path to log file")
+    @click.option(
+        "--log-level",
+        default="INFO",
+        type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]),
+        help="Set the log level (default: INFO)",
+    )
+    def cli(log_file: str | None = None, log_level: str = "INFO"):
+        """
+        Command-line interface function for setting up logging.
+
+        :param log_file: Path to the log file. If provided, a file logger will be added.
+        :param log_level: Log level for the logger. It should be one of the following:
+                           DEBUG, INFO, WARNING, ERROR, CRITICAL.
+        """
+        if log_file:
+            add_filelogger(
+                log_file,
+                log_level.upper(),
+            )  # Add file logger
+
+        # show only time and log level in STDOUT
+        logging.basicConfig(
+            format="%(asctime)s - %(levelname)s - %(message)s",
+        )
+
+        # set the log level for this package
+        logging.getLogger(PACKAGE_ROOT).setLevel(log_level.upper())
+
+        log.debug("Logging setup complete.")
+
+    cli.add_command(validate)
+    cli.add_command(encrypt)
+    cli.add_command(upload)
+    cli.add_command(submit)
+
+    if grz_mode:
+        cli.add_command(list_submissions, name="list")
+        cli.add_command(download)
+        cli.add_command(decrypt)
+        cli.add_command(archive)
+        cli.add_command(clean)
+        cli.add_command(consent)
+        cli.add_command(pruefbericht)
+
+    return cli
+
+
+def main():
+    """
+    Main entry point for the CLI application.
+    """
+    grz_mode = os.environ.get("GRZ_MODE", "false").lower() in {"true", "yes", "1"}
+
+    cli = build_cli(grz_mode=grz_mode)
+    cli()
+
+
+if __name__ == "__main__":
+    main()

grz_cli-0.6.0/src/grz_cli/commands/__init__.py

@@ -0,0 +1 @@
+"""Commands package for grz-cli."""

grz_cli-0.6.0/src/grz_cli/commands/archive.py

@@ -0,0 +1,43 @@
+"""Command for archiving a submission."""
+
+import logging
+from pathlib import Path
+
+import click
+
+from ..workers.worker import Worker
+
+log = logging.getLogger(__name__)
+
+from ..utils.config import read_config
+from .common import config_file, submission_dir, threads
+
+
+@click.command()
+@submission_dir
+@config_file
+@threads
+def archive(
+    submission_dir,
+    config_file,
+    threads,
+):
+    """
+    Archive a submission within a GRZ/GDC.
+    """
+    config = read_config(config_file)
+
+    log.info("Starting archival...")
+
+    submission_dir = Path(submission_dir)
+
+    worker_inst = Worker(
+        metadata_dir=submission_dir / "metadata",
+        files_dir=submission_dir / "files",
+        log_dir=submission_dir / "logs",
+        encrypted_files_dir=submission_dir / "encrypted_files",
+        threads=threads,
+    )
+    worker_inst.archive(config)
+
+    log.info("Archival finished!")

grz_cli-0.6.0/src/grz_cli/commands/clean.py

@@ -0,0 +1,77 @@
+"""Command for cleaning a submission from the S3 inbox."""
+
+import logging
+import sys
+
+import click
+
+from ..transfer import init_s3_resource
+from ..utils.config import read_config
+from .common import config_file, submission_id
+
+log = logging.getLogger(__name__)
+
+
+@click.command()
+@submission_id
+@config_file
+@click.option("--yes-i-really-mean-it", is_flag=True)
+def clean(submission_id, config_file, yes_i_really_mean_it: bool):
+    """
+    Remove all files of a submission from the S3 inbox.
+    """
+    config = read_config(config_file)
+    bucket_name = config.s3_options.bucket
+
+    if yes_i_really_mean_it or click.confirm(
+        f"Are you SURE you want to delete the submission from the inbox bucket ({bucket_name})?",
+        default=False,
+        show_default=True,
+    ):
+        prefix = submission_id
+        prefix = prefix + "/" if not prefix.endswith("/") else prefix
+
+        resource = init_s3_resource(config)
+        bucket = resource.Bucket(bucket_name)
+        log.info(f"Deleting {prefix} from {bucket_name} …")
+
+        responses = bucket.objects.filter(Prefix=prefix).delete()
+        if not responses:
+            sys.exit(f"No objects with prefix '{prefix}' in bucket '{bucket_name}' found for deletion.")
+
+        successfully_deleted_keys = []
+        errors_encountered = []
+        objects_found = 0
+
+        # responses is a list of dicts reporting the result of the deletion API call
+        # because the API does things in batches
+        for response in responses:
+            deleted_batch = response.get("Deleted", [])
+            for deleted_obj in deleted_batch:
+                key = deleted_obj.get("Key")
+                if key:
+                    successfully_deleted_keys.append(key)
+
+            errors_batch = response.get("Errors", [])
+            for error_obj in errors_batch:
+                errors_encountered.append(
+                    {
+                        "Key": error_obj.get("Key", "N/A"),
+                        "Code": error_obj.get("Code", "N/A"),
+                        "Message": error_obj.get("Message", "N/A"),
+                    }
+                )
+
+            objects_found += len(deleted_batch) + len(errors_batch)
+
+        log.info(f"Total objects attempted to delete: {objects_found}")
+        log.info(f"Successfully deleted: {len(successfully_deleted_keys)} objects.")
+        log.info(f"Failed to delete: {len(errors_encountered)} objects.")
+
+        for error in errors_encountered:
+            log.error(f"  - Key: {error['Key']}, Code: {error['Code']}, Message: {error['Message']}")
+
+        if errors_encountered:
+            sys.exit(f"Errors encountered while deleting objects from bucket '{bucket_name}'. See log for details.")
+
+        log.info(f"Deleted '{prefix}' from '{bucket_name}'.")

grz_cli-0.6.0/src/grz_cli/commands/common.py

@@ -0,0 +1,79 @@
+"""
+Common click options for the CLI commands.
+"""
+
+from os import sched_getaffinity
+from pathlib import Path
+
+import click
+import platformdirs
+
+DEFAULT_CONFIG_PATH = Path(platformdirs.user_config_dir("grz-cli")) / "config.yaml"
+
+# Aliases for path types for click options
+# Naming convention: {DIR,FILE}_{Read,Write}_{Exists,Create}
+DIR_R_E = click.Path(
+    exists=True,
+    file_okay=False,
+    dir_okay=True,
+    readable=True,
+    writable=False,
+    resolve_path=True,
+)
+DIR_RW_C = click.Path(
+    exists=False,
+    file_okay=False,
+    dir_okay=True,
+    readable=True,
+    writable=True,
+    resolve_path=True,
+)
+FILE_R_E = click.Path(exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True)
+
+submission_dir = click.option(
+    "--submission-dir",
+    metavar="PATH",
+    type=DIR_R_E,
+    required=True,
+    help="Path to the submission directory containing 'metadata/', 'files/', 'encrypted_files/' and 'logs/' directories",
+)
+
+config_file = click.option(
+    "--config-file",
+    metavar="STRING",
+    type=FILE_R_E,
+    required=False,
+    default=DEFAULT_CONFIG_PATH,
+    help="Path to config file",
+)
+
+threads = click.option(
+    "--threads",
+    default=min(len(sched_getaffinity(0)), 4),
+    type=int,
+    show_default=True,
+    help="Number of threads to use for parallel operations",
+)
+
+submission_id = click.option(
+    "--submission-id",
+    required=True,
+    type=str,
+    metavar="STRING",
+    help="S3 submission ID",
+)
+
+output_dir = click.option(
+    "--output-dir",
+    metavar="PATH",
+    type=DIR_RW_C,
+    required=True,
+    default=None,
+    help="Path to the target submission output directory",
+)
+
+output_json = click.option("--json", "output_json", is_flag=True, help="Output JSON for machine-readability.")
+
+show_details = click.option("--details", "show_details", is_flag=True, help="Show more detailed output.")
+
+force = click.option("--force/--no-force", help="Overwrite files and ignore cached results (dangerous!)")

grz_cli-0.6.0/src/grz_cli/commands/consent.py

@@ -0,0 +1,115 @@
+"""Command for determining whether a submission is consented for research."""
+
+import enum
+import json
+import logging
+import sys
+from pathlib import Path
+from typing import Any
+
+import click
+import rich.console
+import rich.table
+import rich.text
+
+from ..workers.submission import GrzSubmissionMetadata, SubmissionMetadata
+from .common import output_json, show_details, submission_dir
+
+log = logging.getLogger(__name__)
+
+MDAT_WISSENSCHAFTLICH_NUTZEN_EU_DSGVO_NIVEAU = "2.16.840.1.113883.3.1937.777.24.5.3.8"
+
+
+class FhirProvision(enum.StrEnum):
+    """Possible FHIR Provision options."""
+
+    PERMIT = "permit"
+    DENY = "deny"
+
+
+@click.command()
+@submission_dir
+@output_json
+@show_details
+def consent(submission_dir, output_json, show_details):
+    """
+    Check if a submission is consented for research.
+
+    Returns 'true' if consented, 'false' if not.
+    A submission is considered consented if all donors have consented for research, that is
+    the FHIR MII IG Consent profiles all have a "permit" provision for code 2.16.840.1.113883.3.1937.777.24.5.3.8
+    """
+    metadata = SubmissionMetadata(Path(submission_dir) / "metadata" / "metadata.json").content
+
+    consents = _gather_consent_information(metadata)
+    overall_consent = _submission_has_research_consent(consents)
+
+    match output_json, show_details:
+        case True, True:
+            json.dump(consents, sys.stdout)
+        case True, False:
+            json.dump(overall_consent, sys.stdout)
+        case False, True:
+            _print_rich_table(consents)
+        case False, False:
+            click.echo(str(overall_consent).lower())
+
+
+def _submission_has_research_consent(consents):
+    return all(consents.values())
+
+
+def _print_rich_table(consents: dict[str, bool]):
+    console = rich.console.Console()
+    table = rich.table.Table()
+    table.add_column("Donor", no_wrap=True)
+    table.add_column("Research Consent", no_wrap=True)
+    for donor_pseudonym, consent_value in consents.items():
+        research_consent = rich.text.Text(
+            "True" if consent_value else "False",
+            style="green" if consent_value else "red",
+        )
+        table.add_row(
+            donor_pseudonym,
+            research_consent,
+        )
+    console.print(table)
+
+
+def _gather_consent_information(metadata: GrzSubmissionMetadata) -> dict[str, bool]:
+    consents = {donor.donor_pseudonym: False for donor in metadata.donors}
+    for donor in metadata.donors:
+        for research_consent in donor.research_consents:
+            mii_consent = research_consent.scope
+            if isinstance(mii_consent, str):
+                mii_consent = json.loads(mii_consent)
+
+            if top_level_provision := mii_consent.get("provision"):
+                if top_level_provision.get("type") != FhirProvision.DENY:
+                    sys.exit(
+                        f"The root provision type must be deny, not {top_level_provision.get('type')}, "
+                        f"since the profile follows an opt-in consent scheme. "
+                        f"Explicit opt-in consents must be made via nested provisions."
+                    )
+                else:
+                    nested_provisions = top_level_provision.get("provision")
+                    consents[donor.donor_pseudonym] = _check_nested_provisions(nested_provisions)
+
+    return consents
+
+
+def _check_nested_provisions(provisions: list[dict[str, Any]]) -> bool:
+    for provision in provisions:
+        if provision.get("type") == FhirProvision.PERMIT:
+            for codeable_concept in provision.get("code", []):
+                for coding in codeable_concept.get("coding", []):
+                    code = coding.get("code")
+                    if isinstance(code, str):
+                        if code == MDAT_WISSENSCHAFTLICH_NUTZEN_EU_DSGVO_NIVEAU:
+                            return True
+                    elif isinstance(code, dict):
+                        if (value := code.get("value")) and value == MDAT_WISSENSCHAFTLICH_NUTZEN_EU_DSGVO_NIVEAU:
+                            return True
+                    else:
+                        raise ValueError(code, f"Expected str or dict, got {type(code)}")
+    return False

grz_cli-0.6.0/src/grz_cli/commands/decrypt.py

@@ -0,0 +1,45 @@
+"""Command for decrypting a submission."""
+
+import logging
+import sys
+from pathlib import Path
+
+import click
+
+from ..utils.config import read_config
+from ..workers.worker import Worker
+from .common import config_file, force, submission_dir
+
+log = logging.getLogger(__name__)
+
+
+@click.command()
+@submission_dir
+@config_file
+@force
+def decrypt(submission_dir, config_file, force):
+    """
+    Decrypt a submission.
+
+    Decrypting a submission requires the _private_ key of the original recipient.
+    """
+    config = read_config(config_file)
+
+    grz_privkey_path = config.grz_private_key_path
+    if not grz_privkey_path:
+        log.error("GRZ private key path is required for decryption.")
+        sys.exit(1)
+
+    log.info("Starting decryption...")
+
+    submission_dir = Path(submission_dir)
+
+    worker_inst = Worker(
+        metadata_dir=submission_dir / "metadata",
+        files_dir=submission_dir / "files",
+        log_dir=submission_dir / "logs",
+        encrypted_files_dir=submission_dir / "encrypted_files",
+    )
+    worker_inst.decrypt(grz_privkey_path, force=force)
+
+    log.info("Decryption successful!")

grz_cli-0.6.0/src/grz_cli/commands/download.py

@@ -0,0 +1,46 @@
+"""Command for downloading a submission."""
+
+import logging
+from pathlib import Path
+
+import click
+
+from ..utils.config import read_config
+from ..workers.worker import Worker
+from .common import config_file, force, output_dir, submission_id, threads
+
+log = logging.getLogger(__name__)
+
+
+@click.command()
+@submission_id
+@output_dir
+@config_file
+@threads
+@force
+def download(submission_id, output_dir, config_file, threads, force):
+    """
+    Download a submission from a GRZ.
+
+    Downloaded metadata is stored within the `metadata` sub-folder of the submission output directory.
+    Downloaded files are stored within the `encrypted_files` sub-folder of the submission output directory.
+    """
+    config = read_config(config_file)
+
+    log.info("Starting download...")
+
+    submission_dir_path = Path(output_dir)
+    if not submission_dir_path.is_dir():
+        log.debug("Creating submission directory %s", submission_dir_path)
+        submission_dir_path.mkdir(mode=0o770, parents=False, exist_ok=False)
+
+    worker_inst = Worker(
+        metadata_dir=submission_dir_path / "metadata",
+        files_dir=submission_dir_path / "files",
+        log_dir=submission_dir_path / "logs",
+        encrypted_files_dir=submission_dir_path / "encrypted_files",
+        threads=threads,
+    )
+    worker_inst.download(config, submission_id, force=force)
+
+    log.info("Download finished!")