grz-cli 0.3.0__tar.gz → 0.5.0__tar.gz

grz_cli-0.5.0/MANIFEST.in

@@ -0,0 +1,2 @@
+graft src
+include LICENSE

{grz_cli-0.3.0/src/grz_cli.egg-info → grz_cli-0.5.0}/PKG-INFO

@@ -1,17 +1,17 @@
 Metadata-Version: 2.4
 Name: grz-cli
-Version: 0.3.0
+Version: 0.5.0
 Summary: Tool for validation, encryption and upload of MV submissions to GDCs.
 Author-email: Koray Kirli <koraykirli@gmail.com>, Mathias Lesche <mathias.lesche@tu-dresden.de>, "Florian R. Hölzlwimmer" <git.ich@frhoelzlwimmer.de>, Till Hartmann <till.hartmann@bih-charite.de>, Thomas Sell <thomas.sell@bih-charite.de>
-License: MIT
+License-Expression: MIT
 Project-URL: Homepage, https://codebase.helmholtz.cloud/grz-mv-genomseq/grz-cli
 Keywords: GRZ,GDC,S3
 Classifier: Programming Language :: Python :: 3
-Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
-Requires-Dist: boto3<2,>=1.35.39
+License-File: LICENSE
+Requires-Dist: boto3<2,>=1.36
 Requires-Dist: click<9,>=8.1.7
 Requires-Dist: crypt4gh<2,>=1.7
 Requires-Dist: jsonschema<5,>=4.23.0
@@ -19,7 +19,9 @@ Requires-Dist: pyyaml<7,>=6.0.2
 Requires-Dist: tqdm<5,>=4.66.5
 Requires-Dist: pydantic<2.10,>=2.9.2
 Requires-Dist: platformdirs<5,>=4.3.6
-Requires-Dist: grz-pydantic-models>=1.2.0
+Requires-Dist: grz-pydantic-models>=1.2.1
+Requires-Dist: pysam==0.23.*
+Requires-Dist: rich==13.*
 Provides-Extra: test
 Requires-Dist: pytest<9,>=8.3.3; extra == "test"
 Requires-Dist: pytest-cov<6,>=5.0.0; extra == "test"
@@ -28,6 +30,7 @@ Requires-Dist: moto[s3]<6,>=5.0.16; extra == "test"
 Requires-Dist: pytest-sugar<2,>=1.0.0; extra == "test"
 Requires-Dist: ruff<1,>=0.9.6; extra == "test"
 Requires-Dist: numpy<3,>=2.1.2; extra == "test"
+Dynamic: license-file
 
 # GRZ CLI
 
@@ -254,6 +257,8 @@ Option is for the usage at a GDC/GRZ.
 
 ## Testing
 
+Please note that binary files used for testing are managed with [Git LFS](https://git-lfs.com), which will be needed to clone them locally with the git repository.
+
 To run the tests, navigate to the root directory of your project and invoke `pytest`.
 Alternatively, install `uv` and `tox` and run `uv run tox`.
 

{grz_cli-0.3.0 → grz_cli-0.5.0}/README.md

@@ -223,6 +223,8 @@ Option is for the usage at a GDC/GRZ.
 
 ## Testing
 
+Please note that binary files used for testing are managed with [Git LFS](https://git-lfs.com), which will be needed to clone them locally with the git repository.
+
 To run the tests, navigate to the root directory of your project and invoke `pytest`.
 Alternatively, install `uv` and `tox` and run `uv run tox`.
 

{grz_cli-0.3.0 → grz_cli-0.5.0}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 name = "grz-cli"
 description = "Tool for validation, encryption and upload of MV submissions to GDCs."
 readme = "README.md"
-license = { text = "MIT" }
+license = "MIT"
 authors = [
     { name = "Koray Kirli", email = "koraykirli@gmail.com" },
     { name = "Mathias Lesche", email = "mathias.lesche@tu-dresden.de" },
@@ -16,7 +16,7 @@ authors = [
 ]
 dynamic = ["version"]
 dependencies = [
-    "boto3 >=1.35.39,<2",
+    "boto3 >=1.36,<2",
     "click >=8.1.7,<9",
     "crypt4gh >=1.7,<2",
     "jsonschema >=4.23.0,<5",
@@ -24,11 +24,12 @@ dependencies = [
     "tqdm >=4.66.5,<5",
     "pydantic >=2.9.2,<2.10",
     "platformdirs >=4.3.6,<5",
-    "grz-pydantic-models >=1.2.0"
+    "grz-pydantic-models >=1.2.1",
+    "pysam ==0.23.*",
+    "rich ==13.*"
 ]
 classifiers = [
     "Programming Language :: Python :: 3",
-    "License :: OSI Approved :: MIT License",
     "Operating System :: OS Independent"
 ]
 requires-python = ">=3.12"
@@ -42,7 +43,7 @@ version = { attr = "grz_cli.__version__" }
 "Homepage" = "https://codebase.helmholtz.cloud/grz-mv-genomseq/grz-cli"
 
 [project.scripts]
-grz-cli = "grz_cli.cli:cli"
+grz-cli = "grz_cli.cli:main"
 
 [project.optional-dependencies]
 test = [
@@ -206,6 +207,3 @@ follow_untyped_imports = true
 addopts = [
     "--import-mode=importlib",
 ]
-
-[tool.setuptools]
-license-files = []

grz_cli-0.5.0/src/grz_cli/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.5.0"

grz_cli-0.5.0/src/grz_cli/bam_validation.py

@@ -0,0 +1,39 @@
+"""
+Validation methods for BAM files
+
+e.g. warn on headers present, which can contain personally identifying
+information
+"""
+
+import logging
+from collections.abc import Generator
+from os import PathLike
+
+import pysam
+
+log = logging.getLogger(__name__)
+
+
+def validate_bam(bam_path: str | PathLike) -> Generator[str]:
+    """Check if a BAM file is valid for submission"""
+    errors: list[str] = []
+
+    # disable pysam's SQ header presence enforcement (check_sq)
+    #  these are used to optionally sort alignments by a custom contig order,
+    #  which is unnecessary in unmapped BAM files
+    bam_file = pysam.AlignmentFile(str(bam_path), mode="rb", check_sq=False)
+
+    header = bam_file.header.to_dict()
+    # ignore the HD key as it is almost always present and extremely
+    # unlikely to accidentally contain identifying information
+    concerning_keys = header.keys() - {"HD"}
+    if concerning_keys:
+        log.warning(f"Detected a header in BAM file '{bam_path}', ensure it contains no private information:")
+        for key in concerning_keys:
+            for line in header[key]:
+                log.warning(f"@{key}")
+                for tag, value in line.items():
+                    log.warning(f" {tag}: {value}")
+        log.warning("--- End of potentially-concerning BAM header information ---")
+
+    yield from errors

{grz_cli-0.3.0 → grz_cli-0.5.0}/src/grz_cli/cli.py

@@ -3,19 +3,25 @@ CLI module for handling command-line interface operations.
 """
 
 import importlib
+import json
 import logging
 import logging.config
+import os
 import sys
 from os import PathLike, sched_getaffinity
 from pathlib import Path
 from tempfile import NamedTemporaryFile
 
 import click
-import grz_pydantic_models
+import grz_pydantic_models.submission.metadata
 import platformdirs
+import rich.console
+import rich.table
 import yaml
+from pydantic_core import to_jsonable_python
 
 from .constants import PACKAGE_ROOT
+from .download import query_submissions
 from .logging_setup import add_filelogger
 from .models.config import ConfigModel
 from .parser import Worker
@@ -82,7 +88,7 @@ submission_id = click.option(
     required=True,
     type=str,
     metavar="STRING",
-    help="S3 submission prefix (corresponds to the tanG of a submission)",
+    help="S3 submission ID",
 )
 
 output_dir = click.option(
@@ -94,6 +100,16 @@ output_dir = click.option(
     help="Path to the target submission output directory",
 )
 
+output_json = click.option("--json", "output_json", is_flag=True, help="Output JSON for machine-readability.")
+
+
+def read_config(config_path: str | PathLike) -> ConfigModel:
+    """Reads the configuration file and validates it against the schema."""
+    with open(config_path, encoding="utf-8") as f:
+        config = ConfigModel(**yaml.safe_load(f))
+
+    return config
+
 
 class OrderedGroup(click.Group):
     """
@@ -105,48 +121,7 @@ class OrderedGroup(click.Group):
         return list(self.commands.keys())
 
 
-@click.group(
-    cls=OrderedGroup,
-    help="Validate, encrypt, decrypt and upload submissions to a GRZ/GDC.",
-)
-@click.version_option(
-    version=importlib.metadata.version("grz-cli"),
-    prog_name="grz-cli",
-    message=f"%(prog)s v%(version)s (metadata schema versions: {', '.join(grz_pydantic_models.get_supported_versions())})",
-)
-@click.option("--log-file", metavar="FILE", type=str, help="Path to log file")
-@click.option(
-    "--log-level",
-    default="INFO",
-    type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]),
-    help="Set the log level (default: INFO)",
-)
-def cli(log_file: str | None = None, log_level: str = "INFO"):
-    """
-    Command-line interface function for setting up logging.
-
-    :param log_file: Path to the log file. If provided, a file logger will be added.
-    :param log_level: Log level for the logger. It should be one of the following:
-                       DEBUG, INFO, WARNING, ERROR, CRITICAL.
-    """
-    if log_file:
-        add_filelogger(
-            log_file,
-            log_level.upper(),
-        )  # Add file logger
-
-    # show only time and log level in STDOUT
-    logging.basicConfig(
-        format="%(asctime)s - %(levelname)s - %(message)s",
-    )
-
-    # set the log level for this package
-    logging.getLogger(PACKAGE_ROOT).setLevel(log_level.upper())
-
-    log.debug("Logging setup complete.")
-
-
-@cli.command()
+@click.command()
 @submission_dir
 def validate(submission_dir):
     """
@@ -170,7 +145,7 @@ def validate(submission_dir):
     log.info("Validation finished!")
 
 
-@cli.command()
+@click.command()
 @submission_dir
 @config_file
 def encrypt(
@@ -216,41 +191,7 @@ def encrypt(
     log.info("Encryption successful!")
 
 
-@cli.command()
-@submission_dir
-@config_file
-def decrypt(
-    submission_dir,
-    config_file,
-):
-    """
-    Decrypt a submission.
-
-    Decrypting a submission requires the _private_ key of the original recipient.
-    """
-    config = read_config(config_file)
-
-    grz_privkey_path = config.grz_private_key_path
-    if not grz_privkey_path:
-        log.error("GRZ private key path is required for decryption.")
-        sys.exit(1)
-
-    log.info("Starting decryption...")
-
-    submission_dir = Path(submission_dir)
-
-    worker_inst = Worker(
-        metadata_dir=submission_dir / "metadata",
-        files_dir=submission_dir / "files",
-        log_dir=submission_dir / "logs",
-        encrypted_files_dir=submission_dir / "encrypted_files",
-    )
-    worker_inst.decrypt(grz_privkey_path)
-
-    log.info("Encryption successful!")
-
-
-@cli.command()
+@click.command()
 @submission_dir
 @config_file
 @threads
@@ -275,12 +216,34 @@ def upload(
         encrypted_files_dir=submission_dir / "encrypted_files",
         threads=threads,
     )
-    worker_inst.upload(config)
+    # output the generated submission ID
+    print(worker_inst.upload(config))
 
     log.info("Upload finished!")
 
 
-@cli.command()
+@click.command("submit")
+@submission_dir
+@config_file
+@threads
+@click.pass_context
+def submit(ctx, submission_dir, config_file, threads):
+    """
+    Validate, encrypt, and then upload.
+
+    This is a convenience command that performs the following steps in order:
+    1. Validate the submission
+    2. Encrypt the submission
+    3. Upload the encrypted submission
+    """
+    click.echo("Starting submission process...")
+    ctx.invoke(validate, submission_dir=submission_dir)
+    ctx.invoke(encrypt, submission_dir=submission_dir, config_file=config_file)
+    ctx.invoke(upload, submission_dir=submission_dir, config_file=config_file, threads=threads)
+    click.echo("Submission finished!")
+
+
+@click.command()
 @submission_id
 @output_dir
 @config_file
@@ -318,13 +281,140 @@ def download(
     log.info("Download finished!")
 
 
-def read_config(config_path: str | PathLike) -> ConfigModel:
-    """Reads the configuration file and validates it against the schema."""
-    with open(config_path, encoding="utf-8") as f:
-        config = ConfigModel(**yaml.safe_load(f))
+@click.command()
+@config_file
+@output_json
+def list_submissions(config_file, output_json):
+    """
+    List submissions within an inbox from oldest to newest.
+    """
+    config = read_config(config_file)
+    submissions = query_submissions(config)
 
-    return config
+    if output_json:
+        json.dump(to_jsonable_python(submissions), sys.stdout)
+    else:
+        console = rich.console.Console()
+        table = rich.table.Table()
+        table.add_column("ID", no_wrap=True)
+        table.add_column("Status", no_wrap=True)
+        table.add_column("Oldest Upload", overflow="fold")
+        table.add_column("Newest Upload", overflow="fold")
+        for submission in submissions:
+            status_text = rich.text.Text(
+                "Complete" if submission.complete else "Incomplete",
+                style="green" if submission.complete else "yellow",
+            )
+            table.add_row(
+                submission.submission_id,
+                status_text,
+                submission.oldest_upload.astimezone().strftime("%Y-%m-%d %H:%M:%S"),
+                submission.newest_upload.astimezone().strftime("%Y-%m-%d %H:%M:%S"),
+            )
+        console.print(table)
 
 
-if __name__ == "__main__":
+@click.command()
+@submission_dir
+@config_file
+def decrypt(
+    submission_dir,
+    config_file,
+):
+    """
+    Decrypt a submission.
+
+    Decrypting a submission requires the _private_ key of the original recipient.
+    """
+    config = read_config(config_file)
+
+    grz_privkey_path = config.grz_private_key_path
+    if not grz_privkey_path:
+        log.error("GRZ private key path is required for decryption.")
+        sys.exit(1)
+
+    log.info("Starting decryption...")
+
+    submission_dir = Path(submission_dir)
+
+    worker_inst = Worker(
+        metadata_dir=submission_dir / "metadata",
+        files_dir=submission_dir / "files",
+        log_dir=submission_dir / "logs",
+        encrypted_files_dir=submission_dir / "encrypted_files",
+    )
+    worker_inst.decrypt(grz_privkey_path)
+
+    log.info("Encryption successful!")
+
+
+def build_cli(grz_mode=False):
+    """
+    Factory for building the CLI application.
+    """
+
+    @click.group(
+        cls=OrderedGroup,
+        help="Validate, encrypt, decrypt and upload submissions to a GRZ/GDC.",
+    )
+    @click.version_option(
+        version=importlib.metadata.version("grz-cli"),
+        prog_name="grz-cli",
+        message=f"%(prog)s v%(version)s (metadata schema versions: {', '.join(grz_pydantic_models.submission.metadata.get_supported_versions())})",
+    )
+    @click.option("--log-file", metavar="FILE", type=str, help="Path to log file")
+    @click.option(
+        "--log-level",
+        default="INFO",
+        type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]),
+        help="Set the log level (default: INFO)",
+    )
+    def cli(log_file: str | None = None, log_level: str = "INFO"):
+        """
+        Command-line interface function for setting up logging.
+
+        :param log_file: Path to the log file. If provided, a file logger will be added.
+        :param log_level: Log level for the logger. It should be one of the following:
+                           DEBUG, INFO, WARNING, ERROR, CRITICAL.
+        """
+        if log_file:
+            add_filelogger(
+                log_file,
+                log_level.upper(),
+            )  # Add file logger
+
+        # show only time and log level in STDOUT
+        logging.basicConfig(
+            format="%(asctime)s - %(levelname)s - %(message)s",
+        )
+
+        # set the log level for this package
+        logging.getLogger(PACKAGE_ROOT).setLevel(log_level.upper())
+
+        log.debug("Logging setup complete.")
+
+    cli.add_command(validate)
+    cli.add_command(encrypt)
+    cli.add_command(upload)
+    cli.add_command(submit)
+
+    if grz_mode:
+        cli.add_command(list_submissions, name="list")
+        cli.add_command(download)
+        cli.add_command(decrypt)
+
+    return cli
+
+
+def main():
+    """
+    Main entry point for the CLI application.
+    """
+    grz_mode = os.environ.get("GRZ_MODE", "false").lower() in {"true", "yes", "1"}
+
+    cli = build_cli(grz_mode=grz_mode)
     cli()
+
+
+if __name__ == "__main__":
+    main()

{grz_cli-0.3.0 → grz_cli-0.5.0}/src/grz_cli/constants.py

@@ -18,7 +18,6 @@ LOGGING_CONFIG = {
             "level": "INFO",
             "formatter": "standard",
             "class": "logging.StreamHandler",
-            "stream": "ext://sys.stdout",  # Default is stderr
         },
     },
     "loggers": {
@@ -35,3 +34,5 @@ LOGGING_CONFIG = {
         },
     },
 }
+
+TQDM_SMOOTHING: float = 0.0001

{grz_cli-0.3.0 → grz_cli-0.5.0}/src/grz_cli/download.py

@@ -1,24 +1,27 @@
-"""Module for uploading encrypted submissions to a remote storage"""
+"""Module for downloading encrypted submissions to local storage"""
 
 from __future__ import annotations
 
+import datetime
+import itertools
 import logging
 import math
 import re
+from operator import attrgetter, itemgetter
 from os import PathLike
 from pathlib import Path
 from typing import TYPE_CHECKING
 
-import boto3  # type: ignore[import-untyped]
 import botocore.handlers  # type: ignore[import-untyped]
-from boto3 import client as boto3_client  # type: ignore[import-untyped]
 from boto3.s3.transfer import S3Transfer, TransferConfig  # type: ignore[import-untyped]
-from botocore.config import Config as Boto3Config  # type: ignore[import-untyped]
+from pydantic import BaseModel
 from tqdm.auto import tqdm
 
+from .constants import TQDM_SMOOTHING
 from .models.config import ConfigModel
 from .progress_logging import FileProgressLogger
 from .states import DownloadState
+from .transfer import init_s3_client
 
 MULTIPART_THRESHOLD = 8 * 1024 * 1024  # 8MiB, boto3 default
 MULTIPART_CHUNKSIZE = 8 * 1024 * 1024  # 8MiB, boto3 default
@@ -62,32 +65,7 @@ class S3BotoDownloadWorker:
         self._config = config
         self._threads = threads
 
-        self._init_s3_client()
-
-    def _init_s3_client(self):
-        # if user specifies empty strings, this might be an issue
-        def empty_str_to_none(string: str | None) -> str | None:
-            if string == "" or string is None:
-                return None
-            else:
-                return string
-
-        # configure proxies if proxy_url is defined
-        proxy_url = empty_str_to_none(self._config.s3_options.proxy_url)
-        config = Boto3Config(proxies={"http": proxy_url, "https": proxy_url}) if proxy_url is not None else None
-
-        # Initialize S3 client for uploading
-        self._s3_client: boto3.session.Session.client = boto3_client(
-            service_name="s3",
-            region_name=empty_str_to_none(self._config.s3_options.region_name),
-            api_version=empty_str_to_none(self._config.s3_options.api_version),
-            use_ssl=self._config.s3_options.use_ssl,
-            endpoint_url=empty_str_to_none(str(self._config.s3_options.endpoint_url)),
-            aws_access_key_id=empty_str_to_none(self._config.s3_options.access_key),
-            aws_secret_access_key=empty_str_to_none(self._config.s3_options.secret),
-            aws_session_token=empty_str_to_none(self._config.s3_options.session_token),
-            config=config,
-        )
+        self._s3_client = init_s3_client(config)
 
     def prepare_download(
         self,
@@ -155,7 +133,7 @@ class S3BotoDownloadWorker:
         )
 
         transfer = S3Transfer(self._s3_client, config)
-        progress_bar = tqdm(total=filesize, unit="B", unit_scale=True, unit_divisor=1024)
+        progress_bar = tqdm(total=filesize, unit="B", unit_scale=True, unit_divisor=1024, smoothing=TQDM_SMOOTHING)
         transfer.download_file(
             self._config.s3_options.bucket,
             s3_object_id,
@@ -222,3 +200,42 @@ class S3BotoDownloadWorker:
                             file_key,
                             str(full_path),
                         )
+
+
+class SubmissionInboxState(BaseModel):
+    """A summary of the state of a submission in an inbox"""
+
+    submission_id: str
+    complete: bool
+    oldest_upload: datetime.datetime
+    newest_upload: datetime.datetime
+
+
+def query_submissions(config: ConfigModel) -> list[SubmissionInboxState]:
+    """Queries the state of all submissions in the configured bucket."""
+    s3_client = init_s3_client(config)
+    paginator = s3_client.get_paginator("list_objects_v2")
+
+    objects = itertools.chain.from_iterable(
+        page["Contents"] for page in paginator.paginate(Bucket=config.s3_options.bucket)
+    )
+    objects_sorted = sorted(objects, key=itemgetter("Key"))
+    submission2objects = {
+        key: tuple(group) for key, group in itertools.groupby(objects_sorted, key=lambda o: o["Key"].split("/")[0])
+    }
+
+    submissions = []
+    for submission_id, submission_objects in submission2objects.items():
+        submission_objects_sorted = sorted(submission_objects, key=itemgetter("LastModified"))
+        oldest_object = submission_objects_sorted[0]
+        newest_object = submission_objects_sorted[-1]
+        is_complete = len(list(filter(lambda o: o["Key"].endswith("/metadata.json"), submission_objects))) == 1
+        submission = SubmissionInboxState(
+            submission_id=submission_id,
+            complete=is_complete,
+            oldest_upload=oldest_object["LastModified"],
+            newest_upload=newest_object["LastModified"],
+        )
+        submissions.append(submission)
+
+    return sorted(submissions, key=attrgetter("oldest_upload"))