grz-cli 0.4.0__tar.gz → 0.5.0__tar.gz

{grz_cli-0.4.0/src/grz_cli.egg-info → grz_cli-0.5.0}/PKG-INFO

@@ -1,16 +1,16 @@
 Metadata-Version: 2.4
 Name: grz-cli
-Version: 0.4.0
+Version: 0.5.0
 Summary: Tool for validation, encryption and upload of MV submissions to GDCs.
 Author-email: Koray Kirli <koraykirli@gmail.com>, Mathias Lesche <mathias.lesche@tu-dresden.de>, "Florian R. Hölzlwimmer" <git.ich@frhoelzlwimmer.de>, Till Hartmann <till.hartmann@bih-charite.de>, Thomas Sell <thomas.sell@bih-charite.de>
-License: MIT
+License-Expression: MIT
 Project-URL: Homepage, https://codebase.helmholtz.cloud/grz-mv-genomseq/grz-cli
 Keywords: GRZ,GDC,S3
 Classifier: Programming Language :: Python :: 3
-Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
+License-File: LICENSE
 Requires-Dist: boto3<2,>=1.36
 Requires-Dist: click<9,>=8.1.7
 Requires-Dist: crypt4gh<2,>=1.7
@@ -21,6 +21,7 @@ Requires-Dist: pydantic<2.10,>=2.9.2
 Requires-Dist: platformdirs<5,>=4.3.6
 Requires-Dist: grz-pydantic-models>=1.2.1
 Requires-Dist: pysam==0.23.*
+Requires-Dist: rich==13.*
 Provides-Extra: test
 Requires-Dist: pytest<9,>=8.3.3; extra == "test"
 Requires-Dist: pytest-cov<6,>=5.0.0; extra == "test"
@@ -29,6 +30,7 @@ Requires-Dist: moto[s3]<6,>=5.0.16; extra == "test"
 Requires-Dist: pytest-sugar<2,>=1.0.0; extra == "test"
 Requires-Dist: ruff<1,>=0.9.6; extra == "test"
 Requires-Dist: numpy<3,>=2.1.2; extra == "test"
+Dynamic: license-file
 
 # GRZ CLI
 

{grz_cli-0.4.0 → grz_cli-0.5.0}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 name = "grz-cli"
 description = "Tool for validation, encryption and upload of MV submissions to GDCs."
 readme = "README.md"
-license = { text = "MIT" }
+license = "MIT"
 authors = [
     { name = "Koray Kirli", email = "koraykirli@gmail.com" },
     { name = "Mathias Lesche", email = "mathias.lesche@tu-dresden.de" },
@@ -25,11 +25,11 @@ dependencies = [
     "pydantic >=2.9.2,<2.10",
     "platformdirs >=4.3.6,<5",
     "grz-pydantic-models >=1.2.1",
-    "pysam ==0.23.*"
+    "pysam ==0.23.*",
+    "rich ==13.*"
 ]
 classifiers = [
     "Programming Language :: Python :: 3",
-    "License :: OSI Approved :: MIT License",
     "Operating System :: OS Independent"
 ]
 requires-python = ">=3.12"
@@ -43,7 +43,7 @@ version = { attr = "grz_cli.__version__" }
 "Homepage" = "https://codebase.helmholtz.cloud/grz-mv-genomseq/grz-cli"
 
 [project.scripts]
-grz-cli = "grz_cli.cli:cli"
+grz-cli = "grz_cli.cli:main"
 
 [project.optional-dependencies]
 test = [
@@ -207,6 +207,3 @@ follow_untyped_imports = true
 addopts = [
     "--import-mode=importlib",
 ]
-
-[tool.setuptools]
-license-files = []

grz_cli-0.5.0/src/grz_cli/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.5.0"

{grz_cli-0.4.0 → grz_cli-0.5.0}/src/grz_cli/cli.py

@@ -3,8 +3,10 @@ CLI module for handling command-line interface operations.
 """
 
 import importlib
+import json
 import logging
 import logging.config
+import os
 import sys
 from os import PathLike, sched_getaffinity
 from pathlib import Path
@@ -13,9 +15,13 @@ from tempfile import NamedTemporaryFile
 import click
 import grz_pydantic_models.submission.metadata
 import platformdirs
+import rich.console
+import rich.table
 import yaml
+from pydantic_core import to_jsonable_python
 
 from .constants import PACKAGE_ROOT
+from .download import query_submissions
 from .logging_setup import add_filelogger
 from .models.config import ConfigModel
 from .parser import Worker
@@ -94,6 +100,16 @@ output_dir = click.option(
     help="Path to the target submission output directory",
 )
 
+output_json = click.option("--json", "output_json", is_flag=True, help="Output JSON for machine-readability.")
+
+
+def read_config(config_path: str | PathLike) -> ConfigModel:
+    """Reads the configuration file and validates it against the schema."""
+    with open(config_path, encoding="utf-8") as f:
+        config = ConfigModel(**yaml.safe_load(f))
+
+    return config
+
 
 class OrderedGroup(click.Group):
     """
@@ -105,48 +121,7 @@ class OrderedGroup(click.Group):
         return list(self.commands.keys())
 
 
-@click.group(
-    cls=OrderedGroup,
-    help="Validate, encrypt, decrypt and upload submissions to a GRZ/GDC.",
-)
-@click.version_option(
-    version=importlib.metadata.version("grz-cli"),
-    prog_name="grz-cli",
-    message=f"%(prog)s v%(version)s (metadata schema versions: {', '.join(grz_pydantic_models.submission.metadata.get_supported_versions())})",
-)
-@click.option("--log-file", metavar="FILE", type=str, help="Path to log file")
-@click.option(
-    "--log-level",
-    default="INFO",
-    type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]),
-    help="Set the log level (default: INFO)",
-)
-def cli(log_file: str | None = None, log_level: str = "INFO"):
-    """
-    Command-line interface function for setting up logging.
-
-    :param log_file: Path to the log file. If provided, a file logger will be added.
-    :param log_level: Log level for the logger. It should be one of the following:
-                       DEBUG, INFO, WARNING, ERROR, CRITICAL.
-    """
-    if log_file:
-        add_filelogger(
-            log_file,
-            log_level.upper(),
-        )  # Add file logger
-
-    # show only time and log level in STDOUT
-    logging.basicConfig(
-        format="%(asctime)s - %(levelname)s - %(message)s",
-    )
-
-    # set the log level for this package
-    logging.getLogger(PACKAGE_ROOT).setLevel(log_level.upper())
-
-    log.debug("Logging setup complete.")
-
-
-@cli.command()
+@click.command()
 @submission_dir
 def validate(submission_dir):
     """
@@ -170,7 +145,7 @@ def validate(submission_dir):
     log.info("Validation finished!")
 
 
-@cli.command()
+@click.command()
 @submission_dir
 @config_file
 def encrypt(
@@ -216,41 +191,7 @@ def encrypt(
     log.info("Encryption successful!")
 
 
-@cli.command()
-@submission_dir
-@config_file
-def decrypt(
-    submission_dir,
-    config_file,
-):
-    """
-    Decrypt a submission.
-
-    Decrypting a submission requires the _private_ key of the original recipient.
-    """
-    config = read_config(config_file)
-
-    grz_privkey_path = config.grz_private_key_path
-    if not grz_privkey_path:
-        log.error("GRZ private key path is required for decryption.")
-        sys.exit(1)
-
-    log.info("Starting decryption...")
-
-    submission_dir = Path(submission_dir)
-
-    worker_inst = Worker(
-        metadata_dir=submission_dir / "metadata",
-        files_dir=submission_dir / "files",
-        log_dir=submission_dir / "logs",
-        encrypted_files_dir=submission_dir / "encrypted_files",
-    )
-    worker_inst.decrypt(grz_privkey_path)
-
-    log.info("Encryption successful!")
-
-
-@cli.command()
+@click.command()
 @submission_dir
 @config_file
 @threads
@@ -281,7 +222,28 @@ def upload(
     log.info("Upload finished!")
 
 
-@cli.command()
+@click.command("submit")
+@submission_dir
+@config_file
+@threads
+@click.pass_context
+def submit(ctx, submission_dir, config_file, threads):
+    """
+    Validate, encrypt, and then upload.
+
+    This is a convenience command that performs the following steps in order:
+    1. Validate the submission
+    2. Encrypt the submission
+    3. Upload the encrypted submission
+    """
+    click.echo("Starting submission process...")
+    ctx.invoke(validate, submission_dir=submission_dir)
+    ctx.invoke(encrypt, submission_dir=submission_dir, config_file=config_file)
+    ctx.invoke(upload, submission_dir=submission_dir, config_file=config_file, threads=threads)
+    click.echo("Submission finished!")
+
+
+@click.command()
 @submission_id
 @output_dir
 @config_file
@@ -319,13 +281,140 @@ def download(
     log.info("Download finished!")
 
 
-def read_config(config_path: str | PathLike) -> ConfigModel:
-    """Reads the configuration file and validates it against the schema."""
-    with open(config_path, encoding="utf-8") as f:
-        config = ConfigModel(**yaml.safe_load(f))
+@click.command()
+@config_file
+@output_json
+def list_submissions(config_file, output_json):
+    """
+    List submissions within an inbox from oldest to newest.
+    """
+    config = read_config(config_file)
+    submissions = query_submissions(config)
 
-    return config
+    if output_json:
+        json.dump(to_jsonable_python(submissions), sys.stdout)
+    else:
+        console = rich.console.Console()
+        table = rich.table.Table()
+        table.add_column("ID", no_wrap=True)
+        table.add_column("Status", no_wrap=True)
+        table.add_column("Oldest Upload", overflow="fold")
+        table.add_column("Newest Upload", overflow="fold")
+        for submission in submissions:
+            status_text = rich.text.Text(
+                "Complete" if submission.complete else "Incomplete",
+                style="green" if submission.complete else "yellow",
+            )
+            table.add_row(
+                submission.submission_id,
+                status_text,
+                submission.oldest_upload.astimezone().strftime("%Y-%m-%d %H:%M:%S"),
+                submission.newest_upload.astimezone().strftime("%Y-%m-%d %H:%M:%S"),
+            )
+        console.print(table)
 
 
-if __name__ == "__main__":
+@click.command()
+@submission_dir
+@config_file
+def decrypt(
+    submission_dir,
+    config_file,
+):
+    """
+    Decrypt a submission.
+
+    Decrypting a submission requires the _private_ key of the original recipient.
+    """
+    config = read_config(config_file)
+
+    grz_privkey_path = config.grz_private_key_path
+    if not grz_privkey_path:
+        log.error("GRZ private key path is required for decryption.")
+        sys.exit(1)
+
+    log.info("Starting decryption...")
+
+    submission_dir = Path(submission_dir)
+
+    worker_inst = Worker(
+        metadata_dir=submission_dir / "metadata",
+        files_dir=submission_dir / "files",
+        log_dir=submission_dir / "logs",
+        encrypted_files_dir=submission_dir / "encrypted_files",
+    )
+    worker_inst.decrypt(grz_privkey_path)
+
+    log.info("Encryption successful!")
+
+
+def build_cli(grz_mode=False):
+    """
+    Factory for building the CLI application.
+    """
+
+    @click.group(
+        cls=OrderedGroup,
+        help="Validate, encrypt, decrypt and upload submissions to a GRZ/GDC.",
+    )
+    @click.version_option(
+        version=importlib.metadata.version("grz-cli"),
+        prog_name="grz-cli",
+        message=f"%(prog)s v%(version)s (metadata schema versions: {', '.join(grz_pydantic_models.submission.metadata.get_supported_versions())})",
+    )
+    @click.option("--log-file", metavar="FILE", type=str, help="Path to log file")
+    @click.option(
+        "--log-level",
+        default="INFO",
+        type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]),
+        help="Set the log level (default: INFO)",
+    )
+    def cli(log_file: str | None = None, log_level: str = "INFO"):
+        """
+        Command-line interface function for setting up logging.
+
+        :param log_file: Path to the log file. If provided, a file logger will be added.
+        :param log_level: Log level for the logger. It should be one of the following:
+                           DEBUG, INFO, WARNING, ERROR, CRITICAL.
+        """
+        if log_file:
+            add_filelogger(
+                log_file,
+                log_level.upper(),
+            )  # Add file logger
+
+        # show only time and log level in STDOUT
+        logging.basicConfig(
+            format="%(asctime)s - %(levelname)s - %(message)s",
+        )
+
+        # set the log level for this package
+        logging.getLogger(PACKAGE_ROOT).setLevel(log_level.upper())
+
+        log.debug("Logging setup complete.")
+
+    cli.add_command(validate)
+    cli.add_command(encrypt)
+    cli.add_command(upload)
+    cli.add_command(submit)
+
+    if grz_mode:
+        cli.add_command(list_submissions, name="list")
+        cli.add_command(download)
+        cli.add_command(decrypt)
+
+    return cli
+
+
+def main():
+    """
+    Main entry point for the CLI application.
+    """
+    grz_mode = os.environ.get("GRZ_MODE", "false").lower() in {"true", "yes", "1"}
+
+    cli = build_cli(grz_mode=grz_mode)
     cli()
+
+
+if __name__ == "__main__":
+    main()

{grz_cli-0.4.0 → grz_cli-0.5.0}/src/grz_cli/constants.py

@@ -34,3 +34,5 @@ LOGGING_CONFIG = {
         },
     },
 }
+
+TQDM_SMOOTHING: float = 0.0001

{grz_cli-0.4.0 → grz_cli-0.5.0}/src/grz_cli/download.py

@@ -1,24 +1,27 @@
-"""Module for uploading encrypted submissions to a remote storage"""
+"""Module for downloading encrypted submissions to local storage"""
 
 from __future__ import annotations
 
+import datetime
+import itertools
 import logging
 import math
 import re
+from operator import attrgetter, itemgetter
 from os import PathLike
 from pathlib import Path
 from typing import TYPE_CHECKING
 
-import boto3  # type: ignore[import-untyped]
 import botocore.handlers  # type: ignore[import-untyped]
-from boto3 import client as boto3_client  # type: ignore[import-untyped]
 from boto3.s3.transfer import S3Transfer, TransferConfig  # type: ignore[import-untyped]
-from botocore.config import Config as Boto3Config  # type: ignore[import-untyped]
+from pydantic import BaseModel
 from tqdm.auto import tqdm
 
+from .constants import TQDM_SMOOTHING
 from .models.config import ConfigModel
 from .progress_logging import FileProgressLogger
 from .states import DownloadState
+from .transfer import init_s3_client
 
 MULTIPART_THRESHOLD = 8 * 1024 * 1024  # 8MiB, boto3 default
 MULTIPART_CHUNKSIZE = 8 * 1024 * 1024  # 8MiB, boto3 default
@@ -62,35 +65,7 @@ class S3BotoDownloadWorker:
         self._config = config
         self._threads = threads
 
-        self._init_s3_client()
-
-    def _init_s3_client(self):
-        # if user specifies empty strings, this might be an issue
-        def empty_str_to_none(string: str | None) -> str | None:
-            if string == "" or string is None:
-                return None
-            else:
-                return string
-
-        # configure proxies if proxy_url is defined
-        proxy_url = self._config.s3_options.proxy_url
-        config = Boto3Config(
-            proxies={"http": str(proxy_url), "https": str(proxy_url)} if proxy_url is not None else None,
-            request_checksum_calculation=self._config.s3_options.request_checksum_calculation,
-        )
-
-        # Initialize S3 client for uploading
-        self._s3_client: boto3.session.Session.client = boto3_client(
-            service_name="s3",
-            region_name=empty_str_to_none(self._config.s3_options.region_name),
-            api_version=empty_str_to_none(self._config.s3_options.api_version),
-            use_ssl=self._config.s3_options.use_ssl,
-            endpoint_url=empty_str_to_none(str(self._config.s3_options.endpoint_url)),
-            aws_access_key_id=empty_str_to_none(self._config.s3_options.access_key),
-            aws_secret_access_key=empty_str_to_none(self._config.s3_options.secret),
-            aws_session_token=empty_str_to_none(self._config.s3_options.session_token),
-            config=config,
-        )
+        self._s3_client = init_s3_client(config)
 
     def prepare_download(
         self,
@@ -158,7 +133,7 @@ class S3BotoDownloadWorker:
         )
 
         transfer = S3Transfer(self._s3_client, config)
-        progress_bar = tqdm(total=filesize, unit="B", unit_scale=True, unit_divisor=1024)
+        progress_bar = tqdm(total=filesize, unit="B", unit_scale=True, unit_divisor=1024, smoothing=TQDM_SMOOTHING)
         transfer.download_file(
             self._config.s3_options.bucket,
             s3_object_id,
@@ -225,3 +200,42 @@ class S3BotoDownloadWorker:
                             file_key,
                             str(full_path),
                         )
+
+
+class SubmissionInboxState(BaseModel):
+    """A summary of the state of a submission in an inbox"""
+
+    submission_id: str
+    complete: bool
+    oldest_upload: datetime.datetime
+    newest_upload: datetime.datetime
+
+
+def query_submissions(config: ConfigModel) -> list[SubmissionInboxState]:
+    """Queries the state of all submissions in the configured bucket."""
+    s3_client = init_s3_client(config)
+    paginator = s3_client.get_paginator("list_objects_v2")
+
+    objects = itertools.chain.from_iterable(
+        page["Contents"] for page in paginator.paginate(Bucket=config.s3_options.bucket)
+    )
+    objects_sorted = sorted(objects, key=itemgetter("Key"))
+    submission2objects = {
+        key: tuple(group) for key, group in itertools.groupby(objects_sorted, key=lambda o: o["Key"].split("/")[0])
+    }
+
+    submissions = []
+    for submission_id, submission_objects in submission2objects.items():
+        submission_objects_sorted = sorted(submission_objects, key=itemgetter("LastModified"))
+        oldest_object = submission_objects_sorted[0]
+        newest_object = submission_objects_sorted[-1]
+        is_complete = len(list(filter(lambda o: o["Key"].endswith("/metadata.json"), submission_objects))) == 1
+        submission = SubmissionInboxState(
+            submission_id=submission_id,
+            complete=is_complete,
+            oldest_upload=oldest_object["LastModified"],
+            newest_upload=newest_object["LastModified"],
+        )
+        submissions.append(submission)
+
+    return sorted(submissions, key=attrgetter("oldest_upload"))

{grz_cli-0.4.0 → grz_cli-0.5.0}/src/grz_cli/fastq_validation.py

@@ -13,6 +13,7 @@ from typing import TextIO
 
 from tqdm.auto import tqdm
 
+from .constants import TQDM_SMOOTHING
 from .file_operations import TqdmIOWrapper
 
 log = logging.getLogger(__name__)
@@ -51,6 +52,7 @@ def open_fastq(file_path: str | PathLike, progress=True) -> Generator[TextIO, No
                     unit_scale=True,
                     # unit_divisor=1024,  # make use of standard units e.g. KB, MB, etc.
                     miniters=1,
+                    smoothing=TQDM_SMOOTHING,
                 ),
             )
         else:

{grz_cli-0.4.0 → grz_cli-0.5.0}/src/grz_cli/file_operations.py

@@ -24,6 +24,8 @@ import crypt4gh.lib  # type: ignore[import-untyped]
 from nacl.public import PrivateKey
 from tqdm.auto import tqdm
 
+from .constants import TQDM_SMOOTHING
+
 log = logging.getLogger(__name__)
 
 
@@ -69,7 +71,9 @@ def calculate_md5(file_path, chunk_size=2**16, progress=True) -> str:
     md5_hash = hashlib.md5()  # noqa: S324
     with open(file_path, "rb") as f:
         if progress and (total_size > chunk_size):
-            with tqdm(total=total_size, unit="B", unit_scale=True, desc="Calculating MD5") as pbar:
+            with tqdm(
+                total=total_size, unit="B", unit_scale=True, desc="Calculating MD5", smoothing=TQDM_SMOOTHING
+            ) as pbar:
                 while chunk := f.read(chunk_size):
                     md5_hash.update(chunk)
                     pbar.update(len(chunk))
@@ -195,6 +199,7 @@ class Crypt4GH:
                     unit_scale=True,
                     # unit_divisor=1024,  # make use of standard units e.g. KB, MB, etc.
                     miniters=1,
+                    smoothing=TQDM_SMOOTHING,
                 ),
             ) as pbar_in_fd,
         ):

{grz_cli-0.4.0 → grz_cli-0.5.0}/src/grz_cli/models/config.py

@@ -32,14 +32,16 @@ class S3Options(StrictBaseModel):
     The name of the S3 bucket.
     """
 
-    access_key: str
+    access_key: str | None = None
     """
     The access key for the S3 bucket.
+    If undefined, it is read from the AWS_ACCESS_KEY_ID environment variable.
     """
 
-    secret: str
+    secret: str | None = None
     """
     The secret key for the S3 bucket.
+    If undefined, it is read from the AWS_SECRET_ACCESS_KEY environment variable.
     """
 
     session_token: str | None = None
@@ -85,6 +87,11 @@ class S3Options(StrictBaseModel):
     Defaults to None.
     """
 
+    multipart_chunksize: int = 256 * 1024**2
+    """
+    The size of the chunks to use for multipart uploads in bytes.
+    """
+
 
 FilePath = Annotated[Path, AfterValidator(lambda v: v.expanduser()), PathType("file")]
 

grz_cli-0.5.0/src/grz_cli/transfer.py

@@ -0,0 +1,42 @@
+"""
+Common methods for transfering data to and from GRZ buckets.
+"""
+
+import boto3  # type: ignore[import-untyped]
+from boto3 import client as boto3_client  # type: ignore[import-untyped]
+from botocore.config import Config as Boto3Config  # type: ignore[import-untyped]
+
+from .models.config import ConfigModel
+
+
+def _empty_str_to_none(string: str | None) -> str | None:
+    # if user specifies empty strings, this might be an issue
+    if string == "" or string is None:
+        return None
+    else:
+        return string
+
+
+def init_s3_client(config: ConfigModel) -> boto3.session.Session.client:
+    """Create a boto3 Client from a grz-cli configuration."""
+    # configure proxies if proxy_url is defined
+    proxy_url = config.s3_options.proxy_url
+    s3_config = Boto3Config(
+        proxies={"http": str(proxy_url), "https": str(proxy_url)} if proxy_url is not None else None,
+        request_checksum_calculation=config.s3_options.request_checksum_calculation,
+    )
+
+    # Initialize S3 client for uploading
+    s3_client: boto3.session.Session.client = boto3_client(
+        service_name="s3",
+        region_name=_empty_str_to_none(config.s3_options.region_name),
+        api_version=_empty_str_to_none(config.s3_options.api_version),
+        use_ssl=config.s3_options.use_ssl,
+        endpoint_url=_empty_str_to_none(str(config.s3_options.endpoint_url)),
+        aws_access_key_id=_empty_str_to_none(config.s3_options.access_key),
+        aws_secret_access_key=_empty_str_to_none(config.s3_options.secret),
+        aws_session_token=_empty_str_to_none(config.s3_options.session_token),
+        config=s3_config,
+    )
+
+    return s3_client

{grz_cli-0.4.0 → grz_cli-0.5.0}/src/grz_cli/upload.py

@@ -11,19 +11,17 @@ from os.path import getsize
 from pathlib import Path
 from typing import TYPE_CHECKING, override
 
-import boto3  # type: ignore[import-untyped]
 import botocore.handlers  # type: ignore[import-untyped]
-from boto3 import client as boto3_client  # type: ignore[import-untyped]
 from boto3.s3.transfer import S3Transfer, TransferConfig  # type: ignore[import-untyped]
-from botocore.config import Config as Boto3Config  # type: ignore[import-untyped]
 from tqdm.auto import tqdm
 
+from .constants import TQDM_SMOOTHING
 from .models.config import ConfigModel
 from .progress_logging import FileProgressLogger
 from .states import UploadState
+from .transfer import init_s3_client
 
-MULTIPART_THRESHOLD = 8 * 1024 * 1024  # 8MiB, boto3 default
-MULTIPART_CHUNKSIZE = 8 * 1024 * 1024  # 8MiB, boto3 default
+MULTIPART_THRESHOLD = 8 * 1024**2  # 8MiB, boto3 default, largely irrelevant
 MULTIPART_MAX_CHUNKS = 1000  # CEPH S3 limit, AWS limit is 10000
 
 if TYPE_CHECKING:
@@ -87,35 +85,7 @@ class S3BotoUploadWorker(UploadWorker):
         self._config = config
         self._threads = threads
 
-        self._init_s3_client()
-
-    def _init_s3_client(self):
-        # if user specifies empty strings, this might be an issue
-        def empty_str_to_none(string: str | None) -> str | None:
-            if string == "" or string is None:
-                return None
-            else:
-                return string
-
-        # configure proxies if proxy_url is defined
-        proxy_url = self._config.s3_options.proxy_url
-        config = Boto3Config(
-            proxies={"http": str(proxy_url), "https": str(proxy_url)} if proxy_url is not None else None,
-            request_checksum_calculation=self._config.s3_options.request_checksum_calculation,
-        )
-
-        # Initialize S3 client for uploading
-        self._s3_client: boto3.session.Session.client = boto3_client(
-            service_name="s3",
-            region_name=empty_str_to_none(self._config.s3_options.region_name),
-            api_version=empty_str_to_none(self._config.s3_options.api_version),
-            use_ssl=self._config.s3_options.use_ssl,
-            endpoint_url=empty_str_to_none(str(self._config.s3_options.endpoint_url)),
-            aws_access_key_id=empty_str_to_none(self._config.s3_options.access_key),
-            aws_secret_access_key=empty_str_to_none(self._config.s3_options.secret),
-            aws_session_token=empty_str_to_none(self._config.s3_options.session_token),
-            config=config,
-        )
+        self._s3_client = init_s3_client(config)
 
     @override
     def upload_file(self, local_file_path, s3_object_id):
@@ -127,11 +97,12 @@ class S3BotoUploadWorker(UploadWorker):
         self.__log.info(f"Uploading {local_file_path} to {s3_object_id}...")
 
         filesize = getsize(local_file_path)
+        multipart_chunksize = self._config.s3_options.multipart_chunksize
 
         chunksize = (
             math.ceil(filesize / MULTIPART_MAX_CHUNKS)
-            if filesize / MULTIPART_CHUNKSIZE > MULTIPART_MAX_CHUNKS
-            else MULTIPART_CHUNKSIZE
+            if filesize / multipart_chunksize > MULTIPART_MAX_CHUNKS
+            else multipart_chunksize
         )
         self.__log.debug(
             f"Using a chunksize of: {chunksize / 1024**2}MiB, results in {math.ceil(filesize / chunksize)} chunk(s)"
@@ -141,10 +112,11 @@ class S3BotoUploadWorker(UploadWorker):
             multipart_threshold=MULTIPART_THRESHOLD,
             multipart_chunksize=chunksize,
             max_concurrency=self._threads,
+            use_threads=self._threads > 1,
         )
 
         transfer = S3Transfer(self._s3_client, config)
-        progress_bar = tqdm(total=filesize, unit="B", unit_scale=True, unit_divisor=1024)
+        progress_bar = tqdm(total=filesize, unit="B", unit_scale=True, unit_divisor=1024, smoothing=TQDM_SMOOTHING)
         transfer.upload_file(
             local_file_path,
             self._config.s3_options.bucket,
@@ -152,6 +124,25 @@ class S3BotoUploadWorker(UploadWorker):
             callback=lambda bytes_transferred: progress_bar.update(bytes_transferred),
         )
 
+    def _remote_id_exists(self, s3_object_id: str) -> bool:
+        """
+        Determine if a remote ID already exists
+        :param s3_object_id: Remote S3 object ID under which the file should be stored
+        """
+        exists = True
+        try:
+            self._s3_client.head_object(Bucket=self._config.s3_options.bucket, Key=s3_object_id)
+        except self._s3_client.exceptions.NoSuchKey:
+            exists = False
+        except botocore.exceptions.ClientError as error:
+            if error.response["Error"]["Code"] in {"403", "404"}:
+                # backend can return forbidden instead if user has no ListBucket permission
+                exists = False
+            else:
+                raise error
+
+        return exists
+
     @override
     def upload(self, encrypted_submission: EncryptedSubmission):
         """
@@ -160,6 +151,10 @@ class S3BotoUploadWorker(UploadWorker):
         """
         progress_logger = FileProgressLogger[UploadState](self._status_file_path)
         metadata_file_path, metadata_s3_object_id = encrypted_submission.get_metadata_file_path_and_object_id()
+
+        if self._remote_id_exists(metadata_s3_object_id):
+            raise UploadError("Submission already uploaded. Corrections, additions, and followups require a new tanG.")
+
         files_to_upload = encrypted_submission.get_encrypted_files_and_object_id()
         files_to_upload[metadata_file_path] = metadata_s3_object_id
 

{grz_cli-0.4.0 → grz_cli-0.5.0/src/grz_cli.egg-info}/PKG-INFO

@@ -1,16 +1,16 @@
 Metadata-Version: 2.4
 Name: grz-cli
-Version: 0.4.0
+Version: 0.5.0
 Summary: Tool for validation, encryption and upload of MV submissions to GDCs.
 Author-email: Koray Kirli <koraykirli@gmail.com>, Mathias Lesche <mathias.lesche@tu-dresden.de>, "Florian R. Hölzlwimmer" <git.ich@frhoelzlwimmer.de>, Till Hartmann <till.hartmann@bih-charite.de>, Thomas Sell <thomas.sell@bih-charite.de>
-License: MIT
+License-Expression: MIT
 Project-URL: Homepage, https://codebase.helmholtz.cloud/grz-mv-genomseq/grz-cli
 Keywords: GRZ,GDC,S3
 Classifier: Programming Language :: Python :: 3
-Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
+License-File: LICENSE
 Requires-Dist: boto3<2,>=1.36
 Requires-Dist: click<9,>=8.1.7
 Requires-Dist: crypt4gh<2,>=1.7
@@ -21,6 +21,7 @@ Requires-Dist: pydantic<2.10,>=2.9.2
 Requires-Dist: platformdirs<5,>=4.3.6
 Requires-Dist: grz-pydantic-models>=1.2.1
 Requires-Dist: pysam==0.23.*
+Requires-Dist: rich==13.*
 Provides-Extra: test
 Requires-Dist: pytest<9,>=8.3.3; extra == "test"
 Requires-Dist: pytest-cov<6,>=5.0.0; extra == "test"
@@ -29,6 +30,7 @@ Requires-Dist: moto[s3]<6,>=5.0.16; extra == "test"
 Requires-Dist: pytest-sugar<2,>=1.0.0; extra == "test"
 Requires-Dist: ruff<1,>=0.9.6; extra == "test"
 Requires-Dist: numpy<3,>=2.1.2; extra == "test"
+Dynamic: license-file
 
 # GRZ CLI
 

{grz_cli-0.4.0 → grz_cli-0.5.0}/src/grz_cli.egg-info/SOURCES.txt

@@ -14,6 +14,7 @@ src/grz_cli/logging_setup.py
 src/grz_cli/parser.py
 src/grz_cli/progress_logging.py
 src/grz_cli/states.py
+src/grz_cli/transfer.py
 src/grz_cli/upload.py
 src/grz_cli.egg-info/PKG-INFO
 src/grz_cli.egg-info/SOURCES.txt

grz_cli-0.5.0/src/grz_cli.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+grz-cli = grz_cli.cli:main

{grz_cli-0.4.0 → grz_cli-0.5.0}/src/grz_cli.egg-info/requires.txt

@@ -8,6 +8,7 @@ pydantic<2.10,>=2.9.2
 platformdirs<5,>=4.3.6
 grz-pydantic-models>=1.2.1
 pysam==0.23.*
+rich==13.*
 
 [test]
 pytest<9,>=8.3.3

{grz_cli-0.4.0 → grz_cli-0.5.0}/tests/test_cli.py

@@ -42,10 +42,11 @@ def test_validate_submission(
     ]
 
     runner = CliRunner()
-    result = runner.invoke(grz_cli.cli.cli, testargs, catch_exceptions=False)
+    cli = grz_cli.cli.build_cli()
+    result = runner.invoke(cli, testargs, catch_exceptions=False)
 
     # check if re-validation is skipped
-    result = runner.invoke(grz_cli.cli.cli, testargs, catch_exceptions=False)
+    result = runner.invoke(cli, testargs, catch_exceptions=False)
 
     # test if command has correctly checked for:
     # - mismatched md5sums
@@ -75,7 +76,8 @@ def test_encrypt_decrypt_submission(
     ]
 
     runner = CliRunner()
-    result = runner.invoke(grz_cli.cli.cli, testargs, catch_exceptions=False)
+    cli = grz_cli.cli.build_cli(grz_mode=True)
+    result = runner.invoke(cli, testargs, catch_exceptions=False)
 
     assert result.exit_code == 0, result.output
 
@@ -89,7 +91,7 @@ def test_encrypt_decrypt_submission(
     ]
 
     runner = CliRunner()
-    result = runner.invoke(grz_cli.cli.cli, testargs, catch_exceptions=False)
+    result = runner.invoke(cli, testargs, catch_exceptions=False)
 
     assert result.exit_code == 0, result.output
 
@@ -126,7 +128,8 @@ def test_decrypt_submission(working_dir_path, temp_config_file_path):
         temp_config_file_path,
     ]
     runner = CliRunner()
-    result = runner.invoke(grz_cli.cli.cli, testargs, catch_exceptions=False)
+    cli = grz_cli.cli.build_cli(grz_mode=True)
+    result = runner.invoke(cli, testargs, catch_exceptions=False)
 
     assert result.exit_code == 0, result.output
 
@@ -204,7 +207,8 @@ def test_upload_download_submission(
         "grz_cli.models.config.S3Options.__getattr__",
         lambda self, name: None if name == "endpoint_url" else AttributeError,
     ):
-        result = runner.invoke(grz_cli.cli.cli, testargs, catch_exceptions=False)
+        cli = grz_cli.cli.build_cli(grz_mode=True)
+        result = runner.invoke(cli, testargs, catch_exceptions=False)
 
     assert result.exit_code == 0, result.output
     assert len(result.output) != 0, result.stderr
@@ -226,8 +230,8 @@ def test_upload_download_submission(
         temp_config_file_path,
     ]
     runner = CliRunner()
-
-    result = runner.invoke(grz_cli.cli.cli, testargs, catch_exceptions=False)
+    cli = grz_cli.cli.build_cli(grz_mode=True)
+    result = runner.invoke(cli, testargs, catch_exceptions=False)
 
     assert result.exit_code == 0, result.output
 

grz_cli-0.4.0/src/grz_cli/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.4.0"

grz_cli-0.4.0/src/grz_cli.egg-info/entry_points.txt

@@ -1,2 +0,0 @@
-[console_scripts]
-grz-cli = grz_cli.cli:cli