grz-cli 0.1.0__py3-none-any.whl

grz_cli/upload.py

@@ -0,0 +1,243 @@
+"""Module for uploading encrypted submissions to a remote storage"""
+
+from __future__ import annotations
+
+import abc
+import logging
+import math
+import re
+from os import PathLike
+from os.path import getsize
+from pathlib import Path
+from typing import TYPE_CHECKING, override
+
+import boto3  # type: ignore[import-untyped]
+import botocore.handlers  # type: ignore[import-untyped]
+from boto3 import client as boto3_client  # type: ignore[import-untyped]
+from boto3.s3.transfer import S3Transfer, TransferConfig  # type: ignore[import-untyped]
+from botocore.config import Config as Boto3Config  # type: ignore[import-untyped]
+from tqdm.auto import tqdm
+
+from .models.config import ConfigModel
+from .progress_logging import FileProgressLogger
+
+MULTIPART_THRESHOLD = 8 * 1024 * 1024  # 8MiB, boto3 default
+MULTIPART_CHUNKSIZE = 8 * 1024 * 1024  # 8MiB, boto3 default
+MULTIPART_MAX_CHUNKS = 1000  # CEPH S3 limit, AWS limit is 10000
+
+if TYPE_CHECKING:
+    from .parser import EncryptedSubmission
+
+log = logging.getLogger(__name__)
+
+# see discussion: https://github.com/boto/boto3/discussions/4251 for acception bucketnames with : in the name
+botocore.handlers.VALID_BUCKET = re.compile(r"^[:a-zA-Z0-9.\-_]{1,255}$")  # type: ignore[import-untyped]
+
+
+class UploadError(Exception):
+    """Exception raised when an upload fails"""
+
+    pass
+
+
+class UploadWorker(metaclass=abc.ABCMeta):
+    """Worker baseclass for uploading encrypted submissions"""
+
+    @abc.abstractmethod
+    def upload(self, encrypted_submission: EncryptedSubmission):
+        """
+        Upload an encrypted submission
+
+        :param encrypted_submission: The encrypted submission to upload
+        :raises UploadError: when the upload failed
+        """
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def upload_file(self, local_file_path: str | PathLike, s3_object_id: str):
+        """
+        Upload a single file to the specified object ID
+        :param local_file_path: Path to the file to upload
+        :param s3_object_id: Remote S3 object ID under which the file should be stored
+        """
+        raise NotImplementedError()
+
+
+class S3BotoUploadWorker(UploadWorker):
+    """Implementation of an upload worker using boto3 for S3"""
+
+    __log = log.getChild("S3BotoUploadWorker")
+
+    def __init__(
+        self,
+        config: ConfigModel,
+        status_file_path: str | PathLike,
+        threads: int = 1,
+    ):
+        """
+        An upload manager for S3 storage
+
+        :param config: instance of `ConfigModel`
+        :param status_file_path: file for storing upload state. Can be used for e.g. resumable uploads.
+        """
+        super().__init__()
+
+        self._status_file_path = Path(status_file_path)
+        self._config = config
+        self._threads = threads
+
+        self._init_s3_client()
+
+    def _init_s3_client(self):
+        # if user specifies empty strings, this might be an issue
+        def empty_str_to_none(string: str | None) -> str | None:
+            if string == "" or string is None:
+                return None
+            else:
+                return string
+
+        # configure proxies if proxy_url is defined
+        proxy_url = empty_str_to_none(self._config.s3_options.proxy_url)
+        if proxy_url is not None:
+            config = Boto3Config(proxies={"http": proxy_url, "https": proxy_url})
+        else:
+            config = None
+
+        # Initialize S3 client for uploading
+        self._s3_client: boto3.session.Session.client = boto3_client(
+            service_name="s3",
+            region_name=empty_str_to_none(self._config.s3_options.region_name),
+            api_version=empty_str_to_none(self._config.s3_options.api_version),
+            use_ssl=self._config.s3_options.use_ssl,
+            endpoint_url=empty_str_to_none(str(self._config.s3_options.endpoint_url)),
+            aws_access_key_id=empty_str_to_none(self._config.s3_options.access_key),
+            aws_secret_access_key=empty_str_to_none(self._config.s3_options.secret),
+            aws_session_token=empty_str_to_none(self._config.s3_options.session_token),
+            config=config,
+        )
+
+    @override
+    def upload_file(self, local_file_path, s3_object_id):
+        """
+        Upload a single file to the specified object ID
+        :param local_file_path: Path to the file to upload
+        :param s3_object_id: Remote S3 object ID under which the file should be stored
+        """
+        self.__log.info(f"Uploading {local_file_path} to {s3_object_id}...")
+
+        filesize = getsize(local_file_path)
+
+        chunksize = (
+            math.ceil(filesize / MULTIPART_MAX_CHUNKS)
+            if filesize / MULTIPART_CHUNKSIZE > MULTIPART_MAX_CHUNKS
+            else MULTIPART_CHUNKSIZE
+        )
+        self.__log.debug(
+            f"Using a chunksize of: {chunksize / 1024**2}MiB, results in {filesize/chunksize} chunks"
+        )
+
+        config = TransferConfig(
+            multipart_threshold=MULTIPART_THRESHOLD,
+            multipart_chunksize=chunksize,
+            max_concurrency=self._threads,
+        )
+
+        transfer = S3Transfer(self._s3_client, config)
+        progress_bar = tqdm(
+            total=filesize, unit="B", unit_scale=True, unit_divisor=1024
+        )
+        transfer.upload_file(
+            local_file_path,
+            self._config.s3_options.bucket,
+            s3_object_id,
+            callback=lambda bytes_transferred: progress_bar.update(bytes_transferred),
+        )
+
+    @override
+    def upload(self, encrypted_submission: EncryptedSubmission):
+        """
+        Upload an encrypted submission
+        :param encrypted_submission: The encrypted submission to upload
+        """
+        progress_logger = FileProgressLogger(self._status_file_path)
+        metadata_file_path, metadata_s3_object_id = (
+            encrypted_submission.get_metadata_file_path_and_object_id()
+        )
+        files_to_upload = encrypted_submission.get_encrypted_files_and_object_id()
+        files_to_upload[metadata_file_path] = metadata_s3_object_id
+
+        for file_path in files_to_upload:
+            if not Path(file_path).exists():
+                raise UploadError(f"File {file_path} does not exist")
+
+        for file_path, file_metadata in encrypted_submission.encrypted_files.items():
+            logged_state = progress_logger.get_state(file_path, file_metadata)
+            self.__log.debug("state for %s: %s", file_path, logged_state)
+
+            s3_object_id = files_to_upload[file_path]
+
+            if (logged_state is None) or not logged_state.get(
+                "upload_successful", False
+            ):
+                self.__log.info(
+                    "Uploading file: '%s' -> '%s'",
+                    str(file_path),
+                    str(s3_object_id),
+                )
+
+                try:
+                    self.upload_file(file_path, s3_object_id)
+
+                    self.__log.info(f"Upload complete for {str(file_path)}. ")
+                    progress_logger.set_state(
+                        file_path, file_metadata, state={"upload_successful": True}
+                    )
+                except Exception as e:
+                    self.__log.error("Upload failed for '%s'", str(file_path))
+
+                    progress_logger.set_state(
+                        file_path,
+                        file_metadata,
+                        state={"upload_successful": False, "error": str(e)},
+                    )
+
+                    raise e
+            else:
+                self.__log.info(
+                    "File '%s' already uploaded (at '%s')",
+                    str(file_path),
+                    str(s3_object_id),
+                )
+
+        # finally upload the metadata.json file, unconditionally:
+        try:
+            self.upload_file(metadata_file_path, metadata_s3_object_id)
+            self.__log.info(f"Upload complete for {str(metadata_file_path)}. ")
+        except Exception as e:
+            self.__log.error("Upload failed for '%s'", str(metadata_file_path))
+            raise e
+
+    def _check_for_completed_submission(self, s3_object_id: str) -> bool:
+        try:
+            return s3_object_id in self._list_keys(
+                self._config.s3_options.bucket, prefix=str(Path(s3_object_id).parent)
+            )
+        except Exception as e:
+            self.__log.warning(
+                "Exception occured during check for completed submission; assuming submission is incomplete.",
+                exc_info=e,
+            )
+            return False
+
+    # https://stackoverflow.com/a/54014862
+    def _list_keys(self, bucket_name, prefix="/", delimiter="/", start_after=""):
+        s3_paginator = self._s3_client.get_paginator("list_objects_v2")
+        prefix = prefix.lstrip(delimiter)
+        start_after = (
+            (start_after or prefix) if prefix.endswith(delimiter) else start_after
+        )
+        for page in s3_paginator.paginate(
+            Bucket=bucket_name, Prefix=prefix, StartAfter=start_after
+        ):
+            for content in page.get("Contents", ()):
+                yield content["Key"]

grz_cli-0.1.0.dist-info/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 GRZ/GHGA
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

grz_cli-0.1.0.dist-info/METADATA

@@ -0,0 +1,240 @@
+Metadata-Version: 2.1
+Name: grz-cli
+Version: 0.1.0
+Summary: Tool for validation, encryption and upload of MV submissions to GDCs.
+Author-email: Koray Kirli <koraykirli@gmail.com>, Mathias Lesche <mathias.lesche@tu-dresden.de>, "Florian R. Hölzlwimmer" <git.ich@frhoelzlwimmer.de>, Till Hartmann <till.hartmann@bih-charite.de>, Thomas Sell <thomas.sell@bih-charite.de>
+License: MIT
+Project-URL: Homepage, https://codebase.helmholtz.cloud/grz-mv-genomseq/grz-cli
+Keywords: GRZ,GDC,S3
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: boto3~=1.35.39
+Requires-Dist: click~=8.1.7
+Requires-Dist: crypt4gh~=1.7
+Requires-Dist: jsonschema~=4.23.0
+Requires-Dist: pyyaml~=6.0.2
+Requires-Dist: tqdm~=4.66.5
+Requires-Dist: pydantic~=2.9.2
+Requires-Dist: platformdirs~=4.3.6
+Provides-Extra: test
+Requires-Dist: pytest~=8.3.3; extra == "test"
+Requires-Dist: pytest-cov~=5.0.0; extra == "test"
+Requires-Dist: pytest-mock~=3.14.0; extra == "test"
+Requires-Dist: moto[s3]~=5.0.16; extra == "test"
+Requires-Dist: pytest-sugar~=1.0.0; extra == "test"
+Requires-Dist: ruff~=0.7.0; extra == "test"
+Requires-Dist: numpy~=2.1.2; extra == "test"
+
+# GRZ CLI
+
+A command-line tool for validating, encrypting, uploading and downloading submissions to/from a GDC/GRZ (Genomrechenzentrum).
+
+## Table of Contents
+
+- [Introduction](#introduction)
+- [Features](#features)
+- [Installation](#installation)
+- [Usage](#usage)
+  - [Configuration](#configuration)
+  - [Exemplary submission procedure](#exemplary-submission-procedure)
+- [Command-Line Interface](#command-line-interface)
+- [Testing](#testing)
+- [Contributing](#contributing)
+- [License](#license)
+- [Acknowledgements](#acknowledgements)
+
+## Introduction
+
+This tool provides a way to validate files, encrypt/decrypt files using the [crypt4gh](https://crypt4gh.readthedocs.io/en/latest/) library and upload/download the encrypted files to an S3 bucket of a GDC/GRZ. It also logs the progress and outcomes of these operations in a metadata file.
+
+It is recommended to have the following folder structure for a single submission:
+
+```
+EXAMPLE_SUBMISSION
+├── files
+│   ├── aaaaaaaa00000000aaaaaaaa00000000_blood_normal.read1.fastq.gz
+│   ├── aaaaaaaa00000000aaaaaaaa00000000_blood_normal.read2.fastq.gz
+│   ├── aaaaaaaa00000000aaaaaaaa00000000_blood_normal.vcf
+│   ├── aaaaaaaa00000000aaaaaaaa00000000_blood_tumor.read1.fastq.gz
+│   ├── aaaaaaaa00000000aaaaaaaa00000000_blood_tumor.read2.fastq.gz
+│   ├── aaaaaaaa00000000aaaaaaaa00000000_blood_tumor.vcf
+│   ├── target_regions.bed
+└── metadata
+    └── metadata.json
+```
+
+The current version of the tool requires the `working_dir` to have at least as much free disk space as the total size of the data being submitted.
+## Features
+
+- **Validation**: Validate file checksums, basic file metadata and BfArM requirements.
+- **Encryption**: Encrypt files using `crypt4gh`.
+- **Decryption**: Encrypt files using `crypt4gh`.
+- **Upload**: Upload encrypted files directly to a GRZ either (via built-in `boto3`).
+- **Download**: Download encrypted files from a GRZ (via built-in `boto3`).
+- **Logging**: Log progress and results of operations
+
+## Installation
+
+### End-user
+This tool uses the `conda` package manager.
+If `conda` is not yet available on your system, you can install the [Miniforge conda distribution](https://github.com/conda-forge/miniforge) by running the following commands:
+```bash
+curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh"
+bash Miniforge3-$(uname)-$(uname -m).sh
+```
+Next, install the `grz-cli` tool:
+```bash
+# TODO update: This requires membership in the repository
+# download environment.yaml from the git repository:
+git archive --remote=ssh://git@codebase.helmholtz.cloud/grz-mv-genomseq/grz-cli.git dev environment.yaml | tar -xv
+# TODO update: This will work once the repository is publicly accessible:
+# download environment.yaml from the git repository:
+curl -L "https://codebase.helmholtz.cloud/grz-mv-genomseq/grz-cli/-/raw/v0.1.0/environment.yaml?ref_type=heads" > environment.yaml
+
+# create conda environment and activate it
+conda env create -f environment.yaml -n grz-tools
+conda activate grz-tools
+
+# install the grz-cli tool
+# TODO update: This requires membership in the repository
+pip install "git+ssh://git@codebase.helmholtz.cloud/grz-mv-genomseq/grz-cli.git@dev"
+# TODO update: This will work once the repository is publicly accessible:
+pip install "git+https://codebase.helmholtz.cloud/grz-mv-genomseq/grz-cli@v0.1.0"
+```
+
+### Development setup
+For development purposes, you can clone the repository and install the package in editable mode:
+
+```bash
+git clone https://codebase.helmholtz.cloud/grz-mv-genomseq/grz-cli
+# create conda environment and activate it
+conda env create -f grz-cli/environment-dev.yaml -n grz-tools-dev
+conda activate grz-tools-dev
+# install the grz-cli tool
+pip install -e grz-cli/
+```
+
+## Usage
+
+### Configuration
+**The configuration file will be provided by your associated GRZ, please place it into `~/.config/grz-cli/config.yaml`.**
+
+The tool requires a configuration file in YAML format to specify the S3 bucket and other options.
+For an exemplary configuration, see [resources/config.yaml](resources/config.yaml).
+
+S3 access and secret key can be listed either in the config file or as environment variable (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`).
+
+### Exemplary submission procedure
+After preparing your submission as outlined above, you can use the following commands to validate, encrypt and upload the submission: 
+```sh
+# Validate the submission
+grz-cli validate --submission-dir EXAMPLE_SUBMISSION
+
+# Encrypt the submission
+grz-cli encrypt --submission-dir EXAMPLE_SUBMISSION
+
+# Upload the submission
+grz-cli upload --submission-dir EXAMPLE_SUBMISSION
+```
+
+### Troubleshooting
+**In case of issues, please re-run your commands with `grz-cli --log-level DEBUG --log-file <your-log-file.log> [...]` and submit the log file to the GRZ data steward!**
+
+## Command-Line Interface
+
+`grz-cli` provides a command-line interface with the following subcommands:
+
+### validate
+
+It is recommended to run this command before continuing with encryption and upload.
+Progress files are stored relative to the submission directory.
+
+- `--submission-dir`: Path to the submission directory containing both 'metadata/' and 'files/' directories [**Required**]
+
+Example usage:
+
+```bash
+grz_cli validate --submission-dir foo
+```
+
+Option is for the usage at a hospital (Leistungserbringer) and GDC/GRZ.
+
+### encrypt
+
+If a working directory is not provided, then the current directory is used automatically. The log-files are going to be stored in the sub-folder of the working directory.
+Files are stored in a folder named `encrypted_files` as a sub-folder of the working directory.
+
+- `-s, --submission-dir`: Path to the submission directory containing both 'metadata/' and 'files/' directories [**Required**]
+- `-c, --config-file`: Path to config file [_optional_]
+
+```bash
+grz-cli encrypt --submission-dir foo
+```
+
+Option is for the usage at a hospital (Leistungserbringer). Please approach your GDC/GRZ for a valid config file.
+
+### decrypt
+
+Decrypt a submission using the GRZ private key.
+
+- `-s, --submission-dir`: Path to the submission directory containing both 'metadata/' and 'encrypted_files/' directories  [**Required**]
+- `-c,--config-file`: Path to config file [_optional_]
+
+```bash
+grz-cli decrypt --submission-dir foo
+```
+
+Option is for the usage at a GDC/GRZ.
+
+### upload
+
+Upload the submission into a S3 structure of a GRZ.
+
+- `-s, --submission-dir`: Path to the submission directory containing both 'metadata/' and 'encrypted_files/' directories [**Required**]
+- `-c, --config-file`: Path to config file [_optional_]
+
+Example usage:
+
+```bash
+grz-cli upload --submission-dir foo
+```
+
+Option is for the usage at a hospital (Leistungserbringer). Please approach your GDC/GRZ for a valid config file.
+
+### download
+
+Download a submission from a GRZ
+
+- `-s, --submission-id`: S3 submission prefix [**Required**]
+- `-o, --output-dir`: Path to the target submission output directory [**Required**]
+- `-c, --config-file`: Path to config file [_optional_]
+
+Example usage:
+
+```bash
+grz-cli download --submission-id foo --output-dir bar
+```
+
+Option is for the usage at a GDC/GRZ.
+
+## Testing
+
+To run the tests, navigate to the root directory of your project and invoke `pytest`.
+Alternatively, install `uv` and `tox` and run `uv run tox`.
+
+## Contributing
+
+<!-- Add details about how others can contribute to the project -->
+
+## License
+
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
+
+## Acknowledgements
+
+Parts of `cryp4gh` code is used in modified form
+

grz_cli-0.1.0.dist-info/RECORD

@@ -0,0 +1,22 @@
+__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+grz_cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+grz_cli/cli.py,sha256=hByePV42Vc5QFjXEdbxQrEc0-ShZKqmRmxRVsZp9yks,8403
+grz_cli/constants.py,sha256=Tyc_eZNDJAdpFiok8V4N2XHX4W0SV0FcBCar1pTA7xw,1080
+grz_cli/download.py,sha256=1XkjhLdG0qCAORENBPD73NnXki5dffMSSXmfzXkc3_U,9001
+grz_cli/fastq_validation.py,sha256=4muUZMwB-HK5LuhfECJMY_R3OXDH0sBZi81-63uX_XI,3882
+grz_cli/file_operations.py,sha256=scbjpIoUIDcqeMXkSVAQlwZvaB6MK8oRmsrB7saQZyg,12356
+grz_cli/logging_setup.py,sha256=1x67mg9DATOkIRR0NDBBC_I2TICpZPF1Ntz1ImPMfZM,1817
+grz_cli/parser.py,sha256=RfmxySijdm53h21AYZJdr9G8Ag_1SvrH5FqsUaesxJ8,28343
+grz_cli/progress_logging.py,sha256=GMUsPPtLYSk8rlI0uLvVEOr4_nEe0cHiSqKlO84esWA,6393
+grz_cli/upload.py,sha256=OOvhMi-V3bV1bq0y7qgh_xJgTFujpa7VrrnguKf01m8,9074
+grz_cli/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+grz_cli/models/config.py,sha256=-ZH_R1kGMHwGGdP1trclmj9PXX73CIPJmeJ7uVyPMtM,2689
+grz_cli/models/v1_0_0/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+grz_cli/models/v1_0_0/metadata.py,sha256=k2wbO0A2Pupo0RwDyYJkP6GtQspF2pVvW7WAj8V-P08,22079
+grz_cli/resources/grz-schema.json,sha256=S3iaEdRXP_4ZaSxR1ivVPT4pYW7E5674YYRqtxqt6Bw,22104
+grz_cli-0.1.0.dist-info/LICENSE,sha256=3hMrOST0E1zpP4Y_uwI07jQdkCdcWMVdpkgewbx8cd4,1065
+grz_cli-0.1.0.dist-info/METADATA,sha256=trhOfr7j-tQeMWnGx-Uy60l9Wup2LRkLkzzXR0q4nTQ,8977
+grz_cli-0.1.0.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+grz_cli-0.1.0.dist-info/entry_points.txt,sha256=I60_QhAOC5qg07EKxqjlP5eSu_VWl-KEx-7Bh8BUPyk,44
+grz_cli-0.1.0.dist-info/top_level.txt,sha256=oVoiozAelI4VOGy9JLygMnPAkwNw4ry13cwuzMb8I-A,17
+grz_cli-0.1.0.dist-info/RECORD,,

grz_cli-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (75.6.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

grz_cli-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+grz-cli = grz_cli.cli:cli

grz_cli-0.1.0.dist-info/top_level.txt

@@ -0,0 +1,2 @@
+__init__
+grz_cli