grz-common 0.1.0__tar.gz

grz_common-0.1.0/.gitignore

@@ -0,0 +1,195 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+#poetry.toml
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Abstra
+# Abstra is an AI-powered process automation framework.
+# Ignore directories containing user credentials, local state, and settings.
+# Learn more at https://abstra.io/docs
+.abstra/
+
+# Visual Studio Code
+#  Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore 
+#  that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#  and can be added to the global gitignore or merged into this file. However, if you prefer, 
+#  you could uncomment the following to ignore the entire vscode folder
+# .vscode/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Cursor
+#  Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to
+#  exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
+#  refer to https://docs.cursor.com/context/ignore-files
+.cursorignore
+.cursorindexingignore

grz_common-0.1.0/CHANGELOG.md

@@ -0,0 +1,8 @@
+# Changelog
+
+## 0.1.0 (2025-06-11)
+
+
+### Features
+
+* migrate to monorepo configuration ([36c7360](https://github.com/BfArM-MVH/grz-tools/commit/36c736044ce09473cc664b4471117465c5cab9a3))

grz_common-0.1.0/PKG-INFO

@@ -0,0 +1,30 @@
+Metadata-Version: 2.4
+Name: grz-common
+Version: 0.1.0
+Summary: Common library for grz applications
+Project-URL: Homepage, https://github.com/BfArM-MVH/grz-tools
+Project-URL: Repository, https://github.com/BfArM-MVH/grz-tools
+Project-URL: Documentation, https://github.com/BfArM-MVH/grz-tools/tree/main/packages/grz-common
+Project-URL: Issues, https://github.com/BfArM-MVH/grz-tools/issues
+Author-email: Koray Kirli <koraykirli@gmail.com>, Mathias Lesche <mathias.lesche@tu-dresden.de>, "Florian R. Hölzlwimmer" <git.ich@frhoelzlwimmer.de>, Till Hartmann <till.hartmann@bih-charite.de>, Thomas Sell <thomas.sell@bih-charite.de>, Travis Wrightsman <travis.wrightsman@uni-tuebingen.de>
+License-Expression: MIT
+Keywords: GDC,GRZ,S3
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.12
+Requires-Dist: boto3<2,>=1.36
+Requires-Dist: click<9,>=8.1.7
+Requires-Dist: crypt4gh<2,>=1.7
+Requires-Dist: grz-pydantic-models>=1.3.0
+Requires-Dist: jsonschema<5,>=4.23.0
+Requires-Dist: platformdirs<5,>=4.3.6
+Requires-Dist: pydantic-settings<2.10,>=2.9.0
+Requires-Dist: pydantic<2.10,>=2.9.2
+Requires-Dist: pysam==0.23.*
+Requires-Dist: pyyaml<7,>=6.0.2
+Requires-Dist: requests<3,>=2.32.3
+Requires-Dist: rich==13.*
+Requires-Dist: tqdm<5,>=4.66.5
+Description-Content-Type: text/markdown
+
+Common code shared between packages in `grz-tools` (mainly `grz-cli` and `grzctl`).

grz_common-0.1.0/README.md

@@ -0,0 +1 @@
+Common code shared between packages in `grz-tools` (mainly `grz-cli` and `grzctl`).

grz_common-0.1.0/pyproject.toml

@@ -0,0 +1,51 @@
+[project]
+name = "grz-common"
+description = "Common library for grz applications"
+readme = "README.md"
+license = "MIT"
+authors = [
+    { name = "Koray Kirli", email = "koraykirli@gmail.com" },
+    { name = "Mathias Lesche", email = "mathias.lesche@tu-dresden.de" },
+    { name = "Florian R. Hölzlwimmer", email = "git.ich@frhoelzlwimmer.de" },
+    { name = "Till Hartmann", email = "till.hartmann@bih-charite.de" },
+    { name = "Thomas Sell", email = "thomas.sell@bih-charite.de" },
+    { name = "Travis Wrightsman", email = "travis.wrightsman@uni-tuebingen.de" }
+]
+dynamic = ["version"]
+dependencies = [
+    "boto3 >=1.36,<2",
+    "click >=8.1.7,<9",
+    "crypt4gh >=1.7,<2",
+    "jsonschema >=4.23.0,<5",
+    "pyyaml >=6.0.2,<7",
+    "tqdm >=4.66.5,<5",
+    "pydantic >=2.9.2,<2.10",
+    "pydantic-settings >=2.9.0,<2.10",
+    "platformdirs >=4.3.6,<5",
+    "grz-pydantic-models >=1.3.0",
+    "pysam ==0.23.*",
+    "rich ==13.*",
+    "requests >=2.32.3,<3",
+]
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Operating System :: OS Independent"
+]
+requires-python = ">=3.12"
+keywords = ["GRZ", "GDC", "S3"]
+
+[project.urls]
+Homepage = "https://github.com/BfArM-MVH/grz-tools"
+Repository = "https://github.com/BfArM-MVH/grz-tools"
+Documentation = "https://github.com/BfArM-MVH/grz-tools/tree/main/packages/grz-common"
+Issues = "https://github.com/BfArM-MVH/grz-tools/issues"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/grz_common"]
+
+[tool.hatch.version]
+path = "src/grz_common/__init__.py"

grz_common-0.1.0/src/grz_common/__init__.py

@@ -0,0 +1,5 @@
+"""
+Common library for grz applications.
+"""
+
+__version__ = "0.1.0"

grz_common-0.1.0/src/grz_common/cli/__init__.py

@@ -0,0 +1,79 @@
+"""
+Common click options for the CLI commands.
+"""
+
+from os import sched_getaffinity
+from pathlib import Path
+
+import click
+import platformdirs
+
+DEFAULT_CONFIG_PATH = Path(platformdirs.user_config_dir("grz-cli")) / "config.yaml"
+
+# Aliases for path types for click options
+# Naming convention: {DIR,FILE}_{Read,Write}_{Exists,Create}
+DIR_R_E = click.Path(
+    exists=True,
+    file_okay=False,
+    dir_okay=True,
+    readable=True,
+    writable=False,
+    resolve_path=True,
+)
+DIR_RW_C = click.Path(
+    exists=False,
+    file_okay=False,
+    dir_okay=True,
+    readable=True,
+    writable=True,
+    resolve_path=True,
+)
+FILE_R_E = click.Path(exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True)
+
+submission_dir = click.option(
+    "--submission-dir",
+    metavar="PATH",
+    type=DIR_R_E,
+    required=True,
+    help="Path to the submission directory containing 'metadata/', 'files/', 'encrypted_files/' and 'logs/' directories",
+)
+
+config_file = click.option(
+    "--config-file",
+    metavar="STRING",
+    type=FILE_R_E,
+    required=False,
+    default=DEFAULT_CONFIG_PATH,
+    help="Path to config file",
+)
+
+threads = click.option(
+    "--threads",
+    default=min(len(sched_getaffinity(0)), 4),
+    type=int,
+    show_default=True,
+    help="Number of threads to use for parallel operations",
+)
+
+submission_id = click.option(
+    "--submission-id",
+    required=True,
+    type=str,
+    metavar="STRING",
+    help="S3 submission ID",
+)
+
+output_dir = click.option(
+    "--output-dir",
+    metavar="PATH",
+    type=DIR_RW_C,
+    required=True,
+    default=None,
+    help="Path to the target submission output directory",
+)
+
+output_json = click.option("--json", "output_json", is_flag=True, help="Output JSON for machine-readability.")
+
+show_details = click.option("--details", "show_details", is_flag=True, help="Show more detailed output.")
+
+force = click.option("--force/--no-force", help="Overwrite files and ignore cached results (dangerous!)")

grz_common-0.1.0/src/grz_common/constants.py

@@ -0,0 +1,39 @@
+"""Constants for logging configuration, JSON schema validation, and other settings."""
+
+# This will be overridden by the package that imports it
+PACKAGE_ROOT = "grz_common"
+
+LOGGING_FORMAT = "%(asctime)s [%(levelname)s] %(name)s: %(message)s"
+LOGGING_DATEFMT = "%Y-%m-%d %I:%M %p"
+LOGGING_CONFIG = {
+    "version": 1,
+    "disable_existing_loggers": True,
+    "formatters": {
+        "standard": {
+            "format": LOGGING_FORMAT,
+            "datefmt": LOGGING_DATEFMT,
+        },
+    },
+    "handlers": {
+        "default": {
+            "level": "INFO",
+            "formatter": "standard",
+            "class": "logging.StreamHandler",
+        },
+    },
+    "loggers": {
+        "": {  # root logger
+            "handlers": ["default"],
+            "level": "WARNING",
+            "propagate": False,
+        },
+        PACKAGE_ROOT: {"handlers": ["default"], "level": "INFO", "propagate": False},
+        "__main__": {  # if __name__ == '__main__'
+            "handlers": ["default"],
+            "level": "DEBUG",
+            "propagate": False,
+        },
+    },
+}
+
+TQDM_SMOOTHING: float = 0.0001

grz_common-0.1.0/src/grz_common/logging_setup.py

@@ -0,0 +1,60 @@
+"""
+Module: logging_setup
+
+This module provides functions for setting up logging configuration.
+"""
+
+from __future__ import annotations
+
+import logging
+from os import PathLike
+from pathlib import Path
+
+from .constants import LOGGING_DATEFMT, LOGGING_FORMAT, PACKAGE_ROOT
+
+log = logging.getLogger(__name__)
+
+
+def add_filelogger(
+    file_path: str | PathLike | None = None,
+    level: str = "INFO",
+    logger_name: str = PACKAGE_ROOT,
+) -> None:
+    """
+    Add file logging for the specified package.
+
+    This function configures a file logger to capture log messages
+    for the package specified by logger_name. If no file path
+    is provided, a default log file will be created in the user's
+    home directory.
+
+    :param file_path: Optional; the path to the log file. If None,
+                      a default path will be used.
+    :param level: Optional; the logging level. Default is 'INFO'.
+                  Must be a valid logging level name (e.g., 'DEBUG', 'INFO').
+    :param logger_name: Optional; the name of the logger to add the file handler to.
+                        Default is PACKAGE_ROOT.
+    """
+    logger = logging.getLogger(logger_name)
+
+    if file_path is None:
+        default_log_dir = Path.home() / "logs"
+        default_log_dir.mkdir(parents=True, exist_ok=True)
+        file_path = default_log_dir / f"{logger_name}.log"
+        log.warning("No log file path provided, using default: %s", file_path)
+    else:
+        file_path = Path(file_path)
+
+    try:
+        fh = logging.FileHandler(file_path)
+        fh.setLevel(level.upper())
+        fh.setFormatter(logging.Formatter(LOGGING_FORMAT, LOGGING_DATEFMT))
+        logger.addHandler(fh)
+        log.info(
+            "File logger added for %s at %s with level %s.",
+            logger.name,
+            file_path,
+            level.upper(),
+        )
+    except Exception as e:
+        log.error("Failed to add file logger: %s", e)

grz_common-0.1.0/src/grz_common/models/__init__.py

@@ -0,0 +1,3 @@
+"""
+Model classes for grz applications.
+"""

grz_common-0.1.0/src/grz_common/models/base.py

@@ -0,0 +1,59 @@
+from os import PathLike
+from pathlib import Path
+from typing import Annotated, Self
+
+from pydantic import AfterValidator, BaseModel, ConfigDict
+from pydantic.types import PathType
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+FilePath = Annotated[Path, AfterValidator(lambda v: v.expanduser()), PathType("file")]
+
+
+class IgnoringBaseModel(BaseModel):
+    model_config = ConfigDict(
+        extra="ignore",
+        validate_assignment=True,
+        use_enum_values=True,
+    )
+
+    def to_yaml(self, fd):
+        """Reads the configuration file and validates it against the schema."""
+        import yaml
+
+        yaml.dump(self.model_dump(mode="json", exclude_none=True, exclude_unset=True, exclude_defaults=True))
+
+    @classmethod
+    def from_path(cls, path: str | PathLike) -> Self:
+        """Reads the configuration file and validates it against the schema."""
+        import yaml
+
+        with open(path, encoding="utf-8") as f:
+            config = cls(**yaml.safe_load(f))  # noqa:
+
+        return config
+
+
+class IgnoringBaseSettings(BaseSettings):
+    model_config = SettingsConfigDict(
+        extra="ignore",
+        validate_assignment=True,
+        use_enum_values=True,
+        env_nested_delimiter="__",
+        env_prefix="grz_",
+    )
+
+    def to_yaml(self, fd):
+        """Reads the configuration file and validates it against the schema."""
+        import yaml
+
+        yaml.dump(self.model_dump(mode="json", exclude_none=True, exclude_unset=True, exclude_defaults=True), fd)
+
+    @classmethod
+    def from_path(cls, path: str | PathLike) -> Self:
+        """Reads the configuration file and validates it against the schema."""
+        import yaml
+
+        with open(path, encoding="utf-8") as f:
+            config = cls(**yaml.safe_load(f))  # noqa:
+
+        return config

grz_common-0.1.0/src/grz_common/models/keys.py

@@ -0,0 +1,45 @@
+from typing import Self
+
+from grz_common.models.base import FilePath, IgnoringBaseSettings
+from pydantic import field_validator, model_validator
+
+
+class KeyModel(IgnoringBaseSettings):
+    grz_public_key: str | None = None
+    """
+    The public key of the recipient (the associated GRZ).
+    """
+
+    grz_public_key_path: FilePath | None = None
+    """
+    Path to the crypt4gh public key of the recipient (the associated GRZ).
+    """
+
+    grz_private_key_path: FilePath | None = None
+    """
+    Path to the crypt4gh private key of the recipient (optional).
+    """
+
+    submitter_private_key_path: FilePath | None = None
+    """
+    Path to the submitter's private key (optional).
+    """
+
+    @field_validator("grz_public_key")
+    @classmethod
+    def check_grz_public_key(cls, v):
+        if v is not None and "BEGIN CRYPT4GH PUBLIC KEY" not in v and "END CRYPT4GH PUBLIC KEY" not in v:
+            raise ValueError("Invalid public key format")
+        return v
+
+    @model_validator(mode="after")
+    def validate_grz_public_key(self) -> Self:
+        if self.grz_public_key is None and self.grz_public_key_path is None:
+            raise ValueError("Either grz_public_key or grz_public_key_path must be set.")
+        if self.grz_public_key is not None and self.grz_public_key_path is not None:
+            raise ValueError("Only one of grz_public_key or grz_public_key_path must be set.")
+        return self
+
+
+class KeyConfigModel(IgnoringBaseSettings):
+    keys: KeyModel

grz_common-0.1.0/src/grz_common/models/s3.py

@@ -0,0 +1,79 @@
+from pydantic import AnyHttpUrl, AnyUrl
+
+from .base import IgnoringBaseModel, IgnoringBaseSettings
+
+
+class S3Options(IgnoringBaseModel):
+    endpoint_url: AnyHttpUrl
+    """
+    The URL for the S3 service.
+    """
+
+    bucket: str
+    """
+    The name of the S3 bucket.
+    """
+
+    access_key: str | None = None
+    """
+    The access key for the S3 bucket.
+    If undefined, it is read from the AWS_ACCESS_KEY_ID environment variable.
+    """
+
+    secret: str | None = None
+    """
+    The secret key for the S3 bucket.
+    If undefined, it is read from the AWS_SECRET_ACCESS_KEY environment variable.
+    """
+
+    session_token: str | None = None
+    """
+    The session token for temporary credentials (optional).
+    """
+
+    region_name: str | None = None
+    """
+    The region name for the S3 bucket.
+    """
+
+    api_version: str | None = None
+    """
+    The S3 API version.
+    """
+
+    use_ssl: bool = True
+    """
+    Whether to use SSL for S3 operations.
+    """
+
+    proxy_url: AnyUrl | None = None
+    """
+    The proxy URL for S3 operations (optional).
+    """
+
+    request_checksum_calculation: str | None = None
+    """
+    Whether to calculate checksums for S3 request payloads (optional).
+    Valid values are:
+
+    * ``when_supported`` -- When set, a checksum will be calculated for
+      all request payloads of operations modeled with the ``httpChecksum``
+      trait where ``requestChecksumRequired`` is ``true`` or a
+      ``requestAlgorithmMember`` is modeled.
+
+    * ``when_required`` -- When set, a checksum will only be calculated
+      for request payloads of operations modeled with the ``httpChecksum``
+      trait where ``requestChecksumRequired`` is ``true`` or where a
+      ``requestAlgorithmMember`` is modeled and supplied.
+
+    Defaults to None.
+    """
+
+    multipart_chunksize: int = 256 * 1024**2
+    """
+    The size of the chunks to use for multipart uploads in bytes.
+    """
+
+
+class S3ConfigModel(IgnoringBaseSettings):
+    s3: S3Options

grz_common-0.1.0/src/grz_common/progress/__init__.py

@@ -0,0 +1,21 @@
+"""Progress tracking and logging functionality."""
+
+from .progress_logging import FileProgressLogger
+from .states import (
+    DecryptionState,
+    DownloadState,
+    EncryptionState,
+    State,
+    UploadState,
+    ValidationState,
+)
+
+__all__ = [
+    "DecryptionState",
+    "DownloadState",
+    "EncryptionState",
+    "FileProgressLogger",
+    "State",
+    "UploadState",
+    "ValidationState",
+]