divbase-lib 0.1.0.dev0__tar.gz

divbase_lib-0.1.0.dev0/.gitignore

@@ -0,0 +1,38 @@
+# repo secrets 
+.env 
+.env*
+
+# python stuff 
+.venv
+.ruff_cache/
+__pycache__/
+*.py[cod]
+*$py.class
+
+.vscode/
+
+# project specific files
+/sample_metadata.tsv
+sample_metadata_*.tsv
+*.vcf
+*.vcf.gz
+*.vcf.gz.csi
+!tests/fixtures/*.vcf.gz
+tests/fixtures/temp*
+tests/fixtures/merged*
+
+# query job config files
+bcftools_divbase_job_config.json
+
+# benchmarking files
+vcf_dimensions.tsv
+mock*.tsv
+task_records*.json
+
+#MacOS artifacts
+.DS_Store
+
+# mkdocs build cache
+.cache/
+# pypi 
+dist/

divbase_lib-0.1.0.dev0/PKG-INFO

@@ -0,0 +1,37 @@
+Metadata-Version: 2.4
+Name: divbase-lib
+Version: 0.1.0.dev0
+Summary: Library module for Divbase
+Project-URL: Homepage, https://divbase.scilifelab.se
+Project-URL: Documentation, https://scilifelabdatacentre.github.io/divbase
+Project-URL: Repository, https://github.com/ScilifelabDataCentre/divbase
+Project-URL: Issues, https://github.com/ScilifelabDataCentre/divbase/issues
+Author-email: SciLifeLab Data Centre <datacentre@scilifelab.se>
+License-Expression: MIT
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Requires-Python: >=3.12
+Requires-Dist: boto3<2,>=1.42.27
+Requires-Dist: pandas<3,>=2.3.3
+Requires-Dist: pydantic<3,>=2.12.5
+Requires-Dist: pyyaml<7,>=6.0.3
+Description-Content-Type: text/markdown
+
+# divbase-lib
+
+This is library package for Divbase. Code from this library is shared between the Divbase CLI and API packages.
+
+## Installation and usage
+
+It is highly unlikely that you will need to use/install this package directly. Instead, it will be installed automatically by other packages (e.g. Divbase CLI) as a dependency.
+
+## Development
+
+This package is developed in the [DivBase repository](https://github.com/ScilifelabDataCentre/divbase) by Scilifelab Data Centre.

divbase_lib-0.1.0.dev0/README.md

@@ -0,0 +1,11 @@
+# divbase-lib
+
+This is library package for Divbase. Code from this library is shared between the Divbase CLI and API packages.
+
+## Installation and usage
+
+It is highly unlikely that you will need to use/install this package directly. Instead, it will be installed automatically by other packages (e.g. Divbase CLI) as a dependency.
+
+## Development
+
+This package is developed in the [DivBase repository](https://github.com/ScilifelabDataCentre/divbase) by Scilifelab Data Centre.

divbase_lib-0.1.0.dev0/pyproject.toml

@@ -0,0 +1,41 @@
+[project]
+name = "divbase-lib"
+description = "Library module for Divbase"
+readme = "README.md"
+license = "MIT"
+authors = [
+    { name = "SciLifeLab Data Centre", email = "datacentre@scilifelab.se" },
+]
+requires-python = ">=3.12"
+dependencies = [
+    "pyyaml>=6.0.3,<7",
+    "boto3>=1.42.27,<2",
+    "pandas>=2.3.3,<3",
+    "pydantic>=2.12.5,<3"
+]
+dynamic = ["version"]
+
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Environment :: Console",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+]
+
+[project.urls]
+Homepage = "https://divbase.scilifelab.se"
+Documentation = "https://scilifelabdatacentre.github.io/divbase"
+Repository = "https://github.com/ScilifelabDataCentre/divbase"
+Issues = "https://github.com/ScilifelabDataCentre/divbase/issues"
+
+[build-system]
+requires = ["hatchling >= 1.26"]
+build-backend = "hatchling.build"
+
+[tool.hatch.version]
+path = "src/divbase_lib/__init__.py"

divbase_lib-0.1.0.dev0/src/divbase_lib/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0.dev0"

divbase_lib-0.1.0.dev0/src/divbase_lib/api_schemas/auth.py

@@ -0,0 +1,34 @@
+"""
+Schemas for login + access and refresh tokens
+"""
+
+from pydantic import BaseModel, Field
+
+
+class CLILoginResponse(BaseModel):
+    """Response model for API (aka divbase-cli) login endpoint."""
+
+    access_token: str = Field(..., description="Bearer access token for authentication")
+    access_token_expires_at: int = Field(..., description="Unix timestamp when the access token expires")
+    refresh_token: str = Field(..., description="Bearer refresh token for obtaining new access tokens")
+    refresh_token_expires_at: int = Field(..., description="Unix timestamp when the refresh token expires")
+    email: str = Field(..., description="Email of the authenticated user")
+
+
+class RefreshTokenRequest(BaseModel):
+    """Request model for refresh token endpoint."""
+
+    refresh_token: str = Field(..., description="Bearer refresh token for obtaining a new access token")
+
+
+class RefreshTokenResponse(BaseModel):
+    """Response model for refresh token endpoint."""
+
+    access_token: str = Field(..., description="Bearer access token for authentication")
+    expires_at: int = Field(..., description="Unix timestamp when the access token expires")
+
+
+class LogoutRequest(BaseModel):
+    """Request model for logout endpoint."""
+
+    refresh_token: str = Field(..., description="Bearer refresh token to be revoked on logout")

divbase_lib-0.1.0.dev0/src/divbase_lib/api_schemas/project_versions.py

@@ -0,0 +1,62 @@
+"""
+Schemas for project versioning routes.
+
+Project versions are the state of all files in a project's storage bucket at a given time point.
+"""
+
+from pydantic import BaseModel, Field
+
+
+# Request Models
+class CreateVersioningFileRequest(BaseModel):
+    name: str = Field(..., description="Initial version name")
+    description: str = Field(..., description="Initial version description")
+
+
+class AddVersionRequest(BaseModel):
+    name: str = Field(..., description="Name of the new version to add")
+    description: str = Field("", description="Description of the new version")
+
+
+class DeleteVersionRequest(BaseModel):
+    version_name: str = Field(..., description="Name of the version to delete")
+
+
+# Response Models
+class ProjectBasicInfo(BaseModel):
+    """Base model for describing a single project version, not for direct use in an endpoint."""
+
+    name: str = Field(..., description="Version name")
+    description: str | None = Field(..., description="Version description")
+
+
+class AddVersionResponse(ProjectBasicInfo):
+    """Response model for adding a version."""
+
+    created_at: str = Field(..., description="ISO timestamp when version was created")
+
+
+class ProjectVersionInfo(ProjectBasicInfo):
+    """Basic information about a project version. You get a list of these when listing all versions in a project."""
+
+    created_at: str = Field(..., description="ISO timestamp when version was created")
+    is_deleted: bool = Field(..., description="Whether this version has been soft-deleted")
+
+
+class ProjectVersionDetailResponse(ProjectBasicInfo):
+    """Full information about a single project version, including the files at that version."""
+
+    created_at: str = Field(..., description="ISO timestamp when version was created")
+    is_deleted: bool = Field(..., description="Whether this version has been soft-deleted")
+    files: dict[str, str] = Field(..., description="Mapping of file names to their version IDs")
+
+
+class DeleteVersionResponse(BaseModel):
+    """Response model for deleting a version."""
+
+    name: str = Field(..., description="Name of the version that was deleted")
+    already_deleted: bool = Field(
+        False,
+        description="Whether the version was already soft-deleted before this request",
+    )
+    date_deleted: str = Field(..., description="ISO timestamp of when the version was soft-deleted")

divbase_lib-0.1.0.dev0/src/divbase_lib/api_schemas/queries.py

@@ -0,0 +1,64 @@
+"""
+Schemas for query routes.
+"""
+
+from typing import Any, Optional
+
+from pydantic import BaseModel
+
+
+# Request models
+class SampleMetadataQueryRequest(BaseModel):
+    """Request model for sample metadata query route."""
+
+    tsv_filter: str
+    metadata_tsv_name: str
+
+
+class BcftoolsQueryRequest(BaseModel):
+    """Request model for sample metadata query route."""
+
+    tsv_filter: str
+    metadata_tsv_name: str
+    command: str  # TODO add field to decribe that this is bcftools commands
+
+
+# Models for task kwargs and task results. Reused in task history schemas too, hence pydantic models and not just dataclasses.
+class SampleMetadataQueryKwargs(BaseModel):
+    """Keyword arguments for sample metadata query task. Used to pass info to Celery task, and also for recording task history."""
+
+    tsv_filter: str
+    metadata_tsv_name: str
+    bucket_name: str
+    project_id: int
+    project_name: str
+    user_id: int
+
+
+class BcftoolsQueryKwargs(BaseModel):
+    """Keyword arguments for BCFtools query task. Used to pass info to Celery task, and also for recording task history."""
+
+    tsv_filter: str
+    command: str
+    metadata_tsv_name: str
+    bucket_name: str
+    project_id: int
+    project_name: str
+    user_id: int
+
+
+class SampleMetadataQueryTaskResult(BaseModel):
+    """Metadata query task result details. Based on the return of tasks.sample_metadata_query."""
+
+    sample_and_filename_subset: list[dict[str, Any]]
+    unique_sample_ids: list[str]
+    unique_filenames: list[str]
+    query_message: str
+    status: Optional[str] = None
+
+
+class BcftoolsQueryTaskResult(BaseModel):
+    """BCFtools query task result details. Based on the return of tasks.bcftools_query."""
+
+    output_file: str
+    status: Optional[str] = None

divbase_lib-0.1.0.dev0/src/divbase_lib/api_schemas/s3.py

@@ -0,0 +1,51 @@
+"""
+Schemas for DivBase's S3 API routes.
+"""
+
+from pydantic import BaseModel, Field
+
+
+class DownloadObjectRequest(BaseModel):
+    """Request model to upload a single object using a pre-signed URL."""
+
+    name: str = Field(..., description="Name of the object to be downloaded")
+    version_id: str | None = Field(..., description="Version ID of the object, None if latest version")
+
+
+class PreSignedDownloadResponse(BaseModel):
+    """Response model to download a single object using the pre-signed URL and (optionally) version ID."""
+
+    name: str = Field(..., description="Name of the object to be downloaded")
+    pre_signed_url: str = Field(..., description="Pre-signed URL for downloading the object")
+    version_id: str | None = Field(..., description="Version ID of the object, None if latest version")
+
+
+class UploadObjectRequest(BaseModel):
+    """Request model to upload a single object using a pre-signed URL."""
+
+    name: str = Field(..., description="Name of the object to be uploaded")
+    content_length: int = Field(..., description="Size of the file in bytes")
+    md5_hash: str | None = Field(None, description="Optional MD5 hash of the object for integrity check")
+
+
+class PreSignedUploadResponse(BaseModel):
+    """Response model to upload a single object using the pre-signed URL using PUT."""
+
+    name: str = Field(..., description="Name of the object to be uploaded")
+    pre_signed_url: str = Field(..., description="Pre-signed URL to which the file should be uploaded")
+    put_headers: dict[str, str] = Field(..., description="Headers to be included in the PUT request")
+
+
+class CheckFileExistsRequest(BaseModel):
+    """Request model to check if a file already exists in the bucket (using the checksum)"""
+
+    object_name: str
+    md5_checksum: str
+
+
+class ExistingFileResponse(BaseModel):
+    """Response model for reporting a file that already exists in the bucket (using it's checksum)"""
+
+    object_name: str
+    md5_checksum: str
+    matching_object_name: str | None

divbase_lib-0.1.0.dev0/src/divbase_lib/api_schemas/task_history.py

@@ -0,0 +1,50 @@
+"""
+Schemas for task history routes.
+"""
+
+from typing import Any, Optional, Union
+
+from pydantic import BaseModel
+
+from divbase_lib.api_schemas.queries import (
+    BcftoolsQueryKwargs,
+    BcftoolsQueryTaskResult,
+    SampleMetadataQueryKwargs,
+    SampleMetadataQueryTaskResult,
+)
+from divbase_lib.api_schemas.vcf_dimensions import DimensionUpdateKwargs, DimensionUpdateTaskResult
+
+
+class TaskHistoryResult(BaseModel):
+    """
+    Task details as returned by queries to the SQAlchemy+pg results backend.
+    """
+
+    id: int
+    submitter_email: Optional[str] = None
+    status: Optional[str] = None
+    result: Optional[
+        Union[
+            dict[
+                str, Any
+            ],  # Note! This dict must come first here so that error results are preserved and not incorrectly inserted into the result models
+            SampleMetadataQueryTaskResult,
+            BcftoolsQueryTaskResult,
+            DimensionUpdateTaskResult,
+        ]
+    ] = None
+    date_done: Optional[str] = None
+    name: Optional[str] = None
+    args: Optional[str] = None
+    kwargs: Optional[
+        Union[
+            SampleMetadataQueryKwargs,
+            BcftoolsQueryKwargs,
+            DimensionUpdateKwargs,
+        ]
+    ] = None
+    worker: Optional[str] = None
+    created_at: Optional[str] = None
+    started_at: Optional[str] = None
+    completed_at: Optional[str] = None
+    runtime: Optional[float] = None

divbase_lib-0.1.0.dev0/src/divbase_lib/api_schemas/vcf_dimensions.py

@@ -0,0 +1,42 @@
+"""
+Schemas for VCF dimensions routes.
+"""
+
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+
+class DimensionUpdateKwargs(BaseModel):
+    """Keyword arguments for dimension update task. Used to pass info to Celery task, and also for recording task history."""
+
+    bucket_name: str
+    project_id: int
+    project_name: str
+    user_id: int
+
+
+class DimensionUpdateTaskResult(BaseModel):
+    """Dimension update task result details. Based on the return of tasks.update_dimensions_index."""
+
+    status: Optional[str] = None
+    VCF_files_added: Optional[list[str]] = Field(
+        None, description="VCF files that were added to dimensions index by this job"
+    )
+    VCF_files_skipped: Optional[list[str]] = Field(
+        None, description="VCF files skipped by this job (previous DivBase-generated result VCFs)"
+    )
+    VCF_files_deleted: Optional[list[str]] = Field(
+        None, description="VCF files that have been deleted from the project and thus have been dropped from the index"
+    )
+
+
+class DimensionsShowResult(BaseModel):
+    """Result model for showing VCF dimensions for a project."""
+
+    project_id: int
+    project_name: str
+    vcf_file_count: int
+    vcf_files: list[dict]
+    skipped_file_count: int
+    skipped_files: list[dict]

divbase_lib-0.1.0.dev0/src/divbase_lib/exceptions.py

@@ -0,0 +1,132 @@
+"""
+Custom exceptions for DivBase packages.
+
+These are raised by lover-level functions/methods which understand the context of the error.
+
+Note: By adding the `__str__` method to each exception,
+we ensure that when you manually raise a specific exception the error message looks good
+"""
+
+from pathlib import Path
+
+
+class ObjectDoesNotExistError(FileNotFoundError):
+    """Raised when an S3 object/key does not exist in the bucket."""
+
+    def __init__(self, key: str, bucket_name: str):
+        error_message = f"The file/object '{key}' does not exist in the bucket '{bucket_name}'. "
+        super().__init__(error_message)
+        self.key = key
+        self.bucket = bucket_name
+        self.error_message = error_message
+
+    def __str__(self):
+        return self.error_message
+
+
+class BcftoolsEnvironmentError(Exception):
+    """Raised when there's an issue with the execution environment (Docker, etc.)."""
+
+    def __init__(self, container_name: str):
+        self.container_name = container_name
+        error_message = (
+            f"No running container found with name {self.container_name}. Ensure the Docker image is available.\n"
+        )
+        super().__init__(error_message)
+
+        self.error_message = error_message
+
+    def __str__(self):
+        return self.error_message
+
+
+class BcftoolsCommandError(Exception):
+    """Raised when a bcftools command fails to execute properly."""
+
+    def __init__(self, command: str, error_details: Exception = None):
+        self.command = command
+        self.error_details = error_details
+
+        error_message = f"bcftools command failed: '{command}'"
+        if error_details:
+            error_message += f" with error details: {error_details}"
+
+        super().__init__(error_message)
+
+    def __str__(self):
+        if hasattr(self.error_details, "stderr") and self.error_details.stderr:
+            return f"bcftools command failed: '{self.command}' with error: {self.error_details.stderr}"
+        return super().__str__()
+
+
+class BcftoolsPipeEmptyCommandError(Exception):
+    """Raised when an empty command is provided to the bcftools pipe."""
+
+    def __init__(self):
+        error_message = "Empty command provided. Please specify at least one valid bcftools command."
+        super().__init__(error_message)
+        self.error_message = error_message
+
+    def __str__(self):
+        return self.error_message
+
+
+class BcftoolsPipeUnsupportedCommandError(Exception):
+    """Raised when a bcftools command unsupported by the BcftoolsQueryManager class is provided."""
+
+    def __init__(self, command: str, position: int, valid_commands: list[str]):
+        self.command = command
+        self.position = position
+        self.valid_commands = valid_commands
+
+        message = (
+            f"Unsupported bcftools command '{command}' at position {position}. "
+            f"Only the following commands are supported: {', '.join(valid_commands)}"
+        )
+        super().__init__(message)
+
+
+class SidecarNoDataLoadedError(Exception):
+    """Raised when no data is loaded in SidecarQueryManager."""
+
+    def __init__(self, file_path: Path, submethod: str, error_details: str | None = None):
+        self.file_path = file_path
+        self.error_details = error_details
+
+        error_message = f"No data loaded from file '{file_path}', as raised in submethod '{submethod}'."
+        if error_details:
+            error_message += f"More details about the error: {error_details}"
+        super().__init__(error_message)
+        self.error_message = error_message
+
+    def __str__(self):
+        return self.error_message
+
+
+class SidecarInvalidFilterError(Exception):
+    """Raised when an invalid filter is provided to SidecarQueryManager."""
+
+    pass
+
+
+class SidecarColumnNotFoundError(Exception):
+    """Raised when a requested column is not found in the query result."""
+
+    pass
+
+
+class NoVCFFilesFoundError(Exception):
+    """Raised when no VCF files are found in the project bucket."""
+
+    pass
+
+
+class ChecksumVerificationError(Exception):
+    """Raised when a calculated file's checksum does not match the expected value."""
+
+    def __init__(self, expected_checksum: str, calculated_checksum: str):
+        self.expected_checksum = expected_checksum
+        self.calculated_checksum = calculated_checksum
+
+        message = f"Checksum verification failed. Expected: {expected_checksum}, Calculated: {calculated_checksum}"
+        super().__init__(message)

divbase_lib-0.1.0.dev0/src/divbase_lib/s3_checksums.py

@@ -0,0 +1,66 @@
+"""
+Manages creation/validation of S3 object checksums for both file uploads and downloads.
+
+We do not support multipart uploads/downloads at this time.
+Single part uploads/downloads have a limit of 5GBs.
+Docs: https://docs.netapp.com/us-en/storagegrid/s3/put-object.html
+"""
+
+import base64
+import hashlib
+import logging
+from enum import StrEnum
+from pathlib import Path
+from typing import Iterator
+
+from divbase_lib.exceptions import ChecksumVerificationError
+
+logger = logging.getLogger(__name__)
+
+
+def _read_file_chunks(file_path: Path, chunk_size: int = 8192) -> Iterator[bytes]:
+    """Helper function to read a file in 'chunk_size' sized chunks."""
+
+    with file_path.open(mode="rb") as infile:
+        yield from iter(lambda: infile.read(chunk_size), b"")
+
+
+class MD5CheckSumFormat(StrEnum):
+    HEX = "hex"
+    BASE64 = "base64"
+
+
+def calculate_md5_checksum(file_path: Path, output_format: MD5CheckSumFormat) -> str:
+    """
+    Calculate the MD5 checksum of a file.
+    Returns the checksum in either hex-encoded (lowercase) or base64-encoded format.
+    """
+    md5_hash = hashlib.md5()
+
+    for chunk in _read_file_chunks(file_path):
+        md5_hash.update(chunk)
+
+    if output_format == MD5CheckSumFormat.HEX:
+        return md5_hash.hexdigest()
+    elif output_format == MD5CheckSumFormat.BASE64:
+        return base64.b64encode(md5_hash.digest()).decode("utf-8")
+    else:
+        raise ValueError(f"Unknown output format: {output_format}")
+
+
+def verify_downloaded_checksum(file_path: Path, expected_checksum: str) -> None:
+    """
+    Verify a downloaded file against S3's ETag (MD5 checksum in hex format).
+    """
+    calculated_md5 = calculate_md5_checksum(file_path=file_path, output_format=MD5CheckSumFormat.HEX)
+    if calculated_md5 != expected_checksum:
+        raise ChecksumVerificationError(expected_checksum=expected_checksum, calculated_checksum=calculated_md5)
+
+
+def convert_checksum_hex_to_base64(hex_checksum: str) -> str:
+    """
+    Convert a hex-encoded MD5 checksum to base64-encoded format.
+    """
+    raw_bytes = bytes.fromhex(hex_checksum)
+    base64_checksum = base64.b64encode(raw_bytes).decode("utf-8")
+    return base64_checksum