divbase-lib 0.1.0a1__py3-none-any.whl

divbase_lib/api_schemas/vcf_dimensions.py

@@ -0,0 +1,67 @@
+"""
+Schemas for VCF dimensions routes.
+"""
+
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+
+class DimensionUpdateKwargs(BaseModel):
+    """Keyword arguments for dimension update task. Used to pass info to Celery task, and also for recording task history."""
+
+    bucket_name: str
+    project_id: int
+    project_name: str
+    user_id: int
+
+
+class DimensionUpdateTaskResult(BaseModel):
+    """Dimension update task result details. Based on the return of tasks.update_dimensions_index."""
+
+    status: Optional[str] = None
+    VCF_files_added: Optional[list[str]] = Field(
+        None, description="VCF files that were added to dimensions index by this job"
+    )
+    VCF_files_skipped: Optional[list[str]] = Field(
+        None, description="VCF files skipped by this job (previous DivBase-generated result VCFs)"
+    )
+    VCF_files_deleted: Optional[list[str]] = Field(
+        None, description="VCF files that have been deleted from the project and thus have been dropped from the index"
+    )
+
+
+class DimensionsShowResult(BaseModel):
+    """Result model for showing VCF dimensions for a project."""
+
+    project_id: int
+    project_name: str
+    vcf_file_count: int
+    vcf_files: list[dict]
+    skipped_file_count: int
+    skipped_files: list[dict]
+
+
+class DimensionsSamplesResult(BaseModel):
+    """Result model for showing unique samples across project VCFs."""
+
+    unique_samples: list[str]  # Already sorted, by the CRUD function get_unique_samples_by_project_async()
+
+
+class DimensionsScaffoldsResult(BaseModel):
+    """Result model for showing unique scaffolds across project VCFs."""
+
+    unique_scaffolds: list[str]  # Already sorted, by the CRUD function get_unique_scaffolds_by_project_async()
+
+
+class DimensionsVCFFileEntry(BaseModel):
+    """A single VCF file entry in the unique VCF files response."""
+
+    vcf_file_s3_key: str
+    s3_version_id: str
+
+
+class DimensionsVCFFilesResult(BaseModel):
+    """Result model for showing unique VCF file/version entries across project VCFs."""
+
+    unique_vcf_files: list[DimensionsVCFFileEntry]

divbase_lib/divbase_constants.py

@@ -0,0 +1,56 @@
+"""
+Constants that both divbase-api and divbase-cli need to agree on.
+"""
+
+ONE_MiB = 1024 * 1024
+
+# When you download a file that has been uploaded in parts, you have
+# to know the part/chunk size used in order to correctly calculate the composite checksum
+S3_MULTIPART_CHUNK_SIZE = 32 * ONE_MiB
+
+# At what point you swap from single part to multipart upload to S3.
+# If server and client used the same threshold then makes life easier
+# when validating the checksums of files in s3 as single part and multipart uploads use different ETag formats.
+# (No benefit in constraining the download threshold, so not done here)
+S3_MULTIPART_UPLOAD_THRESHOLD = 96 * ONE_MiB
+
+# Max number of items that can be processed in a single API call to divbase-api's S3 routes
+# covers e.g pre-signed urls for upload/download, soft delete and checksum comparisons
+# client has to batch requests if exceeding this limit
+MAX_S3_API_BATCH_SIZE = 100
+
+# How long the pre-signed URLs divbase-api creates are valid for
+SINGLE_PART_UPLOAD_URL_EXPIRATION_SECONDS = 3600  # 1 hour
+MULTI_PART_UPLOAD_URL_EXPIRATION_SECONDS = 36000  # 10 hours
+DOWNLOAD_URL_EXPIRATION_SECONDS = 36000  # 10 hours
+
+# (Not used anywhere, just making it explicit)
+# This is limited by our fixing of the chunk size and S3's limit to the number of chunks allowed (10,000)
+# 320 GiB if using 32 MiB chunks
+LARGEST_FILE_UPLOADABLE_TO_DIVBASE_BYTES = 10_000 * S3_MULTIPART_CHUNK_SIZE
+
+# File types that DivBase supports
+# Whilst we can't realistically limit what file types a user actually uploads,
+# this is here to say what we know should work in DivBase.
+SUPPORTED_DIVBASE_FILE_TYPES = (".tsv", ".vcf.gz", ".csi", ".tbi")
+
+# Characters that are not allowed in file names uploaded to DivBase
+# This is to prevent issues when users try to filter/query files on DivBase using these characters
+# or when downloading files (e.g. ":" is used to specify file versions when downloading files
+UNSUPPORTED_CHARACTERS_IN_FILENAMES = (":", "*", "?", "<", ">", "|", "\\")
+
+# This prefix is used for all *.vcf.gz results files from a query job/task.
+# After the prefix comes the job id which is a rolling integer.
+# E.g. format: result_of_job_<job-id>.vcf.gz , where <job-id> = 1 and is auto-incremented for every new job.
+# NOTE: If you update this, you should also update the S3 lifecycle policies associated with this.
+QUERY_RESULTS_FILE_PREFIX = "result_of_job_"
+
+# CLI includes this in every request to the server,
+# and server uses it to determine if it should:
+# 1. Reject the request as the version is too outdated
+# 2. Add an announcement about a new CLI version being available when the user logs in.
+# 3. Do Nothing as the user's CLI version is up to date.
+CLI_VERSION_HEADER_KEY = "X-CLI-Version"
+
+# Personal Access Tokens (PATs) are used for passwordless auth with DivBase API
+PAT_TOKEN_PREFIX = "divbase_pat_"

divbase_lib/exceptions.py

@@ -0,0 +1,156 @@
+"""
+Custom exceptions for DivBase packages.
+
+These are raised by lover-level functions/methods which understand the context of the error.
+
+Note: By adding the `__str__` method to each exception,
+we ensure that when you manually raise a specific exception the error message looks good
+"""
+
+from pathlib import Path
+
+
+class BcftoolsEnvironmentError(Exception):
+    """Raised when there's an issue with the execution environment (Docker, etc.)."""
+
+    def __init__(self, container_name: str):
+        self.container_name = container_name
+        error_message = (
+            f"No running container found with name {self.container_name}. Ensure the Docker image is available.\n"
+        )
+        super().__init__(error_message)
+
+        self.error_message = error_message
+
+    def __str__(self):
+        return self.error_message
+
+
+class BcftoolsCommandError(Exception):
+    """Raised when a bcftools command fails to execute properly."""
+
+    def __init__(self, command: str, error_details: Exception = None):
+        self.command = command
+        self.error_details = error_details
+
+        error_message = f"bcftools command failed: '{command}'"
+        if error_details:
+            error_message += f" with error details: {error_details}"
+
+        super().__init__(error_message)
+
+    def __str__(self):
+        if hasattr(self.error_details, "stderr") and self.error_details.stderr:
+            return f"bcftools command failed: '{self.command}' with error: {self.error_details.stderr}"
+        return super().__str__()
+
+
+class BcftoolsPipeEmptyCommandError(Exception):
+    """Raised when an empty command is provided to the bcftools pipe."""
+
+    def __init__(self):
+        error_message = "Empty command provided. Please specify at least one valid bcftools command."
+        super().__init__(error_message)
+        self.error_message = error_message
+
+    def __str__(self):
+        return self.error_message
+
+
+class BcftoolsPipeUnsupportedCommandError(Exception):
+    """Raised when a bcftools command unsupported by the BcftoolsQueryManager class is provided."""
+
+    def __init__(self, command: str, position: int, valid_commands: list[str]):
+        self.command = command
+        self.position = position
+        self.valid_commands = valid_commands
+
+        message = (
+            f"Unsupported bcftools command '{command}' at position {position}. "
+            f"Only the following commands are supported: {', '.join(valid_commands)}"
+        )
+        super().__init__(message)
+
+
+class SidecarNoDataLoadedError(Exception):
+    """Raised when no data is loaded in SidecarQueryManager."""
+
+    def __init__(self, file_path: Path, submethod: str, error_details: str | None = None):
+        self.file_path = file_path
+        self.error_details = error_details
+
+        error_message = f"No data loaded from file '{file_path}', as raised in submethod '{submethod}'."
+        if error_details:
+            error_message += f"More details about the error: {error_details}"
+        super().__init__(error_message)
+        self.error_message = error_message
+
+    def __str__(self):
+        return self.error_message
+
+
+class SidecarInvalidFilterError(Exception):
+    """Raised when an invalid filter is provided to SidecarQueryManager."""
+
+    pass
+
+
+class SidecarColumnNotFoundError(Exception):
+    """Raised when a requested column is not found in the query result."""
+
+    pass
+
+
+class SidecarSampleIDError(Exception):
+    """Raised when a Sample_ID is invalid or duplicated when loading sidecar data."""
+
+    pass
+
+
+class SidecarMetadataFormatError(Exception):
+    """Raised when the sidecar metadata TSV file has formatting issues (duplicate columns, empty columns, commas in values, etc.)."""
+
+    pass
+
+
+class TaskUserError(Exception):
+    """
+    Raised in Celery tasks when an error needs to propagate back to the CLI user.
+
+    This is intentionally kept as a simple Exception subclass (no custom __init__)
+    to avoid UnpicklableExceptionWrapper when passing through Celery's JSON serialization/deserialization.
+    Complex exception types such as those inheriting from DivBaseAPIException seem to trigger UnpicklableExceptionWrapper.
+
+    This class is essentially a wrapper to allow to use more complex exceptions in Celery tasks and catch them
+    in the API route handlers to return user-friendly error messages to the CLI.
+    In the Celery task use it like:
+        raise TaskUserError(str(SomeComplexError(...))) from None
+    """
+
+    pass
+
+
+class NoVCFFilesFoundError(Exception):
+    """Raised when no VCF files are found in the project bucket."""
+
+    pass
+
+
+class DimensionsNotUpToDateWithBucketError(Exception):
+    """Raised when VCF files in a bucket are missing or outdated in the dimensions index."""
+
+    pass
+
+
+class ChecksumVerificationError(Exception):
+    """Raised when a calculated file's checksum does not match the expected value."""
+
+    def __init__(self, expected_checksum: str, calculated_checksum: str):
+        self.expected_checksum = expected_checksum
+        self.calculated_checksum = calculated_checksum
+
+        message = (
+            f"Checksum verification failed. Expected: {expected_checksum}, Calculated: {calculated_checksum}"
+            f" The file has been deleted to avoid accidental use of a corrupted file."
+        )
+        super().__init__(message)