divbase-lib 0.1.0.dev2__py3-none-any.whl → 0.1.0.dev5__py3-none-any.whl

divbase_lib/__init__.py

@@ -1 +1 @@
-__version__ = "0.1.0.dev2"
+__version__ = "0.1.0.dev5"

divbase_lib/api_schemas/announcements.py

@@ -0,0 +1,18 @@
+"""
+Schemas for announcements.
+"""
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class AnnouncementResponse(BaseModel):
+    """Response model for an announcement returned by the API."""
+
+    heading: str = Field(..., description="Title of the announcement.")
+    message: str | None = Field(None, description="Detailed message of the announcement.")
+    level: str = Field(
+        ...,
+        description="The announcement level, which can control styling of announcement. Possible values are: info, success, warning, danger.",
+    )
+
+    model_config = ConfigDict(from_attributes=True)

divbase_lib/api_schemas/project_versions.py

@@ -4,6 +4,8 @@ Schemas for project versioning routes.
 Project versions are the state of all files in a project's storage bucket at a given time point.
 """
 
+from typing import TypedDict
+
 from pydantic import BaseModel, Field
 
 
@@ -22,6 +24,12 @@ class DeleteVersionRequest(BaseModel):
     version_name: str = Field(..., description="Name of the version to delete")
 
 
+class UpdateVersionRequest(BaseModel):
+    version_name: str = Field(..., description="Name of the version to update")
+    new_name: str | None = Field(None, description="New name for the version")
+    new_description: str | None = Field(None, description="New description for the version")
+
+
 # Response Models
 class ProjectBasicInfo(BaseModel):
     """Base model for describing a single project version, not for direct use in an endpoint."""
@@ -43,12 +51,23 @@ class ProjectVersionInfo(ProjectBasicInfo):
     is_deleted: bool = Field(..., description="Whether this version has been soft-deleted")
 
 
+# Typed dict used as this is used in the db model to define the contents of the JSONB
+class FileDetails(TypedDict):
+    """Details about a single file in a project version, ETag (is checksum) and size is in bytes."""
+
+    version_id: str
+    etag: str
+    size: int
+
+
 class ProjectVersionDetailResponse(ProjectBasicInfo):
     """Full information about a single project version, including the files at that version."""
 
     created_at: str = Field(..., description="ISO timestamp when version was created")
     is_deleted: bool = Field(..., description="Whether this version has been soft-deleted")
-    files: dict[str, str] = Field(..., description="Mapping of file names to their version IDs")
+    files: dict[str, FileDetails] = Field(
+        ..., description="Info about all files at this version, including: version IDs, etags and file sizes"
+    )
 
 
 class DeleteVersionResponse(BaseModel):
@@ -60,3 +79,9 @@ class DeleteVersionResponse(BaseModel):
         description="Whether the version was already soft-deleted before this request",
     )
     date_deleted: str = Field(..., description="ISO timestamp of when the version was soft-deleted")
+
+
+class UpdateVersionResponse(ProjectBasicInfo):
+    """Response model for updating a version."""
+
+    created_at: str = Field(..., description="ISO timestamp when version was created")

divbase_lib/api_schemas/queries.py

@@ -45,6 +45,7 @@ class BcftoolsQueryKwargs(BaseModel):
     project_id: int
     project_name: str
     user_id: int
+    job_id: int
 
 
 class SampleMetadataQueryTaskResult(BaseModel):
@@ -54,6 +55,7 @@ class SampleMetadataQueryTaskResult(BaseModel):
     unique_sample_ids: list[str]
     unique_filenames: list[str]
     query_message: str
+    warnings: list[str] = []
     status: Optional[str] = None
 
 

divbase_lib/api_schemas/s3.py

@@ -1,12 +1,81 @@
 """
 Schemas for DivBase's S3 API routes.
+
+Pre-signed download URLs do not need to account for single vs multipart as this can be controlled by the client
+using the HTTP range header when downloading (so you only need 1 pre-signed URL per object for download).
+
+Pre-signed upload URLs need to account for single vs multipart uploads hence all the extra schemas below.
 """
 
+from datetime import datetime
+
 from pydantic import BaseModel, Field
 
+from divbase_lib.divbase_constants import S3_MULTIPART_CHUNK_SIZE
+
+MB = 1024 * 1024
+
+
+## list objects models ##
+class ListObjectsRequest(BaseModel):
+    """Request model for listing objects in an S3 bucket."""
+
+    prefix: str | None = Field(None, description="Optional prefix to filter objects by name.")
+    next_token: str | None = Field(
+        None, description="Token to continue listing files from the end of a previous request."
+    )
+
+
+class ObjectDetails(BaseModel):
+    """Details about a single object in an S3 bucket."""
+
+    name: str = Field(..., description="The name of the object in the bucket.")
+    size: int = Field(..., description="The size of the object in bytes.")
+    last_modified: datetime = Field(..., description="The date and time the object was last modified.")
+    etag: str = Field(..., description="The ETag of the object, which is the MD5 checksum.")
+
+
+class ListObjectsResponse(BaseModel):
+    """Response model for listing objects in an S3 bucket."""
+
+    objects: list[ObjectDetails] = Field(
+        ..., description="A list of objects in the bucket.", min_length=0, max_length=1000
+    )
+    next_token: str | None = Field(
+        None, description="Token for fetching the next page of results. If None, no more results."
+    )
+
 
+## list soft-deleted objects models ##
+class SoftDeletedObjectDetails(BaseModel):
+    """Details about a single soft-deleted object in an S3 bucket."""
+
+    name: str = Field(..., description="The name of the object in the bucket.")
+    last_modified: datetime = Field(..., description="The date and time the object was deleted.")
+
+
+## file info models ##
+class ObjectVersionInfo(BaseModel):
+    """Detailed information about a single version of an S3 object."""
+
+    version_id: str = Field(..., description="The version ID of the object.")
+    last_modified: datetime = Field(..., description="The date and time the object version was last modified.")
+    size: int = Field(..., description="The size of the object in bytes.")
+    etag: str = Field(..., description="The ETag of the object, which is the MD5 checksum.")
+    is_latest: bool = Field(..., description="Indicates if this is the latest version of the object.")
+
+
+class ObjectInfoResponse(BaseModel):
+    """Response model for detailed information about all versions of a single object stored in S3."""
+
+    object_name: str = Field(..., description="The name of the object.")
+    is_currently_deleted: bool = Field(..., description="True if the latest version of the object is a delete marker.")
+    versions: list[ObjectVersionInfo] = Field(..., description="A list of all versions of the object.")
+
+
+## download models ##
 class DownloadObjectRequest(BaseModel):
-    """Request model to upload a single object using a pre-signed URL."""
+    """Request model to download a single object using a pre-signed URL."""
 
     name: str = Field(..., description="Name of the object to be downloaded")
     version_id: str | None = Field(..., description="Version ID of the object, None if latest version")
@@ -20,32 +89,126 @@ class PreSignedDownloadResponse(BaseModel):
     version_id: str | None = Field(..., description="Version ID of the object, None if latest version")
 
 
-class UploadObjectRequest(BaseModel):
-    """Request model to upload a single object using a pre-signed URL."""
+### Single-part upload models ###
+class UploadSinglePartObjectRequest(BaseModel):
+    """Request model to upload a single object as a single part using a pre-signed URL."""
 
     name: str = Field(..., description="Name of the object to be uploaded")
     content_length: int = Field(..., description="Size of the file in bytes")
     md5_hash: str | None = Field(None, description="Optional MD5 hash of the object for integrity check")
 
 
-class PreSignedUploadResponse(BaseModel):
-    """Response model to upload a single object using the pre-signed URL using PUT."""
+class PreSignedSinglePartUploadResponse(BaseModel):
+    """Response model to upload a single object as a single part using the pre-signed URL using PUT."""
 
     name: str = Field(..., description="Name of the object to be uploaded")
     pre_signed_url: str = Field(..., description="Pre-signed URL to which the file should be uploaded")
     put_headers: dict[str, str] = Field(..., description="Headers to be included in the PUT request")
 
 
-class CheckFileExistsRequest(BaseModel):
-    """Request model to check if a file already exists in the bucket (using the checksum)"""
+### Multipart upload models ###
+class CreateMultipartUploadRequest(BaseModel):
+    """Request model to create a multipart upload using pre-signed URLs."""
+
+    name: str = Field(..., description="Name of the object to be uploaded")
+    content_length: int = Field(..., description="Size of the file in bytes")
+
+
+class CreateMultipartUploadResponse(BaseModel):
+    """Response model to create a multipart upload using pre-signed URLs."""
+
+    name: str = Field(..., description="Name of the object to be uploaded")
+    upload_id: str = Field(..., description="Upload ID for the multipart upload")
+    number_of_parts: int = Field(..., description="Total number of parts required for the upload", ge=1, le=10000)
+    part_size: int = Field(
+        S3_MULTIPART_CHUNK_SIZE, description="Size of each part in bytes (the last part may be smaller)."
+    )
+
+
+class GetPresignedPartUrlsRequest(BaseModel):
+    """
+    Request model to get pre-signed URLs for multiple parts of a presigned multipart upload.
+
+    You can request up to 100 parts at a time.
+    Part number indexing is 1-based (with max allowed range: 1 to 10000).
+    """
+
+    name: str = Field(..., description="Name of the object to be uploaded")
+    upload_id: str = Field(..., description="Upload ID for the multipart upload")
+    parts_range_start: int = Field(..., description="Starting part number", ge=1, le=10000)
+    parts_range_end: int = Field(..., description="Ending part number", ge=1, le=10000)
+    md5_checksums: list[str] | None = Field(
+        None, description="Optional list of MD5 checksums for each part to be uploaded"
+    )
+
+
+class PresignedUploadPartUrlResponse(BaseModel):
+    """Response model for a pre-signed URL for a single part of a multipart upload."""
+
+    part_number: int = Field(..., description="Part number", ge=1, le=10000)
+    pre_signed_url: str = Field(..., description="Pre-signed URL for uploading this part")
+    headers: dict[str, str] = Field(..., description="Headers to be included in the PUT request for this part")
+
+
+class UploadedPart(BaseModel):
+    """Model representing a part of an object that has been uploaded via multi-part upload."""
+
+    part_number: int = Field(..., description="Part number", ge=1, le=10000)
+    etag: str = Field(description="ETag returned by S3 after uploading the part")
+
+
+class CompleteMultipartUploadRequest(BaseModel):
+    """Request model to complete a multipart upload using pre-signed URLs."""
+
+    name: str = Field(..., description="Name of the object to be uploaded")
+    upload_id: str = Field(..., description="Upload ID for the multipart upload")
+    parts: list[UploadedPart] = Field(..., description="List of parts that have been uploaded")
+
+
+class CompleteMultipartUploadResponse(BaseModel):
+    """Response model to complete a multipart upload using pre-signed URLs."""
+
+    name: str = Field(..., description="Name of the object that was uploaded")
+    version_id: str = Field(..., description="Version ID of the uploaded object")
+    md5_hash: str = Field(..., description="MD5 hash of the uploaded object")
+
+
+class AbortMultipartUploadRequest(BaseModel):
+    """Request model to abort a multipart upload and clean up parts."""
+
+    name: str = Field(..., description="Name of the object being uploaded")
+    upload_id: str = Field(..., description="Upload ID for the multipart upload to be aborted")
+
+
+class AbortMultipartUploadResponse(BaseModel):
+    """Response model to abort a multipart upload."""
+
+    name: str = Field(..., description="Name of the object being uploaded")
+    upload_id: str = Field(..., description="Upload ID for the multipart upload that was aborted")
+
+
+class RestoreObjectsResponse(BaseModel):
+    """Response model for restoring soft-deleted objects in a bucket."""
 
-    object_name: str
-    md5_checksum: str
+    restored: list[str] = Field(
+        ...,
+        description="List of object names that were successfully restored, this includes objects that were already live",
+    )
+    not_restored: list[str] = Field(
+        ...,
+        description=(
+            "List of object names that could not be processed.\n"
+            "This could be due to several reasons:\n"
+            "1. The object does not exist in the bucket (e.g., a typo in the name).\n"
+            "2. The object was hard-deleted and is unrecoverable.\n"
+            "3. An unexpected server error occurred during the restore attempt."
+        ),
+    )
 
 
-class ExistingFileResponse(BaseModel):
-    """Response model for reporting a file that already exists in the bucket (using it's checksum)"""
+## checksum models ##
+class FileChecksumResponse(BaseModel):
+    """Response model for reporting a file's checksum in the bucket."""
 
-    object_name: str
-    md5_checksum: str
-    matching_object_name: str | None
+    object_name: str = Field(..., description="Name of the object in the bucket")
+    md5_checksum: str = Field(..., description="MD5 checksum of the object in the bucket")

divbase_lib/api_schemas/vcf_dimensions.py

@@ -40,3 +40,15 @@ class DimensionsShowResult(BaseModel):
     vcf_files: list[dict]
     skipped_file_count: int
     skipped_files: list[dict]
+
+
+class DimensionsSamplesResult(BaseModel):
+    """Result model for showing unique samples across project VCFs."""
+
+    unique_samples: list[str]  # Already sorted, by the CRUD function get_unique_samples_by_project_async()
+
+
+class DimensionsScaffoldsResult(BaseModel):
+    """Result model for showing unique scaffolds across project VCFs."""
+
+    unique_scaffolds: list[str]  # Already sorted, by the CRUD function get_unique_scaffolds_by_project_async()

divbase_lib/divbase_constants.py

@@ -0,0 +1,53 @@
+"""
+Constants that both divbase-api and divbase-cli need to agree on.
+"""
+
+ONE_MiB = 1024 * 1024
+
+# When you download a file that has been uploaded in parts, you have
+# to know the part/chunk size used in order to correctly calculate the composite checksum
+S3_MULTIPART_CHUNK_SIZE = 32 * ONE_MiB
+
+# At what point you swap from single part to multipart upload to S3.
+# If server and client used the same threshold then makes life easier
+# when validating the checksums of files in s3 as single part and multipart uploads use different ETag formats.
+# (No benefit in constraining the download threshold, so not done here)
+S3_MULTIPART_UPLOAD_THRESHOLD = 96 * ONE_MiB
+
+# Max number of items that can be processed in a single API call to divbase-api's S3 routes
+# covers e.g pre-signed urls for upload/download, soft delete and checksum comparisons
+# client has to batch requests if exceeding this limit
+MAX_S3_API_BATCH_SIZE = 100
+
+# How long the pre-signed URLs divbase-api creates are valid for
+SINGLE_PART_UPLOAD_URL_EXPIRATION_SECONDS = 3600  # 1 hour
+MULTI_PART_UPLOAD_URL_EXPIRATION_SECONDS = 36000  # 10 hours
+DOWNLOAD_URL_EXPIRATION_SECONDS = 36000  # 10 hours
+
+# (Not used anywhere, just making it explicit)
+# This is limited by our fixing of the chunk size and S3's limit to the number of chunks allowed (10,000)
+# 320 GiB if using 32 MiB chunks
+LARGEST_FILE_UPLOADABLE_TO_DIVBASE_BYTES = 10_000 * S3_MULTIPART_CHUNK_SIZE
+
+# File types that DivBase supports
+# Whilst we can't realistically limit what file types a user actually uploads,
+# this is here to say what we know should work in DivBase.
+SUPPORTED_DIVBASE_FILE_TYPES = (".tsv", ".vcf.gz", ".csi", ".tbi")
+
+# Characters that are not allowed in file names uploaded to DivBase
+# This is to prevent issues when users try to filter/query files on DivBase using these characters
+# or when downloading files (e.g. ":" is used to specify file versions when downloading files
+UNSUPPORTED_CHARACTERS_IN_FILENAMES = (":", "*", "?", "<", ">", "|", "\\")
+
+# This prefix is used for all *.vcf.gz results files from a query job/task.
+# After the prefix comes the job id which is a rolling integer.
+# E.g. format: result_of_job_<job-id>.vcf.gz , where <job-id> = 1 and is auto-incremented for every new job.
+# NOTE: If you update this, you should also update the S3 lifecycle policies associated with this.
+QUERY_RESULTS_FILE_PREFIX = "result_of_job_"
+
+# CLI includes this in every request to the server,
+# and server uses it to determine if it should:
+# 1. Reject the request as the version is too outdated
+# 2. Add an announcement about a new CLI version being available when the user logs in.
+# 3. Do Nothing as the user's CLI version is up to date.
+CLI_VERSION_HEADER_KEY = "X-CLI-Version"

divbase_lib/exceptions.py

@@ -10,20 +10,6 @@ we ensure that when you manually raise a specific exception the error message lo
 from pathlib import Path
 
 
-class ObjectDoesNotExistError(FileNotFoundError):
-    """Raised when an S3 object/key does not exist in the bucket."""
-
-    def __init__(self, key: str, bucket_name: str):
-        error_message = f"The file/object '{key}' does not exist in the bucket '{bucket_name}'. "
-        super().__init__(error_message)
-        self.key = key
-        self.bucket = bucket_name
-        self.error_message = error_message
-
-    def __str__(self):
-        return self.error_message
-
-
 class BcftoolsEnvironmentError(Exception):
     """Raised when there's an issue with the execution environment (Docker, etc.)."""
 
@@ -115,12 +101,47 @@ class SidecarColumnNotFoundError(Exception):
     pass
 
 
+class SidecarSampleIDError(Exception):
+    """Raised when a Sample_ID is invalid or duplicated when loading sidecar data."""
+
+    pass
+
+
+class SidecarMetadataFormatError(Exception):
+    """Raised when the sidecar metadata TSV file has formatting issues (duplicate columns, empty columns, commas in values, etc.)."""
+
+    pass
+
+
+class TaskUserError(Exception):
+    """
+    Raised in Celery tasks when an error needs to propagate back to the CLI user.
+
+    This is intentionally kept as a simple Exception subclass (no custom __init__)
+    to avoid UnpicklableExceptionWrapper when passing through Celery's JSON serialization/deserialization.
+    Complex exception types such as those inheriting from DivBaseAPIException seem to trigger UnpicklableExceptionWrapper.
+
+    This class is essentially a wrapper to allow to use more complex exceptions in Celery tasks and catch them
+    in the API route handlers to return user-friendly error messages to the CLI.
+    In the Celery task use it like:
+        raise TaskUserError(str(SomeComplexError(...))) from None
+    """
+
+    pass
+
+
 class NoVCFFilesFoundError(Exception):
     """Raised when no VCF files are found in the project bucket."""
 
     pass
 
 
+class DimensionsNotUpToDateWithBucketError(Exception):
+    """Raised when VCF files in a bucket are missing or outdated in the dimensions index."""
+
+    pass
+
+
 class ChecksumVerificationError(Exception):
     """Raised when a calculated file's checksum does not match the expected value."""
 
@@ -128,5 +149,8 @@ class ChecksumVerificationError(Exception):
         self.expected_checksum = expected_checksum
         self.calculated_checksum = calculated_checksum
 
-        message = f"Checksum verification failed. Expected: {expected_checksum}, Calculated: {calculated_checksum}"
+        message = (
+            f"Checksum verification failed. Expected: {expected_checksum}, Calculated: {calculated_checksum}"
+            f" The file has been deleted to avoid accidental use of a corrupted file."
+        )
         super().__init__(message)