divbase-lib 0.1.0a1__py3-none-any.whl

divbase_lib/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0a1"

divbase_lib/api_schemas/announcements.py

@@ -0,0 +1,18 @@
+"""
+Schemas for announcements.
+"""
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class AnnouncementResponse(BaseModel):
+    """Response model for an announcement returned by the API."""
+
+    heading: str = Field(..., description="Title of the announcement.")
+    message: str | None = Field(None, description="Detailed message of the announcement.")
+    level: str = Field(
+        ...,
+        description="The announcement level, which can control styling of announcement. Possible values are: info, success, warning, danger.",
+    )
+
+    model_config = ConfigDict(from_attributes=True)

divbase_lib/api_schemas/auth.py

@@ -0,0 +1,34 @@
+"""
+Schemas for login + access and refresh tokens
+"""
+
+from pydantic import BaseModel, Field
+
+
+class CLILoginResponse(BaseModel):
+    """Response model for API (aka divbase-cli) login endpoint."""
+
+    access_token: str = Field(..., description="Bearer access token for authentication")
+    access_token_expires_at: int = Field(..., description="Unix timestamp when the access token expires")
+    refresh_token: str = Field(..., description="Bearer refresh token for obtaining new access tokens")
+    refresh_token_expires_at: int = Field(..., description="Unix timestamp when the refresh token expires")
+    email: str = Field(..., description="Email of the authenticated user")
+
+
+class RefreshTokenRequest(BaseModel):
+    """Request model for refresh token endpoint."""
+
+    refresh_token: str = Field(..., description="Bearer refresh token for obtaining a new access token")
+
+
+class RefreshTokenResponse(BaseModel):
+    """Response model for refresh token endpoint."""
+
+    access_token: str = Field(..., description="Bearer access token for authentication")
+    expires_at: int = Field(..., description="Unix timestamp when the access token expires")
+
+
+class LogoutRequest(BaseModel):
+    """Request model for logout endpoint."""
+
+    refresh_token: str = Field(..., description="Bearer refresh token to be revoked on logout")

divbase_lib/api_schemas/personal_access_tokens.py

@@ -0,0 +1,42 @@
+"""
+Schemas for personal access token related endpoints.
+"""
+
+from pydantic import BaseModel, Field, model_validator
+from typing_extensions import Self
+
+# This needs to stay in sync with ProjectRoles in divbase-api.
+# Redefined here to avoid a circular dependency on divbase-api.
+ALLOWED_PROJECT_ROLES = ["read", "edit", "manage"]
+
+
+class PATPermissions(BaseModel):
+    """
+    Personal Access Token permissions model.
+    This is stored in the db as JSONB in the PersonalAccessTokenDB.permissions.
+
+    - all_projects: True = access all the user's projects with same role as user
+                    False = access only the projects listed in `projects`.
+    - projects: {str(project_id): role_string}; only enforced when all_projects=False.
+        Where role_string must be a valid ProjectRoles enum value.
+    - task_history: endpoint level scope.
+        For the 2 task history routes not tied to a specific project.
+        (these are 'task-history id' and for 'task-history user')
+    """
+
+    all_projects: bool = False
+    projects: dict[str, str] = Field(default_factory=dict)
+    task_history: bool = False
+
+    @model_validator(mode="after")
+    def no_project_scope_if_all_projects_true(self) -> Self:
+        if self.all_projects and self.projects:
+            raise ValueError("cannot specify project scope if 'all_projects' is True")
+        return self
+
+    @model_validator(mode="after")
+    def validate_project_roles(self) -> Self:
+        for project_id, role_str in self.projects.items():
+            if role_str not in ALLOWED_PROJECT_ROLES:
+                raise ValueError(f"invalid role '{role_str}' for project {project_id}")
+        return self

divbase_lib/api_schemas/project_versions.py

@@ -0,0 +1,87 @@
+"""
+Schemas for project versioning routes.
+
+Project versions are the state of all files in a project's storage bucket at a given time point.
+"""
+
+from typing import TypedDict
+
+from pydantic import BaseModel, Field
+
+
+# Request Models
+class CreateVersioningFileRequest(BaseModel):
+    name: str = Field(..., description="Initial version name")
+    description: str = Field(..., description="Initial version description")
+
+
+class AddVersionRequest(BaseModel):
+    name: str = Field(..., description="Name of the new version to add")
+    description: str = Field("", description="Description of the new version")
+
+
+class DeleteVersionRequest(BaseModel):
+    version_name: str = Field(..., description="Name of the version to delete")
+
+
+class UpdateVersionRequest(BaseModel):
+    version_name: str = Field(..., description="Name of the version to update")
+    new_name: str | None = Field(None, description="New name for the version")
+    new_description: str | None = Field(None, description="New description for the version")
+
+
+# Response Models
+class ProjectBasicInfo(BaseModel):
+    """Base model for describing a single project version, not for direct use in an endpoint."""
+
+    name: str = Field(..., description="Version name")
+    description: str | None = Field(..., description="Version description")
+
+
+class AddVersionResponse(ProjectBasicInfo):
+    """Response model for adding a version."""
+
+    created_at: str = Field(..., description="ISO timestamp when version was created")
+
+
+class ProjectVersionInfo(ProjectBasicInfo):
+    """Basic information about a project version. You get a list of these when listing all versions in a project."""
+
+    created_at: str = Field(..., description="ISO timestamp when version was created")
+    is_deleted: bool = Field(..., description="Whether this version has been soft-deleted")
+
+
+# Typed dict used as this is used in the db model to define the contents of the JSONB
+class FileDetails(TypedDict):
+    """Details about a single file in a project version, ETag (is checksum) and size is in bytes."""
+
+    version_id: str
+    etag: str
+    size: int
+
+
+class ProjectVersionDetailResponse(ProjectBasicInfo):
+    """Full information about a single project version, including the files at that version."""
+
+    created_at: str = Field(..., description="ISO timestamp when version was created")
+    is_deleted: bool = Field(..., description="Whether this version has been soft-deleted")
+    files: dict[str, FileDetails] = Field(
+        ..., description="Info about all files at this version, including: version IDs, etags and file sizes"
+    )
+
+
+class DeleteVersionResponse(BaseModel):
+    """Response model for deleting a version."""
+
+    name: str = Field(..., description="Name of the version that was deleted")
+    already_deleted: bool = Field(
+        False,
+        description="Whether the version was already soft-deleted before this request",
+    )
+    date_deleted: str = Field(..., description="ISO timestamp of when the version was soft-deleted")
+
+
+class UpdateVersionResponse(ProjectBasicInfo):
+    """Response model for updating a version."""
+
+    created_at: str = Field(..., description="ISO timestamp when version was created")

divbase_lib/api_schemas/queries.py

@@ -0,0 +1,209 @@
+"""
+Schemas for query routes.
+"""
+
+from typing import Optional, Self
+
+from pydantic import BaseModel, ConfigDict, field_validator, model_validator
+
+from divbase_lib.utils import split_semicolon_bcftools_command_segments
+
+# Shared helper functions and models
+
+
+def validate_command_not_empty(command: str) -> str:
+    """
+    Validator function to ensure that the user-submitted bcftools command string is not empty or just whitespace.
+    If there are multiple segments separated by semicolons, it also validates that none of the segments are empty or whitespace.
+
+    Shared helper for command field validators in BcftoolsQueryRequest.command and BcftoolsQueryKwargs.command.
+    """
+    command_string = command.strip()
+    if command_string == "":
+        raise ValueError(
+            "The --command option must be a non-empty bcftools view string. "
+            'If you only want to subset based on samples, use --command "view -s" in combination with one of the sample-selection options: --tsv-filter, --samples, --samples-file.'
+        )
+
+    segments = split_semicolon_bcftools_command_segments(command_string)
+    for position, segment in enumerate(segments, start=1):
+        if segment.strip() == "":
+            raise ValueError(
+                f"The --command option has an empty pipeline segment at position {position}. "
+                "Use semicolons only between complete bcftools view commands."
+            )
+
+    return command
+
+
+class SharedBaseModel(BaseModel):
+    """Shared pydantic BaseModel for VCF query response and kwarg schemas."""
+
+    model_config = ConfigDict(
+        validate_assignment=True,
+        json_schema_extra={
+            "description": ("Exactly one sample-selection mode must be provided: tsv_filter, samples, or all_samples.")
+        },
+    )
+
+
+# Request models
+class SampleMetadataQueryRequest(BaseModel):
+    """Request model for sample metadata query route."""
+
+    tsv_filter: str
+    metadata_tsv_name: str
+
+
+class BcftoolsQueryRequest(SharedBaseModel):
+    """
+    Request model for sample metadata query route.
+    See ConfigDict in SharedBaseModel for validation rules around tsv_filter and samples fields.
+    """
+
+    tsv_filter: str | None = None  # Used for metadata mode only
+    metadata_tsv_name: str | None = None  # Used for metadata mode only
+    command: str  # bcftools command input with --command
+    samples: list[str] | None = None
+    all_samples: bool = False
+
+    @field_validator("command")
+    @classmethod
+    def validate_command(_cls, value: str) -> str:
+        return validate_command_not_empty(value)
+
+    @model_validator(mode="after")
+    def validate_sample_selection_mode(self) -> Self:
+        tsv_filter = self.tsv_filter
+        samples = self.samples
+        all_samples = self.all_samples
+
+        selection_count = 0
+        if tsv_filter is not None:
+            selection_count += 1
+        if samples is not None:
+            selection_count += 1
+        if all_samples:
+            selection_count += 1
+
+        if selection_count > 1:
+            raise ValueError("Only one of tsv_filter, samples, or all_samples may be provided.")
+        if selection_count == 0:
+            raise ValueError("One sample-selection mode must be provided (tsv_filter, samples, or all_samples).")
+
+        if tsv_filter is not None and self.metadata_tsv_name is None:
+            raise ValueError("metadata_tsv_name must be provided when tsv_filter is used.")
+
+        if samples is not None and len(samples) == 0:
+            raise ValueError("samples must contain at least one sample ID when provided.")
+
+        return self
+
+
+# Models for task kwargs and task results. Reused in task history schemas too, hence pydantic models and not just dataclasses.
+class SampleMetadataQueryKwargs(BaseModel):
+    """Keyword arguments for sample metadata query task. Used to pass info to Celery task, and also for recording task history."""
+
+    tsv_filter: str
+    metadata_tsv_name: str
+    bucket_name: str
+    project_id: int
+    project_name: str
+    user_id: int
+
+
+class BcftoolsQueryKwargs(SharedBaseModel):
+    """
+    Keyword arguments for BCFtools query task. Used to pass info to Celery task, and also for recording task history.
+    See ConfigDict in SharedBaseModel for validation rules around tsv_filter and samples fields.
+    """
+
+    tsv_filter: str | None = None  # Used for metadata mode only
+    metadata_tsv_name: str | None = None  # Used for metadata mode only
+    command: str  # bcftools command input with --command
+    bucket_name: str
+    project_id: int
+    project_name: str
+    user_id: int
+    job_id: int
+    samples: list[str] | None = None
+    all_samples: bool = False
+
+    @field_validator("command")
+    @classmethod
+    def validate_command(_cls, value: str) -> str:
+        return validate_command_not_empty(value)
+
+    @model_validator(mode="after")
+    def validate_sample_selection_mode(self) -> Self:
+        tsv_filter = self.tsv_filter
+        samples = self.samples
+        all_samples = self.all_samples
+
+        selection_count = 0
+        if tsv_filter is not None:
+            selection_count += 1
+        if samples is not None:
+            selection_count += 1
+        if all_samples:
+            selection_count += 1
+
+        if selection_count > 1:
+            raise ValueError("Only one of tsv_filter, samples, or all_samples may be provided.")
+        if selection_count == 0:
+            # Backward compatibility for historical task kwargs created before all_samples option was implemented.
+            # To ensure that task-history deserialization does not break for existing tasks.
+            # Could be handled by a backfilling migration instead.
+            self.all_samples = True
+            return self
+
+        if tsv_filter is not None and self.metadata_tsv_name is None:
+            raise ValueError("metadata_tsv_name must be provided when tsv_filter is used.")
+
+        if samples is not None and len(samples) == 0:
+            raise ValueError("samples must contain at least one sample ID when provided.")
+
+        return self
+
+
+class SampleFileMappingResult(BaseModel):
+    sample_id: str
+    filename: str
+
+    @model_validator(mode="before")
+    @classmethod
+    def map_legacy_task_history_keys(cls, data):
+        """
+        Support historical task-history payloads that stored uppercase keys.
+        """
+        if not isinstance(data, dict):
+            return data
+
+        normalized = dict(data)
+        if "sample_id" not in normalized:
+            if "Sample_ID" in normalized:
+                normalized["sample_id"] = normalized["Sample_ID"]
+            elif "#Sample_ID" in normalized:
+                normalized["sample_id"] = normalized["#Sample_ID"]
+        if "filename" not in normalized and "Filename" in normalized:
+            normalized["filename"] = normalized["Filename"]
+
+        return normalized
+
+
+class SampleMetadataQueryTaskResult(BaseModel):
+    """Metadata query task result details. Based on the return of tasks.sample_metadata_query."""
+
+    sample_and_filename_subset: list[SampleFileMappingResult]
+    unique_sample_ids: list[str]
+    unique_filenames: list[str]
+    query_message: str
+    warnings: list[str] = []
+    status: str | None = None
+
+
+class BcftoolsQueryTaskResult(BaseModel):
+    """BCFtools query task result details. Based on the return of tasks.bcftools_query."""
+
+    output_file: str
+    status: Optional[str] = None

divbase_lib/api_schemas/s3.py

@@ -0,0 +1,214 @@
+"""
+Schemas for DivBase's S3 API routes.
+
+Pre-signed download URLs do not need to account for single vs multipart as this can be controlled by the client
+using the HTTP range header when downloading (so you only need 1 pre-signed URL per object for download).
+
+Pre-signed upload URLs need to account for single vs multipart uploads hence all the extra schemas below.
+"""
+
+from datetime import datetime
+
+from pydantic import BaseModel, Field
+
+from divbase_lib.divbase_constants import S3_MULTIPART_CHUNK_SIZE
+
+MB = 1024 * 1024
+
+
+## list objects models ##
+class ListObjectsRequest(BaseModel):
+    """Request model for listing objects in an S3 bucket."""
+
+    prefix: str | None = Field(None, description="Optional prefix to filter objects by name.")
+    next_token: str | None = Field(
+        None, description="Token to continue listing files from the end of a previous request."
+    )
+
+
+class ObjectDetails(BaseModel):
+    """Details about a single object in an S3 bucket."""
+
+    name: str = Field(..., description="The name of the object in the bucket.")
+    size: int = Field(..., description="The size of the object in bytes.")
+    last_modified: datetime = Field(..., description="The date and time the object was last modified.")
+    etag: str = Field(..., description="The ETag of the object, which is the MD5 checksum.")
+
+
+class ListObjectsResponse(BaseModel):
+    """Response model for listing objects in an S3 bucket."""
+
+    objects: list[ObjectDetails] = Field(
+        ..., description="A list of objects in the bucket.", min_length=0, max_length=1000
+    )
+    next_token: str | None = Field(
+        None, description="Token for fetching the next page of results. If None, no more results."
+    )
+
+
+## list soft-deleted objects models ##
+class SoftDeletedObjectDetails(BaseModel):
+    """Details about a single soft-deleted object in an S3 bucket."""
+
+    name: str = Field(..., description="The name of the object in the bucket.")
+    last_modified: datetime = Field(..., description="The date and time the object was deleted.")
+
+
+## file info models ##
+class ObjectVersionInfo(BaseModel):
+    """Detailed information about a single version of an S3 object."""
+
+    version_id: str = Field(..., description="The version ID of the object.")
+    last_modified: datetime = Field(..., description="The date and time the object version was last modified.")
+    size: int = Field(..., description="The size of the object in bytes.")
+    etag: str = Field(..., description="The ETag of the object, which is the MD5 checksum.")
+    is_latest: bool = Field(..., description="Indicates if this is the latest version of the object.")
+
+
+class ObjectInfoResponse(BaseModel):
+    """Response model for detailed information about all versions of a single object stored in S3."""
+
+    object_name: str = Field(..., description="The name of the object.")
+    is_currently_deleted: bool = Field(..., description="True if the latest version of the object is a delete marker.")
+    versions: list[ObjectVersionInfo] = Field(..., description="A list of all versions of the object.")
+
+
+## download models ##
+class DownloadObjectRequest(BaseModel):
+    """Request model to download a single object using a pre-signed URL."""
+
+    name: str = Field(..., description="Name of the object to be downloaded")
+    version_id: str | None = Field(..., description="Version ID of the object, None if latest version")
+
+
+class PreSignedDownloadResponse(BaseModel):
+    """Response model to download a single object using the pre-signed URL and (optionally) version ID."""
+
+    name: str = Field(..., description="Name of the object to be downloaded")
+    pre_signed_url: str = Field(..., description="Pre-signed URL for downloading the object")
+    version_id: str | None = Field(..., description="Version ID of the object, None if latest version")
+
+
+### Single-part upload models ###
+class UploadSinglePartObjectRequest(BaseModel):
+    """Request model to upload a single object as a single part using a pre-signed URL."""
+
+    name: str = Field(..., description="Name of the object to be uploaded")
+    content_length: int = Field(..., description="Size of the file in bytes")
+    md5_hash: str | None = Field(None, description="Optional MD5 hash of the object for integrity check")
+
+
+class PreSignedSinglePartUploadResponse(BaseModel):
+    """Response model to upload a single object as a single part using the pre-signed URL using PUT."""
+
+    name: str = Field(..., description="Name of the object to be uploaded")
+    pre_signed_url: str = Field(..., description="Pre-signed URL to which the file should be uploaded")
+    put_headers: dict[str, str] = Field(..., description="Headers to be included in the PUT request")
+
+
+### Multipart upload models ###
+class CreateMultipartUploadRequest(BaseModel):
+    """Request model to create a multipart upload using pre-signed URLs."""
+
+    name: str = Field(..., description="Name of the object to be uploaded")
+    content_length: int = Field(..., description="Size of the file in bytes")
+
+
+class CreateMultipartUploadResponse(BaseModel):
+    """Response model to create a multipart upload using pre-signed URLs."""
+
+    name: str = Field(..., description="Name of the object to be uploaded")
+    upload_id: str = Field(..., description="Upload ID for the multipart upload")
+    number_of_parts: int = Field(..., description="Total number of parts required for the upload", ge=1, le=10000)
+    part_size: int = Field(
+        S3_MULTIPART_CHUNK_SIZE, description="Size of each part in bytes (the last part may be smaller)."
+    )
+
+
+class GetPresignedPartUrlsRequest(BaseModel):
+    """
+    Request model to get pre-signed URLs for multiple parts of a presigned multipart upload.
+
+    You can request up to 100 parts at a time.
+    Part number indexing is 1-based (with max allowed range: 1 to 10000).
+    """
+
+    name: str = Field(..., description="Name of the object to be uploaded")
+    upload_id: str = Field(..., description="Upload ID for the multipart upload")
+    parts_range_start: int = Field(..., description="Starting part number", ge=1, le=10000)
+    parts_range_end: int = Field(..., description="Ending part number", ge=1, le=10000)
+    md5_checksums: list[str] | None = Field(
+        None, description="Optional list of MD5 checksums for each part to be uploaded"
+    )
+
+
+class PresignedUploadPartUrlResponse(BaseModel):
+    """Response model for a pre-signed URL for a single part of a multipart upload."""
+
+    part_number: int = Field(..., description="Part number", ge=1, le=10000)
+    pre_signed_url: str = Field(..., description="Pre-signed URL for uploading this part")
+    headers: dict[str, str] = Field(..., description="Headers to be included in the PUT request for this part")
+
+
+class UploadedPart(BaseModel):
+    """Model representing a part of an object that has been uploaded via multi-part upload."""
+
+    part_number: int = Field(..., description="Part number", ge=1, le=10000)
+    etag: str = Field(description="ETag returned by S3 after uploading the part")
+
+
+class CompleteMultipartUploadRequest(BaseModel):
+    """Request model to complete a multipart upload using pre-signed URLs."""
+
+    name: str = Field(..., description="Name of the object to be uploaded")
+    upload_id: str = Field(..., description="Upload ID for the multipart upload")
+    parts: list[UploadedPart] = Field(..., description="List of parts that have been uploaded")
+
+
+class CompleteMultipartUploadResponse(BaseModel):
+    """Response model to complete a multipart upload using pre-signed URLs."""
+
+    name: str = Field(..., description="Name of the object that was uploaded")
+    version_id: str = Field(..., description="Version ID of the uploaded object")
+    md5_hash: str = Field(..., description="MD5 hash of the uploaded object")
+
+
+class AbortMultipartUploadRequest(BaseModel):
+    """Request model to abort a multipart upload and clean up parts."""
+
+    name: str = Field(..., description="Name of the object being uploaded")
+    upload_id: str = Field(..., description="Upload ID for the multipart upload to be aborted")
+
+
+class AbortMultipartUploadResponse(BaseModel):
+    """Response model to abort a multipart upload."""
+
+    name: str = Field(..., description="Name of the object being uploaded")
+    upload_id: str = Field(..., description="Upload ID for the multipart upload that was aborted")
+
+
+class RestoreObjectsResponse(BaseModel):
+    """Response model for restoring soft-deleted objects in a bucket."""
+
+    restored: list[str] = Field(
+        ...,
+        description="List of object names that were successfully restored, this includes objects that were already live",
+    )
+    not_restored: list[str] = Field(
+        ...,
+        description=(
+            "List of object names that could not be processed.\n"
+            "This could be due to several reasons:\n"
+            "1. The object does not exist in the bucket (e.g., a typo in the name).\n"
+            "2. The object was hard-deleted and is unrecoverable.\n"
+            "3. An unexpected server error occurred during the restore attempt."
+        ),
+    )
+
+
+## checksum models ##
+class FileChecksumResponse(BaseModel):
+    """Response model for reporting a file's checksum in the bucket."""
+
+    object_name: str = Field(..., description="Name of the object in the bucket")
+    md5_checksum: str = Field(..., description="MD5 checksum of the object in the bucket")

divbase_lib/api_schemas/task_history.py

@@ -0,0 +1,50 @@
+"""
+Schemas for task history routes.
+"""
+
+from typing import Any, Optional, Union
+
+from pydantic import BaseModel
+
+from divbase_lib.api_schemas.queries import (
+    BcftoolsQueryKwargs,
+    BcftoolsQueryTaskResult,
+    SampleMetadataQueryKwargs,
+    SampleMetadataQueryTaskResult,
+)
+from divbase_lib.api_schemas.vcf_dimensions import DimensionUpdateKwargs, DimensionUpdateTaskResult
+
+
+class TaskHistoryResult(BaseModel):
+    """
+    Task details as returned by queries to the SQAlchemy+pg results backend.
+    """
+
+    id: int
+    submitter_email: Optional[str] = None
+    status: Optional[str] = None
+    result: Optional[
+        Union[
+            dict[
+                str, Any
+            ],  # Note! This dict must come first here so that error results are preserved and not incorrectly inserted into the result models
+            SampleMetadataQueryTaskResult,
+            BcftoolsQueryTaskResult,
+            DimensionUpdateTaskResult,
+        ]
+    ] = None
+    date_done: Optional[str] = None
+    name: Optional[str] = None
+    args: Optional[str] = None
+    kwargs: Optional[
+        Union[
+            SampleMetadataQueryKwargs,
+            BcftoolsQueryKwargs,
+            DimensionUpdateKwargs,
+        ]
+    ] = None
+    worker: Optional[str] = None
+    created_at: Optional[str] = None
+    started_at: Optional[str] = None
+    completed_at: Optional[str] = None
+    runtime: Optional[float] = None