divbase-lib 0.1.0a1__tar.gz

divbase_lib-0.1.0a1/.gitignore

@@ -0,0 +1,46 @@
+# repo secrets 
+.env 
+.env*
+
+# python stuff 
+.venv
+.ruff_cache/
+__pycache__/
+*.py[cod]
+*$py.class
+
+.vscode/
+
+# project specific files
+/sample_metadata.tsv
+/sample_metadata_*.tsv
+*.vcf
+*.vcf.gz
+*.vcf.gz.csi
+*.vcf.gz.tbi
+!tests/fixtures/*.vcf.gz
+tests/fixtures/temp*
+tests/fixtures/merged*
+divbase_metadata_template*.tsv
+
+# query job config files
+bcftools_divbase_job_config.json
+
+# benchmarking files
+vcf_dimensions.tsv
+mock*.tsv
+task_records*.json
+split_scaffold_files.txt
+scripts/benchmarking/*.yaml
+scripts/benchmarking/results
+
+#MacOS artifacts
+.DS_Store
+
+# mkdocs build cache
+.cache/
+# mkdocs auto generated files
+docs/cli/_auto_generated/*.md
+
+# pypi 
+dist/

divbase_lib-0.1.0a1/PKG-INFO

@@ -0,0 +1,38 @@
+Metadata-Version: 2.4
+Name: divbase-lib
+Version: 0.1.0a1
+Summary: Library module for Divbase
+Project-URL: Homepage, https://divbase.scilifelab-2-prod.sys.kth.se
+Project-URL: Documentation, https://scilifelabdatacentre.github.io/divbase
+Project-URL: Repository, https://github.com/ScilifelabDataCentre/divbase
+Project-URL: Issues, https://github.com/ScilifelabDataCentre/divbase/issues
+Author-email: SciLifeLab Data Centre <datacentre@scilifelab.se>
+License-Expression: MIT
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Requires-Python: >=3.12
+Requires-Dist: httpx==0.28.1
+Requires-Dist: pandas==2.3.3
+Requires-Dist: pydantic==2.12.5
+Requires-Dist: pyyaml==6.0.3
+Requires-Dist: stamina==25.2.0
+Description-Content-Type: text/markdown
+
+# divbase-lib
+
+This is library package for Divbase. Code from this library is shared between the Divbase CLI and API packages.
+
+## Installation and usage
+
+It is highly unlikely that you will need to use/install this package directly. Instead, it will be installed automatically by other packages (e.g. Divbase CLI) as a dependency.
+
+## Development
+
+This package is developed in the [DivBase repository](https://github.com/ScilifelabDataCentre/divbase) by Scilifelab Data Centre.

divbase_lib-0.1.0a1/README.md

@@ -0,0 +1,11 @@
+# divbase-lib
+
+This is library package for Divbase. Code from this library is shared between the Divbase CLI and API packages.
+
+## Installation and usage
+
+It is highly unlikely that you will need to use/install this package directly. Instead, it will be installed automatically by other packages (e.g. Divbase CLI) as a dependency.
+
+## Development
+
+This package is developed in the [DivBase repository](https://github.com/ScilifelabDataCentre/divbase) by Scilifelab Data Centre.

divbase_lib-0.1.0a1/pyproject.toml

@@ -0,0 +1,49 @@
+[project]
+name = "divbase-lib"
+description = "Library module for Divbase"
+readme = "README.md"
+license = "MIT"
+authors = [
+    { name = "SciLifeLab Data Centre", email = "datacentre@scilifelab.se" },
+]
+requires-python = ">=3.12"
+dependencies = [
+    "pyyaml==6.0.3",
+    "pandas==2.3.3", # TODO - consider major version bump
+    "pydantic==2.12.5",
+    "httpx==0.28.1",
+    "stamina==25.2.0"
+]
+dynamic = ["version"]
+
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Environment :: Console",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+]
+
+[project.urls]
+Homepage = "https://divbase.scilifelab-2-prod.sys.kth.se"
+Documentation = "https://scilifelabdatacentre.github.io/divbase"
+Repository = "https://github.com/ScilifelabDataCentre/divbase"
+Issues = "https://github.com/ScilifelabDataCentre/divbase/issues"
+
+[build-system]
+requires = ["hatchling >= 1.26"]
+build-backend = "hatchling.build"
+
+[tool.hatch.version]
+path = "src/divbase_lib/__init__.py"
+
+[tool.ruff]
+extend = "../../ruff.toml"
+
+[tool.ruff.lint.flake8-tidy-imports.banned-api]
+"divbase_cli" = { msg = "divbase-lib cannot import from divbase-cli, only the other way around works" }
+"divbase_api" = { msg = "divbase-lib cannot import from divbase-api, only the other way around works" }

divbase_lib-0.1.0a1/src/divbase_lib/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0a1"

divbase_lib-0.1.0a1/src/divbase_lib/api_schemas/announcements.py

@@ -0,0 +1,18 @@
+"""
+Schemas for announcements.
+"""
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class AnnouncementResponse(BaseModel):
+    """Response model for an announcement returned by the API."""
+
+    heading: str = Field(..., description="Title of the announcement.")
+    message: str | None = Field(None, description="Detailed message of the announcement.")
+    level: str = Field(
+        ...,
+        description="The announcement level, which can control styling of announcement. Possible values are: info, success, warning, danger.",
+    )
+
+    model_config = ConfigDict(from_attributes=True)

divbase_lib-0.1.0a1/src/divbase_lib/api_schemas/auth.py

@@ -0,0 +1,34 @@
+"""
+Schemas for login + access and refresh tokens
+"""
+
+from pydantic import BaseModel, Field
+
+
+class CLILoginResponse(BaseModel):
+    """Response model for API (aka divbase-cli) login endpoint."""
+
+    access_token: str = Field(..., description="Bearer access token for authentication")
+    access_token_expires_at: int = Field(..., description="Unix timestamp when the access token expires")
+    refresh_token: str = Field(..., description="Bearer refresh token for obtaining new access tokens")
+    refresh_token_expires_at: int = Field(..., description="Unix timestamp when the refresh token expires")
+    email: str = Field(..., description="Email of the authenticated user")
+
+
+class RefreshTokenRequest(BaseModel):
+    """Request model for refresh token endpoint."""
+
+    refresh_token: str = Field(..., description="Bearer refresh token for obtaining a new access token")
+
+
+class RefreshTokenResponse(BaseModel):
+    """Response model for refresh token endpoint."""
+
+    access_token: str = Field(..., description="Bearer access token for authentication")
+    expires_at: int = Field(..., description="Unix timestamp when the access token expires")
+
+
+class LogoutRequest(BaseModel):
+    """Request model for logout endpoint."""
+
+    refresh_token: str = Field(..., description="Bearer refresh token to be revoked on logout")

divbase_lib-0.1.0a1/src/divbase_lib/api_schemas/personal_access_tokens.py

@@ -0,0 +1,42 @@
+"""
+Schemas for personal access token related endpoints.
+"""
+
+from pydantic import BaseModel, Field, model_validator
+from typing_extensions import Self
+
+# This needs to stay in sync with ProjectRoles in divbase-api.
+# Redefined here to avoid a circular dependency on divbase-api.
+ALLOWED_PROJECT_ROLES = ["read", "edit", "manage"]
+
+
+class PATPermissions(BaseModel):
+    """
+    Personal Access Token permissions model.
+    This is stored in the db as JSONB in the PersonalAccessTokenDB.permissions.
+
+    - all_projects: True = access all the user's projects with same role as user
+                    False = access only the projects listed in `projects`.
+    - projects: {str(project_id): role_string}; only enforced when all_projects=False.
+        Where role_string must be a valid ProjectRoles enum value.
+    - task_history: endpoint level scope.
+        For the 2 task history routes not tied to a specific project.
+        (these are 'task-history id' and for 'task-history user')
+    """
+
+    all_projects: bool = False
+    projects: dict[str, str] = Field(default_factory=dict)
+    task_history: bool = False
+
+    @model_validator(mode="after")
+    def no_project_scope_if_all_projects_true(self) -> Self:
+        if self.all_projects and self.projects:
+            raise ValueError("cannot specify project scope if 'all_projects' is True")
+        return self
+
+    @model_validator(mode="after")
+    def validate_project_roles(self) -> Self:
+        for project_id, role_str in self.projects.items():
+            if role_str not in ALLOWED_PROJECT_ROLES:
+                raise ValueError(f"invalid role '{role_str}' for project {project_id}")
+        return self

divbase_lib-0.1.0a1/src/divbase_lib/api_schemas/project_versions.py

@@ -0,0 +1,87 @@
+"""
+Schemas for project versioning routes.
+
+Project versions are the state of all files in a project's storage bucket at a given time point.
+"""
+
+from typing import TypedDict
+
+from pydantic import BaseModel, Field
+
+
+# Request Models
+class CreateVersioningFileRequest(BaseModel):
+    name: str = Field(..., description="Initial version name")
+    description: str = Field(..., description="Initial version description")
+
+
+class AddVersionRequest(BaseModel):
+    name: str = Field(..., description="Name of the new version to add")
+    description: str = Field("", description="Description of the new version")
+
+
+class DeleteVersionRequest(BaseModel):
+    version_name: str = Field(..., description="Name of the version to delete")
+
+
+class UpdateVersionRequest(BaseModel):
+    version_name: str = Field(..., description="Name of the version to update")
+    new_name: str | None = Field(None, description="New name for the version")
+    new_description: str | None = Field(None, description="New description for the version")
+
+
+# Response Models
+class ProjectBasicInfo(BaseModel):
+    """Base model for describing a single project version, not for direct use in an endpoint."""
+
+    name: str = Field(..., description="Version name")
+    description: str | None = Field(..., description="Version description")
+
+
+class AddVersionResponse(ProjectBasicInfo):
+    """Response model for adding a version."""
+
+    created_at: str = Field(..., description="ISO timestamp when version was created")
+
+
+class ProjectVersionInfo(ProjectBasicInfo):
+    """Basic information about a project version. You get a list of these when listing all versions in a project."""
+
+    created_at: str = Field(..., description="ISO timestamp when version was created")
+    is_deleted: bool = Field(..., description="Whether this version has been soft-deleted")
+
+
+# Typed dict used as this is used in the db model to define the contents of the JSONB
+class FileDetails(TypedDict):
+    """Details about a single file in a project version, ETag (is checksum) and size is in bytes."""
+
+    version_id: str
+    etag: str
+    size: int
+
+
+class ProjectVersionDetailResponse(ProjectBasicInfo):
+    """Full information about a single project version, including the files at that version."""
+
+    created_at: str = Field(..., description="ISO timestamp when version was created")
+    is_deleted: bool = Field(..., description="Whether this version has been soft-deleted")
+    files: dict[str, FileDetails] = Field(
+        ..., description="Info about all files at this version, including: version IDs, etags and file sizes"
+    )
+
+
+class DeleteVersionResponse(BaseModel):
+    """Response model for deleting a version."""
+
+    name: str = Field(..., description="Name of the version that was deleted")
+    already_deleted: bool = Field(
+        False,
+        description="Whether the version was already soft-deleted before this request",
+    )
+    date_deleted: str = Field(..., description="ISO timestamp of when the version was soft-deleted")
+
+
+class UpdateVersionResponse(ProjectBasicInfo):
+    """Response model for updating a version."""
+
+    created_at: str = Field(..., description="ISO timestamp when version was created")

divbase_lib-0.1.0a1/src/divbase_lib/api_schemas/queries.py

@@ -0,0 +1,209 @@
+"""
+Schemas for query routes.
+"""
+
+from typing import Optional, Self
+
+from pydantic import BaseModel, ConfigDict, field_validator, model_validator
+
+from divbase_lib.utils import split_semicolon_bcftools_command_segments
+
+# Shared helper functions and models
+
+
+def validate_command_not_empty(command: str) -> str:
+    """
+    Validator function to ensure that the user-submitted bcftools command string is not empty or just whitespace.
+    If there are multiple segments separated by semicolons, it also validates that none of the segments are empty or whitespace.
+
+    Shared helper for command field validators in BcftoolsQueryRequest.command and BcftoolsQueryKwargs.command.
+    """
+    command_string = command.strip()
+    if command_string == "":
+        raise ValueError(
+            "The --command option must be a non-empty bcftools view string. "
+            'If you only want to subset based on samples, use --command "view -s" in combination with one of the sample-selection options: --tsv-filter, --samples, --samples-file.'
+        )
+
+    segments = split_semicolon_bcftools_command_segments(command_string)
+    for position, segment in enumerate(segments, start=1):
+        if segment.strip() == "":
+            raise ValueError(
+                f"The --command option has an empty pipeline segment at position {position}. "
+                "Use semicolons only between complete bcftools view commands."
+            )
+
+    return command
+
+
+class SharedBaseModel(BaseModel):
+    """Shared pydantic BaseModel for VCF query response and kwarg schemas."""
+
+    model_config = ConfigDict(
+        validate_assignment=True,
+        json_schema_extra={
+            "description": ("Exactly one sample-selection mode must be provided: tsv_filter, samples, or all_samples.")
+        },
+    )
+
+
+# Request models
+class SampleMetadataQueryRequest(BaseModel):
+    """Request model for sample metadata query route."""
+
+    tsv_filter: str
+    metadata_tsv_name: str
+
+
+class BcftoolsQueryRequest(SharedBaseModel):
+    """
+    Request model for sample metadata query route.
+    See ConfigDict in SharedBaseModel for validation rules around tsv_filter and samples fields.
+    """
+
+    tsv_filter: str | None = None  # Used for metadata mode only
+    metadata_tsv_name: str | None = None  # Used for metadata mode only
+    command: str  # bcftools command input with --command
+    samples: list[str] | None = None
+    all_samples: bool = False
+
+    @field_validator("command")
+    @classmethod
+    def validate_command(_cls, value: str) -> str:
+        return validate_command_not_empty(value)
+
+    @model_validator(mode="after")
+    def validate_sample_selection_mode(self) -> Self:
+        tsv_filter = self.tsv_filter
+        samples = self.samples
+        all_samples = self.all_samples
+
+        selection_count = 0
+        if tsv_filter is not None:
+            selection_count += 1
+        if samples is not None:
+            selection_count += 1
+        if all_samples:
+            selection_count += 1
+
+        if selection_count > 1:
+            raise ValueError("Only one of tsv_filter, samples, or all_samples may be provided.")
+        if selection_count == 0:
+            raise ValueError("One sample-selection mode must be provided (tsv_filter, samples, or all_samples).")
+
+        if tsv_filter is not None and self.metadata_tsv_name is None:
+            raise ValueError("metadata_tsv_name must be provided when tsv_filter is used.")
+
+        if samples is not None and len(samples) == 0:
+            raise ValueError("samples must contain at least one sample ID when provided.")
+
+        return self
+
+
+# Models for task kwargs and task results. Reused in task history schemas too, hence pydantic models and not just dataclasses.
+class SampleMetadataQueryKwargs(BaseModel):
+    """Keyword arguments for sample metadata query task. Used to pass info to Celery task, and also for recording task history."""
+
+    tsv_filter: str
+    metadata_tsv_name: str
+    bucket_name: str
+    project_id: int
+    project_name: str
+    user_id: int
+
+
+class BcftoolsQueryKwargs(SharedBaseModel):
+    """
+    Keyword arguments for BCFtools query task. Used to pass info to Celery task, and also for recording task history.
+    See ConfigDict in SharedBaseModel for validation rules around tsv_filter and samples fields.
+    """
+
+    tsv_filter: str | None = None  # Used for metadata mode only
+    metadata_tsv_name: str | None = None  # Used for metadata mode only
+    command: str  # bcftools command input with --command
+    bucket_name: str
+    project_id: int
+    project_name: str
+    user_id: int
+    job_id: int
+    samples: list[str] | None = None
+    all_samples: bool = False
+
+    @field_validator("command")
+    @classmethod
+    def validate_command(_cls, value: str) -> str:
+        return validate_command_not_empty(value)
+
+    @model_validator(mode="after")
+    def validate_sample_selection_mode(self) -> Self:
+        tsv_filter = self.tsv_filter
+        samples = self.samples
+        all_samples = self.all_samples
+
+        selection_count = 0
+        if tsv_filter is not None:
+            selection_count += 1
+        if samples is not None:
+            selection_count += 1
+        if all_samples:
+            selection_count += 1
+
+        if selection_count > 1:
+            raise ValueError("Only one of tsv_filter, samples, or all_samples may be provided.")
+        if selection_count == 0:
+            # Backward compatibility for historical task kwargs created before all_samples option was implemented.
+            # To ensure that task-history deserialization does not break for existing tasks.
+            # Could be handled by a backfilling migration instead.
+            self.all_samples = True
+            return self
+
+        if tsv_filter is not None and self.metadata_tsv_name is None:
+            raise ValueError("metadata_tsv_name must be provided when tsv_filter is used.")
+
+        if samples is not None and len(samples) == 0:
+            raise ValueError("samples must contain at least one sample ID when provided.")
+
+        return self
+
+
+class SampleFileMappingResult(BaseModel):
+    sample_id: str
+    filename: str
+
+    @model_validator(mode="before")
+    @classmethod
+    def map_legacy_task_history_keys(cls, data):
+        """
+        Support historical task-history payloads that stored uppercase keys.
+        """
+        if not isinstance(data, dict):
+            return data
+
+        normalized = dict(data)
+        if "sample_id" not in normalized:
+            if "Sample_ID" in normalized:
+                normalized["sample_id"] = normalized["Sample_ID"]
+            elif "#Sample_ID" in normalized:
+                normalized["sample_id"] = normalized["#Sample_ID"]
+        if "filename" not in normalized and "Filename" in normalized:
+            normalized["filename"] = normalized["Filename"]
+
+        return normalized
+
+
+class SampleMetadataQueryTaskResult(BaseModel):
+    """Metadata query task result details. Based on the return of tasks.sample_metadata_query."""
+
+    sample_and_filename_subset: list[SampleFileMappingResult]
+    unique_sample_ids: list[str]
+    unique_filenames: list[str]
+    query_message: str
+    warnings: list[str] = []
+    status: str | None = None
+
+
+class BcftoolsQueryTaskResult(BaseModel):
+    """BCFtools query task result details. Based on the return of tasks.bcftools_query."""
+
+    output_file: str
+    status: Optional[str] = None