deriva-ml 1.17.10__py3-none-any.whl

deriva_ml/core/config.py

@@ -0,0 +1,69 @@
+import getpass
+import logging
+from pathlib import Path
+from typing import Any
+
+from hydra.conf import HydraConf, RunDir
+from hydra.core.hydra_config import HydraConfig
+from hydra_zen import store
+from omegaconf import OmegaConf
+from pydantic import BaseModel, model_validator
+
+from deriva_ml.core.definitions import ML_SCHEMA
+
+
+class DerivaMLConfig(BaseModel):
+    hostname: str
+    catalog_id: str | int = 1
+    domain_schema: str | None = None
+    project_name: str | None = None
+    cache_dir: str | Path | None = None
+    working_dir: str | Path | None = None
+    hydra_runtime_output_dir: str | Path | None = None
+    ml_schema: str = ML_SCHEMA
+    logging_level: Any = logging.WARNING
+    deriva_logging_level: Any = logging.WARNING
+    credential: Any = None
+    use_minid: bool = True
+    check_auth: bool = True
+
+    @model_validator(mode="after")
+    def init_working_dir(self):
+        """
+        Sets up the working directory for the model.
+
+        This method configures the working directory, ensuring that all required
+        file operations are performed in the appropriate location. If the user does not
+        specify a directory, a default directory based on the user's home directory
+        or username will be used.
+
+        This is a repeat of what is in the DerivaML.__init__ bu we put this here so that the working
+        directory is available to hydra.
+
+        Returns:
+            Self: The object instance with the working directory initialized.
+        """
+
+        self.working_dir = DerivaMLConfig.compute_workdir(self.working_dir)
+        self.hydra_runtime_output_dir = Path(HydraConfig.get().runtime.output_dir)
+        return self
+
+    @staticmethod
+    def compute_workdir(working_dir) -> Path:
+        # Create a default working directory if none is provided.  If a working directory is provided, we add the
+        # user name to it to ensure that multiple users do not overwrite each other's work.'
+        working_dir = (Path(working_dir) / getpass.getuser() if working_dir else Path.home()) / "deriva-ml"
+        return working_dir.absolute()
+
+
+OmegaConf.register_new_resolver("compute_workdir", DerivaMLConfig.compute_workdir, replace=True)
+store(
+    HydraConf(
+        run=RunDir("${compute_workdir:${deriva_ml.working_dir}}/hydra/${now:%Y-%m-%d_%H-%M-%S}"),
+        output_subdir="hydra-config",
+    ),
+    group="hydra",
+    name="config",
+)
+
+store.add_to_hydra_store()

deriva_ml/core/constants.py

@@ -0,0 +1,36 @@
+"""
+Constants used throughout the DerivaML package.
+"""
+
+from __future__ import annotations
+
+from typing import NewType, TypeAlias
+
+from pydantic import constr
+
+# Schema name
+ML_SCHEMA = "deriva-ml"
+
+# Special RID for dry runs
+DRY_RUN_RID = "0000"
+
+# Regular expression parts for RIDs
+rid_part = r"(?P<rid>(?:[A-Z\d]{1,4}|[A-Z\d]{1,4}(?:-[A-Z\d]{4})+))"
+snapshot_part = r"(?:@(?P<snapshot>(?:[A-Z\d]{1,4}|[A-Z\d]{1,4}(?:-[A-Z\d]{4})+)))?"
+rid_regex = f"^{rid_part}{snapshot_part}$"
+
+# RID type definition
+BaseRIDString = constr(pattern=rid_regex)
+# RID = TypeVar("RID", bound=BaseRIDString)
+RIDType: TypeAlias = constr(pattern=rid_regex)
+RID = NewType("RID", BaseRIDString)
+
+# System columns in Deriva
+DerivaSystemColumns = ["RID", "RCT", "RMT", "RCB", "RMB"]
+DerivaAssetColumns = {
+    "Filename",
+    "URL",
+    "Length",
+    "MD5",
+    "Description",
+}.union(set(DerivaSystemColumns))

deriva_ml/core/definitions.py

@@ -0,0 +1,74 @@
+"""
+Shared definitions that are used in different DerivaML modules.
+This module re-exports all symbols from the core submodules for backwards compatibility.
+"""
+
+from __future__ import annotations
+
+# Re-export constants
+from deriva_ml.core.constants import (
+    DRY_RUN_RID,
+    ML_SCHEMA,
+    RID,
+    DerivaAssetColumns,
+    DerivaSystemColumns,
+    rid_part,
+    rid_regex,
+    snapshot_part,
+)
+
+# Re-export enums
+from deriva_ml.core.enums import (
+    BaseStrEnum,
+    BuiltinTypes,
+    ExecAssetType,
+    ExecMetadataType,
+    MLAsset,
+    MLTable,
+    MLVocab,
+    Status,
+    UploadState,
+)
+
+# Re-export models
+from deriva_ml.core.ermrest import (
+    ColumnDefinition,
+    FileUploadState,
+    ForeignKeyDefinition,
+    KeyDefinition,
+    TableDefinition,
+    VocabularyTerm,
+)
+
+# Re-export exceptions
+from deriva_ml.core.filespec import FileSpec
+
+__all__ = [
+    # Constants
+    "ML_SCHEMA",
+    "DRY_RUN_RID",
+    "rid_part",
+    "snapshot_part",
+    "rid_regex",
+    "DerivaSystemColumns",
+    "DerivaAssetColumns",
+    "RID",
+    # Enums
+    "BaseStrEnum",
+    "UploadState",
+    "Status",
+    "BuiltinTypes",
+    "MLVocab",
+    "MLTable",
+    "MLAsset",
+    "ExecMetadataType",
+    "ExecAssetType",
+    # Models
+    "FileUploadState",
+    "FileSpec",
+    "VocabularyTerm",
+    "ColumnDefinition",
+    "KeyDefinition",
+    "ForeignKeyDefinition",
+    "TableDefinition",
+]

deriva_ml/core/enums.py

@@ -0,0 +1,222 @@
+"""Enumeration classes for DerivaML.
+
+This module provides enumeration classes used throughout DerivaML for representing states, statuses,
+types, and vocabularies. Each enum class represents a specific set of constants used in the system.
+
+Classes:
+    BaseStrEnum: Base class for string-based enums.
+    UploadState: States for file upload operations.
+    Status: Execution status values.
+    BuiltinTypes: ERMrest built-in data types.
+    MLVocab: Controlled vocabulary types.
+    MLAsset: Asset type identifiers.
+    ExecMetadataType: Execution metadata type identifiers.
+    ExecAssetType: Execution asset type identifiers.
+"""
+
+from enum import Enum
+
+from deriva.core.ermrest_model import builtin_types
+
+
+class BaseStrEnum(str, Enum):
+    """Base class for string-based enumerations.
+
+    Extends both str and Enum to create string enums that are both string-like and enumerated.
+    This provides type safety while maintaining string compatibility.
+
+    Example:
+        >>> class MyEnum(BaseStrEnum):
+        ...     VALUE = "value"
+        >>> isinstance(MyEnum.VALUE, str)  # True
+        >>> isinstance(MyEnum.VALUE, Enum)  # True
+    """
+
+    pass
+
+
+class UploadState(Enum):
+    """File upload operation states.
+
+    Represents the various states a file upload operation can be in, from initiation to completion.
+
+    Attributes:
+        success (int): Upload completed successfully.
+        failed (int): Upload failed.
+        pending (int): Upload is queued.
+        running (int): Upload is in progress.
+        paused (int): Upload is temporarily paused.
+        aborted (int): Upload was aborted.
+        cancelled (int): Upload was cancelled.
+        timeout (int): Upload timed out.
+    """
+
+    success = 0
+    failed = 1
+    pending = 2
+    running = 3
+    paused = 4
+    aborted = 5
+    cancelled = 6
+    timeout = 7
+
+
+class Status(BaseStrEnum):
+    """Execution status values.
+
+    Represents the various states an execution can be in throughout its lifecycle.
+
+    Attributes:
+        initializing (str): Initial setup is in progress.
+        created (str): Execution record has been created.
+        pending (str): Execution is queued.
+        running (str): Execution is in progress.
+        aborted (str): Execution was manually stopped.
+        completed (str): Execution finished successfully.
+        failed (str): Execution encountered an error.
+    """
+
+    initializing = "Initializing"
+    created = "Created"
+    pending = "Pending"
+    running = "Running"
+    aborted = "Aborted"
+    completed = "Completed"
+    failed = "Failed"
+
+
+class BuiltinTypes(Enum):
+    """ERMrest built-in data types.
+
+    Maps ERMrest's built-in data types to their type names. These types are used for defining
+    column types in tables and for type validation.
+
+    Attributes:
+        text (str): Text/string type.
+        int2 (str): 16-bit integer.
+        jsonb (str): Binary JSON.
+        float8 (str): 64-bit float.
+        timestamp (str): Timestamp without timezone.
+        int8 (str): 64-bit integer.
+        boolean (str): Boolean type.
+        json (str): JSON type.
+        float4 (str): 32-bit float.
+        int4 (str): 32-bit integer.
+        timestamptz (str): Timestamp with timezone.
+        date (str): Date type.
+        ermrest_rid (str): Resource identifier.
+        ermrest_rcb (str): Record created by.
+        ermrest_rmb (str): Record modified by.
+        ermrest_rct (str): Record creation time.
+        ermrest_rmt (str): Record modification time.
+        markdown (str): Markdown text.
+        longtext (str): Long text.
+        ermrest_curie (str): Compact URI.
+        ermrest_uri (str): URI type.
+        color_rgb_hex (str): RGB color in hex.
+        serial2 (str): 16-bit auto-incrementing.
+        serial4 (str): 32-bit auto-incrementing.
+        serial8 (str): 64-bit auto-incrementing.
+    """
+
+    text = builtin_types.text.typename
+    int2 = builtin_types.int2.typename
+    jsonb = builtin_types.json.typename
+    float8 = builtin_types.float8.typename
+    timestamp = builtin_types.timestamp.typename
+    int8 = builtin_types.int8.typename
+    boolean = builtin_types.boolean.typename
+    json = builtin_types.json.typename
+    float4 = builtin_types.float4.typename
+    int4 = builtin_types.int4.typename
+    timestamptz = builtin_types.timestamptz.typename
+    date = builtin_types.date.typename
+    ermrest_rid = builtin_types.ermrest_rid.typename
+    ermrest_rcb = builtin_types.ermrest_rcb.typename
+    ermrest_rmb = builtin_types.ermrest_rmb.typename
+    ermrest_rct = builtin_types.ermrest_rct.typename
+    ermrest_rmt = builtin_types.ermrest_rmt.typename
+    markdown = builtin_types.markdown.typename
+    longtext = builtin_types.longtext.typename
+    ermrest_curie = builtin_types.ermrest_curie.typename
+    ermrest_uri = builtin_types.ermrest_uri.typename
+    color_rgb_hex = builtin_types.color_rgb_hex.typename
+    serial2 = builtin_types.serial2.typename
+    serial4 = builtin_types.serial4.typename
+    serial8 = builtin_types.serial8.typename
+
+
+class MLVocab(BaseStrEnum):
+    """Controlled vocabulary type identifiers.
+
+    Defines the names of controlled vocabulary tables used in DerivaML for various types
+    of entities and attributes.
+
+    Attributes:
+        dataset_type (str): Dataset classification vocabulary.
+        workflow_type (str): Workflow classification vocabulary.
+        asset_type (str): Asset classification vocabulary.
+        asset_role (str): Asset role classification vocabulary.
+    """
+
+    dataset_type = "Dataset_Type"
+    workflow_type = "Workflow_Type"
+    asset_type = "Asset_Type"
+    asset_role = "Asset_Role"
+    feature_name = "Feature_Name"
+
+
+class MLAsset(BaseStrEnum):
+    """Asset type identifiers.
+
+    Defines the types of assets that can be associated with executions.
+
+    Attributes:
+        execution_metadata (str): Metadata about an execution.
+        execution_asset (str): Asset produced by an execution.
+    """
+
+    execution_metadata = "Execution_Metadata"
+    execution_asset = "Execution_Asset"
+
+
+class MLTable(BaseStrEnum):
+    dataset = "Dataset"
+    workflow = "Workflow"
+    file = "File"
+    asset = "Asset"
+    execution = "Execution"
+    dataset_version = "Dataset_Version"
+    execution_metadata = "Execution_Metadata"
+    execution_asset = "Execution_Asset"
+
+
+class ExecMetadataType(BaseStrEnum):
+    """Execution metadata type identifiers.
+
+    Defines the types of metadata that can be associated with an execution.
+
+    Attributes:
+        execution_config (str): Execution configuration data.
+        runtime_env (str): Runtime environment information.
+    """
+
+    execution_config = "Execution_Config"
+    runtime_env = "Runtime_Env"
+
+
+class ExecAssetType(BaseStrEnum):
+    """Execution asset type identifiers.
+
+    Defines the types of assets that can be produced during an execution.
+
+    Attributes:
+        input_file (str): Input file used by the execution.
+        output_file (str): Output file produced by the execution.
+        notebook_output (str): Jupyter notebook output from the execution.
+    """
+
+    input_file = "Input_File"
+    output_file = "Output_File"
+    notebook_output = "Notebook_Output"
+    model_file = "Model_File"

deriva_ml/core/ermrest.py

@@ -0,0 +1,288 @@
+"""ERMrest data models for DerivaML.
+
+This module provides Pydantic models that represent ERMrest catalog structures. These models are used
+throughout DerivaML for defining and manipulating catalog elements like tables, columns, and keys.
+
+Classes:
+    FileUploadState: Tracks the state of file uploads.
+    VocabularyTerm: Represents terms in controlled vocabularies.
+    ColumnDefinition: Defines columns in tables.
+    KeyDefinition: Defines primary and unique keys.
+    ForeignKeyDefinition: Defines foreign key relationships.
+    TableDefinition: Defines complete table structures.
+"""
+
+from __future__ import annotations
+
+import warnings
+from typing import Any, Iterable
+
+import deriva.core.ermrest_model as em
+from deriva.core.ermrest_model import builtin_types
+from pydantic import (
+    BaseModel,
+    Field,
+    computed_field,
+    field_validator,
+    model_serializer,
+)
+
+from .constants import RID
+from .enums import BuiltinTypes, UploadState
+
+# Pydantic warnings suppression
+warnings.filterwarnings("ignore", message='Field name "schema"', category=Warning, module="pydantic")
+warnings.filterwarnings(
+    "ignore",
+    message="fields may not start with an underscore",
+    category=Warning,
+    module="pydantic",
+)
+
+
+class FileUploadState(BaseModel):
+    """Tracks the state and result of a file upload operation.
+
+    Attributes:
+        state (UploadState): Current state of the upload (success, failed, etc.).
+        status (str): Detailed status message.
+        result (Any): Upload result data, if any.
+        rid (RID | None): Resource identifier of the uploaded file, if successful.
+    """
+    state: UploadState
+    status: str
+    result: Any
+
+    @computed_field
+    @property
+    def rid(self) -> RID | None:
+        return self.result and self.result["RID"]
+
+
+class VocabularyTerm(BaseModel):
+    """Represents a term in a controlled vocabulary.
+
+    A vocabulary term is a standardized entry in a controlled vocabulary table. Each term has
+    a primary name, optional synonyms, and identifiers for cross-referencing.
+
+    Attributes:
+        name (str): Primary name of the term.
+        synonyms (list[str] | None): Alternative names for the term.
+        id (str): CURIE (Compact URI) identifier.
+        uri (str): Full URI for the term.
+        description (str): Explanation of the term's meaning.
+        rid (str): Resource identifier in the catalog.
+
+    Example:
+        >>> term = VocabularyTerm(
+        ...     Name="epithelial",
+        ...     Synonyms=["epithelium"],
+        ...     ID="tissue:0001",
+        ...     URI="http://example.org/tissue/0001",
+        ...     Description="Epithelial tissue type",
+        ...     RID="1-abc123"
+        ... )
+    """
+    name: str = Field(alias="Name")
+    synonyms: list[str] | None = Field(alias="Synonyms")
+    id: str = Field(alias="ID")
+    uri: str = Field(alias="URI")
+    description: str = Field(alias="Description")
+    rid: str = Field(alias="RID")
+
+    class Config:
+        extra = "ignore"
+
+
+class ColumnDefinition(BaseModel):
+    """Defines a column in an ERMrest table.
+
+    Provides a Pydantic model for defining columns with their types, constraints, and metadata.
+    Maps to deriva_py's Column.define functionality.
+
+    Attributes:
+        name (str): Name of the column.
+        type (BuiltinTypes): ERMrest data type for the column.
+        nullok (bool): Whether NULL values are allowed. Defaults to True.
+        default (Any): Default value for the column.
+        comment (str | None): Description of the column's purpose.
+        acls (dict): Access control lists.
+        acl_bindings (dict): Dynamic access control bindings.
+        annotations (dict): Additional metadata annotations.
+
+    Example:
+        >>> col = ColumnDefinition(
+        ...     name="score",
+        ...     type=BuiltinTypes.float4,
+        ...     nullok=False,
+        ...     comment="Confidence score between 0 and 1"
+        ... )
+    """
+    name: str
+    type: BuiltinTypes
+    nullok: bool = True
+    default: Any = None
+    comment: str | None = None
+    acls: dict = Field(default_factory=dict)
+    acl_bindings: dict = Field(default_factory=dict)
+    annotations: dict = Field(default_factory=dict)
+
+    @field_validator("type", mode="before")
+    @classmethod
+    def extract_type_name(cls, value: Any) -> Any:
+        if isinstance(value, dict):
+            return BuiltinTypes(value["typename"])
+        else:
+            return value
+
+    @model_serializer()
+    def serialize_column_definition(self):
+        return em.Column.define(
+            self.name,
+            builtin_types[self.type.value],
+            nullok=self.nullok,
+            default=self.default,
+            comment=self.comment,
+            acls=self.acls,
+            acl_bindings=self.acl_bindings,
+            annotations=self.annotations,
+        )
+
+
+class KeyDefinition(BaseModel):
+    """Defines a key constraint in an ERMrest table.
+
+    Provides a Pydantic model for defining primary keys and unique constraints.
+    Maps to deriva_py's Key.define functionality.
+
+    Attributes:
+        colnames (Iterable[str]): Names of columns that form the key.
+        constraint_names (Iterable[str]): Names for the key constraints.
+        comment (str | None): Description of the key's purpose.
+        annotations (dict): Additional metadata annotations.
+
+    Example:
+        >>> key = KeyDefinition(
+        ...     colnames=["id", "version"],
+        ...     constraint_names=["unique_id_version"],
+        ...     comment="Unique identifier with version"
+        ... )
+    """
+    colnames: Iterable[str]
+    constraint_names: Iterable[str]
+    comment: str | None = None
+    annotations: dict = Field(default_factory=dict)
+
+    @model_serializer()
+    def serialize_key_definition(self):
+        return em.Key.define(
+            colnames=self.colnames,
+            constraint_names=self.constraint_names,
+            comment=self.comment,
+            annotations=self.annotations,
+        )
+
+
+class ForeignKeyDefinition(BaseModel):
+    """Defines a foreign key relationship between tables.
+
+    Provides a Pydantic model for defining foreign key constraints with referential actions
+    and metadata. Maps to deriva_py's ForeignKey.define functionality.
+
+    Attributes:
+        colnames (Iterable[str]): Names of columns in the referencing table.
+        pk_sname (str): Schema name of the referenced table.
+        pk_tname (str): Name of the referenced table.
+        pk_colnames (Iterable[str]): Names of columns in the referenced table.
+        constraint_names (Iterable[str]): Names for the foreign key constraints.
+        on_update (str): Action on update of referenced row. Defaults to "NO ACTION".
+        on_delete (str): Action on delete of referenced row. Defaults to "NO ACTION".
+        comment (str | None): Description of the relationship.
+        acls (dict): Access control lists.
+        acl_bindings (dict): Dynamic access control bindings.
+        annotations (dict): Additional metadata annotations.
+
+    Example:
+        >>> fk = ForeignKeyDefinition(
+        ...     colnames=["dataset_id"],
+        ...     pk_sname="core",
+        ...     pk_tname="dataset",
+        ...     pk_colnames=["id"],
+        ...     on_delete="CASCADE"
+        ... )
+    """
+    colnames: Iterable[str]
+    pk_sname: str
+    pk_tname: str
+    pk_colnames: Iterable[str]
+    constraint_names: Iterable[str] = Field(default_factory=list)
+    on_update: str = "NO ACTION"
+    on_delete: str = "NO ACTION"
+    comment: str | None = None
+    acls: dict[str, Any] = Field(default_factory=dict)
+    acl_bindings: dict[str, Any] = Field(default_factory=dict)
+    annotations: dict[str, Any] = Field(default_factory=dict)
+
+    @model_serializer()
+    def serialize_fk_definition(self):
+        return em.ForeignKey.define(
+            fk_colnames=self.colnames,
+            pk_sname=self.pk_sname,
+            pk_tname=self.pk_tname,
+            pk_colnames=self.pk_colnames,
+            on_update=self.on_update,
+            on_delete=self.on_delete,
+            comment=self.comment,
+            acls=self.acls,
+            acl_bindings=self.acl_bindings,
+            annotations=self.annotations,
+        )
+
+
+class TableDefinition(BaseModel):
+    """Defines a complete table structure in ERMrest.
+
+    Provides a Pydantic model for defining tables with their columns, keys, and relationships.
+    Maps to deriva_py's Table.define functionality.
+
+    Attributes:
+        name (str): Name of the table.
+        column_defs (Iterable[ColumnDefinition]): Column definitions.
+        key_defs (Iterable[KeyDefinition]): Key constraint definitions.
+        fkey_defs (Iterable[ForeignKeyDefinition]): Foreign key relationship definitions.
+        comment (str | None): Description of the table's purpose.
+        acls (dict): Access control lists.
+        acl_bindings (dict): Dynamic access control bindings.
+        annotations (dict): Additional metadata annotations.
+
+    Example:
+        >>> table = TableDefinition(
+        ...     name="experiment",
+        ...     column_defs=[
+        ...         ColumnDefinition(name="id", type=BuiltinTypes.text),
+        ...         ColumnDefinition(name="date", type=BuiltinTypes.date)
+        ...     ],
+        ...     comment="Experimental data records"
+        ... )
+    """
+    name: str
+    column_defs: Iterable[ColumnDefinition]
+    key_defs: Iterable[KeyDefinition] = Field(default_factory=list)
+    fkey_defs: Iterable[ForeignKeyDefinition] = Field(default_factory=list)
+    comment: str | None = None
+    acls: dict = Field(default_factory=dict)
+    acl_bindings: dict = Field(default_factory=dict)
+    annotations: dict = Field(default_factory=dict)
+
+    @model_serializer()
+    def serialize_table_definition(self):
+        return em.Table.define(
+            tname=self.name,
+            column_defs=[c.model_dump() for c in self.column_defs],
+            key_defs=[k.model_dump() for k in self.key_defs],
+            fkey_defs=[fk.model_dump() for fk in self.fkey_defs],
+            comment=self.comment,
+            acls=self.acls,
+            acl_bindings=self.acl_bindings,
+            annotations=self.annotations,
+        )