deriva-ml 1.17.9__py3-none-any.whl → 1.17.11__py3-none-any.whl

deriva_ml/core/enums.py

@@ -7,7 +7,7 @@ Classes:
     BaseStrEnum: Base class for string-based enums.
     UploadState: States for file upload operations.
     Status: Execution status values.
-    BuiltinTypes: ERMrest built-in data types.
+    BuiltinTypes: Alias for BuiltinType from deriva.core.typed.
     MLVocab: Controlled vocabulary types.
     MLAsset: Asset type identifiers.
     ExecMetadataType: Execution metadata type identifiers.
@@ -16,7 +16,16 @@ Classes:
 
 from enum import Enum
 
-from deriva.core.ermrest_model import builtin_types
+# Import BuiltinType from deriva.core.typed
+from deriva.core.typed import BuiltinType
+
+# Backwards compatibility alias - DerivaML uses plural form
+BuiltinTypes = BuiltinType
+"""Alias for BuiltinType from deriva.core.typed.
+
+This maintains backwards compatibility with existing DerivaML code that uses
+the plural form 'BuiltinTypes'. New code should use BuiltinType directly.
+"""
 
 
 class BaseStrEnum(str, Enum):
@@ -85,78 +94,19 @@ class Status(BaseStrEnum):
     failed = "Failed"
 
 
-class BuiltinTypes(Enum):
-    """ERMrest built-in data types.
-
-    Maps ERMrest's built-in data types to their type names. These types are used for defining
-    column types in tables and for type validation.
-
-    Attributes:
-        text (str): Text/string type.
-        int2 (str): 16-bit integer.
-        jsonb (str): Binary JSON.
-        float8 (str): 64-bit float.
-        timestamp (str): Timestamp without timezone.
-        int8 (str): 64-bit integer.
-        boolean (str): Boolean type.
-        json (str): JSON type.
-        float4 (str): 32-bit float.
-        int4 (str): 32-bit integer.
-        timestamptz (str): Timestamp with timezone.
-        date (str): Date type.
-        ermrest_rid (str): Resource identifier.
-        ermrest_rcb (str): Record created by.
-        ermrest_rmb (str): Record modified by.
-        ermrest_rct (str): Record creation time.
-        ermrest_rmt (str): Record modification time.
-        markdown (str): Markdown text.
-        longtext (str): Long text.
-        ermrest_curie (str): Compact URI.
-        ermrest_uri (str): URI type.
-        color_rgb_hex (str): RGB color in hex.
-        serial2 (str): 16-bit auto-incrementing.
-        serial4 (str): 32-bit auto-incrementing.
-        serial8 (str): 64-bit auto-incrementing.
-    """
-
-    text = builtin_types.text.typename
-    int2 = builtin_types.int2.typename
-    jsonb = builtin_types.json.typename
-    float8 = builtin_types.float8.typename
-    timestamp = builtin_types.timestamp.typename
-    int8 = builtin_types.int8.typename
-    boolean = builtin_types.boolean.typename
-    json = builtin_types.json.typename
-    float4 = builtin_types.float4.typename
-    int4 = builtin_types.int4.typename
-    timestamptz = builtin_types.timestamptz.typename
-    date = builtin_types.date.typename
-    ermrest_rid = builtin_types.ermrest_rid.typename
-    ermrest_rcb = builtin_types.ermrest_rcb.typename
-    ermrest_rmb = builtin_types.ermrest_rmb.typename
-    ermrest_rct = builtin_types.ermrest_rct.typename
-    ermrest_rmt = builtin_types.ermrest_rmt.typename
-    markdown = builtin_types.markdown.typename
-    longtext = builtin_types.longtext.typename
-    ermrest_curie = builtin_types.ermrest_curie.typename
-    ermrest_uri = builtin_types.ermrest_uri.typename
-    color_rgb_hex = builtin_types.color_rgb_hex.typename
-    serial2 = builtin_types.serial2.typename
-    serial4 = builtin_types.serial4.typename
-    serial8 = builtin_types.serial8.typename
-
-
 class MLVocab(BaseStrEnum):
-    """Controlled vocabulary type identifiers.
+    """Controlled vocabulary table identifiers.
 
-    Defines the names of controlled vocabulary tables used in DerivaML for various types
-    of entities and attributes.
+    Defines the names of controlled vocabulary tables used in DerivaML. These tables
+    store standardized terms with descriptions and synonyms for consistent data
+    classification across the catalog.
 
     Attributes:
-        dataset_type (str): Dataset classification vocabulary.
-        workflow_type (str): Workflow classification vocabulary.
-        asset_type (str): Asset classification vocabulary.
-        asset_role (str): Asset role classification vocabulary.
+        dataset_type (str): Dataset classification vocabulary (e.g., "Training", "Test").
+        workflow_type (str): Workflow classification vocabulary (e.g., "Python", "Notebook").
+        asset_type (str): Asset/file type classification vocabulary (e.g., "Image", "CSV").
+        asset_role (str): Asset role vocabulary for execution relationships (e.g., "Input", "Output").
+        feature_name (str): Feature name vocabulary for ML feature definitions.
     """
 
     dataset_type = "Dataset_Type"
@@ -181,11 +131,29 @@ class MLAsset(BaseStrEnum):
 
 
 class MLTable(BaseStrEnum):
+    """Core ML schema table identifiers.
+
+    Defines the names of the core tables in the deriva-ml schema. These tables
+    form the backbone of the ML workflow tracking system.
+
+    Attributes:
+        dataset (str): Dataset table for versioned data collections.
+        workflow (str): Workflow table for computational pipeline definitions.
+        file (str): File table for tracking individual files.
+        asset (str): Asset table for domain-specific file types.
+        execution (str): Execution table for workflow run tracking.
+        execution_execution (str): Execution_Execution table for nested executions.
+        dataset_version (str): Dataset_Version table for version history.
+        execution_metadata (str): Execution_Metadata table for run metadata.
+        execution_asset (str): Execution_Asset table for run outputs.
+    """
+
     dataset = "Dataset"
     workflow = "Workflow"
     file = "File"
     asset = "Asset"
     execution = "Execution"
+    execution_execution = "Execution_Execution"
     dataset_version = "Dataset_Version"
     execution_metadata = "Execution_Metadata"
     execution_asset = "Execution_Asset"
@@ -197,23 +165,29 @@ class ExecMetadataType(BaseStrEnum):
     Defines the types of metadata that can be associated with an execution.
 
     Attributes:
-        execution_config (str): Execution configuration data.
+        execution_config (str): General execution configuration data.
         runtime_env (str): Runtime environment information.
+        hydra_config (str): Hydra YAML configuration files (config.yaml, overrides.yaml).
+        deriva_config (str): DerivaML execution configuration (configuration.json).
     """
 
     execution_config = "Execution_Config"
     runtime_env = "Runtime_Env"
+    hydra_config = "Hydra_Config"
+    deriva_config = "Deriva_Config"
 
 
 class ExecAssetType(BaseStrEnum):
     """Execution asset type identifiers.
 
-    Defines the types of assets that can be produced during an execution.
+    Defines the types of assets that can be produced or consumed during an execution.
+    These types are used to categorize files associated with workflow runs.
 
     Attributes:
-        input_file (str): Input file used by the execution.
+        input_file (str): Input file consumed by the execution.
         output_file (str): Output file produced by the execution.
         notebook_output (str): Jupyter notebook output from the execution.
+        model_file (str): Machine learning model file (e.g., .pkl, .h5, .pt).
     """
 
     input_file = "Input_File"

deriva_ml/core/ermrest.py

@@ -1,34 +1,71 @@
 """ERMrest data models for DerivaML.
 
-This module provides Pydantic models that represent ERMrest catalog structures. These models are used
+This module provides models that represent ERMrest catalog structures. These models are used
 throughout DerivaML for defining and manipulating catalog elements like tables, columns, and keys.
 
+The core definition classes (ColumnDef, KeyDef, ForeignKeyDef, TableDef) are now provided by
+`deriva.core.typed` and re-exported here for backwards compatibility.
+
 Classes:
     FileUploadState: Tracks the state of file uploads.
     VocabularyTerm: Represents terms in controlled vocabularies.
-    ColumnDefinition: Defines columns in tables.
-    KeyDefinition: Defines primary and unique keys.
-    ForeignKeyDefinition: Defines foreign key relationships.
-    TableDefinition: Defines complete table structures.
+    ColumnDefinition: Alias for ColumnDef from deriva.core.typed.
+    KeyDefinition: Alias for KeyDef from deriva.core.typed.
+    ForeignKeyDefinition: Alias for ForeignKeyDef from deriva.core.typed.
+    TableDefinition: Alias for TableDef from deriva.core.typed.
 """
 
 from __future__ import annotations
 
 import warnings
-from typing import Any, Iterable
+from dataclasses import dataclass
+from typing import Any, Protocol
 
-import deriva.core.ermrest_model as em
-from deriva.core.ermrest_model import builtin_types
 from pydantic import (
     BaseModel,
     Field,
+    PrivateAttr,
     computed_field,
-    field_validator,
-    model_serializer,
 )
 
 from .constants import RID
-from .enums import BuiltinTypes, UploadState
+from .enums import UploadState
+
+# Import and re-export typed definitions from deriva.core.typed
+from deriva.core.typed import (
+    ColumnDef,
+    KeyDef,
+    ForeignKeyDef,
+    TableDef,
+    VocabularyTableDef,
+    AssetTableDef,
+    AssociationTableDef,
+    SchemaDef,
+)
+
+# Re-export all typed classes for convenience
+__all__ = [
+    # New typed definitions from deriva.core.typed
+    "ColumnDef",
+    "KeyDef",
+    "ForeignKeyDef",
+    "TableDef",
+    "VocabularyTableDef",
+    "AssetTableDef",
+    "AssociationTableDef",
+    "SchemaDef",
+    # Legacy aliases for backwards compatibility
+    "ColumnDefinition",
+    "KeyDefinition",
+    "ForeignKeyDefinition",
+    "TableDefinition",
+    # DerivaML-specific classes
+    "FileUploadState",
+    "UploadProgress",
+    "UploadCallback",
+    "VocabularyTerm",
+    "VocabularyTermHandle",
+]
 
 # Pydantic warnings suppression
 warnings.filterwarnings("ignore", message='Field name "schema"', category=Warning, module="pydantic")
@@ -40,6 +77,46 @@ warnings.filterwarnings(
 )
 
 
+# =============================================================================
+# Compatibility Aliases
+# =============================================================================
+# These aliases maintain backwards compatibility with existing DerivaML code
+# that uses the old Pydantic-based class names.
+
+ColumnDefinition = ColumnDef
+"""Alias for ColumnDef from deriva.core.typed.
+
+This maintains backwards compatibility with existing DerivaML code.
+New code should use ColumnDef directly.
+"""
+
+KeyDefinition = KeyDef
+"""Alias for KeyDef from deriva.core.typed.
+
+This maintains backwards compatibility with existing DerivaML code.
+New code should use KeyDef directly.
+"""
+
+ForeignKeyDefinition = ForeignKeyDef
+"""Alias for ForeignKeyDef from deriva.core.typed.
+
+This maintains backwards compatibility with existing DerivaML code.
+New code should use ForeignKeyDef directly.
+"""
+
+TableDefinition = TableDef
+"""Alias for TableDef from deriva.core.typed.
+
+This maintains backwards compatibility with existing DerivaML code.
+New code should use TableDef directly.
+"""
+
+
+# =============================================================================
+# DerivaML-Specific Classes
+# =============================================================================
+
+
 class FileUploadState(BaseModel):
     """Tracks the state and result of a file upload operation.
 
@@ -47,7 +124,6 @@ class FileUploadState(BaseModel):
         state (UploadState): Current state of the upload (success, failed, etc.).
         status (str): Detailed status message.
         result (Any): Upload result data, if any.
-        rid (RID | None): Resource identifier of the uploaded file, if successful.
     """
     state: UploadState
     status: str
@@ -59,6 +135,53 @@ class FileUploadState(BaseModel):
         return self.result and self.result["RID"]
 
 
+@dataclass
+class UploadProgress:
+    """Progress information for file uploads.
+
+    This dataclass is passed to upload callbacks to report progress during
+    file upload operations.
+
+    Attributes:
+        file_path: Path to the file being uploaded.
+        file_name: Name of the file being uploaded.
+        bytes_completed: Number of bytes uploaded so far.
+        bytes_total: Total number of bytes to upload.
+        percent_complete: Percentage of upload completed (0-100).
+        phase: Current phase of the upload operation.
+        message: Human-readable status message.
+    """
+    file_path: str = ""
+    file_name: str = ""
+    bytes_completed: int = 0
+    bytes_total: int = 0
+    percent_complete: float = 0.0
+    phase: str = ""
+    message: str = ""
+
+
+class UploadCallback(Protocol):
+    """Protocol for upload progress callbacks.
+
+    Implement this protocol to receive progress updates during file uploads.
+    The callback is invoked with an UploadProgress object containing current
+    upload state information.
+
+    Example:
+        >>> def my_callback(progress: UploadProgress) -> None:
+        ...     print(f"Uploading {progress.file_name}: {progress.percent_complete:.1f}%")
+        ...
+        >>> execution.upload_execution_outputs(progress_callback=my_callback)
+    """
+    def __call__(self, progress: UploadProgress) -> None:
+        """Called with upload progress information.
+
+        Args:
+            progress: Current upload progress state.
+        """
+        ...
+
+
 class VocabularyTerm(BaseModel):
     """Represents a term in a controlled vocabulary.
 
@@ -83,206 +206,116 @@ class VocabularyTerm(BaseModel):
         ...     RID="1-abc123"
         ... )
     """
-    name: str = Field(alias="Name")
-    synonyms: list[str] | None = Field(alias="Synonyms")
+    _name: str = PrivateAttr()
+    _synonyms: list[str] | None = PrivateAttr()
+    _description: str = PrivateAttr()
     id: str = Field(alias="ID")
     uri: str = Field(alias="URI")
-    description: str = Field(alias="Description")
     rid: str = Field(alias="RID")
 
+    def __init__(self, **data):
+        # Extract fields that will be private attrs before calling super
+        name = data.pop("Name", None) or data.pop("name", None)
+        synonyms = data.pop("Synonyms", None) or data.pop("synonyms", None)
+        description = data.pop("Description", None) or data.pop("description", None)
+        super().__init__(**data)
+        self._name = name
+        self._synonyms = synonyms
+        self._description = description
+
+    @property
+    def name(self) -> str:
+        """Primary name of the term."""
+        return self._name
+
+    @property
+    def synonyms(self) -> tuple[str, ...]:
+        """Alternative names for the term (immutable)."""
+        return tuple(self._synonyms or [])
+
+    @property
+    def description(self) -> str:
+        """Explanation of the term's meaning."""
+        return self._description
+
     class Config:
         extra = "ignore"
 
 
-class ColumnDefinition(BaseModel):
-    """Defines a column in an ERMrest table.
+class VocabularyTermHandle(VocabularyTerm):
+    """A VocabularyTerm with methods to modify it in the catalog.
 
-    Provides a Pydantic model for defining columns with their types, constraints, and metadata.
-    Maps to deriva_py's Column.define functionality.
+    This class extends VocabularyTerm to provide mutable access to vocabulary
+    terms. Changes made through property setters are persisted to the catalog.
 
-    Attributes:
-        name (str): Name of the column.
-        type (BuiltinTypes): ERMrest data type for the column.
-        nullok (bool): Whether NULL values are allowed. Defaults to True.
-        default (Any): Default value for the column.
-        comment (str | None): Description of the column's purpose.
-        acls (dict): Access control lists.
-        acl_bindings (dict): Dynamic access control bindings.
-        annotations (dict): Additional metadata annotations.
-
-    Example:
-        >>> col = ColumnDefinition(
-        ...     name="score",
-        ...     type=BuiltinTypes.float4,
-        ...     nullok=False,
-        ...     comment="Confidence score between 0 and 1"
-        ... )
-    """
-    name: str
-    type: BuiltinTypes
-    nullok: bool = True
-    default: Any = None
-    comment: str | None = None
-    acls: dict = Field(default_factory=dict)
-    acl_bindings: dict = Field(default_factory=dict)
-    annotations: dict = Field(default_factory=dict)
-
-    @field_validator("type", mode="before")
-    @classmethod
-    def extract_type_name(cls, value: Any) -> Any:
-        if isinstance(value, dict):
-            return BuiltinTypes(value["typename"])
-        else:
-            return value
-
-    @model_serializer()
-    def serialize_column_definition(self):
-        return em.Column.define(
-            self.name,
-            builtin_types[self.type.value],
-            nullok=self.nullok,
-            default=self.default,
-            comment=self.comment,
-            acls=self.acls,
-            acl_bindings=self.acl_bindings,
-            annotations=self.annotations,
-        )
-
-
-class KeyDefinition(BaseModel):
-    """Defines a key constraint in an ERMrest table.
-
-    Provides a Pydantic model for defining primary keys and unique constraints.
-    Maps to deriva_py's Key.define functionality.
+    The `synonyms` property returns a tuple (immutable) to prevent accidental
+    modification without catalog update. To modify synonyms, assign a new
+    tuple/list to the property.
 
     Attributes:
-        colnames (Iterable[str]): Names of columns that form the key.
-        constraint_names (Iterable[str]): Names for the key constraints.
-        comment (str | None): Description of the key's purpose.
-        annotations (dict): Additional metadata annotations.
+        Inherits all attributes from VocabularyTerm.
 
     Example:
-        >>> key = KeyDefinition(
-        ...     colnames=["id", "version"],
-        ...     constraint_names=["unique_id_version"],
-        ...     comment="Unique identifier with version"
-        ... )
+        >>> term = ml.lookup_term("Dataset_Type", "Training")
+        >>> term.description = "Data used for model training"
+        >>> term.synonyms = ("Train", "TrainingData")
+        >>> term.delete()
     """
-    colnames: Iterable[str]
-    constraint_names: Iterable[str]
-    comment: str | None = None
-    annotations: dict = Field(default_factory=dict)
-
-    @model_serializer()
-    def serialize_key_definition(self):
-        return em.Key.define(
-            colnames=self.colnames,
-            constraint_names=self.constraint_names,
-            comment=self.comment,
-            annotations=self.annotations,
-        )
 
+    _ml: Any = PrivateAttr()
+    _table: str = PrivateAttr()
 
-class ForeignKeyDefinition(BaseModel):
-    """Defines a foreign key relationship between tables.
+    def __init__(self, ml: Any, table: str, **data):
+        """Initialize a VocabularyTermHandle.
 
-    Provides a Pydantic model for defining foreign key constraints with referential actions
-    and metadata. Maps to deriva_py's ForeignKey.define functionality.
+        Args:
+            ml: DerivaML instance for catalog operations.
+            table: Name of the vocabulary table containing this term.
+            **data: Term data (Name, Synonyms, Description, ID, URI, RID).
+        """
+        super().__init__(**data)
+        self._ml = ml
+        self._table = table
 
-    Attributes:
-        colnames (Iterable[str]): Names of columns in the referencing table.
-        pk_sname (str): Schema name of the referenced table.
-        pk_tname (str): Name of the referenced table.
-        pk_colnames (Iterable[str]): Names of columns in the referenced table.
-        constraint_names (Iterable[str]): Names for the foreign key constraints.
-        on_update (str): Action on update of referenced row. Defaults to "NO ACTION".
-        on_delete (str): Action on delete of referenced row. Defaults to "NO ACTION".
-        comment (str | None): Description of the relationship.
-        acls (dict): Access control lists.
-        acl_bindings (dict): Dynamic access control bindings.
-        annotations (dict): Additional metadata annotations.
+    @property
+    def description(self) -> str:
+        """Explanation of the term's meaning."""
+        return self._description
 
-    Example:
-        >>> fk = ForeignKeyDefinition(
-        ...     colnames=["dataset_id"],
-        ...     pk_sname="core",
-        ...     pk_tname="dataset",
-        ...     pk_colnames=["id"],
-        ...     on_delete="CASCADE"
-        ... )
-    """
-    colnames: Iterable[str]
-    pk_sname: str
-    pk_tname: str
-    pk_colnames: Iterable[str]
-    constraint_names: Iterable[str] = Field(default_factory=list)
-    on_update: str = "NO ACTION"
-    on_delete: str = "NO ACTION"
-    comment: str | None = None
-    acls: dict[str, Any] = Field(default_factory=dict)
-    acl_bindings: dict[str, Any] = Field(default_factory=dict)
-    annotations: dict[str, Any] = Field(default_factory=dict)
-
-    @model_serializer()
-    def serialize_fk_definition(self):
-        return em.ForeignKey.define(
-            fk_colnames=self.colnames,
-            pk_sname=self.pk_sname,
-            pk_tname=self.pk_tname,
-            pk_colnames=self.pk_colnames,
-            on_update=self.on_update,
-            on_delete=self.on_delete,
-            comment=self.comment,
-            acls=self.acls,
-            acl_bindings=self.acl_bindings,
-            annotations=self.annotations,
-        )
-
-
-class TableDefinition(BaseModel):
-    """Defines a complete table structure in ERMrest.
-
-    Provides a Pydantic model for defining tables with their columns, keys, and relationships.
-    Maps to deriva_py's Table.define functionality.
+    @description.setter
+    def description(self, value: str) -> None:
+        """Update the term's description in the catalog.
 
-    Attributes:
-        name (str): Name of the table.
-        column_defs (Iterable[ColumnDefinition]): Column definitions.
-        key_defs (Iterable[KeyDefinition]): Key constraint definitions.
-        fkey_defs (Iterable[ForeignKeyDefinition]): Foreign key relationship definitions.
-        comment (str | None): Description of the table's purpose.
-        acls (dict): Access control lists.
-        acl_bindings (dict): Dynamic access control bindings.
-        annotations (dict): Additional metadata annotations.
+        Args:
+            value: New description for the term.
+        """
+        self._ml._update_term_description(self._table, self.name, value)
+        self._description = value
 
-    Example:
-        >>> table = TableDefinition(
-        ...     name="experiment",
-        ...     column_defs=[
-        ...         ColumnDefinition(name="id", type=BuiltinTypes.text),
-        ...         ColumnDefinition(name="date", type=BuiltinTypes.date)
-        ...     ],
-        ...     comment="Experimental data records"
-        ... )
-    """
-    name: str
-    column_defs: Iterable[ColumnDefinition]
-    key_defs: Iterable[KeyDefinition] = Field(default_factory=list)
-    fkey_defs: Iterable[ForeignKeyDefinition] = Field(default_factory=list)
-    comment: str | None = None
-    acls: dict = Field(default_factory=dict)
-    acl_bindings: dict = Field(default_factory=dict)
-    annotations: dict = Field(default_factory=dict)
-
-    @model_serializer()
-    def serialize_table_definition(self):
-        return em.Table.define(
-            tname=self.name,
-            column_defs=[c.model_dump() for c in self.column_defs],
-            key_defs=[k.model_dump() for k in self.key_defs],
-            fkey_defs=[fk.model_dump() for fk in self.fkey_defs],
-            comment=self.comment,
-            acls=self.acls,
-            acl_bindings=self.acl_bindings,
-            annotations=self.annotations,
-        )
+    @property
+    def synonyms(self) -> tuple[str, ...]:
+        """Alternative names for the term (immutable).
+
+        Returns a tuple to prevent accidental modification without catalog update.
+        To modify synonyms, assign a new tuple/list to this property.
+        """
+        return tuple(self._synonyms or [])
+
+    @synonyms.setter
+    def synonyms(self, value: list[str] | tuple[str, ...]) -> None:
+        """Replace all synonyms for this term in the catalog.
+
+        Args:
+            value: New list of synonyms (replaces all existing synonyms).
+        """
+        new_synonyms = list(value)
+        self._ml._update_term_synonyms(self._table, self.name, new_synonyms)
+        self._synonyms = new_synonyms
+
+    def delete(self) -> None:
+        """Delete this term from the vocabulary.
+
+        Raises:
+            DerivaMLException: If the term is currently in use by other records.
+        """
+        self._ml.delete_term(self._table, self.name)