deriva-ml 1.17.10__py3-none-any.whl → 1.17.11__py3-none-any.whl

deriva_ml/core/mixins/feature.py

@@ -0,0 +1,365 @@
+"""Feature management mixin for DerivaML.
+
+This module provides the FeatureMixin class which handles
+feature operations including creating, looking up, deleting,
+and listing feature values.
+"""
+
+from __future__ import annotations
+
+from itertools import chain
+from typing import TYPE_CHECKING, Any, Callable, Iterable
+
+# Deriva imports - use importlib to avoid shadowing by local 'deriva.py' files
+import importlib
+datapath = importlib.import_module("deriva.core.datapath")
+_ermrest_model = importlib.import_module("deriva.core.ermrest_model")
+Key = _ermrest_model.Key
+Table = _ermrest_model.Table
+
+from pydantic import ConfigDict, validate_call
+
+from deriva_ml.core.definitions import ColumnDefinition, VocabularyTerm
+from deriva_ml.core.exceptions import DerivaMLException
+from deriva_ml.feature import Feature, FeatureRecord
+
+if TYPE_CHECKING:
+    from deriva_ml.model.catalog import DerivaModel
+
+
+class FeatureMixin:
+    """Mixin providing feature management operations.
+
+    This mixin requires the host class to have:
+        - model: DerivaModel instance
+        - ml_schema: str - name of the ML schema
+        - domain_schema: str - name of the domain schema
+        - pathBuilder(): method returning catalog path builder
+        - add_term(): method for adding vocabulary terms (from VocabularyMixin)
+        - apply_catalog_annotations(): method to update navbar (from DerivaML base class)
+
+    Methods:
+        create_feature: Create a new feature definition
+        feature_record_class: Get pydantic model class for feature records
+        delete_feature: Remove a feature definition
+        lookup_feature: Retrieve a Feature object
+        find_features: Find all features in the catalog, optionally filtered by table
+        list_feature_values: Get all values for a feature
+    """
+
+    # Type hints for IDE support - actual attributes/methods from host class
+    model: "DerivaModel"
+    ml_schema: str
+    domain_schemas: frozenset[str]
+    default_schema: str | None
+    pathBuilder: Callable[[], Any]
+    add_term: Callable[..., VocabularyTerm]
+    apply_catalog_annotations: Callable[[], None]
+
+    def create_feature(
+        self,
+        target_table: Table | str,
+        feature_name: str,
+        terms: list[Table | str] | None = None,
+        assets: list[Table | str] | None = None,
+        metadata: list[ColumnDefinition | Table | Key | str] | None = None,
+        optional: list[str] | None = None,
+        comment: str = "",
+        update_navbar: bool = True,
+    ) -> type[FeatureRecord]:
+        """Creates a new feature definition.
+
+        A feature represents a measurable property or characteristic that can be associated with records in the target
+        table. Features can include vocabulary terms, asset references, and additional metadata.
+
+        **Side Effects**:
+        This method dynamically creates:
+        1. A new association table in the domain schema to store feature values
+        2. A Pydantic model class (subclass of FeatureRecord) for creating validated feature instances
+
+        The returned Pydantic model class provides type-safe construction of feature records with
+        automatic validation of values against the feature's definition (vocabulary terms, asset
+        references, etc.). Use this class to create feature instances that can be inserted into
+        the catalog.
+
+        Args:
+            target_table: Table to associate the feature with (name or Table object).
+            feature_name: Unique name for the feature within the target table.
+            terms: Optional vocabulary tables/names whose terms can be used as feature values.
+            assets: Optional asset tables/names that can be referenced by this feature.
+            metadata: Optional columns, tables, or keys to include in a feature definition.
+            optional: Column names that are not required when creating feature instances.
+            comment: Description of the feature's purpose and usage.
+            update_navbar: If True (default), automatically updates the navigation bar to include
+                the new feature table. Set to False during batch feature creation to avoid
+                redundant updates, then call apply_catalog_annotations() once at the end.
+
+        Returns:
+            type[FeatureRecord]: A dynamically generated Pydantic model class for creating
+                validated feature instances. The class has fields corresponding to the feature's
+                terms, assets, and metadata columns.
+
+        Raises:
+            DerivaMLException: If a feature definition is invalid or conflicts with existing features.
+
+        Examples:
+            Create a feature with confidence score:
+                >>> DiagnosisFeature = ml.create_feature(
+                ...     target_table="Image",
+                ...     feature_name="Diagnosis",
+                ...     terms=["Diagnosis_Type"],
+                ...     metadata=[ColumnDefinition(name="confidence", type=BuiltinTypes.float4)],
+                ...     comment="Clinical diagnosis label"
+                ... )
+                >>> # Use the returned class to create validated feature instances
+                >>> record = DiagnosisFeature(
+                ...     Image="1-ABC",  # Target record RID
+                ...     Diagnosis_Type="Normal",  # Vocabulary term
+                ...     confidence=0.95,
+                ...     Execution="2-XYZ"  # Execution that produced this value
+                ... )
+        """
+        # Initialize empty collections if None provided
+        terms = terms or []
+        assets = assets or []
+        metadata = metadata or []
+        optional = optional or []
+
+        def normalize_metadata(m: Key | Table | ColumnDefinition | str | dict) -> Key | Table | dict:
+            """Helper function to normalize metadata references.
+
+            Handles:
+            - str: Table name, converted to Table object
+            - ColumnDefinition: Dataclass with to_dict() method
+            - dict: Already in dict format (from Column.define())
+            - Key/Table: Passed through unchanged
+            """
+            if isinstance(m, str):
+                return self.model.name_to_table(m)
+            elif isinstance(m, dict):
+                # Already a dict (e.g., from Column.define())
+                return m
+            elif hasattr(m, 'to_dict'):
+                # ColumnDefinition or similar dataclass
+                return m.to_dict()
+            else:
+                return m
+
+        # Validate asset and term tables
+        if not all(map(self.model.is_asset, assets)):
+            raise DerivaMLException("Invalid create_feature asset table.")
+        if not all(map(self.model.is_vocabulary, terms)):
+            raise DerivaMLException("Invalid create_feature asset table.")
+
+        # Get references to required tables
+        target_table = self.model.name_to_table(target_table)
+        execution = self.model.schemas[self.ml_schema].tables["Execution"]
+        feature_name_table = self.model.schemas[self.ml_schema].tables["Feature_Name"]
+
+        # Add feature name to vocabulary
+        feature_name_term = self.add_term("Feature_Name", feature_name, description=comment)
+        atable_name = f"Execution_{target_table.name}_{feature_name_term.name}"
+        # Create an association table implementing the feature
+        atable = self.model.create_table(
+            target_table.define_association(
+                table_name=atable_name,
+                associates=[execution, target_table, feature_name_table],
+                metadata=[normalize_metadata(m) for m in chain(assets, terms, metadata)],
+                comment=comment,
+            )
+        )
+        # Configure optional columns and default feature name
+        for c in optional:
+            atable.columns[c].alter(nullok=True)
+        atable.columns["Feature_Name"].alter(default=feature_name_term.name)
+
+        # Update navbar to include the new feature table
+        if update_navbar:
+            self.apply_catalog_annotations()
+
+        # Return feature record class for creating instances
+        return self.feature_record_class(target_table, feature_name)
+
+    def feature_record_class(self, table: str | Table, feature_name: str) -> type[FeatureRecord]:
+        """Returns a dynamically generated Pydantic model class for creating feature records.
+
+        Each feature has a unique set of columns based on its definition (terms, assets, metadata).
+        This method returns a Pydantic class with fields corresponding to those columns, providing:
+
+        - **Type validation**: Values are validated against expected types (str, int, float, Path)
+        - **Required field checking**: Non-nullable columns must be provided
+        - **Default values**: Feature_Name is pre-filled with the feature's name
+
+        **Field types in the generated class:**
+        - `{TargetTable}` (str): Required. RID of the target record (e.g., Image RID)
+        - `Execution` (str, optional): RID of the execution for provenance tracking
+        - `Feature_Name` (str): Pre-filled with the feature name
+        - Term columns (str): Accept vocabulary term names
+        - Asset columns (str | Path): Accept asset RIDs or file paths
+        - Value columns: Accept values matching the column type (int, float, str)
+
+        Use `lookup_feature()` to inspect the feature's structure and see what columns
+        are available.
+
+        Args:
+            table: The table containing the feature, either as name or Table object.
+            feature_name: Name of the feature to create a record class for.
+
+        Returns:
+            type[FeatureRecord]: A Pydantic model class for creating validated feature records.
+                The class name follows the pattern `{TargetTable}Feature{FeatureName}`.
+
+        Raises:
+            DerivaMLException: If the feature doesn't exist or the table is invalid.
+
+        Example:
+            >>> # Get the dynamically generated class
+            >>> DiagnosisFeature = ml.feature_record_class("Image", "Diagnosis")
+            >>>
+            >>> # Create a validated feature record
+            >>> record = DiagnosisFeature(
+            ...     Image="1-ABC",           # Target record RID
+            ...     Diagnosis_Type="Normal", # Vocabulary term
+            ...     confidence=0.95,         # Metadata column
+            ...     Execution="2-XYZ"        # Provenance
+            ... )
+            >>>
+            >>> # Convert to dict for insertion
+            >>> record.model_dump()
+            {'Image': '1-ABC', 'Diagnosis_Type': 'Normal', 'confidence': 0.95, ...}
+        """
+        # Look up a feature and return its record class
+        return self.lookup_feature(table, feature_name).feature_record_class()
+
+    def delete_feature(self, table: Table | str, feature_name: str) -> bool:
+        """Removes a feature definition and its data.
+
+        Deletes the feature and its implementation table from the catalog. This operation cannot be undone and
+        will remove all feature values associated with this feature.
+
+        Args:
+            table: The table containing the feature, either as name or Table object.
+            feature_name: Name of the feature to delete.
+
+        Returns:
+            bool: True if the feature was successfully deleted, False if it didn't exist.
+
+        Raises:
+            DerivaMLException: If deletion fails due to constraints or permissions.
+
+        Example:
+            >>> success = ml.delete_feature("samples", "obsolete_feature")
+            >>> print("Deleted" if success else "Not found")
+        """
+        # Get table reference and find feature
+        table = self.model.name_to_table(table)
+        try:
+            # Find and delete the feature's implementation table
+            feature = next(f for f in self.model.find_features(table) if f.feature_name == feature_name)
+            feature.feature_table.drop()
+            return True
+        except StopIteration:
+            return False
+
+    def lookup_feature(self, table: str | Table, feature_name: str) -> Feature:
+        """Retrieves a Feature object.
+
+        Looks up and returns a Feature object that provides an interface to work with an existing feature
+        definition in the catalog.
+
+        Args:
+            table: The table containing the feature, either as name or Table object.
+            feature_name: Name of the feature to look up.
+
+        Returns:
+            Feature: An object representing the feature and its implementation.
+
+        Raises:
+            DerivaMLException: If the feature doesn't exist in the specified table.
+
+        Example:
+            >>> feature = ml.lookup_feature("samples", "expression_level")
+            >>> print(feature.feature_name)
+            'expression_level'
+        """
+        return self.model.lookup_feature(table, feature_name)
+
+    def find_features(self, table: str | Table | None = None) -> list[Feature]:
+        """Find features in the catalog.
+
+        Catalog-level operation to find feature definitions. If a table is specified,
+        returns only features for that table. If no table is specified, returns all
+        features across all tables in the catalog.
+
+        Args:
+            table: Optional table to find features for. If None, returns all features
+                in the catalog.
+
+        Returns:
+            A list of Feature instances describing the features.
+
+        Examples:
+            Find all features in the catalog:
+                >>> all_features = ml.find_features()
+                >>> for f in all_features:
+                ...     print(f"{f.target_table.name}.{f.feature_name}")
+
+            Find features for a specific table:
+                >>> image_features = ml.find_features("Image")
+                >>> print([f.feature_name for f in image_features])
+        """
+        return list(self.model.find_features(table))
+
+    @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
+    def list_feature_values(
+        self, table: Table | str, feature_name: str
+    ) -> Iterable[FeatureRecord]:
+        """Retrieves all values for a feature as typed FeatureRecord instances.
+
+        Returns an iterator of dynamically-generated FeatureRecord objects for each
+        feature value. Each record is an instance of a Pydantic model specific to
+        this feature, with typed attributes for all columns including the Execution
+        that created the feature value.
+
+        Args:
+            table: The table containing the feature, either as name or Table object.
+            feature_name: Name of the feature to retrieve values for.
+
+        Returns:
+            Iterable[FeatureRecord]: An iterator of FeatureRecord instances.
+                Each instance has:
+                - Execution: RID of the execution that created this feature value
+                - Feature_Name: Name of the feature
+                - All feature-specific columns as typed attributes
+                - model_dump() method to convert back to a dictionary
+
+        Raises:
+            DerivaMLException: If the feature doesn't exist or cannot be accessed.
+
+        Example:
+            >>> # Get typed feature records
+            >>> for record in ml.list_feature_values("Image", "Quality"):
+            ...     print(f"Image {record.Image}: {record.ImageQuality}")
+            ...     print(f"Created by execution: {record.Execution}")
+
+            >>> # Convert records to dictionaries
+            >>> records = list(ml.list_feature_values("Image", "Quality"))
+            >>> dicts = [r.model_dump() for r in records]
+        """
+        # Get table and feature
+        table = self.model.name_to_table(table)
+        feature = self.lookup_feature(table, feature_name)
+
+        # Get the dynamically-generated FeatureRecord subclass for this feature
+        record_class = feature.feature_record_class()
+
+        # Build and execute query for feature values
+        pb = self.pathBuilder()
+        raw_values = pb.schemas[feature.feature_table.schema.name].tables[feature.feature_table.name].entities().fetch()
+
+        for raw_value in raw_values:
+            # Create a record instance from the raw dictionary
+            # Filter to only include fields that the record class expects
+            field_names = set(record_class.model_fields.keys())
+            filtered_data = {k: v for k, v in raw_value.items() if k in field_names}
+            yield record_class(**filtered_data)

deriva_ml/core/mixins/file.py

@@ -0,0 +1,263 @@
+"""File management mixin for DerivaML.
+
+This module provides the FileMixin class which handles
+file operations including adding and listing files.
+"""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from itertools import chain
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Callable, Iterable
+from urllib.parse import urlsplit
+
+# Deriva imports - use importlib to avoid shadowing by local 'deriva.py' files
+import importlib
+datapath = importlib.import_module("deriva.core.datapath")
+
+from deriva_ml.core.definitions import RID, FileSpec, MLTable, MLVocab, VocabularyTerm
+from deriva_ml.core.exceptions import DerivaMLInvalidTerm, DerivaMLTableTypeError
+from deriva_ml.dataset.aux_classes import DatasetVersion
+from deriva_ml.dataset.history import iso_to_snap
+
+if TYPE_CHECKING:
+    from deriva.core.ermrest_catalog import ResolveRidResult
+
+    from deriva_ml.dataset.dataset import Dataset
+    from deriva_ml.model.catalog import DerivaModel
+
+
+class FileMixin:
+    """Mixin providing file management operations.
+
+    This mixin requires the host class to have:
+        - model: DerivaModel instance
+        - ml_schema: str - name of the ML schema
+        - pathBuilder(): method returning catalog path builder
+        - resolve_rid(): method for RID resolution (from RidResolutionMixin)
+        - lookup_term(): method for vocabulary lookup (from VocabularyMixin)
+        - list_vocabulary_terms(): method for listing vocab terms (from VocabularyMixin)
+        - find_datasets(): method for finding datasets (from DatasetMixin)
+
+    Methods:
+        add_files: Add files to the catalog with metadata
+        list_files: List files in the catalog
+        _bootstrap_versions: Initialize dataset versions
+        _synchronize_dataset_versions: Sync dataset versions
+        _set_version_snapshot: Update version snapshots
+    """
+
+    # Type hints for IDE support - actual attributes/methods from host class
+    model: "DerivaModel"
+    ml_schema: str
+    pathBuilder: Callable[[], Any]
+    resolve_rid: Callable[[RID], "ResolveRidResult"]
+    lookup_term: Callable[[str, str], VocabularyTerm]
+    list_vocabulary_terms: Callable[[str], list[VocabularyTerm]]
+    find_datasets: Callable[..., Iterable["Dataset"]]
+
+    def add_files(
+        self,
+        files: Iterable[FileSpec],
+        execution_rid: RID,
+        dataset_types: str | list[str] | None = None,
+        description: str = "",
+    ) -> "Dataset":
+        """Adds files to the catalog with their metadata.
+
+        Registers files in the catalog along with their metadata (MD5, length, URL) and associates them with
+        specified file types. Links files to the specified execution record for provenance tracking.
+
+        Args:
+            files: File specifications containing MD5 checksum, length, and URL.
+            execution_rid: Execution RID to associate files with (required for provenance).
+            dataset_types: One or more dataset type terms from File_Type vocabulary.
+            description: Description of the files.
+
+        Returns:
+            Dataset: Dataset that represents the newly added files.
+
+        Raises:
+            DerivaMLException: If file_types are invalid or execution_rid is not an execution record.
+
+        Examples:
+            Add files via an execution:
+                >>> with ml.create_execution(config) as exe:
+                ...     files = [FileSpec(url="path/to/file.txt", md5="abc123", length=1000)]
+                ...     dataset = exe.add_files(files, dataset_types="text")
+        """
+        # Import here to avoid circular imports
+        from deriva_ml.dataset.dataset import Dataset
+
+        if self.resolve_rid(execution_rid).table.name != "Execution":
+            raise DerivaMLTableTypeError("Execution", execution_rid)
+
+        filespec_list = list(files)
+
+        # Get a list of all defined file types and their synonyms.
+        defined_types = set(
+            chain.from_iterable([[t.name] + list(t.synonyms or []) for t in self.list_vocabulary_terms(MLVocab.asset_type)])
+        )
+
+        # Get a list of all of the file types used in the filespec_list
+        spec_types = set(chain.from_iterable(filespec.file_types for filespec in filespec_list))
+
+        # Now make sure that all of the file types and dataset_types in the spec list are defined.
+        if spec_types - defined_types:
+            raise DerivaMLInvalidTerm(MLVocab.asset_type.name, f"{spec_types - defined_types}")
+
+        # Normalize dataset_types, make sure File type is included.
+        if isinstance(dataset_types, list):
+            dataset_types = ["File"] + dataset_types if "File" not in dataset_types else dataset_types
+        else:
+            dataset_types = ["File", dataset_types] if dataset_types else ["File"]
+        for ds_type in dataset_types:
+            self.lookup_term(MLVocab.dataset_type, ds_type)
+
+        # Add files to the file table, and collect up the resulting entries by directory name.
+        pb = self.pathBuilder()
+        file_records = list(
+            pb.schemas[self.ml_schema].tables["File"].insert([f.model_dump(by_alias=True) for f in filespec_list])
+        )
+
+        # Get the name of the association table between file_table and file_type and add file_type records
+        atable = self.model.find_association(MLTable.file, MLVocab.asset_type)[0].name
+        # Need to get a link between file record and file_types.
+        type_map = {
+            file_spec.md5: file_spec.file_types + ([] if "File" in file_spec.file_types else [])
+            for file_spec in filespec_list
+        }
+        file_type_records = [
+            {MLVocab.asset_type.value: file_type, "File": file_record["RID"]}
+            for file_record in file_records
+            for file_type in type_map[file_record["MD5"]]
+        ]
+        pb.schemas[self.ml_schema].tables[atable].insert(file_type_records)
+
+        # Link files to the execution for provenance tracking.
+        pb.schemas[self.ml_schema].File_Execution.insert(
+            [
+                {"File": file_record["RID"], "Execution": execution_rid, "Asset_Role": "Output"}
+                for file_record in file_records
+            ]
+        )
+
+        # Now create datasets to capture the original directory structure of the files.
+        dir_rid_map = defaultdict(list)
+        for e in file_records:
+            dir_rid_map[Path(urlsplit(e["URL"]).path).parent].append(e["RID"])
+
+        nested_datasets = []
+        path_length = 0
+        dataset = None
+        # Start with the longest path so we get subdirectories first.
+        for p, rids in sorted(dir_rid_map.items(), key=lambda kv: len(kv[0].parts), reverse=True):
+            dataset = Dataset.create_dataset(
+                self,  # type: ignore[arg-type]
+                dataset_types=dataset_types,
+                execution_rid=execution_rid,
+                description=description,
+            )
+            members = rids
+            if len(p.parts) < path_length:
+                # Going up one level in directory, so Create nested dataset
+                members = [m.dataset_rid for m in nested_datasets] + rids
+                nested_datasets = []
+            dataset.add_dataset_members(members=members, execution_rid=execution_rid)
+            nested_datasets.append(dataset)
+            path_length = len(p.parts)
+
+        return dataset
+
+    def _bootstrap_versions(self) -> None:
+        """Initialize dataset versions for datasets that don't have versions."""
+        datasets = [ds.dataset_rid for ds in self.find_datasets()]
+        ds_version = [
+            {
+                "Dataset": d,
+                "Version": "0.1.0",
+                "Description": "Dataset at the time of conversion to versioned datasets",
+            }
+            for d in datasets
+        ]
+        schema_path = self.pathBuilder().schemas[self.ml_schema]
+        version_path = schema_path.tables["Dataset_Version"]
+        dataset_path = schema_path.tables["Dataset"]
+        history = list(version_path.insert(ds_version))
+        dataset_versions = [{"RID": h["Dataset"], "Version": h["Version"]} for h in history]
+        dataset_path.update(dataset_versions)
+
+    def _synchronize_dataset_versions(self) -> None:
+        """Synchronize dataset versions with the latest version in Dataset_Version table."""
+        schema_path = self.pathBuilder().schemas[self.ml_schema]
+        dataset_version_path = schema_path.tables["Dataset_Version"]
+        # Get the maximum version number for each dataset.
+        versions = {}
+        for v in dataset_version_path.entities().fetch():
+            if v["Version"] > versions.get("Dataset", DatasetVersion(0, 0, 0)):
+                versions[v["Dataset"]] = v
+        dataset_path = schema_path.tables["Dataset"]
+        dataset_path.update([{"RID": dataset, "Version": version["RID"]} for dataset, version in versions.items()])
+
+    def _set_version_snapshot(self) -> None:
+        """Update the Snapshot column of the Dataset_Version table to the correct time."""
+        dataset_version_path = self.pathBuilder().schemas[self.model.ml_schema].tables["Dataset_Version"]
+        versions = dataset_version_path.entities().fetch()
+        dataset_version_path.update(
+            [{"RID": h["RID"], "Snapshot": iso_to_snap(h["RCT"])} for h in versions if not h["Snapshot"]]
+        )
+
+    def list_files(self, file_types: list[str] | None = None) -> list[dict[str, Any]]:
+        """Lists files in the catalog with their metadata.
+
+        Returns a list of files with their metadata including URL, MD5 hash, length, description,
+        and associated file types. Files can be optionally filtered by type.
+
+        Args:
+            file_types: Filter results to only include these file types.
+
+        Returns:
+            list[dict[str, Any]]: List of file records, each containing:
+                - RID: Resource identifier
+                - URL: File location
+                - MD5: File hash
+                - Length: File size
+                - Description: File description
+                - File_Types: List of associated file types
+
+        Examples:
+            List all files:
+                >>> files = ml.list_files()
+                >>> for f in files:
+                ...     print(f"{f['RID']}: {f['URL']}")
+
+            Filter by file type:
+                >>> image_files = ml.list_files(["image", "png"])
+        """
+        asset_type_atable, file_fk, asset_type_fk = self.model.find_association("File", "Asset_Type")
+        ml_path = self.pathBuilder().schemas[self.ml_schema]
+        file = ml_path.File
+        asset_type = ml_path.tables[asset_type_atable.name]
+
+        path = file.path
+        path = path.link(asset_type.alias("AT"), on=file.RID == asset_type.columns[file_fk], join_type="left")
+        if file_types:
+            path = path.filter(asset_type.columns[asset_type_fk] == datapath.Any(*file_types))
+        path = path.attributes(
+            path.File.RID,
+            path.File.URL,
+            path.File.MD5,
+            path.File.Length,
+            path.File.Description,
+            path.AT.columns[asset_type_fk],
+        )
+
+        file_map = {}
+        for f in path.fetch():
+            entry = file_map.setdefault(f["RID"], {**f, "File_Types": []})
+            if ft := f.get("Asset_Type"):  # assign-and-test in one go
+                entry["File_Types"].append(ft)
+
+        # Now get rid of the File_Type key and return the result
+        return [(f, f.pop("Asset_Type"))[0] for f in file_map.values()]