deriva-ml 1.14.0__py3-none-any.whl → 1.14.26__py3-none-any.whl

deriva_ml/feature.py

@@ -1,27 +1,48 @@
-"""
-This module provides the implementation of the Feature capability in deriva-ml
+"""Feature implementation for deriva-ml.
+
+This module provides classes for defining and managing features in deriva-ml. Features represent measurable
+properties or characteristics that can be associated with records in a table. The module includes:
+
+- Feature: Main class for defining and managing features
+- FeatureRecord: Base class for feature records using pydantic models
+
+Typical usage example:
+    >>> feature = Feature(association_result, model)
+    >>> FeatureClass = feature.feature_record_class()
+    >>> record = FeatureClass(value="high", confidence=0.95)
 """
 
-from deriva.core.ermrest_model import FindAssociationResult, Column
 from pathlib import Path
-from pydantic import BaseModel, create_model
-from typing import Optional, Type, ClassVar, TYPE_CHECKING
 from types import UnionType
+from typing import TYPE_CHECKING, ClassVar, Optional, Type
+
+from deriva.core.ermrest_model import Column, FindAssociationResult
+from pydantic import BaseModel, create_model
 
 if TYPE_CHECKING:
-    from .deriva_model import DerivaModel
+    from model.catalog import DerivaModel
 
 
 class FeatureRecord(BaseModel):
-    """Base class for feature records.  Feature records are pydantic models which are dynamically generated and
-    describe all the columns of a feature.
+    """Base class for dynamically generated feature record models.
 
-    Attributes:
-        Execution (str):
-        Feature_Name (str):
-        feature:
-    Returns:
+    This class serves as the base for pydantic models that represent feature records. Each feature record
+    contains the values and metadata associated with a feature instance.
 
+    Attributes:
+        Execution (Optional[str]): RID of the execution that created this feature record.
+        Feature_Name (str): Name of the feature this record belongs to.
+        feature (ClassVar[Optional[Feature]]): Reference to the Feature object that created this record.
+
+    Example:
+        >>> class GeneFeature(FeatureRecord):
+        ...     value: str
+        ...     confidence: float
+        >>> record = GeneFeature(
+        ...     Feature_Name="expression",
+        ...     value="high",
+        ...     confidence=0.95
+        ... )
     """
 
     # model_dump of this feature should be compatible with feature table columns.
@@ -34,53 +55,61 @@ class FeatureRecord(BaseModel):
 
     @classmethod
     def feature_columns(cls) -> set[Column]:
-        """
+        """Returns all columns specific to this feature.
 
         Returns:
-          A set of feature column names.
-
+            set[Column]: Set of feature-specific columns, excluding system and relationship columns.
         """
         return cls.feature.feature_columns
 
     @classmethod
     def asset_columns(cls) -> set[Column]:
-        """
-
-        Args:
+        """Returns columns that reference asset tables.
 
         Returns:
-          A set of asset column names.
-
+            set[Column]: Set of columns that contain references to asset tables.
         """
         return cls.feature.asset_columns
 
     @classmethod
     def term_columns(cls) -> set[Column]:
-        """
-
-        Args:
+        """Returns columns that reference vocabulary terms.
 
         Returns:
-          :return: set of term column names.
-
+            set[Column]: Set of columns that contain references to controlled vocabulary terms.
         """
         return cls.feature.term_columns
 
     @classmethod
     def value_columns(cls) -> set[Column]:
-        """
-
-        Args:
+        """Returns columns that contain direct values.
 
         Returns:
-          A set of value column names.
-
+            set[Column]: Set of columns containing direct values (not references to assets or terms).
         """
         return cls.feature.value_columns
 
 
 class Feature:
-    """Wrapper for results of Table.find_associations()"""
+    """Manages feature definitions and their relationships in the catalog.
+
+    A Feature represents a measurable property or characteristic that can be associated with records in a table.
+    Features can include asset references, controlled vocabulary terms, and custom metadata fields.
+
+    Attributes:
+        feature_table: Table containing the feature implementation.
+        target_table: Table that the feature is associated with.
+        feature_name: Name of the feature (from Feature_Name column default).
+        feature_columns: Set of columns specific to this feature.
+        asset_columns: Set of columns referencing asset tables.
+        term_columns: Set of columns referencing vocabulary tables.
+        value_columns: Set of columns containing direct values.
+
+    Example:
+        >>> feature = Feature(association_result, model)
+        >>> print(f"Feature {feature.feature_name} on {feature.target_table.name}")
+        >>> print("Asset columns:", [c.name for c in feature.asset_columns])
+    """
 
     def __init__(self, atable: FindAssociationResult, model: "DerivaModel") -> None:
         self.feature_table = atable.table
@@ -98,9 +127,7 @@ class Feature:
             self.target_table.name,
             "Execution",
         }
-        self.feature_columns = {
-            c for c in self.feature_table.columns if c.name not in skip_columns
-        }
+        self.feature_columns = {c for c in self.feature_table.columns if c.name not in skip_columns}
 
         assoc_fkeys = {atable.self_fkey} | atable.other_fkeys
 
@@ -117,9 +144,7 @@ class Feature:
             if fk not in assoc_fkeys and self._model.is_vocabulary(fk.pk_table)
         }
 
-        self.value_columns = self.feature_columns - (
-            self.asset_columns | self.term_columns
-        )
+        self.value_columns = self.feature_columns - (self.asset_columns | self.term_columns)
 
     def feature_record_class(self) -> type[FeatureRecord]:
         """Create a pydantic model for entries into the specified feature table
@@ -129,14 +154,25 @@ class Feature:
         """
 
         def map_type(c: Column) -> UnionType | Type[str] | Type[int] | Type[float]:
-            """Map a deriva type into a pydantic model type.
+            """Maps a Deriva column type to a Python/pydantic type.
+
+            Converts ERMrest column types to appropriate Python types for use in pydantic models.
+            Special handling is provided for asset columns which can accept either strings or Path objects.
 
             Args:
-                c: column to be mapped
-                c: Column:
+                c: ERMrest column to map to a Python type.
 
             Returns:
-                A pydantic model type
+                UnionType | Type[str] | Type[int] | Type[float]: Appropriate Python type for the column:
+                    - str | Path for asset columns
+                    - str for text columns
+                    - int for integer columns
+                    - float for floating point columns
+                    - str for all other types
+
+            Example:
+                >>> col = Column(name="score", type="float4")
+                >>> typ = map_type(col)  # Returns float
             """
             if c.name in {c.name for c in self.asset_columns}:
                 return str | Path
@@ -168,7 +204,10 @@ class Feature:
             ),  # Set default value for Feature_Name
             self.target_table.name: (str, ...),
         }
-        docstring = f"Class to capture fields in a feature {self.feature_name} on table {self.target_table}. Feature columns include:\n"
+        docstring = (
+            f"Class to capture fields in a feature {self.feature_name} on table {self.target_table}. "
+            "Feature columns include:\n"
+        )
         docstring += "\n".join([f"    {c.name}" for c in self.feature_columns])
 
         model = create_model(
@@ -177,9 +216,7 @@ class Feature:
             __doc__=docstring,
             **feature_columns,
         )
-        model.feature = (
-            self  # Set value of class variable within the feature class definition.
-        )
+        model.feature = self  # Set value of class variable within the feature class definition.
 
         return model
 

deriva_ml/{deriva_model.py → model/catalog.py}

@@ -1,27 +1,55 @@
 """
-`deriva_ml_base.py` is the core module for the Deriva ML project.  This module implements the DerivaML class, which is
-the primary interface to the Deriva based catalogs.  The module also implements the Feature and Vocabulary functions
-in the DerivaML.
-
-DerivaML and its associated classes all depend on a catalog that implements a `deriva-ml` schema with tables and
-relationships that follow a specific data model.
+Model management for Deriva ML catalogs.
 
+This module provides the DerivaModel class which augments the standard Deriva model class with
+ML-specific functionality. It handles schema management, feature definitions, and asset tracking.
 """
 
-from deriva.core.ermrest_model import Table, Column, Model, FindAssociationResult
+from __future__ import annotations
+
+# Standard library imports
+from collections import Counter
+from typing import Any, Callable, Final, Iterable, NewType, TypeAlias
+
 from deriva.core.ermrest_catalog import ErmrestCatalog
-from .feature import Feature
 
-from .deriva_definitions import (
-    DerivaMLException,
+# Deriva imports
+from deriva.core.ermrest_model import Column, FindAssociationResult, Model, Schema, Table
+
+# Third-party imports
+from pydantic import ConfigDict, validate_call
+
+from deriva_ml.core.definitions import (
     ML_SCHEMA,
-    DerivaSystemColumns,
+    DerivaAssetColumns,
     TableDefinition,
 )
+from deriva_ml.core.exceptions import DerivaMLException, DerivaMLTableTypeError
 
-from collections import Counter
-from pydantic import validate_call, ConfigDict
-from typing import Iterable, Optional, Any
+# Local imports
+from deriva_ml.feature import Feature
+
+try:
+    from icecream import ic
+except ImportError:  # Graceful fallback if IceCream isn't installed.
+    ic = lambda *a: None if not a else (a[0] if len(a) == 1 else a)  # noqa
+
+
+# Define common types:
+TableInput: TypeAlias = str | Table
+SchemaDict: TypeAlias = dict[str, Schema]
+FeatureList: TypeAlias = Iterable[Feature]
+SchemaName = NewType("SchemaName", str)
+ColumnSet: TypeAlias = set[Column]
+AssociationResult: TypeAlias = FindAssociationResult
+TableSet: TypeAlias = set[Table]
+PathList: TypeAlias = list[list[Table]]
+
+# Define constants:
+VOCAB_COLUMNS: Final[set[str]] = {"NAME", "URI", "SYNONYMS", "DESCRIPTION", "ID"}
+ASSET_COLUMNS: Final[set[str]] = {"Filename", "URL", "Length", "MD5", "Description"}
+
+FilterPredicate = Callable[[Table], bool]
 
 
 class DerivaModel:
@@ -30,9 +58,8 @@ class DerivaModel:
     This class provides a number of DerivaML specific methods that augment the interface in the deriva model class.
 
     Attributes:
-        domain_schema: Schema name for domain specific tables and relationships.
+        domain_schema: Schema name for domain-specific tables and relationships.
         model: ERMRest model for the catalog.
-        schemas: ERMRest model for the catalog.
         catalog: ERMRest catalog for the model
         hostname: ERMRest catalog for the model
         ml_schema: The ML schema for the catalog.
@@ -41,7 +68,10 @@ class DerivaModel:
     """
 
     def __init__(
-        self, model: Model, ml_schema: str = ML_SCHEMA, domain_schema: str = ""
+        self,
+        model: Model,
+        ml_schema: str = ML_SCHEMA,
+        domain_schema: str | None = None,
     ):
         """Create and initialize a DerivaML instance.
 
@@ -49,64 +79,66 @@ class DerivaModel:
         This class is intended to be used as a base class on which domain-specific interfaces are built.
 
         Args:
+            model: The ERMRest model for the catalog.
+            ml_schema: The ML schema name.
+            domain_schema: The domain schema name.
         """
         self.model = model
         self.configuration = None
         self.catalog: ErmrestCatalog = self.model.catalog
-        self.hostname = (
-            self.catalog.deriva_server.server
-            if isinstance(self.catalog, ErmrestCatalog)
-            else "localhost"
-        )
-        self.schemas = self.model.schemas
+        self.hostname = self.catalog.deriva_server.server if isinstance(self.catalog, ErmrestCatalog) else "localhost"
 
         self.ml_schema = ml_schema
-        builtin_schemas = ["public", self.ml_schema, "www", "WWW"]
-        try:
-            self.domain_schema = (
-                domain_schema
-                or [
-                    s for s in self.model.schemas.keys() if s not in builtin_schemas
-                ].pop()
-            )
-        except IndexError:
-            # No domain schema defined.
+        builtin_schemas = ("public", self.ml_schema, "www", "WWW")
+        if domain_schema:
             self.domain_schema = domain_schema
+        else:
+            if len(user_schemas := {k for k in self.model.schemas.keys()} - set(builtin_schemas)) == 1:
+                self.domain_schema = user_schemas.pop()
+            else:
+                raise DerivaMLException(f"Ambiguous domain schema: {user_schemas}")
+
+    def refresh_model(self) -> None:
+        self.model = self.catalog.getCatalogModel()
+
+    @property
+    def schemas(self) -> dict[str, Schema]:
+        return self.model.schemas
 
     @property
     def chaise_config(self) -> dict[str, Any]:
         """Return the chaise configuration."""
         return self.model.chaise_config
 
-    def __getattr__(self, name):
+    def __getattr__(self, name: str) -> Any:
         # Called only if `name` is not found in Manager.  Delegate attributes to model class.
         return getattr(self.model, name)
 
-    def name_to_table(self, table: str | Table) -> Table:
+    def name_to_table(self, table: TableInput) -> Table:
         """Return the table object corresponding to the given table name.
 
         If the table name appears in more than one schema, return the first one you find.
 
         Args:
           table: A ERMRest table object or a string that is the name of the table.
-          table: str | Table:
 
         Returns:
           Table object.
         """
         if isinstance(table, Table):
             return table
-        for s in self.model.schemas.values():
+        if table in (s := self.model.schemas[self.domain_schema].tables):
+            return s[table]
+        for s in [self.model.schemas[sname] for sname in [self.domain_schema, self.ml_schema, "WWW"]]:
             if table in s.tables.keys():
                 return s.tables[table]
         raise DerivaMLException(f"The table {table} doesn't exist.")
 
-    def is_vocabulary(self, table_name: str | Table) -> bool:
+    def is_vocabulary(self, table_name: TableInput) -> bool:
         """Check if a given table is a controlled vocabulary table.
 
         Args:
           table_name: A ERMRest table object or the name of the table.
-          table_name: str | Table:
 
         Returns:
           Table object if the table is a controlled vocabulary, False otherwise.
@@ -126,7 +158,7 @@ class DerivaModel:
         pure: bool = True,
         min_arity: int = 2,
         max_arity: int = 2,
-    ) -> bool | set | int:
+    ) -> bool | set[str] | int:
         """Check the specified table to see if it is an association table.
 
         Args:
@@ -140,12 +172,10 @@ class DerivaModel:
 
         """
         table = self.name_to_table(table_name)
-        return table.is_association(
-            unqualified=unqualified, pure=pure, min_arity=min_arity, max_arity=max_arity
-        )
+        return table.is_association(unqualified=unqualified, pure=pure, min_arity=min_arity, max_arity=max_arity)
 
-    def find_association(self, table1: Table | str, table2: Table | str) -> Table:
-        """Given two tables, return an association table that connects the two.
+    def find_association(self, table1: Table | str, table2: Table | str) -> tuple[Table, Column, Column]:
+        """Given two tables, return an association table that connects the two and the two columns used to link them..
 
         Raises:
             DerivaML exception if there is either not an association table or more than one association table.
@@ -154,22 +184,21 @@ class DerivaModel:
         table2 = self.name_to_table(table2)
 
         tables = [
-            a.table
+            (a.table, a.self_fkey.columns[0].name, other_key.columns[0].name)
             for a in table1.find_associations(pure=False)
-            if a.other_fkeys.pop().pk_table == table2
+            if len(a.other_fkeys) == 1 and (other_key := a.other_fkeys.pop()).pk_table == table2
         ]
+
         if len(tables) == 1:
             return tables[0]
         elif len(tables) == 0:
-            raise DerivaMLException(
-                f"No association tables found between {table1.name} and {table2.name}."
-            )
+            raise DerivaMLException(f"No association tables found between {table1.name} and {table2.name}.")
         else:
             raise DerivaMLException(
                 f"There are {len(tables)} association tables between {table1.name} and {table2.name}."
             )
 
-    def is_asset(self, table_name: str | Table) -> bool:
+    def is_asset(self, table_name: TableInput) -> bool:
         """True if the specified table is an asset table.
 
         Args:
@@ -185,24 +214,14 @@ class DerivaModel:
 
     def find_assets(self, with_metadata: bool = False) -> list[Table]:
         """Return the list of asset tables in the current model"""
-        return [
-            t
-            for s in self.model.schemas.values()
-            for t in s.tables.values()
-            if self.is_asset(t)
-        ]
+        return [t for s in self.model.schemas.values() for t in s.tables.values() if self.is_asset(t)]
 
     def find_vocabularies(self) -> list[Table]:
         """Return a list of all the controlled vocabulary tables in the domain schema."""
-        return [
-            t
-            for s in self.model.schemas.values()
-            for t in s.tables.values()
-            if self.is_vocabulary(t)
-        ]
+        return [t for s in self.model.schemas.values() for t in s.tables.values() if self.is_vocabulary(t)]
 
     @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
-    def find_features(self, table: Table | str) -> Iterable[Feature]:
+    def find_features(self, table: TableInput) -> Iterable[Feature]:
         """List the names of the features in the specified table.
 
         Args:
@@ -215,15 +234,13 @@ class DerivaModel:
         table = self.name_to_table(table)
 
         def is_feature(a: FindAssociationResult) -> bool:
-            """
+            """Check if association represents a feature.
 
             Args:
-              a: FindAssociationResult:
-
+                a: Association result to check
             Returns:
-
+                bool: True if association represents a feature
             """
-            # return {'Feature_Name', 'Execution'}.issubset({c.name for c in a.table.columns})
             return {
                 "Feature_Name",
                 "Execution",
@@ -231,12 +248,10 @@ class DerivaModel:
             }.issubset({c.name for c in a.table.columns})
 
         return [
-            Feature(a, self)
-            for a in table.find_associations(min_arity=3, max_arity=3, pure=False)
-            if is_feature(a)
+            Feature(a, self) for a in table.find_associations(min_arity=3, max_arity=3, pure=False) if is_feature(a)
         ]
 
-    def lookup_feature(self, table: str | Table, feature_name: str) -> Feature:
+    def lookup_feature(self, table: TableInput, feature_name: str) -> Feature:
         """Lookup the named feature associated with the provided table.
 
         Args:
@@ -252,31 +267,20 @@ class DerivaModel:
         """
         table = self.name_to_table(table)
         try:
-            return [
-                f for f in self.find_features(table) if f.feature_name == feature_name
-            ][0]
+            return [f for f in self.find_features(table) if f.feature_name == feature_name][0]
         except IndexError:
-            raise DerivaMLException(
-                f"Feature {table.name}:{feature_name} doesn't exist."
-            )
+            raise DerivaMLException(f"Feature {table.name}:{feature_name} doesn't exist.")
 
     def asset_metadata(self, table: str | Table) -> set[str]:
         """Return the metadata columns for an asset table."""
 
         table = self.name_to_table(table)
-        asset_columns = {
-            "Filename",
-            "URL",
-            "Length",
-            "MD5",
-            "Description",
-        }.union(set(DerivaSystemColumns))
 
         if not self.is_asset(table):
-            raise DerivaMLException(f"{table.name} is not an asset table.")
-        return {c.name for c in table.columns} - asset_columns
+            raise DerivaMLTableTypeError("asset table", table.name)
+        return {c.name for c in table.columns} - DerivaAssetColumns
 
-    def apply(self):
+    def apply(self) -> None:
         """Call ERMRestModel.apply"""
         if self.catalog == "file-system":
             raise DerivaMLException("Cannot apply() to non-catalog model.")
@@ -284,45 +288,38 @@ class DerivaModel:
             self.model.apply()
 
     def _table_relationship(
-        self, table1: Table | str, table2: Table | str
+        self,
+        table1: TableInput,
+        table2: TableInput,
     ) -> tuple[Column, Column]:
         """Return columns used to relate two tables."""
         table1 = self.name_to_table(table1)
         table2 = self.name_to_table(table2)
         relationships = [
-            (fk.foreign_key_columns[0], fk.referenced_columns[0])
-            for fk in table1.foreign_keys
-            if fk.pk_table == table2
+            (fk.foreign_key_columns[0], fk.referenced_columns[0]) for fk in table1.foreign_keys if fk.pk_table == table2
         ]
         relationships.extend(
-            [
-                (fk.referenced_columns[0], fk.foreign_key_columns[0])
-                for fk in table1.referenced_by
-                if fk.table == table2
-            ]
+            [(fk.referenced_columns[0], fk.foreign_key_columns[0]) for fk in table1.referenced_by if fk.table == table2]
         )
         if len(relationships) != 1:
-            raise DerivaMLException(
-                f"Ambiguous linkage between {table1.name} and {table2.name}"
-            )
+            raise DerivaMLException(f"Ambiguous linkage between {table1.name} and {table2.name}")
         return relationships[0]
 
     def _schema_to_paths(
         self,
-        root: Table = None,
-        path: Optional[list[Table]] = None,
+        root: Table | None = None,
+        path: list[Table] | None = None,
     ) -> list[list[Table]]:
-        """Recursively walk over the domain schema graph and extend the current path.
-
-        Walk a schema graph and return a list all the paths through the graph.
+        """Return a list of paths through the schema graph.
 
         Args:
-            path: Source path so far
+            root: The root table to start from.
+            path: The current path being built.
 
         Returns:
-          A list of all the paths through the graph.  Each path is a list of tables.
-
+            A list of paths through the schema graph.
         """
+        path = path or []
 
         root = root or self.model.schemas[self.ml_schema].tables["Dataset"]
         path = path.copy() if path else []
@@ -332,21 +329,11 @@ class DerivaModel:
 
         def find_arcs(table: Table) -> set[Table]:
             """Given a path through the model, return the FKs that link the tables"""
-            arc_list = [fk.pk_table for fk in table.foreign_keys] + [
-                fk.table for fk in table.referenced_by
-            ]
-            arc_list = [
-                t
-                for t in arc_list
-                if t.schema.name in {self.domain_schema, self.ml_schema}
-            ]
+            arc_list = [fk.pk_table for fk in table.foreign_keys] + [fk.table for fk in table.referenced_by]
+            arc_list = [t for t in arc_list if t.schema.name in {self.domain_schema, self.ml_schema}]
             domain_tables = [t for t in arc_list if t.schema.name == self.domain_schema]
-            if multiple_columns := [
-                c for c, cnt in Counter(domain_tables).items() if cnt > 1
-            ]:
-                raise DerivaMLException(
-                    f"Ambiguous relationship in {table.name} {multiple_columns}"
-                )
+            if multiple_columns := [c for c, cnt in Counter(domain_tables).items() if cnt > 1]:
+                raise DerivaMLException(f"Ambiguous relationship in {table.name} {multiple_columns}")
             return set(arc_list)
 
         def is_nested_dataset_loopback(n1: Table, n2: Table) -> bool:
@@ -354,9 +341,7 @@ class DerivaModel:
             # If we have node_name <- node_name_dataset-> Dataset then we are looping
             # back around to a new dataset element
             dataset_table = self.model.schemas[self.ml_schema].tables["Dataset"]
-            assoc_table = [
-                a for a in dataset_table.find_associations() if a.table == n2
-            ]
+            assoc_table = [a for a in dataset_table.find_associations() if a.table == n2]
             return len(assoc_table) == 1 and n1 != dataset_table
 
         # Don't follow vocabulary terms back to their use.
@@ -372,9 +357,7 @@ class DerivaModel:
             if is_nested_dataset_loopback(root, child):
                 continue
             if child in path:
-                raise DerivaMLException(
-                    f"Cycle in schema path: {child.name} path:{[p.name for p in path]}"
-                )
+                raise DerivaMLException(f"Cycle in schema path: {child.name} path:{[p.name for p in path]}")
 
             paths.extend(self._schema_to_paths(child, path))
         return paths
@@ -382,6 +365,4 @@ class DerivaModel:
     @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
     def create_table(self, table_def: TableDefinition) -> Table:
         """Create a new table from TableDefinition."""
-        return self.model.schemas[self.domain_schema].create_table(
-            table_def.model_dump()
-        )
+        return self.model.schemas[self.domain_schema].create_table(table_def.model_dump())