deriva-ml 1.17.10__py3-none-any.whl → 1.17.12__py3-none-any.whl

deriva_ml/core/mixins/path_builder.py

@@ -0,0 +1,145 @@
+"""Path builder mixin for DerivaML.
+
+This module provides the PathBuilderMixin class which handles
+catalog path building and table access utilities.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Iterable
+
+# Deriva imports - use importlib to avoid shadowing by local 'deriva.py' files
+import importlib
+datapath = importlib.import_module("deriva.core.datapath")
+_ermrest_catalog = importlib.import_module("deriva.core.ermrest_catalog")
+_ermrest_model = importlib.import_module("deriva.core.ermrest_model")
+
+SchemaWrapper = datapath._SchemaWrapper
+ErmrestCatalog = _ermrest_catalog.ErmrestCatalog
+ErmrestSnapshot = _ermrest_catalog.ErmrestSnapshot
+Table = _ermrest_model.Table
+
+import pandas as pd
+
+from deriva_ml.dataset.upload import table_path as _table_path
+
+if TYPE_CHECKING:
+    from deriva_ml.model.catalog import DerivaModel
+
+
+class PathBuilderMixin:
+    """Mixin providing path building and table access utilities.
+
+    This mixin requires the host class to have:
+        - catalog: ErmrestCatalog or ErmrestSnapshot instance
+        - domain_schema: str - name of the domain schema
+        - model: DerivaModel instance
+        - working_dir: Path - working directory path
+
+    Methods:
+        pathBuilder: Get catalog path builder for queries
+        domain_path: Property returning path builder for domain schema
+        table_path: Get local filesystem path for table CSV files
+        get_table_as_dataframe: Get table contents as pandas DataFrame
+        get_table_as_dict: Get table contents as dictionaries
+    """
+
+    # Type hints for IDE support - actual attributes from host class
+    catalog: ErmrestCatalog | ErmrestSnapshot
+    domain_schemas: frozenset[str]
+    default_schema: str | None
+    model: "DerivaModel"
+    working_dir: Path
+
+    def pathBuilder(self) -> SchemaWrapper:
+        """Returns catalog path builder for queries.
+
+        The path builder provides a fluent interface for constructing complex queries against the catalog.
+        This is a core component used by many other methods to interact with the catalog.
+
+        Returns:
+            datapath._CatalogWrapper: A new instance of the catalog path builder.
+
+        Example:
+            >>> path = ml.pathBuilder.schemas['my_schema'].tables['my_table']
+            >>> results = path.entities().fetch()
+        """
+        return self.catalog.getPathBuilder()
+
+    def domain_path(self, schema: str | None = None) -> datapath.DataPath:
+        """Returns path builder for a domain schema.
+
+        Provides a convenient way to access tables and construct queries within a domain-specific schema.
+
+        Args:
+            schema: Schema name to get path builder for. If None, uses default_schema.
+
+        Returns:
+            datapath._CatalogWrapper: Path builder object scoped to the specified domain schema.
+
+        Raises:
+            DerivaMLException: If no schema specified and default_schema is not set.
+
+        Example:
+            >>> domain = ml.domain_path()  # Uses default schema
+            >>> results = domain.my_table.entities().fetch()
+            >>> # Or with explicit schema:
+            >>> domain = ml.domain_path("my_schema")
+        """
+        schema = schema or self.model._require_default_schema()
+        return self.pathBuilder().schemas[schema]
+
+    def table_path(self, table: str | Table, schema: str | None = None) -> Path:
+        """Returns a local filesystem path for table CSV files.
+
+        Generates a standardized path where CSV files should be placed when preparing to upload data to a table.
+        The path follows the project's directory structure conventions.
+
+        Args:
+            table: Name of the table or Table object to get the path for.
+            schema: Schema name for the path. If None, uses the table's schema or default_schema.
+
+        Returns:
+            Path: Filesystem path where the CSV file should be placed.
+
+        Example:
+            >>> path = ml.table_path("experiment_results")
+            >>> df.to_csv(path) # Save data for upload
+        """
+        table_obj = self.model.name_to_table(table)
+        # Use table's schema if available, otherwise use provided schema or default
+        schema = schema or table_obj.schema.name
+        return _table_path(
+            self.working_dir,
+            schema=schema,
+            table=table_obj.name,
+        )
+
+    def get_table_as_dataframe(self, table: str) -> pd.DataFrame:
+        """Get table contents as a pandas DataFrame.
+
+        Retrieves all contents of a table from the catalog.
+
+        Args:
+            table: Name of the table to retrieve.
+
+        Returns:
+            DataFrame containing all table contents.
+        """
+        return pd.DataFrame(list(self.get_table_as_dict(table)))
+
+    def get_table_as_dict(self, table: str) -> Iterable[dict[str, Any]]:
+        """Get table contents as dictionaries.
+
+        Retrieves all contents of a table from the catalog.
+
+        Args:
+            table: Name of the table to retrieve.
+
+        Returns:
+            Iterable yielding dictionaries for each row.
+        """
+        table_obj = self.model.name_to_table(table)
+        pb = self.pathBuilder()
+        yield from pb.schemas[table_obj.schema.name].tables[table_obj.name].entities().fetch()

deriva_ml/core/mixins/rid_resolution.py

@@ -0,0 +1,204 @@
+"""RID resolution mixin for DerivaML.
+
+This module provides the RidResolutionMixin class which handles
+Resource Identifier (RID) resolution and retrieval operations.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any
+
+# Deriva imports - use importlib to avoid shadowing by local 'deriva.py' files
+import importlib
+_datapath = importlib.import_module("deriva.core.datapath")
+_ermrest_catalog = importlib.import_module("deriva.core.ermrest_catalog")
+_ermrest_model = importlib.import_module("deriva.core.ermrest_model")
+
+AnyQuantifier = _datapath.Any
+ErmrestCatalog = _ermrest_catalog.ErmrestCatalog
+ErmrestSnapshot = _ermrest_catalog.ErmrestSnapshot
+ResolveRidResult = _ermrest_catalog.ResolveRidResult
+Table = _ermrest_model.Table
+
+from deriva_ml.core.definitions import RID
+from deriva_ml.core.exceptions import DerivaMLException
+
+if TYPE_CHECKING:
+    from deriva_ml.model.catalog import DerivaModel
+
+
+@dataclass
+class BatchRidResult:
+    """Result of batch RID resolution.
+
+    Attributes:
+        rid: The resolved RID (normalized form).
+        table: The Table object containing this RID.
+        table_name: The name of the table containing this RID.
+        schema_name: The name of the schema containing this RID.
+    """
+
+    rid: RID
+    table: Table
+    table_name: str
+    schema_name: str
+
+
+class RidResolutionMixin:
+    """Mixin providing RID resolution and retrieval operations.
+
+    This mixin requires the host class to have:
+        - catalog: ErmrestCatalog or ErmrestSnapshot instance
+        - model: DerivaModel instance (with .model attribute for ermrest model)
+        - pathBuilder(): method returning catalog path builder
+
+    Methods:
+        resolve_rid: Resolve a RID to its catalog location
+        resolve_rids: Batch resolve multiple RIDs efficiently
+        retrieve_rid: Retrieve the complete record for a RID
+    """
+
+    # Type hints for IDE support - actual attributes from host class
+    catalog: ErmrestCatalog | ErmrestSnapshot
+    model: "DerivaModel"
+    pathBuilder: Any  # Callable returning path builder
+
+    def resolve_rid(self, rid: RID) -> ResolveRidResult:
+        """Resolves RID to catalog location.
+
+        Looks up a RID and returns information about where it exists in the catalog, including schema,
+        table, and column metadata.
+
+        Args:
+            rid: Resource Identifier to resolve.
+
+        Returns:
+            ResolveRidResult: Named tuple containing:
+                - schema: Schema name
+                - table: Table name
+                - columns: Column definitions
+                - datapath: Path builder for accessing the entity
+
+        Raises:
+            DerivaMLException: If RID doesn't exist in catalog.
+
+        Examples:
+            >>> result = ml.resolve_rid("1-abc123")
+            >>> print(f"Found in {result.schema}.{result.table}")
+            >>> data = result.datapath.entities().fetch()
+        """
+        try:
+            # Attempt to resolve RID using catalog model
+            return self.catalog.resolve_rid(rid, self.model.model)
+        except KeyError as _e:
+            raise DerivaMLException(f"Invalid RID {rid}")
+
+    def retrieve_rid(self, rid: RID) -> dict[str, Any]:
+        """Retrieves complete record for RID.
+
+        Fetches all column values for the entity identified by the RID.
+
+        Args:
+            rid: Resource Identifier of the record to retrieve.
+
+        Returns:
+            dict[str, Any]: Dictionary containing all column values for the entity.
+
+        Raises:
+            DerivaMLException: If the RID doesn't exist in the catalog.
+
+        Example:
+            >>> record = ml.retrieve_rid("1-abc123")
+            >>> print(f"Name: {record['name']}, Created: {record['creation_date']}")
+        """
+        # Resolve RID and fetch the first (only) matching record
+        return self.resolve_rid(rid).datapath.entities().fetch()[0]
+
+    def resolve_rids(
+        self,
+        rids: set[RID] | list[RID],
+        candidate_tables: list[Table] | None = None,
+    ) -> dict[RID, BatchRidResult]:
+        """Batch resolve multiple RIDs efficiently.
+
+        Resolves multiple RIDs in batched queries, significantly faster than
+        calling resolve_rid() for each RID individually. Instead of N network
+        calls for N RIDs, this makes one query per candidate table.
+
+        Args:
+            rids: Set or list of RIDs to resolve.
+            candidate_tables: Optional list of Table objects to search in.
+                If not provided, searches all tables in domain and ML schemas.
+
+        Returns:
+            dict[RID, BatchRidResult]: Mapping from each resolved RID to its
+                BatchRidResult containing table information.
+
+        Raises:
+            DerivaMLException: If any RID cannot be resolved.
+
+        Example:
+            >>> results = ml.resolve_rids(["1-ABC", "2-DEF", "3-GHI"])
+            >>> for rid, info in results.items():
+            ...     print(f"{rid} is in table {info.table_name}")
+        """
+        rids = set(rids)
+        if not rids:
+            return {}
+
+        results: dict[RID, BatchRidResult] = {}
+        remaining_rids = set(rids)
+
+        # Determine which tables to search
+        if candidate_tables is None:
+            # Search all tables in domain and ML schemas
+            candidate_tables = []
+            for schema_name in [*self.model.domain_schemas, self.model.ml_schema]:
+                schema = self.model.model.schemas.get(schema_name)
+                if schema:
+                    candidate_tables.extend(schema.tables.values())
+
+        pb = self.pathBuilder()
+
+        # Query each candidate table for matching RIDs
+        for table in candidate_tables:
+            if not remaining_rids:
+                break
+
+            schema_name = table.schema.name
+            table_name = table.name
+
+            # Build a query with RID filter for all remaining RIDs
+            table_path = pb.schemas[schema_name].tables[table_name]
+
+            # Use ERMrest's Any quantifier for IN-style query
+            # Query only for RID column to minimize data transfer
+            try:
+                # Filter: RID = any(rid1, rid2, ...) - ERMrest's way of doing IN clause
+                found_entities = list(
+                    table_path.filter(table_path.RID == AnyQuantifier(*remaining_rids))
+                    .attributes(table_path.RID)
+                    .fetch()
+                )
+            except Exception:
+                # Table might not support this query, skip it
+                continue
+
+            # Process found RIDs
+            for entity in found_entities:
+                rid = entity["RID"]
+                if rid in remaining_rids:
+                    results[rid] = BatchRidResult(
+                        rid=rid,
+                        table=table,
+                        table_name=table_name,
+                        schema_name=schema_name,
+                    )
+                    remaining_rids.remove(rid)
+
+        # Check if any RIDs were not found
+        if remaining_rids:
+            raise DerivaMLException(f"Invalid RIDs: {remaining_rids}")
+
+        return results