deriva-ml 1.10.1__py3-none-any.whl → 1.12.0__py3-none-any.whl

deriva_ml/deriva_ml_base.py

@@ -31,7 +31,6 @@ from deriva.core.datapath import DataPathException
 from deriva.core.deriva_server import DerivaServer
 from deriva.core.ermrest_catalog import ResolveRidResult
 from deriva.core.ermrest_model import Key, Table
-from deriva.core.hatrac_store import HatracStore
 from deriva.core.utils.globus_auth_utils import GlobusNativeLogin
 from pydantic import validate_call, ConfigDict
 from requests import RequestException
@@ -42,24 +41,17 @@ from .dataset import Dataset
 from .dataset_aux_classes import DatasetSpec
 from .dataset_bag import DatasetBag
 from .deriva_model import DerivaModel
-from .upload import (
-    table_path,
-    execution_rids,
-    execution_metadata_dir,
-    upload_directory,
-    UploadAssetDirectory,
-)
+from .upload import table_path, execution_rids, asset_file_path
 from .deriva_definitions import ColumnDefinition
-from .deriva_definitions import ExecMetadataVocab
 from .deriva_definitions import (
     RID,
     Status,
-    FileUploadState,
     DerivaMLException,
     ML_SCHEMA,
     VocabularyTerm,
     MLVocab,
     FileSpec,
+    TableDefinition,
 )
 
 try:
@@ -273,10 +265,13 @@ class DerivaML(Dataset):
             is_notebook = True
         else:
             stack = inspect.stack()
+            # Get the caller's filename, which is two up the stack from here.
             if len(stack) > 1:
-                filename = Path(
-                    stack[2].filename
-                )  # Get the caller's filename, which is two up the stack from here.
+                filename = Path(stack[2].filename)
+                if not filename.exists():
+                    # Begin called from command line interpreter.
+                    filename = "REPL"
+                # Get the caller's filename, which is two up the stack from here.
             else:
                 raise DerivaMLException(
                     "Looking for caller failed"
@@ -334,7 +329,6 @@ class DerivaML(Dataset):
         """Return a local file path in which to place a CSV to add values to a table on upload.
 
         Args:
-          table: return:
           table: str | Table:
 
         Returns:
@@ -346,30 +340,6 @@ class DerivaML(Dataset):
             table=self.model.name_to_table(table).name,
         )
 
-    def asset_dir(
-        self, table: str | Table, prefix: Optional[str | Path] = None
-    ) -> UploadAssetDirectory:
-        """Return a local file path in which to place a files for an asset table.  T
-
-        Args:
-            table: Location of where to place files.  Defaults to execution_assets_path.
-            prefix: Root path to asset directory.
-
-        Returns:
-            Path to the directory in which asset files should be placed.
-        """
-        table = self.model.name_to_table(table)
-        if not self.model.is_asset(table):
-            raise DerivaMLException(f"The table {table} is not an asset table.")
-
-        prefix = Path(prefix) if prefix else self.working_dir
-        return UploadAssetDirectory(
-            model=self.model,
-            prefix=prefix,
-            schema=table.schema.name,
-            table=table.name,
-        )
-
     def download_dir(self, cached: bool = False) -> Path:
         """Location where downloaded files are placed.
 
@@ -532,10 +502,17 @@ class DerivaML(Dataset):
             )
         )
 
+    def create_table(self, table: TableDefinition) -> Table:
+        """Create a table from a table definition."""
+        return self.model.schemas[self.domain_schema].create_table(table.model_dump())
+
+    @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
     def create_asset(
         self,
         asset_name: str,
         column_defs: Optional[Iterable[ColumnDefinition]] = None,
+        fkey_defs: Optional[Iterable[ColumnDefinition]] = None,
+        referenced_tables: Optional[Iterable[Table]] = None,
         comment: str = "",
         schema: Optional[str] = None,
     ) -> Table:
@@ -544,6 +521,8 @@ class DerivaML(Dataset):
         Args:
             asset_name: Name of the asset table.
             column_defs: Iterable of ColumnDefinition objects to provide additional metadata for asset.
+            fkey_defs: Iterable of ForeignKeyDefinition objects to provide additional metadata for asset.
+            referenced_tables: Iterable of Table objects to which asset should provide foreign-key references to.
             comment: Description of the asset table. (Default value = '')
             schema: Schema in which to create the asset table.  Defaults to domain_schema.
             asset_name: str:
@@ -553,17 +532,82 @@ class DerivaML(Dataset):
             Table object for the asset table.
         """
         column_defs = column_defs or []
+        fkey_defs = fkey_defs or []
+        referenced_tables = referenced_tables or []
         schema = schema or self.domain_schema
+
+        self.add_term(
+            MLVocab.asset_type, asset_name, description=f"A {asset_name} asset"
+        )
         asset_table = self.model.schemas[schema].create_table(
             Table.define_asset(
                 schema,
                 asset_name,
                 column_defs=[c.model_dump() for c in column_defs],
+                fkey_defs=[fk.model_dump() for fk in fkey_defs],
                 comment=comment,
             )
         )
+
+        self.model.schemas[self.domain_schema].create_table(
+            Table.define_association(
+                [
+                    (asset_table.name, asset_table),
+                    ("Asset_Type", self.model.name_to_table("Asset_Type")),
+                ]
+            )
+        )
+        for t in referenced_tables:
+            asset_table.create_reference(self.model.name_to_table(t))
+        # Create a table to track execution that creates the asset
+        atable = self.model.schemas[self.domain_schema].create_table(
+            Table.define_association(
+                [
+                    (asset_name, asset_table),
+                    (
+                        "Execution",
+                        self.model.schemas[self.ml_schema].tables["Execution"],
+                    ),
+                ]
+            )
+        )
+        atable.create_reference(self.model.name_to_table("Asset_Role"))
         return asset_table
 
+    # @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
+    def list_assets(self, asset_table: Table | str):
+        """Return the contents of an asset table"""
+
+        if not self.model.is_asset(asset_table):
+            raise DerivaMLException(f"Table {asset_table.name} is not an asset")
+        asset_table = self.model.name_to_table(asset_table)
+        pb = self._model.catalog.getPathBuilder()
+        asset_path = pb.schemas[asset_table.schema.name].tables[asset_table.name]
+
+        asset_type_table = self._model.find_association(asset_table, MLVocab.asset_type)
+        type_path = pb.schemas[asset_type_table.schema.name].tables[
+            asset_type_table.name
+        ]
+
+        # Get a list of all the asset_type values associated with this dataset_table.
+        assets = []
+        for asset in asset_path.entities().fetch():
+            asset_types = (
+                type_path.filter(type_path.columns[asset_table.name] == asset["RID"])
+                .attributes(type_path.Asset_Type)
+                .fetch()
+            )
+            assets.append(
+                asset
+                | {
+                    MLVocab.asset_type.value: [
+                        asset_type[MLVocab.asset_type.value]
+                        for asset_type in asset_types
+                    ]
+                }
+            )
+        return assets
+
     @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
     def create_feature(
         self,
@@ -717,24 +761,6 @@ class DerivaML(Dataset):
         """
         return self.model.find_features(table)
 
-    @validate_call
-    def add_features(self, features: Iterable[FeatureRecord]) -> int:
-        """Add a set of new feature values to the catalog.
-
-        Args:
-          features: Iterable[FeatureRecord]:
-
-        Returns:
-            Number of attributes added
-        """
-        features = list(features)
-        feature_table = features[0].feature.feature_table
-        feature_path = self.pathBuilder.schemas[feature_table.schema.name].tables[
-            feature_table.name
-        ]
-        entries = feature_path.insert(f.model_dump() for f in features)
-        return len(entries)
-
     # noinspection PyProtectedMember
     @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
     def list_feature_values(
@@ -838,7 +864,8 @@ class DerivaML(Dataset):
             raise DerivaMLException(f"The table {table} is not a controlled vocabulary")
         schema_name, table_name = vocab_table.schema.name, vocab_table.name
         schema_path = self.catalog.getPathBuilder().schemas[schema_name]
-        for term in schema_path.tables[table_name].entities():
+
+        for term in schema_path.tables[table_name].entities().fetch():
             if term_name == term["Name"] or (
                 term["Synonyms"] and term_name in term["Synonyms"]
             ):
@@ -891,65 +918,6 @@ class DerivaML(Dataset):
             snapshot_catalog=DerivaML(self.host_name, self._version_snapshot(dataset)),
         )
 
-    @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
-    def download_asset(self, asset_rid: RID, dest_dir: Path) -> Path:
-        """Download an asset from a URL and place it in a local directory.
-
-        Args:
-            asset_rid: URL of the asset.
-            dest_dir: Destination directory for the asset.
-
-        Returns:
-            A  Path object to the downloaded asset.
-        """
-        table = self.resolve_rid(asset_rid).table
-        if not self.model.is_asset(table):
-            raise DerivaMLException(f"RID {asset_rid}  is not for an asset table.")
-
-        tpath = self.pathBuilder.schemas[table.schema.name].tables[table.name]
-        asset_metadata = list(tpath.filter(tpath.RID == asset_rid).entities())[0]
-        asset_url = asset_metadata["URL"]
-        asset_filename = dest_dir / asset_metadata["Filename"]
-
-        hs = HatracStore("https", self.host_name, self.credential)
-        hs.get_obj(path=asset_url, destfilename=asset_filename.as_posix())
-        return Path(asset_filename)
-
-    @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
-    def upload_assets(
-        self,
-        assets_dir: str | Path | UploadAssetDirectory,
-    ) -> dict[Any, FileUploadState] | None:
-        """Upload assets from a directory.
-
-        This routine assumes that the current upload specification includes a configuration for the specified directory.
-        Every asset in the specified directory is uploaded
-
-        Args:
-            assets_dir: Directory containing the assets to upload.
-
-        Returns:
-            Results of the upload operation.
-
-        Raises:
-            DerivaMLException: If there is an issue uploading the assets.
-        """
-
-        def path_to_asset(path: str) -> str:
-            """Pull the asset name out of a path to that asset in the filesystem"""
-            components = path.split("/")
-            return components[
-                components.index("asset") + 2
-            ]  # Look for asset in the path to find the name
-
-        if isinstance(assets_dir, UploadAssetDirectory):
-            assets_dir = assets_dir.path
-
-        if not self.model.is_asset(Path(assets_dir).name):
-            raise DerivaMLException("Directory does not have name of an asset table.")
-        results = upload_directory(self.model, assets_dir)
-        return {path_to_asset(p): r for p, r in results.items()}
-
     def _update_status(
         self, new_status: Status, status_detail: str, execution_rid: RID
     ):
@@ -1177,13 +1145,17 @@ class DerivaML(Dataset):
             if self._is_notebook
             else f"git hash-object {self.executable_path}"
         )
-        checksum = subprocess.run(
-            cmd,
-            capture_output=True,
-            text=True,
-            check=True,
-            shell=True,
-        ).stdout.strip()
+        checksum = (
+            subprocess.run(
+                cmd,
+                capture_output=True,
+                text=True,
+                check=False,
+                shell=True,
+            ).stdout.strip()
+            if self.executable_path != "REPL"
+            else "1"
+        )
 
         return Workflow(
             name=name,
@@ -1205,7 +1177,9 @@ class DerivaML(Dataset):
 
         """
 
-        # Get repo URL from local github repo.
+        # Get repo URL from local gitHub repo.
+        if self.executable_path == "REPL":
+            return "REPL", True
         try:
             result = subprocess.run(
                 ["git", "remote", "get-url", "origin"],
@@ -1261,7 +1235,7 @@ class DerivaML(Dataset):
 
         Args:
             configuration: ExecutionConfiguration:
-            dryrun: Do not create an execution record or upload results.
+            dry_run: Do not create an execution record or upload results.
 
         Returns:
             An execution object.
@@ -1274,6 +1248,7 @@ class DerivaML(Dataset):
     # @validate_call
     def restore_execution(self, execution_rid: Optional[RID] = None) -> "Execution":
         """Return an Execution object for a previously started execution with the specified RID."""
+
         from .execution import Execution
 
         # Find path to execution
@@ -1283,13 +1258,11 @@ class DerivaML(Dataset):
                 raise DerivaMLException(f"Multiple execution RIDs were found {e_rids}.")
 
             execution_rid = e_rids[0]
-        cfile = (
-            execution_metadata_dir(
-                self.working_dir,
-                exec_rid=execution_rid,
-                metadata_type=ExecMetadataVocab.execution_config.value,
-            )
-            / "configuration.json"
+        cfile = asset_file_path(
+            prefix=self.working_dir,
+            exec_rid=execution_rid,
+            file_name="configuration.json",
+            asset_table=self.model.name_to_table("Execution_Metadata"),
         )
         configuration = ExecutionConfiguration.load_configuration(cfile)
         return Execution(configuration, self, reload=execution_rid)

deriva_ml/deriva_model.py

@@ -27,6 +27,8 @@ from typing import Iterable, Optional
 class DerivaModel:
     """Augmented interface to deriva model class.
 
+    This class provides a number of DerivaML specific methods that augment the interface in the deriva model class.
+
     Attributes:
         domain_schema: Schema name for domain specific tables and relationships.
         model: ERMRest model for the catalog.
@@ -71,6 +73,10 @@ class DerivaModel:
             # No domain schema defined.
             self.domain_schema = domain_schema
 
+    def __getattr__(self, name):
+        # Called only if `name` is not found in Manager.  Delegate attributes to model class.
+        return getattr(self.model, name)
+
     def name_to_table(self, table: str | Table) -> Table:
         """Return the table object corresponding to the given table name.
 
@@ -126,6 +132,31 @@ class DerivaModel:
         table = self.name_to_table(table_name)
         return table.is_association(unqualified=unqualified, pure=pure)
 
+    def find_association(self, table1: Table | str, table2: Table | str) -> Table:
+        """Given two tables, return an association table that connects the two.
+
+        Raises:
+            DerivaML exception if there is either not an association table or more than one association table.
+        """
+        table1 = self.name_to_table(table1)
+        table2 = self.name_to_table(table2)
+
+        tables = [
+            a.table
+            for a in table1.find_associations(pure=False)
+            if a.other_fkeys.pop().pk_table == table2
+        ]
+        if len(tables) == 1:
+            return tables[0]
+        elif len(tables) == 0:
+            raise DerivaMLException(
+                f"No association tables found between {table1.name} and {table2.name}."
+            )
+        else:
+            raise DerivaMLException(
+                f"There are {len(tables)} association tables between {table1.name} and {table2.name}."
+            )
+
     def is_asset(self, table_name: str | Table) -> bool:
         """True if the specified table is an asset table.