deriva-ml 1.14.0__py3-none-any.whl → 1.14.26__py3-none-any.whl

deriva_ml/deriva_ml_base.py

@@ -1,1046 +0,0 @@
-"""
-`deriva_ml_base.py` is the core module for the Deriva ML project.  This module implements the DerivaML class, which is
-the primary interface to the Deriva based catalogs.  The module also implements the Feature and Vocabulary functions
-in the DerivaML.
-
-DerivaML and its associated classes all depend on a catalog that implements a `deriva-ml` schema with tables and
-relationships that follow a specific data model.
-
-"""
-
-from __future__ import annotations
-
-import getpass
-import logging
-from datetime import datetime
-from itertools import chain
-from pathlib import Path
-import requests
-
-from typing import Optional, Any, Iterable, TYPE_CHECKING
-
-from deriva.core import (
-    get_credential,
-    urlquote,
-    format_exception,
-    DEFAULT_SESSION_CONFIG,
-)
-import deriva.core.datapath as datapath
-from deriva.core.datapath import DataPathException
-from deriva.core.deriva_server import DerivaServer
-from deriva.core.ermrest_catalog import ResolveRidResult
-from deriva.core.ermrest_model import Key, Table
-from deriva.core.utils.globus_auth_utils import GlobusNativeLogin
-from pydantic import validate_call, ConfigDict
-
-from .execution_configuration import ExecutionConfiguration, Workflow
-from .feature import Feature, FeatureRecord
-from .dataset import Dataset
-from .dataset_aux_classes import DatasetSpec
-from .dataset_bag import DatasetBag
-from .deriva_model import DerivaModel
-from .upload import table_path, execution_rids, asset_file_path
-from .deriva_definitions import ColumnDefinition
-from .deriva_definitions import (
-    RID,
-    Status,
-    DerivaMLException,
-    ML_SCHEMA,
-    VocabularyTerm,
-    MLVocab,
-    FileSpec,
-    TableDefinition,
-)
-from .schema_setup.annotations import asset_annotation
-
-try:
-    from icecream import ic
-except ImportError:  # Graceful fallback if IceCream isn't installed.
-    ic = lambda *a: None if not a else (a[0] if len(a) == 1 else a)  # noqa
-
-
-if TYPE_CHECKING:
-    from .execution import Execution
-
-
-class DerivaML(Dataset):
-    """Base class for ML operations on a Deriva catalog.
-
-    This class is intended to be used as a base class on which more domain specific interfaces are built.
-
-    Attributes:
-        host_name: Hostname of the Deriva server.
-        catalog_id: Catalog ID. Either and identifier, or a catalog name.
-        domain_schema: Schema name for domain specific tables and relationships.
-        model: ERMRest model for the catalog
-    """
-
-    def __init__(
-        self,
-        hostname: str,
-        catalog_id: str | int,
-        domain_schema: Optional[str] = None,
-        project_name: Optional[str] = None,
-        cache_dir: Optional[str] = None,
-        working_dir: Optional[str] = None,
-        ml_schema: str = ML_SCHEMA,
-        logging_level=logging.INFO,
-        credential=None,
-        use_minid=True,
-    ):
-        """Create and initialize a DerivaML instance.
-
-        This method will connect to a catalog, and initialize local configuration for the ML execution.
-        This class is intended to be used as a base class on which domain-specific interfaces are built.
-
-        Args:
-            hostname: Hostname of the Deriva server.
-            catalog_id: Catalog ID. Either an identifier or a catalog name.
-            domain_schema: Schema name for domain-specific tables and relationships.
-            project_name: Project name. Defaults to name of domain schema.
-            cache_dir: Directory path for caching data downloaded from the Deriva server as bdbag.
-            working_dir: Directory path for storing data used by or generated by any computations.
-            use_minid: Use the MINID serice when downloading dataset bags.
-        """
-        self.credential = credential or get_credential(hostname)
-        server = DerivaServer(
-            "https",
-            hostname,
-            credentials=self.credential,
-            session_config=self._get_session_config(),
-        )
-        self.catalog = server.connect_ermrest(catalog_id)
-        self.model = DerivaModel(
-            self.catalog.getCatalogModel(), domain_schema=domain_schema
-        )
-
-        default_workdir = self.__class__.__name__ + "_working"
-        self.working_dir = (
-            Path(working_dir) / getpass.getuser()
-            if working_dir
-            else Path.home() / "deriva-ml"
-        ) / default_workdir
-
-        self.working_dir.mkdir(parents=True, exist_ok=True)
-        self.cache_dir = Path(cache_dir) if cache_dir else self.working_dir / "cache"
-
-        self.cache_dir.mkdir(parents=True, exist_ok=True)
-
-        # Initialize dataset class.
-        super().__init__(
-            self.model, self.cache_dir, self.working_dir, use_minid=use_minid
-        )
-        self._logger = logging.getLogger("deriva_ml")
-        self._logger.setLevel(logging_level)
-
-        self.host_name = hostname
-        self.catalog_id = catalog_id
-        self.ml_schema = ml_schema
-        self.configuration = None
-        self._execution: Optional[Execution] = None
-        self.domain_schema = self.model.domain_schema
-        self.project_name = project_name or self.domain_schema
-        self.start_time = datetime.now()
-        self.status = Status.pending.value
-
-        logging.basicConfig(
-            level=logging_level,
-            format="%(asctime)s - %(name)s.%(levelname)s - %(message)s",
-        )
-
-        # Set logging level for Deriva library
-        deriva_logger = logging.getLogger("deriva")
-        deriva_logger.setLevel(logging_level)
-
-    def __del__(self):
-        try:
-            if self._execution and self._execution.status != Status.completed:
-                self._execution.update_status(Status.aborted, "Execution Aborted")
-        except (AttributeError, requests.HTTPError):
-            pass
-
-    @staticmethod
-    def _get_session_config():
-        """ """
-        session_config = DEFAULT_SESSION_CONFIG.copy()
-        session_config.update(
-            {
-                # our PUT/POST to ermrest is idempotent
-                "allow_retry_on_all_methods": True,
-                # do more retries before aborting
-                "retry_read": 8,
-                "retry_connect": 5,
-                # increase delay factor * 2**(n-1) for Nth retry
-                "retry_backoff_factor": 5,
-            }
-        )
-        return session_config
-
-    # noinspection PyProtectedMember
-    @property
-    def pathBuilder(self) -> datapath._CatalogWrapper:
-        """Get a new instance of a pathBuilder object."""
-        return self.catalog.getPathBuilder()
-
-    @property
-    def domain_path(self):
-        """Get a new instance of a pathBuilder object to the domain schema"""
-
-        return self.pathBuilder.schemas[self.domain_schema]
-
-    def table_path(self, table: str | Table) -> Path:
-        """Return a local file path in which to place a CSV to add values to a table on upload.
-
-        Args:
-          table: str | Table:
-
-        Returns:
-            Path to a CSV file in which to add values to a table on upload.
-        """
-        return table_path(
-            self.working_dir,
-            schema=self.domain_schema,
-            table=self.model.name_to_table(table).name,
-        )
-
-    def download_dir(self, cached: bool = False) -> Path:
-        """Location where downloaded files are placed.
-
-        Args:
-          cached: bool:  (Default value = False)
-
-        Returns:
-
-        """
-        return self.cache_dir if cached else self.working_dir
-
-    @staticmethod
-    def globus_login(host: str) -> None:
-        """Log  into the specified host using Globus.
-
-        Args:
-            host:
-
-        Returns:
-
-        """
-        gnl = GlobusNativeLogin(host=host)
-        if gnl.is_logged_in([host]):
-            print("You are already logged in.")
-        else:
-            gnl.login(
-                [host],
-                no_local_server=True,
-                no_browser=True,
-                refresh_tokens=True,
-                update_bdbag_keychain=True,
-            )
-            print("Login Successful")
-
-    def chaise_url(self, table: RID | Table) -> str:
-        """Return a Chaise URL to the specified table.
-
-        Args:
-            table: Table or RID to be visited
-            table: str | Table:
-
-        Returns:
-            URL to the table in Chaise format.
-        """
-        table_obj = self.model.name_to_table(table)
-        try:
-            uri = self.catalog.get_server_uri().replace(
-                "ermrest/catalog/", "chaise/recordset/#"
-            )
-        except DerivaMLException:
-            # Perhaps we have a RID....
-            uri = self.cite(table)
-        return f"{uri}/{urlquote(table_obj.schema.name)}:{urlquote(table_obj.name)}"
-
-    def cite(self, entity: dict | str) -> str:
-        """Return a citation URL for the provided entity.
-
-        Args:
-            entity: A dict that contains the column values for a specific entity or a RID.
-
-        Returns:
-            The URI for the provided entity.
-
-        Raises:
-            DerivaMLException: if provided RID does not exist.
-        """
-        if isinstance(entity, str) and entity.startswith(
-            f"https://{self.host_name}/id/{self.catalog_id}/"
-        ):
-            # Already got a citation...
-            return entity
-        try:
-            self.resolve_rid(
-                rid := entity if isinstance(entity, str) else entity["RID"]
-            )
-            return f"https://{self.host_name}/id/{self.catalog_id}/{rid}@{self.catalog.latest_snapshot().snaptime}"
-        except KeyError as e:
-            raise DerivaMLException(f"Entity {e} does not have RID column")
-        except DerivaMLException as _e:
-            raise DerivaMLException("Entity RID does not exist")
-
-    def user_list(self) -> list[dict[str, str]]:
-        """List of users in the catalog
-
-        Args:
-
-        Returns:
-          A list of dictionaries containing user information.
-
-        """
-        user_path = self.pathBuilder.public.ERMrest_Client.path
-        return [
-            {"ID": u["ID"], "Full_Name": u["Full_Name"]}
-            for u in user_path.entities().fetch()
-        ]
-
-    def resolve_rid(self, rid: RID) -> ResolveRidResult:
-        """Return a named tuple with information about the specified RID.
-
-        Args:
-            rid: RID of the object of interest
-
-        Returns:
-            ResolveRidResult which has information about the specified RID.
-
-        Raises:
-          DerivaMLException: if the RID doesn't exist.
-        """
-        try:
-            return self.catalog.resolve_rid(rid, self.model.model)
-        except KeyError as _e:
-            raise DerivaMLException(f"Invalid RID {rid}")
-
-    def retrieve_rid(self, rid: RID) -> dict[str, Any]:
-        """Return a dictionary that represents the values of the specified RID.
-
-        Args:
-            rid: RID of the object of interest
-
-        Returns:
-          A dictionary that represents the values of the specified RID.
-
-        Raises:
-          DerivaMLException: if the RID doesn't exist.
-        """
-        return self.resolve_rid(rid).datapath.entities().fetch()[0]
-
-    def add_page(self, title: str, content: str) -> None:
-        """
-
-        Args:
-          title: str:
-          content: str:
-
-        Returns:
-
-        """
-        self.pathBuilder.www.tables[self.domain_schema].insert(
-            [{"Title": title, "Content": content}]
-        )
-
-    def create_vocabulary(
-        self, vocab_name: str, comment: str = "", schema: Optional[str] = None
-    ) -> Table:
-        """Create a controlled vocabulary table with the given vocab name.
-
-        Args:
-            vocab_name: Name of the controlled vocabulary table.
-            comment: Description of the vocabulary table. (Default value = '')
-            schema: Schema in which to create the controlled vocabulary table.  Defaults to domain_schema.
-            vocab_name: str:
-
-        Returns:
-            An ERMRest table object for the newly created vocabulary table.
-        """
-        schema = schema or self.domain_schema
-        return self.model.schemas[schema].create_table(
-            Table.define_vocabulary(
-                vocab_name, f"{self.project_name}:{{RID}}", comment=comment
-            )
-        )
-
-    def create_table(self, table: TableDefinition) -> Table:
-        """Create a table from a table definition."""
-        return self.model.schemas[self.domain_schema].create_table(table.model_dump())
-
-    @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
-    def create_asset(
-        self,
-        asset_name: str,
-        column_defs: Optional[Iterable[ColumnDefinition]] = None,
-        fkey_defs: Optional[Iterable[ColumnDefinition]] = None,
-        referenced_tables: Optional[Iterable[Table]] = None,
-        comment: str = "",
-        schema: Optional[str] = None,
-    ) -> Table:
-        """Create an asset table with the given asset name.
-
-        Args:
-            asset_name: Name of the asset table.
-            column_defs: Iterable of ColumnDefinition objects to provide additional metadata for asset.
-            fkey_defs: Iterable of ForeignKeyDefinition objects to provide additional metadata for asset.
-            referenced_tables: Iterable of Table objects to which asset should provide foreign-key references to.
-            comment: Description of the asset table. (Default value = '')
-            schema: Schema in which to create the asset table.  Defaults to domain_schema.
-            asset_name: str:
-            schema: str:  (Default value = None)
-
-        Returns:
-            Table object for the asset table.
-        """
-        column_defs = column_defs or []
-        fkey_defs = fkey_defs or []
-        referenced_tables = referenced_tables or []
-        schema = schema or self.domain_schema
-
-        self.add_term(
-            MLVocab.asset_type, asset_name, description=f"A {asset_name} asset"
-        )
-        asset_table = self.model.schemas[schema].create_table(
-            Table.define_asset(
-                schema,
-                asset_name,
-                column_defs=[c.model_dump() for c in column_defs],
-                fkey_defs=[fk.model_dump() for fk in fkey_defs],
-                comment=comment,
-            )
-        )
-
-        self.model.schemas[self.domain_schema].create_table(
-            Table.define_association(
-                [
-                    (asset_table.name, asset_table),
-                    ("Asset_Type", self.model.name_to_table("Asset_Type")),
-                ]
-            )
-        )
-        for t in referenced_tables:
-            asset_table.create_reference(self.model.name_to_table(t))
-        # Create a table to track execution that creates the asset
-        atable = self.model.schemas[self.domain_schema].create_table(
-            Table.define_association(
-                [
-                    (asset_name, asset_table),
-                    (
-                        "Execution",
-                        self.model.schemas[self.ml_schema].tables["Execution"],
-                    ),
-                ]
-            )
-        )
-        atable.create_reference(self.model.name_to_table("Asset_Role"))
-
-        asset_annotation(asset_table)
-        return asset_table
-
-    # @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
-    def list_assets(self, asset_table: Table | str):
-        """Return the contents of an asset table"""
-
-        if not self.model.is_asset(asset_table):
-            raise DerivaMLException(f"Table {asset_table.name} is not an asset")
-        asset_table = self.model.name_to_table(asset_table)
-        pb = self._model.catalog.getPathBuilder()
-        asset_path = pb.schemas[asset_table.schema.name].tables[asset_table.name]
-
-        asset_type_table = self._model.find_association(asset_table, MLVocab.asset_type)
-        type_path = pb.schemas[asset_type_table.schema.name].tables[
-            asset_type_table.name
-        ]
-
-        # Get a list of all the asset_type values associated with this dataset_table.
-        assets = []
-        for asset in asset_path.entities().fetch():
-            asset_types = (
-                type_path.filter(type_path.columns[asset_table.name] == asset["RID"])
-                .attributes(type_path.Asset_Type)
-                .fetch()
-            )
-            assets.append(
-                asset
-                | {
-                    MLVocab.asset_type.value: [
-                        asset_type[MLVocab.asset_type.value]
-                        for asset_type in asset_types
-                    ]
-                }
-            )
-        return assets
-
-    @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
-    def create_feature(
-        self,
-        target_table: Table | str,
-        feature_name: str,
-        terms: Optional[list[Table | str]] = None,
-        assets: Optional[list[Table | str]] = None,
-        metadata: Optional[Iterable[ColumnDefinition | Table | Key | str]] = None,
-        optional: Optional[list[str]] = None,
-        comment: str = "",
-    ) -> type[FeatureRecord]:
-        """Create a new feature that can be associated with a table.
-
-        The feature can associate a controlled vocabulary term, an asset, or any other values with a
-        specific instance of an object and  execution.
-
-        Args:
-            feature_name: Name of the new feature to be defined
-            target_table: table name or object on which the feature is to be associated
-            terms: List of controlled vocabulary terms that will be part of the feature value
-            assets: List of asset table names or objects that will be part of the feature value
-            metadata: List of other value types that are associated with the feature
-            optional: List of columns that are optional in the feature
-            comment: return: A Feature class that can be used to create instances of the feature.
-
-        Returns:
-            A Feature class that can be used to create instances of the feature.
-
-        Raises:
-            DerivaException: If the feature cannot be created.
-        """
-
-        terms = terms or []
-        assets = assets or []
-        metadata = metadata or []
-        optional = optional or []
-
-        def normalize_metadata(m: Key | Table | ColumnDefinition | str):
-            """
-
-            Args:
-              m: Key | Table | ColumnDefinition | str:
-
-            Returns:
-
-            """
-            if isinstance(m, str):
-                return self.model.name_to_table(m)
-            elif isinstance(m, ColumnDefinition):
-                return m.model_dump()
-            else:
-                return m
-
-        # Make sure that the provided assets or terms are actually assets or terms.
-        if not all(map(self.model.is_asset, assets)):
-            raise DerivaMLException("Invalid create_feature asset table.")
-        if not all(map(self.model.is_vocabulary, terms)):
-            raise DerivaMLException("Invalid create_feature asset table.")
-
-        # Get references to the necessary tables and make sure that the
-        # provided feature name exists.
-        target_table = self.model.name_to_table(target_table)
-        execution = self.model.schemas[self.ml_schema].tables["Execution"]
-        feature_name_table = self.model.schemas[self.ml_schema].tables["Feature_Name"]
-        feature_name_term = self.add_term(
-            "Feature_Name", feature_name, description=comment
-        )
-        atable_name = f"Execution_{target_table.name}_{feature_name_term.name}"
-
-        # Now create the association table that implements the feature.
-        atable = self.model.schemas[self.domain_schema].create_table(
-            target_table.define_association(
-                table_name=atable_name,
-                associates=[execution, target_table, feature_name_table],
-                metadata=[
-                    normalize_metadata(m) for m in chain(assets, terms, metadata)
-                ],
-                comment=comment,
-            )
-        )
-        # Now set optional terms.
-        for c in optional:
-            atable.columns[c].alter(nullok=True)
-        atable.columns["Feature_Name"].alter(default=feature_name_term.name)
-        return self.feature_record_class(target_table, feature_name)
-
-    def feature_record_class(
-        self, table: str | Table, feature_name: str
-    ) -> type[FeatureRecord]:
-        """Create a pydantic model for entries into the specified feature table.
-
-        For information on how to
-        See the pydantic documentation for more details about the pydantic model.
-
-        Args:
-            table: table name or object on which the feature is to be associated
-            feature_name: name of the feature to be created
-            table: str | Table:
-            feature_name: str:
-
-        Returns:
-            A Feature class that can be used to create instances of the feature.
-        """
-        return self.lookup_feature(table, feature_name).feature_record_class()
-
-    def delete_feature(self, table: Table | str, feature_name: str) -> bool:
-        """
-
-        Args:
-          table: Table | str:
-          feature_name: str:
-
-        Returns:
-        """
-        table = self.model.name_to_table(table)
-        try:
-            feature = next(
-                f for f in self.find_features(table) if f.feature_name == feature_name
-            )
-            feature.feature_table.drop()
-            return True
-        except StopIteration:
-            return False
-
-    def lookup_feature(self, table: str | Table, feature_name: str) -> Feature:
-        """Lookup the named feature associated with the provided table.
-
-        Args:
-            table: param feature_name:
-            table: str | Table:
-            feature_name: str:
-
-        Returns:
-            A Feature class that represents the requested feature.
-
-        Raises:
-          DerivaMLException: If the feature cannot be found.
-        """
-        return self.model.lookup_feature(table, feature_name)
-
-    @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
-    def find_features(self, table: Table | str) -> Iterable[Feature]:
-        """List the names of the features in the specified table.
-
-        Args:
-            table: The table to find features for.
-            table: Table | str:
-
-        Returns:
-            An iterable of FeatureResult instances that describe the current features in the table.
-        """
-        return self.model.find_features(table)
-
-    # noinspection PyProtectedMember
-    @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
-    def list_feature_values(
-        self, table: Table | str, feature_name: str
-    ) -> datapath._ResultSet:
-        """Return a datapath ResultSet containing all values of a feature associated with a table.
-
-        Args:
-            table: param feature_name:
-            table: Table | str:
-            feature_name: str:
-
-        Returns:
-
-        """
-        table = self.model.name_to_table(table)
-        feature = self.lookup_feature(table, feature_name)
-        pb = self.catalog.getPathBuilder()
-        return (
-            pb.schemas[feature.feature_table.schema.name]
-            .tables[feature.feature_table.name]
-            .entities()
-            .fetch()
-        )
-
-    @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
-    def add_term(
-        self,
-        table: str | Table,
-        term_name: str,
-        description: str,
-        synonyms: Optional[list[str]] = None,
-        exists_ok: bool = True,
-    ) -> VocabularyTerm:
-        """Creates a new control vocabulary term in the control vocabulary table.
-
-        Args:
-
-        Args:
-            table: The name of the control vocabulary table.
-            term_name: The name of the new control vocabulary.
-            description: The description of the new control vocabulary.
-            synonyms: Optional list of synonyms for the new control vocabulary. Defaults to an empty list.
-            exists_ok: Optional flag indicating whether to allow creation if the control vocabulary name
-                already exists. Defaults to True.
-
-        Returns:
-          The RID of the newly created control vocabulary.
-
-        Raises:
-          DerivaException: If the control vocabulary name already exists and exist_ok is False.
-        """
-        synonyms = synonyms or []
-        table = self.model.name_to_table(table)
-        pb = self.catalog.getPathBuilder()
-        if not (self.model.is_vocabulary(table)):
-            raise DerivaMLException(f"The table {table} is not a controlled vocabulary")
-
-        schema_name = table.schema.name
-        table_name = table.name
-        try:
-            term_id = VocabularyTerm.model_validate(
-                pb.schemas[schema_name]
-                .tables[table_name]
-                .insert(
-                    [
-                        {
-                            "Name": term_name,
-                            "Description": description,
-                            "Synonyms": synonyms,
-                        }
-                    ],
-                    defaults={"ID", "URI"},
-                )[0]
-            )
-        except DataPathException:
-            term_id = self.lookup_term(table, term_name)
-            if not exists_ok:
-                raise DerivaMLException(f"{term_name} already exists")
-            # Check vocabulary
-        return term_id
-
-    @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
-    def lookup_term(self, table: str | Table, term_name: str) -> VocabularyTerm:
-        """Given a term name, return the vocabulary record.  Can provide either the term name
-         or a synonym for the term.  Generate an exception if the term is not in the vocabulary.
-
-        Args:
-            table: The name of the controlled vocabulary table or a ERMRest table object.
-            term_name: The name of the term to look up.
-
-        Returns:
-          The entry the associated term or synonym.
-
-        Raises:
-          DerivaException: If the schema or vocabulary table doesn't exist, or if the term is not
-            found in the vocabulary.
-        """
-        vocab_table = self.model.name_to_table(table)
-        if not self.model.is_vocabulary(vocab_table):
-            raise DerivaMLException(f"The table {table} is not a controlled vocabulary")
-        schema_name, table_name = vocab_table.schema.name, vocab_table.name
-        schema_path = self.catalog.getPathBuilder().schemas[schema_name]
-
-        for term in schema_path.tables[table_name].entities().fetch():
-            if term_name == term["Name"] or (
-                term["Synonyms"] and term_name in term["Synonyms"]
-            ):
-                return VocabularyTerm.model_validate(term)
-        raise DerivaMLException(f"Term {term_name} is not in vocabulary {table_name}")
-
-    def list_vocabulary_terms(self, table: str | Table) -> list[VocabularyTerm]:
-        """Return a list of terms that are in a vocabulary table.
-
-        Args:
-            table: The name of the controlled vocabulary table or a ERMRest table object.
-            table: str | Table:
-
-        Returns:
-            The list of terms that are in a vocabulary table.
-
-        Raises:
-            DerivaMLException: If the schema or vocabulary table doesn't exist, or if the table is not
-                a controlled vocabulary.
-        """
-        pb = self.catalog.getPathBuilder()
-        table = self.model.name_to_table(table)
-        if not (self.model.is_vocabulary(table)):
-            raise DerivaMLException(f"The table {table} is not a controlled vocabulary")
-
-        return [
-            VocabularyTerm(**v)
-            for v in pb.schemas[table.schema.name].tables[table.name].entities().fetch()
-        ]
-
-    @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
-    def download_dataset_bag(
-        self,
-        dataset: DatasetSpec,
-        execution_rid: Optional[RID] = None,
-    ) -> DatasetBag:
-        """Download a dataset onto the local file system.  Create a MINID for the dataset if one doesn't already exist.
-
-        Args:
-            dataset: Specification of the dataset to be downloaded.
-            execution_rid: Execution RID for the dataset.
-
-        Returns:
-            Tuple consisting of the path to the dataset, the RID of the dataset that was downloaded and the MINID
-            for the dataset.
-        """
-        return self._download_dataset_bag(
-            dataset=dataset,
-            execution_rid=execution_rid,
-            snapshot_catalog=DerivaML(self.host_name, self._version_snapshot(dataset)),
-        )
-
-    def _update_status(
-        self, new_status: Status, status_detail: str, execution_rid: RID
-    ):
-        """Update the status of an execution in the catalog.
-
-        Args:
-            new_status: New status.
-            status_detail: Details of the status.
-            execution_rid: Resource Identifier (RID) of the execution.
-            new_status: Status:
-            status_detail: str:
-             execution_rid: RID:
-
-        Returns:
-
-        """
-        self.status = new_status.value
-        self.pathBuilder.schemas[self.ml_schema].Execution.update(
-            [
-                {
-                    "RID": execution_rid,
-                    "Status": self.status,
-                    "Status_Detail": status_detail,
-                }
-            ]
-        )
-
-    @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
-    def add_files(
-        self,
-        files: Iterable[FileSpec],
-        file_types: str | list[str],
-        execution_rid: Optional[RID] = None,
-    ) -> Iterable[RID]:
-        """Add a new file to the File table in the catalog.
-
-        The input is an iterator of FileSpec objects which provide the MD5 checksum, length, and URL.
-
-        Args:
-            file_types: One or more file types.  Must be a term from the File_Type controlled vocabulary.
-            files: A sequence of file specifications that describe the files to add.
-            execution_rid: Resource Identifier (RID) of the execution to associate with the file.
-
-        Returns:
-            Iterable of the RIDs of the files that were added.
-        """
-        defined_types = self.list_vocabulary_terms(MLVocab.file_type)
-        if execution_rid and self.resolve_rid(execution_rid).table.name != "Execution":
-            raise DerivaMLException(
-                f"RID {execution_rid} is not for an execution table."
-            )
-
-        def check_file_type(dtype: str) -> bool:
-            """Make sure that the specified string is either the name or synonym for a file type term."""
-            for term in defined_types:
-                if dtype == term.name or (term.synonyms and file_type in term.synonyms):
-                    return True
-            return False
-
-        file_types = [file_types] if isinstance(file_types, str) else file_types
-        pb = self._model.catalog.getPathBuilder()
-        for file_type in file_types:
-            if not check_file_type(file_type):
-                raise DerivaMLException("File type must be a vocabulary term.")
-        file_table_path = pb.schemas[self.ml_schema].tables["File"]
-        file_rids = [
-            e["RID"] for e in file_table_path.insert([f.model_dump() for f in files])
-        ]
-
-        # Get the name of the association table between file_table and file_type.
-        atable = next(
-            self._model.schemas[self._ml_schema]
-            .tables[MLVocab.file_type]
-            .find_associations()
-        ).name
-        pb.schemas[self._ml_schema].tables[atable].insert(
-            [
-                {"File_Type": file_type, "File": file_rid}
-                for file_rid in file_rids
-                for file_type in file_types
-            ]
-        )
-
-        if execution_rid:
-            # Get the name of the association table between file_table and execution.
-            pb.schemas[self._ml_schema].File_Execution.insert(
-                [
-                    {"File": file_rid, "Execution": execution_rid}
-                    for file_rid in file_rids
-                ]
-            )
-        return file_rids
-
-    def list_files(
-        self, file_types: Optional[list[str]] = None
-    ) -> list[dict[str, Any]]:
-        """Return the contents of the file table.  Denormalized file types into the file record."""
-        ml_path = self.pathBuilder.schemas[self._ml_schema]
-        file_path = ml_path.File
-        type_path = ml_path.File_File_Type
-
-        path = file_path.link(
-            type_path, on=file_path.RID == type_path.File, join_type="left"
-        )
-        path = path.File.attributes(
-            path.File.RID,
-            path.File.URL,
-            path.File.MD5,
-            path.File.Length,
-            path.File.Description,
-            path.File_File_Type.File_Type,
-        )
-        file_map = {}
-        for f in path.fetch():
-            entry = file_map.setdefault(f["RID"], {**f, "File_Types": []})
-            if ft := f.get("File_Type"):  # assign-and-test in one go
-                entry["File_Types"].append(ft)
-
-        # Now get rid of the File_Type key and return the result
-        return [(f, f.pop("File_Type"))[0] for f in file_map.values()]
-
-    def list_workflows(self) -> list[Workflow]:
-        """Return a list of all the workflows in the catalog."""
-        workflow_path = self.pathBuilder.schemas[self.ml_schema].Workflow
-        return [
-            Workflow(
-                name=w["Name"],
-                url=w["URL"],
-                workflow_type=w["Workflow_Type"],
-                version=w["Version"],
-                description=w["Description"],
-                rid=w["RID"],
-                checksum=w["Checksum"],
-            )
-            for w in workflow_path.entities().fetch()
-        ]
-
-    def add_workflow(self, workflow: Workflow) -> RID:
-        """Add a workflow to the Workflow table.
-
-        Args:
-            workflow: An instance of a Workflow object.
-
-        Returns:
-          - str: Resource Identifier (RID) of the added workflow.
-
-        """
-
-        # Check to make sure that the workflow is not already in the table. If it's not, add it.
-
-        if workflow_rid := self.lookup_workflow(workflow.url):
-            return workflow_rid
-
-        ml_schema_path = self.pathBuilder.schemas[self.ml_schema]
-        try:
-            # Record doesn't exist already
-            workflow_record = {
-                "URL": workflow.url,
-                "Name": workflow.name,
-                "Description": workflow.description,
-                "Checksum": workflow.checksum,
-                "Version": workflow.version,
-                MLVocab.workflow_type: self.lookup_term(
-                    MLVocab.workflow_type, workflow.workflow_type
-                ).name,
-            }
-            workflow_rid = ml_schema_path.Workflow.insert([workflow_record])[0]["RID"]
-        except Exception as e:
-            error = format_exception(e)
-            raise DerivaMLException(f"Failed to insert workflow. Error: {error}")
-        return workflow_rid
-
-    def lookup_workflow(self, url: str) -> Optional[RID]:
-        """Given a URL, look in the workflow table to find a matching workflow."""
-        workflow_path = self.pathBuilder.schemas[self.ml_schema].Workflow
-        try:
-            url_column = workflow_path.URL
-            return list(workflow_path.filter(url_column == url).entities())[0]["RID"]
-        except IndexError:
-            return None
-
-    def create_workflow(
-        self, name: str, workflow_type: str, description: str = ""
-    ) -> Workflow:
-        """Identify current executing program and return a workflow RID for it
-
-        Determine the notebook or script that is currently being executed. Assume that this is
-        being executed from a cloned GitHub repository.  Determine the remote repository name for
-        this object.  Then either retrieve an existing workflow for this executable or create
-        a new one.
-
-        Args:
-            name: The name of the workflow.
-            workflow_type: The type of the workflow.
-            description: The description of the workflow.
-
-        Returns:
-            A workflow object.
-        """
-        # Make sure type is correct.
-        self.lookup_term(MLVocab.workflow_type, workflow_type)
-
-        return Workflow.create_workflow(name, workflow_type, description)
-
-    # @validate_call
-    def create_execution(
-        self, configuration: ExecutionConfiguration, dry_run: bool = False
-    ) -> "Execution":
-        """Create an execution object
-
-        Given an execution configuration, initialize the local compute environment to prepare for executing an
-        ML or analytic routine.  This routine has a number of side effects.
-
-        1. The datasets specified in the configuration are downloaded and placed in the cache-dir. If a version is
-        not specified in the configuration, then a new minor version number is created for the dataset and downloaded.
-
-        2. If any execution assets are provided in the configuration, they are downloaded and placed in the working directory.
-
-
-        Args:
-            configuration: ExecutionConfiguration:
-            dry_run: Do not create an execution record or upload results.
-
-        Returns:
-            An execution object.
-        """
-        from .execution import Execution
-
-        self._execution = Execution(configuration, self, dry_run=dry_run)
-        return self._execution
-
-    # @validate_call
-    def restore_execution(self, execution_rid: Optional[RID] = None) -> "Execution":
-        """Return an Execution object for a previously started execution with the specified RID."""
-
-        from .execution import Execution
-
-        # Find path to execution
-        if not execution_rid:
-            e_rids = execution_rids(self.working_dir)
-            if len(e_rids) != 1:
-                raise DerivaMLException(f"Multiple execution RIDs were found {e_rids}.")
-
-            execution_rid = e_rids[0]
-        cfile = asset_file_path(
-            prefix=self.working_dir,
-            exec_rid=execution_rid,
-            file_name="configuration.json",
-            asset_table=self.model.name_to_table("Execution_Metadata"),
-            metadata={},
-        )
-
-        if cfile.exists():
-            configuration = ExecutionConfiguration.load_configuration(cfile)
-        else:
-            execution = self.retrieve_rid(execution_rid)
-            configuration = ExecutionConfiguration(
-                workflow=execution["Workflow"],
-                description=execution["Description"],
-            )
-        return Execution(configuration, self, reload=execution_rid)