deriva-ml 1.17.10__py3-none-any.whl → 1.17.11__py3-none-any.whl

deriva_ml/core/base.py

@@ -14,56 +14,53 @@ Typical usage example:
 from __future__ import annotations  # noqa: I001
 
 # Standard library imports
-from collections import defaultdict
 import logging
 from datetime import datetime
-from itertools import chain
 from pathlib import Path
-from typing import Dict, Iterable, List, cast, TYPE_CHECKING, Any
+from typing import Dict, List, cast, TYPE_CHECKING, Any
 from typing_extensions import Self
-from urllib.parse import urlsplit
-
 
 # Third-party imports
 import requests
-from pydantic import ConfigDict, validate_call
-
-# Deriva imports
-from deriva.core import DEFAULT_SESSION_CONFIG, format_exception, get_credential, urlquote
-
-import deriva.core.datapath as datapath
-from deriva.core.datapath import DataPathException, _SchemaWrapper as SchemaWrapper
-from deriva.core.deriva_server import DerivaServer
-from deriva.core.ermrest_catalog import ResolveRidResult
-from deriva.core.ermrest_model import Key, Table
-from deriva.core.utils.core_utils import DEFAULT_LOGGER_OVERRIDES
-from deriva.core.utils.globus_auth_utils import GlobusNativeLogin
-
-from deriva_ml.core.exceptions import DerivaMLInvalidTerm
-from deriva_ml.core.definitions import (
-    ML_SCHEMA,
-    RID,
-    ColumnDefinition,
-    FileSpec,
-    MLVocab,
-    MLTable,
-    Status,
-    TableDefinition,
-    VocabularyTerm,
-)
+
+# Deriva imports - use importlib to avoid shadowing by local 'deriva.py' files
+import importlib
+_deriva_core = importlib.import_module("deriva.core")
+_deriva_server = importlib.import_module("deriva.core.deriva_server")
+_ermrest_catalog = importlib.import_module("deriva.core.ermrest_catalog")
+_ermrest_model = importlib.import_module("deriva.core.ermrest_model")
+_core_utils = importlib.import_module("deriva.core.utils.core_utils")
+_globus_auth_utils = importlib.import_module("deriva.core.utils.globus_auth_utils")
+
+DEFAULT_SESSION_CONFIG = _deriva_core.DEFAULT_SESSION_CONFIG
+get_credential = _deriva_core.get_credential
+urlquote = _deriva_core.urlquote
+DerivaServer = _deriva_server.DerivaServer
+ErmrestCatalog = _ermrest_catalog.ErmrestCatalog
+ErmrestSnapshot = _ermrest_catalog.ErmrestSnapshot
+Table = _ermrest_model.Table
+DEFAULT_LOGGER_OVERRIDES = _core_utils.DEFAULT_LOGGER_OVERRIDES
+deriva_tags = _core_utils.tag
+GlobusNativeLogin = _globus_auth_utils.GlobusNativeLogin
+
 from deriva_ml.core.config import DerivaMLConfig
-from deriva_ml.core.exceptions import DerivaMLTableTypeError, DerivaMLException
-from deriva_ml.dataset.aux_classes import DatasetSpec
-from deriva_ml.dataset.dataset import Dataset
-from deriva_ml.dataset.dataset_bag import DatasetBag
-from deriva_ml.dataset.upload import asset_file_path, execution_rids, table_path
-
-# Local imports
-from deriva_ml.execution.execution_configuration import ExecutionConfiguration
-from deriva_ml.execution.workflow import Workflow
-from deriva_ml.feature import Feature, FeatureRecord
-from deriva_ml.model.catalog import DerivaModel
-from deriva_ml.schema.annotations import asset_annotation
+from deriva_ml.core.definitions import ML_SCHEMA, RID, Status, TableDefinition, VocabularyTableDef
+from deriva_ml.core.exceptions import DerivaMLException
+from deriva_ml.core.logging_config import apply_logger_overrides, configure_logging
+from deriva_ml.dataset.upload import bulk_upload_configuration
+from deriva_ml.interfaces import DerivaMLCatalog
+from deriva_ml.core.mixins import (
+    AnnotationMixin,
+    VocabularyMixin,
+    RidResolutionMixin,
+    PathBuilderMixin,
+    WorkflowMixin,
+    FeatureMixin,
+    DatasetMixin,
+    AssetMixin,
+    ExecutionMixin,
+    FileMixin,
+)
 
 # Optional debug imports
 try:
@@ -75,12 +72,25 @@ except ImportError:  # Graceful fallback if IceCream isn't installed.
 
 if TYPE_CHECKING:
     from deriva_ml.execution.execution import Execution
+    from deriva_ml.model.catalog import DerivaModel
 
 # Stop pycharm from complaining about undefined references.
 ml: DerivaML
 
 
-class DerivaML(Dataset):
+class DerivaML(
+    PathBuilderMixin,
+    RidResolutionMixin,
+    VocabularyMixin,
+    WorkflowMixin,
+    FeatureMixin,
+    DatasetMixin,
+    AssetMixin,
+    ExecutionMixin,
+    FileMixin,
+    AnnotationMixin,
+    DerivaMLCatalog,
+):
     """Core class for machine learning operations on a Deriva catalog.
 
     This class provides core functionality for managing ML workflows, features, and datasets in a Deriva catalog.
@@ -105,26 +115,79 @@ class DerivaML(Dataset):
         >>> ml.add_term('vocabulary_table', 'new_term', description='Description of term')
     """
 
+    # Class-level type annotations for DerivaMLCatalog protocol compliance
+    ml_schema: str
+    domain_schemas: frozenset[str]
+    default_schema: str | None
+    model: DerivaModel
+    cache_dir: Path
+    working_dir: Path
+    catalog: ErmrestCatalog | ErmrestSnapshot
+    catalog_id: str | int
+
     @classmethod
     def instantiate(cls, config: DerivaMLConfig) -> Self:
+        """Create a DerivaML instance from a configuration object.
+
+        This method is the preferred way to instantiate DerivaML when using hydra-zen
+        for configuration management. It accepts a DerivaMLConfig (Pydantic model) and
+        unpacks it to create the instance.
+
+        This pattern allows hydra-zen's `instantiate()` to work with DerivaML:
+
+        Example with hydra-zen:
+            >>> from hydra_zen import builds, instantiate
+            >>> from deriva_ml import DerivaML
+            >>> from deriva_ml.core.config import DerivaMLConfig
+            >>>
+            >>> # Create a structured config using hydra-zen
+            >>> DerivaMLConf = builds(DerivaMLConfig, populate_full_signature=True)
+            >>>
+            >>> # Configure for your environment
+            >>> conf = DerivaMLConf(
+            ...     hostname='deriva.example.org',
+            ...     catalog_id='42',
+            ...     domain_schema='my_domain',
+            ... )
+            >>>
+            >>> # Instantiate the config to get a DerivaMLConfig object
+            >>> config = instantiate(conf)
+            >>>
+            >>> # Create the DerivaML instance
+            >>> ml = DerivaML.instantiate(config)
+
+        Args:
+            config: A DerivaMLConfig object containing all configuration parameters.
+
+        Returns:
+            A new DerivaML instance configured according to the config object.
+
+        Note:
+            The DerivaMLConfig class integrates with Hydra's configuration system
+            and registers custom resolvers for computing working directories.
+            See `deriva_ml.core.config` for details on configuration options.
+        """
         return cls(**config.model_dump())
 
     def __init__(
         self,
         hostname: str,
         catalog_id: str | int,
-        domain_schema: str | None = None,
+        domain_schemas: set[str] | None = None,
+        default_schema: str | None = None,
         project_name: str | None = None,
         cache_dir: str | Path | None = None,
         working_dir: str | Path | None = None,
         hydra_runtime_output_dir: str | Path | None = None,
         ml_schema: str = ML_SCHEMA,
-        logging_level=logging.WARNING,
-        deriva_logging_level=logging.WARNING,
-        credential=None,
-        use_minid: bool = True,
+        logging_level: int = logging.WARNING,
+        deriva_logging_level: int = logging.WARNING,
+        credential: dict | None = None,
+        s3_bucket: str | None = None,
+        use_minid: bool | None = None,
         check_auth: bool = True,
-    ):
+        clean_execution_dir: bool = True,
+    ) -> None:
         """Initializes a DerivaML instance.
 
         This method will connect to a catalog and initialize local configuration for the ML execution.
@@ -133,17 +196,28 @@ class DerivaML(Dataset):
         Args:
             hostname: Hostname of the Deriva server.
             catalog_id: Catalog ID. Either an identifier or a catalog name.
-            domain_schema: Schema name for domain-specific tables and relationships. Defaults to the name of the
-                schema that is not one of the standard schemas.  If there is more than one user-defined schema, then
-                this argument must be provided a value.
+            domain_schemas: Optional set of domain schema names. If None, auto-detects all
+                non-system schemas. Use this when working with catalogs that have multiple
+                user-defined schemas.
+            default_schema: The default schema for table creation operations. If None and
+                there is exactly one domain schema, that schema is used. If there are multiple
+                domain schemas, this must be specified for table creation to work without
+                explicit schema parameters.
             ml_schema: Schema name for ML schema. Used if you have a non-standard configuration of deriva-ml.
-            project_name: Project name. Defaults to name of domain schema.
+            project_name: Project name. Defaults to name of default_schema.
             cache_dir: Directory path for caching data downloaded from the Deriva server as bdbag. If not provided,
                 will default to working_dir.
             working_dir: Directory path for storing data used by or generated by any computations. If no value is
                 provided, will default to  ${HOME}/deriva_ml
-            use_minid: Use the MINID service when downloading dataset bags.
+            s3_bucket: S3 bucket URL for dataset bag storage (e.g., 's3://my-bucket'). If provided,
+                enables MINID creation and S3 upload for dataset exports. If None, MINID functionality
+                is disabled regardless of use_minid setting.
+            use_minid: Use the MINID service when downloading dataset bags. Only effective when
+                s3_bucket is configured. If None (default), automatically set to True when s3_bucket
+                is provided, False otherwise.
             check_auth: Check if the user has access to the catalog.
+            clean_execution_dir: Whether to automatically clean up execution working directories
+                after successful upload. Defaults to True. Set to False to retain local copies.
         """
         # Get or use provided credentials for server access
         self.credential = credential or get_credential(hostname)
@@ -164,32 +238,46 @@ class DerivaML(Dataset):
                 "Please check your credentials and make sure you have logged in."
             )
         self.catalog = server.connect_ermrest(catalog_id)
-        self.model = DerivaModel(self.catalog.getCatalogModel(), domain_schema=domain_schema)
+        # Import here to avoid circular imports
+        from deriva_ml.model.catalog import DerivaModel
+        self.model = DerivaModel(
+            self.catalog.getCatalogModel(),
+            ml_schema=ml_schema,
+            domain_schemas=domain_schemas,
+            default_schema=default_schema,
+        )
+
+        # Store S3 bucket configuration and resolve use_minid
+        self.s3_bucket = s3_bucket
+        if use_minid is None:
+            # Auto mode: enable MINID if s3_bucket is configured
+            self.use_minid = s3_bucket is not None
+        elif use_minid and s3_bucket is None:
+            # User requested MINID but no S3 bucket configured - disable MINID
+            self.use_minid = False
+        else:
+            self.use_minid = use_minid
 
         # Set up working and cache directories
-        self.working_dir = DerivaMLConfig.compute_workdir(working_dir)
+        self.working_dir = DerivaMLConfig.compute_workdir(working_dir, catalog_id)
         self.working_dir.mkdir(parents=True, exist_ok=True)
         self.hydra_runtime_output_dir = hydra_runtime_output_dir
 
         self.cache_dir = Path(cache_dir) if cache_dir else self.working_dir / "cache"
         self.cache_dir.mkdir(parents=True, exist_ok=True)
 
-        # Initialize dataset functionality from the parent class
-        super().__init__(self.model, self.cache_dir, self.working_dir, use_minid=use_minid)
-
-        # Set up logging
-        self._logger = logging.getLogger("deriva_ml")
-        self._logger.setLevel(logging_level)
+        # Set up logging using centralized configuration
+        # This configures deriva_ml, Hydra, and deriva-py loggers without
+        # affecting the root logger or calling basicConfig()
+        self._logger = configure_logging(
+            level=logging_level,
+            deriva_level=deriva_logging_level,
+        )
         self._logging_level = logging_level
         self._deriva_logging_level = deriva_logging_level
 
-        # Configure deriva logging level
-        logger_config = DEFAULT_LOGGER_OVERRIDES
-        # allow for reconfiguration of module-specific logging levels
-        [logging.getLogger(name).setLevel(level) for name, level in logger_config.items()]
-        logging.getLogger("root").setLevel(deriva_logging_level)
-        logging.getLogger("bagit").setLevel(deriva_logging_level)
-        logging.getLogger("bdbag").setLevel(deriva_logging_level)
+        # Apply deriva's default logger overrides for fine-grained control
+        apply_logger_overrides(DEFAULT_LOGGER_OVERRIDES)
 
         # Store instance configuration
         self.host_name = hostname
@@ -197,22 +285,14 @@ class DerivaML(Dataset):
         self.ml_schema = ml_schema
         self.configuration = None
         self._execution: Execution | None = None
-        self.domain_schema = self.model.domain_schema
-        self.project_name = project_name or self.domain_schema
+        self.domain_schemas = self.model.domain_schemas
+        self.default_schema = self.model.default_schema
+        self.project_name = project_name or self.default_schema or "deriva-ml"
         self.start_time = datetime.now()
         self.status = Status.pending.value
+        self.clean_execution_dir = clean_execution_dir
 
-        # Configure logging format
-        logging.basicConfig(
-            level=logging_level,
-            format="%(asctime)s - %(name)s.%(levelname)s - %(message)s",
-        )
-
-        # Set Deriva library logging level
-        deriva_logger = logging.getLogger("deriva")
-        deriva_logger.setLevel(logging_level)
-
-    def __del__(self):
+    def __del__(self) -> None:
         """Cleanup method to handle incomplete executions."""
         try:
             # Mark execution as aborted if not completed
@@ -222,7 +302,7 @@ class DerivaML(Dataset):
             pass
 
     @staticmethod
-    def _get_session_config():
+    def _get_session_config() -> dict:
         """Returns customized HTTP session configuration.
 
         Configures retry behavior and connection settings for HTTP requests to the Deriva server. Settings include:
@@ -254,58 +334,23 @@ class DerivaML(Dataset):
         )
         return session_config
 
-    @property
-    def pathBuilder(self) -> SchemaWrapper:
-        """Returns catalog path builder for queries.
+    def is_snapshot(self) -> bool:
+        return hasattr(self.catalog, "_snaptime")
 
-        The path builder provides a fluent interface for constructing complex queries against the catalog.
-        This is a core component used by many other methods to interact with the catalog.
-
-        Returns:
-            datapath._CatalogWrapper: A new instance of the catalog path builder.
-
-        Example:
-            >>> path = ml.pathBuilder.schemas['my_schema'].tables['my_table']
-            >>> results = path.entities().fetch()
-        """
-        return self.catalog.getPathBuilder()
+    def catalog_snapshot(self, version_snapshot: str) -> Self:
+        """Returns a DerivaML instance for a specific snapshot of the catalog."""
+        return DerivaML(
+            self.host_name,
+            version_snapshot,
+            logging_level=self._logging_level,
+            deriva_logging_level=self._deriva_logging_level,
+        )
 
     @property
-    def domain_path(self) -> datapath.DataPath:
-        """Returns path builder for domain schema.
-
-        Provides a convenient way to access tables and construct queries within the domain-specific schema.
-
-        Returns:
-            datapath._CatalogWrapper: Path builder object scoped to the domain schema.
-
-        Example:
-            >>> domain = ml.domain_path
-            >>> results = domain.my_table.entities().fetch()
-        """
-        return self.pathBuilder.schemas[self.domain_schema]
-
-    def table_path(self, table: str | Table) -> Path:
-        """Returns a local filesystem path for table CSV files.
+    def _dataset_table(self) -> Table:
+        return self.model.schemas[self.model.ml_schema].tables["Dataset"]
 
-        Generates a standardized path where CSV files should be placed when preparing to upload data to a table.
-        The path follows the project's directory structure conventions.
-
-        Args:
-            table: Name of the table or Table object to get the path for.
-
-        Returns:
-            Path: Filesystem path where the CSV file should be placed.
-
-        Example:
-            >>> path = ml.table_path("experiment_results")
-            >>> df.to_csv(path) # Save data for upload
-        """
-        return table_path(
-            self.working_dir,
-            schema=self.domain_schema,
-            table=self.model.name_to_table(table).name,
-        )
+    # pathBuilder, domain_path, table_path moved to PathBuilderMixin
 
     def download_dir(self, cached: bool = False) -> Path:
         """Returns the appropriate download directory.
@@ -384,27 +429,37 @@ class DerivaML(Dataset):
             uri = self.cite(cast(str, table))
         return f"{uri}/{urlquote(table_obj.schema.name)}:{urlquote(table_obj.name)}"
 
-    def cite(self, entity: Dict[str, Any] | str) -> str:
-        """Generates permanent citation URL.
+    def cite(self, entity: Dict[str, Any] | str, current: bool = False) -> str:
+        """Generates citation URL for an entity.
 
-        Creates a versioned URL that can be used to reference a specific entity in the catalog. The URL includes
-        the catalog snapshot time to ensure version stability.
+        Creates a URL that can be used to reference a specific entity in the catalog.
+        By default, includes the catalog snapshot time to ensure version stability
+        (permanent citation). With current=True, returns a URL to the current state.
 
         Args:
             entity: Either a RID string or a dictionary containing entity data with a 'RID' key.
+            current: If True, return URL to current catalog state (no snapshot).
+                     If False (default), return permanent citation URL with snapshot time.
 
         Returns:
-            str: Permanent citation URL in format: https://{host}/id/{catalog}/{rid}@{snapshot_time}
+            str: Citation URL. Format depends on `current` parameter:
+                - current=False: https://{host}/id/{catalog}/{rid}@{snapshot_time}
+                - current=True: https://{host}/id/{catalog}/{rid}
 
         Raises:
             DerivaMLException: If an entity doesn't exist or lacks a RID.
 
         Examples:
-            Using a RID string:
+            Permanent citation (default):
                 >>> url = ml.cite("1-abc123")
                 >>> print(url)
                 'https://deriva.org/id/1/1-abc123@2024-01-01T12:00:00'
 
+            Current catalog URL:
+                >>> url = ml.cite("1-abc123", current=True)
+                >>> print(url)
+                'https://deriva.org/id/1/1-abc123'
+
             Using a dictionary:
                 >>> url = ml.cite({"RID": "1-abc123"})
         """
@@ -413,9 +468,12 @@ class DerivaML(Dataset):
             return entity
 
         try:
-            # Resolve RID and create citation URL with snapshot time
+            # Resolve RID and create citation URL
             self.resolve_rid(rid := entity if isinstance(entity, str) else entity["RID"])
-            return f"https://{self.host_name}/id/{self.catalog_id}/{rid}@{self.catalog.latest_snapshot().snaptime}"
+            base_url = f"https://{self.host_name}/id/{self.catalog_id}/{rid}"
+            if current:
+                return base_url
+            return f"{base_url}@{self.catalog.latest_snapshot().snaptime}"
         except KeyError as e:
             raise DerivaMLException(f"Entity {e} does not have RID column")
         except DerivaMLException as _e:
@@ -439,59 +497,247 @@ class DerivaML(Dataset):
             ...     print(f"{user['Full_Name']} ({user['ID']})")
         """
         # Get the user table path and fetch basic user info
-        user_path = self.pathBuilder.public.ERMrest_Client.path
+        user_path = self.pathBuilder().public.ERMrest_Client.path
         return [{"ID": u["ID"], "Full_Name": u["Full_Name"]} for u in user_path.entities().fetch()]
 
-    def resolve_rid(self, rid: RID) -> ResolveRidResult:
-        """Resolves RID to catalog location.
+    # resolve_rid, retrieve_rid moved to RidResolutionMixin
 
-        Looks up a RID and returns information about where it exists in the catalog, including schema,
-        table, and column metadata.
-
-        Args:
-            rid: Resource Identifier to resolve.
-
-        Returns:
-            ResolveRidResult: Named tuple containing:
-                - schema: Schema name
-                - table: Table name
-                - columns: Column definitions
-                - datapath: Path builder for accessing the entity
-
-        Raises:
-            DerivaMLException: If RID doesn't exist in catalog.
-
-        Examples:
-            >>> result = ml.resolve_rid("1-abc123")
-            >>> print(f"Found in {result.schema}.{result.table}")
-            >>> data = result.datapath.entities().fetch()
-        """
-        try:
-            # Attempt to resolve RID using catalog model
-            return self.catalog.resolve_rid(rid, self.model.model)
-        except KeyError as _e:
-            raise DerivaMLException(f"Invalid RID {rid}")
-
-    def retrieve_rid(self, rid: RID) -> dict[str, Any]:
-        """Retrieves complete record for RID.
-
-        Fetches all column values for the entity identified by the RID.
+    def apply_catalog_annotations(
+        self,
+        navbar_brand_text: str = "ML Data Browser",
+        head_title: str = "Catalog ML",
+    ) -> None:
+        """Apply catalog-level annotations including the navigation bar and display settings.
+
+        This method configures the Chaise web interface for the catalog. Chaise is Deriva's
+        web-based data browser that provides a user-friendly interface for exploring and
+        managing catalog data. This method sets up annotations that control how Chaise
+        displays and organizes the catalog.
+
+        **Navigation Bar Structure**:
+        The method creates a navigation bar with the following menus:
+        - **User Info**: Links to Users, Groups, and RID Lease tables
+        - **Deriva-ML**: Core ML tables (Workflow, Execution, Dataset, Dataset_Version, etc.)
+        - **WWW**: Web content tables (Page, File)
+        - **{Domain Schema}**: All domain-specific tables (excludes vocabularies and associations)
+        - **Vocabulary**: All controlled vocabulary tables from both ML and domain schemas
+        - **Assets**: All asset tables from both ML and domain schemas
+        - **Features**: All feature tables with entries named "TableName:FeatureName"
+        - **Catalog Registry**: Link to the ermrest registry
+        - **Documentation**: Links to ML notebook instructions and Deriva-ML docs
+
+        **Display Settings**:
+        - Underscores in table/column names displayed as spaces
+        - System columns (RID) shown in compact and entry views
+        - Default table set to Dataset
+        - Faceting and record deletion enabled
+        - Export configurations available to all users
+
+        **Bulk Upload Configuration**:
+        Configures upload patterns for asset tables, enabling drag-and-drop file uploads
+        through the Chaise interface.
+
+        Call this after creating the domain schema and all tables to initialize the catalog's
+        web interface. The navigation menus are dynamically built based on the current schema
+        structure, automatically organizing tables into appropriate categories.
 
         Args:
-            rid: Resource Identifier of the record to retrieve.
-
-        Returns:
-            dict[str, Any]: Dictionary containing all column values for the entity.
-
-        Raises:
-            DerivaMLException: If the RID doesn't exist in the catalog.
+            navbar_brand_text: Text displayed in the navigation bar brand area.
+            head_title: Title displayed in the browser tab.
 
         Example:
-            >>> record = ml.retrieve_rid("1-abc123")
-            >>> print(f"Name: {record['name']}, Created: {record['creation_date']}")
+            >>> ml = DerivaML('deriva.example.org', 'my_catalog')
+            >>> # After creating domain schema and tables...
+            >>> ml.apply_catalog_annotations()
+            >>> # Or with custom branding:
+            >>> ml.apply_catalog_annotations("My Project Browser", "My ML Project")
         """
-        # Resolve RID and fetch the first (only) matching record
-        return self.resolve_rid(rid).datapath.entities().fetch()[0]
+        catalog_id = self.model.catalog.catalog_id
+        ml_schema = self.ml_schema
+
+        # Build domain schema menu items (one menu per domain schema)
+        domain_schema_menus = []
+        for domain_schema in sorted(self.domain_schemas):
+            if domain_schema not in self.model.schemas:
+                continue
+            domain_schema_menus.append({
+                "name": domain_schema,
+                "children": [
+                    {
+                        "name": tname,
+                        "url": f"/chaise/recordset/#{catalog_id}/{domain_schema}:{tname}",
+                    }
+                    for tname in self.model.schemas[domain_schema].tables
+                    # Don't include controlled vocabularies, association tables, or feature tables.
+                    if not (
+                        self.model.is_vocabulary(tname)
+                        or self.model.is_association(tname, pure=False, max_arity=3)
+                    )
+                ],
+            })
+
+        # Build vocabulary menu items (ML schema + all domain schemas)
+        vocab_children = [{"name": f"{ml_schema} Vocabularies", "header": True}]
+        vocab_children.extend([
+            {
+                "url": f"/chaise/recordset/#{catalog_id}/{ml_schema}:{tname}",
+                "name": tname,
+            }
+            for tname in self.model.schemas[ml_schema].tables
+            if self.model.is_vocabulary(tname)
+        ])
+        for domain_schema in sorted(self.domain_schemas):
+            if domain_schema not in self.model.schemas:
+                continue
+            vocab_children.append({"name": f"{domain_schema} Vocabularies", "header": True})
+            vocab_children.extend([
+                {
+                    "url": f"/chaise/recordset/#{catalog_id}/{domain_schema}:{tname}",
+                    "name": tname,
+                }
+                for tname in self.model.schemas[domain_schema].tables
+                if self.model.is_vocabulary(tname)
+            ])
+
+        # Build asset menu items (ML schema + all domain schemas)
+        asset_children = [
+            {
+                "url": f"/chaise/recordset/#{catalog_id}/{ml_schema}:{tname}",
+                "name": tname,
+            }
+            for tname in self.model.schemas[ml_schema].tables
+            if self.model.is_asset(tname)
+        ]
+        for domain_schema in sorted(self.domain_schemas):
+            if domain_schema not in self.model.schemas:
+                continue
+            asset_children.extend([
+                {
+                    "url": f"/chaise/recordset/#{catalog_id}/{domain_schema}:{tname}",
+                    "name": tname,
+                }
+                for tname in self.model.schemas[domain_schema].tables
+                if self.model.is_asset(tname)
+            ])
+
+        catalog_annotation = {
+            deriva_tags.display: {"name_style": {"underline_space": True}},
+            deriva_tags.chaise_config: {
+                "headTitle": head_title,
+                "navbarBrandText": navbar_brand_text,
+                "systemColumnsDisplayEntry": ["RID"],
+                "systemColumnsDisplayCompact": ["RID"],
+                "defaultTable": {"table": "Dataset", "schema": "deriva-ml"},
+                "deleteRecord": True,
+                "showFaceting": True,
+                "shareCiteAcls": True,
+                "exportConfigsSubmenu": {"acls": {"show": ["*"], "enable": ["*"]}},
+                "resolverImplicitCatalog": False,
+                "navbarMenu": {
+                    "newTab": False,
+                    "children": [
+                        {
+                            "name": "User Info",
+                            "children": [
+                                {
+                                    "url": f"/chaise/recordset/#{catalog_id}/public:ERMrest_Client",
+                                    "name": "Users",
+                                },
+                                {
+                                    "url": f"/chaise/recordset/#{catalog_id}/public:ERMrest_Group",
+                                    "name": "Groups",
+                                },
+                                {
+                                    "url": f"/chaise/recordset/#{catalog_id}/public:ERMrest_RID_Lease",
+                                    "name": "ERMrest RID Lease",
+                                },
+                            ],
+                        },
+                        {  # All the primary tables in deriva-ml schema.
+                            "name": "Deriva-ML",
+                            "children": [
+                                {
+                                    "url": f"/chaise/recordset/#{catalog_id}/{ml_schema}:Workflow",
+                                    "name": "Workflow",
+                                },
+                                {
+                                    "url": f"/chaise/recordset/#{catalog_id}/{ml_schema}:Execution",
+                                    "name": "Execution",
+                                },
+                                {
+                                    "url": f"/chaise/recordset/#{catalog_id}/{ml_schema}:Execution_Metadata",
+                                    "name": "Execution Metadata",
+                                },
+                                {
+                                    "url": f"/chaise/recordset/#{catalog_id}/{ml_schema}:Execution_Asset",
+                                    "name": "Execution Asset",
+                                },
+                                {
+                                    "url": f"/chaise/recordset/#{catalog_id}/{ml_schema}:Dataset",
+                                    "name": "Dataset",
+                                },
+                                {
+                                    "url": f"/chaise/recordset/#{catalog_id}/{ml_schema}:Dataset_Version",
+                                    "name": "Dataset Version",
+                                },
+                            ],
+                        },
+                        {  # WWW schema tables.
+                            "name": "WWW",
+                            "children": [
+                                {
+                                    "url": f"/chaise/recordset/#{catalog_id}/WWW:Page",
+                                    "name": "Page",
+                                },
+                                {
+                                    "url": f"/chaise/recordset/#{catalog_id}/WWW:File",
+                                    "name": "File",
+                                },
+                            ],
+                        },
+                        *domain_schema_menus,  # One menu per domain schema
+                        {  # Vocabulary menu with all controlled vocabularies.
+                            "name": "Vocabulary",
+                            "children": vocab_children,
+                        },
+                        {  # List of all asset tables.
+                            "name": "Assets",
+                            "children": asset_children,
+                        },
+                        {  # List of all feature tables in the catalog.
+                            "name": "Features",
+                            "children": [
+                                {
+                                    "url": f"/chaise/recordset/#{catalog_id}/{f.feature_table.schema.name}:{f.feature_table.name}",
+                                    "name": f"{f.target_table.name}:{f.feature_name}",
+                                }
+                                for f in self.model.find_features()
+                            ],
+                        },
+                        {
+                            "url": "/chaise/recordset/#0/ermrest:registry@sort(RID)",
+                            "name": "Catalog Registry",
+                        },
+                        {
+                            "name": "Documentation",
+                            "children": [
+                                {
+                                    "url": "https://github.com/informatics-isi-edu/deriva-ml/blob/main/docs/ml_workflow_instruction.md",
+                                    "name": "ML Notebook Instruction",
+                                },
+                                {
+                                    "url": "https://informatics-isi-edu.github.io/deriva-ml/",
+                                    "name": "Deriva-ML Documentation",
+                                },
+                            ],
+                        },
+                    ],
+                },
+            },
+            deriva_tags.bulk_upload: bulk_upload_configuration(model=self.model),
+        }
+        self.model.annotations.update(catalog_annotation)
+        self.model.apply()
 
     def add_page(self, title: str, content: str) -> None:
         """Adds page to web interface.
@@ -513,9 +759,15 @@ class DerivaML(Dataset):
             ... )
         """
         # Insert page into www tables with title and content
-        self.pathBuilder.www.tables[self.domain_schema].insert([{"Title": title, "Content": content}])
-
-    def create_vocabulary(self, vocab_name: str, comment: str = "", schema: str | None = None) -> Table:
+        # Use default schema or first domain schema for www tables
+        schema = self.default_schema or (sorted(self.domain_schemas)[0] if self.domain_schemas else None)
+        if schema is None:
+            raise DerivaMLException("No domain schema available for adding pages")
+        self.pathBuilder().www.tables[schema].insert([{"Title": title, "Content": content}])
+
+    def create_vocabulary(
+        self, vocab_name: str, comment: str = "", schema: str | None = None, update_navbar: bool = True
+    ) -> Table:
         """Creates a controlled vocabulary table.
 
         A controlled vocabulary table maintains a list of standardized terms and their definitions. Each term can have
@@ -525,6 +777,9 @@ class DerivaML(Dataset):
             vocab_name: Name for the new vocabulary table. Must be a valid SQL identifier.
             comment: Description of the vocabulary's purpose and usage. Defaults to empty string.
             schema: Schema name to create the table in. If None, uses domain_schema.
+            update_navbar: If True (default), automatically updates the navigation bar to include
+                the new vocabulary table. Set to False during batch table creation to avoid
+                redundant updates, then call apply_catalog_annotations() once at the end.
 
         Returns:
             Table: ERMRest table object representing the newly created vocabulary table.
@@ -540,988 +795,483 @@ class DerivaML(Dataset):
                 ...     comment="Standard tissue classifications",
                 ...     schema="bio_schema"
                 ... )
+
+            Create multiple vocabularies without updating navbar until the end:
+
+                >>> ml.create_vocabulary("Species", update_navbar=False)
+                >>> ml.create_vocabulary("Tissue_Type", update_navbar=False)
+                >>> ml.apply_catalog_annotations()  # Update navbar once
         """
-        # Use domain schema if none specified
-        schema = schema or self.domain_schema
+        # Use default schema if none specified
+        schema = schema or self.model._require_default_schema()
 
         # Create and return vocabulary table with RID-based URI pattern
         try:
             vocab_table = self.model.schemas[schema].create_table(
-                Table.define_vocabulary(vocab_name, f"{self.project_name}:{{RID}}", comment=comment)
+                VocabularyTableDef(
+                    name=vocab_name,
+                    curie_template=f"{self.project_name}:{{RID}}",
+                    comment=comment,
+                )
             )
         except ValueError:
             raise DerivaMLException(f"Table {vocab_name} already exist")
-        return vocab_table
-
-    def create_table(self, table: TableDefinition) -> Table:
-        """Creates a new table in the catalog.
-
-        Creates a table using the provided TableDefinition object, which specifies the table structure including
-        columns, keys, and foreign key relationships.
-
-        Args:
-            table: A TableDefinition object containing the complete specification of the table to create.
-
-        Returns:
-            Table: The newly created ERMRest table object.
-
-        Raises:
-            DerivaMLException: If table creation fails or the definition is invalid.
-
-        Example:
-
-            >>> table_def = TableDefinition(
-            ...     name="experiments",
-            ...     column_definitions=[
-            ...         ColumnDefinition(name="name", type=BuiltinTypes.text),
-            ...         ColumnDefinition(name="date", type=BuiltinTypes.date)
-            ...     ]
-            ... )
-            >>> new_table = ml.create_table(table_def)
-        """
-        # Create table in domain schema using provided definition
-        return self.model.schemas[self.domain_schema].create_table(table.model_dump())
-
-    @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
-    def create_asset(
-        self,
-        asset_name: str,
-        column_defs: Iterable[ColumnDefinition] | None = None,
-        fkey_defs: Iterable[ColumnDefinition] | None = None,
-        referenced_tables: Iterable[Table] | None = None,
-        comment: str = "",
-        schema: str | None = None,
-    ) -> Table:
-        """Creates an asset table.
-
-        Args:
-            asset_name: Name of the asset table.
-            column_defs: Iterable of ColumnDefinition objects to provide additional metadata for asset.
-            fkey_defs: Iterable of ForeignKeyDefinition objects to provide additional metadata for asset.
-            referenced_tables: Iterable of Table objects to which asset should provide foreign-key references to.
-            comment: Description of the asset table. (Default value = '')
-            schema: Schema in which to create the asset table.  Defaults to domain_schema.
-
-        Returns:
-            Table object for the asset table.
-        """
-        # Initialize empty collections if None provided
-        column_defs = column_defs or []
-        fkey_defs = fkey_defs or []
-        referenced_tables = referenced_tables or []
-        schema = schema or self.domain_schema
-
-        # Add an asset type to vocabulary
-        self.add_term(MLVocab.asset_type, asset_name, description=f"A {asset_name} asset")
-
-        # Create the main asset table
-        asset_table = self.model.schemas[schema].create_table(
-            Table.define_asset(
-                schema,
-                asset_name,
-                column_defs=[c.model_dump() for c in column_defs],
-                fkey_defs=[fk.model_dump() for fk in fkey_defs],
-                comment=comment,
-            )
-        )
 
-        # Create an association table between asset and asset type
-        self.model.schemas[self.domain_schema].create_table(
-            Table.define_association(
-                [
-                    (asset_table.name, asset_table),
-                    ("Asset_Type", self.model.name_to_table("Asset_Type")),
-                ]
-            )
-        )
-
-        # Create references to other tables if specified
-        for t in referenced_tables:
-            asset_table.create_reference(self.model.name_to_table(t))
-
-        # Create an association table for tracking execution
-        atable = self.model.schemas[self.domain_schema].create_table(
-            Table.define_association(
-                [
-                    (asset_name, asset_table),
-                    (
-                        "Execution",
-                        self.model.schemas[self.ml_schema].tables["Execution"],
-                    ),
-                ]
-            )
-        )
-        atable.create_reference(self.model.name_to_table("Asset_Role"))
+        # Update navbar to include the new vocabulary table
+        if update_navbar:
+            self.apply_catalog_annotations()
 
-        # Add asset annotations
-        asset_annotation(asset_table)
-        return asset_table
+        return vocab_table
 
-    def list_assets(self, asset_table: Table | str) -> list[dict[str, Any]]:
-        """Lists contents of an asset table.
+    def create_table(self, table: TableDefinition, schema: str | None = None, update_navbar: bool = True) -> Table:
+        """Creates a new table in the domain schema.
 
-        Returns a list of assets with their types for the specified asset table.
+        Creates a table using the provided TableDefinition object, which specifies the table structure
+        including columns, keys, and foreign key relationships. The table is created in the domain
+        schema associated with this DerivaML instance.
 
-        Args:
-            asset_table: Table or name of the asset table to list assets for.
+        **Required Classes**:
+        Import the following classes from deriva_ml to define tables:
 
-        Returns:
-            list[dict[str, Any]]: List of asset records, each containing:
-                - RID: Resource identifier
-                - Type: Asset type
-                - Metadata: Asset metadata
+        - ``TableDefinition``: Defines the complete table structure
+        - ``ColumnDefinition``: Defines individual columns with types and constraints
+        - ``KeyDefinition``: Defines unique key constraints (optional)
+        - ``ForeignKeyDefinition``: Defines foreign key relationships to other tables (optional)
+        - ``BuiltinTypes``: Enum of available column data types
 
-        Raises:
-            DerivaMLException: If the table is not an asset table or doesn't exist.
-
-        Example:
-            >>> assets = ml.list_assets("tissue_types")
-            >>> for asset in assets:
-            ...     print(f"{asset['RID']}: {asset['Type']}")
-        """
-        # Validate and get asset table reference
-        asset_table = self.model.name_to_table(asset_table)
-        if not self.model.is_asset(asset_table):
-            raise DerivaMLException(f"Table {asset_table.name} is not an asset")
-
-        # Get path builders for asset and type tables
-        pb = self._model.catalog.getPathBuilder()
-        asset_path = pb.schemas[asset_table.schema.name].tables[asset_table.name]
-        (
-            asset_type_table,
-            _,
-            _,
-        ) = self._model.find_association(asset_table, MLVocab.asset_type)
-        type_path = pb.schemas[asset_type_table.schema.name].tables[asset_type_table.name]
-
-        # Build a list of assets with their types
-        assets = []
-        for asset in asset_path.entities().fetch():
-            # Get associated asset types for each asset
-            asset_types = (
-                type_path.filter(type_path.columns[asset_table.name] == asset["RID"])
-                .attributes(type_path.Asset_Type)
-                .fetch()
-            )
-            # Combine asset data with its types
-            assets.append(
-                asset | {MLVocab.asset_type.value: [asset_type[MLVocab.asset_type.value] for asset_type in asset_types]}
-            )
-        return assets
-
-    @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
-    def create_feature(
-        self,
-        target_table: Table | str,
-        feature_name: str,
-        terms: list[Table | str] | None = None,
-        assets: list[Table | str] | None = None,
-        metadata: list[ColumnDefinition | Table | Key | str] | None = None,
-        optional: list[str] | None = None,
-        comment: str = "",
-    ) -> type[FeatureRecord]:
-        """Creates a new feature definition.
-
-        A feature represents a measurable property or characteristic that can be associated with records in the target
-        table. Features can include vocabulary terms, asset references, and additional metadata.
+        **Available Column Types** (BuiltinTypes enum):
+        ``text``, ``int2``, ``int4``, ``int8``, ``float4``, ``float8``, ``boolean``,
+        ``date``, ``timestamp``, ``timestamptz``, ``json``, ``jsonb``, ``markdown``,
+        ``ermrest_uri``, ``ermrest_rid``, ``ermrest_rcb``, ``ermrest_rmb``,
+        ``ermrest_rct``, ``ermrest_rmt``
 
         Args:
-            target_table: Table to associate the feature with (name or Table object).
-            feature_name: Unique name for the feature within the target table.
-            terms: Optional vocabulary tables/names whose terms can be used as feature values.
-            assets: Optional asset tables/names that can be referenced by this feature.
-            metadata: Optional columns, tables, or keys to include in a feature definition.
-            optional: Column names that are not required when creating feature instances.
-            comment: Description of the feature's purpose and usage.
+            table: A TableDefinition object containing the complete specification of the table to create.
+            update_navbar: If True (default), automatically updates the navigation bar to include
+                the new table. Set to False during batch table creation to avoid redundant updates,
+                then call apply_catalog_annotations() once at the end.
 
         Returns:
-            type[FeatureRecord]: Feature class for creating validated instances.
+            Table: The newly created ERMRest table object.
 
         Raises:
-            DerivaMLException: If a feature definition is invalid or conflicts with existing features.
+            DerivaMLException: If table creation fails or the definition is invalid.
 
         Examples:
-            Create a feature with confidence score:
-                >>> feature_class = ml.create_feature(
-                ...     target_table="samples",
-                ...     feature_name="expression_level",
-                ...     terms=["expression_values"],
-                ...     metadata=[ColumnDefinition(name="confidence", type=BuiltinTypes.float4)],
-                ...     comment="Gene expression measurement"
+            **Simple table with basic columns**:
+
+                >>> from deriva_ml import TableDefinition, ColumnDefinition, BuiltinTypes
+                >>>
+                >>> table_def = TableDefinition(
+                ...     name="Experiment",
+                ...     column_defs=[
+                ...         ColumnDefinition(name="Name", type=BuiltinTypes.text, nullok=False),
+                ...         ColumnDefinition(name="Date", type=BuiltinTypes.date),
+                ...         ColumnDefinition(name="Description", type=BuiltinTypes.markdown),
+                ...         ColumnDefinition(name="Score", type=BuiltinTypes.float4),
+                ...     ],
+                ...     comment="Records of experimental runs"
                 ... )
-        """
-        # Initialize empty collections if None provided
-        terms = terms or []
-        assets = assets or []
-        metadata = metadata or []
-        optional = optional or []
-
-        def normalize_metadata(m: Key | Table | ColumnDefinition | str):
-            """Helper function to normalize metadata references."""
-            if isinstance(m, str):
-                return self.model.name_to_table(m)
-            elif isinstance(m, ColumnDefinition):
-                return m.model_dump()
-            else:
-                return m
-
-        # Validate asset and term tables
-        if not all(map(self.model.is_asset, assets)):
-            raise DerivaMLException("Invalid create_feature asset table.")
-        if not all(map(self.model.is_vocabulary, terms)):
-            raise DerivaMLException("Invalid create_feature asset table.")
-
-        # Get references to required tables
-        target_table = self.model.name_to_table(target_table)
-        execution = self.model.schemas[self.ml_schema].tables["Execution"]
-        feature_name_table = self.model.schemas[self.ml_schema].tables["Feature_Name"]
-
-        # Add feature name to vocabulary
-        feature_name_term = self.add_term("Feature_Name", feature_name, description=comment)
-        atable_name = f"Execution_{target_table.name}_{feature_name_term.name}"
-        # Create an association table implementing the feature
-        atable = self.model.schemas[self.domain_schema].create_table(
-            target_table.define_association(
-                table_name=atable_name,
-                associates=[execution, target_table, feature_name_table],
-                metadata=[normalize_metadata(m) for m in chain(assets, terms, metadata)],
-                comment=comment,
-            )
-        )
-        # Configure optional columns and default feature name
-        for c in optional:
-            atable.columns[c].alter(nullok=True)
-        atable.columns["Feature_Name"].alter(default=feature_name_term.name)
-
-        # Return feature record class for creating instances
-        return self.feature_record_class(target_table, feature_name)
-
-    def feature_record_class(self, table: str | Table, feature_name: str) -> type[FeatureRecord]:
-        """Returns a pydantic model class for feature records.
-
-        Creates a typed interface for creating new instances of the specified feature. The returned class includes
-        validation and type checking based on the feature's definition.
-
-        Args:
-            table: The table containing the feature, either as name or Table object.
-            feature_name: Name of the feature to create a record class for.
-
-        Returns:
-            type[FeatureRecord]: A pydantic model class for creating validated feature records.
-
-        Raises:
-            DerivaMLException: If the feature doesn't exist or the table is invalid.
+                >>> experiment_table = ml.create_table(table_def)
 
-        Example:
-            >>> ExpressionFeature = ml.feature_record_class("samples", "expression_level")
-            >>> feature = ExpressionFeature(value="high", confidence=0.95)
-        """
-        # Look up a feature and return its record class
-        return self.lookup_feature(table, feature_name).feature_record_class()
+            **Table with foreign key to another table**:
 
-    def delete_feature(self, table: Table | str, feature_name: str) -> bool:
-        """Removes a feature definition and its data.
-
-        Deletes the feature and its implementation table from the catalog. This operation cannot be undone and
-        will remove all feature values associated with this feature.
+                >>> from deriva_ml import (
+                ...     TableDefinition, ColumnDefinition, ForeignKeyDefinition, BuiltinTypes
+                ... )
+                >>>
+                >>> # Create a Sample table that references Subject
+                >>> sample_def = TableDefinition(
+                ...     name="Sample",
+                ...     column_defs=[
+                ...         ColumnDefinition(name="Name", type=BuiltinTypes.text, nullok=False),
+                ...         ColumnDefinition(name="Subject", type=BuiltinTypes.text, nullok=False),
+                ...         ColumnDefinition(name="Collection_Date", type=BuiltinTypes.date),
+                ...     ],
+                ...     fkey_defs=[
+                ...         ForeignKeyDefinition(
+                ...             colnames=["Subject"],
+                ...             pk_sname=ml.default_schema,  # Schema of referenced table
+                ...             pk_tname="Subject",          # Name of referenced table
+                ...             pk_colnames=["RID"],         # Column(s) in referenced table
+                ...             on_delete="CASCADE",         # Delete samples when subject deleted
+                ...         )
+                ...     ],
+                ...     comment="Biological samples collected from subjects"
+                ... )
+                >>> sample_table = ml.create_table(sample_def)
 
-        Args:
-            table: The table containing the feature, either as name or Table object.
-            feature_name: Name of the feature to delete.
+            **Table with unique key constraint**:
 
-        Returns:
-            bool: True if the feature was successfully deleted, False if it didn't exist.
+                >>> from deriva_ml import (
+                ...     TableDefinition, ColumnDefinition, KeyDefinition, BuiltinTypes
+                ... )
+                >>>
+                >>> protocol_def = TableDefinition(
+                ...     name="Protocol",
+                ...     column_defs=[
+                ...         ColumnDefinition(name="Name", type=BuiltinTypes.text, nullok=False),
+                ...         ColumnDefinition(name="Version", type=BuiltinTypes.text, nullok=False),
+                ...         ColumnDefinition(name="Description", type=BuiltinTypes.markdown),
+                ...     ],
+                ...     key_defs=[
+                ...         KeyDefinition(
+                ...             colnames=["Name", "Version"],
+                ...             constraint_names=[["myschema", "Protocol_Name_Version_key"]],
+                ...             comment="Each protocol name+version must be unique"
+                ...         )
+                ...     ],
+                ...     comment="Experimental protocols with versioning"
+                ... )
+                >>> protocol_table = ml.create_table(protocol_def)
 
-        Raises:
-            DerivaMLException: If deletion fails due to constraints or permissions.
+            **Batch creation without navbar updates**:
 
-        Example:
-            >>> success = ml.delete_feature("samples", "obsolete_feature")
-            >>> print("Deleted" if success else "Not found")
+                >>> ml.create_table(table1_def, update_navbar=False)
+                >>> ml.create_table(table2_def, update_navbar=False)
+                >>> ml.create_table(table3_def, update_navbar=False)
+                >>> ml.apply_catalog_annotations()  # Update navbar once at the end
         """
-        # Get table reference and find feature
-        table = self.model.name_to_table(table)
-        try:
-            # Find and delete the feature's implementation table
-            feature = next(f for f in self.model.find_features(table) if f.feature_name == feature_name)
-            feature.feature_table.drop()
-            return True
-        except StopIteration:
-            return False
+        # Use default schema if none specified
+        schema = schema or self.model._require_default_schema()
 
-    def lookup_feature(self, table: str | Table, feature_name: str) -> Feature:
-        """Retrieves a Feature object.
+        # Create table in domain schema using provided definition
+        # Handle both TableDefinition (dataclass with to_dict) and plain dicts
+        table_dict = table.to_dict() if hasattr(table, 'to_dict') else table
+        new_table = self.model.schemas[schema].create_table(table_dict)
 
-        Looks up and returns a Feature object that provides an interface to work with an existing feature
-        definition in the catalog.
+        # Update navbar to include the new table
+        if update_navbar:
+            self.apply_catalog_annotations()
 
-        Args:
-            table: The table containing the feature, either as name or Table object.
-            feature_name: Name of the feature to look up.
+        return new_table
 
-        Returns:
-            Feature: An object representing the feature and its implementation.
+    # =========================================================================
+    # Cache and Directory Management
+    # =========================================================================
 
-        Raises:
-            DerivaMLException: If the feature doesn't exist in the specified table.
+    def clear_cache(self, older_than_days: int | None = None) -> dict[str, int]:
+        """Clear the dataset cache directory.
 
-        Example:
-            >>> feature = ml.lookup_feature("samples", "expression_level")
-            >>> print(feature.feature_name)
-            'expression_level'
-        """
-        return self.model.lookup_feature(table, feature_name)
-
-    @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
-    def list_feature_values(self, table: Table | str, feature_name: str) -> datapath._ResultSet:
-        """Retrieves all values for a feature.
-
-        Returns all instances of the specified feature that have been created, including their associated
-        metadata and references.
+        Removes cached dataset bags from the cache directory. Can optionally filter
+        by age to only remove old cache entries.
 
         Args:
-            table: The table containing the feature, either as name or Table object.
-            feature_name: Name of the feature to retrieve values for.
+            older_than_days: If provided, only remove cache entries older than this
+                many days. If None, removes all cache entries.
 
         Returns:
-            datapath._ResultSet: A result set containing all feature values and their metadata.
-
-        Raises:
-            DerivaMLException: If the feature doesn't exist or cannot be accessed.
+            dict with keys:
+                - 'files_removed': Number of files removed
+                - 'dirs_removed': Number of directories removed
+                - 'bytes_freed': Total bytes freed
+                - 'errors': Number of removal errors
 
         Example:
-            >>> values = ml.list_feature_values("samples", "expression_level")
-            >>> for value in values:
-            ...     print(f"Sample {value['RID']}: {value['value']}")
+            >>> ml = DerivaML('deriva.example.org', 'my_catalog')
+            >>> # Clear all cache
+            >>> result = ml.clear_cache()
+            >>> print(f"Freed {result['bytes_freed'] / 1e6:.1f} MB")
+            >>>
+            >>> # Clear cache older than 7 days
+            >>> result = ml.clear_cache(older_than_days=7)
         """
-        # Get table and feature references
-        table = self.model.name_to_table(table)
-        feature = self.lookup_feature(table, feature_name)
-
-        # Build and execute query for feature values
-        pb = self.catalog.getPathBuilder()
-        return pb.schemas[feature.feature_table.schema.name].tables[feature.feature_table.name].entities().fetch()
+        import shutil
+        import time
 
-    @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
-    def add_term(
-        self,
-        table: str | Table,
-        term_name: str,
-        description: str,
-        synonyms: list[str] | None = None,
-        exists_ok: bool = True,
-    ) -> VocabularyTerm:
-        """Adds a term to a vocabulary table.
+        stats = {'files_removed': 0, 'dirs_removed': 0, 'bytes_freed': 0, 'errors': 0}
 
-        Creates a new standardized term with description and optional synonyms in a vocabulary table.
-        Can either create a new term or return an existing one if it already exists.
+        if not self.cache_dir.exists():
+            return stats
 
-        Args:
-            table: Vocabulary table to add term to (name or Table object).
-            term_name: Primary name of the term (must be unique within vocabulary).
-            description: Explanation of term's meaning and usage.
-            synonyms: Alternative names for the term.
-            exists_ok: If True, return the existing term if found. If False, raise error.
-
-        Returns:
-            VocabularyTerm: Object representing the created or existing term.
-
-        Raises:
-            DerivaMLException: If a term exists and exists_ok=False, or if the table is not a vocabulary table.
-
-        Examples:
-            Add a new tissue type:
-                >>> term = ml.add_term(
-                ...     table="tissue_types",
-                ...     term_name="epithelial",
-                ...     description="Epithelial tissue type",
-                ...     synonyms=["epithelium"]
-                ... )
-
-            Attempt to add an existing term:
-                >>> term = ml.add_term("tissue_types", "epithelial", "...", exists_ok=True)
-        """
-        # Initialize an empty synonyms list if None
-        synonyms = synonyms or []
-
-        # Get table reference and validate if it is a vocabulary table
-        table = self.model.name_to_table(table)
-        pb = self.catalog.getPathBuilder()
-        if not (self.model.is_vocabulary(table)):
-            raise DerivaMLTableTypeError("vocabulary", table.name)
-
-        # Get schema and table names for path building
-        schema_name = table.schema.name
-        table_name = table.name
+        cutoff_time = None
+        if older_than_days is not None:
+            cutoff_time = time.time() - (older_than_days * 24 * 60 * 60)
 
         try:
-            # Attempt to insert a new term
-            term_id = VocabularyTerm.model_validate(
-                pb.schemas[schema_name]
-                .tables[table_name]
-                .insert(
-                    [
-                        {
-                            "Name": term_name,
-                            "Description": description,
-                            "Synonyms": synonyms,
-                        }
-                    ],
-                    defaults={"ID", "URI"},
-                )[0]
-            )
-        except DataPathException:
-            # Term exists - look it up or raise an error
-            term_id = self.lookup_term(table, term_name)
-            if not exists_ok:
-                raise DerivaMLInvalidTerm(table.name, term_name, msg="term already exists")
-        return term_id
-
-    @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
-    def lookup_term(self, table: str | Table, term_name: str) -> VocabularyTerm:
-        """Finds a term in a vocabulary table.
-
-        Searches for a term in the specified vocabulary table, matching either the primary name
-        or any of its synonyms.
-
-        Args:
-            table: Vocabulary table to search in (name or Table object).
-            term_name: Name or synonym of the term to find.
+            for entry in self.cache_dir.iterdir():
+                try:
+                    # Check age if filtering
+                    if cutoff_time is not None:
+                        entry_mtime = entry.stat().st_mtime
+                        if entry_mtime > cutoff_time:
+                            continue  # Skip recent entries
+
+                    # Calculate size before removal
+                    if entry.is_dir():
+                        entry_size = sum(f.stat().st_size for f in entry.rglob('*') if f.is_file())
+                        shutil.rmtree(entry)
+                        stats['dirs_removed'] += 1
+                    else:
+                        entry_size = entry.stat().st_size
+                        entry.unlink()
+                        stats['files_removed'] += 1
+
+                    stats['bytes_freed'] += entry_size
+                except (OSError, PermissionError) as e:
+                    self._logger.warning(f"Failed to remove cache entry {entry}: {e}")
+                    stats['errors'] += 1
+
+        except OSError as e:
+            self._logger.error(f"Failed to iterate cache directory: {e}")
+            stats['errors'] += 1
+
+        return stats
+
+    def get_cache_size(self) -> dict[str, int | float]:
+        """Get the current size of the cache directory.
 
         Returns:
-            VocabularyTerm: The matching vocabulary term.
+            dict with keys:
+                - 'total_bytes': Total size in bytes
+                - 'total_mb': Total size in megabytes
+                - 'file_count': Number of files
+                - 'dir_count': Number of directories
 
-        Raises:
-            DerivaMLVocabularyException: If the table is not a vocabulary table, or term is not found.
-
-        Examples:
-            Look up by primary name:
-                >>> term = ml.lookup_term("tissue_types", "epithelial")
-                >>> print(term.description)
-
-            Look up by synonym:
-                >>> term = ml.lookup_term("tissue_types", "epithelium")
+        Example:
+            >>> ml = DerivaML('deriva.example.org', 'my_catalog')
+            >>> size = ml.get_cache_size()
+            >>> print(f"Cache size: {size['total_mb']:.1f} MB ({size['file_count']} files)")
         """
-        # Get and validate vocabulary table reference
-        vocab_table = self.model.name_to_table(table)
-        if not self.model.is_vocabulary(vocab_table):
-            raise DerivaMLException(f"The table {table} is not a controlled vocabulary")
+        stats = {'total_bytes': 0, 'total_mb': 0.0, 'file_count': 0, 'dir_count': 0}
 
-        # Get schema and table paths
-        schema_name, table_name = vocab_table.schema.name, vocab_table.name
-        schema_path = self.catalog.getPathBuilder().schemas[schema_name]
+        if not self.cache_dir.exists():
+            return stats
 
-        # Search for term by name or synonym
-        for term in schema_path.tables[table_name].entities().fetch():
-            if term_name == term["Name"] or (term["Synonyms"] and term_name in term["Synonyms"]):
-                return VocabularyTerm.model_validate(term)
+        for entry in self.cache_dir.rglob('*'):
+            if entry.is_file():
+                stats['total_bytes'] += entry.stat().st_size
+                stats['file_count'] += 1
+            elif entry.is_dir():
+                stats['dir_count'] += 1
 
-        # Term not found
-        raise DerivaMLInvalidTerm(table_name, term_name)
+        stats['total_mb'] = stats['total_bytes'] / (1024 * 1024)
+        return stats
 
-    def list_vocabulary_terms(self, table: str | Table) -> list[VocabularyTerm]:
-        """Lists all terms in a vocabulary table.
+    def list_execution_dirs(self) -> list[dict[str, any]]:
+        """List execution working directories.
 
-        Retrieves all terms, their descriptions, and synonyms from a controlled vocabulary table.
-
-        Args:
-            table: Vocabulary table to list terms from (name or Table object).
+        Returns information about each execution directory in the working directory,
+        useful for identifying orphaned or incomplete execution outputs.
 
         Returns:
-            list[VocabularyTerm]: List of vocabulary terms with their metadata.
+            List of dicts, each containing:
+                - 'execution_rid': The execution RID (directory name)
+                - 'path': Full path to the directory
+                - 'size_bytes': Total size in bytes
+                - 'size_mb': Total size in megabytes
+                - 'modified': Last modification time (datetime)
+                - 'file_count': Number of files
 
-        Raises:
-            DerivaMLException: If table doesn't exist or is not a vocabulary table.
-
-        Examples:
-            >>> terms = ml.list_vocabulary_terms("tissue_types")
-            >>> for term in terms:
-            ...     print(f"{term.name}: {term.description}")
-            ...     if term.synonyms:
-            ...         print(f"  Synonyms: {', '.join(term.synonyms)}")
+        Example:
+            >>> ml = DerivaML('deriva.example.org', 'my_catalog')
+            >>> dirs = ml.list_execution_dirs()
+            >>> for d in dirs:
+            ...     print(f"{d['execution_rid']}: {d['size_mb']:.1f} MB")
         """
-        # Get path builder and table reference
-        pb = self.catalog.getPathBuilder()
-        table = self.model.name_to_table(table.value if isinstance(table, MLVocab) else table)
+        from datetime import datetime
+        from deriva_ml.dataset.upload import upload_root
 
-        # Validate table is a vocabulary table
-        if not (self.model.is_vocabulary(table)):
-            raise DerivaMLException(f"The table {table} is not a controlled vocabulary")
+        results = []
+        exec_root = upload_root(self.working_dir) / "execution"
 
-        # Fetch and convert all terms to VocabularyTerm objects
-        return [VocabularyTerm(**v) for v in pb.schemas[table.schema.name].tables[table.name].entities().fetch()]
+        if not exec_root.exists():
+            return results
 
-    @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
-    def download_dataset_bag(
-        self,
-        dataset: DatasetSpec,
-        execution_rid: RID | None = None,
-    ) -> DatasetBag:
-        """Downloads a dataset to the local filesystem and creates a MINID if needed.
+        for entry in exec_root.iterdir():
+            if entry.is_dir():
+                size_bytes = sum(f.stat().st_size for f in entry.rglob('*') if f.is_file())
+                file_count = sum(1 for f in entry.rglob('*') if f.is_file())
+                mtime = datetime.fromtimestamp(entry.stat().st_mtime)
 
-        Downloads a dataset specified by DatasetSpec to the local filesystem. If the dataset doesn't have
-        a MINID (Minimal Viable Identifier), one will be created. The dataset can optionally be associated
-        with an execution record.
+                results.append({
+                    'execution_rid': entry.name,
+                    'path': str(entry),
+                    'size_bytes': size_bytes,
+                    'size_mb': size_bytes / (1024 * 1024),
+                    'modified': mtime,
+                    'file_count': file_count,
+                })
 
-        Args:
-            dataset: Specification of the dataset to download, including version and materialization options.
-            execution_rid: Optional execution RID to associate the download with.
-
-        Returns:
-            DatasetBag: Object containing:
-                - path: Local filesystem path to downloaded dataset
-                - rid: Dataset's Resource Identifier
-                - minid: Dataset's Minimal Viable Identifier
+        return sorted(results, key=lambda x: x['modified'], reverse=True)
 
-        Examples:
-            Download with default options:
-                >>> spec = DatasetSpec(rid="1-abc123")
-                >>> bag = ml.download_dataset_bag(dataset=spec)
-                >>> print(f"Downloaded to {bag.path}")
-
-            Download with execution tracking:
-                >>> bag = ml.download_dataset_bag(
-                ...     dataset=DatasetSpec(rid="1-abc123", materialize=True),
-                ...     execution_rid="1-xyz789"
-                ... )
-        """
-        if not self._is_dataset_rid(dataset.rid):
-            raise DerivaMLTableTypeError("Dataset", dataset.rid)
-        return self._download_dataset_bag(
-            dataset=dataset,
-            execution_rid=execution_rid,
-            snapshot_catalog=DerivaML(
-                self.host_name,
-                self._version_snapshot(dataset),
-                logging_level=self._logging_level,
-                deriva_logging_level=self._deriva_logging_level,
-            ),
-        )
-
-    def _update_status(self, new_status: Status, status_detail: str, execution_rid: RID):
-        """Update the status of an execution in the catalog.
-
-        Args:
-            new_status: New status.
-            status_detail: Details of the status.
-            execution_rid: Resource Identifier (RID) of the execution.
-            new_status: Status:
-            status_detail: str:
-             execution_rid: RID:
-
-        Returns:
-
-        """
-        self.status = new_status.value
-        self.pathBuilder.schemas[self.ml_schema].Execution.update(
-            [
-                {
-                    "RID": execution_rid,
-                    "Status": self.status,
-                    "Status_Detail": status_detail,
-                }
-            ]
-        )
-
-    @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
-    def add_files(
+    def clean_execution_dirs(
         self,
-        files: Iterable[FileSpec],
-        dataset_types: str | list[str] | None = None,
-        description: str = "",
-        execution_rid: RID | None = None,
-    ) -> RID:
-        """Adds files to the catalog with their metadata.
+        older_than_days: int | None = None,
+        exclude_rids: list[str] | None = None,
+    ) -> dict[str, int]:
+        """Clean up execution working directories.
 
-        Registers files in the catalog along with their metadata (MD5, length, URL) and associates them with
-        specified file types. Optionally links files to an execution record.
+        Removes execution output directories from the local working directory.
+        Use this to free up disk space from completed or orphaned executions.
 
         Args:
-            files: File specifications containing MD5 checksum, length, and URL.
-            dataset_types: One or more dataset type terms from File_Type vocabulary.
-            description: Description of the files.
-            execution_rid: Optional execution RID to associate files with.
+            older_than_days: If provided, only remove directories older than this
+                many days. If None, removes all execution directories (except excluded).
+            exclude_rids: List of execution RIDs to preserve (never remove).
 
         Returns:
-            RID: Resource of dataset that represents the newly added files.
-
-        Raises:
-            DerivaMLException: If file_types are invalid or execution_rid is not an execution record.
-
-        Examples:
-            Add a single file type:
-                >>> files = [FileSpec(url="path/to/file.txt", md5="abc123", length=1000)]
-                >>> rids = ml.add_files(files, file_types="text")
-
-            Add multiple file types:
-                >>> rids = ml.add_files(
-                ...     files=[FileSpec(url="image.png", md5="def456", length=2000)],
-                ...     file_types=["image", "png"],
-                ...     execution_rid="1-xyz789"
-                ... )
-        """
-        if execution_rid and self.resolve_rid(execution_rid).table.name != "Execution":
-            raise DerivaMLTableTypeError("Execution", execution_rid)
-
-        filespec_list = list(files)
+            dict with keys:
+                - 'dirs_removed': Number of directories removed
+                - 'bytes_freed': Total bytes freed
+                - 'errors': Number of removal errors
 
-        # Get a list of all defined file types and their synonyms.
-        defined_types = set(
-            chain.from_iterable([[t.name] + t.synonyms for t in self.list_vocabulary_terms(MLVocab.asset_type)])
-        )
-
-        # Get a list of al of the file types used in the filespec_list
-        spec_types = set(chain.from_iterable(filespec.file_types for filespec in filespec_list))
-
-        # Now make sure that all of the file types and dataset_types in the spec list are defined.
-        if spec_types - defined_types:
-            raise DerivaMLInvalidTerm(MLVocab.asset_type.name, f"{spec_types - defined_types}")
-
-        # Normalize dataset_types, make sure FIle type is included.
-        if isinstance(dataset_types, list):
-            dataset_types = ["File"] + dataset_types if "File" not in dataset_types else dataset_types
-        else:
-            dataset_types = ["File", dataset_types] if dataset_types else ["File"]
-        for ds_type in dataset_types:
-            self.lookup_term(MLVocab.dataset_type, ds_type)
-
-        # Add files to the file table, and collect up the resulting entries by directory name.
-        pb = self._model.catalog.getPathBuilder()
-        file_records = list(
-            pb.schemas[self.ml_schema].tables["File"].insert([f.model_dump(by_alias=True) for f in filespec_list])
-        )
-
-        # Get the name of the association table between file_table and file_type and add file_type records
-        atable = self.model.find_association(MLTable.file, MLVocab.asset_type)[0].name
-        # Need to get a link between file record and file_types.
-        type_map = {
-            file_spec.md5: file_spec.file_types + ([] if "File" in file_spec.file_types else [])
-            for file_spec in filespec_list
-        }
-        file_type_records = [
-            {MLVocab.asset_type.value: file_type, "File": file_record["RID"]}
-            for file_record in file_records
-            for file_type in type_map[file_record["MD5"]]
-        ]
-        pb.schemas[self._ml_schema].tables[atable].insert(file_type_records)
-
-        if execution_rid:
-            # Get the name of the association table between file_table and execution.
-            pb.schemas[self._ml_schema].File_Execution.insert(
-                [
-                    {"File": file_record["RID"], "Execution": execution_rid, "Asset_Role": "Output"}
-                    for file_record in file_records
-                ]
-            )
-
-        # Now create datasets to capture the original directory structure of the files.
-        dir_rid_map = defaultdict(list)
-        for e in file_records:
-            dir_rid_map[Path(urlsplit(e["URL"]).path).parent].append(e["RID"])
-
-        nested_datasets = []
-        path_length = 0
-        dataset = None
-        # Start with the longest path so we get subdirectories first.
-        for p, rids in sorted(dir_rid_map.items(), key=lambda kv: len(kv[0].parts), reverse=True):
-            dataset = self.create_dataset(
-                dataset_types=dataset_types, execution_rid=execution_rid, description=description
-            )
-            members = rids
-            if len(p.parts) < path_length:
-                # Going up one level in directory, so Create nested dataset
-                members = nested_datasets + rids
-                nested_datasets = []
-            self.add_dataset_members(dataset_rid=dataset, members=members, execution_rid=execution_rid)
-            nested_datasets.append(dataset)
-            path_length = len(p.parts)
-
-        return dataset
-
-    def list_files(self, file_types: list[str] | None = None) -> list[dict[str, Any]]:
-        """Lists files in the catalog with their metadata.
-
-        Returns a list of files with their metadata including URL, MD5 hash, length, description,
-        and associated file types. Files can be optionally filtered by type.
-
-        Args:
-            file_types: Filter results to only include these file types.
-
-        Returns:
-            list[dict[str, Any]]: List of file records, each containing:
-                - RID: Resource identifier
-                - URL: File location
-                - MD5: File hash
-                - Length: File size
-                - Description: File description
-                - File_Types: List of associated file types
-
-        Examples:
-            List all files:
-                >>> files = ml.list_files()
-                >>> for f in files:
-                ...     print(f"{f['RID']}: {f['URL']}")
-
-            Filter by file type:
-                >>> image_files = ml.list_files(["image", "png"])
+        Example:
+            >>> ml = DerivaML('deriva.example.org', 'my_catalog')
+            >>> # Clean all execution dirs older than 30 days
+            >>> result = ml.clean_execution_dirs(older_than_days=30)
+            >>> print(f"Freed {result['bytes_freed'] / 1e9:.2f} GB")
+            >>>
+            >>> # Clean all except specific executions
+            >>> result = ml.clean_execution_dirs(exclude_rids=['1-ABC', '1-DEF'])
         """
+        import shutil
+        import time
+        from deriva_ml.dataset.upload import upload_root
 
-        asset_type_atable, file_fk, asset_type_fk = self.model.find_association("File", "Asset_Type")
-        ml_path = self.pathBuilder.schemas[self._ml_schema]
-        file = ml_path.File
-        asset_type = ml_path.tables[asset_type_atable.name]
-
-        path = file.path
-        path = path.link(asset_type.alias("AT"), on=file.RID == asset_type.columns[file_fk], join_type="left")
-        if file_types:
-            path = path.filter(asset_type.columns[asset_type_fk] == datapath.Any(*file_types))
-        path = path.attributes(
-            path.File.RID,
-            path.File.URL,
-            path.File.MD5,
-            path.File.Length,
-            path.File.Description,
-            path.AT.columns[asset_type_fk],
-        )
+        stats = {'dirs_removed': 0, 'bytes_freed': 0, 'errors': 0}
+        exclude_rids = set(exclude_rids or [])
 
-        file_map = {}
-        for f in path.fetch():
-            entry = file_map.setdefault(f["RID"], {**f, "File_Types": []})
-            if ft := f.get("Asset_Type"):  # assign-and-test in one go
-                entry["File_Types"].append(ft)
+        exec_root = upload_root(self.working_dir) / "execution"
+        if not exec_root.exists():
+            return stats
 
-        # Now get rid of the File_Type key and return the result
-        return [(f, f.pop("Asset_Type"))[0] for f in file_map.values()]
+        cutoff_time = None
+        if older_than_days is not None:
+            cutoff_time = time.time() - (older_than_days * 24 * 60 * 60)
 
-    def list_workflows(self) -> list[Workflow]:
-        """Lists all workflows in the catalog.
+        for entry in exec_root.iterdir():
+            if not entry.is_dir():
+                continue
 
-        Retrieves all workflow definitions, including their names, URLs, types, versions,
-        and descriptions.
+            # Skip excluded RIDs
+            if entry.name in exclude_rids:
+                continue
 
-        Returns:
-            list[Workflow]: List of workflow objects, each containing:
-                - name: Workflow name
-                - url: Source code URL
-                - workflow_type: Type of workflow
-                - version: Version identifier
-                - description: Workflow description
-                - rid: Resource identifier
-                - checksum: Source code checksum
-
-        Examples:
-            >>> workflows = ml.list_workflows()
-            >>> for w in workflows:
-                    print(f"{w.name} (v{w.version}): {w.description}")
-                    print(f"  Source: {w.url}")
-        """
-        # Get a workflow table path and fetch all workflows
-        workflow_path = self.pathBuilder.schemas[self.ml_schema].Workflow
-        return [
-            Workflow(
-                name=w["Name"],
-                url=w["URL"],
-                workflow_type=w["Workflow_Type"],
-                version=w["Version"],
-                description=w["Description"],
-                rid=w["RID"],
-                checksum=w["Checksum"],
-            )
-            for w in workflow_path.entities().fetch()
-        ]
+            try:
+                # Check age if filtering
+                if cutoff_time is not None:
+                    entry_mtime = entry.stat().st_mtime
+                    if entry_mtime > cutoff_time:
+                        continue
 
-    def add_workflow(self, workflow: Workflow) -> RID:
-        """Adds a workflow to the catalog.
+                # Calculate size before removal
+                entry_size = sum(f.stat().st_size for f in entry.rglob('*') if f.is_file())
+                shutil.rmtree(entry)
+                stats['dirs_removed'] += 1
+                stats['bytes_freed'] += entry_size
 
-        Registers a new workflow in the catalog or returns the RID of an existing workflow with the same
-        URL or checksum.
+            except (OSError, PermissionError) as e:
+                self._logger.warning(f"Failed to remove execution dir {entry}: {e}")
+                stats['errors'] += 1
 
-        Each workflow represents a specific computational process or analysis pipeline.
+        return stats
 
-        Args:
-            workflow: Workflow object containing name, URL, type, version, and description.
+    def get_storage_summary(self) -> dict[str, any]:
+        """Get a summary of local storage usage.
 
         Returns:
-            RID: Resource Identifier of the added or existing workflow.
-
-        Raises:
-            DerivaMLException: If workflow insertion fails or required fields are missing.
-
-        Examples:
-            >>> workflow = Workflow(
-            ...     name="Gene Analysis",
-            ...     url="https://github.com/org/repo/workflows/gene_analysis.py",
-            ...     workflow_type="python_script",
-            ...     version="1.0.0",
-            ...     description="Analyzes gene expression patterns"
-            ... )
-            >>> workflow_rid = ml.add_workflow(workflow)
-        """
-        # Check if a workflow already exists by URL
-        if workflow_rid := self.lookup_workflow(workflow.checksum or workflow.url):
-            return workflow_rid
-
-        # Get an ML schema path for the workflow table
-        ml_schema_path = self.pathBuilder.schemas[self.ml_schema]
-
-        try:
-            # Create a workflow record
-            workflow_record = {
-                "URL": workflow.url,
-                "Name": workflow.name,
-                "Description": workflow.description,
-                "Checksum": workflow.checksum,
-                "Version": workflow.version,
-                MLVocab.workflow_type: self.lookup_term(MLVocab.workflow_type, workflow.workflow_type).name,
-            }
-            # Insert a workflow and get its RID
-            workflow_rid = ml_schema_path.Workflow.insert([workflow_record])[0]["RID"]
-        except Exception as e:
-            error = format_exception(e)
-            raise DerivaMLException(f"Failed to insert workflow. Error: {error}")
-        return workflow_rid
-
-    def lookup_workflow(self, url_or_checksum: str) -> RID | None:
-        """Finds a workflow by URL.
-
-        Args:
-            url_or_checksum: URL or checksum of the workflow.
-        Returns:
-            RID: Resource Identifier of the workflow if found, None otherwise.
+            dict with keys:
+                - 'working_dir': Path to working directory
+                - 'cache_dir': Path to cache directory
+                - 'cache_size_mb': Cache size in MB
+                - 'cache_file_count': Number of files in cache
+                - 'execution_dir_count': Number of execution directories
+                - 'execution_size_mb': Total size of execution directories in MB
+                - 'total_size_mb': Combined size in MB
 
         Example:
-            >>> rid = ml.lookup_workflow("https://github.com/org/repo/workflow.py")
-            >>> if rid:
-            ...     print(f"Found workflow: {rid}")
-        """
-        # Get a workflow table path
-        workflow_path = self.pathBuilder.schemas[self.ml_schema].Workflow
-        try:
-            # Search for workflow by URL
-            url_column = workflow_path.URL
-            checksum_column = workflow_path.Checksum
-            return list(
-                workflow_path.path.filter(
-                    (url_column == url_or_checksum) | (checksum_column == url_or_checksum)
-                ).entities()
-            )[0]["RID"]
-        except IndexError:
-            return None
-
-    def create_workflow(self, name: str, workflow_type: str, description: str = "") -> Workflow:
-        """Creates a new workflow definition.
-
-        Creates a Workflow object that represents a computational process or analysis pipeline. The workflow type
-        must be a term from the controlled vocabulary. This method is typically used to define new analysis
-        workflows before execution.
-
-        Args:
-            name: Name of the workflow.
-            workflow_type: Type of workflow (must exist in workflow_type vocabulary).
-            description: Description of what the workflow does.
-
-        Returns:
-            Workflow: New workflow object ready for registration.
-
-        Raises:
-            DerivaMLException: If workflow_type is not in the vocabulary.
-
-        Examples:
-            >>> workflow = ml.create_workflow(
-            ...     name="RNA Analysis",
-            ...     workflow_type="python_notebook",
-            ...     description="RNA sequence analysis pipeline"
-            ... )
-            >>> rid = ml.add_workflow(workflow)
-        """
-        # Validate workflow type exists in vocabulary
-        self.lookup_term(MLVocab.workflow_type, workflow_type)
-
-        # Create and return a new workflow object
-        return Workflow(name=name, workflow_type=workflow_type, description=description)
-
-    def create_execution(
-        self, configuration: ExecutionConfiguration, workflow: Workflow | RID | None = None, dry_run: bool = False
-    ) -> "Execution":
-        """Creates an execution environment.
-
-        Given an execution configuration, initialize the local compute environment to prepare for executing an
-        ML or analytic routine.  This routine has a number of side effects.
-
-        1. The datasets specified in the configuration are downloaded and placed in the cache-dir. If a version is
-        not specified in the configuration, then a new minor version number is created for the dataset and downloaded.
-
-        2. If any execution assets are provided in the configuration, they are downloaded
-        and placed in the working directory.
-
-
-        Args:
-            configuration: ExecutionConfiguration:
-            workflow: Workflow object representing the workflow to execute if not present in the ExecutionConfiguration.
-            dry_run: Do not create an execution record or upload results.
-
-        Returns:
-            An execution object.
+            >>> ml = DerivaML('deriva.example.org', 'my_catalog')
+            >>> summary = ml.get_storage_summary()
+            >>> print(f"Total storage: {summary['total_size_mb']:.1f} MB")
+            >>> print(f"  Cache: {summary['cache_size_mb']:.1f} MB")
+            >>> print(f"  Executions: {summary['execution_size_mb']:.1f} MB")
         """
-        # Import here to avoid circular dependency
-        from deriva_ml.execution.execution import Execution
+        cache_stats = self.get_cache_size()
+        exec_dirs = self.list_execution_dirs()
+
+        exec_size_mb = sum(d['size_mb'] for d in exec_dirs)
+
+        return {
+            'working_dir': str(self.working_dir),
+            'cache_dir': str(self.cache_dir),
+            'cache_size_mb': cache_stats['total_mb'],
+            'cache_file_count': cache_stats['file_count'],
+            'execution_dir_count': len(exec_dirs),
+            'execution_size_mb': exec_size_mb,
+            'total_size_mb': cache_stats['total_mb'] + exec_size_mb,
+        }
 
-        # Create and store an execution instance
-        self._execution = Execution(configuration, self, workflow=workflow, dry_run=dry_run)
-        return self._execution
+    # =========================================================================
+    # Schema Validation
+    # =========================================================================
 
-    def restore_execution(self, execution_rid: RID | None = None) -> Execution:
-        """Restores a previous execution.
+    def validate_schema(self, strict: bool = False) -> "SchemaValidationReport":
+        """Validate that the catalog's ML schema matches the expected structure.
 
-        Given an execution RID, retrieves the execution configuration and restores the local compute environment.
-        This routine has a number of side effects.
+        This method inspects the catalog schema and verifies that it contains all
+        the required tables, columns, vocabulary terms, and relationships that are
+        created by the ML schema initialization routines in create_schema.py.
 
-        1. The datasets specified in the configuration are downloaded and placed in the cache-dir. If a version is
-        not specified in the configuration, then a new minor version number is created for the dataset and downloaded.
+        The validation checks:
+        - All required ML tables exist (Dataset, Execution, Workflow, etc.)
+        - All required columns exist with correct types
+        - All required vocabulary tables exist (Asset_Type, Dataset_Type, etc.)
+        - All required vocabulary terms are initialized
+        - All association tables exist for relationships
 
-        2. If any execution assets are provided in the configuration, they are downloaded and placed
-        in the working directory.
+        In strict mode, the validator also reports errors for:
+        - Extra tables not in the expected schema
+        - Extra columns not in the expected table definitions
 
         Args:
-            execution_rid: Resource Identifier (RID) of the execution to restore.
+            strict: If True, extra tables and columns are reported as errors.
+                   If False (default), they are reported as informational items.
+                   Use strict=True to verify a clean ML catalog matches exactly.
+                   Use strict=False to validate a catalog that may have domain extensions.
 
         Returns:
-            Execution: An execution object representing the restored execution environment.
-
-        Raises:
-            DerivaMLException: If execution_rid is not valid or execution cannot be restored.
+            SchemaValidationReport with validation results. Key attributes:
+                - is_valid: True if no errors were found
+                - errors: List of error-level issues
+                - warnings: List of warning-level issues
+                - info: List of informational items
+                - to_text(): Human-readable report
+                - to_dict(): JSON-serializable dictionary
 
         Example:
-            >>> execution = ml.restore_execution("1-abc123")
+            >>> ml = DerivaML('localhost', 'my_catalog')
+            >>> report = ml.validate_schema(strict=False)
+            >>> if report.is_valid:
+            ...     print("Schema is valid!")
+            ... else:
+            ...     print(report.to_text())
+
+            >>> # Strict validation for a fresh ML catalog
+            >>> report = ml.validate_schema(strict=True)
+            >>> print(f"Found {len(report.errors)} errors, {len(report.warnings)} warnings")
+
+            >>> # Get report as dictionary for JSON/logging
+            >>> import json
+            >>> print(json.dumps(report.to_dict(), indent=2))
+
+        Note:
+            This method validates the ML schema (typically 'deriva-ml'), not the
+            domain schema. Domain-specific tables and columns are not checked
+            unless they are part of the ML schema itself.
+
+        See Also:
+            - deriva_ml.schema.validation.SchemaValidationReport
+            - deriva_ml.schema.validation.validate_ml_schema
         """
-        # Import here to avoid circular dependency
-        from deriva_ml.execution.execution import Execution
-
-        # If no RID provided, try to find single execution in working directory
-        if not execution_rid:
-            e_rids = execution_rids(self.working_dir)
-            if len(e_rids) != 1:
-                raise DerivaMLException(f"Multiple execution RIDs were found {e_rids}.")
-            execution_rid = e_rids[0]
-
-        # Try to load configuration from a file
-        cfile = asset_file_path(
-            prefix=self.working_dir,
-            exec_rid=execution_rid,
-            file_name="configuration.json",
-            asset_table=self.model.name_to_table("Execution_Metadata"),
-            metadata={},
-        )
+        from deriva_ml.schema.validation import SchemaValidationReport, validate_ml_schema
+        return validate_ml_schema(self, strict=strict)
 
-        # Load configuration from a file or create from an execution record
-        if cfile.exists():
-            configuration = ExecutionConfiguration.load_configuration(cfile)
-        else:
-            execution = self.retrieve_rid(execution_rid)
-            configuration = ExecutionConfiguration(
-                workflow=execution["Workflow"],
-                description=execution["Description"],
-            )
+    # Methods moved to mixins:
+    # - create_asset, list_assets -> AssetMixin
+    # - create_feature, feature_record_class, delete_feature, lookup_feature, list_feature_values -> FeatureMixin
+    # - find_datasets, create_dataset, lookup_dataset, delete_dataset, list_dataset_element_types,
+    #   add_dataset_element_type, download_dataset_bag -> DatasetMixin
+    # - _update_status, create_execution, restore_execution -> ExecutionMixin
+    # - add_files, list_files, _bootstrap_versions, _synchronize_dataset_versions, _set_version_snapshot -> FileMixin
 
-        # Create and return an execution instance
-        return Execution(configuration, self, reload=execution_rid)