deriva-ml 1.17.9__py3-none-any.whl → 1.17.11__py3-none-any.whl

deriva_ml/core/config.py

@@ -1,9 +1,62 @@
+"""Configuration management for DerivaML.
+
+This module provides the DerivaMLConfig class for managing DerivaML instance
+configuration. It integrates with hydra-zen for configuration management and supports
+both programmatic and structured configuration.
+
+The configuration handles:
+    - Server connection settings (hostname, catalog_id, credentials)
+    - Schema configuration (domain_schema, ml_schema)
+    - Directory paths (working_dir, cache_dir)
+    - Logging levels for both DerivaML and underlying Deriva libraries
+    - Feature toggles (use_minid, check_auth)
+
+Integration with hydra-zen:
+    The module registers a custom resolver for computing working directories
+    and configures Hydra's output directory structure for reproducible runs.
+    Use hydra-zen's `builds()` and `store` to create composable configurations.
+
+Example:
+    Programmatic configuration:
+        >>> config = DerivaMLConfig(
+        ...     hostname='deriva.example.org',
+        ...     catalog_id='my_catalog',
+        ...     working_dir='/path/to/work'
+        ... )
+        >>> ml = DerivaML.instantiate(config)
+
+    With hydra-zen:
+        >>> from hydra_zen import builds, instantiate, store, zen
+        >>> from deriva_ml import DerivaML
+        >>> from deriva_ml.core.config import DerivaMLConfig
+        >>>
+        >>> # Create a structured config for DerivaML
+        >>> DerivaMLConf = builds(DerivaMLConfig, populate_full_signature=True)
+        >>>
+        >>> # Store configurations for different environments
+        >>> store(DerivaMLConf(
+        ...     hostname='dev.example.org',
+        ...     catalog_id='1',
+        ... ), name='dev')
+        >>>
+        >>> store(DerivaMLConf(
+        ...     hostname='prod.example.org',
+        ...     catalog_id='52',
+        ... ), name='prod')
+        >>>
+        >>> # Use with Hydra's @hydra.main or zen() wrapper
+        >>> @zen(DerivaMLConf)
+        ... def my_task(cfg: DerivaMLConfig):
+        ...     ml = DerivaML.instantiate(cfg)
+        ...     # ... do work with ml instance
+"""
+
 import getpass
 import logging
 from pathlib import Path
 from typing import Any
 
-from hydra.conf import HydraConf, RunDir
+from hydra.conf import HydraConf, RunDir, SweepDir
 from hydra.core.hydra_config import HydraConfig
 from hydra_zen import store
 from omegaconf import OmegaConf
@@ -13,9 +66,52 @@ from deriva_ml.core.definitions import ML_SCHEMA
 
 
 class DerivaMLConfig(BaseModel):
+    """Configuration model for DerivaML instances.
+
+    This Pydantic model defines all configurable parameters for a DerivaML instance.
+    It can be used directly or via Hydra configuration files.
+
+    Attributes:
+        hostname: Hostname of the Deriva server (e.g., 'deriva.example.org').
+        catalog_id: Catalog identifier, either numeric ID or catalog name.
+        domain_schemas: Optional set of domain schema names. If None, auto-detects all
+            non-system schemas. Use this when working with catalogs that have multiple
+            user-defined schemas.
+        default_schema: The default schema for table creation operations. If None and
+            there is exactly one domain schema, that schema is used. If there are multiple
+            domain schemas, this must be specified for table creation to work without
+            explicit schema parameters.
+        project_name: Project name for organizing outputs. Defaults to default_schema.
+        cache_dir: Directory for caching downloaded datasets. Defaults to working_dir/cache.
+        working_dir: Base directory for computation data. Defaults to ~/deriva-ml.
+        hydra_runtime_output_dir: Hydra's runtime output directory (set automatically).
+        ml_schema: Schema name for ML tables. Defaults to 'deriva-ml'.
+        logging_level: Logging level for DerivaML. Defaults to WARNING.
+        deriva_logging_level: Logging level for Deriva libraries. Defaults to WARNING.
+        credential: Authentication credentials. If None, retrieved automatically.
+        s3_bucket: S3 bucket URL for dataset bag storage (e.g., 's3://my-bucket').
+            If provided, enables MINID creation and S3 upload for dataset exports.
+            If None, MINID functionality is disabled regardless of use_minid setting.
+        use_minid: Whether to use MINID service for dataset bags. Only effective when
+            s3_bucket is configured. Defaults to True when s3_bucket is set, False otherwise.
+        check_auth: Whether to verify authentication on connection. Defaults to True.
+        clean_execution_dir: Whether to automatically clean execution working directories
+            after successful upload. Defaults to True. Set to False to retain local copies
+            of execution outputs for debugging or manual inspection.
+
+    Example:
+        >>> config = DerivaMLConfig(
+        ...     hostname='deriva.example.org',
+        ...     catalog_id=1,
+        ...     default_schema='my_domain',
+        ...     logging_level=logging.INFO
+        ... )
+    """
+
     hostname: str
     catalog_id: str | int = 1
-    domain_schema: str | None = None
+    domain_schemas: set[str] | None = None
+    default_schema: str | None = None
     project_name: str | None = None
     cache_dir: str | Path | None = None
     working_dir: str | Path | None = None
@@ -24,46 +120,98 @@ class DerivaMLConfig(BaseModel):
     logging_level: Any = logging.WARNING
     deriva_logging_level: Any = logging.WARNING
     credential: Any = None
-    use_minid: bool = True
+    s3_bucket: str | None = None
+    use_minid: bool | None = None  # None means "auto" - True if s3_bucket is set
     check_auth: bool = True
+    clean_execution_dir: bool = True
 
     @model_validator(mode="after")
-    def init_working_dir(self):
-        """
-        Sets up the working directory for the model.
+    def init_working_dir(self) -> "DerivaMLConfig":
+        """Initialize working directory and resolve use_minid after model validation.
+
+        Sets up the working directory path, computing a default if not specified.
+        Also captures Hydra's runtime output directory for logging and outputs.
 
-        This method configures the working directory, ensuring that all required
-        file operations are performed in the appropriate location. If the user does not
-        specify a directory, a default directory based on the user's home directory
-        or username will be used.
+        Resolves the use_minid flag based on s3_bucket configuration:
+        - If use_minid is explicitly set, use that value (but it only takes effect if s3_bucket is set)
+        - If use_minid is None (auto), set it to True if s3_bucket is configured, False otherwise
 
-        This is a repeat of what is in the DerivaML.__init__ bu we put this here so that the working
-        directory is available to hydra.
+        This validator runs after all field validation and ensures the working
+        directory is available for Hydra configuration resolution.
 
         Returns:
-            Self: The object instance with the working directory initialized.
+            Self: The configuration instance with initialized paths.
         """
-
-        self.working_dir = DerivaMLConfig.compute_workdir(self.working_dir)
+        self.working_dir = DerivaMLConfig.compute_workdir(self.working_dir, self.catalog_id)
         self.hydra_runtime_output_dir = Path(HydraConfig.get().runtime.output_dir)
+
+        # Resolve use_minid based on s3_bucket configuration
+        if self.use_minid is None:
+            # Auto mode: enable MINID if s3_bucket is configured
+            self.use_minid = self.s3_bucket is not None
+        elif self.use_minid and self.s3_bucket is None:
+            # User requested MINID but no S3 bucket configured - disable MINID
+            self.use_minid = False
+
         return self
 
     @staticmethod
-    def compute_workdir(working_dir) -> Path:
-        # Create a default working directory if none is provided.  If a working directory is provided, we add the
-        # user name to it to ensure that multiple users do not overwrite each other's work.'
-        working_dir = (Path(working_dir) / getpass.getuser() if working_dir else Path.home()) / "deriva-ml"
-        return working_dir.absolute()
+    def compute_workdir(working_dir: str | Path | None, catalog_id: str | int | None = None) -> Path:
+        """Compute the effective working directory path.
 
+        Creates a standardized working directory path. If a base directory is provided,
+        appends the current username to prevent conflicts between users. If no directory
+        is provided, uses ~/.deriva-ml. The catalog_id is appended to
+        separate data from different catalogs.
+
+        Args:
+            working_dir: Base working directory path, or None for default.
+            catalog_id: Catalog identifier to include in the path. If None, no
+                       catalog subdirectory is created.
+
+        Returns:
+            Path: Absolute path to the working directory.
 
+        Example:
+            >>> DerivaMLConfig.compute_workdir('/shared/data', '52')
+            PosixPath('/shared/data/username/deriva-ml/52')
+            >>> DerivaMLConfig.compute_workdir(None, 1)
+            PosixPath('/home/username/.deriva-ml/1')
+        """
+        # Append username and deriva-ml to provided path, or use ~/.deriva-ml as base
+        if working_dir:
+            base_dir = Path(working_dir) / getpass.getuser() / "deriva-ml"
+        else:
+            base_dir = Path.home() / ".deriva-ml"
+        # Append catalog_id if provided
+        if catalog_id is not None:
+            base_dir = base_dir / str(catalog_id)
+        return base_dir.absolute()
+
+
+# =============================================================================
+# Hydra Integration
+# =============================================================================
+
+# Register custom resolver for computing working directories in Hydra configs
+# This allows ${compute_workdir:${working_dir},${catalog_id}} syntax in YAML configuration files
 OmegaConf.register_new_resolver("compute_workdir", DerivaMLConfig.compute_workdir, replace=True)
+
+# Configure Hydra's output directory structure for reproducible runs
+# Outputs are organized by timestamp under the computed working directory
+# For multirun/sweep, outputs go to a sweep subdirectory with job number subfolders
 store(
     HydraConf(
-        run=RunDir("${compute_workdir:${deriva_ml.working_dir}}/hydra/${now:%Y-%m-%d_%H-%M-%S}"),
+        run=RunDir("${compute_workdir:${deriva_ml.working_dir},${deriva_ml.catalog_id}}/hydra/${now:%Y-%m-%d_%H-%M-%S}"),
+        sweep=SweepDir(
+            dir="${compute_workdir:${deriva_ml.working_dir},${deriva_ml.catalog_id}}/hydra-sweep/${now:%Y-%m-%d_%H-%M-%S}",
+            subdir="${hydra.job.num}",
+        ),
         output_subdir="hydra-config",
     ),
     group="hydra",
     name="config",
 )
 
+# Add the configuration to Hydra's store for discovery
 store.add_to_hydra_store()

deriva_ml/core/constants.py

@@ -1,36 +1,137 @@
-"""
-Constants used throughout the DerivaML package.
+"""Constants used throughout the DerivaML package.
+
+This module defines fundamental constants, type aliases, and regular expressions
+used for validating and working with Deriva catalog structures.
+
+Constants:
+    ML_SCHEMA: Default schema name for ML-related tables ('deriva-ml').
+    DRY_RUN_RID: Special RID used for dry-run operations without database changes.
+
+Type Aliases:
+    RID: Annotated string type for Resource Identifiers with validation.
+
+Regular Expressions:
+    rid_part: Pattern for matching the RID portion of an identifier.
+    snapshot_part: Pattern for matching optional snapshot timestamps.
+    rid_regex: Complete pattern for validating RID strings.
+
+Column Sets:
+    DerivaSystemColumns: Standard Deriva system columns present in all tables.
+    DerivaAssetColumns: Columns specific to asset tables (files, etc.).
+
+Example:
+    >>> from deriva_ml.core.constants import RID, ML_SCHEMA
+    >>> def process_entity(rid: RID) -> None:
+    ...     # RID is validated by Pydantic
+    ...     pass
 """
 
 from __future__ import annotations
 
-from typing import NewType, TypeAlias
+from typing import Annotated
 
-from pydantic import constr
+from pydantic import StringConstraints
 
-# Schema name
+# =============================================================================
+# Schema Constants
+# =============================================================================
+
+# Default schema name for ML-related tables in the catalog
 ML_SCHEMA = "deriva-ml"
 
-# Special RID for dry runs
+# Special RID value used for dry-run operations that don't modify the database
 DRY_RUN_RID = "0000"
 
-# Regular expression parts for RIDs
+# System schemas that are part of Deriva infrastructure (not user domain schemas)
+# These are excluded when auto-detecting domain schemas
+SYSTEM_SCHEMAS: frozenset[str] = frozenset({"public", "www", "WWW"})
+
+
+def is_system_schema(schema_name: str, ml_schema: str = ML_SCHEMA) -> bool:
+    """Check if a schema is a system or ML schema (not a domain schema).
+
+    System schemas are Deriva infrastructure schemas (public, www, WWW) and the
+    ML schema (deriva-ml by default). Domain schemas are user-defined schemas
+    containing business logic tables.
+
+    Args:
+        schema_name: Name of the schema to check.
+        ml_schema: Name of the ML schema (default: 'deriva-ml').
+
+    Returns:
+        True if the schema is a system or ML schema, False if it's a domain schema.
+
+    Example:
+        >>> is_system_schema("public")
+        True
+        >>> is_system_schema("deriva-ml")
+        True
+        >>> is_system_schema("my_project")
+        False
+    """
+    return schema_name.lower() in {s.lower() for s in SYSTEM_SCHEMAS} or schema_name == ml_schema
+
+
+def get_domain_schemas(all_schemas: set[str] | list[str], ml_schema: str = ML_SCHEMA) -> frozenset[str]:
+    """Return all domain schemas from a collection of schema names.
+
+    Filters out system schemas (public, www, WWW) and the ML schema to return
+    only user-defined domain schemas.
+
+    Args:
+        all_schemas: Collection of schema names to filter.
+        ml_schema: Name of the ML schema to exclude (default: 'deriva-ml').
+
+    Returns:
+        Frozen set of domain schema names.
+
+    Example:
+        >>> get_domain_schemas(["public", "deriva-ml", "my_project", "www"])
+        frozenset({'my_project'})
+    """
+    return frozenset(s for s in all_schemas if not is_system_schema(s, ml_schema))
+
+# =============================================================================
+# RID Regular Expression Components
+# =============================================================================
+
+# Pattern for the RID portion: 1-4 alphanumeric chars, optionally followed by
+# hyphen-separated groups of exactly 4 alphanumeric chars (e.g., "1ABC" or "1ABC-DEF2-3GHI")
 rid_part = r"(?P<rid>(?:[A-Z\d]{1,4}|[A-Z\d]{1,4}(?:-[A-Z\d]{4})+))"
+
+# Pattern for optional snapshot timestamp suffix (e.g., "@2024-01-01T12:00:00")
+# Uses the same format as RID for the snapshot identifier
 snapshot_part = r"(?:@(?P<snapshot>(?:[A-Z\d]{1,4}|[A-Z\d]{1,4}(?:-[A-Z\d]{4})+)))?"
+
+# Complete regex for validating RID strings with optional snapshot
 rid_regex = f"^{rid_part}{snapshot_part}$"
 
-# RID type definition
-BaseRIDString = constr(pattern=rid_regex)
-# RID = TypeVar("RID", bound=BaseRIDString)
-RIDType: TypeAlias = constr(pattern=rid_regex)
-RID = NewType("RID", BaseRIDString)
+# =============================================================================
+# Type Aliases
+# =============================================================================
 
-# System columns in Deriva
+# RID type with Pydantic validation - ensures strings match the RID format
+# Used throughout the codebase for type hints and runtime validation
+RID = Annotated[str, StringConstraints(pattern=rid_regex)]
+
+# =============================================================================
+# Column Definitions
+# =============================================================================
+
+# Standard Deriva system columns present in every table:
+# - RID: Resource Identifier (unique key)
+# - RCT: Record Creation Time
+# - RMT: Record Modification Time
+# - RCB: Record Created By (user ID)
+# - RMB: Record Modified By (user ID)
 DerivaSystemColumns = ["RID", "RCT", "RMT", "RCB", "RMB"]
+
+# Columns specific to asset tables (files, images, etc.)
+# Includes system columns plus asset-specific metadata
 DerivaAssetColumns = {
-    "Filename",
-    "URL",
-    "Length",
-    "MD5",
-    "Description",
-}.union(set(DerivaSystemColumns))
+    "Filename",    # Original filename
+    "URL",         # Hatrac storage URL
+    "Length",      # File size in bytes
+    "MD5",         # MD5 checksum for integrity verification
+    "Description", # Optional description of the asset
+}.union(set(DerivaSystemColumns))

deriva_ml/core/definitions.py

@@ -1,23 +1,52 @@
-"""
-Shared definitions that are used in different DerivaML modules.
-This module re-exports all symbols from the core submodules for backwards compatibility.
+"""Shared definitions for DerivaML modules.
+
+This module serves as the central location for type definitions, constants, enums,
+and data models used throughout DerivaML. It re-exports symbols from specialized
+submodules for convenience and backwards compatibility.
+
+The module consolidates:
+    - Constants: Schema names, RID patterns, column definitions
+    - Enums: Status codes, upload states, built-in types, vocabulary identifiers
+    - Models: Dataclass-based models for ERMrest structures (tables, columns, keys)
+    - Utilities: FileSpec for file metadata handling
+
+Core definition classes (ColumnDef, KeyDef, ForeignKeyDef, TableDef) are provided by
+`deriva.core.typed` and re-exported here. Legacy aliases (ColumnDefinition, etc.)
+are maintained for backwards compatibility.
+
+This is the recommended import location for most DerivaML type definitions:
+    >>> from deriva_ml.core.definitions import RID, MLVocab, TableDef
+
+For more specialized imports, you can import directly from submodules:
+    >>> from deriva_ml.core.constants import ML_SCHEMA
+    >>> from deriva_ml.core.enums import Status
+    >>> from deriva.core.typed import ColumnDef
 """
 
 from __future__ import annotations
 
-# Re-export constants
+# =============================================================================
+# Re-exported Constants
+# =============================================================================
+# From constants.py: Schema names, RID patterns, and column definitions
 from deriva_ml.core.constants import (
     DRY_RUN_RID,
     ML_SCHEMA,
     RID,
+    SYSTEM_SCHEMAS,
     DerivaAssetColumns,
     DerivaSystemColumns,
+    get_domain_schemas,
+    is_system_schema,
     rid_part,
     rid_regex,
     snapshot_part,
 )
 
-# Re-export enums
+# =============================================================================
+# Re-exported Enums
+# =============================================================================
+# From enums.py: Status codes, type identifiers, and vocabulary names
 from deriva_ml.core.enums import (
     BaseStrEnum,
     BuiltinTypes,
@@ -29,46 +58,127 @@ from deriva_ml.core.enums import (
     Status,
     UploadState,
 )
+# Also export BuiltinType directly (BuiltinTypes is the backwards-compatible alias)
+from deriva.core.typed import BuiltinType
 
-# Re-export models
+# =============================================================================
+# Re-exported ERMrest Models
+# =============================================================================
+# From ermrest.py: Dataclass-based models for catalog structure definitions
+# New typed classes from deriva.core.typed
 from deriva_ml.core.ermrest import (
+    # New dataclass-based definitions from deriva.core.typed
+    ColumnDef,
+    KeyDef,
+    ForeignKeyDef,
+    TableDef,
+    VocabularyTableDef,
+    AssetTableDef,
+    AssociationTableDef,
+    SchemaDef,
+    # Legacy aliases for backwards compatibility
     ColumnDefinition,
-    FileUploadState,
-    ForeignKeyDefinition,
     KeyDefinition,
+    ForeignKeyDefinition,
     TableDefinition,
+    # DerivaML-specific classes
+    FileUploadState,
+    UploadCallback,
+    UploadProgress,
     VocabularyTerm,
+    VocabularyTermHandle,
+)
+
+# =============================================================================
+# Re-exported Exceptions
+# =============================================================================
+# From exceptions.py: Exception hierarchy for DerivaML errors
+from deriva_ml.core.exceptions import (
+    DerivaMLAuthenticationError,
+    DerivaMLConfigurationError,
+    DerivaMLCycleError,
+    DerivaMLDataError,
+    DerivaMLDatasetNotFound,
+    DerivaMLException,
+    DerivaMLExecutionError,
+    DerivaMLInvalidTerm,
+    DerivaMLNotFoundError,
+    DerivaMLReadOnlyError,
+    DerivaMLSchemaError,
+    DerivaMLTableNotFound,
+    DerivaMLTableTypeError,
+    DerivaMLUploadError,
+    DerivaMLValidationError,
+    DerivaMLWorkflowError,
 )
 
-# Re-export exceptions
+# =============================================================================
+# Re-exported Utilities
+# =============================================================================
+# From filespec.py: File metadata and specification handling
 from deriva_ml.core.filespec import FileSpec
 
 __all__ = [
     # Constants
     "ML_SCHEMA",
     "DRY_RUN_RID",
+    "SYSTEM_SCHEMAS",
     "rid_part",
     "snapshot_part",
     "rid_regex",
     "DerivaSystemColumns",
     "DerivaAssetColumns",
     "RID",
+    # Schema classification helpers
+    "is_system_schema",
+    "get_domain_schemas",
     # Enums
     "BaseStrEnum",
     "UploadState",
     "Status",
+    "BuiltinType",
     "BuiltinTypes",
     "MLVocab",
     "MLTable",
     "MLAsset",
     "ExecMetadataType",
     "ExecAssetType",
-    # Models
-    "FileUploadState",
-    "FileSpec",
-    "VocabularyTerm",
+    # Typed definitions from deriva.core.typed
+    "ColumnDef",
+    "KeyDef",
+    "ForeignKeyDef",
+    "TableDef",
+    "VocabularyTableDef",
+    "AssetTableDef",
+    "AssociationTableDef",
+    "SchemaDef",
+    # Legacy aliases for backwards compatibility
     "ColumnDefinition",
     "KeyDefinition",
     "ForeignKeyDefinition",
     "TableDefinition",
+    # DerivaML-specific models
+    "FileUploadState",
+    "FileSpec",
+    "VocabularyTerm",
+    "VocabularyTermHandle",
+    "UploadProgress",
+    "UploadCallback",
+    # Exceptions
+    "DerivaMLException",
+    "DerivaMLConfigurationError",
+    "DerivaMLSchemaError",
+    "DerivaMLAuthenticationError",
+    "DerivaMLDataError",
+    "DerivaMLNotFoundError",
+    "DerivaMLDatasetNotFound",
+    "DerivaMLTableNotFound",
+    "DerivaMLInvalidTerm",
+    "DerivaMLTableTypeError",
+    "DerivaMLValidationError",
+    "DerivaMLCycleError",
+    "DerivaMLExecutionError",
+    "DerivaMLWorkflowError",
+    "DerivaMLUploadError",
+    "DerivaMLReadOnlyError",
 ]