deriva-ml 1.17.10__py3-none-any.whl → 1.17.12__py3-none-any.whl

deriva_ml/core/exceptions.py

@@ -1,28 +1,311 @@
-"""
-Custom exceptions used throughout the DerivaML package.
+"""Custom exceptions for the DerivaML package.
+
+This module defines the exception hierarchy for DerivaML. All DerivaML-specific
+exceptions inherit from DerivaMLException, making it easy to catch all library
+errors with a single except clause.
+
+Exception Hierarchy:
+    DerivaMLException (base class for all DerivaML errors)
+    │
+    ├── DerivaMLConfigurationError (configuration and initialization)
+    │   ├── DerivaMLSchemaError (schema/catalog structure issues)
+    │   └── DerivaMLAuthenticationError (authentication failures)
+    │
+    ├── DerivaMLDataError (data access and validation)
+    │   ├── DerivaMLNotFoundError (entity not found)
+    │   │   ├── DerivaMLDatasetNotFound (dataset lookup failures)
+    │   │   ├── DerivaMLTableNotFound (table lookup failures)
+    │   │   └── DerivaMLInvalidTerm (vocabulary term not found)
+    │   ├── DerivaMLTableTypeError (wrong table type)
+    │   ├── DerivaMLValidationError (data validation failures)
+    │   └── DerivaMLCycleError (cycle detected in relationships)
+    │
+    ├── DerivaMLExecutionError (execution lifecycle)
+    │   ├── DerivaMLWorkflowError (workflow issues)
+    │   └── DerivaMLUploadError (asset upload failures)
+    │
+    └── DerivaMLReadOnlyError (write operation on read-only resource)
+
+Example:
+    >>> from deriva_ml.core.exceptions import DerivaMLException, DerivaMLNotFoundError
+    >>> try:
+    ...     dataset = ml.lookup_dataset("invalid_rid")
+    ... except DerivaMLDatasetNotFound as e:
+    ...     print(f"Dataset not found: {e}")
+    ... except DerivaMLNotFoundError as e:
+    ...     print(f"Entity not found: {e}")
+    ... except DerivaMLException as e:
+    ...     print(f"DerivaML error: {e}")
 """
 
 
 class DerivaMLException(Exception):
-    """Exception class specific to DerivaML module.
+    """Base exception class for all DerivaML errors.
+
+    This is the root exception for all DerivaML-specific errors. Catching this
+    exception will catch any error raised by the DerivaML library.
+
+    Attributes:
+        _msg: The error message stored for later access.
 
     Args:
-        msg (str): Optional message for the exception.
+        msg: Descriptive error message. Defaults to empty string.
+
+    Example:
+        >>> raise DerivaMLException("Failed to connect to catalog")
+        DerivaMLException: Failed to connect to catalog
     """
 
-    def __init__(self, msg=""):
+    def __init__(self, msg: str = "") -> None:
         super().__init__(msg)
         self._msg = msg
 
 
-class DerivaMLInvalidTerm(DerivaMLException):
-    """Exception class for invalid terms in DerivaML controlled vocabulary."""
-    def __init__(self, vocabulary, term: str, msg: str = "Term doesn't exist"):
-        """Exception indicating undefined term type"""
+# =============================================================================
+# Configuration and Initialization Errors
+# =============================================================================
+
+
+class DerivaMLConfigurationError(DerivaMLException):
+    """Exception raised for configuration and initialization errors.
+
+    Raised when there are issues with DerivaML configuration, catalog
+    initialization, or schema setup.
+
+    Example:
+        >>> raise DerivaMLConfigurationError("Invalid catalog configuration")
+    """
+
+    pass
+
+
+class DerivaMLSchemaError(DerivaMLConfigurationError):
+    """Exception raised for schema or catalog structure issues.
+
+    Raised when the catalog schema is invalid, missing required tables,
+    or has structural problems that prevent normal operation.
+
+    Example:
+        >>> raise DerivaMLSchemaError("Ambiguous domain schema: ['Schema1', 'Schema2']")
+    """
+
+    pass
+
+
+class DerivaMLAuthenticationError(DerivaMLConfigurationError):
+    """Exception raised for authentication failures.
+
+    Raised when authentication with the catalog fails or credentials are invalid.
+
+    Example:
+        >>> raise DerivaMLAuthenticationError("Failed to authenticate with catalog")
+    """
+
+    pass
+
+
+# =============================================================================
+# Data Access and Validation Errors
+# =============================================================================
+
+
+class DerivaMLDataError(DerivaMLException):
+    """Exception raised for data access and validation issues.
+
+    Base class for errors related to data lookup, validation, and integrity.
+
+    Example:
+        >>> raise DerivaMLDataError("Invalid data format")
+    """
+
+    pass
+
+
+class DerivaMLNotFoundError(DerivaMLDataError):
+    """Exception raised when an entity cannot be found.
+
+    Raised when a lookup operation fails to find the requested entity
+    (dataset, table, term, etc.) in the catalog or bag.
+
+    Example:
+        >>> raise DerivaMLNotFoundError("Entity '1-ABC' not found in catalog")
+    """
+
+    pass
+
+
+class DerivaMLDatasetNotFound(DerivaMLNotFoundError):
+    """Exception raised when a dataset cannot be found.
+
+    Raised when attempting to look up a dataset that doesn't exist in the
+    catalog or downloaded bag.
+
+    Args:
+        dataset_rid: The RID of the dataset that was not found.
+        msg: Additional context. Defaults to "Dataset not found".
+
+    Example:
+        >>> raise DerivaMLDatasetNotFound("1-ABC")
+        DerivaMLDatasetNotFound: Dataset 1-ABC not found
+    """
+
+    def __init__(self, dataset_rid: str, msg: str = "Dataset not found") -> None:
+        super().__init__(f"{msg}: {dataset_rid}")
+        self.dataset_rid = dataset_rid
+
+
+class DerivaMLTableNotFound(DerivaMLNotFoundError):
+    """Exception raised when a table cannot be found.
+
+    Raised when attempting to access a table that doesn't exist in the
+    catalog schema or downloaded bag.
+
+    Args:
+        table_name: The name of the table that was not found.
+        msg: Additional context. Defaults to "Table not found".
+
+    Example:
+        >>> raise DerivaMLTableNotFound("MyTable")
+        DerivaMLTableNotFound: Table not found: MyTable
+    """
+
+    def __init__(self, table_name: str, msg: str = "Table not found") -> None:
+        super().__init__(f"{msg}: {table_name}")
+        self.table_name = table_name
+
+
+class DerivaMLInvalidTerm(DerivaMLNotFoundError):
+    """Exception raised when a vocabulary term is not found or invalid.
+
+    Raised when attempting to look up or use a term that doesn't exist in
+    a controlled vocabulary table, or when a term name/synonym cannot be resolved.
+
+    Args:
+        vocabulary: Name of the vocabulary table being searched.
+        term: The term name that was not found.
+        msg: Additional context about the error. Defaults to "Term doesn't exist".
+
+    Example:
+        >>> raise DerivaMLInvalidTerm("Diagnosis", "unknown_condition")
+        DerivaMLInvalidTerm: Invalid term unknown_condition in vocabulary Diagnosis: Term doesn't exist.
+    """
+
+    def __init__(self, vocabulary: str, term: str, msg: str = "Term doesn't exist") -> None:
         super().__init__(f"Invalid term {term} in vocabulary {vocabulary}: {msg}.")
+        self.vocabulary = vocabulary
+        self.term = term
+
+
+class DerivaMLTableTypeError(DerivaMLDataError):
+    """Exception raised when a RID or table is not of the expected type.
+
+    Raised when an operation requires a specific table type (e.g., Dataset,
+    Execution) but receives a RID or table reference of a different type.
+
+    Args:
+        table_type: The expected table type (e.g., "Dataset", "Execution").
+        table: The actual table name or RID that was provided.
+
+    Example:
+        >>> raise DerivaMLTableTypeError("Dataset", "1-ABC123")
+        DerivaMLTableTypeError: Table 1-ABC123 is not of type Dataset.
+    """
+
+    def __init__(self, table_type: str, table: str) -> None:
+        super().__init__(f"Table {table} is not of type {table_type}.")
+        self.table_type = table_type
+        self.table = table
+
+
+class DerivaMLValidationError(DerivaMLDataError):
+    """Exception raised when data validation fails.
+
+    Raised when input data fails validation, such as invalid RID format,
+    mismatched metadata, or constraint violations.
+
+    Example:
+        >>> raise DerivaMLValidationError("Invalid RID format: ABC")
+    """
+
+    pass
+
+
+class DerivaMLCycleError(DerivaMLDataError):
+    """Exception raised when a cycle is detected in relationships.
+
+    Raised when creating dataset hierarchies or other relationships that
+    would result in a circular dependency.
+
+    Args:
+        cycle_nodes: List of nodes involved in the cycle.
+        msg: Additional context. Defaults to "Cycle detected".
+
+    Example:
+        >>> raise DerivaMLCycleError(["Dataset1", "Dataset2", "Dataset1"])
+    """
+
+    def __init__(self, cycle_nodes: list[str], msg: str = "Cycle detected") -> None:
+        super().__init__(f"{msg}: {cycle_nodes}")
+        self.cycle_nodes = cycle_nodes
+
+
+# =============================================================================
+# Execution Lifecycle Errors
+# =============================================================================
+
+
+class DerivaMLExecutionError(DerivaMLException):
+    """Exception raised for execution lifecycle issues.
+
+    Base class for errors related to workflow execution, asset management,
+    and provenance tracking.
+
+    Example:
+        >>> raise DerivaMLExecutionError("Execution failed to initialize")
+    """
+
+    pass
+
+
+class DerivaMLWorkflowError(DerivaMLExecutionError):
+    """Exception raised for workflow-related issues.
+
+    Raised when there are problems with workflow lookup, creation, or
+    Git integration for workflow tracking.
+
+    Example:
+        >>> raise DerivaMLWorkflowError("Not executing in a Git repository")
+    """
+
+    pass
+
+
+class DerivaMLUploadError(DerivaMLExecutionError):
+    """Exception raised for asset upload failures.
+
+    Raised when uploading assets to the catalog fails, including file
+    uploads, metadata insertion, and provenance recording.
+
+    Example:
+        >>> raise DerivaMLUploadError("Failed to upload execution assets")
+    """
+
+    pass
+
+
+# =============================================================================
+# Read-Only Resource Errors
+# =============================================================================
+
+
+class DerivaMLReadOnlyError(DerivaMLException):
+    """Exception raised when attempting write operations on read-only resources.
+
+    Raised when attempting to modify data in a downloaded bag or other
+    read-only context where write operations are not supported.
+
+    Example:
+        >>> raise DerivaMLReadOnlyError("Cannot create datasets in a downloaded bag")
+    """
 
-class DerivaMLTableTypeError(DerivaMLException):
-    """RID for table is not of correct type."""
-    def __init__(self, table_type, table: str):
-        """Exception indicating undefined term type"""
-        super().__init__(f"Table  {table} is not of type {table_type}.")
+    pass

deriva_ml/core/filespec.py

@@ -1,5 +1,26 @@
-"""
-File-related utility functions for DerivaML.
+"""File specification utilities for DerivaML.
+
+This module provides the FileSpec class for creating and managing file metadata
+in the Deriva catalog. FileSpec objects represent files with their checksums,
+sizes, and type classifications, ready for insertion into the File table.
+
+Key Features:
+    - Automatic MD5 checksum computation
+    - URL normalization (local paths converted to tag URIs)
+    - Support for file type classification
+    - Batch processing of directories
+    - JSONL serialization/deserialization
+
+Example:
+    Create FileSpec from a local file:
+        >>> specs = list(FileSpec.create_filespecs(
+        ...     path="/data/images/sample.png",
+        ...     description="Sample image",
+        ...     file_types=["Image", "PNG"]
+        ... ))
+
+    Read FileSpecs from a JSONL file:
+        >>> specs = list(FileSpec.read_filespec("files.jsonl"))
 """
 
 from __future__ import annotations
@@ -12,25 +33,44 @@ from typing import Callable, Generator
 from urllib.parse import urlparse
 
 import deriva.core.utils.hash_utils as hash_utils
-from pydantic import BaseModel, Field, conlist, field_validator, validate_call
+from pydantic import BaseModel, Field, field_validator, validate_call
 
 
 class FileSpec(BaseModel):
-    """An entry into the File table
+    """Specification for a file to be added to the Deriva catalog.
+
+    Represents file metadata required for creating entries in the File table.
+    Handles URL normalization, ensuring local file paths are converted to
+    tag URIs that uniquely identify the file's origin.
 
     Attributes:
-        url: The File url to the url.
-        description: The description of the file.
-        md5: The MD5 hash of the file.
-        length: The length of the file in bytes.
-        file_types: A list of file types.  Each files_type should be a defined term in MLVocab.file_type vocabulary.
+        url: File location as URL or local path. Local paths are converted to tag URIs.
+        md5: MD5 checksum for integrity verification.
+        length: File size in bytes.
+        description: Optional description of the file's contents or purpose.
+        file_types: List of file type classifications from the Asset_Type vocabulary.
+
+    Note:
+        The 'File' type is automatically added to file_types if not present when
+        using create_filespecs().
+
+    Example:
+        >>> spec = FileSpec(
+        ...     url="/data/results.csv",
+        ...     md5="d41d8cd98f00b204e9800998ecf8427e",
+        ...     length=1024,
+        ...     description="Analysis results",
+        ...     file_types=["CSV", "Data"]
+        ... )
     """
 
-    url: str = Field(alias="URL", validation_alias="url")
-    md5: str = Field(alias="MD5", validation_alias="md5")
-    length: int = Field(alias="Length", validation_alias="length")
-    description: str | None = Field(default="", alias="Description", validation_alias="description")
-    file_types: conlist(str) | None = []
+    model_config = {"populate_by_name": True}
+
+    url: str = Field(alias="URL")
+    md5: str = Field(alias="MD5")
+    length: int = Field(alias="Length")
+    description: str | None = Field(default="", alias="Description")
+    file_types: list[str] | None = Field(default_factory=list)
 
     @field_validator("url")
     @classmethod
@@ -61,22 +101,39 @@ class FileSpec(BaseModel):
     def create_filespecs(
         cls, path: Path | str, description: str, file_types: list[str] | Callable[[Path], list[str]] | None = None
     ) -> Generator[FileSpec, None, None]:
-        """Given a file or directory, generate the sequence of corresponding FileSpecs suitable to create a File table.
+        """Generate FileSpec objects for a file or directory.
+
+        Creates FileSpec objects with computed MD5 checksums for each file found.
+        For directories, recursively processes all files. The 'File' type is
+        automatically prepended to file_types if not already present.
 
         Args:
-            path: Path to the file or directory.
-            description: The description of the file(s)
-            file_types: A list of file types or a function that takes a file path and returns a list of file types.
+            path: Path to a file or directory. If directory, all files are processed recursively.
+            description: Description to apply to all generated FileSpecs.
+            file_types: Either a static list of file types, or a callable that takes a Path
+                and returns a list of types for that specific file. Allows dynamic type
+                assignment based on file extension, content, etc.
 
-        Returns:
-            An iterable of FileSpecs for each file in the directory.
-        """
+        Yields:
+            FileSpec: A specification for each file with computed checksums and metadata.
+
+        Example:
+            Static file types:
+                >>> specs = FileSpec.create_filespecs("/data/images", "Images", ["Image"])
 
+            Dynamic file types based on extension:
+                >>> def get_types(path):
+                ...     ext = path.suffix.lower()
+                ...     return {"png": ["PNG", "Image"], ".jpg": ["JPEG", "Image"]}.get(ext, [])
+                >>> specs = FileSpec.create_filespecs("/data", "Mixed files", get_types)
+        """
         path = Path(path)
         file_types = file_types or []
+        # Convert static list to callable for uniform handling
         file_types_fn = file_types if callable(file_types) else lambda _x: file_types
 
         def create_spec(file_path: Path) -> FileSpec:
+            """Create a FileSpec for a single file with computed hashes."""
             hashes = hash_utils.compute_file_hashes(file_path, hashes=frozenset(["md5", "sha256"]))
             md5 = hashes["md5"][0]
             type_list = file_types_fn(file_path)
@@ -85,21 +142,31 @@ class FileSpec(BaseModel):
                 md5=md5,
                 description=description,
                 url=file_path.as_posix(),
+                # Ensure 'File' type is always included
                 file_types=type_list if "File" in type_list else ["File"] + type_list,
             )
 
+        # Handle both single files and directories (recursive)
         files = [path] if path.is_file() else [f for f in Path(path).rglob("*") if f.is_file()]
         return (create_spec(file) for file in files)
 
     @staticmethod
     def read_filespec(path: Path | str) -> Generator[FileSpec, None, None]:
-        """Get FileSpecs from a JSON lines file.
+        """Read FileSpec objects from a JSON Lines file.
+
+        Parses a JSONL file where each line is a JSON object representing a FileSpec.
+        Empty lines are skipped. This is useful for batch processing pre-computed
+        file specifications.
 
         Args:
-         path: Path to the .jsonl file (string or Path).
+            path: Path to the .jsonl file containing FileSpec data.
 
         Yields:
-             A FileSpec object.
+            FileSpec: Parsed FileSpec object for each valid line.
+
+        Example:
+            >>> for spec in FileSpec.read_filespec("files.jsonl"):
+            ...     print(f"{spec.url}: {spec.md5}")
         """
         path = Path(path)
         with path.open("r", encoding="utf-8") as f:
@@ -110,7 +177,11 @@ class FileSpec(BaseModel):
                 yield FileSpec(**json.loads(line))
 
 
-# Hack round pydantic validate_call and forward reference.
-_raw = FileSpec.create_filespecs.__func__
-# wrap it with validate_call, then re‐make it a classmethod
-FileSpec.create_filespecs = classmethod(validate_call(_raw))
+# =============================================================================
+# Pydantic Workaround
+# =============================================================================
+# Workaround for Pydantic's validate_call decorator not working directly with
+# classmethods that have forward references. We extract the underlying function,
+# wrap it with validate_call, and re-create the classmethod.
+_raw = FileSpec.create_filespecs.__func__  # type: ignore[attr-defined]
+FileSpec.create_filespecs = classmethod(validate_call(_raw))  # type: ignore[arg-type]