deriva-ml 1.17.10__py3-none-any.whl

deriva_ml/core/exceptions.py

@@ -0,0 +1,28 @@
+"""
+Custom exceptions used throughout the DerivaML package.
+"""
+
+
+class DerivaMLException(Exception):
+    """Exception class specific to DerivaML module.
+
+    Args:
+        msg (str): Optional message for the exception.
+    """
+
+    def __init__(self, msg=""):
+        super().__init__(msg)
+        self._msg = msg
+
+
+class DerivaMLInvalidTerm(DerivaMLException):
+    """Exception class for invalid terms in DerivaML controlled vocabulary."""
+    def __init__(self, vocabulary, term: str, msg: str = "Term doesn't exist"):
+        """Exception indicating undefined term type"""
+        super().__init__(f"Invalid term {term} in vocabulary {vocabulary}: {msg}.")
+
+class DerivaMLTableTypeError(DerivaMLException):
+    """RID for table is not of correct type."""
+    def __init__(self, table_type, table: str):
+        """Exception indicating undefined term type"""
+        super().__init__(f"Table  {table} is not of type {table_type}.")

deriva_ml/core/filespec.py

@@ -0,0 +1,116 @@
+"""
+File-related utility functions for DerivaML.
+"""
+
+from __future__ import annotations
+
+import json
+from datetime import date
+from pathlib import Path
+from socket import gethostname
+from typing import Callable, Generator
+from urllib.parse import urlparse
+
+import deriva.core.utils.hash_utils as hash_utils
+from pydantic import BaseModel, Field, conlist, field_validator, validate_call
+
+
+class FileSpec(BaseModel):
+    """An entry into the File table
+
+    Attributes:
+        url: The File url to the url.
+        description: The description of the file.
+        md5: The MD5 hash of the file.
+        length: The length of the file in bytes.
+        file_types: A list of file types.  Each files_type should be a defined term in MLVocab.file_type vocabulary.
+    """
+
+    url: str = Field(alias="URL", validation_alias="url")
+    md5: str = Field(alias="MD5", validation_alias="md5")
+    length: int = Field(alias="Length", validation_alias="length")
+    description: str | None = Field(default="", alias="Description", validation_alias="description")
+    file_types: conlist(str) | None = []
+
+    @field_validator("url")
+    @classmethod
+    def validate_file_url(cls, url: str) -> str:
+        """Examine the provided URL. If it's a local path, convert it into a tag URL.
+
+        Args:
+            url: The URL to validate and potentially convert
+
+        Returns:
+            The validated/converted URL
+
+        Raises:
+            ValidationError: If the URL is not a file URL
+        """
+        url_parts = urlparse(url)
+        if url_parts.scheme == "tag":
+            # Already a tag URL, so just return it.
+            return url
+        elif (not url_parts.scheme) or url_parts.scheme == "file":
+            # There is no scheme part of the URL, or it is a file URL, so it is a local file path.
+            # Convert to a tag URL.
+            return f"tag://{gethostname()},{date.today()}:file://{url_parts.path}"
+        else:
+            raise ValueError("url is not a file URL")
+
+    @classmethod
+    def create_filespecs(
+        cls, path: Path | str, description: str, file_types: list[str] | Callable[[Path], list[str]] | None = None
+    ) -> Generator[FileSpec, None, None]:
+        """Given a file or directory, generate the sequence of corresponding FileSpecs suitable to create a File table.
+
+        Args:
+            path: Path to the file or directory.
+            description: The description of the file(s)
+            file_types: A list of file types or a function that takes a file path and returns a list of file types.
+
+        Returns:
+            An iterable of FileSpecs for each file in the directory.
+        """
+
+        path = Path(path)
+        file_types = file_types or []
+        file_types_fn = file_types if callable(file_types) else lambda _x: file_types
+
+        def create_spec(file_path: Path) -> FileSpec:
+            hashes = hash_utils.compute_file_hashes(file_path, hashes=frozenset(["md5", "sha256"]))
+            md5 = hashes["md5"][0]
+            type_list = file_types_fn(file_path)
+            return FileSpec(
+                length=path.stat().st_size,
+                md5=md5,
+                description=description,
+                url=file_path.as_posix(),
+                file_types=type_list if "File" in type_list else ["File"] + type_list,
+            )
+
+        files = [path] if path.is_file() else [f for f in Path(path).rglob("*") if f.is_file()]
+        return (create_spec(file) for file in files)
+
+    @staticmethod
+    def read_filespec(path: Path | str) -> Generator[FileSpec, None, None]:
+        """Get FileSpecs from a JSON lines file.
+
+        Args:
+         path: Path to the .jsonl file (string or Path).
+
+        Yields:
+             A FileSpec object.
+        """
+        path = Path(path)
+        with path.open("r", encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+                yield FileSpec(**json.loads(line))
+
+
+# Hack round pydantic validate_call and forward reference.
+_raw = FileSpec.create_filespecs.__func__
+# wrap it with validate_call, then re‐make it a classmethod
+FileSpec.create_filespecs = classmethod(validate_call(_raw))

deriva_ml/dataset/__init__.py

@@ -0,0 +1,12 @@
+from .aux_classes import DatasetSpec, DatasetSpecConfig, DatasetVersion, VersionPart
+from .dataset import Dataset
+from .dataset_bag import DatasetBag
+
+__all__ = [
+    "Dataset",
+    "DatasetSpec",
+    "DatasetSpecConfig",
+    "DatasetBag",
+    "DatasetVersion",
+    "VersionPart",
+]

deriva_ml/dataset/aux_classes.py

@@ -0,0 +1,225 @@
+"""
+THis module defines the DataSet class with is used to manipulate n
+"""
+
+from enum import Enum
+from typing import Any, Optional, SupportsInt
+
+from hydra_zen import hydrated_dataclass
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    Field,
+    computed_field,
+    conlist,
+    field_serializer,
+    field_validator,
+    model_validator,
+)
+from semver import Version
+
+from deriva_ml.core.definitions import RID
+
+
+class VersionPart(Enum):
+    """Simple enumeration for semantic versioning.
+
+    Attributes:
+        major (int): Major version number
+        minor (int): Minor version number
+        patch (int): Patch version number
+
+    """
+
+    major = "major"
+    minor = "minor"
+    patch = "patch"
+
+
+class DatasetVersion(Version):
+    """Represent the version associated with a dataset using semantic versioning.
+
+    Methods:
+        replace(major, minor, patch): Replace the major and minor versions
+    """
+
+    def __init__(self, major: SupportsInt, minor: SupportsInt = 0, patch: SupportsInt = 0):
+        """Initialize a DatasetVersion object.
+
+        Args:
+            major: Major version number. Used to indicate schema changes.
+            minor: Minor version number.  Used to indicate additional members added, or change in member values.
+            patch: Patch number of the dataset.  Used to indicate minor clean-up and edits
+        """
+        super().__init__(major, minor, patch)
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+
+        Returns:
+            dictionary of version information
+
+        """
+        return {"major": self.major, "minor": self.minor, "patch": self.patch}
+
+    def to_tuple(self) -> tuple[int, int, int]:
+        """
+
+        Returns:
+            tuple of version information
+
+        """
+        return self.major, self.minor, self.patch
+
+    @classmethod
+    def parse(cls, version: str, optional_minor_an_path=False) -> "DatasetVersion":
+        v = Version.parse(version)
+        return DatasetVersion(v.major, v.minor, v.patch)
+
+    def increment_version(self, component: VersionPart) -> "DatasetVersion":
+        match component:
+            case VersionPart.major:
+                return self.bump_major()
+            case VersionPart.minor:
+                return self.bump_minor()
+            case VersionPart.patch:
+                return self.bump_patch()
+            case _:
+                return self
+
+
+class DatasetHistory(BaseModel):
+    """
+    Class representing a dataset history.
+
+    Attributes:
+        dataset_version (DatasetVersion): A DatasetVersion object which captures the semantic versioning of the dataset.
+        dataset_rid (RID): The RID of the dataset.
+        version_rid (RID): The RID of the version record for the dataset in the Dataset_Version table.
+        minid (str): The URL that represents the handle of the dataset bag.  This will be None if a MINID has not
+                     been created yet.
+        snapshot (str): Catalog snapshot ID of when the version record was created.
+    """
+
+    dataset_version: DatasetVersion
+    dataset_rid: RID
+    version_rid: RID
+    execution_rid: Optional[RID] = None
+    description: str | None = ""
+    minid: str | None = None
+    snapshot: str | None = None
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    @field_validator("description", mode="after")
+    def _default_description(cls, v) -> str:
+        return v or ""
+
+
+class DatasetMinid(BaseModel):
+    """Represent information about a MINID that refers to a dataset
+
+    Attributes:
+        dataset_version (DatasetVersion): A DatasetVersion object which captures the semantic versioning of the dataset.
+        metadata (dict): A dictionary containing metadata from the MINID landing page.
+        minid (str): The URL that represents the handle of the MINID associated with the dataset.
+        bag_url (str): The URL to the dataset bag
+        identifier (str): The identifier of the MINID in CURI form
+        landing_page (str): The URL to the landing page of the MINID
+        version_rid (str): RID of the dataset version.
+        checksum (str): The checksum of the MINID in SHA256 form
+
+    """
+
+    dataset_version: DatasetVersion
+    metadata: dict[str, str | int] = {}
+    minid: str = Field(alias="compact_uri", default=None)
+    bag_url: str = Field(alias="location")
+    identifier: Optional[str] = None
+    landing_page: Optional[str] = None
+    version_rid: RID = Field(alias="RID")
+    checksum: str = Field(alias="checksums", default="")
+
+    @computed_field
+    @property
+    def dataset_rid(self) -> str:
+        rid_parts = self.version_rid.split("@")
+        return rid_parts[0]
+
+    @computed_field
+    @property
+    def dataset_snapshot(self) -> str:
+        return self.version_rid.split("@")[1]
+
+    @model_validator(mode="before")
+    @classmethod
+    def insert_metadata(cls, data: Any) -> Any:
+        if isinstance(data, dict):
+            if "metadata" in data:
+                data = data | data["metadata"]
+        return data
+
+    @field_validator("bag_url", mode="before")
+    @classmethod
+    def convert_location_to_str(cls, value: list[str] | str) -> str:
+        return value[0] if isinstance(value, list) else value
+
+    @field_validator("checksum", mode="before")
+    @classmethod
+    def convert_checksum_to_value(cls, checksums: list[dict]) -> str:
+        checksum_value = ""
+        for checksum in checksums:
+            if checksum.get("function") == "sha256":
+                checksum_value = checksum.get("value")
+                break
+        return checksum_value
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+
+class DatasetSpec(BaseModel):
+    """Represent a dataset_table in an execution configuration dataset_table list
+
+    Attributes:
+        rid (RID): A dataset_table RID
+        materialize (bool): If False do not materialize datasets, only download table data, no assets.  Defaults to True
+        version (DatasetVersion): The version of the dataset.  Should follow semantic versioning.
+    """
+
+    rid: RID
+    version: DatasetVersion | conlist(item_type=int, min_length=3, max_length=3) | tuple[int, int, int] | str
+    materialize: bool = True
+    description: str = ""
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    @field_validator("version", mode="before")
+    @classmethod
+    def version_field_validator(cls, v: Any) -> Any:
+        if isinstance(v, dict):
+            return DatasetVersion(**v)
+        elif isinstance(v, str):
+            return DatasetVersion.parse(v)
+        elif (isinstance(v, list) or isinstance(v, tuple)) and len(v) == 3:
+            return DatasetVersion(int(v[0]), int(v[1]), int(v[2]))
+        else:
+            return v
+
+    @model_validator(mode="before")
+    @classmethod
+    def _check_bare_rid(cls, data: Any) -> dict[str, str | bool]:
+        # If you are just given a string, assume it's a rid and put into dict for further validation.
+        return {"rid": data} if isinstance(data, str) else data
+
+    @field_serializer("version")
+    def serialize_version(self, version: DatasetVersion) -> dict[str, Any]:
+        return version.to_dict()
+
+
+# Interface for hydra-zen
+@hydrated_dataclass(DatasetSpec)
+class DatasetSpecConfig:
+    rid: str
+    version: str
+    materialize: bool = True
+    description: str = ""