deriva-ml 1.14.47__tar.gz → 1.17.0__tar.gz

{deriva_ml-1.14.47 → deriva_ml-1.17.0}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.4
 Name: deriva-ml
-Version: 1.14.47
+Version: 1.17.0
 Summary: Utilities to simplify use of Dervia and Pandas to create reproducable ML pipelines
 Author-email: ISRD <isrd-dev@isi.edu>
-Requires-Python: >=3.10
+Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: bump-my-version
@@ -12,15 +12,18 @@ Requires-Dist: deriva~=1.7.10
 Requires-Dist: deepdiff
 Requires-Dist: nbconvert
 Requires-Dist: pandas
-Requires-Dist: regex~=2024.7.24
+Requires-Dist: pip-system-certs
 Requires-Dist: pydantic>=2.11
-Requires-Dist: semver>3.0.0
-Requires-Dist: setuptools>=64
-Requires-Dist: setuptools-scm>=8.0
-Requires-Dist: nbstripout
 Requires-Dist: papermill
 Requires-Dist: pandas-stubs==2.2.3.250527
 Requires-Dist: pyyaml
+Requires-Dist: regex~=2024.7.24
+Requires-Dist: semver>3.0.0
+Requires-Dist: setuptools>=80
+Requires-Dist: setuptools-scm>=8.0
+Requires-Dist: nbstripout
+Requires-Dist: hydra_zen
+Requires-Dist: SQLAlchemy
 Dynamic: license-file
 
 # DerivaML

{deriva_ml-1.14.47 → deriva_ml-1.17.0}/pyproject.toml

@@ -6,7 +6,7 @@ authors = [
 ]
 description = "Utilities to simplify use of Dervia and Pandas to create reproducable ML pipelines"
 readme = "README.md"
-requires-python = ">=3.10"
+requires-python = ">=3.11"
 dependencies = [
     "bump-my-version",
     "bdbag",
@@ -14,15 +14,18 @@ dependencies = [
     "deepdiff",
     "nbconvert",
     "pandas",
-    "regex~=2024.7.24",
+    "pip-system-certs",
     "pydantic>=2.11",
+    "papermill",
+    "pandas-stubs==2.2.3.250527",
+    "pyyaml",
+    "regex~=2024.7.24",
     "semver>3.0.0",
-    "setuptools>=64",
+    "setuptools>=80",
     "setuptools-scm>=8.0",
     "nbstripout",
-    "papermill",
-    "pandas-stubs==2.2.3.250527",
-    "pyyaml"
+    "hydra_zen",
+    "SQLAlchemy"
 ]
 
 [project.scripts]
@@ -38,10 +41,11 @@ deriva-ml-check-catalog-schema = "deriva_ml.schema.check_schema:main"
 [project.optional-dependencies]
 
 [build-system]
-requires = ["setuptools>=45", "setuptools_scm[toml]>=6.2", "wheel"]
+requires = ["setuptools>=80", "setuptools_scm[toml]>=8", "wheel"]
 build-backend = "setuptools.build_meta"
 
 [tool.uv]
+python-preference = "only-managed"
 
 [tool.uv.sources]
 #bdbag = {git = "https://github.com/fair-research/bdbag", branch = "master" }
@@ -113,6 +117,7 @@ dev = [
     "pytest>=8.4.1",
     "pytest-mock",
     "pytest-coverage>=0.0",
+    "pip-system-certs",
     "ruff"
 ]
 lint = [

deriva_ml-1.17.0/src/deriva_ml/__init__.py

@@ -0,0 +1,77 @@
+from importlib.metadata import PackageNotFoundError, version
+from typing import TYPE_CHECKING
+
+# Safe imports - no circular dependencies
+from deriva_ml.core.config import DerivaMLConfig
+from deriva_ml.core.definitions import (
+    RID,
+    BuiltinTypes,
+    ColumnDefinition,
+    DerivaAssetColumns,
+    DerivaSystemColumns,
+    ExecAssetType,
+    ExecMetadataType,
+    FileSpec,
+    FileUploadState,
+    ForeignKeyDefinition,
+    KeyDefinition,
+    MLAsset,
+    MLVocab,
+    TableDefinition,
+    UploadState,
+)
+from deriva_ml.core.exceptions import (
+    DerivaMLException,
+    DerivaMLInvalidTerm,
+    DerivaMLTableTypeError,
+)
+
+# Type-checking only - avoid circular import at runtime
+if TYPE_CHECKING:
+    from deriva_ml.core.base import DerivaML
+
+
+# Lazy import function for runtime usage
+def __getattr__(name):
+    """Lazy import to avoid circular dependencies."""
+    if name == "DerivaML":
+        from deriva_ml.core.base import DerivaML
+
+        return DerivaML
+    elif name == "Execution":
+        from deriva_ml.execution.execution import Execution
+
+        return Execution
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+
+__all__ = [
+    "DerivaML",  # Lazy-loaded
+    "DerivaMLConfig",
+    # Exceptions
+    "DerivaMLException",
+    "DerivaMLInvalidTerm",
+    "DerivaMLTableTypeError",
+    # Definitions
+    "RID",
+    "BuiltinTypes",
+    "ColumnDefinition",
+    "DerivaSystemColumns",
+    "DerivaAssetColumns",
+    "ExecAssetType",
+    "ExecMetadataType",
+    "FileSpec",
+    "FileUploadState",
+    "ForeignKeyDefinition",
+    "KeyDefinition",
+    "MLAsset",
+    "MLVocab",
+    "TableDefinition",
+    "UploadState",
+]
+
+try:
+    __version__ = version("deriva_ml")
+except PackageNotFoundError:
+    # package is not installed
+    pass

{deriva_ml-1.14.47 → deriva_ml-1.17.0}/src/deriva_ml/core/__init__.py

@@ -1,4 +1,5 @@
 from deriva_ml.core.base import DerivaML
+from deriva_ml.core.config import DerivaMLConfig
 from deriva_ml.core.definitions import (
     RID,
     BuiltinTypes,
@@ -17,12 +18,11 @@ from deriva_ml.core.exceptions import DerivaMLException, DerivaMLInvalidTerm, De
 
 __all__ = [
     "DerivaML",
-
+    "DerivaMLConfig",
     # Exceptions
     "DerivaMLException",
     "DerivaMLInvalidTerm",
     "DerivaMLTableTypeError",
-
     # Definitions
     "RID",
     "BuiltinTypes",

{deriva_ml-1.14.47 → deriva_ml-1.17.0}/src/deriva_ml/core/base.py

@@ -15,12 +15,11 @@ from __future__ import annotations  # noqa: I001
 
 # Standard library imports
 from collections import defaultdict
-import getpass
 import logging
 from datetime import datetime
 from itertools import chain
 from pathlib import Path
-from typing import Dict, Iterable, List, cast, TYPE_CHECKING, Any
+from typing import Dict, Iterable, List, cast, TYPE_CHECKING, Any, Self
 from urllib.parse import urlsplit
 
 
@@ -29,18 +28,14 @@ import requests
 from pydantic import ConfigDict, validate_call
 
 # Deriva imports
-from deriva.core import (
-    DEFAULT_SESSION_CONFIG,
-    format_exception,
-    get_credential,
-    urlquote,
-)
+from deriva.core import DEFAULT_SESSION_CONFIG, format_exception, get_credential, urlquote
 
 import deriva.core.datapath as datapath
 from deriva.core.datapath import DataPathException, _SchemaWrapper as SchemaWrapper
 from deriva.core.deriva_server import DerivaServer
 from deriva.core.ermrest_catalog import ResolveRidResult
 from deriva.core.ermrest_model import Key, Table
+from deriva.core.utils.core_utils import DEFAULT_LOGGER_OVERRIDES
 from deriva.core.utils.globus_auth_utils import GlobusNativeLogin
 
 from deriva_ml.core.exceptions import DerivaMLInvalidTerm
@@ -55,6 +50,7 @@ from deriva_ml.core.definitions import (
     TableDefinition,
     VocabularyTerm,
 )
+from deriva_ml.core.config import DerivaMLConfig
 from deriva_ml.core.exceptions import DerivaMLTableTypeError, DerivaMLException
 from deriva_ml.dataset.aux_classes import DatasetSpec
 from deriva_ml.dataset.dataset import Dataset
@@ -108,6 +104,10 @@ class DerivaML(Dataset):
         >>> ml.add_term('vocabulary_table', 'new_term', description='Description of term')
     """
 
+    @classmethod
+    def instantiate(cls, config: DerivaMLConfig) -> Self:
+        return cls(**config.model_dump())
+
     def __init__(
         self,
         hostname: str,
@@ -116,8 +116,10 @@ class DerivaML(Dataset):
         project_name: str | None = None,
         cache_dir: str | Path | None = None,
         working_dir: str | Path | None = None,
+        hydra_runtime_output_dir: str | Path | None = None,
         ml_schema: str = ML_SCHEMA,
         logging_level=logging.WARNING,
+        deriva_logging_level=logging.WARNING,
         credential=None,
         use_minid: bool = True,
         check_auth: bool = True,
@@ -152,7 +154,6 @@ class DerivaML(Dataset):
             credentials=self.credential,
             session_config=self._get_session_config(),
         )
-
         try:
             if check_auth and server.get_authn_session():
                 pass
@@ -161,17 +162,14 @@ class DerivaML(Dataset):
                 "You are not authorized to access this catalog. "
                 "Please check your credentials and make sure you have logged in."
             )
-
         self.catalog = server.connect_ermrest(catalog_id)
         self.model = DerivaModel(self.catalog.getCatalogModel(), domain_schema=domain_schema)
 
         # Set up working and cache directories
-        default_workdir = self.__class__.__name__ + "_working"
-        self.working_dir = (
-            Path(working_dir) / getpass.getuser() if working_dir else Path.home() / "deriva-ml"
-        ) / default_workdir
-
+        self.working_dir = DerivaMLConfig.compute_workdir(working_dir)
         self.working_dir.mkdir(parents=True, exist_ok=True)
+        self.hydra_runtime_output_dir = hydra_runtime_output_dir
+
         self.cache_dir = Path(cache_dir) if cache_dir else self.working_dir / "cache"
         self.cache_dir.mkdir(parents=True, exist_ok=True)
 
@@ -181,6 +179,15 @@ class DerivaML(Dataset):
         # Set up logging
         self._logger = logging.getLogger("deriva_ml")
         self._logger.setLevel(logging_level)
+        self._logging_level = logging_level
+        self._deriva_logging_level = deriva_logging_level
+
+        # Configure deriva logging level
+        logger_config = DEFAULT_LOGGER_OVERRIDES
+        # allow for reconfiguration of module-specific logging levels
+        [logging.getLogger(name).setLevel(level) for name, level in logger_config.items()]
+        logging.getLogger("bagit").setLevel(deriva_logging_level)
+        logging.getLogger("bdbag").setLevel(deriva_logging_level)
 
         # Store instance configuration
         self.host_name = hostname
@@ -1081,7 +1088,12 @@ class DerivaML(Dataset):
         return self._download_dataset_bag(
             dataset=dataset,
             execution_rid=execution_rid,
-            snapshot_catalog=DerivaML(self.host_name, self._version_snapshot(dataset)),
+            snapshot_catalog=DerivaML(
+                self.host_name,
+                self._version_snapshot(dataset),
+                logging_level=self._logging_level,
+                deriva_logging_level=self._deriva_logging_level,
+            ),
         )
 
     def _update_status(self, new_status: Status, status_detail: str, execution_rid: RID):

deriva_ml-1.17.0/src/deriva_ml/core/config.py

@@ -0,0 +1,67 @@
+import logging
+from pathlib import Path
+from typing import Any
+
+from hydra.conf import HydraConf, RunDir
+from hydra.core.hydra_config import HydraConfig
+from hydra_zen import store
+from omegaconf import OmegaConf
+from pydantic import BaseModel, model_validator
+
+from deriva_ml.core.definitions import ML_SCHEMA
+
+
+class DerivaMLConfig(BaseModel):
+    hostname: str
+    catalog_id: str | int = 1
+    domain_schema: str | None = None
+    project_name: str | None = None
+    cache_dir: str | Path | None = None
+    working_dir: str | Path | None = None
+    hydra_runtime_output_dir: str | Path | None = None
+    ml_schema: str = ML_SCHEMA
+    logging_level: Any = logging.WARNING
+    deriva_logging_level: Any = logging.WARNING
+    credential: Any = None
+    use_minid: bool = True
+    check_auth: bool = True
+
+    @model_validator(mode="after")
+    def init_working_dir(self):
+        """
+        Sets up the working directory for the model.
+
+        This method configures the working directory, ensuring that all required
+        file operations are performed in the appropriate location. If the user does not
+        specify a directory, a default directory based on the user's home directory
+        or username will be used.
+
+        This is a repeat of what is in the DerivaML.__init__ bu we put this here so that the working
+        directory is available to hydra.
+
+        Returns:
+            Self: The object instance with the working directory initialized.
+        """
+
+        self.working_dir = DerivaMLConfig.compute_workdir(self.working_dir)
+        self.hydra_runtime_output_dir = Path(HydraConfig.get().runtime.output_dir)
+        return self
+
+    @staticmethod
+    def compute_workdir(working_dir) -> Path:
+        # Create a default working directory if none is provided
+        working_dir = Path(working_dir) if working_dir else Path.home() / "deriva-ml"
+        return working_dir.absolute()
+
+
+OmegaConf.register_new_resolver("compute_workdir", DerivaMLConfig.compute_workdir, replace=True)
+store(
+    HydraConf(
+        run=RunDir("${compute_workdir:${deriva_ml.working_dir}}/hydra/${now:%Y-%m-%d_%H-%M-%S}"),
+        output_subdir="hydra-config",
+    ),
+    group="hydra",
+    name="config",
+)
+
+store.add_to_hydra_store()

deriva_ml-1.17.0/src/deriva_ml/dataset/__init__.py

@@ -0,0 +1,12 @@
+from .aux_classes import DatasetSpec, DatasetSpecConfig, DatasetVersion, VersionPart
+from .dataset import Dataset
+from .dataset_bag import DatasetBag
+
+__all__ = [
+    "Dataset",
+    "DatasetSpec",
+    "DatasetSpecConfig",
+    "DatasetBag",
+    "DatasetVersion",
+    "VersionPart",
+]

{deriva_ml-1.14.47 → deriva_ml-1.17.0}/src/deriva_ml/dataset/aux_classes.py

@@ -3,8 +3,9 @@ THis module defines the DataSet class with is used to manipulate n
 """
 
 from enum import Enum
-from typing import Any, Optional, SupportsInt
+from typing import Any, Optional, SupportsInt, overload
 
+from hydra_zen import hydrated_dataclass
 from pydantic import (
     BaseModel,
     ConfigDict,
@@ -42,6 +43,9 @@ class DatasetVersion(Version):
         replace(major, minor, patch): Replace the major and minor versions
     """
 
+    @overload
+    def __init__(self, version: str): ...
+    @overload
     def __init__(self, major: SupportsInt, minor: SupportsInt = 0, patch: SupportsInt = 0):
         """Initialize a DatasetVersion object.
 
@@ -50,6 +54,21 @@ class DatasetVersion(Version):
             minor: Minor version number.  Used to indicate additional members added, or change in member values.
             patch: Patch number of the dataset.  Used to indicate minor clean-up and edits
         """
+        ...
+
+    def __init__(self, *args):
+        """Initialize a DatasetVersion object.
+
+        Args:
+            major: Major version number. Used to indicate schema changes.
+            minor: Minor version number.  Used to indicate additional members added, or change in member values.
+            patch: Patch number of the dataset.  Used to indicate minor clean-up and edits
+        """
+        if len(args) == 1 and isinstance(args[0], str):
+            v = Version.parse(args[0])
+            major, minor, patch = v.major, v.minor, v.patch
+        else:
+            major, minor, patch = args
         super().__init__(major, minor, patch)
 
     def to_dict(self) -> dict[str, Any]:
@@ -182,8 +201,9 @@ class DatasetSpec(BaseModel):
     """
 
     rid: RID
-    materialize: bool = True
     version: DatasetVersion | conlist(item_type=int, min_length=3, max_length=3) | tuple[int, int, int] | str
+    materialize: bool = True
+    description: str = ""
 
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
@@ -208,3 +228,12 @@ class DatasetSpec(BaseModel):
     @field_serializer("version")
     def serialize_version(self, version: DatasetVersion) -> dict[str, Any]:
         return version.to_dict()
+
+
+# Interface for hydra-zen
+@hydrated_dataclass(DatasetSpec)
+class DatasetSpecConfig:
+    rid: str
+    version: str
+    materialize: bool = True
+    description: str = ""

{deriva_ml-1.14.47 → deriva_ml-1.17.0}/src/deriva_ml/dataset/dataset.py

@@ -22,14 +22,16 @@ Typical usage example:
 
 from __future__ import annotations
 
-# Standard library imports
 import json
 import logging
 from collections import defaultdict
+
+# Standard library imports
 from graphlib import TopologicalSorter
 from pathlib import Path
 from tempfile import TemporaryDirectory
 from typing import TYPE_CHECKING, Any, Callable, Iterable, Iterator
+from urllib.parse import urlparse
 
 import deriva.core.utils.hash_utils as hash_utils
 import requests
@@ -1039,7 +1041,6 @@ class Dataset:
                     envars={"RID": dataset.rid},
                 )
                 minid_page_url = exporter.export()[0]  # Get the MINID launch page
-
             except (
                 DerivaDownloadError,
                 DerivaDownloadConfigurationError,
@@ -1095,7 +1096,8 @@ class Dataset:
 
         # Check or create MINID
         minid_url = version_record.minid
-        if not minid_url:
+        # If we either don't have a MINID, or we have a MINID, but we don't want to use it, generate a new one.
+        if (not minid_url) or (not self._use_minid):
             if not create:
                 raise DerivaMLException(f"Minid for dataset {rid} doesn't exist")
             if self._use_minid:
@@ -1105,7 +1107,6 @@ class Dataset:
         # Return based on MINID usage
         if self._use_minid:
             return self._fetch_minid_metadata(minid_url, dataset.version)
-
         return DatasetMinid(
             dataset_version=dataset.version,
             RID=f"{rid}@{version_record.snapshot}",
@@ -1138,7 +1139,8 @@ class Dataset:
         with TemporaryDirectory() as tmp_dir:
             if self._use_minid:
                 # Get bag from S3
-                archive_path = fetch_single_file(minid.bag_url, output_path=tmp_dir)
+                bag_path = Path(tmp_dir) / Path(urlparse(minid.bag_url).path).name
+                archive_path = fetch_single_file(minid.bag_url, output_path=bag_path)
             else:
                 exporter = DerivaExport(host=self._model.catalog.deriva_server.server, output_dir=tmp_dir)
                 archive_path = exporter.retrieve_file(minid.bag_url)