deriva-ml 1.16.0__tar.gz → 1.17.1__tar.gz

{deriva_ml-1.16.0 → deriva_ml-1.17.1}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.4
 Name: deriva-ml
-Version: 1.16.0
+Version: 1.17.1
 Summary: Utilities to simplify use of Dervia and Pandas to create reproducable ML pipelines
 Author-email: ISRD <isrd-dev@isi.edu>
-Requires-Python: >=3.10
+Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: bump-my-version
@@ -12,16 +12,18 @@ Requires-Dist: deriva~=1.7.10
 Requires-Dist: deepdiff
 Requires-Dist: nbconvert
 Requires-Dist: pandas
-Requires-Dist: regex~=2024.7.24
+Requires-Dist: pip-system-certs
 Requires-Dist: pydantic>=2.11
-Requires-Dist: semver>3.0.0
-Requires-Dist: setuptools>=64
-Requires-Dist: setuptools-scm>=8.0
-Requires-Dist: nbstripout
 Requires-Dist: papermill
 Requires-Dist: pandas-stubs==2.2.3.250527
 Requires-Dist: pyyaml
+Requires-Dist: regex~=2024.7.24
+Requires-Dist: semver>3.0.0
+Requires-Dist: setuptools>=80
+Requires-Dist: setuptools-scm>=8.0
+Requires-Dist: nbstripout
 Requires-Dist: hydra_zen
+Requires-Dist: SQLAlchemy
 Dynamic: license-file
 
 # DerivaML

{deriva_ml-1.16.0 → deriva_ml-1.17.1}/pyproject.toml

@@ -6,7 +6,7 @@ authors = [
 ]
 description = "Utilities to simplify use of Dervia and Pandas to create reproducable ML pipelines"
 readme = "README.md"
-requires-python = ">=3.10"
+requires-python = ">=3.11"
 dependencies = [
     "bump-my-version",
     "bdbag",
@@ -14,16 +14,18 @@ dependencies = [
     "deepdiff",
     "nbconvert",
     "pandas",
-    "regex~=2024.7.24",
+    "pip-system-certs",
     "pydantic>=2.11",
-    "semver>3.0.0",
-    "setuptools>=64",
-    "setuptools-scm>=8.0",
-    "nbstripout",
     "papermill",
     "pandas-stubs==2.2.3.250527",
     "pyyaml",
+    "regex~=2024.7.24",
+    "semver>3.0.0",
+    "setuptools>=80",
+    "setuptools-scm>=8.0",
+    "nbstripout",
     "hydra_zen",
+    "SQLAlchemy"
 ]
 
 [project.scripts]
@@ -39,10 +41,11 @@ deriva-ml-check-catalog-schema = "deriva_ml.schema.check_schema:main"
 [project.optional-dependencies]
 
 [build-system]
-requires = ["setuptools>=45", "setuptools_scm[toml]>=6.2", "wheel"]
+requires = ["setuptools>=80", "setuptools_scm[toml]>=8", "wheel"]
 build-backend = "setuptools.build_meta"
 
 [tool.uv]
+python-preference = "only-managed"
 
 [tool.uv.sources]
 #bdbag = {git = "https://github.com/fair-research/bdbag", branch = "master" }
@@ -77,13 +80,10 @@ pre_commit_hooks = []
 post_commit_hooks = []
 
 [tool.pytest]
-mock_use_standalone_module = true
-
-[tool.pytest.ini_options]
 testpaths = ["tests"]
 python_files = ["test_*.py"]
 #addopts = "-v --cov=deriva_ml --cov-report=term-missing  --import-mode=importlib"
-addopts = "-v  --import-mode=importlib"
+addopts = ["-v", "--import-mode=importlib"]
 
 
 [tool.ruff]
@@ -114,6 +114,7 @@ dev = [
     "pytest>=8.4.1",
     "pytest-mock",
     "pytest-coverage>=0.0",
+    "pip-system-certs",
     "ruff"
 ]
 lint = [

{deriva_ml-1.16.0 → deriva_ml-1.17.1}/src/deriva_ml/__init__.py

@@ -25,9 +25,6 @@ from deriva_ml.core.exceptions import (
     DerivaMLInvalidTerm,
     DerivaMLTableTypeError,
 )
-from deriva_ml.dataset.aux_classes import DatasetConfig, DatasetConfigList, DatasetSpec, DatasetVersion
-
-from .execution import Execution, ExecutionConfiguration, Workflow
 
 # Type-checking only - avoid circular import at runtime
 if TYPE_CHECKING:
@@ -51,13 +48,6 @@ def __getattr__(name):
 __all__ = [
     "DerivaML",  # Lazy-loaded
     "DerivaMLConfig",
-    "DatasetConfig",
-    "DatasetConfigList",
-    "DatasetSpec",
-    "DatasetVersion",
-    "Execution",
-    "ExecutionConfiguration",
-    "Workflow",
     # Exceptions
     "DerivaMLException",
     "DerivaMLInvalidTerm",

{deriva_ml-1.16.0 → deriva_ml-1.17.1}/src/deriva_ml/core/base.py

@@ -19,7 +19,7 @@ import logging
 from datetime import datetime
 from itertools import chain
 from pathlib import Path
-from typing import Dict, Iterable, List, cast, TYPE_CHECKING, Any
+from typing import Dict, Iterable, List, cast, TYPE_CHECKING, Any, Self
 from urllib.parse import urlsplit
 
 
@@ -28,13 +28,14 @@ import requests
 from pydantic import ConfigDict, validate_call
 
 # Deriva imports
-from deriva.core import DEFAULT_SESSION_CONFIG, format_exception, get_credential, urlquote, init_logging
+from deriva.core import DEFAULT_SESSION_CONFIG, format_exception, get_credential, urlquote
 
 import deriva.core.datapath as datapath
 from deriva.core.datapath import DataPathException, _SchemaWrapper as SchemaWrapper
 from deriva.core.deriva_server import DerivaServer
 from deriva.core.ermrest_catalog import ResolveRidResult
 from deriva.core.ermrest_model import Key, Table
+from deriva.core.utils.core_utils import DEFAULT_LOGGER_OVERRIDES
 from deriva.core.utils.globus_auth_utils import GlobusNativeLogin
 
 from deriva_ml.core.exceptions import DerivaMLInvalidTerm
@@ -103,6 +104,10 @@ class DerivaML(Dataset):
         >>> ml.add_term('vocabulary_table', 'new_term', description='Description of term')
     """
 
+    @classmethod
+    def instantiate(cls, config: DerivaMLConfig) -> Self:
+        return cls(**config.model_dump())
+
     def __init__(
         self,
         hostname: str,
@@ -149,7 +154,6 @@ class DerivaML(Dataset):
             credentials=self.credential,
             session_config=self._get_session_config(),
         )
-
         try:
             if check_auth and server.get_authn_session():
                 pass
@@ -158,7 +162,6 @@ class DerivaML(Dataset):
                 "You are not authorized to access this catalog. "
                 "Please check your credentials and make sure you have logged in."
             )
-
         self.catalog = server.connect_ermrest(catalog_id)
         self.model = DerivaModel(self.catalog.getCatalogModel(), domain_schema=domain_schema)
 
@@ -176,9 +179,13 @@ class DerivaML(Dataset):
         # Set up logging
         self._logger = logging.getLogger("deriva_ml")
         self._logger.setLevel(logging_level)
+        self._logging_level = logging_level
+        self._deriva_logging_level = deriva_logging_level
 
         # Configure deriva logging level
-        init_logging(deriva_logging_level)
+        logger_config = DEFAULT_LOGGER_OVERRIDES
+        # allow for reconfiguration of module-specific logging levels
+        [logging.getLogger(name).setLevel(level) for name, level in logger_config.items()]
         logging.getLogger("bagit").setLevel(deriva_logging_level)
         logging.getLogger("bdbag").setLevel(deriva_logging_level)
 
@@ -1081,7 +1088,12 @@ class DerivaML(Dataset):
         return self._download_dataset_bag(
             dataset=dataset,
             execution_rid=execution_rid,
-            snapshot_catalog=DerivaML(self.host_name, self._version_snapshot(dataset)),
+            snapshot_catalog=DerivaML(
+                self.host_name,
+                self._version_snapshot(dataset),
+                logging_level=self._logging_level,
+                deriva_logging_level=self._deriva_logging_level,
+            ),
         )
 
     def _update_status(self, new_status: Status, status_detail: str, execution_rid: RID):

deriva_ml-1.17.1/src/deriva_ml/dataset/__init__.py

@@ -0,0 +1,12 @@
+from .aux_classes import DatasetSpec, DatasetSpecConfig, DatasetVersion, VersionPart
+from .dataset import Dataset
+from .dataset_bag import DatasetBag
+
+__all__ = [
+    "Dataset",
+    "DatasetSpec",
+    "DatasetSpecConfig",
+    "DatasetBag",
+    "DatasetVersion",
+    "VersionPart",
+]

{deriva_ml-1.16.0 → deriva_ml-1.17.1}/src/deriva_ml/dataset/aux_classes.py

@@ -212,18 +212,10 @@ class DatasetSpec(BaseModel):
         return version.to_dict()
 
 
+# Interface for hydra-zen
 @hydrated_dataclass(DatasetSpec)
-class DatasetConfig:
+class DatasetSpecConfig:
     rid: str
     version: str
     materialize: bool = True
     description: str = ""
-
-class DatasetList(BaseModel):
-    datasets: list[DatasetSpec]
-    description: str = ""
-
-@hydrated_dataclass(DatasetList)
-class DatasetConfigList:
-    datasets: list[DatasetConfig]
-    description: str = ""

{deriva_ml-1.16.0 → deriva_ml-1.17.1}/src/deriva_ml/dataset/dataset.py

@@ -31,6 +31,7 @@ from graphlib import TopologicalSorter
 from pathlib import Path
 from tempfile import TemporaryDirectory
 from typing import TYPE_CHECKING, Any, Callable, Iterable, Iterator
+from urllib.parse import urlparse
 
 import deriva.core.utils.hash_utils as hash_utils
 import requests
@@ -1040,7 +1041,6 @@ class Dataset:
                     envars={"RID": dataset.rid},
                 )
                 minid_page_url = exporter.export()[0]  # Get the MINID launch page
-
             except (
                 DerivaDownloadError,
                 DerivaDownloadConfigurationError,
@@ -1096,7 +1096,8 @@ class Dataset:
 
         # Check or create MINID
         minid_url = version_record.minid
-        if not minid_url:
+        # If we either don't have a MINID, or we have a MINID, but we don't want to use it, generate a new one.
+        if (not minid_url) or (not self._use_minid):
             if not create:
                 raise DerivaMLException(f"Minid for dataset {rid} doesn't exist")
             if self._use_minid:
@@ -1106,7 +1107,6 @@ class Dataset:
         # Return based on MINID usage
         if self._use_minid:
             return self._fetch_minid_metadata(minid_url, dataset.version)
-
         return DatasetMinid(
             dataset_version=dataset.version,
             RID=f"{rid}@{version_record.snapshot}",
@@ -1139,7 +1139,8 @@ class Dataset:
         with TemporaryDirectory() as tmp_dir:
             if self._use_minid:
                 # Get bag from S3
-                archive_path = fetch_single_file(minid.bag_url, output_path=tmp_dir)
+                bag_path = Path(tmp_dir) / Path(urlparse(minid.bag_url).path).name
+                archive_path = fetch_single_file(minid.bag_url, output_path=bag_path)
             else:
                 exporter = DerivaExport(host=self._model.catalog.deriva_server.server, output_dir=tmp_dir)
                 archive_path = exporter.retrieve_file(minid.bag_url)