deriva-ml 1.17.4__py3-none-any.whl → 1.17.6__py3-none-any.whl

deriva_ml/__init__.py

@@ -1,3 +1,5 @@
+# We will be loading get_version from setuptools_scm and it will emit a UserWarning about it being deprecated.
+
 from importlib.metadata import PackageNotFoundError, version
 from typing import TYPE_CHECKING
 

deriva_ml/core/base.py

@@ -187,6 +187,7 @@ class DerivaML(Dataset):
         logger_config = DEFAULT_LOGGER_OVERRIDES
         # allow for reconfiguration of module-specific logging levels
         [logging.getLogger(name).setLevel(level) for name, level in logger_config.items()]
+        logging.getLogger("root").setLevel(deriva_logging_level)
         logging.getLogger("bagit").setLevel(deriva_logging_level)
         logging.getLogger("bdbag").setLevel(deriva_logging_level)
 
@@ -1439,7 +1440,9 @@ class DerivaML(Dataset):
         # Create and return a new workflow object
         return Workflow(name=name, workflow_type=workflow_type, description=description)
 
-    def create_execution(self, configuration: ExecutionConfiguration, dry_run: bool = False) -> "Execution":
+    def create_execution(
+        self, configuration: ExecutionConfiguration, workflow: Workflow | RID | None = None, dry_run: bool = False
+    ) -> "Execution":
         """Creates an execution environment.
 
         Given an execution configuration, initialize the local compute environment to prepare for executing an
@@ -1454,6 +1457,7 @@ class DerivaML(Dataset):
 
         Args:
             configuration: ExecutionConfiguration:
+            workflow: Workflow object representing the workflow to execute if not present in the ExecutionConfiguration.
             dry_run: Do not create an execution record or upload results.
 
         Returns:
@@ -1463,7 +1467,7 @@ class DerivaML(Dataset):
         from deriva_ml.execution.execution import Execution
 
         # Create and store an execution instance
-        self._execution = Execution(configuration, self, dry_run=dry_run)
+        self._execution = Execution(configuration, self, workflow=workflow, dry_run=dry_run)
         return self._execution
 
     def restore_execution(self, execution_rid: RID | None = None) -> Execution:

deriva_ml/core/config.py

@@ -1,3 +1,4 @@
+import getpass
 import logging
 from pathlib import Path
 from typing import Any
@@ -49,8 +50,9 @@ class DerivaMLConfig(BaseModel):
 
     @staticmethod
     def compute_workdir(working_dir) -> Path:
-        # Create a default working directory if none is provided
-        working_dir = Path(working_dir) if working_dir else Path.home() / "deriva-ml"
+        # Create a default working directory if none is provided.  If a working directory is provided, we add the
+        # user name to it to ensure that multiple users do not overwrite each other's work.'
+        working_dir = (Path(working_dir) / getpass.getuser() if working_dir else Path.home()) / "deriva-ml"
         return working_dir.absolute()
 
 

deriva_ml/execution/__init__.py

@@ -1,7 +1,7 @@
 from typing import TYPE_CHECKING
 
 # Safe imports - no circular dependencies
-from deriva_ml.execution.execution_configuration import ExecutionConfiguration, AssetRIDConfig
+from deriva_ml.execution.execution_configuration import AssetRIDConfig, ExecutionConfiguration
 from deriva_ml.execution.workflow import Workflow
 
 if TYPE_CHECKING:
@@ -22,5 +22,5 @@ __all__ = [
     "Execution",  # Lazy-loaded
     "ExecutionConfiguration",
     "Workflow",
-    "AssetRIDConfig"
+    "AssetRIDConfig",
 ]

deriva_ml/execution/execution.py

@@ -216,6 +216,7 @@ class Execution:
         self,
         configuration: ExecutionConfiguration,
         ml_object: DerivaML,
+        workflow: Workflow | RID | None = None,
         reload: RID | None = None,
         dry_run: bool = False,
     ):
@@ -227,6 +228,8 @@ class Execution:
         Args:
             configuration: Settings and parameters for the execution.
             ml_object: DerivaML instance managing the execution.
+            workflow: Optional workflow RID or Workflow object.  If not specified, the workflow RID is taken from
+              the ExecutionConfiguration object
             reload: Optional RID of existing execution to reload.
             dry_run: If True, don't create catalog records or upload results.
 
@@ -234,7 +237,7 @@ class Execution:
             DerivaMLException: If initialization fails or configuration is invalid.
         """
 
-        self.asset_paths: list[AssetFilePath] = []
+        self.asset_paths: dict[str, list[AssetFilePath]] = {}
         self.configuration = configuration
         self._ml_object = ml_object
         self._model = ml_object.model
@@ -253,6 +256,8 @@ class Execution:
         self._dry_run = dry_run
 
         # Make sure we have a good workflow.
+        if workflow:
+            self.configuration.workflow = workflow
         if isinstance(self.configuration.workflow, Workflow):
             self._ml_object.lookup_term(MLVocab.workflow_type, configuration.workflow.workflow_type)
             self.workflow_rid = (

deriva_ml/execution/execution_configuration.py

@@ -26,9 +26,11 @@ import json
 import sys
 from dataclasses import dataclass
 from pathlib import Path
+from typing import Any
 
 from hydra_zen import builds
-from pydantic import BaseModel, ConfigDict, Field
+from omegaconf import DictConfig
+from pydantic import BaseModel, ConfigDict, Field, field_validator
 
 from deriva_ml.core.definitions import RID
 from deriva_ml.dataset.aux_classes import DatasetSpec
@@ -67,7 +69,7 @@ class ExecutionConfiguration(BaseModel):
 
     datasets: list[DatasetSpec] = []
     assets: list[RID] = []
-    workflow: RID | Workflow
+    workflow: RID | Workflow | None = None
     description: str = ""
     argv: list[str] = Field(default_factory=lambda: sys.argv)
 
@@ -80,6 +82,10 @@ class ExecutionConfiguration(BaseModel):
     #          config_list: DatasetList = value
     #          value = config_list.datasets
     #      return value
+    @field_validator("assets", mode="before")
+    @classmethod
+    def validate_assets(cls, value: Any) -> Any:
+        return [v.rid if isinstance(v, DictConfig) or isinstance(v, AssetRID) else v for v in value]
 
     @staticmethod
     def load_configuration(path: Path) -> ExecutionConfiguration:

deriva_ml/execution/workflow.py

@@ -3,6 +3,7 @@ import logging
 import os
 import subprocess
 import sys
+import warnings
 from pathlib import Path
 from typing import Any
 
@@ -130,7 +131,6 @@ class Workflow(BaseModel):
             self.git_root = Workflow._get_git_root(path)
 
         self.version = Workflow.get_dynamic_version(root=str(self.git_root or Path.cwd()))
-
         self._logger = logging.getLogger("deriva_ml")
         return self
 
@@ -392,10 +392,19 @@ class Workflow(BaseModel):
 
         Works under uv / Python 3.10+ by forcing setuptools to use stdlib distutils.
         """
-        # Ensure setuptools doesn't try to override stdlib distutils
+        # 1) Tell setuptools to use stdlib distutils (or no override) to avoid
+        #    the '_distutils_hack' assertion you hit.
         os.environ.setdefault("SETUPTOOLS_USE_DISTUTILS", "stdlib")
 
-        from setuptools_scm import get_version  # imported *after* env var is set
+        warnings.filterwarnings(
+            "ignore",
+            category=UserWarning,
+            module="_distutils_hack",
+        )
+        try:
+            from setuptools_scm import get_version
+        except Exception as e:  # ImportError or anything environment-specific
+            raise RuntimeError(f"setuptools_scm is not available: {e}") from e
 
         if root is None:
             # Adjust this to point at your repo root if needed

deriva_ml/schema/create_schema.py

@@ -310,6 +310,7 @@ def initialize_ml_schema(model: Model, schema_name: str = "deriva-ml"):
             },
             {"Name": "File", "Description": "A file that is not managed by Hatrac"},
             {"Name": "Input_File", "Description": "A file input to an execution."},
+            {"Name": "Output_File", "Description": "A file output from an execution."},
             {"Name": "Model_File", "Description": "The ML model."},
             {
                 "Name": "Notebook_Output",

{deriva_ml-1.17.4.dist-info → deriva_ml-1.17.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deriva-ml
-Version: 1.17.4
+Version: 1.17.6
 Summary: Utilities to simplify use of Dervia and Pandas to create reproducable ML pipelines
 Author-email: ISRD <isrd-dev@isi.edu>
 Requires-Python: >=3.10

{deriva_ml-1.17.4.dist-info → deriva_ml-1.17.6.dist-info}/RECORD

@@ -1,5 +1,5 @@
 deriva_ml/.DS_Store,sha256=gb-f5IXVed_gS5Be1Z6WxCYjrI_r5SdblvfFpIOY4ro,8196
-deriva_ml/__init__.py,sha256=YCG7P4PUtO_b-aIIYb4KhKHcfnb8Wz_YeAL-c0HiQlA,1775
+deriva_ml/__init__.py,sha256=OqhpZEC9xDpQ1vKNQSb_b0W32LlNGnfAyQaIkOXJNAs,1887
 deriva_ml/bump_version.py,sha256=eN2G5G_OeiuFxhOdjjwfxD8Rmv6dFvzIm0y_1x4Mif4,4020
 deriva_ml/demo_catalog.py,sha256=FfXPlDfzy29K9g2Fr_KmYyRhmxP2eSaqm8_Xcji8fUM,15352
 deriva_ml/feature.py,sha256=6-aphkxdKjWa9oPSGFWxHcwAc_8hmWj-7I4M178YG5Y,8470
@@ -7,8 +7,8 @@ deriva_ml/install_kernel.py,sha256=b62XY0SLViYO_Zye5r1Pl9qhYZyu_fk4KAO8NS1pxgM,2
 deriva_ml/run_notebook.py,sha256=_pds1q3WcfWqhCBqKeznbwSv5n7OND8FkL6JQ2Jkfmc,8093
 deriva_ml/test.py,sha256=BqmQXR9IyQP9h8pWttk0dzyJod2CwcfYbSUZS-Q5r4k,4460
 deriva_ml/core/__init__.py,sha256=Ko8GsWc7K_eDFW0-GaNS6gOWYP8cWHWir-ChSQaHntE,856
-deriva_ml/core/base.py,sha256=zTOxrAonj59hKqaaMvsIbvBEHrCkpdIUZfGi4q_6qks,62554
-deriva_ml/core/config.py,sha256=dF4rOLFmbk1DEkQimqbiH4pC519nRZWpwKItARNMiZ4,2244
+deriva_ml/core/base.py,sha256=uy2wgQ41yakoKy0faRcrg9Z-yQ_t2hFwC_mLkJn3sck,62813
+deriva_ml/core/config.py,sha256=HKo_cTuPWbu-IMm5Nw57Wzn1B_Mhm7Wuiq6HC-Tmt2E,2423
 deriva_ml/core/constants.py,sha256=6wBJ8qMxe-dbCjRGrjUIX-RK0mTWrLDTeUpaVbLFoM8,888
 deriva_ml/core/definitions.py,sha256=uq_8uYFBVBVHS691Ri2kdQsN37z0GNYTaZskJIb_ocM,1385
 deriva_ml/core/enums.py,sha256=sSN4B4OynbB-AXwxRszoFr-KWIWIAfhVa06EzAEHwVc,7194
@@ -21,11 +21,11 @@ deriva_ml/dataset/dataset.py,sha256=d860WuCL0-Pz6TyRpGVzhpPWDMco01-I5LT4dZjYxsQ,
 deriva_ml/dataset/dataset_bag.py,sha256=ori3BuYVqfeHkVCjNSKuZh7oMdC6uufsszicpTPODiw,19944
 deriva_ml/dataset/history.py,sha256=FK5AYYz11p4E4FWMVg4r7UPWOD4eobrq3b3xMjWF59g,3197
 deriva_ml/dataset/upload.py,sha256=n1aXSbOx1hghCDxuF8yf03jZmOLMueXL-rSnQMrfHq0,16535
-deriva_ml/execution/__init__.py,sha256=5kKpPwQbxhmRn7Npz7DpavuCxYwCQaDdl4-6z62hbds,705
+deriva_ml/execution/__init__.py,sha256=1ngO7avVUqSOS1o09OzS8jFp5mf-rrLxEBHzlIm5R28,706
 deriva_ml/execution/environment.py,sha256=B7nywqxFTRUWgyu8n7rFoKcVC9on422kjeFG2FPQfvg,9302
-deriva_ml/execution/execution.py,sha256=pso488INQ9eZx_VO6XHv4N6e3CwclOtgTEDE-qyGnNg,46311
-deriva_ml/execution/execution_configuration.py,sha256=lftW9tAzpCiQw83vLT0IG5JjlL4K6Q-qHjkhVquh8-E,5384
-deriva_ml/execution/workflow.py,sha256=80mFNRd-4H_DOJUT00aO4k9OFYSj1UOQBJTGm_kTlf8,14439
+deriva_ml/execution/execution.py,sha256=l2dAtalhKt042Yi6vjY9N0sJ49kx-wnxbgEGlYaY4jA,46604
+deriva_ml/execution/execution_configuration.py,sha256=Bxfa_JLbJoEkimRgJ0mDbCKyaKnsSn8j7XON0UbiP4I,5684
+deriva_ml/execution/workflow.py,sha256=wchHbL2_VrROvbNGtEQb4FYxOtQzlqmwR0VyfnadDvE,14795
 deriva_ml/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deriva_ml/model/catalog.py,sha256=O6_Ll4Uxg6DyxoBXT9P9CPTt9jx1guVTeX1L3KW1A5c,19645
 deriva_ml/model/database.py,sha256=BG5FSisl9tWTBnf5k9dNnijOIDyCUDeRhN_inkmIqTw,31132
@@ -33,13 +33,13 @@ deriva_ml/protocols/dataset.py,sha256=1TyaT--89Elcs-nCvVyJxUj4cDaLztZOuSOzzj1cBM
 deriva_ml/schema/__init__.py,sha256=yV-MfzCF3FA4OOz7mZwMM2q6-x1vgOJ057kUvikFF6E,130
 deriva_ml/schema/annotations.py,sha256=CMcRqYUlyW8iLCYp6sYJsncaRNtp4kFKoxcg-i-t-50,18302
 deriva_ml/schema/check_schema.py,sha256=6dadLYHPqRex6AYVClmsESI8WhC7-rb-XnGf2G298xw,3609
-deriva_ml/schema/create_schema.py,sha256=9qK9_8SRQT-DwcEwTGSkhi3j2NaoH5EVgthvV2kO-gg,13042
+deriva_ml/schema/create_schema.py,sha256=jB6ZIlQVBMRTWOEnQN6XO6gxMNPdN3t4Q3VdDs8c6SE,13130
 deriva_ml/schema/deriva-ml-reference.json,sha256=AEOMIgwKO3dNMMWHb0lxaXyamvfAEbUPh8qw0aAtsUQ,242460
 deriva_ml/schema/policy.json,sha256=5ykB8nnZFl-oCHzlAwppCFKJHWJFIkYognUMVEanfY8,1826
 deriva_ml/schema/table_comments_utils.py,sha256=4flCqnZAaqg_uSZ9I18pNUWAZoLfmMCXbmI5uERY5vM,2007
-deriva_ml-1.17.4.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-deriva_ml-1.17.4.dist-info/METADATA,sha256=rCfFB1SBVMXGd4aNxpgelYzZQjyxg6HnB4idrmV3tKo,1272
-deriva_ml-1.17.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-deriva_ml-1.17.4.dist-info/entry_points.txt,sha256=XsHSbfp7S1cKMjHoPUdFIaFcp9lHXHS6CV1zb_MEXkg,463
-deriva_ml-1.17.4.dist-info/top_level.txt,sha256=I1Q1dkH96cRghdsFRVqwpa2M7IqJpR2QPUNNc5-Bnpw,10
-deriva_ml-1.17.4.dist-info/RECORD,,
+deriva_ml-1.17.6.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+deriva_ml-1.17.6.dist-info/METADATA,sha256=F-A3XQG1DZ_MyGCRZ-zijKA4kivzYNDKy0rh_bLL2UA,1272
+deriva_ml-1.17.6.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+deriva_ml-1.17.6.dist-info/entry_points.txt,sha256=XsHSbfp7S1cKMjHoPUdFIaFcp9lHXHS6CV1zb_MEXkg,463
+deriva_ml-1.17.6.dist-info/top_level.txt,sha256=I1Q1dkH96cRghdsFRVqwpa2M7IqJpR2QPUNNc5-Bnpw,10
+deriva_ml-1.17.6.dist-info/RECORD,,