deriva-ml 1.9.1__tar.gz → 1.10.1__tar.gz

{deriva_ml-1.9.1/src/deriva_ml.egg-info → deriva_ml-1.10.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deriva-ml
-Version: 1.9.1
+Version: 1.10.1
 Summary: Utilities to simplify use of Dervia and Pandas to create reproducable ML pipelines
 Author-email: ISRD <isrd-dev@isi.edu>
 Requires-Python: >=3.10

{deriva_ml-1.9.1 → deriva_ml-1.10.1}/release.sh

@@ -33,6 +33,6 @@ python -m build
 NEW_TAG=$(git describe --tags --abbrev=0)
 echo "New version tag: $NEW_TAG"
 
-twine upload dist/*${NEW_TAG/v/}
+twine upload dist/*${NEW_TAG/v/}*
 
 echo "Release process complete!"

{deriva_ml-1.9.1 → deriva_ml-1.10.1}/src/deriva_ml/demo_catalog.py

@@ -295,7 +295,9 @@ def create_demo_catalog(
         dataset_table = deriva_ml.dataset_table
         dataset_table.annotations.update(
             Dataset(
-                deriva_ml.model, deriva_ml.cache_dir
+                deriva_ml.model,
+                cache_dir=deriva_ml.cache_dir,
+                working_dir=deriva_ml.working_dir,
             )._generate_dataset_annotations()
         )
         deriva_ml.model.apply()

{deriva_ml-1.9.1 → deriva_ml-1.10.1}/src/deriva_ml/deriva_ml_base.py

@@ -1136,8 +1136,8 @@ class DerivaML(Dataset):
             return None
 
     def create_workflow(
-        self, name: str, workflow_type: str, description: str = "", create: bool = True
-    ) -> RID | None:
+        self, name: str, workflow_type: str, description: str = ""
+    ) -> Workflow:
         """Identify current executing program and return a workflow RID for it
 
         Determine the notebook or script that is currently being executed. Assume that  this is
@@ -1149,10 +1149,21 @@ class DerivaML(Dataset):
             name: The name of the workflow.
             workflow_type: The type of the workflow.
             description: The description of the workflow.
-            create: Whether to create a new workflow.
         """
         # Make sure type is correct.
         self.lookup_term(MLVocab.workflow_type, workflow_type)
+
+        try:
+            subprocess.run(
+                "git rev-parse --is-inside-work-tree",
+                capture_output=True,
+                text=True,
+                shell=True,
+                check=True,
+            )
+        except subprocess.CalledProcessError:
+            raise DerivaMLException("Not executing in a Git repository.")
+
         github_url, is_dirty = self._github_url()
 
         if is_dirty:
@@ -1174,14 +1185,13 @@ class DerivaML(Dataset):
             shell=True,
         ).stdout.strip()
 
-        workflow = Workflow(
+        return Workflow(
             name=name,
             url=github_url,
             checksum=checksum,
             description=description,
             workflow_type=workflow_type,
         )
-        return self.add_workflow(workflow) if create else None
 
     def _github_url(self) -> tuple[str, bool]:
         """Return a GitHUB URL for the latest commit of the script from which this routine is called.
@@ -1238,7 +1248,9 @@ class DerivaML(Dataset):
         return url, is_dirty
 
     # @validate_call
-    def create_execution(self, configuration: ExecutionConfiguration) -> "Execution":
+    def create_execution(
+        self, configuration: ExecutionConfiguration, dry_run: bool = False
+    ) -> "Execution":
         """Create an execution object
 
         Given an execution configuration, initialize the local compute environment to prepare for executing an
@@ -1249,18 +1261,14 @@ class DerivaML(Dataset):
 
         Args:
             configuration: ExecutionConfiguration:
+            dryrun: Do not create an execution record or upload results.
 
         Returns:
             An execution object.
         """
         from .execution import Execution
 
-        if self._execution:
-            DerivaMLException(
-                "Only one execution can be created for a Deriva ML instance."
-            )
-        else:
-            self._execution = Execution(configuration, self)
+        self._execution = Execution(configuration, self, dry_run=dry_run)
         return self._execution
 
     # @validate_call

{deriva_ml-1.9.1 → deriva_ml-1.10.1}/src/deriva_ml/execution.py

@@ -18,6 +18,7 @@ from typing import Iterable, Any, Optional
 from deriva.core import format_exception
 from deriva.core.ermrest_model import Table
 from pydantic import validate_call, ConfigDict
+import sys
 
 from .deriva_definitions import MLVocab, ExecMetadataVocab
 from .deriva_definitions import (
@@ -30,7 +31,7 @@ from .deriva_definitions import (
 from .deriva_ml_base import DerivaML, FeatureRecord
 from .dataset_aux_classes import DatasetSpec, DatasetVersion, VersionPart
 from .dataset_bag import DatasetBag
-from .execution_configuration import ExecutionConfiguration
+from .execution_configuration import ExecutionConfiguration, Workflow
 from .execution_environment import get_execution_environment
 from .upload import (
     execution_metadata_dir,
@@ -96,6 +97,7 @@ class Execution:
         configuration: ExecutionConfiguration,
         ml_object: "DerivaML",
         reload: Optional[RID] = None,
+        dry_run: bool = False,
     ):
         """
 
@@ -107,23 +109,36 @@ class Execution:
         self.asset_paths: list[Path] = []
         self.configuration = configuration
         self._ml_object = ml_object
+        self._logger = ml_object._logger
         self.start_time = None
         self.stop_time = None
         self.status = Status.created
         self.uploaded_assets: list[Path] = []
+        self.configuration.argv = sys.argv
 
         self.dataset_rids: list[RID] = []
         self.datasets: list[DatasetBag] = []
+        self.parameters = self.configuration.parameters
 
         self._working_dir = self._ml_object.working_dir
         self._cache_dir = self._ml_object.cache_dir
+        self._dry_run = dry_run
 
-        self.workflow_rid = self.configuration.workflow
-
-        if self._ml_object.resolve_rid(configuration.workflow).table.name != "Workflow":
-            raise DerivaMLException(
-                "Workflow specified in execution configuration is not a Workflow"
+        if isinstance(self.configuration.workflow, Workflow):
+            self.workflow_rid = (
+                self._ml_object.add_workflow(self.configuration.workflow)
+                if not self._dry_run
+                else "0000"
             )
+        else:
+            self.workflow_rid = self.configuration.workflow
+            if (
+                self._ml_object.resolve_rid(configuration.workflow).table.name
+                != "Workflow"
+            ):
+                raise DerivaMLException(
+                    "Workflow specified in execution configuration is not a Workflow"
+                )
 
         for d in self.configuration.datasets:
             if self._ml_object.resolve_rid(d.rid).table.name != "Dataset":
@@ -142,6 +157,10 @@ class Execution:
         schema_path = self._ml_object.pathBuilder.schemas[self._ml_object.ml_schema]
         if reload:
             self.execution_rid = reload
+            if self.execution_rid == "0000":
+                self._dry_run = True
+        elif self._dry_run:
+            self.execution_rid = "0000"
         else:
             self.execution_rid = schema_path.Execution.insert(
                 [
@@ -189,7 +208,7 @@ class Execution:
             self.dataset_rids.append(dataset.rid)
         # Update execution info
         schema_path = self._ml_object.pathBuilder.schemas[self._ml_object.ml_schema]
-        if self.dataset_rids and not reload:
+        if self.dataset_rids and not (reload or self._dry_run):
             schema_path.Dataset_Execution.insert(
                 [
                     {"Dataset": d, "Execution": self.execution_rid}
@@ -203,7 +222,7 @@ class Execution:
             self._ml_object.download_asset(asset_rid=a, dest_dir=self._asset_dir())
             for a in self.configuration.assets
         ]
-        if self.asset_paths and not reload:
+        if self.asset_paths and not (reload or self._dry_run):
             self._update_execution_asset_table(self.configuration.assets)
 
         # Save configuration details for later upload
@@ -242,6 +261,11 @@ class Execution:
             msg: Additional information about the status
         """
         self.status = status
+        self._logger.info(msg)
+
+        if self._dry_run:
+            return
+
         self._ml_object.pathBuilder.schemas[self._ml_object.ml_schema].Execution.update(
             [
                 {
@@ -278,7 +302,7 @@ class Execution:
 
         self.start_time = datetime.now()
         self.uploaded_assets = None
-        self.update_status(Status.initializing, "Start ML algorithm ...")
+        self.update_status(Status.initializing, "Start execution  ...")
 
     def execution_stop(self) -> None:
         """Finish the execution and update the duration and status of execution."""
@@ -288,13 +312,11 @@ class Execution:
         minutes, seconds = divmod(remainder, 60)
         duration = f"{round(hours, 0)}H {round(minutes, 0)}min {round(seconds, 4)}sec"
 
-        if self._ml_object._is_notebook:
-            self._create_notebook_checkpoint()
-
         self.update_status(Status.completed, "Algorithm execution ended.")
-        self._ml_object.pathBuilder.schemas[self._ml_object.ml_schema].Execution.update(
-            [{"RID": self.execution_rid, "Duration": duration}]
-        )
+        if not self._dry_run:
+            self._ml_object.pathBuilder.schemas[
+                self._ml_object.ml_schema
+            ].Execution.update([{"RID": self.execution_rid, "Duration": duration}])
 
     def _upload_execution_dirs(self) -> dict[str, FileUploadState]:
         """Upload execution assets at _working_dir/Execution_asset.
@@ -402,6 +424,8 @@ class Execution:
             Uploaded assets with key as assets' suborder name, values as an
             ordered dictionary with RID and metadata in the Execution_Asset table.
         """
+        if self._dry_run:
+            return {}
         try:
             uploaded_assets = self._upload_execution_dirs()
             self.update_status(Status.completed, "Successfully end the execution.")

{deriva_ml-1.9.1 → deriva_ml-1.10.1}/src/deriva_ml/execution_configuration.py

@@ -1,14 +1,11 @@
 from __future__ import annotations
 
 import json
-from typing import Optional
+from typing import Optional, Any
 
-from pydantic import (
-    BaseModel,
-    conlist,
-    ConfigDict,
-)
+from pydantic import BaseModel, conlist, ConfigDict, field_validator, Field
 from pathlib import Path
+import sys
 
 
 from .dataset_aux_classes import DatasetSpec
@@ -43,17 +40,30 @@ class ExecutionConfiguration(BaseModel):
         datasets: List of dataset specifications which specify the dataset RID, version and if the dataset
             should be materialized.
         assets: List of assets to be downloaded prior to execution.  The values must be RIDs in an asset table
+        parameters: Either a dictionary or a path to a JSON file that contains configuration parameters for the execution.
         workflow: A RID for a workflow instance.  Must have a name, URI to the workflow instance, and a type.
         description: A description of the execution.  Can use Markdown format.
     """
 
     datasets: conlist(DatasetSpec) = []
     assets: list[RID | str] = []  # List of RIDs to model files.
-    workflow: RID
+    workflow: RID | Workflow
+    parameters: dict[str, Any] = {}
     description: str = ""
+    argv: conlist(str) = Field(default_factory=lambda: sys.argv)
 
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
+    @field_validator("parameters", mode="before")
+    @classmethod
+    def validate_parameters(cls, value: Any) -> Any:
+        """If parameter is a file, assume that it has JSON contents for configuration parameters"""
+        if isinstance(value, str) or isinstance(value, Path):
+            with open(value, "r") as f:
+                return json.load(f)
+        else:
+            return value
+
     @staticmethod
     def load_configuration(path: Path) -> ExecutionConfiguration:
         """Create a ExecutionConfiguration from a JSON configuration file.

{deriva_ml-1.9.1 → deriva_ml-1.10.1/src/deriva_ml.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deriva-ml
-Version: 1.9.1
+Version: 1.10.1
 Summary: Utilities to simplify use of Dervia and Pandas to create reproducable ML pipelines
 Author-email: ISRD <isrd-dev@isi.edu>
 Requires-Python: >=3.10