deriva-ml 1.14.29__tar.gz → 1.14.31__tar.gz

{deriva_ml-1.14.29/src/deriva_ml.egg-info → deriva_ml-1.14.31}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deriva-ml
-Version: 1.14.29
+Version: 1.14.31
 Summary: Utilities to simplify use of Dervia and Pandas to create reproducable ML pipelines
 Author-email: ISRD <isrd-dev@isi.edu>
 Requires-Python: >=3.10

{deriva_ml-1.14.29 → deriva_ml-1.14.31}/src/deriva_ml/core/base.py

@@ -1305,7 +1305,9 @@ class DerivaML(Dataset):
     def add_workflow(self, workflow: Workflow) -> RID:
         """Adds a workflow to the catalog.
 
-        Registers a new workflow in the catalog or returns the RID of an existing workflow with the same URL.
+        Registers a new workflow in the catalog or returns the RID of an existing workflow with the same
+        URL or checksum.
+
         Each workflow represents a specific computational process or analysis pipeline.
 
         Args:
@@ -1328,11 +1330,12 @@ class DerivaML(Dataset):
             >>> workflow_rid = ml.add_workflow(workflow)
         """
         # Check if a workflow already exists by URL
-        if workflow_rid := self.lookup_workflow(workflow.url):
+        if workflow_rid := self.lookup_workflow(workflow.checksum or workflow.url):
             return workflow_rid
 
         # Get an ML schema path for the workflow table
         ml_schema_path = self.pathBuilder.schemas[self.ml_schema]
+
         try:
             # Create a workflow record
             workflow_record = {
@@ -1350,12 +1353,11 @@ class DerivaML(Dataset):
             raise DerivaMLException(f"Failed to insert workflow. Error: {error}")
         return workflow_rid
 
-    def lookup_workflow(self, url: str) -> RID | None:
+    def lookup_workflow(self, url_or_checksum: str) -> RID | None:
         """Finds a workflow by URL.
 
         Args:
-            url: URL of the workflow to find.
-
+            url_or_checksum: URL or checksum of the workflow.
         Returns:
             RID: Resource Identifier of the workflow if found, None otherwise.
 
@@ -1369,7 +1371,12 @@ class DerivaML(Dataset):
         try:
             # Search for workflow by URL
             url_column = workflow_path.URL
-            return list(workflow_path.filter(url_column == url).entities())[0]["RID"]
+            checksum_column = workflow_path.Checksum
+            return list(
+                workflow_path.path.filter(
+                    (url_column == url_or_checksum) | (checksum_column == url_or_checksum)
+                ).entities()
+            )[0]["RID"]
         except IndexError:
             return None
 
@@ -1403,7 +1410,7 @@ class DerivaML(Dataset):
         self.lookup_term(MLVocab.workflow_type, workflow_type)
 
         # Create and return a new workflow object
-        return Workflow.create_workflow(name, workflow_type, description)
+        return Workflow(name=name, workflow_type=workflow_type, description=description)
 
     def create_execution(self, configuration: ExecutionConfiguration, dry_run: bool = False) -> "Execution":
         """Creates an execution environment.

{deriva_ml-1.14.29 → deriva_ml-1.14.31}/src/deriva_ml/execution/workflow.py

@@ -7,10 +7,7 @@ from pathlib import Path
 from typing import Any
 
 import requests
-from pydantic import (
-    BaseModel,
-    PrivateAttr,
-)
+from pydantic import BaseModel, PrivateAttr, model_validator
 from requests import RequestException
 
 from deriva_ml.core.definitions import RID
@@ -80,26 +77,18 @@ class Workflow(BaseModel):
     """
 
     name: str
-    url: str
     workflow_type: str
-    version: str | None = None
     description: str | None = None
+    url: str | None = None
+    version: str | None = None
     rid: RID | None = None
     checksum: str | None = None
     is_notebook: bool = False
 
-    _logger: Any = PrivateAttr()
-
-    def __post_init__(self):
-        """Initializes logging for the workflow."""
-        self._logger = logging.getLogger("deriva_ml")
+    _logger: logging.Logger = PrivateAttr(default=10)
 
-    @staticmethod
-    def create_workflow(
-        name: str,
-        workflow_type: str,
-        description: str = "",
-    ) -> "Workflow":
+    @model_validator(mode="after")
+    def setup_url_checksum(self) -> "Workflow":
         """Creates a workflow from the current execution context.
 
         Identifies the currently executing program (script or notebook) and creates
@@ -128,24 +117,20 @@ class Workflow(BaseModel):
             ...     description="Process sample data"
             ... )
         """
+        """Initializes logging for the workflow."""
 
         # Check to see if execution file info is being passed in by calling program.
         if "DERIVA_ML_WORKFLOW_URL" in os.environ:
-            github_url = os.environ["DERIVA_ML_WORKFLOW_URL"]
-            checksum = os.environ["DERIVA_ML_WORKFLOW_CHECKSUM"]
-            is_notebook = True
-        else:
-            path, is_notebook = Workflow._get_python_script()
-            github_url, checksum = Workflow.get_url_and_checksum(path)
-
-        return Workflow(
-            name=name,
-            url=github_url,
-            checksum=checksum,
-            description=description,
-            workflow_type=workflow_type,
-            is_notebook=is_notebook,
-        )
+            self.url = os.environ["DERIVA_ML_WORKFLOW_URL"]
+            self.checksum = os.environ["DERIVA_ML_WORKFLOW_CHECKSUM"]
+            self.is_notebook = True
+
+        if not self.url:
+            path, self.is_notebook = Workflow._get_python_script()
+            self.url, self.checksum = Workflow.get_url_and_checksum(path)
+
+        self._logger = logging.getLogger("deriva_ml")
+        return self
 
     @staticmethod
     def get_url_and_checksum(executable_path: Path) -> tuple[str, str]:

{deriva_ml-1.14.29 → deriva_ml-1.14.31}/src/deriva_ml/run_notebook.py

@@ -19,9 +19,7 @@ class DerivaMLRunNotebookCLI(BaseCLI):
     def __init__(self, description, epilog, **kwargs):
         BaseCLI.__init__(self, description, epilog, **kwargs)
         Workflow._check_nbstrip_status()
-        self.parser.add_argument(
-            "notebook_file", type=Path, help="Path to the notebook file"
-        )
+        self.parser.add_argument("notebook_file", type=Path, help="Path to the notebook file")
 
         self.parser.add_argument(
             "--file",
@@ -39,7 +37,7 @@ class DerivaMLRunNotebookCLI(BaseCLI):
 
         self.parser.add_argument(
             "--log-output",
-            action="store_false",
+            action="store_true",
             help="Display logging output from notebook.",
         )
 
@@ -60,9 +58,7 @@ class DerivaMLRunNotebookCLI(BaseCLI):
             help="Provide a parameter name and value to inject into the notebook.",
         )
 
-        self.parser.add_argument(
-            "--kernel", "-k", nargs=1, help="Name of kernel to run..", default=None
-        )
+        self.parser.add_argument("--kernel", "-k", nargs=1, help="Name of kernel to run..", default=None)
 
     @staticmethod
     def _coerce_number(val: str):
@@ -95,7 +91,7 @@ class DerivaMLRunNotebookCLI(BaseCLI):
                 parameters |= json.load(f)
 
         if not (notebook_file.is_file() and notebook_file.suffix == ".ipynb"):
-            print("Notebook file must be an ipynb file.")
+            print(f"Notebook file must be an ipynb file: {notebook_file.name}.")
             exit(1)
 
         os.environ["DERIVA_HOST"] = args.host
@@ -106,22 +102,20 @@ class DerivaMLRunNotebookCLI(BaseCLI):
         notebook_parameters = pm.inspect_notebook(notebook_file)
         if args.inspect:
             for param, value in notebook_parameters.items():
-                print(
-                    f"{param}:{value['inferred_type_name']}  (default {value['default']})"
-                )
+                print(f"{param}:{value['inferred_type_name']}  (default {value['default']})")
             return
         else:
             notebook_parameters = (
-                {"host": args.host, "catalog": args.catalog}
+                {"host": args.host, "catalog_id": args.catalog, "catalog": args.catalog}
                 | {k: v["default"] for k, v in notebook_parameters.items()}
                 | parameters
             )
             print(f"Running notebook {notebook_file.name} with parameters:")
             for param, value in notebook_parameters.items():
                 print(f"  {param}:{value}")
-            self.run_notebook(notebook_file.resolve(), parameters, args.kernel)
+            self.run_notebook(notebook_file.resolve(), parameters, kernel=args.kernel, log=args.log_output)
 
-    def run_notebook(self, notebook_file, parameters, kernel=None):
+    def run_notebook(self, notebook_file, parameters, kernel=None, log=False):
         url, checksum = Workflow.get_url_and_checksum(Path(notebook_file))
         os.environ["DERIVA_ML_WORKFLOW_URL"] = url
         os.environ["DERIVA_ML_WORKFLOW_CHECKSUM"] = checksum
@@ -133,6 +127,7 @@ class DerivaMLRunNotebookCLI(BaseCLI):
                 output_path=notebook_output,
                 parameters=parameters,
                 kernel_name=kernel,
+                log_output=log,
             )
             catalog_id = execution_rid = None
             with Path(notebook_output).open("r") as f:
@@ -171,15 +166,13 @@ class DerivaMLRunNotebookCLI(BaseCLI):
 
 def main():
     """Main entry point for the notebook runner CLI.
-    
+
     Creates and runs the DerivaMLRunNotebookCLI instance.
-    
+
     Returns:
         None. Executes the CLI.
     """
-    cli = DerivaMLRunNotebookCLI(
-        description="Deriva ML Execution Script Demo", epilog=""
-    )
+    cli = DerivaMLRunNotebookCLI(description="Deriva ML Execution Script Demo", epilog="")
     cli.main()
 
 

{deriva_ml-1.14.29 → deriva_ml-1.14.31/src/deriva_ml.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deriva-ml
-Version: 1.14.29
+Version: 1.14.31
 Summary: Utilities to simplify use of Dervia and Pandas to create reproducable ML pipelines
 Author-email: ISRD <isrd-dev@isi.edu>
 Requires-Python: >=3.10

{deriva_ml-1.14.29 → deriva_ml-1.14.31}/tests/execution/test_execution.py

@@ -2,7 +2,7 @@
 Tests for the execution module.
 """
 
-import re
+import json
 import subprocess
 from tempfile import TemporaryDirectory
 
@@ -24,6 +24,9 @@ class TestWorkflow:
         ml_instance.add_term(vc.asset_type, "Test Model", description="Model for our Test workflow")
         ml_instance.add_term(vc.workflow_type, "Test Workflow", description="A ML Workflow that uses Deriva ML API")
         print("Running workflow-test.py ...")
+        workflow_table = ml_instance.pathBuilder.schemas[ml_instance.ml_schema].Workflow
+        workflows = list(workflow_table.entities().fetch())
+        assert 0 == len(workflows)
         result = subprocess.run(
             [
                 "python",
@@ -34,46 +37,64 @@ class TestWorkflow:
             capture_output=True,
             text=True,
         )
-        print(result.stdout)
-        m = re.match(".*url='(?P<url>.*?)'", result.stdout)
-        url = m["url"]
-        m = re.match(".*checksum='(?P<checksum>.*?)'", result.stdout)
-        checksum = m["checksum"]
-        m = re.match(".*is_notebook=(?P<is_notebook>True|False)", result.stdout)
-        is_notebook = m["is_notebook"]
-        print("URL", url)
-        print("checksum", checksum)
-        print("is_notebook", is_notebook)
-        assert is_notebook == "False"
-        assert url.endswith("workflow-test.py")
-
-    def test_workflow_creation_notebook(self, test_ml):
+
+        workflows = list(workflow_table.entities().fetch())
+        assert 1 == len(workflows)
+        workflow_rid = workflows[0]["RID"]
+        workflow_url = workflows[0]["URL"]
+
+        workflow_rid = ml_instance.lookup_workflow(workflow_url)
+
+        assert workflow_url.endswith("workflow-test.py")
+
+        # Make sure that workflow is not duplicated if created again.
+        result = subprocess.run(
+            [
+                "python",
+                "execution/workflow-test.py",
+                ml_instance.catalog.deriva_server.server,
+                ml_instance.catalog_id,
+            ],
+            capture_output=True,
+            text=True,
+        )
+        new_workflow = result.stdout.strip()
+        assert new_workflow == workflow_rid
+
+    def test_workflow_creation_notebook(self, test_ml, tmp_path):
         ml_instance = test_ml
         ml_instance.add_term(vc.asset_type, "Test Model", description="Model for our Test workflow")
         ml_instance.add_term(vc.workflow_type, "Test Workflow", description="A ML Workflow that uses Deriva ML API")
+        workflow_table = ml_instance.pathBuilder.schemas[ml_instance.ml_schema].Workflow
+        workflows = list(workflow_table.entities().fetch())
+        assert 0 == len(workflows)
+
+        config_file = tmp_path / "config.json"
+        with config_file.open("w") as fp:
+            json.dump({"host": ml_instance.catalog.deriva_server.server, "catalog_id": ml_instance.catalog_id}, fp)
+
         print("Running notebook...")
         result = subprocess.run(
             [
                 "deriva-ml-run-notebook",
-                "execution/workflow-test.ipnb",
+                "execution/workflow-test.ipynb",
                 "--host",
-                "localhost",
+                ml_instance.catalog.deriva_server.server,
+                "--catalog",
+                ml_instance.catalog_id,
+                "--log-output",
             ],
             capture_output=True,
             text=True,
         )
         print(result)
-        m = re.match(".*url='(?P<url>.*?)'", result.stdout)
-        url = m["url"]
-        m = re.match(".*checksum='(?P<checksum>.*?)'", result.stdout)
-        checksum = m["checksum"]
-        m = re.match(".*is_notebook=(?P<is_notebook>True|False)", result.stdout)
-        is_notebook = m["is_notebook"]
-        print("URL", url)
-        print("checksum", checksum)
-        print("is_notebook", is_notebook)
-        assert is_notebook == "False"
-        assert url.endswith("workflow-test.py")
+        workflows = list(workflow_table.entities().fetch())
+        assert 1 == len(workflows)
+        workflow_rid = workflows[0]["RID"]
+        workflow_url = workflows[0]["URL"]
+
+        print(workflow_url)
+        print(workflow_rid)
 
 
 class TestExecution:

{deriva_ml-1.14.29 → deriva_ml-1.14.31}/tests/execution/workflow-test.ipynb

@@ -11,63 +11,68 @@
    ]
   },
   {
-   "metadata": {
-    "ExecuteTime": {
-     "end_time": "2025-04-18T20:05:17.505817Z",
-     "start_time": "2025-04-18T20:05:16.235245Z"
-    }
-   },
    "cell_type": "code",
+   "execution_count": null,
+   "id": "1",
+   "metadata": {},
+   "outputs": [],
    "source": [
     "from deriva_ml import DerivaML, MLVocab as vc\n",
-    "import os"
-   ],
-   "id": "2c6cbcfe94814682",
-   "outputs": [],
-   "execution_count": 1
+    "import os\n",
+    "import logging\n",
+    "logger = logging.getLogger()"
+   ]
   },
   {
-   "metadata": {},
    "cell_type": "markdown",
-   "source": "## Parameters cell\n",
-   "id": "2af30ff79278ba00"
+   "id": "2",
+   "metadata": {},
+   "source": [
+    "## Parameters cell\n"
+   ]
   },
   {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3",
    "metadata": {
     "tags": [
      "parameters"
     ]
    },
-   "cell_type": "code",
    "outputs": [],
-   "execution_count": null,
    "source": [
     "hostname = None\n",
     "catalog_id = None"
-   ],
-   "id": "3"
+   ]
   },
   {
-   "metadata": {},
    "cell_type": "code",
-   "outputs": [],
    "execution_count": null,
+   "id": "4",
+   "metadata": {},
+   "outputs": [],
    "source": [
     "# Modify these to your desired server and catalog.\n",
     "hostname = hostname or os.environ.get(\"DERIVA_HOST\")\n",
     "catalog_id = catalog_id or os.environ.get(\"DERIVA_CATALOG_ID\")\n",
     "\n",
     "# Change this line to call the domain specific class derived from DerivaML\n",
-    "deriva_ml = DerivaML(hostname=hostname, catalog_id=catalog_id)\n",
-    "deriva_ml.add_term(vc.asset_type, \"Test Model\", description=\"Model for our Test workflow\")\n",
-    "deriva_ml.add_term(vc.workflow_type, \"Test Workflow\", description=\"A ML Workflow that uses Deriva ML API\")\n",
-    "api_workflow = deriva_ml.create_workflow(\n",
+    "ml_instance = DerivaML(hostname, catalog_id)\n",
+    "logger.info(\"Got ML instance:\")\n",
+    "\n",
+    "ml_instance.add_term(vc.asset_type, \"Test Model\", description=\"Model for our Test workflow\")\n",
+    "ml_instance.add_term(vc.workflow_type, \"Test Workflow\", description=\"A ML Workflow that uses Deriva ML API\")\n",
+    "print(\"Added terms to ML instance\")\n",
+    "api_workflow = ml_instance.create_workflow(\n",
     "    name=\"Test Workflow One\",\n",
     "    workflow_type=\"Test Workflow\",\n",
     "    description=\"A test operation\",\n",
-    ")\n"
-   ],
-   "id": "1"
+    ")\n",
+    "logger.info(f\"URL: {api_workflow.url}\")\n",
+    "rid = ml_instance.add_workflow(api_workflow)\n",
+    "logger.info(f\"RID {rid}\")\n"
+   ]
   }
  ],
  "metadata": {

{deriva_ml-1.14.29 → deriva_ml-1.14.31}/tests/execution/workflow-test.py

@@ -16,4 +16,5 @@ api_workflow = ml_instance.create_workflow(
     workflow_type="Test Workflow",
     description="A test operation",
 )
-print(api_workflow)
+rid = ml_instance.add_workflow(api_workflow)
+print(rid)