deriva-ml 1.14.28__py3-none-any.whl → 1.14.30__py3-none-any.whl

deriva_ml/core/base.py

@@ -1305,7 +1305,9 @@ class DerivaML(Dataset):
     def add_workflow(self, workflow: Workflow) -> RID:
         """Adds a workflow to the catalog.
 
-        Registers a new workflow in the catalog or returns the RID of an existing workflow with the same URL.
+        Registers a new workflow in the catalog or returns the RID of an existing workflow with the same
+        URL or checksum.
+
         Each workflow represents a specific computational process or analysis pipeline.
 
         Args:
@@ -1328,11 +1330,12 @@ class DerivaML(Dataset):
             >>> workflow_rid = ml.add_workflow(workflow)
         """
         # Check if a workflow already exists by URL
-        if workflow_rid := self.lookup_workflow(workflow.url):
+        if workflow_rid := self.lookup_workflow(workflow.checksum or workflow.url):
             return workflow_rid
 
         # Get an ML schema path for the workflow table
         ml_schema_path = self.pathBuilder.schemas[self.ml_schema]
+
         try:
             # Create a workflow record
             workflow_record = {
@@ -1350,12 +1353,11 @@ class DerivaML(Dataset):
             raise DerivaMLException(f"Failed to insert workflow. Error: {error}")
         return workflow_rid
 
-    def lookup_workflow(self, url: str) -> RID | None:
+    def lookup_workflow(self, url_or_checksum: str) -> RID | None:
         """Finds a workflow by URL.
 
         Args:
-            url: URL of the workflow to find.
-
+            url_or_checksum: URL or checksum of the workflow.
         Returns:
             RID: Resource Identifier of the workflow if found, None otherwise.
 
@@ -1369,7 +1371,12 @@ class DerivaML(Dataset):
         try:
             # Search for workflow by URL
             url_column = workflow_path.URL
-            return list(workflow_path.filter(url_column == url).entities())[0]["RID"]
+            checksum_column = workflow_path.Checksum
+            return list(
+                workflow_path.path.filter(
+                    (url_column == url_or_checksum) | (checksum_column == url_or_checksum)
+                ).entities()
+            )[0]["RID"]
         except IndexError:
             return None
 
@@ -1403,7 +1410,7 @@ class DerivaML(Dataset):
         self.lookup_term(MLVocab.workflow_type, workflow_type)
 
         # Create and return a new workflow object
-        return Workflow.create_workflow(name, workflow_type, description)
+        return Workflow(name=name, workflow_type=workflow_type, description=description)
 
     def create_execution(self, configuration: ExecutionConfiguration, dry_run: bool = False) -> "Execution":
         """Creates an execution environment.

deriva_ml/execution/workflow.py

@@ -7,10 +7,6 @@ from pathlib import Path
 from typing import Any
 
 import requests
-from pydantic import (
-    BaseModel,
-    PrivateAttr,
-)
 from requests import RequestException
 
 from deriva_ml.core.definitions import RID
@@ -52,7 +48,7 @@ except ImportError:
         return get_connection_file()
 
 
-class Workflow(BaseModel):
+class Workflow:
     """Represents a computational workflow in DerivaML.
 
     A workflow defines a computational process or analysis pipeline. Each workflow has
@@ -79,27 +75,17 @@ class Workflow(BaseModel):
         ... )
     """
 
-    name: str
-    url: str
-    workflow_type: str
-    version: str | None = None
-    description: str | None = None
-    rid: RID | None = None
-    checksum: str | None = None
-    is_notebook: bool = False
-
-    _logger: Any = PrivateAttr()
-
-    def __post_init__(self):
-        """Initializes logging for the workflow."""
-        self._logger = logging.getLogger("deriva_ml")
-
-    @staticmethod
-    def create_workflow(
+    def __init__(
+        self,
         name: str,
         workflow_type: str,
-        description: str = "",
-    ) -> "Workflow":
+        description: str | None = None,
+        url: str | None = None,
+        version: str | None = None,
+        rid: RID | None = None,
+        checksum: str | None = None,
+        is_notebook: bool = False,
+    ):
         """Creates a workflow from the current execution context.
 
         Identifies the currently executing program (script or notebook) and creates
@@ -128,24 +114,27 @@ class Workflow(BaseModel):
             ...     description="Process sample data"
             ... )
         """
+        self.name = name
+        self.url = url
+        self.workflow_type = workflow_type
+        self.version = version
+        self.description = description
+        self.rid = rid
+        self.checksum = checksum
+        self.is_notebook = is_notebook
+        """Initializes logging for the workflow."""
 
         # Check to see if execution file info is being passed in by calling program.
         if "DERIVA_ML_WORKFLOW_URL" in os.environ:
-            github_url = os.environ["DERIVA_ML_WORKFLOW_URL"]
-            checksum = os.environ["DERIVA_ML_WORKFLOW_CHECKSUM"]
-            is_notebook = True
-        else:
-            path, is_notebook = Workflow._get_python_script()
-            github_url, checksum = Workflow.get_url_and_checksum(path)
-
-        return Workflow(
-            name=name,
-            url=github_url,
-            checksum=checksum,
-            description=description,
-            workflow_type=workflow_type,
-            is_notebook=is_notebook,
-        )
+            self.url = os.environ["DERIVA_ML_WORKFLOW_URL"]
+            self.checksum = os.environ["DERIVA_ML_WORKFLOW_CHECKSUM"]
+            self.is_notebook = True
+
+        if not self.url:
+            path, self.is_notebook = Workflow._get_python_script()
+            self.url, self.checksum = Workflow.get_url_and_checksum(path)
+
+        self._logger = logging.getLogger("deriva_ml")
 
     @staticmethod
     def get_url_and_checksum(executable_path: Path) -> tuple[str, str]:
@@ -386,7 +375,7 @@ class Workflow(BaseModel):
         """Get SHA-1 hash of latest commit of the file in the repository"""
 
         result = subprocess.run(
-            ["git", "log", "-n", "1", "--pretty=format:%H--", executable_path],
+            ["git", "log", "-n", "1", "--pretty=format:%H", executable_path],
             cwd=repo_root,
             capture_output=True,
             text=True,

deriva_ml/run_notebook.py

@@ -19,9 +19,7 @@ class DerivaMLRunNotebookCLI(BaseCLI):
     def __init__(self, description, epilog, **kwargs):
         BaseCLI.__init__(self, description, epilog, **kwargs)
         Workflow._check_nbstrip_status()
-        self.parser.add_argument(
-            "notebook_file", type=Path, help="Path to the notebook file"
-        )
+        self.parser.add_argument("notebook_file", type=Path, help="Path to the notebook file")
 
         self.parser.add_argument(
             "--file",
@@ -39,7 +37,7 @@ class DerivaMLRunNotebookCLI(BaseCLI):
 
         self.parser.add_argument(
             "--log-output",
-            action="store_false",
+            action="store_true",
             help="Display logging output from notebook.",
         )
 
@@ -60,9 +58,7 @@ class DerivaMLRunNotebookCLI(BaseCLI):
             help="Provide a parameter name and value to inject into the notebook.",
         )
 
-        self.parser.add_argument(
-            "--kernel", "-k", nargs=1, help="Name of kernel to run..", default=None
-        )
+        self.parser.add_argument("--kernel", "-k", nargs=1, help="Name of kernel to run..", default=None)
 
     @staticmethod
     def _coerce_number(val: str):
@@ -95,7 +91,7 @@ class DerivaMLRunNotebookCLI(BaseCLI):
                 parameters |= json.load(f)
 
         if not (notebook_file.is_file() and notebook_file.suffix == ".ipynb"):
-            print("Notebook file must be an ipynb file.")
+            print(f"Notebook file must be an ipynb file: {notebook_file.name}.")
             exit(1)
 
         os.environ["DERIVA_HOST"] = args.host
@@ -106,22 +102,20 @@ class DerivaMLRunNotebookCLI(BaseCLI):
         notebook_parameters = pm.inspect_notebook(notebook_file)
         if args.inspect:
             for param, value in notebook_parameters.items():
-                print(
-                    f"{param}:{value['inferred_type_name']}  (default {value['default']})"
-                )
+                print(f"{param}:{value['inferred_type_name']}  (default {value['default']})")
             return
         else:
             notebook_parameters = (
-                {"host": args.host, "catalog": args.catalog}
+                {"host": args.host, "catalog_id": args.catalog, "catalog": args.catalog}
                 | {k: v["default"] for k, v in notebook_parameters.items()}
                 | parameters
             )
             print(f"Running notebook {notebook_file.name} with parameters:")
             for param, value in notebook_parameters.items():
                 print(f"  {param}:{value}")
-            self.run_notebook(notebook_file.resolve(), parameters, args.kernel)
+            self.run_notebook(notebook_file.resolve(), parameters, kernel=args.kernel, log=args.log_output)
 
-    def run_notebook(self, notebook_file, parameters, kernel=None):
+    def run_notebook(self, notebook_file, parameters, kernel=None, log=False):
         url, checksum = Workflow.get_url_and_checksum(Path(notebook_file))
         os.environ["DERIVA_ML_WORKFLOW_URL"] = url
         os.environ["DERIVA_ML_WORKFLOW_CHECKSUM"] = checksum
@@ -133,6 +127,7 @@ class DerivaMLRunNotebookCLI(BaseCLI):
                 output_path=notebook_output,
                 parameters=parameters,
                 kernel_name=kernel,
+                log_output=log,
             )
             catalog_id = execution_rid = None
             with Path(notebook_output).open("r") as f:
@@ -171,15 +166,13 @@ class DerivaMLRunNotebookCLI(BaseCLI):
 
 def main():
     """Main entry point for the notebook runner CLI.
-    
+
     Creates and runs the DerivaMLRunNotebookCLI instance.
-    
+
     Returns:
         None. Executes the CLI.
     """
-    cli = DerivaMLRunNotebookCLI(
-        description="Deriva ML Execution Script Demo", epilog=""
-    )
+    cli = DerivaMLRunNotebookCLI(description="Deriva ML Execution Script Demo", epilog="")
     cli.main()
 
 

{deriva_ml-1.14.28.dist-info → deriva_ml-1.14.30.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deriva-ml
-Version: 1.14.28
+Version: 1.14.30
 Summary: Utilities to simplify use of Dervia and Pandas to create reproducable ML pipelines
 Author-email: ISRD <isrd-dev@isi.edu>
 Requires-Python: >=3.10

{deriva_ml-1.14.28.dist-info → deriva_ml-1.14.30.dist-info}/RECORD

@@ -1,9 +1,9 @@
 deriva_ml/__init__.py,sha256=_aMdxGG4mRTcXodLZLNpXqH8v5uqMbqFUryE9KqNSB8,1158
 deriva_ml/demo_catalog.py,sha256=JjPAIac_hKPh5krEhGJydjXquRnivi7kQoR8W4Khp-s,14928
 deriva_ml/feature.py,sha256=L1XUXLWGnUGCjkxX5KsGu0I8SaUTJG7eDs__yUCWuCY,8445
-deriva_ml/run_notebook.py,sha256=SNjqRMDjy7zYnf6TwC7w0Y7nvqsgoYzW2NMxWD9BzUc,6479
+deriva_ml/run_notebook.py,sha256=BvLtkD_uizukqkAYoSUb1teIfuBWtKKcHSyJoAnAC4c,6496
 deriva_ml/core/__init__.py,sha256=V_i90pc5PB1F4UdOO6DZWzpEFaZDTaPRU-EzKXQ19eI,787
-deriva_ml/core/base.py,sha256=Rsa6fExlEJwQkU1PHf5ZK051ZB9oBTHrHx-_Omij__Y,60823
+deriva_ml/core/base.py,sha256=-cV-mgADLwK2BAUxOauRB99GUDQpPS7qZP33DLXpRCY,61087
 deriva_ml/core/constants.py,sha256=6wBJ8qMxe-dbCjRGrjUIX-RK0mTWrLDTeUpaVbLFoM8,888
 deriva_ml/core/definitions.py,sha256=uq_8uYFBVBVHS691Ri2kdQsN37z0GNYTaZskJIb_ocM,1385
 deriva_ml/core/enums.py,sha256=sSN4B4OynbB-AXwxRszoFr-KWIWIAfhVa06EzAEHwVc,7194
@@ -20,7 +20,7 @@ deriva_ml/execution/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSu
 deriva_ml/execution/environment.py,sha256=B7nywqxFTRUWgyu8n7rFoKcVC9on422kjeFG2FPQfvg,9302
 deriva_ml/execution/execution.py,sha256=tXWkFLDoSre836x6MMkcmhtmr3zP5_VoSioQ72-XmvE,44298
 deriva_ml/execution/execution_configuration.py,sha256=Rw4VWkBCZN9yatvSKdTqEWTfu470lpcVKfHFR0uN0jI,6248
-deriva_ml/execution/workflow.py,sha256=E3npefABfJthlphH6QXHBjHGgDdRcoXzd7YowQOeUHg,13813
+deriva_ml/execution/workflow.py,sha256=65z9p0u4EKduLykn-noWRgf14B1lRPeYJr_0SbqYlA4,13681
 deriva_ml/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deriva_ml/model/catalog.py,sha256=dzTBcRlqgEVkPY32AUax_iu75RgFiT4Pu5au7rmrv8k,14068
 deriva_ml/model/database.py,sha256=MlXQQFgFmGxZbRx-unRFoRttXwpJspV4v2AIgppttCU,14805
@@ -32,9 +32,9 @@ deriva_ml/schema/create_schema.py,sha256=0ydJSZEg3C3-m8hWPN6k2MoUvm-RWxAlKFzVChx
 deriva_ml/schema/deriva-ml-reference.json,sha256=AEOMIgwKO3dNMMWHb0lxaXyamvfAEbUPh8qw0aAtsUQ,242460
 deriva_ml/schema/policy.json,sha256=5ykB8nnZFl-oCHzlAwppCFKJHWJFIkYognUMVEanfY8,1826
 deriva_ml/schema/table_comments_utils.py,sha256=4flCqnZAaqg_uSZ9I18pNUWAZoLfmMCXbmI5uERY5vM,2007
-deriva_ml-1.14.28.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-deriva_ml-1.14.28.dist-info/METADATA,sha256=4OlbcNCpEy3cjuU_nekQ14A_feGdqhQinktMv3IwIMw,1034
-deriva_ml-1.14.28.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-deriva_ml-1.14.28.dist-info/entry_points.txt,sha256=dkf_z7E4V6_3_5Xjsm0hcixNg6ASHDw6NfYQuBvF1Wc,363
-deriva_ml-1.14.28.dist-info/top_level.txt,sha256=I1Q1dkH96cRghdsFRVqwpa2M7IqJpR2QPUNNc5-Bnpw,10
-deriva_ml-1.14.28.dist-info/RECORD,,
+deriva_ml-1.14.30.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+deriva_ml-1.14.30.dist-info/METADATA,sha256=c3IN9kLi-Es1kMa6LAxJ-hgr6BqFnOPfw7gd6o_xvpM,1034
+deriva_ml-1.14.30.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+deriva_ml-1.14.30.dist-info/entry_points.txt,sha256=dkf_z7E4V6_3_5Xjsm0hcixNg6ASHDw6NfYQuBvF1Wc,363
+deriva_ml-1.14.30.dist-info/top_level.txt,sha256=I1Q1dkH96cRghdsFRVqwpa2M7IqJpR2QPUNNc5-Bnpw,10
+deriva_ml-1.14.30.dist-info/RECORD,,