deriva-ml 1.14.42__py3-none-any.whl → 1.14.44__py3-none-any.whl

deriva_ml/dataset/dataset.py

@@ -1289,8 +1289,8 @@ class Dataset:
                 {
                     "processor": "fetch",
                     "processor_params": {
-                        "query_path": f"/attribute/{spath}/!(URL::null::)/url:=URL,length:=Length,filename:=Filename,md5:=MD5",
-                        "output_path": f"asset/{table.name}",
+                        "query_path": f"/attribute/{spath}/!(URL::null::)/url:=URL,length:=Length,filename:=Filename,md5:=MD5,asset_rid:=RID",
+                        "output_path": "asset/{asset_rid}/" + table.name,
                     },
                 }
             )
@@ -1341,9 +1341,9 @@ class Dataset:
                     "source": {
                         "skip_root_path": False,
                         "api": "attribute",
-                        "path": f"{spath}/!(URL::null::)/url:=URL,length:=Length,filename:=Filename,md5:=MD5",
+                        "path": f"{spath}/!(URL::null::)/url:=URL,length:=Length,filename:=Filename,md5:=MD5, asset_rid:=RID",
                     },
-                    "destination": {"name": f"asset/{table.name}", "type": "fetch"},
+                    "destination": {"name": "asset/{asset_rid}/" + table.name, "type": "fetch"},
                 }
             )
         return exports

deriva_ml/dataset/upload.py

@@ -306,7 +306,7 @@ def upload_directory(model: DerivaModel, directory: Path | str) -> dict[Any, Fil
         )
         try:
             uploader.getUpdatedConfig()
-            uploader.scanDirectory(directory)
+            uploader.scanDirectory(directory, purge_state=True)
             results = {
                 path: FileUploadState(
                     state=UploadState(result["State"]),

deriva_ml/execution/execution.py

@@ -236,6 +236,7 @@ class Execution:
         Raises:
             DerivaMLException: If initialization fails or configuration is invalid.
         """
+
         self.asset_paths: list[AssetFilePath] = []
         self.configuration = configuration
         self._ml_object = ml_object
@@ -378,6 +379,9 @@ class Execution:
             # save runtime env
             self._save_runtime_environment()
 
+            # Now upload the files so we have the info in case the execution fails.
+            self.uploaded_assets = self._upload_execution_dirs()
+
         self.start_time = datetime.now()
         self.update_status(Status.pending, "Initialize status finished.")
 
@@ -886,10 +890,10 @@ class Execution:
 
         file_name = Path(file_name)
         asset_path = asset_file_path(
-            self._working_dir,
-            self.execution_rid,
-            self._model.name_to_table(asset_name),
-            file_name.name,
+            prefix=self._working_dir,
+            exec_rid=self.execution_rid,
+            asset_table=self._model.name_to_table(asset_name),
+            file_name=file_name.name,
             metadata=kwargs,
         )
 

deriva_ml/execution/workflow.py

@@ -318,7 +318,7 @@ class Workflow(BaseModel):
             ]
             # Get the caller's filename, which is two up the stack from here.
             filename = Path(stack[-1])
-            if not (filename.exists() or Workflow._in_repl()):
+            if not (filename.exists()) or Workflow._in_repl():
                 # Being called from the command line interpreter.
                 filename = Path.cwd() / Path("REPL")
             # Get the caller's filename, which is two up the stack from here.

deriva_ml/run_notebook.py

@@ -6,11 +6,13 @@ import tempfile
 from datetime import datetime
 from pathlib import Path
 
+import nbformat
 import papermill as pm
 import regex as re
 from deriva.core import BaseCLI
+from nbconvert import MarkdownExporter
 
-from deriva_ml import DerivaML, ExecAssetType, MLAsset, Workflow
+from deriva_ml import DerivaML, ExecAssetType, Execution, ExecutionConfiguration, MLAsset, Workflow
 
 
 class DerivaMLRunNotebookCLI(BaseCLI):
@@ -120,8 +122,8 @@ class DerivaMLRunNotebookCLI(BaseCLI):
         os.environ["DERIVA_ML_WORKFLOW_URL"] = url
         os.environ["DERIVA_ML_WORKFLOW_CHECKSUM"] = checksum
         os.environ["DERIVA_ML_NOTEBOOK_PATH"] = notebook_file.as_posix()
-
         with tempfile.TemporaryDirectory() as tmpdirname:
+            print(f"Running notebook {notebook_file.name} with parameters:")
             notebook_output = Path(tmpdirname) / Path(notebook_file).name
             pm.execute_notebook(
                 input_path=notebook_file,
@@ -130,6 +132,7 @@ class DerivaMLRunNotebookCLI(BaseCLI):
                 kernel_name=kernel,
                 log_output=log,
             )
+            print(f"Notebook output saved to {notebook_output}")
             catalog_id = execution_rid = None
             with Path(notebook_output).open("r") as f:
                 for line in f:
@@ -143,25 +146,56 @@ class DerivaMLRunNotebookCLI(BaseCLI):
             if not execution_rid:
                 print("Execution RID not found.")
                 exit(1)
-            print("Uploaded notebook output for Execution RID:", execution_rid)
 
-            ml_instance = DerivaML(hostname=hostname, catalog_id=catalog_id)
+            ml_instance = DerivaML(hostname=hostname, catalog_id=catalog_id, working_dir=tmpdirname)
+            workflow_rid = ml_instance.retrieve_rid(execution_rid)["Workflow"]
+
+            execution = Execution(
+                configuration=ExecutionConfiguration(workflow=workflow_rid),
+                ml_object=ml_instance,
+                reload=execution_rid,
+            )
+
+            # Generate an HTML version of the output notebook.
+            notebook_output_md = notebook_output.with_suffix(".md")
+            with notebook_output.open() as f:
+                nb = nbformat.read(f, as_version=4)
+            # Convert to Markdown
+            exporter = MarkdownExporter()
+            (body, resources) = exporter.from_notebook_node(nb)
+
+            with notebook_output_md.open("w") as f:
+                f.write(body)
+            nb = nbformat.read(notebook_output, as_version=4)
 
-            execution = ml_instance.restore_execution(execution_rid)
             execution.asset_file_path(
                 asset_name=MLAsset.execution_asset,
                 file_name=notebook_output,
                 asset_types=ExecAssetType.notebook_output,
             )
+
+            execution.asset_file_path(
+                asset_name=MLAsset.execution_asset,
+                file_name=notebook_output_md,
+                asset_types=ExecAssetType.notebook_output,
+            )
+            execution.asset_file_path(
+                asset_name=MLAsset.execution_asset,
+                file_name=notebook_output_md,
+                asset_types=ExecAssetType.notebook_output,
+            )
+            print("parameter....")
+
             parameter_file = execution.asset_file_path(
                 asset_name=MLAsset.execution_asset,
                 file_name=f"notebook-parameters-{datetime.now().strftime('%Y%m%d-%H%M%S')}.json",
                 asset_types=ExecAssetType.input_file.value,
             )
+
             with Path(parameter_file).open("w") as f:
                 json.dump(parameters, f)
-
             execution.upload_execution_outputs()
+
             print(ml_instance.cite(execution_rid))
 
 
@@ -178,4 +212,8 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
+    try:
+        main()
+    except Exception as e:
+        print(e)
+        exit(1)

deriva_ml/schema/create_schema.py

@@ -309,7 +309,12 @@ def initialize_ml_schema(model: Model, schema_name: str = "deriva-ml"):
                 "Description": "A file generated by an execution",
             },
             {"Name": "File", "Description": "A file that is not managed by Hatrac"},
+            {"Name": "Input_File", "Description": "A file input to an execution."},
             {"Name": "Model_File", "Description": "The ML model."},
+            {
+                "Name": "Notebook_Output",
+                "Description": "A Jupyter notebook with output cells filled from an execution.",
+            },
         ],
         defaults={"ID", "URI"},
     )

{deriva_ml-1.14.42.dist-info → deriva_ml-1.14.44.dist-info}/METADATA

@@ -1,14 +1,16 @@
 Metadata-Version: 2.4
 Name: deriva-ml
-Version: 1.14.42
+Version: 1.14.44
 Summary: Utilities to simplify use of Dervia and Pandas to create reproducable ML pipelines
 Author-email: ISRD <isrd-dev@isi.edu>
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: bump-my-version
+Requires-Dist: bdbag
 Requires-Dist: deriva~=1.7.10
 Requires-Dist: deepdiff
+Requires-Dist: nbconvert
 Requires-Dist: pandas
 Requires-Dist: regex~=2024.7.24
 Requires-Dist: pydantic>=2.11

{deriva_ml-1.14.42.dist-info → deriva_ml-1.14.44.dist-info}/RECORD

@@ -3,7 +3,7 @@ deriva_ml/bump_version.py,sha256=KpHmkpEztly2QHYL4dyaIGdEMyP4F0D89rawyh5EDTs,398
 deriva_ml/demo_catalog.py,sha256=JjPAIac_hKPh5krEhGJydjXquRnivi7kQoR8W4Khp-s,14928
 deriva_ml/feature.py,sha256=6-aphkxdKjWa9oPSGFWxHcwAc_8hmWj-7I4M178YG5Y,8470
 deriva_ml/install_kernel.py,sha256=b62XY0SLViYO_Zye5r1Pl9qhYZyu_fk4KAO8NS1pxgM,2165
-deriva_ml/run_notebook.py,sha256=QDeYKv8Ug9lx36e-CMogDu1Y4omOFlkTXvMxLriulJQ,6604
+deriva_ml/run_notebook.py,sha256=U3Bz-PdYYMIo_KZwDwlyKJtnZEMKwJjLh5Jn-M8ChUI,8039
 deriva_ml/core/__init__.py,sha256=V_i90pc5PB1F4UdOO6DZWzpEFaZDTaPRU-EzKXQ19eI,787
 deriva_ml/core/base.py,sha256=5H0GEMlKXUVRXBYXJQ25kxdscB6uX1wIKSUAM_CBOhI,61228
 deriva_ml/core/constants.py,sha256=6wBJ8qMxe-dbCjRGrjUIX-RK0mTWrLDTeUpaVbLFoM8,888
@@ -14,15 +14,15 @@ deriva_ml/core/exceptions.py,sha256=4MZNPOyN-UMaGeY9sqJDVwh_iOmz1ntp4usSyCNqVMg,
 deriva_ml/core/filespec.py,sha256=BQAAcRXfXq1lDcsKlokLOOXCBtEZpPgXxrFOIZYAgLg,4229
 deriva_ml/dataset/__init__.py,sha256=ukl2laJqa9J2AVqb4zlpIYc-3RaAlfRR33NMIQaoNrQ,104
 deriva_ml/dataset/aux_classes.py,sha256=9mZAln7_rrzaRbKhKA6dJOp3xeD6dHOC9NXOtJKROo4,6933
-deriva_ml/dataset/dataset.py,sha256=drL0YZovPpQC9G_eYe-fWuFQFysK2X-ad0vL4ZsYkf0,64396
+deriva_ml/dataset/dataset.py,sha256=AU27ZtzDSpCodtbq9T-8AtqiA-x8r78wQvFBOCgaqsQ,64451
 deriva_ml/dataset/dataset_bag.py,sha256=mPIZRX5aTbVRcJbCFtdkmlnexquF8NE-onbVK_8IxVk,14224
 deriva_ml/dataset/history.py,sha256=FK5AYYz11p4E4FWMVg4r7UPWOD4eobrq3b3xMjWF59g,3197
-deriva_ml/dataset/upload.py,sha256=Ad5JDfGvkIvefE-plP8SN9pNAxHzYrBoid5isz_bnNs,16411
+deriva_ml/dataset/upload.py,sha256=i_7KLfRSd2-THqZ1aG2OFAFGoyb8dJBCZZ5t1ftrtMQ,16429
 deriva_ml/execution/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deriva_ml/execution/environment.py,sha256=B7nywqxFTRUWgyu8n7rFoKcVC9on422kjeFG2FPQfvg,9302
-deriva_ml/execution/execution.py,sha256=NJT4qzZvvBXAlh73NVM39VE-uinSVBu2mHtuZD35G1M,44591
+deriva_ml/execution/execution.py,sha256=JH2alK-7DoRzfGvjsrNhD_tZqoZkb736bxbbgXDP33o,44780
 deriva_ml/execution/execution_configuration.py,sha256=Rw4VWkBCZN9yatvSKdTqEWTfu470lpcVKfHFR0uN0jI,6248
-deriva_ml/execution/workflow.py,sha256=E5GkE1m76eWvL1_i88h8sBkNMNJEPH-3QXyqNSzSoec,13604
+deriva_ml/execution/workflow.py,sha256=7CwPrgs3FKQHiEVus0PpK9w5hVKLKZnCrlu_nT8GFe8,13604
 deriva_ml/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deriva_ml/model/catalog.py,sha256=dzTBcRlqgEVkPY32AUax_iu75RgFiT4Pu5au7rmrv8k,14068
 deriva_ml/model/database.py,sha256=SBkYFf0qwbGmvL0Xtn_n5DCz4roGfrhuYrM8G69Cy9Y,14837
@@ -30,13 +30,13 @@ deriva_ml/model/sql_mapper.py,sha256=_0QsJEVSgSPtxrWKSgjfPZCQ1aMVcjR_Tk2OxLhWEvY
 deriva_ml/schema/__init__.py,sha256=yV-MfzCF3FA4OOz7mZwMM2q6-x1vgOJ057kUvikFF6E,130
 deriva_ml/schema/annotations.py,sha256=TuQ3vWFnK0160fRmtvsCkHx9qAcRa63MSyERB4x5a98,18197
 deriva_ml/schema/check_schema.py,sha256=6dadLYHPqRex6AYVClmsESI8WhC7-rb-XnGf2G298xw,3609
-deriva_ml/schema/create_schema.py,sha256=IrnSfN0ufS3M31MD8M6ZWyfJidKllLPqDFBUDAIDPY0,12789
+deriva_ml/schema/create_schema.py,sha256=9qK9_8SRQT-DwcEwTGSkhi3j2NaoH5EVgthvV2kO-gg,13042
 deriva_ml/schema/deriva-ml-reference.json,sha256=AEOMIgwKO3dNMMWHb0lxaXyamvfAEbUPh8qw0aAtsUQ,242460
 deriva_ml/schema/policy.json,sha256=5ykB8nnZFl-oCHzlAwppCFKJHWJFIkYognUMVEanfY8,1826
 deriva_ml/schema/table_comments_utils.py,sha256=4flCqnZAaqg_uSZ9I18pNUWAZoLfmMCXbmI5uERY5vM,2007
-deriva_ml-1.14.42.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-deriva_ml-1.14.42.dist-info/METADATA,sha256=kwydUUccmWn5SwtxH_HciAqeIyq2w1Pmk_IUo0VQvfc,1122
-deriva_ml-1.14.42.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-deriva_ml-1.14.42.dist-info/entry_points.txt,sha256=XsHSbfp7S1cKMjHoPUdFIaFcp9lHXHS6CV1zb_MEXkg,463
-deriva_ml-1.14.42.dist-info/top_level.txt,sha256=I1Q1dkH96cRghdsFRVqwpa2M7IqJpR2QPUNNc5-Bnpw,10
-deriva_ml-1.14.42.dist-info/RECORD,,
+deriva_ml-1.14.44.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+deriva_ml-1.14.44.dist-info/METADATA,sha256=Kq-OKAhKOm-JW2HbMoNsZIRWy8oT6v7-CYz4whIqCbo,1168
+deriva_ml-1.14.44.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+deriva_ml-1.14.44.dist-info/entry_points.txt,sha256=XsHSbfp7S1cKMjHoPUdFIaFcp9lHXHS6CV1zb_MEXkg,463
+deriva_ml-1.14.44.dist-info/top_level.txt,sha256=I1Q1dkH96cRghdsFRVqwpa2M7IqJpR2QPUNNc5-Bnpw,10
+deriva_ml-1.14.44.dist-info/RECORD,,