deriva-ml 1.14.41__py3-none-any.whl → 1.14.43__py3-none-any.whl

deriva_ml/dataset/upload.py

@@ -296,6 +296,7 @@ def upload_directory(model: DerivaModel, directory: Path | str) -> dict[Any, Fil
 
         with spec_file.open("w+") as cfile:
             json.dump(bulk_upload_configuration(model), cfile)
+        print("Creating GenericUploader...")
         uploader = GenericUploader(
             server={
                 "host": model.hostname,
@@ -306,7 +307,7 @@ def upload_directory(model: DerivaModel, directory: Path | str) -> dict[Any, Fil
         )
         try:
             uploader.getUpdatedConfig()
-            uploader.scanDirectory(directory)
+            uploader.scanDirectory(directory, purge_state=True)
             results = {
                 path: FileUploadState(
                     state=UploadState(result["State"]),
@@ -317,6 +318,7 @@ def upload_directory(model: DerivaModel, directory: Path | str) -> dict[Any, Fil
             }
         finally:
             uploader.cleanup()
+            print(f"Cleanup called...{uploader}")
         return results
 
 

deriva_ml/execution/execution.py

@@ -236,6 +236,7 @@ class Execution:
         Raises:
             DerivaMLException: If initialization fails or configuration is invalid.
         """
+
         self.asset_paths: list[AssetFilePath] = []
         self.configuration = configuration
         self._ml_object = ml_object
@@ -378,6 +379,9 @@ class Execution:
             # save runtime env
             self._save_runtime_environment()
 
+            # Now upload the files so we have the info in case the execution fails.
+            self.uploaded_assets = self._upload_execution_dirs()
+
         self.start_time = datetime.now()
         self.update_status(Status.pending, "Initialize status finished.")
 
@@ -886,10 +890,10 @@ class Execution:
 
         file_name = Path(file_name)
         asset_path = asset_file_path(
-            self._working_dir,
-            self.execution_rid,
-            self._model.name_to_table(asset_name),
-            file_name.name,
+            prefix=self._working_dir,
+            exec_rid=self.execution_rid,
+            asset_table=self._model.name_to_table(asset_name),
+            file_name=file_name.name,
             metadata=kwargs,
         )
 

deriva_ml/execution/workflow.py

@@ -121,7 +121,7 @@ class Workflow(BaseModel):
         if "DERIVA_ML_WORKFLOW_URL" in os.environ:
             self.url = os.environ["DERIVA_ML_WORKFLOW_URL"]
             self.checksum = os.environ["DERIVA_ML_WORKFLOW_CHECKSUM"]
-            self.git_root = Workflow._get_git_root(Path(os.environ["DERIVA_ML_NOTEBOOK_PATHL"]))
+            self.git_root = Workflow._get_git_root(Path(os.environ["DERIVA_ML_NOTEBOOK_PATH"]))
             self.is_notebook = True
 
         if not self.url:
@@ -318,14 +318,12 @@ class Workflow(BaseModel):
             ]
             # Get the caller's filename, which is two up the stack from here.
             filename = Path(stack[-1])
-            if not (filename.exists() or Workflow._in_repl()):
+            if not (filename.exists()) or Workflow._in_repl():
                 # Being called from the command line interpreter.
                 filename = Path.cwd() / Path("REPL")
             # Get the caller's filename, which is two up the stack from here.
-            elif "PYTEST_CURRENT_TEST" in os.environ:
+            elif (not filename.exists()) and "PYTEST_CURRENT_TEST" in os.environ:
                 filename = Path.cwd() / Path("pytest")
-            else:
-                raise DerivaMLException("Looking for caller failed")  # Stack is too shallow
         return filename, is_notebook
 
     @staticmethod

deriva_ml/run_notebook.py

@@ -6,11 +6,13 @@ import tempfile
 from datetime import datetime
 from pathlib import Path
 
+import nbformat
 import papermill as pm
 import regex as re
 from deriva.core import BaseCLI
+from nbconvert import HTMLExporter
 
-from deriva_ml import DerivaML, ExecAssetType, MLAsset, Workflow
+from deriva_ml import DerivaML, ExecAssetType, Execution, ExecutionConfiguration, MLAsset, Workflow
 
 
 class DerivaMLRunNotebookCLI(BaseCLI):
@@ -120,8 +122,8 @@ class DerivaMLRunNotebookCLI(BaseCLI):
         os.environ["DERIVA_ML_WORKFLOW_URL"] = url
         os.environ["DERIVA_ML_WORKFLOW_CHECKSUM"] = checksum
         os.environ["DERIVA_ML_NOTEBOOK_PATH"] = notebook_file.as_posix()
-
         with tempfile.TemporaryDirectory() as tmpdirname:
+            print(f"Running notebook {notebook_file.name} with parameters:")
             notebook_output = Path(tmpdirname) / Path(notebook_file).name
             pm.execute_notebook(
                 input_path=notebook_file,
@@ -130,6 +132,7 @@ class DerivaMLRunNotebookCLI(BaseCLI):
                 kernel_name=kernel,
                 log_output=log,
             )
+            print(f"Notebook output saved to {notebook_output}")
             catalog_id = execution_rid = None
             with Path(notebook_output).open("r") as f:
                 for line in f:
@@ -143,25 +146,51 @@ class DerivaMLRunNotebookCLI(BaseCLI):
             if not execution_rid:
                 print("Execution RID not found.")
                 exit(1)
-            print("Uploaded notebook output for Execution RID:", execution_rid)
 
-            ml_instance = DerivaML(hostname=hostname, catalog_id=catalog_id)
+            ml_instance = DerivaML(hostname=hostname, catalog_id=catalog_id, working_dir=tmpdirname)
+            workflow_rid = ml_instance.retrieve_rid(execution_rid)["Workflow"]
+
+            execution = Execution(
+                configuration=ExecutionConfiguration(workflow=workflow_rid),
+                ml_object=ml_instance,
+                reload=execution_rid,
+            )
+
+            # Generate an HTML version of the output notebook.
+            nb = nbformat.read(notebook_output, as_version=4)
+            html_exporter = HTMLExporter(template_name="classic")
+            body, resources = html_exporter.from_notebook_node(nb)
+            notebook_output_html = notebook_output.with_suffix(".html")
+            notebook_output_html.write_text(body, encoding="utf-8")
 
-            execution = ml_instance.restore_execution(execution_rid)
             execution.asset_file_path(
                 asset_name=MLAsset.execution_asset,
                 file_name=notebook_output,
                 asset_types=ExecAssetType.notebook_output,
             )
+
+            execution.asset_file_path(
+                asset_name=MLAsset.execution_asset,
+                file_name=notebook_output_html,
+                asset_types=ExecAssetType.notebook_output,
+            )
+            execution.asset_file_path(
+                asset_name=MLAsset.execution_asset,
+                file_name=notebook_output_html,
+                asset_types=ExecAssetType.notebook_output,
+            )
+            print("parameter....")
+
             parameter_file = execution.asset_file_path(
                 asset_name=MLAsset.execution_asset,
                 file_name=f"notebook-parameters-{datetime.now().strftime('%Y%m%d-%H%M%S')}.json",
                 asset_types=ExecAssetType.input_file.value,
             )
+
             with Path(parameter_file).open("w") as f:
                 json.dump(parameters, f)
-
             execution.upload_execution_outputs()
+
             print(ml_instance.cite(execution_rid))
 
 
@@ -178,4 +207,8 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
+    try:
+        main()
+    except Exception as e:
+        print(e)
+        exit(1)

deriva_ml/schema/create_schema.py

@@ -309,7 +309,12 @@ def initialize_ml_schema(model: Model, schema_name: str = "deriva-ml"):
                 "Description": "A file generated by an execution",
             },
             {"Name": "File", "Description": "A file that is not managed by Hatrac"},
+            {"Name": "Input_File", "Description": "A file input to an execution."},
             {"Name": "Model_File", "Description": "The ML model."},
+            {
+                "Name": "Notebook_Output",
+                "Description": "A Jupyter notebook with output cells filled from an execution.",
+            },
         ],
         defaults={"ID", "URI"},
     )

{deriva_ml-1.14.41.dist-info → deriva_ml-1.14.43.dist-info}/METADATA

@@ -1,14 +1,16 @@
 Metadata-Version: 2.4
 Name: deriva-ml
-Version: 1.14.41
+Version: 1.14.43
 Summary: Utilities to simplify use of Dervia and Pandas to create reproducable ML pipelines
 Author-email: ISRD <isrd-dev@isi.edu>
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: bump-my-version
+Requires-Dist: bdbag
 Requires-Dist: deriva~=1.7.10
 Requires-Dist: deepdiff
+Requires-Dist: nbconvert
 Requires-Dist: pandas
 Requires-Dist: regex~=2024.7.24
 Requires-Dist: pydantic>=2.11

{deriva_ml-1.14.41.dist-info → deriva_ml-1.14.43.dist-info}/RECORD

@@ -3,7 +3,7 @@ deriva_ml/bump_version.py,sha256=KpHmkpEztly2QHYL4dyaIGdEMyP4F0D89rawyh5EDTs,398
 deriva_ml/demo_catalog.py,sha256=JjPAIac_hKPh5krEhGJydjXquRnivi7kQoR8W4Khp-s,14928
 deriva_ml/feature.py,sha256=6-aphkxdKjWa9oPSGFWxHcwAc_8hmWj-7I4M178YG5Y,8470
 deriva_ml/install_kernel.py,sha256=b62XY0SLViYO_Zye5r1Pl9qhYZyu_fk4KAO8NS1pxgM,2165
-deriva_ml/run_notebook.py,sha256=QDeYKv8Ug9lx36e-CMogDu1Y4omOFlkTXvMxLriulJQ,6604
+deriva_ml/run_notebook.py,sha256=2pGv6dNTH0TuCz7E6GBS4MMzF5IyH-Tbv1i46rsJLLQ,7923
 deriva_ml/core/__init__.py,sha256=V_i90pc5PB1F4UdOO6DZWzpEFaZDTaPRU-EzKXQ19eI,787
 deriva_ml/core/base.py,sha256=5H0GEMlKXUVRXBYXJQ25kxdscB6uX1wIKSUAM_CBOhI,61228
 deriva_ml/core/constants.py,sha256=6wBJ8qMxe-dbCjRGrjUIX-RK0mTWrLDTeUpaVbLFoM8,888
@@ -17,12 +17,12 @@ deriva_ml/dataset/aux_classes.py,sha256=9mZAln7_rrzaRbKhKA6dJOp3xeD6dHOC9NXOtJKR
 deriva_ml/dataset/dataset.py,sha256=drL0YZovPpQC9G_eYe-fWuFQFysK2X-ad0vL4ZsYkf0,64396
 deriva_ml/dataset/dataset_bag.py,sha256=mPIZRX5aTbVRcJbCFtdkmlnexquF8NE-onbVK_8IxVk,14224
 deriva_ml/dataset/history.py,sha256=FK5AYYz11p4E4FWMVg4r7UPWOD4eobrq3b3xMjWF59g,3197
-deriva_ml/dataset/upload.py,sha256=Ad5JDfGvkIvefE-plP8SN9pNAxHzYrBoid5isz_bnNs,16411
+deriva_ml/dataset/upload.py,sha256=7C9v4CmH3OGjBKdPl-OPvygseuAEtDjy3atSwmhgGBs,16524
 deriva_ml/execution/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deriva_ml/execution/environment.py,sha256=B7nywqxFTRUWgyu8n7rFoKcVC9on422kjeFG2FPQfvg,9302
-deriva_ml/execution/execution.py,sha256=NJT4qzZvvBXAlh73NVM39VE-uinSVBu2mHtuZD35G1M,44591
+deriva_ml/execution/execution.py,sha256=JH2alK-7DoRzfGvjsrNhD_tZqoZkb736bxbbgXDP33o,44780
 deriva_ml/execution/execution_configuration.py,sha256=Rw4VWkBCZN9yatvSKdTqEWTfu470lpcVKfHFR0uN0jI,6248
-deriva_ml/execution/workflow.py,sha256=qMH3PlJRy6W5r8uKveZm8pbS5UEEWdUvZ1xyyQjdAOA,13688
+deriva_ml/execution/workflow.py,sha256=7CwPrgs3FKQHiEVus0PpK9w5hVKLKZnCrlu_nT8GFe8,13604
 deriva_ml/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deriva_ml/model/catalog.py,sha256=dzTBcRlqgEVkPY32AUax_iu75RgFiT4Pu5au7rmrv8k,14068
 deriva_ml/model/database.py,sha256=SBkYFf0qwbGmvL0Xtn_n5DCz4roGfrhuYrM8G69Cy9Y,14837
@@ -30,13 +30,13 @@ deriva_ml/model/sql_mapper.py,sha256=_0QsJEVSgSPtxrWKSgjfPZCQ1aMVcjR_Tk2OxLhWEvY
 deriva_ml/schema/__init__.py,sha256=yV-MfzCF3FA4OOz7mZwMM2q6-x1vgOJ057kUvikFF6E,130
 deriva_ml/schema/annotations.py,sha256=TuQ3vWFnK0160fRmtvsCkHx9qAcRa63MSyERB4x5a98,18197
 deriva_ml/schema/check_schema.py,sha256=6dadLYHPqRex6AYVClmsESI8WhC7-rb-XnGf2G298xw,3609
-deriva_ml/schema/create_schema.py,sha256=IrnSfN0ufS3M31MD8M6ZWyfJidKllLPqDFBUDAIDPY0,12789
+deriva_ml/schema/create_schema.py,sha256=9qK9_8SRQT-DwcEwTGSkhi3j2NaoH5EVgthvV2kO-gg,13042
 deriva_ml/schema/deriva-ml-reference.json,sha256=AEOMIgwKO3dNMMWHb0lxaXyamvfAEbUPh8qw0aAtsUQ,242460
 deriva_ml/schema/policy.json,sha256=5ykB8nnZFl-oCHzlAwppCFKJHWJFIkYognUMVEanfY8,1826
 deriva_ml/schema/table_comments_utils.py,sha256=4flCqnZAaqg_uSZ9I18pNUWAZoLfmMCXbmI5uERY5vM,2007
-deriva_ml-1.14.41.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-deriva_ml-1.14.41.dist-info/METADATA,sha256=WTixw85aw7Qz8eFwBUcCCnf3WtjrLqng9XTc9S5ToZg,1122
-deriva_ml-1.14.41.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-deriva_ml-1.14.41.dist-info/entry_points.txt,sha256=XsHSbfp7S1cKMjHoPUdFIaFcp9lHXHS6CV1zb_MEXkg,463
-deriva_ml-1.14.41.dist-info/top_level.txt,sha256=I1Q1dkH96cRghdsFRVqwpa2M7IqJpR2QPUNNc5-Bnpw,10
-deriva_ml-1.14.41.dist-info/RECORD,,
+deriva_ml-1.14.43.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+deriva_ml-1.14.43.dist-info/METADATA,sha256=ktO6NAE5LHzYsCkS-CEK2OELB_Wqc97_3X7rhjLgrfY,1168
+deriva_ml-1.14.43.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+deriva_ml-1.14.43.dist-info/entry_points.txt,sha256=XsHSbfp7S1cKMjHoPUdFIaFcp9lHXHS6CV1zb_MEXkg,463
+deriva_ml-1.14.43.dist-info/top_level.txt,sha256=I1Q1dkH96cRghdsFRVqwpa2M7IqJpR2QPUNNc5-Bnpw,10
+deriva_ml-1.14.43.dist-info/RECORD,,