deriva-ml 1.14.38__tar.gz → 1.14.40__tar.gz

{deriva_ml-1.14.38 → deriva_ml-1.14.40}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deriva-ml
-Version: 1.14.38
+Version: 1.14.40
 Summary: Utilities to simplify use of Dervia and Pandas to create reproducable ML pipelines
 Author-email: ISRD <isrd-dev@isi.edu>
 Requires-Python: >=3.10

{deriva_ml-1.14.38 → deriva_ml-1.14.40}/src/deriva_ml/core/base.py

@@ -134,8 +134,10 @@ class DerivaML(Dataset):
                 this argument must be provided a value.
             ml_schema: Schema name for ML schema. Used if you have a non-standard configuration of deriva-ml.
             project_name: Project name. Defaults to name of domain schema.
-            cache_dir: Directory path for caching data downloaded from the Deriva server as bdbag.
-            working_dir: Directory path for storing data used by or generated by any computations.
+            cache_dir: Directory path for caching data downloaded from the Deriva server as bdbag. If not provided,
+                will default to working_dir.
+            working_dir: Directory path for storing data used by or generated by any computations. If no value is
+                provided, will default to  ${HOME}/deriva_ml
             use_minid: Use the MINID service when downloading dataset bags.
         """
         # Get or use provided credentials for server access

{deriva_ml-1.14.38 → deriva_ml-1.14.40}/src/deriva_ml/execution/workflow.py

@@ -121,6 +121,7 @@ class Workflow(BaseModel):
         if "DERIVA_ML_WORKFLOW_URL" in os.environ:
             self.url = os.environ["DERIVA_ML_WORKFLOW_URL"]
             self.checksum = os.environ["DERIVA_ML_WORKFLOW_CHECKSUM"]
+            self.git_root = Workflow._get_git_root(Path(os.environ["DERIVA_ML_NOTEBOOK_PATHL"]))
             self.is_notebook = True
 
         if not self.url:
@@ -320,6 +321,11 @@ class Workflow(BaseModel):
             if not (filename.exists() or Workflow._in_repl()):
                 # Being called from the command line interpreter.
                 filename = Path.cwd() / Path("REPL")
+            # Get the caller's filename, which is two up the stack from here.
+            elif "PYTEST_CURRENT_TEST" in os.environ:
+                filename = Path.cwd() / Path("pytest")
+            else:
+                raise DerivaMLException("Looking for caller failed")  # Stack is too shallow
         return filename, is_notebook
 
     @staticmethod

{deriva_ml-1.14.38 → deriva_ml-1.14.40}/src/deriva_ml/install_kernel.py

@@ -1,4 +1,3 @@
-# your_pkg/install_kernel.py
 import re
 import sys
 from argparse import ArgumentParser

{deriva_ml-1.14.38 → deriva_ml-1.14.40}/src/deriva_ml/model/database.py

@@ -326,7 +326,7 @@ class DatabaseModel(DerivaModel, metaclass=DatabaseModelMeta):
         except ValueError:
             tname = table
             for sname in [self.domain_schema, self.ml_schema, "WWW"]:  # Be careful of File table.
-                if table in self.model.schemas[sname].tables:
+                if sname in self.model.schemas and table in self.model.schemas[sname].tables:
                     break
         try:
             _ = self.model.schemas[sname].tables[tname]

{deriva_ml-1.14.38 → deriva_ml-1.14.40}/src/deriva_ml/run_notebook.py

@@ -119,6 +119,7 @@ class DerivaMLRunNotebookCLI(BaseCLI):
         url, checksum = Workflow.get_url_and_checksum(Path(notebook_file))
         os.environ["DERIVA_ML_WORKFLOW_URL"] = url
         os.environ["DERIVA_ML_WORKFLOW_CHECKSUM"] = checksum
+        os.environ["DERIVA_ML_NOTEBOOK_PATH"] = notebook_file
 
         with tempfile.TemporaryDirectory() as tmpdirname:
             notebook_output = Path(tmpdirname) / Path(notebook_file).name

{deriva_ml-1.14.38 → deriva_ml-1.14.40}/src/deriva_ml.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deriva-ml
-Version: 1.14.38
+Version: 1.14.40
 Summary: Utilities to simplify use of Dervia and Pandas to create reproducable ML pipelines
 Author-email: ISRD <isrd-dev@isi.edu>
 Requires-Python: >=3.10

{deriva_ml-1.14.38 → deriva_ml-1.14.40}/tests/dataset/test_download.py

@@ -1,7 +1,7 @@
 from pathlib import Path
 
 # Local imports
-from deriva_ml import DatasetSpec, DerivaML, MLVocab
+from deriva_ml import DatasetSpec, DerivaML, MLVocab, TableDefinition, VersionPart
 from deriva_ml.demo_catalog import DatasetDescription
 from tests.test_utils import MLDatasetCatalog
 
@@ -98,7 +98,6 @@ class TestDatasetDownload:
 
         self.compare_datasets(ml_instance, dataset_test, dataset_spec)
 
-
     def test_dataset_download_recurse(self, dataset_test, tmp_path):
         hostname = dataset_test.catalog.hostname
         catalog_id = dataset_test.catalog.catalog_id
@@ -147,3 +146,28 @@ class TestDatasetDownload:
 
         self.compare_datasets(ml_instance, dataset_test, current_spec)
         self.compare_datasets(ml_instance, dataset_test, new_spec)
+
+    def test_dataset_download_schemas(self, dataset_test, tmp_path):
+        hostname = dataset_test.catalog.hostname
+        catalog_id = dataset_test.catalog.catalog_id
+        ml_instance = DerivaML(hostname, catalog_id, working_dir=tmp_path, use_minid=False)
+        dataset_description = dataset_test.dataset_description
+
+        current_version = ml_instance.dataset_version(dataset_description.rid)
+        current_spec = DatasetSpec(rid=dataset_description.rid, version=current_version)
+        ml_instance.create_table(
+            TableDefinition(
+                name="NewTable",
+                column_defs=[],
+            )
+        )
+        new_version = ml_instance.increment_dataset_version(
+            dataset_rid=dataset_description.rid, component=VersionPart.minor
+        )
+        new_spec = DatasetSpec(rid=dataset_description.rid, version=new_version)
+
+        current_bag = ml_instance.download_dataset_bag(current_spec)
+        new_bag = ml_instance.download_dataset_bag(new_spec)
+
+        assert "NewTable" in new_bag.model.schemas[ml_instance.domain_schema].tables
+        assert "NewTable" not in current_bag.model.schemas[ml_instance.domain_schema].tables

{deriva_ml-1.14.38 → deriva_ml-1.14.40}/tests/execution/test_execution.py

@@ -7,6 +7,8 @@ from pathlib import Path
 from tempfile import TemporaryDirectory
 
 from deriva_ml import (
+    BuiltinTypes,
+    ColumnDefinition,
     DatasetSpec,
     DerivaML,
     ExecAssetType,
@@ -47,7 +49,7 @@ class TestWorkflow:
         workflow_url = workflows[0]["URL"]
 
         workflow_rid = ml_instance.lookup_workflow(workflow_url)
-
+        print(f"Workflow url: {workflow_url}")
         assert workflow_url.endswith("workflow-test.py")
 
         # Make sure that workflow is not duplicated if created again.
@@ -186,10 +188,48 @@ class TestExecution:
         # Create manual execution
         test_execution = ml_instance.create_execution(config)
         with test_execution.execute() as execution:
+            assert execution.asset_paths["Execution_Asset"][0].asset_types[0] == "Model_File"
             assert 1 == len(execution.asset_paths)
             assert 1 == len(execution.datasets)
             assert execution.datasets[0].dataset_rid == dataset_rid
 
+    def test_download_asset(self, test_ml, tmp_path):
+        ml_instance = test_ml
+
+        # Create a workflow
+        ml_instance.add_term(vc.asset_type, "Test Model", description="Model for our Test workflow")
+        ml_instance.add_term(vc.workflow_type, "Test Workflow", description="A ML Workflow that uses Deriva ML API")
+        api_workflow = ml_instance.create_workflow(
+            name="Test Workflow One",
+            workflow_type="Test Workflow",
+            description="A test operation",
+        )
+        ml_instance.create_asset(
+            "BarAsset",
+            column_defs=[ColumnDefinition(name="foo", type=BuiltinTypes.int4)],
+        )
+        manual_execution = ml_instance.create_execution(
+            ExecutionConfiguration(description="Sample Execution", workflow=api_workflow)
+        )
+
+        with manual_execution.execute() as execution:
+            model_file = execution.asset_file_path(
+                "BarAsset", "API_Model/modelfile.txt", asset_types=ExecAssetType.model_file, foo=23
+            )
+            print(model_file)
+            with model_file.open("w") as fp:
+                fp.write("My model")
+            # Now upload the file and retrieve the RID of the new asset from the returned results.
+        uploaded_assets = manual_execution.upload_execution_outputs()
+        print(uploaded_assets)
+        assert 1 == len(uploaded_assets["deriva-ml/Execution_Asset"])
+
+        file = manual_execution.download_asset(asset_rid, tmpdir, update_catalog=False)
+        assert file.name == "modelfile.txt"
+
+        assert "BarAsset" in [a.name for a in ml_instance.model.find_assets()]
+        assert ml_instance.model.asset_metadata("BarAsset") == {"foo"}
+
     @staticmethod
     def create_execution_asset(ml_instance: DerivaML, api_workflow):
         manual_execution = ml_instance.create_execution(