deriva-ml 1.8.2__py3-none-any.whl → 1.8.4__py3-none-any.whl

deriva_ml/VERSION.py

@@ -1 +1 @@
-__version__ = "1.8.2"
+__version__ = "1.8.4"

deriva_ml/dataset.py

@@ -6,6 +6,7 @@ accessible via a DerivaML class instance.
 
 """
 
+from __future__ import annotations
 from bdbag.fetch.fetcher import fetch_single_file
 from bdbag import bdbag_api as bdb
 from collections import defaultdict
@@ -37,7 +38,7 @@ from pydantic import (
 import requests
 
 from tempfile import TemporaryDirectory, NamedTemporaryFile
-from typing import Any, Callable, Optional, Iterable, Iterator
+from typing import Any, Callable, Optional, Iterable, Iterator, TYPE_CHECKING
 
 from deriva_ml import DatasetBag
 from .deriva_definitions import ML_SCHEMA, DerivaMLException, MLVocab, Status, RID
@@ -52,6 +53,9 @@ from .dataset_aux_classes import (
     DatasetSpec,
 )
 
+if TYPE_CHECKING:
+    from .deriva_ml_base import DerivaML
+
 
 class Dataset:
     """
@@ -83,29 +87,32 @@ class Dataset:
         else:
             return not list(rid_info.datapath.entities().fetch())[0]["Deleted"]
 
-    def _insert_dataset_version(
+    def _insert_dataset_versions(
         self,
-        dataset_rid: RID,
-        dataset_version: DatasetVersion,
+        dataset_list: list[DatasetSpec],
         description: Optional[str] = "",
         execution_rid: Optional[RID] = None,
     ) -> RID:
         schema_path = self._model.catalog.getPathBuilder().schemas[self._ml_schema]
-        version_path = schema_path.tables["Dataset_Version"]
-        version_rid = version_path.insert(
-            [
-                {
-                    "Dataset": dataset_rid,
-                    "Version": str(dataset_version),
-                    "Description": description,
-                    "Execution": execution_rid,
-                }
-            ]
-        )[0]["RID"]
-        schema_path.tables["Dataset"].update(
-            [{"RID": dataset_rid, "Version": version_rid}]
-        )
-        return version_rid
+
+        # Construct version records for insert
+        version_records = [
+            {
+                "Dataset": dataset.rid,
+                "Version": str(dataset.version),
+                "Description": description,
+                "Execution": execution_rid,
+            }
+            for dataset in dataset_list
+        ]
+
+        # Insert version records and construct entities for updating the dataset version column.
+        version_rids = [
+            {"Version": v["RID"], "RID": v["Dataset"]}
+            for v in schema_path.tables["Dataset_Version"].insert(version_records)
+        ]
+        schema_path.tables["Dataset"].update(version_rids)
+        return version_rids
 
     def _bootstrap_versions(self):
         datasets = [ds["RID"] for ds in self.find_datasets()]
@@ -237,16 +244,20 @@ class Dataset:
         Raises:
           DerivaMLException: if provided RID is not to a dataset_table.
         """
-        for dataset in self._build_dataset_graph(dataset_rid=dataset_rid):
-            version = self.dataset_version(dataset)
-            new_version = version.increment_version(component)
-            self._insert_dataset_version(
-                dataset,
-                new_version,
-                description=description,
-                execution_rid=execution_rid,
+
+        # Find all of the datasets that are reachable from this dataset and determine their new version numbers.
+        related_datasets = list(self._build_dataset_graph(dataset_rid=dataset_rid))
+        version_update_list = [
+            DatasetSpec(
+                rid=ds_rid,
+                version=self.dataset_version(ds_rid).increment_version(component),
             )
-        return self.dataset_version(dataset_rid)
+            for ds_rid in related_datasets
+        ]
+        updated_versions = self._insert_dataset_versions(
+            version_update_list, description=description, execution_rid=execution_rid
+        )
+        return [d.version for d in version_update_list if d.rid == dataset_rid][0]
 
     @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
     def create_dataset(
@@ -323,9 +334,8 @@ class Dataset:
             pb.schemas[self._ml_schema].Dataset_Execution.insert(
                 [{"Dataset": dataset_rid, "Execution": execution_rid}]
             )
-        self._insert_dataset_version(
-            dataset_rid,
-            dataset_version=version,
+        self._insert_dataset_versions(
+            [DatasetSpec(rid=dataset_rid, version=version)],
             execution_rid=execution_rid,
             description="Initial dataset creation.",
         )
@@ -455,7 +465,7 @@ class Dataset:
             dataset_rid: param recurse: If this is a nested dataset_table, list the members of the contained datasets
             dataset_rid: RID:
             recurse:  (Default value = False)
-            limit: If provided, the maxiumum number of members to return for each element type.
+            limit: If provided, the maximum number of members to return for each element type.
 
         Returns:
             Dictionary of entities associated with a specific dataset_table.  Key is the table from which the elements
@@ -697,11 +707,25 @@ class Dataset:
           list of RIDs of nested datasets.
 
         """
-        children = [d["RID"] for d in self.list_dataset_members(dataset_rid)["Dataset"]]
-        if recurse:
-            for child in children.copy():
-                children.extend(self.list_dataset_children(child, recurse=recurse))
-        return children
+        dataset_dataset_path = (
+            self._model.catalog.getPathBuilder()
+            .schemas[self._ml_schema]
+            .tables["Dataset_Dataset"]
+        )
+        nested_datasets = list(dataset_dataset_path.entities().fetch())
+
+        def find_children(rid: RID):
+            children = [
+                child["Nested_Dataset"]
+                for child in nested_datasets
+                if child["Dataset"] == rid
+            ]
+            if recurse:
+                for child in children.copy():
+                    children.extend(find_children(child))
+            return children
+
+        return find_children(dataset_rid)
 
     def _vocabulary_specification(
         self, writer: Callable[[str, str, Table], list[dict[str, Any]]]
@@ -727,20 +751,19 @@ class Dataset:
         ]
 
     def _table_paths(
-        self, dataset: DatasetSpec = None
-    ) -> Iterator[tuple[list[str], list[str], list[Table]]]:
+        self, dataset: DatasetSpec = None, snapshot_catalog: Optional[DerivaML] = None
+    ) -> Iterator[tuple[str, str, Table]]:
 
-        dataset_dataset = self._model.schemas[self._ml_schema].tables["Dataset_Dataset"]
-
-        paths = self._collect_paths(dataset and dataset.rid)
+        paths = self._collect_paths(dataset and dataset.rid, snapshot_catalog)
 
         def source_path(path: tuple[Table, ...]):
+            """Convert a tuple representing a path into a source path component with FK linkage"""
             path = list(path)
             p = [f"{self._model.ml_schema}:Dataset/RID={{Dataset_RID}}"]
             for table in path[1:]:
-                if table == dataset_dataset:
+                if table.name == "Dataset_Dataset":
                     p.append("(RID)=(deriva-ml:Dataset_Dataset:Dataset)")
-                elif table == self.dataset_table:
+                elif table.name == "Dataset":
                     p.append("(Nested_Dataset)=(deriva-ml:Dataset:RID)")
                 elif table.name == "Dataset_Version":
                     p.append(f"(RID)=({self._model.ml_schema}:Dataset_Version:Dataset)")
@@ -751,63 +774,76 @@ class Dataset:
         src_paths = ["/".join(source_path(p)) for p in paths]
         dest_paths = ["/".join([t.name for t in p]) for p in paths]
         target_tables = [p[-1] for p in paths]
-
         return zip(src_paths, dest_paths, target_tables)
 
     def _collect_paths(
         self,
         dataset_rid: Optional[RID] = None,
+        snapshot_catalog: Optional[DerivaML] = None,
         dataset_nesting_depth: Optional[int] = None,
     ) -> set[tuple[Table, ...]]:
 
-        dataset_nesting_depth = (
-            self._dataset_nesting_depth()
-            if dataset_nesting_depth is None
-            else dataset_nesting_depth
-        )
-        dataset_dataset = self._model.schemas[self._ml_schema].tables["Dataset_Dataset"]
-
-        # Figure out which paths we don't need to query for this dataset.  If no dataset is provided, use them all.
-        dataset_elements = (
-            [
-                self._model.name_to_table(e)
-                for e, m in self.list_dataset_members(
+        snapshot_catalog = snapshot_catalog or self
+        dataset_table = snapshot_catalog._model.schemas[self._ml_schema].tables[
+            "Dataset"
+        ]
+        dataset_dataset = snapshot_catalog._model.schemas[self._ml_schema].tables[
+            "Dataset_Dataset"
+        ]
+        dataset_associations = [
+            a
+            for a in self.dataset_table.find_associations()
+            if a.table.schema.name != self._ml_schema
+            or a.table.name == "Dataset_Dataset"
+        ]
+        if dataset_rid:
+            # Get a list of the members of the dataset so we can figure out which tables to query.
+            dataset_elements = [
+                snapshot_catalog._model.name_to_table(e)
+                for e, m in snapshot_catalog.list_dataset_members(
                     dataset_rid=dataset_rid, limit=1
                 ).items()
                 if m
             ]
-            if dataset_rid
-            else self.list_dataset_element_types()
-        )
-
-        dataset_associations = [a.table for a in self.dataset_table.find_associations()]
-        included_associations = [
-            a.table
-            for a in self.dataset_table.find_associations()
-            if a.other_fkeys.pop().pk_table in dataset_elements
-        ]
+            included_associations = [
+                a.table
+                for a in dataset_table.find_associations()
+                if a.other_fkeys.pop().pk_table in dataset_elements
+            ]
+        else:
+            included_associations = dataset_associations
         # Get the paths through the schema and filter out all of dataset paths not used by this dataset.
         paths = {
             tuple(p)
-            for p in self._model._schema_to_paths()
+            for p in snapshot_catalog._model._schema_to_paths()
             if (len(p) == 1)
-            or (p[1] not in dataset_associations)
-            or (p[1] in included_associations)
+            or (p[1] not in dataset_associations)  # Tables in the domain schema
+            or (
+                p[1] in included_associations
+            )  # Tables that include members of the dataset
         }
         # Now get paths for nested datasets
         nested_paths = set()
         if dataset_rid:
-            for c in self.list_dataset_children(dataset_rid=dataset_rid):
-                nested_paths |= self._collect_paths(c)
+            for c in snapshot_catalog.list_dataset_children(dataset_rid=dataset_rid):
+                nested_paths |= self._collect_paths(
+                    c, snapshot_catalog=snapshot_catalog
+                )
         else:
+            # Initialize nesting depth if not already provided.
+            dataset_nesting_depth = (
+                self._dataset_nesting_depth()
+                if dataset_nesting_depth is None
+                else dataset_nesting_depth
+            )
             if dataset_nesting_depth:
                 nested_paths = self._collect_paths(
                     dataset_nesting_depth=dataset_nesting_depth - 1
                 )
         if nested_paths:
             paths |= {
-                tuple([self.dataset_table]),
-                (self.dataset_table, dataset_dataset),
+                tuple([dataset_table]),
+                (dataset_table, dataset_dataset),
             }
         paths |= {(self.dataset_table, dataset_dataset) + p for p in nested_paths}
         return paths
@@ -863,6 +899,7 @@ class Dataset:
         self,
         writer: Callable[[str, str, Table], list[dict[str, Any]]],
         dataset: DatasetSpec,
+        snapshot_catalog: Optional[DerivaML] = None,
     ) -> list[dict[str, Any]]:
         """Output a download/export specification for a dataset_table.  Each element of the dataset_table will be placed in its own dir
         The top level data directory of the resulting BDBag will have one subdirectory for element type. the subdirectory
@@ -902,21 +939,24 @@ class Dataset:
             A dataset_table specification.
         """
         element_spec = []
-        for path in self._table_paths(dataset=dataset):
+        for path in self._table_paths(
+            dataset=dataset, snapshot_catalog=snapshot_catalog
+        ):
             element_spec.extend(writer(*path))
         return self._vocabulary_specification(writer) + element_spec
 
-    @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
-    def download_dataset_bag(
+    def _download_dataset_bag(
         self,
         dataset: DatasetSpec,
         execution_rid: Optional[RID] = None,
+        snapshot_catalog: Optional[DerivaML] = None,
     ) -> DatasetBag:
         """Download a dataset onto the local file system.  Create a MINID for the dataset if one doesn't already exist.
 
         Args:
             dataset: Specification of the dataset to be downloaded.
             execution_rid: Execution RID for the dataset.
+            snapshot_catalog: Snapshot catalog for the dataset version if specified.
 
         Returns:
             Tuple consisting of the path to the dataset, the RID of the dataset that was downloaded and the MINID
@@ -927,16 +967,17 @@ class Dataset:
             and self._model.catalog.resolve_rid(execution_rid).table.name != "Execution"
         ):
             raise DerivaMLException(f"RID {execution_rid} is not an execution")
-        minid = self.get_dataset_minid(dataset)
+        minid = self._get_dataset_minid(dataset, snapshot_catalog=snapshot_catalog)
 
         bag_path = (
             self._materialize_dataset_bag(minid, execution_rid=execution_rid)
             if dataset.materialize
-            else self._download_dataset_bag(minid)
+            else self._download_dataset_minid(minid)
         )
         return DatabaseModel(minid, bag_path).get_dataset()
 
     def _version_snapshot(self, dataset: DatasetSpec) -> str:
+        """Return a catalog with snapshot for the specified dataset version"""
         version_record = [
             h
             for h in self.dataset_history(dataset_rid=dataset.rid)
@@ -944,13 +985,17 @@ class Dataset:
         ][0]
         return f"{self._model.catalog.catalog_id}@{iso_to_snap(version_record.timestamp.isoformat())}"
 
-    def _create_dataset_minid(self, dataset: DatasetSpec) -> str:
+    def _create_dataset_minid(
+        self, dataset: DatasetSpec, snapshot_catalog: Optional[DerivaML] = None
+    ) -> str:
         with TemporaryDirectory() as tmp_dir:
             # Generate a download specification file for the current catalog schema. By default, this spec
             # will generate a minid and place the bag into S3 storage.
             spec_file = f"{tmp_dir}/download_spec.json"
             with open(spec_file, "w", encoding="utf-8") as ds:
-                json.dump(self._generate_dataset_download_spec(dataset), ds)
+                json.dump(
+                    self._generate_dataset_download_spec(dataset, snapshot_catalog), ds
+                )
             try:
                 self._logger.info(
                     f"Downloading dataset minid for catalog: {dataset.rid}@{str(dataset.version)}"
@@ -987,14 +1032,17 @@ class Dataset:
             version_path.update([{"RID": version_rid, "Minid": minid_page_url}])
         return minid_page_url
 
-    @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
-    def get_dataset_minid(
-        self, dataset: DatasetSpec, create: bool = True
+    def _get_dataset_minid(
+        self,
+        dataset: DatasetSpec,
+        snapshot_catalog: Optional[DerivaML] = None,
+        create: bool = True,
     ) -> DatasetMinid:
         """Return a MINID to the specified dataset.  If no version is specified, use the latest.
 
         Args:
             dataset: Specification of the dataset.
+            snapshot_catalog: Snapshot catalog for the dataset version if specified.
             create: Create a new MINID if one doesn't already exist.
 
         Returns:
@@ -1025,12 +1073,12 @@ class Dataset:
                         f"Minid for dataset {dataset.rid} doesn't exist"
                     )
                 self._logger.info("Creating new MINID for dataset %s", dataset.rid)
-                minid_url = self._create_dataset_minid(dataset)
+                minid_url = self._create_dataset_minid(dataset, snapshot_catalog)
             # If provided a MINID, use the MINID metadata to get the checksum and download the bag.
         r = requests.get(minid_url, headers={"accept": "application/json"})
         return DatasetMinid(dataset_version=dataset.version, **r.json())
 
-    def _download_dataset_bag(self, minid: DatasetMinid) -> Path:
+    def _download_dataset_minid(self, minid: DatasetMinid) -> Path:
         """Given a RID to a dataset_table, or a MINID to an existing bag, download the bag file, extract it and validate
         that all the metadata is correct
 
@@ -1097,7 +1145,7 @@ class Dataset:
             return True
 
         # request metadata
-        bag_path = self._download_dataset_bag(minid)
+        bag_path = self._download_dataset_minid(minid)
         bag_dir = bag_path.parent
         validated_check = bag_dir / "validated_check.txt"
 
@@ -1112,7 +1160,9 @@ class Dataset:
         return Path(bag_path)
 
     def _export_outputs(
-        self, dataset: Optional[DatasetSpec] = None
+        self,
+        dataset: Optional[DatasetSpec] = None,
+        snapshot_catalog: Optional[DerivaML] = None,
     ) -> list[dict[str, Any]]:
         """Return and output specification for the datasets in the provided model
 
@@ -1150,9 +1200,13 @@ class Dataset:
                 "source": {"api": "schema", "skip_root_path": True},
                 "destination": {"type": "json", "name": "schema"},
             },
-        ] + self._dataset_specification(writer, dataset)
+        ] + self._dataset_specification(
+            writer, dataset, snapshot_catalog=snapshot_catalog
+        )
 
-    def _processor_params(self, dataset: DatasetSpec) -> list[dict[str, Any]]:
+    def _processor_params(
+        self, dataset: DatasetSpec, snapshot_catalog: Optional[DerivaML] = None
+    ) -> list[dict[str, Any]]:
         """
         Returns:
           a download specification for the datasets in the provided model.
@@ -1178,7 +1232,7 @@ class Dataset:
                 "processor": "json",
                 "processor_params": {"query_path": "/schema", "output_path": "schema"},
             }
-        ] + self._dataset_specification(writer, dataset)
+        ] + self._dataset_specification(writer, dataset, snapshot_catalog)
 
     @staticmethod
     def _download_dataset_element(
@@ -1257,7 +1311,9 @@ class Dataset:
             )
         return exports
 
-    def _generate_dataset_download_spec(self, dataset: DatasetSpec) -> dict[str, Any]:
+    def _generate_dataset_download_spec(
+        self, dataset: DatasetSpec, snapshot_catalog: Optional[DerivaML]
+    ) -> dict[str, Any]:
         """
 
         Returns:
@@ -1315,7 +1371,7 @@ class Dataset:
                         },
                     },
                 ]
-                + self._processor_params(dataset),
+                + self._processor_params(dataset, snapshot_catalog),
             },
         }
 

deriva_ml/deriva_definitions.py

@@ -139,7 +139,6 @@ class FileSpec(BaseModel):
         if url_parts.scheme == "tag":
             return v
         elif not url_parts.scheme:
-            print(v)
             return f'tag://{gethostname()},{date.today()}:file://{v}'
         else:
             raise ValidationError("url is not a file URL")

deriva_ml/deriva_ml_base.py

@@ -15,8 +15,11 @@ import logging
 from datetime import datetime
 import hashlib
 from itertools import chain
+import inspect
 from pathlib import Path
 import requests
+from setuptools_git_versioning import get_latest_file_commit
+import subprocess
 from typing import Optional, Any, Iterable, TYPE_CHECKING
 from deriva.core import (
     ErmrestCatalog,
@@ -35,6 +38,8 @@ from pydantic import validate_call, ConfigDict
 from .execution_configuration import ExecutionConfiguration, Workflow
 from .feature import Feature, FeatureRecord
 from .dataset import Dataset
+from .dataset_aux_classes import DatasetSpec
+from .dataset_bag import DatasetBag
 from .deriva_model import DerivaModel
 from .upload import (
     table_path,
@@ -56,6 +61,18 @@ from .deriva_definitions import (
     FileSpec,
 )
 
+try:
+    from icecream import ic
+except ImportError:  # Graceful fallback if IceCream isn't installed.
+    ic = lambda *a: None if not a else (a[0] if len(a) == 1 else a)  # noqa
+
+
+try:
+    from IPython import get_ipython
+except ImportError:  # Graceful fallback if IPython isn't installed.
+    get_ipython = lambda: None
+
+
 if TYPE_CHECKING:
     from .execution import Execution
 
@@ -132,6 +149,17 @@ class DerivaML(Dataset):
         self.version = model_version
         self.configuration = None
         self._execution: Optional[Execution] = None
+        self._notebook = None
+        try:
+            from IPython import get_ipython
+
+            ipython = get_ipython()
+            # Check if running in Jupyter's ZMQ kernel (used by notebooks)
+            if ipython is not None and "IPKernelApp" in ipython.config:
+                self._notebook = Path(ipython.user_ns.get("__session__"))
+            # Check if running in Jupyter's ZMQ kernel (used by notebooks)
+        except (ImportError, AttributeError):
+            pass
 
         self.domain_schema = self.model.domain_schema
         self.project_name = project_name or self.domain_schema
@@ -705,6 +733,28 @@ class DerivaML(Dataset):
             for v in pb.schemas[table.schema.name].tables[table.name].entities().fetch()
         ]
 
+    @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
+    def download_dataset_bag(
+        self,
+        dataset: DatasetSpec,
+        execution_rid: Optional[RID] = None,
+    ) -> DatasetBag:
+        """Download a dataset onto the local file system.  Create a MINID for the dataset if one doesn't already exist.
+
+        Args:
+            dataset: Specification of the dataset to be downloaded.
+            execution_rid: Execution RID for the dataset.
+
+        Returns:
+            Tuple consisting of the path to the dataset, the RID of the dataset that was downloaded and the MINID
+            for the dataset.
+        """
+        return self._download_dataset_bag(
+            dataset=dataset,
+            execution_rid=execution_rid,
+            snapshot_catalog=DerivaML(self.host_name, self._version_snapshot(dataset)),
+        )
+
     @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
     def download_asset(self, asset_rid: RID, dest_dir: Path) -> Path:
         """Download an asset from a URL and place it in a local directory.
@@ -808,8 +858,10 @@ class DerivaML(Dataset):
             Iterable of the RIDs of the files that were added.
         """
         defined_types = self.list_vocabulary_terms(MLVocab.file_type)
-        if execution_rid and self.resolve_rid(execution_rid).table.name != 'Execution':
-            raise DerivaMLException(f'RID {execution_rid} is not for an execution table.')
+        if execution_rid and self.resolve_rid(execution_rid).table.name != "Execution":
+            raise DerivaMLException(
+                f"RID {execution_rid} is not for an execution table."
+            )
 
         def check_file_type(dtype: str) -> bool:
             for term in defined_types:
@@ -862,18 +914,11 @@ class DerivaML(Dataset):
         self, file_types: Optional[list[str]] = None
     ) -> list[dict[str, Any]]:
         """Return the contents of the file table.  Denormalized file types into the file record."""
-        atable = next(
-            self._model.schemas[self._ml_schema]
-            .tables[MLVocab.dataset_type]
-            .find_associations()
-        ).name
         ml_path = self.pathBuilder.schemas[self._ml_schema]
-        atable_path = ml_path.tables[atable]
         file_path = ml_path.File
         type_path = ml_path.File_File_Type
 
         # Get a list of all the dataset_type values associated with this dataset_table.
-        files = []
         path = file_path.link(type_path)
         path = path.attributes(
             path.File.RID,
@@ -885,10 +930,12 @@ class DerivaML(Dataset):
         )
         file_map = {}
         for f in path.fetch():
-            file_map.setdefault(f['RID'], f | {'File_Types': []})['File_Types'].append(f['File_Type'])
+            file_map.setdefault(f["RID"], f | {"File_Types": []})["File_Types"].append(
+                f["File_Type"]
+            )
 
         # Now get rid of the File_Type key and return the result
-        return [ (f, f.pop('File_Type'))[0] for f in file_map.values()]
+        return [(f, f.pop("File_Type"))[0] for f in file_map.values()]
 
     def list_workflows(self) -> list[Workflow]:
         """Return a list of all the workflows in the catalog."""
@@ -901,6 +948,7 @@ class DerivaML(Dataset):
                 version=w["Version"],
                 description=w["Description"],
                 rid=w["RID"],
+                checksum=w["Checksum"],
             )
             for w in workflow_path.entities().fetch()
         ]
@@ -917,33 +965,18 @@ class DerivaML(Dataset):
         """
 
         # Check to make sure that the workflow is not already in the table. If it's not, add it.
-        def get_checksum(url) -> str:
-            """Get the checksum of a file from a URL."""
-            try:
-                response = requests.get(url)
-                response.raise_for_status()
-            except Exception:
-                raise DerivaMLException(f"Invalid URL: {url}")
-            else:
-                sha256_hash = hashlib.sha256()
-                sha256_hash.update(response.content)
-                checksum = "SHA-256: " + sha256_hash.hexdigest()
-            return checksum
+
+        if workflow_rid := self.lookup_workflow(workflow.url):
+            return workflow_rid
 
         ml_schema_path = self.pathBuilder.schemas[self.ml_schema]
         try:
-            url_column = ml_schema_path.Workflow.URL
-            workflow_record = list(
-                ml_schema_path.Workflow.filter(url_column == workflow.url).entities()
-            )[0]
-            workflow_rid = workflow_record["RID"]
-        except IndexError:
             # Record doesn't exist already
             workflow_record = {
                 "URL": workflow.url,
                 "Name": workflow.name,
                 "Description": workflow.description,
-                "Checksum": get_checksum(workflow.url),
+                "Checksum": workflow.checksum,
                 "Version": workflow.version,
                 MLVocab.workflow_type: self.lookup_term(
                     MLVocab.workflow_type, workflow.workflow_type
@@ -955,6 +988,125 @@ class DerivaML(Dataset):
             raise DerivaMLException(f"Failed to insert workflow. Error: {error}")
         return workflow_rid
 
+    def lookup_workflow(self, url: str) -> Optional[RID]:
+        workflow_path = self.pathBuilder.schemas[self.ml_schema].Workflow
+        try:
+            url_column = workflow_path.URL
+            return list(workflow_path.filter(url_column == url).entities())[0]["RID"]
+        except IndexError:
+            return None
+
+    def create_workflow(
+        self, name: str, workflow_type: str, description: str = "", create: bool = True
+    ) -> RID:
+        """Identify current executing program and return a workflow RID for it
+
+        Determane the notebook of script that is currently being executed. Assume that  this is
+        being executed from a cloned GitHub repository.  Determine the remote repository name for
+        this object.  Then either retrieve an existing workflow for this executable of create
+        a new one.
+
+        Args:
+            name: The name of the workflow.
+            workflow_type: The type of the workflow.
+            description: The description of the workflow.
+            create: Whether or not to create a new workflow.
+        """
+        # Make sure type is correct.
+        self.lookup_term(MLVocab.workflow_type, workflow_type)
+        filename, github_url, is_dirty = self._github_url()
+
+        if is_dirty:
+            self._logger.warning(
+                f"File {filename} has been modified since last commit. Consider commiting before executing"
+            )
+
+        sha256_hash = hashlib.sha256()
+        if self._notebook:
+            # If you are in a notebook, strip out the outputs before computing the checksum.
+            result = subprocess.run(
+                ["nbstripout", "-t", filename],
+                capture_output=True,
+                text=False,
+                check=True,
+            )
+            sha256_hash.update(result.stdout)
+        else:
+            with open(filename, "rb") as f:
+                sha256_hash.update(f.read())
+        checksum = "SHA-256:" + sha256_hash.hexdigest()
+
+        workflow = Workflow(
+            name=name,
+            url=github_url,
+            checksum=checksum,
+            description=description,
+            workflow_type=workflow_type,
+        )
+        return self.add_workflow(workflow) if create else None
+
+    def _github_url(self) -> tuple[str, str, bool]:
+        """Return a GitHUB URL for the latest commit of the script from which this routine is called.
+
+        This routine is used to be called from a script or notebook (e.g. python -m file). It assumes that
+        the file is in a gitHUB repository and commited.  It returns a URL to the last commited version of this
+        file in GitHUB.
+
+        Returns: A tuple with the filename, gethub_url and a boolaen to indicated if uncommited changes
+            have been made to the file.
+
+        """
+
+        # Get the name of the script that is calling this function.
+        if self._notebook:
+            # Try to get the __session__ variable from the user namespace.
+            filename = Path("").absolute().parent / self._notebook
+        else:
+            stack = inspect.stack()
+            if len(stack) > 1:
+                filename = Path(
+                    stack[2].filename
+                )  # Get the caller's filename, which is two up the stack from here.
+            else:
+                raise DerivaMLException(
+                    f"Looking for caller failed"
+                )  # Stack is too shallow
+
+        # Get repo URL from local github repo.
+        try:
+            result = subprocess.run(
+                ["git", "remote", "get-url", "origin"], capture_output=True, text=True
+            )
+            github_url = result.stdout.strip().removesuffix(".git")
+        except subprocess.CalledProcessError:
+            raise DerivaMLException(f"No GIT remote found")
+
+        # Find the root directory for the repository
+        repo_root = filename
+        while repo_root != repo_root.root:
+            if (repo_root / ".git").exists():
+                break
+            else:
+                repo_root = repo_root.parent
+
+        # Now check to see if file has been modified since the last commit.
+        try:
+            result = subprocess.run(
+                ["git", "status", "--porcelain"],
+                capture_output=True,
+                text=True,
+                check=True,
+            )
+            is_dirty = bool(
+                " M " in result.stdout.strip()
+            )  # Returns True if output indicates a modified file
+        except subprocess.CalledProcessError:
+            is_dirty = False  # If Git command fails, assume no changes
+
+        sha = get_latest_file_commit(filename)
+        url = f"{github_url}/blob/{sha}/{filename.relative_to(repo_root)}"
+        return filename, url, is_dirty
+
     # @validate_call
     def create_execution(self, configuration: ExecutionConfiguration) -> "Execution":
         """Create an execution object

deriva_ml/deriva_ml_execute.py

@@ -0,0 +1,104 @@
+from sympy import cxxcode
+
+from  deriva_ml import DerivaML, execution_configuration
+
+def execute(host, catalog, script):
+    workflow_rid = foobar
+    execution_configuration = cxxcode(
+
+    )
+    ml_instance = DerivaML()
+    ml_instance.create_execution(configuration)
+    script
+
+
+from deriva_ml import DerivaML, ExecutionConfiguration, DatasetSpec, RID, DerivaMLException
+import os
+import sys
+import json
+import traceback
+import argparse
+import requests
+from requests.exceptions import HTTPError, ConnectionError
+from deriva.transfer import GenericDownloader
+from deriva.transfer.download import DerivaDownloadError, DerivaDownloadConfigurationError, \
+    DerivaDownloadAuthenticationError, DerivaDownloadAuthorizationError, DerivaDownloadTimeoutError, \
+    DerivaDownloadBaggingError
+from deriva.core import BaseCLI, KeyValuePairArgs, format_credential, format_exception, urlparse
+
+
+class DerivaMLExecCLI(BaseCLI):
+    def __init__(self, description, epilog, **kwargs):
+
+        BaseCLI.__init__(self, description, epilog, **kwargs)
+        self.parser.add_argument("--catalog", default=1, metavar="<1>", help="Catalog number. Default: 1")
+        self.parser.add_argument("--timeout", metavar="<seconds>",
+                                 help="Total number of seconds elapsed before the download is aborted.")
+        self.parser.add_argument("output_dir", metavar="<output dir>", help="Path to an output directory.")
+        self.parser.add_argument("envars", metavar="[key=value key=value ...]",
+                                 nargs=argparse.REMAINDER, action=KeyValuePairArgs, default={},
+                                 help="Variable length of whitespace-delimited key=value pair arguments used for "
+                                      "string interpolation in specific parts of the configuration file. "
+                                      "For example: key1=value1 key2=value2")
+
+    def main(self):
+        try:
+            args = self.parse_cli()
+        except ValueError as e:
+            sys.stderr.write(str(e))
+            return 2
+        if not args.quiet:
+            sys.stderr.write("\n")
+
+        try:
+            try:
+                ml_instance = DerivaML(args.hostname, args.catalog)
+                downloaded = self.execute()
+                sys.stdout.write("\n%s\n" % (json.dumps(downloaded)))
+            except ConnectionError as e:
+                raise DerivaDownloadError("Connection error occurred. %s" % format_exception(e))
+            except HTTPError as e:
+                if e.response.status_code == requests.codes.unauthorized:
+                    raise DerivaDownloadAuthenticationError(
+                        "The requested service requires authentication and a valid login session could "
+                        "not be found for the specified host. Server responded: %s" % e)
+                elif e.response.status_code == requests.codes.forbidden:
+                    raise DerivaDownloadAuthorizationError(
+                        "A requested operation was forbidden. Server responded: %s" % e)
+        except (DerivaDownloadError, DerivaDownloadConfigurationError, DerivaDownloadAuthenticationError,
+                DerivaDownloadAuthorizationError, DerivaDownloadTimeoutError, DerivaDownloadBaggingError) as e:
+            sys.stderr.write(("\n" if not args.quiet else "") + format_exception(e))
+            if args.debug:
+                traceback.print_exc()
+            return 1
+        except:
+            sys.stderr.write("An unexpected error occurred.")
+            traceback.print_exc()
+            return 1
+        finally:
+            if not args.quiet:
+                sys.stderr.write("\n\n")
+        return 0
+
+
+def do_stuff():
+    pass
+
+def main(datasets: list[RID], model: list[RID], hostname: str, catalog_id: str):
+    my_url = DerivaML.github_url()
+    ml_instance = DerivaML(hostname, catalog_id)
+    ml_instance.lookup_workflow(my_url)
+    config = ExecutionConfiguration(
+        datasets=[DatasetSpec(rid=dataset,
+                              version=ml_instance.dataset_version(dataset)) for dataset in datasets],
+        assets=model,
+        workflow= ml_instance.lookup_workflow(my_url)
+    )
+    execution = ml_instance.create_execution(config)
+    with execution as e:
+        do_stuff()
+    execution.upload_execution_outputs()
+
+if __name__ == "__main__":
+    main(datasets, model, hostname, catalog_id)
+if __file__ == matplotlib_inline

deriva_ml/execution.py

@@ -12,6 +12,7 @@ import os
 import shutil
 from datetime import datetime
 from pathlib import Path
+import requests
 from tempfile import NamedTemporaryFile
 from typing import Iterable, Any, Optional
 from deriva.core import format_exception
@@ -28,7 +29,6 @@ from .deriva_definitions import (
 )
 from .deriva_ml_base import DerivaML, FeatureRecord
 from .dataset_aux_classes import DatasetSpec, DatasetVersion, VersionPart
-from .dataset import Dataset
 from .dataset_bag import DatasetBag
 from .execution_configuration import ExecutionConfiguration
 from .execution_environment import get_execution_environment
@@ -51,6 +51,12 @@ except ImportError:  # Graceful fallback if IceCream isn't installed.
     ic = lambda *a: None if not a else (a[0] if len(a) == 1 else a)  # noqa
 
 
+try:
+    from jupyter_server.serverapp import list_running_servers
+except ImportError:
+    list_running_servers = lambda: []
+
+
 class Execution:
     """The Execution class is used to capture the context of an activity within DerivaML.  While these are primarily
     computational, manual processes can be represented by an execution as well.
@@ -100,6 +106,7 @@ class Execution:
         self.configuration = configuration
         self._ml_object = ml_object
         self.start_time = None
+        self.stop_time = None
         self.status = Status.created
         self.uploaded_assets: list[Path] = []
 
@@ -221,8 +228,9 @@ class Execution:
         Returns:
             the location of the unpacked and validated dataset_table bag and the RID of the bag
         """
-        ds = Dataset(self._ml_object.model, cache_dir=self._cache_dir)
-        return ds.download_dataset_bag(dataset, execution_rid=self.execution_rid)
+        return self._ml_object.download_dataset_bag(
+            dataset, execution_rid=self.execution_rid
+        )
 
     @validate_call
     def update_status(self, status: Status, msg: str) -> None:
@@ -243,6 +251,35 @@ class Execution:
             ]
         )
 
+    def _create_notebook_checkpoint(self):
+        """Trigger a checkpoint creation using Jupyter's API."""
+        notebook_name = self._ml_object._notebook
+        servers = list_running_servers()
+        # Look for the server running this notebook.
+        root = Path("").absolute().parent.as_posix()
+        servers = list(list_running_servers())
+        # Jupyterhub seems to handle root_dir differently then server case.
+        server = (
+            servers
+            if len(servers) == 1
+            else [s for s in servers if s["root_dir"] == root]
+        )[0]
+        notebook_url = f"{server['url']}api/contents/{notebook_name}"
+
+        # Get notebook content
+        response = requests.get(
+            notebook_url, headers={"Authorization": f"Token {server['token']}"}
+        )
+        if response.status_code == 200:
+            notebook_content = response.json()["content"]
+            # Execution metadata cannot be in a directory, so map path into filename.
+            checkpoint_path = (
+                self.execution_metadata_path(ExecMetadataVocab.runtime_env.value)
+                / f"{notebook_name.as_posix().replace('/','_')}.checkpoint"
+            )
+            with open(checkpoint_path, "w", encoding="utf-8") as f:
+                json.dump(notebook_content, f)
+
     def execution_start(self) -> None:
         """Start an execution, uploading status to catalog"""
 
@@ -252,11 +289,15 @@ class Execution:
 
     def execution_stop(self) -> None:
         """Finish the execution and update the duration and status of execution."""
-        duration = datetime.now() - self.start_time
+        self.stop_time = datetime.now()
+        duration = self.stop_time - self.start_time
         hours, remainder = divmod(duration.total_seconds(), 3600)
         minutes, seconds = divmod(remainder, 60)
         duration = f"{round(hours, 0)}H {round(minutes, 0)}min {round(seconds, 4)}sec"
 
+        if self._ml_object._notebook:
+            self._create_notebook_checkpoint()
+
         self.update_status(Status.completed, "Algorithm execution ended.")
         self._ml_object.pathBuilder.schemas[self._ml_object.ml_schema].Execution.update(
             [{"RID": self.execution_rid, "Duration": duration}]

deriva_ml/execution_configuration.py

@@ -33,18 +33,18 @@ class Workflow(BaseModel):
     version: Optional[str] = None
     description: Optional[str] = ""
     rid: Optional[RID] = None
+    checksum: Optional[str]
+
 
 
 class ExecutionConfiguration(BaseModel):
     """Define the parameters that are used to configure a specific execution.
 
     Attributes:
-        datasets: List of dataset_table RIDS, MINIDS for datasets to be downloaded prior to execution.  By default,
-                     all  the datasets are materialized. However, if the assets associated with a dataset_table are not
-                     needed, a dictionary that defines the rid and the materialization parameter for the
-                     download_dataset_bag method can be specified, e.g.  datasets=[{'rid': RID, 'materialize': True}].
+        datasets: List of dataset specifications which specify the dataset RID, version and if the dataset
+            should be materialized.
         assets: List of assets to be downloaded prior to execution.  The values must be RIDs in an asset table
-        workflow: A workflow instance.  Must have a name, URI to the workflow instance, and a type.
+        workflow: A RID for a workflow instance.  Must have a name, URI to the workflow instance, and a type.
         description: A description of the execution.  Can use Markdown format.
     """
 

deriva_ml/upload.py

@@ -70,8 +70,11 @@ exec_asset_regex = (
 exec_metadata_dir_regex = (
     exec_dir_regex + r"/execution-metadata/(?P<execution_metadata_type>[-\w]+)"
 )
+
+# May have more than one suffix
 exec_metadata_regex = (
-    exec_metadata_dir_regex + r"/(?P<filename>[-\w]+)[.](?P<file_ext>[a-z0-9]*)$"
+    exec_metadata_dir_regex
+    + r"/(?P<filename>[-\w]+([.][\w]+)*)[.](?P<file_ext>[a-z0-9]*)$"
 )
 feature_dir_regex = exec_dir_regex + r"/feature"
 feature_table_dir_regex = (

{deriva_ml-1.8.2.dist-info → deriva_ml-1.8.4.dist-info}/METADATA

@@ -1,16 +1,18 @@
 Metadata-Version: 2.2
 Name: deriva-ml
-Version: 1.8.2
+Version: 1.8.4
 Summary: Utilities to simplify use of Dervia and Pandas to create reproducable ML pipelines
 Author-email: ISRD <isrd-dev@isi.edu>
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: deriva~=1.7.6
+Requires-Dist: deriva~=1.7.7
 Requires-Dist: pandas
 Requires-Dist: regex~=2024.7.24
 Requires-Dist: pydantic>=2.10.6
 Requires-Dist: semver>3.0.0
+Requires-Dist: setuptools-git-versioning<3,>=2.0
+Requires-Dist: nbstripout
 
 Deriva-ML is a python libary to simplify the process of creating and executing reproducible machine learning workflows
 using a deriva catalog.

{deriva_ml-1.8.2.dist-info → deriva_ml-1.8.4.dist-info}/RECORD

@@ -1,20 +1,21 @@
-deriva_ml/VERSION.py,sha256=d6593s-XBNvVxri9lr2qLUDZQ3Zk3-VXHEwdb4pj8qA,22
+deriva_ml/VERSION.py,sha256=8kdJa8mgK7VES73y02oBbzwoXZCUs42GzbJ4UU-L_3I,22
 deriva_ml/__init__.py,sha256=0PHNB8gRDALLtaffRmU7wCUgWbRHVQZcjuPJxMLNEco,856
 deriva_ml/database_model.py,sha256=uhoyVyd8MQmY8J9ovCH8fjxhZDxxXNkdJyYdeyEGPXA,13898
-deriva_ml/dataset.py,sha256=5STHbjWomTCPl8isdlcDgLk_K9DLCfACajBAreUAXTQ,58272
+deriva_ml/dataset.py,sha256=xC6QPUp4MZcJiEnOEU3NnzoLBL9RcJWtPTyzIQP0Ivw,60666
 deriva_ml/dataset_aux_classes.py,sha256=YxjQnu2kS9kK_f8bGqhmgE6ty9GNeitCxfvReT9vaM0,6537
 deriva_ml/dataset_bag.py,sha256=e6IHv3saZUnZRfl0EjfnlV2NnmPeOagYYv3PuZqS1l0,11501
 deriva_ml/demo_catalog.py,sha256=xQPhFlflqwJskNQrQ-jdBSnGzBm2-aONBgcRxfsdNKM,11045
-deriva_ml/deriva_definitions.py,sha256=MGl29ogCzqrlRilMhSuR5tECo4NSHP4CLbJAXRtPH6E,8914
-deriva_ml/deriva_ml_base.py,sha256=ShDZlG9F4XrGRUcUINT3bb_P_UdvV1FqSnnPsjGTCLU,36443
+deriva_ml/deriva_definitions.py,sha256=pZLPoUxiuJ-uGglmQ6sF9oVXsSUuOnPEqywoec78XNM,8893
+deriva_ml/deriva_ml_base.py,sha256=3iA1OaPU-6Q7ixt87uDmPuHHZ5P-FyHvX0AKfi4tKp0,42224
+deriva_ml/deriva_ml_execute.py,sha256=y_rGjc97eidBuzy-AaQGe93vuTbWbkNkK9rpReqV0IY,4433
 deriva_ml/deriva_model.py,sha256=LV3FjIhIlz13ckZSmu0aOJhT9EVE0-M9oVMudfkxb0g,12004
-deriva_ml/execution.py,sha256=UcXWY1W5Mt_Yzuayd3Pjd-lKzLlMV5QXZFcLvE6Lt0E,28390
-deriva_ml/execution_configuration.py,sha256=nMeaG1qYdIgu4BV5atSUlcL8VZ3O6ohGY5iBhtD_LQ4,3700
+deriva_ml/execution.py,sha256=c7dbk4HvEh7E4BLlBrf_azUxxhRSUmLQa_6G8t8OKVY,29929
+deriva_ml/execution_configuration.py,sha256=bjnZwXN6M7YPy5dFQwoGEBU8YjhQRSe1FW0rL0V9TaM,3422
 deriva_ml/execution_environment.py,sha256=bCRKrCELDbGQDo7_FKfw7e8iMzVjSRZK3baKkqH5-_0,3264
 deriva_ml/feature.py,sha256=7e8WYPCfJSrGxJh9oUTduYSnB5ekybRhXa_0HIigS_w,5459
 deriva_ml/history.py,sha256=qTDLDs8Ow_6r7mDO0gZm0Fg81SWKOAgtCU5pzZoDRgM,2828
 deriva_ml/test_functions.py,sha256=-eqLHjjCQCLBNAr1ofbZekNiCOfMISSACRxT_YHER8I,4396
-deriva_ml/upload.py,sha256=HCOChW6bALW_gt0sWUs_81bNPsb72TNs4o0FQsGSLM4,22222
+deriva_ml/upload.py,sha256=CKtT-gBln3pnAll9TFaiPhFSHC-bzg9oE4ruh_OSOqY,22270
 deriva_ml/build/lib/schema_setup/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deriva_ml/build/lib/schema_setup/alter_annotation.py,sha256=pkwk0WystN69JfAFK4iBJZAZVQKbRs-gN9IFYuS9rfg,1739
 deriva_ml/build/lib/schema_setup/annotation_temp.py,sha256=Euygu8wNklZFUbR6mz-pDWJemlzdsIn9d6j0f6fCfgE,9102
@@ -26,9 +27,9 @@ deriva_ml/schema_setup/annotations.py,sha256=Uogm9YkRtoKSdgfQlICqRywbCATppwBO-Xr
 deriva_ml/schema_setup/create_schema.py,sha256=jwziMWJPbjRgjiRBT-KtidnXI8YNEFO74A9fwfptjHY,10626
 deriva_ml/schema_setup/policy.json,sha256=77sf0Imy6CAQV0_VwwbA56_KROJ05WXsvT-Wjtkk538,1633
 deriva_ml/schema_setup/table_comments_utils.py,sha256=-2_ubEpoH7ViLVb-ZfW9wZbQ26DTKNgjkCABMzGu4i4,2140
-deriva_ml-1.8.2.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-deriva_ml-1.8.2.dist-info/METADATA,sha256=DnSPqOt32ddlxTwuxGo9iL3DSbUKLIiMitRDMbxZcYQ,556
-deriva_ml-1.8.2.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
-deriva_ml-1.8.2.dist-info/entry_points.txt,sha256=ZiOvrYj022x544TQwi018ujeHRRDahNmwJnzn5ThacM,242
-deriva_ml-1.8.2.dist-info/top_level.txt,sha256=I1Q1dkH96cRghdsFRVqwpa2M7IqJpR2QPUNNc5-Bnpw,10
-deriva_ml-1.8.2.dist-info/RECORD,,
+deriva_ml-1.8.4.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+deriva_ml-1.8.4.dist-info/METADATA,sha256=F14U7NvY310NBB4wGp3-OVmAUXvMy_sDNuS1ZmRjwek,631
+deriva_ml-1.8.4.dist-info/WHEEL,sha256=beeZ86-EfXScwlR_HKu4SllMC9wUEj_8Z_4FJ3egI2w,91
+deriva_ml-1.8.4.dist-info/entry_points.txt,sha256=ZiOvrYj022x544TQwi018ujeHRRDahNmwJnzn5ThacM,242
+deriva_ml-1.8.4.dist-info/top_level.txt,sha256=I1Q1dkH96cRghdsFRVqwpa2M7IqJpR2QPUNNc5-Bnpw,10
+deriva_ml-1.8.4.dist-info/RECORD,,

{deriva_ml-1.8.2.dist-info → deriva_ml-1.8.4.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (76.0.0)
+Generator: setuptools (76.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any