deriva-ml 1.12.2__tar.gz → 1.13.0__tar.gz

{deriva_ml-1.12.2/src/deriva_ml.egg-info → deriva_ml-1.13.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deriva-ml
-Version: 1.12.2
+Version: 1.13.0
 Summary: Utilities to simplify use of Dervia and Pandas to create reproducable ML pipelines
 Author-email: ISRD <isrd-dev@isi.edu>
 Requires-Python: >=3.10
@@ -15,6 +15,7 @@ Requires-Dist: semver>3.0.0
 Requires-Dist: setuptools>=64
 Requires-Dist: setuptools-scm<=6.0
 Requires-Dist: nbstripout
+Requires-Dist: papermill
 Dynamic: license-file
 
 # DerivaML

{deriva_ml-1.12.2 → deriva_ml-1.13.0}/pyproject.toml

@@ -22,6 +22,7 @@ dependencies = [
      "setuptools>=64",
     "setuptools-scm<=6.0",
     "nbstripout",
+    "papermill"
 ]
 
 [tool.setuptools.package-data]
@@ -54,6 +55,8 @@ post_commit_hooks = []
 deriva-ml-table-comments-utils = "deriva_ml.schema_setup.table_comments_utils:main"
 deriva-ml-create-schema = "deriva_ml.schema_setup.create_schema:main"
 deriva-ml-alter-annotation = "deriva_ml.schema_setup.alter_annotation:main"
+deriva-ml-run-notebook = "deriva_ml.run_notebook:main"
+
 
 [metadata]
 license = "Apache 2.0"

{deriva_ml-1.12.2 → deriva_ml-1.13.0}/src/deriva_ml/__init__.py

@@ -14,6 +14,7 @@ __all__ = [
     "BuiltinTypes",
     "UploadState",
     "MLVocab",
+    "MLAsset",
     "ExecMetadataVocab",
     "RID",
     "DerivaSystemColumns",
@@ -32,6 +33,7 @@ from .deriva_definitions import (
     RID,
     DerivaMLException,
     MLVocab,
+    MLAsset,
     ExecMetadataVocab,
     DerivaSystemColumns,
 )
@@ -49,4 +51,3 @@ try:
 except PackageNotFoundError:
     # package is not installed
     pass
-

{deriva_ml-1.12.2 → deriva_ml-1.13.0}/src/deriva_ml/dataset.py

@@ -41,7 +41,14 @@ from tempfile import TemporaryDirectory, NamedTemporaryFile
 from typing import Any, Callable, Optional, Iterable, Iterator, TYPE_CHECKING
 
 from deriva_ml import DatasetBag
-from .deriva_definitions import ML_SCHEMA, DerivaMLException, MLVocab, Status, RID
+from .deriva_definitions import (
+    ML_SCHEMA,
+    DerivaMLException,
+    MLVocab,
+    Status,
+    RID,
+    DRY_RUN_RID,
+)
 from .history import iso_to_snap
 from .deriva_model import DerivaModel
 from .database_model import DatabaseModel
@@ -957,7 +964,7 @@ class Dataset:
             for the dataset.
         """
         if (
-            execution_rid
+            execution_rid != DRY_RUN_RID
             and self._model.catalog.resolve_rid(execution_rid).table.name != "Execution"
         ):
             raise DerivaMLException(f"RID {execution_rid} is not an execution")

{deriva_ml-1.12.2 → deriva_ml-1.13.0}/src/deriva_ml/deriva_definitions.py

@@ -21,6 +21,7 @@ from pydantic import (
 from socket import gethostname
 
 ML_SCHEMA = "deriva-ml"
+DRY_RUN_RID = "0000"
 
 # We are going to use schema as a field name and this collides with method in pydantic base class
 warnings.filterwarnings(
@@ -191,6 +192,11 @@ class MLVocab(StrEnum):
     asset_role = "Asset_Role"
 
 
+class MLAsset(StrEnum):
+    execution_metadata = "Execution_Metadata"
+    execution_asset = "Execution_Asset"
+
+
 class ExecMetadataVocab(StrEnum):
     """
     Predefined execution metadata types.

{deriva_ml-1.12.2 → deriva_ml-1.13.0}/src/deriva_ml/deriva_ml_base.py

@@ -14,17 +14,16 @@ import getpass
 import logging
 from datetime import datetime
 from itertools import chain
-import inspect
-import setuptools_scm
 from pathlib import Path
 import requests
-import subprocess
+
 from typing import Optional, Any, Iterable, TYPE_CHECKING
+
 from deriva.core import (
     get_credential,
     urlquote,
-    DEFAULT_SESSION_CONFIG,
     format_exception,
+    DEFAULT_SESSION_CONFIG,
 )
 import deriva.core.datapath as datapath
 from deriva.core.datapath import DataPathException
@@ -33,7 +32,6 @@ from deriva.core.ermrest_catalog import ResolveRidResult
 from deriva.core.ermrest_model import Key, Table
 from deriva.core.utils.globus_auth_utils import GlobusNativeLogin
 from pydantic import validate_call, ConfigDict
-from requests import RequestException
 
 from .execution_configuration import ExecutionConfiguration, Workflow
 from .feature import Feature, FeatureRecord
@@ -60,33 +58,6 @@ except ImportError:  # Graceful fallback if IceCream isn't installed.
     ic = lambda *a: None if not a else (a[0] if len(a) == 1 else a)  # noqa
 
 
-try:
-    from IPython import get_ipython
-except ImportError:  # Graceful fallback if IPython isn't installed.
-
-    def get_ipython():
-        """Dummy routine in case you are not running in IPython."""
-        return None
-
-
-try:
-    from jupyter_server.serverapp import list_running_servers
-except ImportError:
-
-    def list_running_servers():
-        """Dummy routine in case you are not running in Jupyter."""
-        return []
-
-
-try:
-    from ipykernel import get_connection_file
-except ImportError:
-
-    def get_connection_file():
-        """Dummy routine in case you are not running in Jupyter."""
-        return ""
-
-
 if TYPE_CHECKING:
     from .execution import Execution
 
@@ -165,7 +136,6 @@ class DerivaML(Dataset):
         self.version = model_version
         self.configuration = None
         self._execution: Optional[Execution] = None
-        self.executable_path, self._is_notebook = self._get_python_script()
         self.domain_schema = self.model.domain_schema
         self.project_name = project_name or self.domain_schema
         self.start_time = datetime.now()
@@ -192,102 +162,6 @@ class DerivaML(Dataset):
         except (AttributeError, requests.HTTPError):
             pass
 
-    def _check_nbstrip_status(self) -> None:
-        """Check to see if nbstrip is installed"""
-        try:
-            if subprocess.run(
-                ["nbstripout", "--is-installed"],
-                check=False,
-                capture_output=True,
-            ).returncode:
-                self._logger.warning(
-                    "nbstripout is not installed in repository. Please run nbstripout --install"
-                )
-        except subprocess.CalledProcessError:
-            self._logger.error("nbstripout is not found.")
-
-    @staticmethod
-    def _get_notebook_session() -> tuple[dict[str, Any] | None, dict[str, Any] | None]:
-        """Return the absolute path of the current notebook."""
-        # Get the kernel's connection file and extract the kernel ID
-        try:
-            if not (connection_file := Path(get_connection_file()).name):
-                return None, None
-        except RuntimeError:
-            return None, None
-
-        kernel_id = connection_file.split("-", 1)[1].split(".")[0]
-
-        # Look through the running server sessions to find the matching kernel ID
-        for server in list_running_servers():
-            try:
-                # If a token is required for authentication, include it in headers
-                token = server.get("token", "")
-                headers = {}
-                if token:
-                    headers["Authorization"] = f"token {token}"
-
-                try:
-                    sessions_url = server["url"] + "api/sessions"
-                    response = requests.get(sessions_url, headers=headers)
-                    response.raise_for_status()
-                    sessions = response.json()
-                except RequestException as e:
-                    raise e
-                for sess in sessions:
-                    if sess["kernel"]["id"] == kernel_id:
-                        return server, sess
-            except Exception as _e:
-                # Ignore servers we can't connect to.
-                pass
-        return None, None
-
-    def _get_notebook_path(self) -> Path | None:
-        """Return the absolute path of the current notebook."""
-
-        server, session = DerivaML._get_notebook_session()
-        if server and session:
-            self._check_nbstrip_status()
-            relative_path = session["notebook"]["path"]
-            # Join the notebook directory with the relative path
-            return Path(server["root_dir"]) / relative_path
-        else:
-            return None
-
-    def _get_python_script(self) -> tuple[Path, bool]:
-        """Return the path to the currently executing script"""
-        is_notebook = False
-        if filename := self._get_notebook_path():
-            is_notebook = True
-        else:
-            stack = inspect.stack()
-            # Get the caller's filename, which is two up the stack from here.
-            if len(stack) > 1:
-                filename = Path(stack[2].filename)
-                if not filename.exists():
-                    # Begin called from command line interpreter.
-                    filename = "REPL"
-                # Get the caller's filename, which is two up the stack from here.
-            else:
-                raise DerivaMLException(
-                    "Looking for caller failed"
-                )  # Stack is too shallow
-        return filename, is_notebook
-
-    def _get_git_root(self):
-        try:
-            result = subprocess.run(
-                ["git", "rev-parse", "--show-toplevel"],
-                cwd=self.executable_path.parent,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.DEVNULL,
-                text=True,
-                check=True,
-            )
-            return result.stdout.strip()
-        except subprocess.CalledProcessError:
-            return None  # Not in a git repository
-
     @staticmethod
     def _get_session_config():
         """ """
@@ -311,10 +185,6 @@ class DerivaML(Dataset):
         """Get a new instance of a pathBuilder object."""
         return self.catalog.getPathBuilder()
 
-    def get_version(self) -> str:
-        """Return the version number of the executable"""
-        return setuptools_scm.get_version(root=self._get_git_root())
-
     @property
     def domain_path(self):
         """Get a new instance of a pathBuilder object to the domain schema"""
@@ -1117,105 +987,7 @@ class DerivaML(Dataset):
         # Make sure type is correct.
         self.lookup_term(MLVocab.workflow_type, workflow_type)
 
-        try:
-            subprocess.run(
-                "git rev-parse --is-inside-work-tree",
-                capture_output=True,
-                text=True,
-                shell=True,
-                check=True,
-            )
-        except subprocess.CalledProcessError:
-            raise DerivaMLException("Not executing in a Git repository.")
-
-        github_url, is_dirty = self._github_url()
-
-        if is_dirty:
-            self._logger.warning(
-                f"File {self.executable_path} has been modified since last commit. Consider commiting before executing"
-            )
-
-        # If you are in a notebook, strip out the outputs before computing the checksum.
-        cmd = (
-            f"nbstripout {self.executable_path} | git hash-object --stdin"
-            if self._is_notebook
-            else f"git hash-object {self.executable_path}"
-        )
-        checksum = (
-            subprocess.run(
-                cmd,
-                capture_output=True,
-                text=True,
-                check=False,
-                shell=True,
-            ).stdout.strip()
-            if self.executable_path != "REPL"
-            else "1"
-        )
-
-        return Workflow(
-            name=name,
-            url=github_url,
-            checksum=checksum,
-            description=description,
-            workflow_type=workflow_type,
-        )
-
-    def _github_url(self) -> tuple[str, bool]:
-        """Return a GitHUB URL for the latest commit of the script from which this routine is called.
-
-        This routine is used to be called from a script or notebook (e.g. python -m file). It assumes that
-        the file is in a gitHUB repository and commited.  It returns a URL to the last commited version of this
-        file in GitHUB.
-
-        Returns: A tuple with the gethub_url and a boolean to indicated if uncommited changes
-            have been made to the file.
-
-        """
-
-        # Get repo URL from local gitHub repo.
-        if self.executable_path == "REPL":
-            return "REPL", True
-        try:
-            result = subprocess.run(
-                ["git", "remote", "get-url", "origin"],
-                capture_output=True,
-                text=True,
-                cwd=self.executable_path.parent,
-            )
-            github_url = result.stdout.strip().removesuffix(".git")
-        except subprocess.CalledProcessError:
-            raise DerivaMLException("No GIT remote found")
-
-        # Find the root directory for the repository
-        repo_root = self._get_git_root()
-
-        # Now check to see if file has been modified since the last commit.
-        try:
-            result = subprocess.run(
-                ["git", "status", "--porcelain"],
-                cwd=self.executable_path.parent,
-                capture_output=True,
-                text=True,
-                check=True,
-            )
-            is_dirty = bool(
-                "M " in result.stdout.strip()
-            )  # Returns True if output indicates a modified file
-        except subprocess.CalledProcessError:
-            is_dirty = False  # If Git command fails, assume no changes
-
-        """Get SHA-1 hash of latest commit of the file in the repository"""
-        result = subprocess.run(
-            ["git", "log", "-n", "1", "--pretty=format:%H--", self.executable_path],
-            cwd=self.executable_path.parent,
-            capture_output=True,
-            text=True,
-            check=True,
-        )
-        sha = result.stdout.strip()
-        url = f"{github_url}/blob/{sha}/{self.executable_path.relative_to(repo_root)}"
-        return url, is_dirty
+        return Workflow.create_workflow(name, workflow_type, description)
 
     # @validate_call
     def create_execution(
@@ -1259,6 +1031,15 @@ class DerivaML(Dataset):
             exec_rid=execution_rid,
             file_name="configuration.json",
             asset_table=self.model.name_to_table("Execution_Metadata"),
+            metadata={},
         )
-        configuration = ExecutionConfiguration.load_configuration(cfile)
+
+        if cfile.exists():
+            configuration = ExecutionConfiguration.load_configuration(cfile)
+        else:
+            execution = self.retrieve_rid(execution_rid)
+            configuration = ExecutionConfiguration(
+                workflow=execution["Workflow"],
+                description=execution["Description"],
+            )
         return Execution(configuration, self, reload=execution_rid)

{deriva_ml-1.12.2 → deriva_ml-1.13.0}/src/deriva_ml/execution.py

@@ -5,21 +5,31 @@ This module defined the Execution class which is used to interact with the state
 from __future__ import annotations
 
 from collections import defaultdict
+from datetime import datetime
 import json
 import logging
 import os
-import shutil
-from datetime import datetime
 from pathlib import Path
-from typing import Iterable, Any, Optional
 
-from deriva.core import format_exception
 from pydantic import validate_call, ConfigDict
+import regex as re
 import sys
-from deriva.core.hatrac_store import HatracStore
+import shutil
+from typing import Iterable, Any, Optional
 
+from deriva.core import format_exception
+from deriva.core.datapath import DataPathException
+from deriva.core.hatrac_store import HatracStore
 from .deriva_definitions import ExecMetadataVocab
-from .deriva_definitions import RID, Status, FileUploadState, DerivaMLException, MLVocab
+from .deriva_definitions import (
+    RID,
+    Status,
+    FileUploadState,
+    DerivaMLException,
+    MLVocab,
+    MLAsset,
+    DRY_RUN_RID,
+)
 from .deriva_ml_base import DerivaML, FeatureRecord
 from .dataset_aux_classes import DatasetSpec, DatasetVersion, VersionPart
 from .dataset_bag import DatasetBag
@@ -45,11 +55,14 @@ except ImportError:  # Graceful fallback if IceCream isn't installed.
 
 
 try:
-    from jupyter_server.serverapp import list_running_servers
+    from IPython.display import display, Markdown
 except ImportError:
 
-    def list_running_servers():
-        return []
+    def display(s):
+        print(s)
+
+    def Markdown(s):
+        return s
 
 
 class AssetFilePath(type(Path())):
@@ -167,7 +180,7 @@ class Execution:
             self.workflow_rid = (
                 self._ml_object.add_workflow(self.configuration.workflow)
                 if not self._dry_run
-                else "0000"
+                else DRY_RUN_RID
             )
         else:
             self.workflow_rid = self.configuration.workflow
@@ -195,10 +208,10 @@ class Execution:
         schema_path = self._ml_object.pathBuilder.schemas[self._ml_object.ml_schema]
         if reload:
             self.execution_rid = reload
-            if self.execution_rid == "0000":
+            if self.execution_rid == DRY_RUN_RID:
                 self._dry_run = True
         elif self._dry_run:
-            self.execution_rid = "0000"
+            self.execution_rid = DRY_RUN_RID
         else:
             self.execution_rid = schema_path.Execution.insert(
                 [
@@ -209,6 +222,15 @@ class Execution:
                 ]
             )[0]["RID"]
 
+        if (
+            isinstance(self.configuration.workflow, Workflow)
+            and self.configuration.workflow.is_notebook
+        ):
+            # Put execution_rid into cell output so we can find it later.
+            display(
+                Markdown(f"Execution RID: {self._ml_object.cite(self.execution_rid)}")
+            )
+
         # Create a directory for execution rid so we can recover state in case of a crash.
         execution_root(prefix=self._ml_object.working_dir, exec_rid=self.execution_rid)
         self._initialize_execution(reload)
@@ -272,16 +294,20 @@ class Execution:
             )
 
         # Save configuration details for later upload
-        cfile = self.asset_file_path(
-            asset_name="Execution_Metadata",
-            file_name="configuration.json",
-            asset_types=ExecMetadataVocab.execution_config.value,
-        )
-        with open(cfile.as_posix(), "w", encoding="utf-8") as config_file:
-            json.dump(self.configuration.model_dump(), config_file)
+        if not reload:
+            cfile = self.asset_file_path(
+                asset_name=MLAsset.execution_metadata,
+                file_name="configuration.json",
+                asset_types=ExecMetadataVocab.execution_config.value,
+            )
+            with open(cfile.as_posix(), "w", encoding="utf-8") as config_file:
+                json.dump(self.configuration.model_dump(), config_file)
+
+            for parameter_file in self.configuration.parameters:
+                self.asset_file_path(MLAsset.execution_assets, parameter_file)
 
-        # save runtime env
-        self._save_runtime_environment()
+            # save runtime env
+            self._save_runtime_environment()
 
         self.start_time = datetime.now()
         self.update_status(Status.pending, "Initialize status finished.")
@@ -625,9 +651,20 @@ class Execution:
         with open(feature_file, "r") as feature_values:
             entities = [json.loads(line.strip()) for line in feature_values]
         # Update the asset columns in the feature and add to the catalog.
-        self._ml_object.domain_path.tables[feature_table].insert(
-            [map_path(e) for e in entities]
-        )
+        try:
+            self._ml_object.domain_path.tables[feature_table].insert(
+                [map_path(e) for e in entities]
+            )
+        except DataPathException as e:
+            if re.match(
+                rf'DETAIL: +Key +\("Execution", +"{target_table}", +"Feature_Name"\)=\(.*\) already exists',
+                e.message,
+            ):
+                self._logger.info(
+                    f"Skipping reload of feature values for {feature_table}"
+                )
+            else:
+                raise e
 
     def _update_asset_execution_table(
         self,
@@ -652,16 +689,27 @@ class Execution:
             asset_exe = self._model.find_association(asset_table_name, "Execution")
             asset_exe_path = pb.schemas[asset_exe.schema.name].tables[asset_exe.name]
 
-            asset_exe_path.insert(
-                [
-                    {
-                        asset_table_name: asset_path.asset_rid,
-                        "Execution": self.execution_rid,
-                        "Asset_Role": asset_role,
-                    }
-                    for asset_path in asset_list
-                ]
-            )
+            try:
+                asset_exe_path.insert(
+                    [
+                        {
+                            asset_table_name: asset_path.asset_rid,
+                            "Execution": self.execution_rid,
+                            "Asset_Role": asset_role,
+                        }
+                        for asset_path in asset_list
+                    ]
+                )
+            except DataPathException as e:
+                if re.match(
+                    rf'DETAIL: +Key +\("{asset_table_name}", +"Execution"\)=\(.*\) already exists',
+                    e.message,
+                ):
+                    self._logger.info(
+                        f"Skipping reload of execution assocations for {asset_table_name}"
+                    )
+                else:
+                    raise e
 
             # Now add in the type names via the asset_asset_type association table.
             # Get the list of types for each file in the asset.
@@ -687,19 +735,30 @@ class Execution:
             type_path = pb.schemas[asset_asset_type.schema.name].tables[
                 asset_asset_type.name
             ]
-            type_path.insert(
-                [
-                    {asset_table_name: asset.asset_rid, "Asset_Type": t}
-                    for asset in asset_list
-                    for t in asset_type_map[asset.file_name]
-                ]
-            )
+            try:
+                type_path.insert(
+                    [
+                        {asset_table_name: asset.asset_rid, "Asset_Type": t}
+                        for asset in asset_list
+                        for t in asset_type_map[asset.file_name]
+                    ]
+                )
+            except DataPathException as e:
+                if re.match(
+                    rf'DETAIL: +Key +\("{asset_table_name}", +"Asset_Type"\)=\(.*\) already exists',
+                    e.message,
+                ):
+                    self._logger.info(
+                        f"Skipping reload of execution asset types for {asset_table_name}"
+                    )
+                else:
+                    raise e
 
     @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
     def asset_file_path(
         self,
         asset_name: str,
-        file_name: str,
+        file_name: str | Path,
         asset_types: Optional[list[str] | str] = None,
         copy_file=False,
         **kwargs,