deriva-ml 1.8.10__py3-none-any.whl → 1.8.11__py3-none-any.whl

deriva_ml/deriva_ml_base.py

@@ -19,7 +19,6 @@ import setuptools_scm
 from pathlib import Path
 import requests
 import subprocess
-import shutil
 from typing import Optional, Any, Iterable, TYPE_CHECKING
 from deriva.core import (
     get_credential,
@@ -34,6 +33,7 @@ from deriva.core.ermrest_catalog import ResolveRidResult
 from deriva.core.ermrest_model import Key, Table
 from deriva.core.hatrac_store import HatracStore
 from pydantic import validate_call, ConfigDict
+from requests import RequestException
 
 from .execution_configuration import ExecutionConfiguration, Workflow
 from .feature import Feature, FeatureRecord
@@ -72,6 +72,15 @@ try:
 except ImportError:  # Graceful fallback if IPython isn't installed.
     get_ipython = lambda: None
 
+try:
+    from jupyter_server.serverapp import list_running_servers
+except ImportError:
+    list_running_servers = lambda: []
+
+try:
+    from ipykernel import get_connection_file
+except ImportError:
+    get_connection_file = lambda: ""
 
 if TYPE_CHECKING:
     from .execution import Execution
@@ -151,8 +160,7 @@ class DerivaML(Dataset):
         self.version = model_version
         self.configuration = None
         self._execution: Optional[Execution] = None
-        self._script_path, self._is_notebook = self._get_python_script()
-        self._notebook = self._get_python_notebook()
+        self.executable_path, self._is_notebook = self._get_python_script()
         self.domain_schema = self.model.domain_schema
         self.project_name = project_name or self.domain_schema
         self.start_time = datetime.now()
@@ -179,38 +187,77 @@ class DerivaML(Dataset):
         except (AttributeError, requests.HTTPError):
             pass
 
-    def _get_python_notebook(self) -> Path | None:
+    def _check_nbstrip_status(self) -> None:
         """Figure out if you are running in a Jupyter notebook
 
         Returns:
             A Path to the notebook file that is currently being executed.
         """
-        notebook = None
         try:
-            ipython = get_ipython()
-            # Check if running in Jupyter's ZMQ kernel (used by notebooks)
-            if ipython is not None and "IPKernelApp" in ipython.config:
-                notebook = Path(ipython.user_ns.get("__session__"))
-                # Check if running in Jupyter's ZMQ kernel (used by notebooks)
+            if subprocess.run(
+                ["nbstripout", "--is-installed"],
+                check=False,
+                capture_output=True,
+            ).returncode:
+                self._logger.warning(
+                    "nbstripout is not installed in repository. Please run nbstripout --install"
+                )
+        except subprocess.CalledProcessError:
+            self._logger.error("nbstripout is not found.")
+
+    def _get_notebook_session(
+        self,
+    ) -> tuple[dict[str, Any] | None, dict[str, Any] | None]:
+        """Return the absolute path of the current notebook."""
+        # Get the kernel's connection file and extract the kernel ID
+        try:
+            if not (connection_file := Path(get_connection_file()).name):
+                return None, None
+        except RuntimeError:
+            return None, None
+
+        kernel_id = connection_file.split("-", 1)[1].split(".")[0]
+
+        # Look through the running server sessions to find the matching kernel ID
+        for server in list_running_servers():
+            try:
+                # If a token is required for authentication, include it in headers
+                token = server.get("token", "")
+                headers = {}
+                if token:
+                    headers["Authorization"] = f"token {token}"
+
                 try:
-                    if subprocess.run(
-                        [shutil.which("nbstripout"), "--is-installed"],
-                        check=False,
-                        capture_output=True,
-                    ).returncode:
-                        self._logger.warning(
-                            "nbstripout is not installed in repository. Please run nbstripout --install"
-                        )
-                except subprocess.CalledProcessError:
-                    self._logger.error("nbstripout is not found.")
-        except (ImportError, AttributeError):
-            pass
-        return notebook
+                    sessions_url = server["url"] + "api/sessions"
+                    response = requests.get(sessions_url, headers=headers)
+                    response.raise_for_status()
+                    sessions = response.json()
+                except RequestException as e:
+                    raise e
+                for sess in sessions:
+                    if sess["kernel"]["id"] == kernel_id:
+                        return server, sess
+            except Exception as _e:
+                # Ignore servers we can't connect to.
+                pass
+        return None, None
+
+    def _get_notebook_path(self) -> Path | None:
+        """Return the absolute path of the current notebook."""
+
+        server, session = self._get_notebook_session()
+        if server and session:
+            self._check_nbstrip_status()
+            relative_path = session["notebook"]["path"]
+            # Join the notebook directory with the relative path
+            return Path(server["root_dir"]) / relative_path
+        else:
+            return None
 
     def _get_python_script(self) -> tuple[Path, bool]:
         """Return the path to the currently executing script"""
         is_notebook = False
-        if filename := self._get_python_notebook():
+        if filename := self._get_notebook_path():
             is_notebook = True
         else:
             stack = inspect.stack()
@@ -228,11 +275,11 @@ class DerivaML(Dataset):
         try:
             result = subprocess.run(
                 ["git", "rev-parse", "--show-toplevel"],
-                cwd=self._script_path.parent,
+                cwd=self.executable_path.parent,
                 stdout=subprocess.PIPE,
                 stderr=subprocess.DEVNULL,
                 text=True,
-                check=True
+                check=True,
             )
             return result.stdout.strip()
         except subprocess.CalledProcessError:
@@ -262,6 +309,7 @@ class DerivaML(Dataset):
         return self.catalog.getPathBuilder()
 
     def get_version(self) -> str:
+        """Return the version number of the executable"""
         return setuptools_scm.get_version(root=self._get_git_root())
 
     @property
@@ -1040,6 +1088,7 @@ class DerivaML(Dataset):
         return workflow_rid
 
     def lookup_workflow(self, url: str) -> Optional[RID]:
+        """Given a URL, look in the workflow table to find a matching workflow."""
         workflow_path = self.pathBuilder.schemas[self.ml_schema].Workflow
         try:
             url_column = workflow_path.URL
@@ -1069,20 +1118,21 @@ class DerivaML(Dataset):
 
         if is_dirty:
             self._logger.warning(
-                f"File {self._script_path} has been modified since last commit. Consider commiting before executing"
+                f"File {self.executable_path} has been modified since last commit. Consider commiting before executing"
             )
 
         # If you are in a notebook, strip out the outputs before computing the checksum.
         cmd = (
-            f"nbstripout {self._script_path} | git hash-object --stdin"
+            f"nbstripout {self.executable_path} | git hash-object --stdin"
             if self._is_notebook
-            else f"git hash-object {self._script_path}"
+            else f"git hash-object {self.executable_path}"
         )
         checksum = subprocess.run(
             cmd,
             capture_output=True,
             text=True,
             check=True,
+            shell=True,
         ).stdout.strip()
 
         workflow = Workflow(
@@ -1109,8 +1159,10 @@ class DerivaML(Dataset):
         # Get repo URL from local github repo.
         try:
             result = subprocess.run(
-                ["git", "remote", "get-url", "origin"], capture_output=True, text=True,
-                cwd=self._script_path.parent,
+                ["git", "remote", "get-url", "origin"],
+                capture_output=True,
+                text=True,
+                cwd=self.executable_path.parent,
             )
             github_url = result.stdout.strip().removesuffix(".git")
         except subprocess.CalledProcessError:
@@ -1123,7 +1175,7 @@ class DerivaML(Dataset):
         try:
             result = subprocess.run(
                 ["git", "status", "--porcelain"],
-                cwd=self._script_path.parent,
+                cwd=self.executable_path.parent,
                 capture_output=True,
                 text=True,
                 check=True,
@@ -1136,14 +1188,14 @@ class DerivaML(Dataset):
 
         """Get SHA-1 hash of latest commit of the file in the repository"""
         result = subprocess.run(
-            ["git", "log", "-n", "1", "--pretty=format:%H" "--", self._script_path],
-            cwd=self._script_path.parent,
+            ["git", "log", "-n", "1", "--pretty=format:%H" "--", self.executable_path],
+            cwd=self.executable_path.parent,
             capture_output=True,
             text=True,
             check=True,
         )
         sha = result.stdout.strip()
-        url = f"{github_url}/blob/{sha}/{self._script_path.relative_to(repo_root)}"
+        url = f"{github_url}/blob/{sha}/{self.executable_path.relative_to(repo_root)}"
         return url, is_dirty
 
     # @validate_call
@@ -1174,6 +1226,7 @@ class DerivaML(Dataset):
 
     # @validate_call
     def restore_execution(self, execution_rid: Optional[RID] = None) -> "Execution":
+        """Return an Execution object for a previously started execution with the specified RID."""
         from .execution import Execution
 
         # Find path to execution

deriva_ml/execution.py

@@ -253,17 +253,9 @@ class Execution:
 
     def _create_notebook_checkpoint(self):
         """Trigger a checkpoint creation using Jupyter's API."""
-        notebook_name = self._ml_object._notebook
-
-        # Look for the server running this notebook.
-        root = Path("").absolute().parent.as_posix()
-        servers = list(list_running_servers())
-        # Jupyterhub seems to handle root_dir differently then server case.
-        server = (
-            servers
-            if len(servers) == 1
-            else [s for s in servers if s["root_dir"] == root]
-        )[0]
+
+        server, session = self._ml_object._get_notebook_session()
+        notebook_name = session["notebook"]["path"]
         notebook_url = f"{server['url']}api/contents/{notebook_name}"
 
         # Get notebook content
@@ -275,7 +267,7 @@ class Execution:
             # Execution metadata cannot be in a directory, so map path into filename.
             checkpoint_path = (
                 self.execution_metadata_path(ExecMetadataVocab.runtime_env.value)
-                / f"{notebook_name.as_posix().replace('/','_')}.checkpoint"
+                / f"{notebook_name.replace('/','_')}.checkpoint"
             )
             with open(checkpoint_path, "w", encoding="utf-8") as f:
                 json.dump(notebook_content, f)
@@ -295,7 +287,7 @@ class Execution:
         minutes, seconds = divmod(remainder, 60)
         duration = f"{round(hours, 0)}H {round(minutes, 0)}min {round(seconds, 4)}sec"
 
-        if self._ml_object._notebook:
+        if self._ml_object._is_notebook:
             self._create_notebook_checkpoint()
 
         self.update_status(Status.completed, "Algorithm execution ended.")

{deriva_ml-1.8.10.dist-info → deriva_ml-1.8.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deriva-ml
-Version: 1.8.10
+Version: 1.8.11
 Summary: Utilities to simplify use of Dervia and Pandas to create reproducable ML pipelines
 Author-email: ISRD <isrd-dev@isi.edu>
 Requires-Python: >=3.10

{deriva_ml-1.8.10.dist-info → deriva_ml-1.8.11.dist-info}/RECORD

@@ -5,9 +5,9 @@ deriva_ml/dataset_aux_classes.py,sha256=YxjQnu2kS9kK_f8bGqhmgE6ty9GNeitCxfvReT9v
 deriva_ml/dataset_bag.py,sha256=e6IHv3saZUnZRfl0EjfnlV2NnmPeOagYYv3PuZqS1l0,11501
 deriva_ml/demo_catalog.py,sha256=xQPhFlflqwJskNQrQ-jdBSnGzBm2-aONBgcRxfsdNKM,11045
 deriva_ml/deriva_definitions.py,sha256=pZLPoUxiuJ-uGglmQ6sF9oVXsSUuOnPEqywoec78XNM,8893
-deriva_ml/deriva_ml_base.py,sha256=aVyGsFERZtpjNxfaVYzvKa7J0Ma-U3DEibfjnbr7lFQ,43817
+deriva_ml/deriva_ml_base.py,sha256=KbmJ0-mGuZn7-wuzQemzof8o8mA_3-UVlQCrnDsKod0,45741
 deriva_ml/deriva_model.py,sha256=LV3FjIhIlz13ckZSmu0aOJhT9EVE0-M9oVMudfkxb0g,12004
-deriva_ml/execution.py,sha256=VlapQGPDQI2MOmYnA5-hpf-XM6Fu4hPLpFjNN5q9Udo,29889
+deriva_ml/execution.py,sha256=uDblqngcldgR7X4W1PfMV4iPWkxwQYSr9CBmXNlIv1E,29572
 deriva_ml/execution_configuration.py,sha256=bjnZwXN6M7YPy5dFQwoGEBU8YjhQRSe1FW0rL0V9TaM,3422
 deriva_ml/execution_environment.py,sha256=bCRKrCELDbGQDo7_FKfw7e8iMzVjSRZK3baKkqH5-_0,3264
 deriva_ml/feature.py,sha256=7e8WYPCfJSrGxJh9oUTduYSnB5ekybRhXa_0HIigS_w,5459
@@ -25,9 +25,9 @@ deriva_ml/schema_setup/annotations.py,sha256=Uogm9YkRtoKSdgfQlICqRywbCATppwBO-Xr
 deriva_ml/schema_setup/create_schema.py,sha256=jwziMWJPbjRgjiRBT-KtidnXI8YNEFO74A9fwfptjHY,10626
 deriva_ml/schema_setup/policy.json,sha256=77sf0Imy6CAQV0_VwwbA56_KROJ05WXsvT-Wjtkk538,1633
 deriva_ml/schema_setup/table_comments_utils.py,sha256=-2_ubEpoH7ViLVb-ZfW9wZbQ26DTKNgjkCABMzGu4i4,2140
-deriva_ml-1.8.10.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-deriva_ml-1.8.10.dist-info/METADATA,sha256=Mhx0joyR1gPEX8G6ZoEpvxNVW4sUG9C_S5TIA6ueZKk,670
-deriva_ml-1.8.10.dist-info/WHEEL,sha256=1tXe9gY0PYatrMPMDd6jXqjfpz_B-Wqm32CPfRC58XU,91
-deriva_ml-1.8.10.dist-info/entry_points.txt,sha256=ZiOvrYj022x544TQwi018ujeHRRDahNmwJnzn5ThacM,242
-deriva_ml-1.8.10.dist-info/top_level.txt,sha256=I1Q1dkH96cRghdsFRVqwpa2M7IqJpR2QPUNNc5-Bnpw,10
-deriva_ml-1.8.10.dist-info/RECORD,,
+deriva_ml-1.8.11.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+deriva_ml-1.8.11.dist-info/METADATA,sha256=RyttxTkz_MCnWX1hZK2g7ffPyd54txE6AAZ2GMSpQ54,670
+deriva_ml-1.8.11.dist-info/WHEEL,sha256=1tXe9gY0PYatrMPMDd6jXqjfpz_B-Wqm32CPfRC58XU,91
+deriva_ml-1.8.11.dist-info/entry_points.txt,sha256=ZiOvrYj022x544TQwi018ujeHRRDahNmwJnzn5ThacM,242
+deriva_ml-1.8.11.dist-info/top_level.txt,sha256=I1Q1dkH96cRghdsFRVqwpa2M7IqJpR2QPUNNc5-Bnpw,10
+deriva_ml-1.8.11.dist-info/RECORD,,