deriva-ml 1.8.5__py3-none-any.whl → 1.8.11__py3-none-any.whl

deriva_ml/__init__.py

@@ -41,3 +41,12 @@ from .execution_configuration import (
     Workflow,
 )
 from .execution import Execution
+
+from importlib.metadata import version, PackageNotFoundError
+
+try:
+    __version__ = version("deriva_ml")
+except PackageNotFoundError:
+    # package is not installed
+    pass
+

deriva_ml/deriva_ml_base.py

@@ -13,17 +13,14 @@ from __future__ import annotations
 import getpass
 import logging
 from datetime import datetime
-import hashlib
 from itertools import chain
 import inspect
+import setuptools_scm
 from pathlib import Path
 import requests
-from setuptools_git_versioning import get_latest_file_commit
 import subprocess
-import shutil
 from typing import Optional, Any, Iterable, TYPE_CHECKING
 from deriva.core import (
-    ErmrestCatalog,
     get_credential,
     urlquote,
     DEFAULT_SESSION_CONFIG,
@@ -36,6 +33,7 @@ from deriva.core.ermrest_catalog import ResolveRidResult
 from deriva.core.ermrest_model import Key, Table
 from deriva.core.hatrac_store import HatracStore
 from pydantic import validate_call, ConfigDict
+from requests import RequestException
 
 from .execution_configuration import ExecutionConfiguration, Workflow
 from .feature import Feature, FeatureRecord
@@ -74,6 +72,15 @@ try:
 except ImportError:  # Graceful fallback if IPython isn't installed.
     get_ipython = lambda: None
 
+try:
+    from jupyter_server.serverapp import list_running_servers
+except ImportError:
+    list_running_servers = lambda: []
+
+try:
+    from ipykernel import get_connection_file
+except ImportError:
+    get_connection_file = lambda: ""
 
 if TYPE_CHECKING:
     from .execution import Execution
@@ -144,6 +151,8 @@ class DerivaML(Dataset):
 
         # Initialize dataset class.
         super().__init__(self.model, self.cache_dir)
+        self._logger = logging.getLogger("deriva_ml")
+        self._logger.setLevel(logging_level)
 
         self.host_name = hostname
         self.catalog_id = catalog_id
@@ -151,37 +160,11 @@ class DerivaML(Dataset):
         self.version = model_version
         self.configuration = None
         self._execution: Optional[Execution] = None
-        self._notebook = None
-        try:
-            from IPython import get_ipython
-
-            ipython = get_ipython()
-            # Check if running in Jupyter's ZMQ kernel (used by notebooks)
-            if ipython is not None and "IPKernelApp" in ipython.config:
-                self._notebook = Path(ipython.user_ns.get("__session__"))
-                # Check if running in Jupyter's ZMQ kernel (used by notebooks)
-                try:
-                    if subprocess.run(
-                        [shutil.which("nbstripout"), "--is-installed"],
-                        check=False,
-                        capture_output=True,
-                    ).returncode:
-                        self._logger.warn(
-                            "nbstripout is not installed in repository. Please run nbstripout --install"
-                        )
-                except subprocess.CalledProcessError:
-                    self._logger.error("nbstripout is not found.")
-
-        except (ImportError, AttributeError):
-            pass
-
+        self.executable_path, self._is_notebook = self._get_python_script()
         self.domain_schema = self.model.domain_schema
         self.project_name = project_name or self.domain_schema
-
         self.start_time = datetime.now()
         self.status = Status.pending.value
-        self._logger = logging.getLogger("deriva_ml")
-        self._logger.setLevel(logging_level)
 
         logging.basicConfig(
             level=logging_level,
@@ -204,6 +187,104 @@ class DerivaML(Dataset):
         except (AttributeError, requests.HTTPError):
             pass
 
+    def _check_nbstrip_status(self) -> None:
+        """Figure out if you are running in a Jupyter notebook
+
+        Returns:
+            A Path to the notebook file that is currently being executed.
+        """
+        try:
+            if subprocess.run(
+                ["nbstripout", "--is-installed"],
+                check=False,
+                capture_output=True,
+            ).returncode:
+                self._logger.warning(
+                    "nbstripout is not installed in repository. Please run nbstripout --install"
+                )
+        except subprocess.CalledProcessError:
+            self._logger.error("nbstripout is not found.")
+
+    def _get_notebook_session(
+        self,
+    ) -> tuple[dict[str, Any] | None, dict[str, Any] | None]:
+        """Return the absolute path of the current notebook."""
+        # Get the kernel's connection file and extract the kernel ID
+        try:
+            if not (connection_file := Path(get_connection_file()).name):
+                return None, None
+        except RuntimeError:
+            return None, None
+
+        kernel_id = connection_file.split("-", 1)[1].split(".")[0]
+
+        # Look through the running server sessions to find the matching kernel ID
+        for server in list_running_servers():
+            try:
+                # If a token is required for authentication, include it in headers
+                token = server.get("token", "")
+                headers = {}
+                if token:
+                    headers["Authorization"] = f"token {token}"
+
+                try:
+                    sessions_url = server["url"] + "api/sessions"
+                    response = requests.get(sessions_url, headers=headers)
+                    response.raise_for_status()
+                    sessions = response.json()
+                except RequestException as e:
+                    raise e
+                for sess in sessions:
+                    if sess["kernel"]["id"] == kernel_id:
+                        return server, sess
+            except Exception as _e:
+                # Ignore servers we can't connect to.
+                pass
+        return None, None
+
+    def _get_notebook_path(self) -> Path | None:
+        """Return the absolute path of the current notebook."""
+
+        server, session = self._get_notebook_session()
+        if server and session:
+            self._check_nbstrip_status()
+            relative_path = session["notebook"]["path"]
+            # Join the notebook directory with the relative path
+            return Path(server["root_dir"]) / relative_path
+        else:
+            return None
+
+    def _get_python_script(self) -> tuple[Path, bool]:
+        """Return the path to the currently executing script"""
+        is_notebook = False
+        if filename := self._get_notebook_path():
+            is_notebook = True
+        else:
+            stack = inspect.stack()
+            if len(stack) > 1:
+                filename = Path(
+                    stack[2].filename
+                )  # Get the caller's filename, which is two up the stack from here.
+            else:
+                raise DerivaMLException(
+                    f"Looking for caller failed"
+                )  # Stack is too shallow
+        return filename, is_notebook
+
+    def _get_git_root(self):
+        try:
+            result = subprocess.run(
+                ["git", "rev-parse", "--show-toplevel"],
+                cwd=self.executable_path.parent,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.DEVNULL,
+                text=True,
+                check=True,
+            )
+            return result.stdout.strip()
+        except subprocess.CalledProcessError:
+            return None  # Not in a git repository
+
     @staticmethod
     def _get_session_config():
         """ """
@@ -227,6 +308,10 @@ class DerivaML(Dataset):
         """Get a new instance of a pathBuilder object."""
         return self.catalog.getPathBuilder()
 
+    def get_version(self) -> str:
+        """Return the version number of the executable"""
+        return setuptools_scm.get_version(root=self._get_git_root())
+
     @property
     def domain_path(self):
         """Get a new instance of a pathBuilder object to the domain schema"""
@@ -1003,6 +1088,7 @@ class DerivaML(Dataset):
         return workflow_rid
 
     def lookup_workflow(self, url: str) -> Optional[RID]:
+        """Given a URL, look in the workflow table to find a matching workflow."""
         workflow_path = self.pathBuilder.schemas[self.ml_schema].Workflow
         try:
             url_column = workflow_path.URL
@@ -1028,27 +1114,26 @@ class DerivaML(Dataset):
         """
         # Make sure type is correct.
         self.lookup_term(MLVocab.workflow_type, workflow_type)
-        filename, github_url, is_dirty = self._github_url()
+        github_url, is_dirty = self._github_url()
 
         if is_dirty:
             self._logger.warning(
-                f"File {filename} has been modified since last commit. Consider commiting before executing"
+                f"File {self.executable_path} has been modified since last commit. Consider commiting before executing"
             )
 
-        sha256_hash = hashlib.sha256()
-        if self._notebook:
-            # If you are in a notebook, strip out the outputs before computing the checksum.
-            result = subprocess.run(
-                ["nbstripout", "-t", filename],
-                capture_output=True,
-                text=False,
-                check=True,
-            )
-            sha256_hash.update(result.stdout)
-        else:
-            with open(filename, "rb") as f:
-                sha256_hash.update(f.read())
-        checksum = "SHA-256:" + sha256_hash.hexdigest()
+        # If you are in a notebook, strip out the outputs before computing the checksum.
+        cmd = (
+            f"nbstripout {self.executable_path} | git hash-object --stdin"
+            if self._is_notebook
+            else f"git hash-object {self.executable_path}"
+        )
+        checksum = subprocess.run(
+            cmd,
+            capture_output=True,
+            text=True,
+            check=True,
+            shell=True,
+        ).stdout.strip()
 
         workflow = Workflow(
             name=name,
@@ -1059,54 +1144,38 @@ class DerivaML(Dataset):
         )
         return self.add_workflow(workflow) if create else None
 
-    def _github_url(self) -> tuple[Path, str, bool]:
+    def _github_url(self) -> tuple[str, bool]:
         """Return a GitHUB URL for the latest commit of the script from which this routine is called.
 
         This routine is used to be called from a script or notebook (e.g. python -m file). It assumes that
         the file is in a gitHUB repository and commited.  It returns a URL to the last commited version of this
         file in GitHUB.
 
-        Returns: A tuple with the filename, gethub_url and a boolean to indicated if uncommited changes
+        Returns: A tuple with the gethub_url and a boolean to indicated if uncommited changes
             have been made to the file.
 
         """
 
-        # Get the name of the script that is calling this function.
-        if self._notebook:
-            # Try to get the __session__ variable from the user namespace.
-            filename = Path("").absolute().parent / self._notebook
-        else:
-            stack = inspect.stack()
-            if len(stack) > 1:
-                filename = Path(
-                    stack[2].filename
-                )  # Get the caller's filename, which is two up the stack from here.
-            else:
-                raise DerivaMLException(
-                    f"Looking for caller failed"
-                )  # Stack is too shallow
-
         # Get repo URL from local github repo.
         try:
             result = subprocess.run(
-                ["git", "remote", "get-url", "origin"], capture_output=True, text=True
+                ["git", "remote", "get-url", "origin"],
+                capture_output=True,
+                text=True,
+                cwd=self.executable_path.parent,
             )
             github_url = result.stdout.strip().removesuffix(".git")
         except subprocess.CalledProcessError:
             raise DerivaMLException(f"No GIT remote found")
 
         # Find the root directory for the repository
-        repo_root = filename
-        while repo_root != repo_root.root:
-            if (repo_root / ".git").exists():
-                break
-            else:
-                repo_root = repo_root.parent
+        repo_root = self._get_git_root()
 
         # Now check to see if file has been modified since the last commit.
         try:
             result = subprocess.run(
                 ["git", "status", "--porcelain"],
+                cwd=self.executable_path.parent,
                 capture_output=True,
                 text=True,
                 check=True,
@@ -1117,9 +1186,17 @@ class DerivaML(Dataset):
         except subprocess.CalledProcessError:
             is_dirty = False  # If Git command fails, assume no changes
 
-        sha = get_latest_file_commit(filename)
-        url = f"{github_url}/blob/{sha}/{filename.relative_to(repo_root)}"
-        return filename, url, is_dirty
+        """Get SHA-1 hash of latest commit of the file in the repository"""
+        result = subprocess.run(
+            ["git", "log", "-n", "1", "--pretty=format:%H" "--", self.executable_path],
+            cwd=self.executable_path.parent,
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+        sha = result.stdout.strip()
+        url = f"{github_url}/blob/{sha}/{self.executable_path.relative_to(repo_root)}"
+        return url, is_dirty
 
     # @validate_call
     def create_execution(self, configuration: ExecutionConfiguration) -> "Execution":
@@ -1149,6 +1226,7 @@ class DerivaML(Dataset):
 
     # @validate_call
     def restore_execution(self, execution_rid: Optional[RID] = None) -> "Execution":
+        """Return an Execution object for a previously started execution with the specified RID."""
         from .execution import Execution
 
         # Find path to execution

deriva_ml/execution.py

@@ -253,17 +253,9 @@ class Execution:
 
     def _create_notebook_checkpoint(self):
         """Trigger a checkpoint creation using Jupyter's API."""
-        notebook_name = self._ml_object._notebook
-
-        # Look for the server running this notebook.
-        root = Path("").absolute().parent.as_posix()
-        servers = list(list_running_servers())
-        # Jupyterhub seems to handle root_dir differently then server case.
-        server = (
-            servers
-            if len(servers) == 1
-            else [s for s in servers if s["root_dir"] == root]
-        )[0]
+
+        server, session = self._ml_object._get_notebook_session()
+        notebook_name = session["notebook"]["path"]
         notebook_url = f"{server['url']}api/contents/{notebook_name}"
 
         # Get notebook content
@@ -275,7 +267,7 @@ class Execution:
             # Execution metadata cannot be in a directory, so map path into filename.
             checkpoint_path = (
                 self.execution_metadata_path(ExecMetadataVocab.runtime_env.value)
-                / f"{notebook_name.as_posix().replace('/','_')}.checkpoint"
+                / f"{notebook_name.replace('/','_')}.checkpoint"
             )
             with open(checkpoint_path, "w", encoding="utf-8") as f:
                 json.dump(notebook_content, f)
@@ -295,7 +287,7 @@ class Execution:
         minutes, seconds = divmod(remainder, 60)
         duration = f"{round(hours, 0)}H {round(minutes, 0)}min {round(seconds, 4)}sec"
 
-        if self._ml_object._notebook:
+        if self._ml_object._is_notebook:
             self._create_notebook_checkpoint()
 
         self.update_status(Status.completed, "Algorithm execution ended.")

{deriva_ml-1.8.5.dist-info → deriva_ml-1.8.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deriva-ml
-Version: 1.8.5
+Version: 1.8.11
 Summary: Utilities to simplify use of Dervia and Pandas to create reproducable ML pipelines
 Author-email: ISRD <isrd-dev@isi.edu>
 Requires-Python: >=3.10
@@ -11,7 +11,8 @@ Requires-Dist: pandas
 Requires-Dist: regex~=2024.7.24
 Requires-Dist: pydantic>=2.10.6
 Requires-Dist: semver>3.0.0
-Requires-Dist: setuptools-git-versioning<3,>=2.0
+Requires-Dist: setuptools>=64
+Requires-Dist: setuptools-scm<=6.0
 Requires-Dist: nbstripout
 Dynamic: license-file
 

{deriva_ml-1.8.5.dist-info → deriva_ml-1.8.11.dist-info}/RECORD

@@ -1,14 +1,13 @@
-deriva_ml/VERSION.py,sha256=Dtbi4ISKI_kkTsaWuM-q8NfE3DySQu91TTP04Yhd8d8,22
-deriva_ml/__init__.py,sha256=DyHiqklSer7q7oPGAemkzg5Qcq2swMZf9ALwJhGf6Jo,905
+deriva_ml/__init__.py,sha256=r1Z9N5vtZkAET7emqhpAx2bf_xJUp5wHOc4_DIplsG8,1082
 deriva_ml/database_model.py,sha256=uhoyVyd8MQmY8J9ovCH8fjxhZDxxXNkdJyYdeyEGPXA,13898
 deriva_ml/dataset.py,sha256=xC6QPUp4MZcJiEnOEU3NnzoLBL9RcJWtPTyzIQP0Ivw,60666
 deriva_ml/dataset_aux_classes.py,sha256=YxjQnu2kS9kK_f8bGqhmgE6ty9GNeitCxfvReT9vaM0,6537
 deriva_ml/dataset_bag.py,sha256=e6IHv3saZUnZRfl0EjfnlV2NnmPeOagYYv3PuZqS1l0,11501
 deriva_ml/demo_catalog.py,sha256=xQPhFlflqwJskNQrQ-jdBSnGzBm2-aONBgcRxfsdNKM,11045
 deriva_ml/deriva_definitions.py,sha256=pZLPoUxiuJ-uGglmQ6sF9oVXsSUuOnPEqywoec78XNM,8893
-deriva_ml/deriva_ml_base.py,sha256=e2UtT3TlDpFQrG6z0DaB2iV22wmi4TLP7qXF3hvb8to,42868
+deriva_ml/deriva_ml_base.py,sha256=KbmJ0-mGuZn7-wuzQemzof8o8mA_3-UVlQCrnDsKod0,45741
 deriva_ml/deriva_model.py,sha256=LV3FjIhIlz13ckZSmu0aOJhT9EVE0-M9oVMudfkxb0g,12004
-deriva_ml/execution.py,sha256=VlapQGPDQI2MOmYnA5-hpf-XM6Fu4hPLpFjNN5q9Udo,29889
+deriva_ml/execution.py,sha256=uDblqngcldgR7X4W1PfMV4iPWkxwQYSr9CBmXNlIv1E,29572
 deriva_ml/execution_configuration.py,sha256=bjnZwXN6M7YPy5dFQwoGEBU8YjhQRSe1FW0rL0V9TaM,3422
 deriva_ml/execution_environment.py,sha256=bCRKrCELDbGQDo7_FKfw7e8iMzVjSRZK3baKkqH5-_0,3264
 deriva_ml/feature.py,sha256=7e8WYPCfJSrGxJh9oUTduYSnB5ekybRhXa_0HIigS_w,5459
@@ -26,9 +25,9 @@ deriva_ml/schema_setup/annotations.py,sha256=Uogm9YkRtoKSdgfQlICqRywbCATppwBO-Xr
 deriva_ml/schema_setup/create_schema.py,sha256=jwziMWJPbjRgjiRBT-KtidnXI8YNEFO74A9fwfptjHY,10626
 deriva_ml/schema_setup/policy.json,sha256=77sf0Imy6CAQV0_VwwbA56_KROJ05WXsvT-Wjtkk538,1633
 deriva_ml/schema_setup/table_comments_utils.py,sha256=-2_ubEpoH7ViLVb-ZfW9wZbQ26DTKNgjkCABMzGu4i4,2140
-deriva_ml-1.8.5.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-deriva_ml-1.8.5.dist-info/METADATA,sha256=uuAq67MUZyY2LP8NbZ8RMJ5q-aX3pJV5ioYZqqLbuFA,653
-deriva_ml-1.8.5.dist-info/WHEEL,sha256=tTnHoFhvKQHCh4jz3yCn0WPTYIy7wXx3CJtJ7SJGV7c,91
-deriva_ml-1.8.5.dist-info/entry_points.txt,sha256=ZiOvrYj022x544TQwi018ujeHRRDahNmwJnzn5ThacM,242
-deriva_ml-1.8.5.dist-info/top_level.txt,sha256=I1Q1dkH96cRghdsFRVqwpa2M7IqJpR2QPUNNc5-Bnpw,10
-deriva_ml-1.8.5.dist-info/RECORD,,
+deriva_ml-1.8.11.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+deriva_ml-1.8.11.dist-info/METADATA,sha256=RyttxTkz_MCnWX1hZK2g7ffPyd54txE6AAZ2GMSpQ54,670
+deriva_ml-1.8.11.dist-info/WHEEL,sha256=1tXe9gY0PYatrMPMDd6jXqjfpz_B-Wqm32CPfRC58XU,91
+deriva_ml-1.8.11.dist-info/entry_points.txt,sha256=ZiOvrYj022x544TQwi018ujeHRRDahNmwJnzn5ThacM,242
+deriva_ml-1.8.11.dist-info/top_level.txt,sha256=I1Q1dkH96cRghdsFRVqwpa2M7IqJpR2QPUNNc5-Bnpw,10
+deriva_ml-1.8.11.dist-info/RECORD,,

{deriva_ml-1.8.5.dist-info → deriva_ml-1.8.11.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (77.0.1)
+Generator: setuptools (77.0.3)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

deriva_ml/VERSION.py

@@ -1 +0,0 @@
-__version__ = "1.8.5"