deriva-ml 1.8.10__py3-none-any.whl → 1.9.0__py3-none-any.whl

deriva_ml/database_model.py

@@ -1,12 +1,15 @@
-"""Ths module constains the definition of the DatabaseModel class.  The role of this class is to provide an nterface between the BDBag representation
+"""Ths module contains the definition of the DatabaseModel class.  The role of this class is to provide an nterface between the BDBag representation
 of a dataset and a sqllite database in which the contents of the bag are stored.
 """
+
+from __future__ import annotations
+
 import logging
 import sqlite3
 
 from csv import reader
 from pathlib import Path
-from typing import Any, Optional
+from typing import Any, Optional, Generator
 from urllib.parse import urlparse
 
 from deriva.core.ermrest_model import Model
@@ -20,7 +23,7 @@ from .dataset_bag import DatasetBag
 class DatabaseModelMeta(type):
     """Use metaclass to ensure that there is onl one instance per path"""
 
-    _paths_loaded: dict[Path:"DatabaseModel"] = {}
+    _paths_loaded: dict[Path, "DatabaseModel"] = {}
 
     def __call__(cls, *args, **kwargs):
         logger = logging.getLogger("deriva_ml")
@@ -47,7 +50,7 @@ class DatabaseModel(DerivaModel, metaclass=DatabaseModelMeta):
     Because of nested datasets, it's possible that more than one dataset rid is in a bag, or that a dataset rid might
     appear in more than one database. To help manage this, a global list of all the datasets that have been loaded
     into DatabaseModels, is kept in the class variable `_rid_map`.
-    
+
     Because you can load diffent versions of a dataset simultaniously, the dataset RID and version number are tracked, and a new
     sqllite instance is created for every new dataset version present.
 
@@ -315,6 +318,26 @@ class DatabaseModel(DerivaModel, metaclass=DatabaseModelMeta):
             )
         return datasets
 
+    def get_table_as_dict(self, table: str) -> Generator[dict[str, Any], None, None]:
+        """Retrieve the contents of the specified table as a dictionary.
+
+        Args:
+            table: Table to retrieve data from. f schema is not provided as part of the table name,
+                the method will attempt to locate the schema for the table.
+
+        Returns:
+          A generator producing dictionaries containing the contents of the specified table as name/value pairs.
+        """
+        table_name = self.normalize_table_name(table)
+        with self.dbase as dbase:
+            col_names = [
+                c[1]
+                for c in dbase.execute(f'PRAGMA table_info("{table_name}")').fetchall()
+            ]
+            result = self.dbase.execute(f'SELECT * FROM "{table_name}"')
+            while row := result.fetchone():
+                yield dict(zip(col_names, row))
+
     def normalize_table_name(self, table: str) -> str:
         """Attempt to insert the schema into a table name if it's not provided.
 

deriva_ml/dataset.py

@@ -92,7 +92,7 @@ class Dataset:
         dataset_list: list[DatasetSpec],
         description: Optional[str] = "",
         execution_rid: Optional[RID] = None,
-    ) -> RID:
+    ) -> list[dict[str, Any]]:
         schema_path = self._model.catalog.getPathBuilder().schemas[self._ml_schema]
 
         # Construct version records for insert
@@ -245,7 +245,7 @@ class Dataset:
           DerivaMLException: if provided RID is not to a dataset_table.
         """
 
-        # Find all of the datasets that are reachable from this dataset and determine their new version numbers.
+        # Find all the datasets that are reachable from this dataset and determine their new version numbers.
         related_datasets = list(self._build_dataset_graph(dataset_rid=dataset_rid))
         version_update_list = [
             DatasetSpec(
@@ -254,7 +254,7 @@ class Dataset:
             )
             for ds_rid in related_datasets
         ]
-        updated_versions = self._insert_dataset_versions(
+        self._insert_dataset_versions(
             version_update_list, description=description, execution_rid=execution_rid
         )
         return [d.version for d in version_update_list if d.rid == dataset_rid][0]
@@ -751,9 +751,10 @@ class Dataset:
         ]
 
     def _table_paths(
-        self, dataset: DatasetSpec = None, snapshot_catalog: Optional[DerivaML] = None
+        self,
+        dataset: Optional[DatasetSpec] = None,
+        snapshot_catalog: Optional[DerivaML] = None,
     ) -> Iterator[tuple[str, str, Table]]:
-
         paths = self._collect_paths(dataset and dataset.rid, snapshot_catalog)
 
         def source_path(path: tuple[Table, ...]):
@@ -779,17 +780,20 @@ class Dataset:
     def _collect_paths(
         self,
         dataset_rid: Optional[RID] = None,
-        snapshot_catalog: Optional[DerivaML] = None,
+        snapshot: Optional[Dataset] = None,
         dataset_nesting_depth: Optional[int] = None,
     ) -> set[tuple[Table, ...]]:
 
-        snapshot_catalog = snapshot_catalog or self
+        snapshot_catalog = snapshot if snapshot else self
+
         dataset_table = snapshot_catalog._model.schemas[self._ml_schema].tables[
             "Dataset"
         ]
         dataset_dataset = snapshot_catalog._model.schemas[self._ml_schema].tables[
             "Dataset_Dataset"
         ]
+
+        # Figure out what types of elements the dataset contains.
         dataset_associations = [
             a
             for a in self.dataset_table.find_associations()
@@ -812,7 +816,8 @@ class Dataset:
             ]
         else:
             included_associations = dataset_associations
-        # Get the paths through the schema and filter out all of dataset paths not used by this dataset.
+
+        # Get the paths through the schema and filter out all the dataset paths not used by this dataset.
         paths = {
             tuple(p)
             for p in snapshot_catalog._model._schema_to_paths()
@@ -827,7 +832,7 @@ class Dataset:
         if dataset_rid:
             for c in snapshot_catalog.list_dataset_children(dataset_rid=dataset_rid):
                 nested_paths |= self._collect_paths(
-                    c, snapshot_catalog=snapshot_catalog
+                    c, snapshot=snapshot_catalog
                 )
         else:
             # Initialize nesting depth if not already provided.

deriva_ml/dataset_bag.py

@@ -109,7 +109,7 @@ class DatasetBag:
         for ts, on in paths:
             tables = " JOIN ".join(ts)
             on_expression = " and ".join(
-                [f"{column_name(l)}={column_name(r)}" for l, r in on]
+                [f"{column_name(left)}={column_name(right)}" for left, right in on]
             )
             sql.append(
                 f"SELECT {select_args} FROM {tables} ON {on_expression} WHERE {dataset_table_name}.RID IN ({datasets})"

deriva_ml/demo_catalog.py

@@ -5,6 +5,7 @@ import logging
 from random import random, randint
 import tempfile
 from tempfile import TemporaryDirectory
+from typing import Optional
 import itertools
 
 from deriva.config.acl_config import AclConfig
@@ -18,7 +19,6 @@ from requests import HTTPError
 from deriva_ml import (
     DerivaML,
     ExecutionConfiguration,
-    Workflow,
     MLVocab,
     BuiltinTypes,
     ColumnDefinition,
@@ -169,12 +169,9 @@ def create_demo_features(ml_instance):
         description="Model for our API workflow",
     )
 
-    api_workflow = ml_instance.add_workflow(
-        Workflow(
-            name="API Workflow",
-            url="https://github.com/informatics-isi-edu/deriva-ml/blob/main/pyproject.toml",
-            workflow_type="API Workflow",
-        )
+    api_workflow = ml_instance.create_workflow(
+        name="API Workflow",
+        workflow_type="API Workflow",
     )
 
     api_execution = ml_instance.create_execution(
@@ -322,7 +319,11 @@ def create_demo_catalog(
 
 class DemoML(DerivaML):
     def __init__(
-        self, hostname, catalog_id, cache_dir: str = None, working_dir: str = None
+        self,
+        hostname,
+        catalog_id,
+        cache_dir: Optional[str] = None,
+        working_dir: Optional[str] = None,
     ):
         super().__init__(
             hostname=hostname,

deriva_ml/deriva_definitions.py

@@ -8,7 +8,7 @@ from enum import Enum
 from typing import Any, Iterable, Optional, Annotated
 
 import deriva.core.ermrest_model as em
-from urllib.parse import urlparse, urljoin
+from urllib.parse import urlparse
 from deriva.core.ermrest_model import builtin_types
 from pydantic import (
     BaseModel,
@@ -139,13 +139,18 @@ class FileSpec(BaseModel):
         if url_parts.scheme == "tag":
             return v
         elif not url_parts.scheme:
-            return f'tag://{gethostname()},{date.today()}:file://{v}'
+            return f"tag://{gethostname()},{date.today()}:file://{v}"
         else:
             raise ValidationError("url is not a file URL")
 
     @model_serializer()
     def serialize_filespec(self):
-        return {'URL': self.url, 'Description': self.description, 'MD5': self.md5, 'Length': self.length}
+        return {
+            "URL": self.url,
+            "Description": self.description,
+            "MD5": self.md5,
+            "Length": self.length,
+        }
 
 
 class VocabularyTerm(BaseModel):

deriva_ml/deriva_ml_base.py

@@ -19,7 +19,6 @@ import setuptools_scm
 from pathlib import Path
 import requests
 import subprocess
-import shutil
 from typing import Optional, Any, Iterable, TYPE_CHECKING
 from deriva.core import (
     get_credential,
@@ -33,7 +32,9 @@ from deriva.core.deriva_server import DerivaServer
 from deriva.core.ermrest_catalog import ResolveRidResult
 from deriva.core.ermrest_model import Key, Table
 from deriva.core.hatrac_store import HatracStore
+from deriva.core.utils.globus_auth_utils import GlobusNativeLogin
 from pydantic import validate_call, ConfigDict
+from requests import RequestException
 
 from .execution_configuration import ExecutionConfiguration, Workflow
 from .feature import Feature, FeatureRecord
@@ -70,7 +71,28 @@ except ImportError:  # Graceful fallback if IceCream isn't installed.
 try:
     from IPython import get_ipython
 except ImportError:  # Graceful fallback if IPython isn't installed.
-    get_ipython = lambda: None
+
+    def get_ipython():
+        """Dummy routine in case you are not running in IPython."""
+        return None
+
+
+try:
+    from jupyter_server.serverapp import list_running_servers
+except ImportError:
+
+    def list_running_servers():
+        """Dummy routine in case you are not running in Jupyter."""
+        return []
+
+
+try:
+    from ipykernel import get_connection_file
+except ImportError:
+
+    def get_connection_file():
+        """Dummy routine in case you are not running in Jupyter."""
+        return ""
 
 
 if TYPE_CHECKING:
@@ -93,8 +115,8 @@ class DerivaML(Dataset):
         self,
         hostname: str,
         catalog_id: str | int,
-        domain_schema: str = None,
-        project_name: str = None,
+        domain_schema: Optional[str] = None,
+        project_name: Optional[str] = None,
         cache_dir: Optional[str] = None,
         working_dir: Optional[str] = None,
         model_version: str = "1",
@@ -151,8 +173,7 @@ class DerivaML(Dataset):
         self.version = model_version
         self.configuration = None
         self._execution: Optional[Execution] = None
-        self._script_path, self._is_notebook = self._get_python_script()
-        self._notebook = self._get_python_notebook()
+        self.executable_path, self._is_notebook = self._get_python_script()
         self.domain_schema = self.model.domain_schema
         self.project_name = project_name or self.domain_schema
         self.start_time = datetime.now()
@@ -179,38 +200,76 @@ class DerivaML(Dataset):
         except (AttributeError, requests.HTTPError):
             pass
 
-    def _get_python_notebook(self) -> Path | None:
+    def _check_nbstrip_status(self) -> None:
         """Figure out if you are running in a Jupyter notebook
 
         Returns:
             A Path to the notebook file that is currently being executed.
         """
-        notebook = None
         try:
-            ipython = get_ipython()
-            # Check if running in Jupyter's ZMQ kernel (used by notebooks)
-            if ipython is not None and "IPKernelApp" in ipython.config:
-                notebook = Path(ipython.user_ns.get("__session__"))
-                # Check if running in Jupyter's ZMQ kernel (used by notebooks)
+            if subprocess.run(
+                ["nbstripout", "--is-installed"],
+                check=False,
+                capture_output=True,
+            ).returncode:
+                self._logger.warning(
+                    "nbstripout is not installed in repository. Please run nbstripout --install"
+                )
+        except subprocess.CalledProcessError:
+            self._logger.error("nbstripout is not found.")
+
+    @staticmethod
+    def _get_notebook_session() -> tuple[dict[str, Any] | None, dict[str, Any] | None]:
+        """Return the absolute path of the current notebook."""
+        # Get the kernel's connection file and extract the kernel ID
+        try:
+            if not (connection_file := Path(get_connection_file()).name):
+                return None, None
+        except RuntimeError:
+            return None, None
+
+        kernel_id = connection_file.split("-", 1)[1].split(".")[0]
+
+        # Look through the running server sessions to find the matching kernel ID
+        for server in list_running_servers():
+            try:
+                # If a token is required for authentication, include it in headers
+                token = server.get("token", "")
+                headers = {}
+                if token:
+                    headers["Authorization"] = f"token {token}"
+
                 try:
-                    if subprocess.run(
-                        [shutil.which("nbstripout"), "--is-installed"],
-                        check=False,
-                        capture_output=True,
-                    ).returncode:
-                        self._logger.warning(
-                            "nbstripout is not installed in repository. Please run nbstripout --install"
-                        )
-                except subprocess.CalledProcessError:
-                    self._logger.error("nbstripout is not found.")
-        except (ImportError, AttributeError):
-            pass
-        return notebook
+                    sessions_url = server["url"] + "api/sessions"
+                    response = requests.get(sessions_url, headers=headers)
+                    response.raise_for_status()
+                    sessions = response.json()
+                except RequestException as e:
+                    raise e
+                for sess in sessions:
+                    if sess["kernel"]["id"] == kernel_id:
+                        return server, sess
+            except Exception as _e:
+                # Ignore servers we can't connect to.
+                pass
+        return None, None
+
+    def _get_notebook_path(self) -> Path | None:
+        """Return the absolute path of the current notebook."""
+
+        server, session = self._get_notebook_session()
+        if server and session:
+            self._check_nbstrip_status()
+            relative_path = session["notebook"]["path"]
+            # Join the notebook directory with the relative path
+            return Path(server["root_dir"]) / relative_path
+        else:
+            return None
 
     def _get_python_script(self) -> tuple[Path, bool]:
         """Return the path to the currently executing script"""
         is_notebook = False
-        if filename := self._get_python_notebook():
+        if filename := self._get_notebook_path():
             is_notebook = True
         else:
             stack = inspect.stack()
@@ -220,7 +279,7 @@ class DerivaML(Dataset):
                 )  # Get the caller's filename, which is two up the stack from here.
             else:
                 raise DerivaMLException(
-                    f"Looking for caller failed"
+                    "Looking for caller failed"
                 )  # Stack is too shallow
         return filename, is_notebook
 
@@ -228,11 +287,11 @@ class DerivaML(Dataset):
         try:
             result = subprocess.run(
                 ["git", "rev-parse", "--show-toplevel"],
-                cwd=self._script_path.parent,
+                cwd=self.executable_path.parent,
                 stdout=subprocess.PIPE,
                 stderr=subprocess.DEVNULL,
                 text=True,
-                check=True
+                check=True,
             )
             return result.stdout.strip()
         except subprocess.CalledProcessError:
@@ -262,6 +321,7 @@ class DerivaML(Dataset):
         return self.catalog.getPathBuilder()
 
     def get_version(self) -> str:
+        """Return the version number of the executable"""
         return setuptools_scm.get_version(root=self._get_git_root())
 
     @property
@@ -287,7 +347,7 @@ class DerivaML(Dataset):
         )
 
     def asset_dir(
-        self, table: str | Table, prefix: str | Path = None
+        self, table: str | Table, prefix: Optional[str | Path] = None
     ) -> UploadAssetDirectory:
         """Return a local file path in which to place a files for an asset table.  T
 
@@ -321,6 +381,29 @@ class DerivaML(Dataset):
         """
         return self.cache_dir if cached else self.working_dir
 
+    @staticmethod
+    def globus_login(host: str) -> None:
+        """Log  into the specified host using Globus.
+
+        Args:
+            host:
+
+        Returns:
+
+        """
+        gnl = GlobusNativeLogin(host=host)
+        if gnl.is_logged_in([host]):
+            print("You are already logged in.")
+        else:
+            gnl.login(
+                [host],
+                no_local_server=True,
+                no_browser=True,
+                refresh_tokens=True,
+                update_bdbag_keychain=True,
+            )
+            print("Login Successful")
+
     def chaise_url(self, table: RID | Table) -> str:
         """Return a Chaise URL to the specified table.
 
@@ -331,15 +414,15 @@ class DerivaML(Dataset):
         Returns:
             URL to the table in Chaise format.
         """
+        table_obj = self.model.name_to_table(table)
         try:
-            table = self.model.name_to_table(table)
             uri = self.catalog.get_server_uri().replace(
                 "ermrest/catalog/", "chaise/recordset/#"
             )
         except DerivaMLException:
             # Perhaps we have a RID....
             uri = self.cite(table)
-        return f"{uri}/{urlquote(table.schema.name)}:{urlquote(table.name)}"
+        return f"{uri}/{urlquote(table_obj.schema.name)}:{urlquote(table_obj.name)}"
 
     def cite(self, entity: dict | str) -> str:
         """Return a citation URL for the provided entity.
@@ -353,7 +436,9 @@ class DerivaML(Dataset):
         Raises:
             DerivaMLException: if provided RID does not exist.
         """
-        if entity.startswith(f"https://{self.host_name}/id/{self.catalog_id}/"):
+        if isinstance(entity, str) and entity.startswith(
+            f"https://{self.host_name}/id/{self.catalog_id}/"
+        ):
             # Already got a citation...
             return entity
         try:
@@ -450,9 +535,9 @@ class DerivaML(Dataset):
     def create_asset(
         self,
         asset_name: str,
-        column_defs: Iterable[ColumnDefinition] = None,
+        column_defs: Optional[Iterable[ColumnDefinition]] = None,
         comment: str = "",
-        schema: str = None,
+        schema: Optional[str] = None,
     ) -> Table:
         """Create an asset table with the given asset name.
 
@@ -484,9 +569,9 @@ class DerivaML(Dataset):
         self,
         target_table: Table | str,
         feature_name: str,
-        terms: list[Table | str] = None,
-        assets: list[Table | str] = None,
-        metadata: Iterable[ColumnDefinition | Table | Key | str] = None,
+        terms: Optional[list[Table | str]] = None,
+        assets: Optional[list[Table | str]] = None,
+        metadata: Optional[Iterable[ColumnDefinition | Table | Key | str]] = None,
         optional: Optional[list[str]] = None,
         comment: str = "",
     ) -> type[FeatureRecord]:
@@ -851,6 +936,7 @@ class DerivaML(Dataset):
         """
 
         def path_to_asset(path: str) -> str:
+            """Pull the asset name out of a path to that asset in the filesystem"""
             components = path.split("/")
             return components[
                 components.index("asset") + 2
@@ -915,6 +1001,7 @@ class DerivaML(Dataset):
             )
 
         def check_file_type(dtype: str) -> bool:
+            """Make sure that the specified string is either the name or synonym for a file type term."""
             for term in defined_types:
                 if dtype == term.name or (term.synonyms and file_type in term.synonyms):
                     return True
@@ -1040,6 +1127,7 @@ class DerivaML(Dataset):
         return workflow_rid
 
     def lookup_workflow(self, url: str) -> Optional[RID]:
+        """Given a URL, look in the workflow table to find a matching workflow."""
         workflow_path = self.pathBuilder.schemas[self.ml_schema].Workflow
         try:
             url_column = workflow_path.URL
@@ -1049,7 +1137,7 @@ class DerivaML(Dataset):
 
     def create_workflow(
         self, name: str, workflow_type: str, description: str = "", create: bool = True
-    ) -> RID:
+    ) -> RID | None:
         """Identify current executing program and return a workflow RID for it
 
         Determine the notebook or script that is currently being executed. Assume that  this is
@@ -1069,20 +1157,21 @@ class DerivaML(Dataset):
 
         if is_dirty:
             self._logger.warning(
-                f"File {self._script_path} has been modified since last commit. Consider commiting before executing"
+                f"File {self.executable_path} has been modified since last commit. Consider commiting before executing"
             )
 
         # If you are in a notebook, strip out the outputs before computing the checksum.
         cmd = (
-            f"nbstripout {self._script_path} | git hash-object --stdin"
+            f"nbstripout {self.executable_path} | git hash-object --stdin"
             if self._is_notebook
-            else f"git hash-object {self._script_path}"
+            else f"git hash-object {self.executable_path}"
         )
         checksum = subprocess.run(
             cmd,
             capture_output=True,
             text=True,
             check=True,
+            shell=True,
         ).stdout.strip()
 
         workflow = Workflow(
@@ -1109,12 +1198,14 @@ class DerivaML(Dataset):
         # Get repo URL from local github repo.
         try:
             result = subprocess.run(
-                ["git", "remote", "get-url", "origin"], capture_output=True, text=True,
-                cwd=self._script_path.parent,
+                ["git", "remote", "get-url", "origin"],
+                capture_output=True,
+                text=True,
+                cwd=self.executable_path.parent,
             )
             github_url = result.stdout.strip().removesuffix(".git")
         except subprocess.CalledProcessError:
-            raise DerivaMLException(f"No GIT remote found")
+            raise DerivaMLException("No GIT remote found")
 
         # Find the root directory for the repository
         repo_root = self._get_git_root()
@@ -1123,7 +1214,7 @@ class DerivaML(Dataset):
         try:
             result = subprocess.run(
                 ["git", "status", "--porcelain"],
-                cwd=self._script_path.parent,
+                cwd=self.executable_path.parent,
                 capture_output=True,
                 text=True,
                 check=True,
@@ -1136,14 +1227,14 @@ class DerivaML(Dataset):
 
         """Get SHA-1 hash of latest commit of the file in the repository"""
         result = subprocess.run(
-            ["git", "log", "-n", "1", "--pretty=format:%H" "--", self._script_path],
-            cwd=self._script_path.parent,
+            ["git", "log", "-n", "1", "--pretty=format:%H--", self.executable_path],
+            cwd=self.executable_path.parent,
             capture_output=True,
             text=True,
             check=True,
         )
         sha = result.stdout.strip()
-        url = f"{github_url}/blob/{sha}/{self._script_path.relative_to(repo_root)}"
+        url = f"{github_url}/blob/{sha}/{self.executable_path.relative_to(repo_root)}"
         return url, is_dirty
 
     # @validate_call
@@ -1174,6 +1265,7 @@ class DerivaML(Dataset):
 
     # @validate_call
     def restore_execution(self, execution_rid: Optional[RID] = None) -> "Execution":
+        """Return an Execution object for a previously started execution with the specified RID."""
         from .execution import Execution
 
         # Find path to execution

deriva_ml/deriva_model.py

@@ -21,7 +21,7 @@ from .deriva_definitions import (
 
 from collections import Counter
 from pydantic import validate_call, ConfigDict
-from typing import Iterable
+from typing import Iterable, Optional
 
 
 class DerivaModel:
@@ -267,7 +267,7 @@ class DerivaModel:
     def _schema_to_paths(
         self,
         root: Table = None,
-        path: list[Table] = None,
+        path: Optional[list[Table]] = None,
     ) -> list[list[Table]]:
         """Recursively walk over the domain schema graph and extend the current path.