deriva-ml 1.6.8__py3-none-any.whl → 1.8.0__py3-none-any.whl

deriva_ml/VERSION.py

@@ -1 +1 @@
-__version__ = "1.6.8"
+__version__ = "1.8.0"

deriva_ml/__init__.py

@@ -2,6 +2,7 @@ __all__ = [
     "DerivaML",
     "DerivaMLException",
     "FileUploadState",
+    "FileSpec",
     "ExecutionConfiguration",
     "Workflow",
     "DatasetBag",
@@ -26,6 +27,7 @@ from .deriva_definitions import (
     BuiltinTypes,
     UploadState,
     FileUploadState,
+    FileSpec,
     RID,
     DerivaMLException,
     MLVocab,

deriva_ml/database_model.py

@@ -1,14 +1,15 @@
+"""Ths module constains the definition of the DatabaseModel class.  The role of this class is to provide an nterface between the BDBag representation
+of a dataset and a sqllite database in which the contents of the bag are stored.
+"""
 import logging
 import sqlite3
 
 from csv import reader
 from pathlib import Path
-from typing import Any, Generator, Optional
+from typing import Any, Optional
 from urllib.parse import urlparse
 
-import pandas as pd
 from deriva.core.ermrest_model import Model
-from pydantic import validate_call
 
 from .deriva_definitions import ML_SCHEMA, MLVocab, RID, DerivaMLException
 from .dataset_aux_classes import DatasetVersion, DatasetMinid
@@ -16,7 +17,21 @@ from .deriva_model import DerivaModel
 from .dataset_bag import DatasetBag
 
 
-class DatabaseModel(DerivaModel):
+class DatabaseModelMeta(type):
+    """Use metaclass to ensure that there is onl one instance per path"""
+
+    _paths_loaded: dict[Path:"DatabaseModel"] = {}
+
+    def __call__(cls, *args, **kwargs):
+        logger = logging.getLogger("deriva_ml")
+        bag_path: Path = args[1]
+        if bag_path.as_posix() not in cls._paths_loaded:
+            logger.info(f"Loading {bag_path}")
+            cls._paths_loaded[bag_path] = super().__call__(*args, **kwargs)
+        return cls._paths_loaded[bag_path]
+
+
+class DatabaseModel(DerivaModel, metaclass=DatabaseModelMeta):
     """Read in the contents of a BDBag and create a local SQLite database.
 
         As part of its initialization, this routine will create a sqlite database that has the contents of all the tables
@@ -32,6 +47,9 @@ class DatabaseModel(DerivaModel):
     Because of nested datasets, it's possible that more than one dataset rid is in a bag, or that a dataset rid might
     appear in more than one database. To help manage this, a global list of all the datasets that have been loaded
     into DatabaseModels, is kept in the class variable `_rid_map`.
+    
+    Because you can load diffent versions of a dataset simultaniously, the dataset RID and version number are tracked, and a new
+    sqllite instance is created for every new dataset version present.
 
     Attributes:
         bag_path (Path): path to the local copy of the BDBag
@@ -42,29 +60,9 @@ class DatabaseModel(DerivaModel):
         dataset_table  (Table): the dataset table in the ERMRest model.
     """
 
-    # Keep track of what databases we have loaded.
-    _paths_loaded: dict[Path:"DatabaseModel"] = {}
-
     # Maintain a global map of RIDS to versions and databases.
     _rid_map: dict[RID, list[tuple[DatasetVersion, "DatabaseModel"]]] = {}
 
-    @classmethod
-    @validate_call
-    def register(cls, minid: DatasetMinid, bag_path: Path):
-        """Register a new minid in the list of local databases if it's new, otherwise, return an existing DatabaseModel.
-
-        Args:
-            minid: MINID to the databag that is to be loaded.
-            bag_path: Path to the bag on the local filesystem./
-
-        Returns:
-            A DatabaseModel instance to the loaded bag.
-        """
-        o = cls._paths_loaded.get(bag_path.as_posix())
-        if o:
-            return o
-        return cls(minid, bag_path)
-
     @staticmethod
     def rid_lookup(dataset_rid: RID) -> list[tuple[DatasetVersion, "DatabaseModel"]]:
         """Return a list of DatasetVersion/DatabaseModel instances corresponding to the given RID.
@@ -84,13 +82,12 @@ class DatabaseModel(DerivaModel):
             raise DerivaMLException(f"Dataset {dataset_rid} not found")
 
     def __init__(self, minid: DatasetMinid, bag_path: Path):
-        """Create a new DatabaseModel.  This should only be called via the static Register method
+        """Create a new DatabaseModel.
 
         Args:
             minid: Minid for the specified bag.
             bag_path:  Path to the local copy of the BDBag.
         """
-        DatabaseModel._paths_loaded[bag_path.as_posix()] = self
 
         self.bag_path = bag_path
         self.minid = minid
@@ -342,60 +339,6 @@ class DatabaseModel(DerivaModel):
         except KeyError:
             raise DerivaMLException(f'Table name "{table}" does not exist.')
 
-    def get_table(self, table: str) -> Generator[tuple, None, None]:
-        """Retrieve the contents of the specified table. If schema is not provided as part of the table name,
-        the method will attempt to locate the schema for the table.
-
-        Args:
-            table: return: A generator that yields tuples of column values.
-
-        Returns:
-          A generator that yields tuples of column values.
-
-        """
-        table_name = self.normalize_table_name(table)
-        result = self.dbase.execute(f'SELECT * FROM "{table_name}"')
-        while row := result.fetchone():
-            yield row
-
-    def get_table_as_dataframe(self, table: str) -> pd.DataFrame:
-        """Retrieve the contents of the specified table as a dataframe.
-
-
-        If schema is not provided as part of the table name,
-        the method will attempt to locate the schema for the table.
-
-        Args:
-            table: Table to retrieve data from.
-
-        Returns:
-          A dataframe containing the contents of the specified table.
-        """
-        table_name = self.normalize_table_name(table)
-        return pd.read_sql(f'SELECT * FROM "{table_name}"', con=self.dbase)
-
-    def get_table_as_dict(self, table: str) -> Generator[dict[str, Any], None, None]:
-        """Retrieve the contents of the specified table as a dictionary.
-
-        Args:
-            table: Table to retrieve data from. f schema is not provided as part of the table name,
-                the method will attempt to locate the schema for the table.
-
-        Returns:
-          A generator producing dictionaries containing the contents of the specified table as name/value pairs.
-        """
-        table_name = self.normalize_table_name(table)
-        with self.dbase:
-            col_names = [
-                c[1]
-                for c in self.dbase.execute(
-                    f'PRAGMA table_info("{table_name}")'
-                ).fetchall()
-            ]
-            result = self.dbase.execute(f'SELECT * FROM "{table_name}"')
-            while row := result.fetchone():
-                yield dict(zip(col_names, row))
-
     def delete_database(self):
         """