deriva-ml 1.8.0__py3-none-any.whl → 1.8.2__py3-none-any.whl

deriva_ml/VERSION.py

@@ -1 +1 @@
-__version__ = "1.8.0"
+__version__ = "1.8.2"

deriva_ml/dataset.py

@@ -447,7 +447,7 @@ class Dataset:
 
     # @validate_call
     def list_dataset_members(
-        self, dataset_rid: RID, recurse: bool = False
+        self, dataset_rid: RID, recurse: bool = False, limit: Optional[int] = None
     ) -> dict[str, list[dict[str, Any]]]:
         """Return a list of entities associated with a specific dataset_table.
 
@@ -455,6 +455,7 @@ class Dataset:
             dataset_rid: param recurse: If this is a nested dataset_table, list the members of the contained datasets
             dataset_rid: RID:
             recurse:  (Default value = False)
+            limit: If provided, the maxiumum number of members to return for each element type.
 
         Returns:
             Dictionary of entities associated with a specific dataset_table.  Key is the table from which the elements
@@ -492,7 +493,9 @@ class Dataset:
                 target_path,
                 on=(member_path.columns[member_column] == target_path.columns["RID"]),
             )
-            target_entities = list(path.entities().fetch())
+            target_entities = list(
+                path.entities().fetch(limit=limit) if limit else path.entities().fetch()
+            )
             members[target_table.name].extend(target_entities)
             if recurse and target_table == self.dataset_table:
                 # Get the members for all the nested datasets and add to the member list.
@@ -723,21 +726,16 @@ class Dataset:
             for o in writer(f"{table.schema.name}:{table.name}", table.name, table)
         ]
 
-    def _table_paths(self) -> Iterator[tuple[list[str], list[str], list[Table]]]:
+    def _table_paths(
+        self, dataset: DatasetSpec = None
+    ) -> Iterator[tuple[list[str], list[str], list[Table]]]:
 
         dataset_dataset = self._model.schemas[self._ml_schema].tables["Dataset_Dataset"]
-        paths = self._model._schema_to_paths()
-        nested_paths = paths
-
-        for i in range(self._dataset_nesting_depth()):
-            if i == 0:
-                paths.extend([[self.dataset_table, dataset_dataset]])
-            nested_paths = [
-                [self.dataset_table, dataset_dataset] + p for p in nested_paths
-            ]
-            paths.extend(nested_paths)
 
-        def source_path(path):
+        paths = self._collect_paths(dataset and dataset.rid)
+
+        def source_path(path: tuple[Table, ...]):
+            path = list(path)
             p = [f"{self._model.ml_schema}:Dataset/RID={{Dataset_RID}}"]
             for table in path[1:]:
                 if table == dataset_dataset:
@@ -756,7 +754,65 @@ class Dataset:
 
         return zip(src_paths, dest_paths, target_tables)
 
-    def _dataset_nesting_depth(self):
+    def _collect_paths(
+        self,
+        dataset_rid: Optional[RID] = None,
+        dataset_nesting_depth: Optional[int] = None,
+    ) -> set[tuple[Table, ...]]:
+
+        dataset_nesting_depth = (
+            self._dataset_nesting_depth()
+            if dataset_nesting_depth is None
+            else dataset_nesting_depth
+        )
+        dataset_dataset = self._model.schemas[self._ml_schema].tables["Dataset_Dataset"]
+
+        # Figure out which paths we don't need to query for this dataset.  If no dataset is provided, use them all.
+        dataset_elements = (
+            [
+                self._model.name_to_table(e)
+                for e, m in self.list_dataset_members(
+                    dataset_rid=dataset_rid, limit=1
+                ).items()
+                if m
+            ]
+            if dataset_rid
+            else self.list_dataset_element_types()
+        )
+
+        dataset_associations = [a.table for a in self.dataset_table.find_associations()]
+        included_associations = [
+            a.table
+            for a in self.dataset_table.find_associations()
+            if a.other_fkeys.pop().pk_table in dataset_elements
+        ]
+        # Get the paths through the schema and filter out all of dataset paths not used by this dataset.
+        paths = {
+            tuple(p)
+            for p in self._model._schema_to_paths()
+            if (len(p) == 1)
+            or (p[1] not in dataset_associations)
+            or (p[1] in included_associations)
+        }
+        # Now get paths for nested datasets
+        nested_paths = set()
+        if dataset_rid:
+            for c in self.list_dataset_children(dataset_rid=dataset_rid):
+                nested_paths |= self._collect_paths(c)
+        else:
+            if dataset_nesting_depth:
+                nested_paths = self._collect_paths(
+                    dataset_nesting_depth=dataset_nesting_depth - 1
+                )
+        if nested_paths:
+            paths |= {
+                tuple([self.dataset_table]),
+                (self.dataset_table, dataset_dataset),
+            }
+        paths |= {(self.dataset_table, dataset_dataset) + p for p in nested_paths}
+        return paths
+
+    def _dataset_nesting_depth(self, dataset_rid: Optional[RID] = None) -> int:
         """Determine the maximum dataset nesting depth in the current catalog.
 
         Returns:
@@ -766,7 +822,7 @@ class Dataset:
         def children_depth(
             dataset_rid: RID, nested_datasets: dict[RID, list[RID]]
         ) -> int:
-            """Return the number of nested datasets in the current catalog"""
+            """Return the number of nested datasets for the dataset_rid if provided, otherwise in the current catalog"""
             try:
                 children = nested_datasets[dataset_rid]
                 return (
@@ -783,8 +839,19 @@ class Dataset:
             .schemas[self._ml_schema]
             .tables["Dataset_Dataset"]
         )
+        dataset_children = (
+            [
+                {
+                    "Dataset": dataset_rid,
+                    "Nested_Dataset": c,
+                }  # Make uniform with return from datapath
+                for c in self.list_dataset_children(dataset_rid)
+            ]
+            if dataset_rid
+            else pb.entities().fetch()
+        )
         nested_dataset = defaultdict(list)
-        for ds in pb.entities().fetch():
+        for ds in dataset_children:
             nested_dataset[ds["Dataset"]].append(ds["Nested_Dataset"])
         return (
             max(map(lambda d: children_depth(d, dict(nested_dataset)), nested_dataset))
@@ -793,7 +860,9 @@ class Dataset:
         )
 
     def _dataset_specification(
-        self, writer: Callable[[str, str, Table], list[dict[str, Any]]]
+        self,
+        writer: Callable[[str, str, Table], list[dict[str, Any]]],
+        dataset: DatasetSpec,
     ) -> list[dict[str, Any]]:
         """Output a download/export specification for a dataset_table.  Each element of the dataset_table will be placed in its own dir
         The top level data directory of the resulting BDBag will have one subdirectory for element type. the subdirectory
@@ -833,7 +902,7 @@ class Dataset:
             A dataset_table specification.
         """
         element_spec = []
-        for path in self._table_paths():
+        for path in self._table_paths(dataset=dataset):
             element_spec.extend(writer(*path))
         return self._vocabulary_specification(writer) + element_spec
 
@@ -892,7 +961,7 @@ class Dataset:
                     config_file=spec_file,
                     output_dir=tmp_dir,
                     defer_download=True,
-                    timeout=(10, 300),
+                    timeout=(10, 610),
                     envars={"Dataset_RID": dataset.rid},
                 )
                 minid_page_url = exporter.export()[0]  # Get the MINID launch page
@@ -1042,7 +1111,9 @@ class Dataset:
             validated_check.touch()
         return Path(bag_path)
 
-    def _export_outputs(self) -> list[dict[str, Any]]:
+    def _export_outputs(
+        self, dataset: Optional[DatasetSpec] = None
+    ) -> list[dict[str, Any]]:
         """Return and output specification for the datasets in the provided model
 
         Returns:
@@ -1079,9 +1150,9 @@ class Dataset:
                 "source": {"api": "schema", "skip_root_path": True},
                 "destination": {"type": "json", "name": "schema"},
             },
-        ] + self._dataset_specification(writer)
+        ] + self._dataset_specification(writer, dataset)
 
-    def _processor_params(self) -> list[dict[str, Any]]:
+    def _processor_params(self, dataset: DatasetSpec) -> list[dict[str, Any]]:
         """
         Returns:
           a download specification for the datasets in the provided model.
@@ -1107,7 +1178,7 @@ class Dataset:
                 "processor": "json",
                 "processor_params": {"query_path": "/schema", "output_path": "schema"},
             }
-        ] + self._dataset_specification(writer)
+        ] + self._dataset_specification(writer, dataset)
 
     @staticmethod
     def _download_dataset_element(
@@ -1244,7 +1315,7 @@ class Dataset:
                         },
                     },
                 ]
-                + self._processor_params(),
+                + self._processor_params(dataset),
             },
         }
 

deriva_ml/dataset_aux_classes.py

@@ -187,6 +187,14 @@ class DatasetSpec(BaseModel):
 
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
+    @field_validator("version", mode="before")
+    @classmethod
+    def version_field_validator(cls, v: Any) -> Any:
+        if isinstance(v, dict):
+            return DatasetVersion(**v)
+        else:
+            return v
+
     @model_validator(mode="before")
     @classmethod
     def _check_bare_rid(cls, data: Any) -> dict[str, str | bool]:

deriva_ml/demo_catalog.py

@@ -294,6 +294,7 @@ def create_demo_catalog(
             project_name=project_name,
             logging_level=logging.WARN,
         )
+        working_dir = deriva_ml.working_dir
         dataset_table = deriva_ml.dataset_table
         dataset_table.annotations.update(
             Dataset(

deriva_ml/deriva_ml_base.py

@@ -115,10 +115,12 @@ class DerivaML(Dataset):
             if working_dir
             else Path.home() / "deriva-ml"
         ) / default_workdir
+
         self.working_dir.mkdir(parents=True, exist_ok=True)
         self.cache_dir = (
             Path(cache_dir) if cache_dir else Path.home() / "deriva-ml" / "cache"
         )
+
         self.cache_dir.mkdir(parents=True, exist_ok=True)
 
         # Initialize dataset class.

deriva_ml/deriva_model.py

@@ -265,7 +265,9 @@ class DerivaModel:
         return relationships[0]
 
     def _schema_to_paths(
-        self, root: Table = None, path: list[Table] = None
+        self,
+        root: Table = None,
+        path: list[Table] = None,
     ) -> list[list[Table]]:
         """Recursively walk over the domain schema graph and extend the current path.
 
@@ -278,6 +280,7 @@ class DerivaModel:
           A list of all the paths through the graph.  Each path is a list of tables.
 
         """
+
         root = root or self.model.schemas[self.ml_schema].tables["Dataset"]
         path = path.copy() if path else []
         parent = path[-1] if path else None  # Table that we are coming from.

deriva_ml/execution_configuration.py

@@ -1,12 +1,12 @@
 from __future__ import annotations
 
 import json
-from typing import Optional
+from typing import Optional, Any
 
 from pydantic import (
     BaseModel,
     conlist,
-    ConfigDict,
+    ConfigDict, field_validator,
 )
 from pathlib import Path
 

deriva_ml/test_functions.py

@@ -96,12 +96,12 @@ def execution_test(ml_instance):
         vc.workflow_type, "ML Demo", description="A ML Workflow that uses Deriva ML API"
     )
 
-    api_workflow = Workflow(
+    api_workflow = ml_instance.add_workflow(Workflow(
         name="Manual Workflow",
         url="https://github.com/informatics-isi-edu/deriva-ml/blob/main/docs/Notebooks/DerivaML%20Execution.ipynb",
         workflow_type="Manual Workflow",
         description="A manual operation",
-    )
+    ))
 
     manual_execution = ml_instance.create_execution(
         ExecutionConfiguration(description="Sample Execution", workflow=api_workflow)

{deriva_ml-1.8.0.dist-info → deriva_ml-1.8.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: deriva-ml
-Version: 1.8.0
+Version: 1.8.2
 Summary: Utilities to simplify use of Dervia and Pandas to create reproducable ML pipelines
 Author-email: ISRD <isrd-dev@isi.edu>
 Requires-Python: >=3.10

{deriva_ml-1.8.0.dist-info → deriva_ml-1.8.2.dist-info}/RECORD

@@ -1,19 +1,19 @@
-deriva_ml/VERSION.py,sha256=Oc_xF94AMAHKZkZlB5rBt1iO0TXWFalg65MP4T2qt-A,22
+deriva_ml/VERSION.py,sha256=d6593s-XBNvVxri9lr2qLUDZQ3Zk3-VXHEwdb4pj8qA,22
 deriva_ml/__init__.py,sha256=0PHNB8gRDALLtaffRmU7wCUgWbRHVQZcjuPJxMLNEco,856
 deriva_ml/database_model.py,sha256=uhoyVyd8MQmY8J9ovCH8fjxhZDxxXNkdJyYdeyEGPXA,13898
-deriva_ml/dataset.py,sha256=b0FqUHKsaIKyyjq3O_P9lSP5I5aWAkEDp4nrQn3rzqw,55524
-deriva_ml/dataset_aux_classes.py,sha256=8cnhVTuaIS7NdxF34oKvzH3U1Rx9sc6jiTC6vOmAPF0,6313
+deriva_ml/dataset.py,sha256=5STHbjWomTCPl8isdlcDgLk_K9DLCfACajBAreUAXTQ,58272
+deriva_ml/dataset_aux_classes.py,sha256=YxjQnu2kS9kK_f8bGqhmgE6ty9GNeitCxfvReT9vaM0,6537
 deriva_ml/dataset_bag.py,sha256=e6IHv3saZUnZRfl0EjfnlV2NnmPeOagYYv3PuZqS1l0,11501
-deriva_ml/demo_catalog.py,sha256=eWUiQ1DMkMXFZEYqCGVlLJHOZd7uE5L6-Zr3kl8YUI0,11001
+deriva_ml/demo_catalog.py,sha256=xQPhFlflqwJskNQrQ-jdBSnGzBm2-aONBgcRxfsdNKM,11045
 deriva_ml/deriva_definitions.py,sha256=MGl29ogCzqrlRilMhSuR5tECo4NSHP4CLbJAXRtPH6E,8914
-deriva_ml/deriva_ml_base.py,sha256=GFz6SSjcwFt5KdJyWb7kcjr1srZjyQtanpdo81LlXCI,36441
-deriva_ml/deriva_model.py,sha256=F5zDw-MDV55POAjKB6dorpa7P4KTSN9hjDCN4E9zB9A,11986
+deriva_ml/deriva_ml_base.py,sha256=ShDZlG9F4XrGRUcUINT3bb_P_UdvV1FqSnnPsjGTCLU,36443
+deriva_ml/deriva_model.py,sha256=LV3FjIhIlz13ckZSmu0aOJhT9EVE0-M9oVMudfkxb0g,12004
 deriva_ml/execution.py,sha256=UcXWY1W5Mt_Yzuayd3Pjd-lKzLlMV5QXZFcLvE6Lt0E,28390
-deriva_ml/execution_configuration.py,sha256=PeBB4ZOcZwwMeRpg0QR1sCKD2AwSYdpzIl_PcVllYc0,3678
+deriva_ml/execution_configuration.py,sha256=nMeaG1qYdIgu4BV5atSUlcL8VZ3O6ohGY5iBhtD_LQ4,3700
 deriva_ml/execution_environment.py,sha256=bCRKrCELDbGQDo7_FKfw7e8iMzVjSRZK3baKkqH5-_0,3264
 deriva_ml/feature.py,sha256=7e8WYPCfJSrGxJh9oUTduYSnB5ekybRhXa_0HIigS_w,5459
 deriva_ml/history.py,sha256=qTDLDs8Ow_6r7mDO0gZm0Fg81SWKOAgtCU5pzZoDRgM,2828
-deriva_ml/test_functions.py,sha256=stunfznC759qCrhUBVeymDNBikrfBhqe4pLTw8p6PzA,4370
+deriva_ml/test_functions.py,sha256=-eqLHjjCQCLBNAr1ofbZekNiCOfMISSACRxT_YHER8I,4396
 deriva_ml/upload.py,sha256=HCOChW6bALW_gt0sWUs_81bNPsb72TNs4o0FQsGSLM4,22222
 deriva_ml/build/lib/schema_setup/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deriva_ml/build/lib/schema_setup/alter_annotation.py,sha256=pkwk0WystN69JfAFK4iBJZAZVQKbRs-gN9IFYuS9rfg,1739
@@ -26,9 +26,9 @@ deriva_ml/schema_setup/annotations.py,sha256=Uogm9YkRtoKSdgfQlICqRywbCATppwBO-Xr
 deriva_ml/schema_setup/create_schema.py,sha256=jwziMWJPbjRgjiRBT-KtidnXI8YNEFO74A9fwfptjHY,10626
 deriva_ml/schema_setup/policy.json,sha256=77sf0Imy6CAQV0_VwwbA56_KROJ05WXsvT-Wjtkk538,1633
 deriva_ml/schema_setup/table_comments_utils.py,sha256=-2_ubEpoH7ViLVb-ZfW9wZbQ26DTKNgjkCABMzGu4i4,2140
-deriva_ml-1.8.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-deriva_ml-1.8.0.dist-info/METADATA,sha256=X_PTMVr4ZhOe7XBaEKWk2nPRpfvB3ayZIK5MRdPjQV8,556
-deriva_ml-1.8.0.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-deriva_ml-1.8.0.dist-info/entry_points.txt,sha256=ZiOvrYj022x544TQwi018ujeHRRDahNmwJnzn5ThacM,242
-deriva_ml-1.8.0.dist-info/top_level.txt,sha256=I1Q1dkH96cRghdsFRVqwpa2M7IqJpR2QPUNNc5-Bnpw,10
-deriva_ml-1.8.0.dist-info/RECORD,,
+deriva_ml-1.8.2.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+deriva_ml-1.8.2.dist-info/METADATA,sha256=DnSPqOt32ddlxTwuxGo9iL3DSbUKLIiMitRDMbxZcYQ,556
+deriva_ml-1.8.2.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
+deriva_ml-1.8.2.dist-info/entry_points.txt,sha256=ZiOvrYj022x544TQwi018ujeHRRDahNmwJnzn5ThacM,242
+deriva_ml-1.8.2.dist-info/top_level.txt,sha256=I1Q1dkH96cRghdsFRVqwpa2M7IqJpR2QPUNNc5-Bnpw,10
+deriva_ml-1.8.2.dist-info/RECORD,,

{deriva_ml-1.8.0.dist-info → deriva_ml-1.8.2.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.8.2)
+Generator: setuptools (76.0.0)
 Root-Is-Purelib: true
 Tag: py3-none-any