deriva-ml 1.16.0__py3-none-any.whl → 1.17.0__py3-none-any.whl

deriva_ml/execution/workflow.py

@@ -9,6 +9,7 @@ from typing import Any
 import requests
 from pydantic import BaseModel, PrivateAttr, model_validator
 from requests import RequestException
+from setuptools_scm import get_version
 
 from deriva_ml.core.definitions import RID
 from deriva_ml.core.exceptions import DerivaMLException
@@ -129,6 +130,13 @@ class Workflow(BaseModel):
             self.url, self.checksum = Workflow.get_url_and_checksum(path)
             self.git_root = Workflow._get_git_root(path)
 
+        self.version = get_version(
+            root=str(self.git_root or Path.cwd()),
+            search_parent_directories=True,
+            # Optional but recommended: provide a safe fallback when tags are absent
+            fallback_version="0.0",
+        )
+
         self._logger = logging.getLogger("deriva_ml")
         return self
 

deriva_ml/model/catalog.py

@@ -8,7 +8,7 @@ ML-specific functionality. It handles schema management, feature definitions, an
 from __future__ import annotations
 
 # Standard library imports
-from collections import Counter
+from collections import Counter, defaultdict
 from graphlib import CycleError, TopologicalSorter
 from typing import Any, Callable, Final, Iterable, NewType, TypeAlias
 
@@ -312,7 +312,10 @@ class DerivaModel:
 
         return [t for a in dataset_table.find_associations() if domain_table(t := a.other_fkeys.pop().pk_table)]
 
-    def _prepare_wide_table(self, dataset: DatasetLike, dataset_rid: RID, include_tables: list[str] | None) -> tuple:
+    def _prepare_wide_table(self,
+                            dataset,
+                            dataset_rid: RID,
+                            include_tables: list[str]) -> tuple[dict[str, Any], list[tuple]]:
         """
         Generates details of a wide table from the model
 
@@ -327,7 +330,7 @@ class DerivaModel:
         # Skip over tables that we don't want to include in the denormalized dataset.
         # Also, strip off the Dataset/Dataset_X part of the path so we don't include dataset columns in the denormalized
         # table.
-        include_tables = set(include_tables) if include_tables else set()
+        include_tables = set(include_tables)
         for t in include_tables:
             # Check to make sure the table is in the catalog.
             _ = self.name_to_table(t)
@@ -335,8 +338,11 @@ class DerivaModel:
         table_paths = [
             path
             for path in self._schema_to_paths()
-            if (not include_tables) or include_tables.intersection({p.name for p in path})
+            if path[-1].name in include_tables and include_tables.intersection({p.name for p in path})
         ]
+        paths_by_element = defaultdict(list)
+        for p in table_paths:
+            paths_by_element[p[2].name].append(p)
 
         # Get the names of all of the tables that can be dataset elements.
         dataset_element_tables = {
@@ -344,58 +350,57 @@ class DerivaModel:
         }
 
         skip_columns = {"RCT", "RMT", "RCB", "RMB"}
-        tables = {}
-        graph = {}
-        for path in table_paths:
-            for left, right in zip(path[0:], path[1:]):
-                graph.setdefault(left.name, set()).add(right.name)
-
-        # New lets remove any cycles that we may have in the graph.
-        # We will use a topological sort to find the order in which we need to join the tables.
-        # If we find a cycle, we will remove the table from the graph and splice in an additional ON clause.
-        # We will then repeat the process until there are no cycles.
-        graph_has_cycles = True
-        join_tables = []
-        while graph_has_cycles:
-            try:
-                ts = TopologicalSorter(graph)
-                join_tables = list(reversed(list(ts.static_order())))
-                graph_has_cycles = False
-            except CycleError as e:
-                cycle_nodes = e.args[1]
-                if len(cycle_nodes) > 3:
-                    raise DerivaMLException(f"Unexpected cycle found when normalizing dataset {cycle_nodes}")
-                # Remove cycle from graph and splice in additional ON constraint.
-                graph[cycle_nodes[1]].remove(cycle_nodes[0])
-
-        # The Dataset_Version table is a special case as it points to dataset and dataset to version.
-        if "Dataset_Version" in join_tables:
-            join_tables.remove("Dataset_Version")
-
-        for path in table_paths:
-            for left, right in zip(path[0:], path[1:]):
-                if right.name == "Dataset_Version":
-                    # The Dataset_Version table is a special case as it points to dataset and dataset to version.
-                    continue
-                if join_tables.index(right.name) < join_tables.index(left.name):
-                    continue
-                table_relationship = self._table_relationship(left, right)
-                tables.setdefault(self.normalize_table_name(right.name), set()).add(
-                    (table_relationship[0], table_relationship[1])
-                )
-
+        element_tables = {}
+        for element_table, paths in paths_by_element.items():
+            graph = {}
+            for path in paths:
+                for left, right in zip(path[0:], path[1:]):
+                    graph.setdefault(left.name, set()).add(right.name)
+
+            # New lets remove any cycles that we may have in the graph.
+            # We will use a topological sort to find the order in which we need to join the tables.
+            # If we find a cycle, we will remove the table from the graph and splice in an additional ON clause.
+            # We will then repeat the process until there are no cycles.
+            graph_has_cycles = True
+            element_join_tables = []
+            element_join_conditions = {}
+            while graph_has_cycles:
+                try:
+                    ts = TopologicalSorter(graph)
+                    element_join_tables = list(reversed(list(ts.static_order())))
+                    graph_has_cycles = False
+                except CycleError as e:
+                    cycle_nodes = e.args[1]
+                    if len(cycle_nodes) > 3:
+                        raise DerivaMLException(f"Unexpected cycle found when normalizing dataset {cycle_nodes}")
+                    # Remove cycle from graph and splice in additional ON constraint.
+                    graph[cycle_nodes[1]].remove(cycle_nodes[0])
+
+            # The Dataset_Version table is a special case as it points to dataset and dataset to version.
+            if "Dataset_Version" in element_join_tables:
+                element_join_tables.remove("Dataset_Version")
+
+            for path in paths:
+                for left, right in zip(path[0:], path[1:]):
+                    if right.name == "Dataset_Version":
+                        # The Dataset_Version table is a special case as it points to dataset and dataset to version.
+                        continue
+                    if element_join_tables.index(right.name) < element_join_tables.index(left.name):
+                        continue
+                    table_relationship = self._table_relationship(left, right)
+                    element_join_conditions.setdefault(right.name, set()).add(
+                        (table_relationship[0], table_relationship[1])
+                    )
+            element_tables[element_table] = (element_join_tables, element_join_conditions)
         # Get the list of columns that will appear in the final denormalized dataset.
         denormalized_columns = [
             (table_name, c.name)
-            for table_name in join_tables
+            for table_name in include_tables
             if not self.is_association(table_name)  # Don't include association columns in the denormalized view.'
             for c in self.name_to_table(table_name).columns
-            if c.name not in skip_columns
+            if (not include_tables or table_name in include_tables) and (c.name not in skip_columns)
         ]
-
-        # List of dataset ids to include in the denormalized view.
-        dataset_rids = dataset.list_dataset_children(recurse=True)
-        return join_tables, tables, denormalized_columns, dataset_rids, dataset_element_tables
+        return element_tables, denormalized_columns
 
     def _table_relationship(
         self,