pytrilogy 0.0.1.109__py3-none-any.whl → 0.0.1.111__py3-none-any.whl

trilogy/core/processing/node_generators/node_merge_node.py

@@ -10,6 +10,7 @@ from trilogy.utility import unique
 from trilogy.core.exceptions import AmbiguousRelationshipResolutionException
 from trilogy.core.processing.utility import padding
 from trilogy.core.processing.graph_utils import extract_mandatory_subgraphs
+from trilogy.core.enums import PurposeLineage
 
 LOGGER_PREFIX = "[GEN_MERGE_NODE]"
 
@@ -65,13 +66,13 @@ def identify_ds_join_paths(
         ]
         if partial and not accept_partial:
             return None
-        # join_candidates.append({"paths": paths, "datasource": datasource})
+
         return PathInfo(
             paths=paths,
             datasource=datasource,
             reduced_concepts=reduce_path_concepts(paths, g),
             concept_subgraphs=extract_mandatory_subgraphs(paths, g),
-        )  # {"paths": paths, "datasource": datasource}
+        )
     return None
 
 
@@ -88,14 +89,7 @@ def gen_merge_node(
     join_candidates: List[PathInfo] = []
     # anchor on datasources
     final_all_concepts = []
-    # implicit_upstream = {}
     for x in all_concepts:
-        # if x.derivation in (PurposeLineage.AGGREGATE, PurposeLineage.BASIC):
-        #     final_all_concepts +=resolve_function_parent_concepts(x)
-        # elif x.derivation == PurposeLineage.FILTER:
-        #     final_all_concepts +=resolve_filter_parent_concepts(x)
-        # else:
-        #     final_all_concepts.append(x)
         final_all_concepts.append(x)
     for datasource in environment.datasources.values():
         path = identify_ds_join_paths(final_all_concepts, g, datasource, accept_partial)
@@ -104,18 +98,25 @@ def gen_merge_node(
     join_candidates.sort(key=lambda x: sum([len(v) for v in x.paths.values()]))
     if not join_candidates:
         return None
-    for join_candidate in join_candidates:
-        logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} Join candidate: {join_candidate.paths}"
-        )
-    join_additions: List[set[str]] = []
+    join_additions: list[set[str]] = []
     for candidate in join_candidates:
         join_additions.append(candidate.reduced_concepts)
-    if not all(
-        [x.issubset(y) or y.issubset(x) for x in join_additions for y in join_additions]
-    ):
+
+    common: set[str] = set()
+    final_candidates: list[set[str]] = []
+    # find all values that show up in every join_additions
+    for ja in join_additions:
+        if not common:
+            common = ja
+        else:
+            common = common.intersection(ja)
+        if all(ja.issubset(y) for y in join_additions):
+            final_candidates.append(ja)
+
+    if not final_candidates:
+        filtered_paths = [x.difference(common) for x in join_additions]
         raise AmbiguousRelationshipResolutionException(
-            f"Ambiguous concept join resolution - possible paths =  {join_additions}. Include an additional concept to disambiguate",
+            f"Ambiguous concept join resolution fetching {[x.address for x in all_concepts]} - unique values in possible paths = {filtered_paths}. Include an additional concept to disambiguate",
             join_additions,
         )
     if not join_candidates:
@@ -123,9 +124,10 @@ def gen_merge_node(
             f"{padding(depth)}{LOGGER_PREFIX} No additional join candidates could be found"
         )
         return None
-    shortest: PathInfo = sorted(join_candidates, key=lambda x: len(x.reduced_concepts))[
-        0
-    ]
+    shortest: PathInfo = sorted(
+        [x for x in join_candidates if x.reduced_concepts in final_candidates],
+        key=lambda x: len(x.reduced_concepts),
+    )[0]
     logger.info(f"{padding(depth)}{LOGGER_PREFIX} final path is {shortest.paths}")
     # logger.info(f'{padding(depth)}{LOGGER_PREFIX} final reduced concepts are {shortest.concs}')
     parents = []
@@ -145,11 +147,20 @@ def gen_merge_node(
                 f"{padding(depth)}{LOGGER_PREFIX} Unable to instantiate target subgraph"
             )
             return None
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} finished subgraph fetch for {[c.address for c in graph]}, have parent {type(parent)}"
+        )
         parents.append(parent)
 
     return MergeNode(
-        input_concepts=[environment.concepts[x] for x in shortest.reduced_concepts],
-        output_concepts=all_concepts,
+        input_concepts=[
+            environment.concepts[x]
+            for x in shortest.reduced_concepts
+            if environment.concepts[x].derivation != PurposeLineage.MERGE
+        ],
+        output_concepts=[
+            x for x in all_concepts if x.derivation != PurposeLineage.MERGE
+        ],
         environment=environment,
         g=g,
         parents=parents,

trilogy/core/processing/node_generators/rowset_node.py

@@ -12,7 +12,7 @@ from typing import List
 
 from trilogy.core.enums import JoinType, PurposeLineage
 from trilogy.constants import logger
-from trilogy.core.processing.utility import padding
+from trilogy.core.processing.utility import padding, unique
 from trilogy.core.processing.node_generators.common import concept_to_relevant_joins
 
 
@@ -35,30 +35,57 @@ def gen_rowset_node(
     lineage: RowsetItem = concept.lineage
     rowset: RowsetDerivationStatement = lineage.rowset
     select: SelectStatement | MultiSelectStatement = lineage.rowset.select
+    parents: List[StrategyNode] = []
     if where := select.where_clause:
-        targets = select.output_components + where.conditional.concept_arguments
+        targets = select.output_components + where.conditional.row_arguments
+        for sub_select in where.conditional.existence_arguments:
+            logger.info(
+                f"{padding(depth)}{LOGGER_PREFIX} generating parent existence node with {[x.address for x in sub_select]}"
+            )
+            parent_check = source_concepts(
+                mandatory_list=sub_select,
+                environment=environment,
+                g=g,
+                depth=depth + 1,
+                history=history,
+            )
+            if not parent_check:
+                logger.info(
+                    f"{padding(depth)}{LOGGER_PREFIX} Cannot generate parent existence node for rowset node for {concept}"
+                )
+                return None
+            parents.append(parent_check)
     else:
         targets = select.output_components
     node: StrategyNode = source_concepts(
-        mandatory_list=targets,
+        mandatory_list=unique(targets, "address"),
         environment=environment,
         g=g,
         depth=depth + 1,
         history=history,
     )
+
+    # add our existence concepts in
+    if parents:
+        node.parents += parents
+        for parent in parents:
+            for x in parent.output_concepts:
+                if x.address not in node.output_lcl:
+                    node.existence_concepts.append(x)
     if not node:
         logger.info(
             f"{padding(depth)}{LOGGER_PREFIX} Cannot generate rowset node for {concept}"
         )
         return None
     node.conditions = select.where_clause.conditional if select.where_clause else None
-    # rebuild any cached info with the new condition clause
-
     enrichment = set([x.address for x in local_optional])
-    rowset_relevant = [
+    rowset_relevant = [x for x in rowset.derived_concepts]
+    select_hidden = set([x.address for x in select.hidden_components])
+    rowset_hidden = [
         x
         for x in rowset.derived_concepts
-        # if x.address == concept.address or x.address in enrichment
+        if isinstance(x.lineage, RowsetItem)
+        and x.lineage.content.address in select_hidden
     ]
     additional_relevant = [
         x for x in select.output_components if x.address in enrichment
@@ -71,7 +98,7 @@ def gen_rowset_node(
     if select.where_clause:
         for item in additional_relevant:
             node.partial_concepts.append(item)
-    node.hidden_concepts = [
+    node.hidden_concepts = rowset_hidden + [
         x
         for x in node.output_concepts
         if x.address not in [y.address for y in local_optional + [concept]]
@@ -81,9 +108,11 @@ def gen_rowset_node(
     # but don't include anything aggregate at this point
     node.rebuild_cache()
     assert node.resolution_cache
+
     node.resolution_cache.grain = concept_list_to_grain(
         node.output_concepts, parent_sources=node.resolution_cache.datasources
     )
+
     possible_joins = concept_to_relevant_joins(additional_relevant)
     if not local_optional:
         logger.info(

trilogy/core/processing/node_generators/select_node.py

@@ -53,7 +53,7 @@ def dm_to_strategy_node(
         # we have to group
         else:
             logger.info(
-                f"{padding(depth)}{LOGGER_PREFIX} not all grain components are in output {str(dm.matched)}, group to actual grain"
+                f"{padding(depth)}{LOGGER_PREFIX} not all grain components {target_grain} are in output {str(dm.matched)}, group to actual grain"
             )
             force_group = True
     elif all([x in dm.matched for x in datasource.grain.components]):
@@ -76,7 +76,7 @@ def dm_to_strategy_node(
         partial_concepts=dm.partial.concepts,
         accept_partial=accept_partial,
         datasource=datasource,
-        grain=Grain(components=dm.matched.concepts),
+        grain=datasource.grain,
     )
     # we need to nest the group node one further
     if force_group is True:
@@ -317,13 +317,19 @@ def gen_select_node_from_table(
         )
         if target_grain and target_grain.issubset(datasource.grain):
 
-            if all([x in all_lcl for x in target_grain.components]):
+            if (
+                all([x in all_lcl for x in target_grain.components])
+                and target_grain == datasource.grain
+            ):
+                logger.info(
+                    f"{padding(depth)}{LOGGER_PREFIX} target grain components match all lcl, group to false"
+                )
                 force_group = False
             # if we are not returning the grain
             # we have to group
             else:
                 logger.info(
-                    f"{padding(depth)}{LOGGER_PREFIX} not all grain components are in output {str(all_lcl)}, group to actual grain"
+                    f"{padding(depth)}{LOGGER_PREFIX} not all grain components {target_grain} are in output {str(all_lcl)}, group to actual grain"
                 )
                 force_group = True
         elif all([x in all_lcl for x in datasource.grain.components]):
@@ -363,7 +369,7 @@ def gen_select_node_from_table(
         else:
             candidate = bcandidate
         logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} found select node with {datasource.identifier}, returning {candidate.output_lcl}"
+            f"{padding(depth)}{LOGGER_PREFIX} found select node with {datasource.identifier}, force group is {force_group}, returning {candidate.output_lcl}"
         )
         candidates[datasource.identifier] = candidate
         scores[datasource.identifier] = -len(partial_concepts)
@@ -467,6 +473,8 @@ def gen_select_node(
         target_grain = Grain()
         for ac in all_concepts:
             target_grain += ac.grain
+    if target_grain.abstract:
+        target_grain = Grain(components=all_concepts)
     if materialized_lcl != all_lcl:
         logger.info(
             f"{padding(depth)}{LOGGER_PREFIX} Skipping select node generation for {concept.address} "
@@ -513,13 +521,15 @@ def gen_select_node(
                 [c.address in [x.address for x in p.partial_concepts] for p in parents]
             )
         ]
-        force_group = False
+        force_group = None
+        inferred_grain = sum([x.grain for x in parents if x.grain], Grain())
         for candidate in parents:
             if candidate.grain and not candidate.grain.issubset(target_grain):
                 force_group = True
         if len(parents) == 1:
             candidate = parents[0]
         else:
+
             candidate = MergeNode(
                 output_concepts=[concept] + found,
                 input_concepts=[concept] + found,
@@ -528,13 +538,13 @@ def gen_select_node(
                 parents=parents,
                 depth=depth,
                 partial_concepts=all_partial,
-                grain=sum([x.grain for x in parents if x.grain], Grain()),
+                grain=inferred_grain,
             )
         candidate.depth += 1
-        source_grain = candidate.grain
+        # source_grain = candidate.grain
         if force_group:
             logger.info(
-                f"{padding(depth)}{LOGGER_PREFIX} datasource grain {source_grain} does not match target grain {target_grain} for select, adding group node"
+                f"{padding(depth)}{LOGGER_PREFIX} datasource grain {inferred_grain} does not match target grain {target_grain} for select, adding group node"
             )
             return GroupNode(
                 output_concepts=candidate.output_concepts,
@@ -545,6 +555,10 @@ def gen_select_node(
                 depth=depth,
                 partial_concepts=candidate.partial_concepts,
             )
+        else:
+            logger.info(
+                f"{padding(depth)}{LOGGER_PREFIX} datasource grain {inferred_grain} matches target grain {target_grain} for select, returning without group"
+            )
         return candidate
 
     if not accept_partial_optional:

trilogy/core/processing/node_generators/unnest_node.py

@@ -2,7 +2,11 @@ from typing import List
 
 
 from trilogy.core.models import Concept, Function
-from trilogy.core.processing.nodes import UnnestNode, History
+from trilogy.core.processing.nodes import SelectNode, UnnestNode, History, StrategyNode
+from trilogy.core.processing.utility import padding
+from trilogy.constants import logger
+
+LOGGER_PREFIX = "[GEN_ROWSET_NODE]"
 
 
 def gen_unnest_node(
@@ -13,7 +17,7 @@ def gen_unnest_node(
     depth: int,
     source_concepts,
     history: History | None = None,
-) -> UnnestNode | None:
+) -> StrategyNode | None:
     arguments = []
     if isinstance(concept.lineage, Function):
         arguments = concept.lineage.concept_arguments
@@ -26,8 +30,12 @@ def gen_unnest_node(
             history=history,
         )
         if not parent:
+            logger.info(
+                f"{padding(depth)}{LOGGER_PREFIX} could not find unnest node parents"
+            )
             return None
-    return UnnestNode(
+
+    base = UnnestNode(
         unnest_concept=concept,
         input_concepts=arguments + local_optional,
         output_concepts=[concept] + local_optional,
@@ -35,3 +43,16 @@ def gen_unnest_node(
         g=g,
         parents=([parent] if (arguments or local_optional) else []),
     )
+    # we need to sometimes nest an unnest node,
+    # as unnest operations are not valid in all situations
+    # TODO: inline this node when we can detect it's safe
+    new = SelectNode(
+        input_concepts=[concept] + local_optional,
+        output_concepts=[concept] + local_optional,
+        environment=environment,
+        g=g,
+        parents=[base],
+    )
+    qds = new.resolve()
+    assert qds.source_map[concept.address] == {base.resolve()}
+    return new

trilogy/core/processing/node_generators/window_node.py

@@ -59,19 +59,21 @@ def gen_window_node(
         parents=[
             parent_node,
         ],
+        depth=depth,
     )
     window_node = MergeNode(
         parents=[_window_node],
         environment=environment,
         g=g,
-        input_concepts=_window_node.input_concepts,
+        input_concepts=[concept] + _window_node.input_concepts,
         output_concepts=_window_node.output_concepts,
         grain=_window_node.grain,
         force_group=False,
+        depth=depth,
     )
     if not local_optional:
         return window_node
-    logger.info(f"{padding(depth)}{LOGGER_PREFIX} group node requires enrichment")
+    logger.info(f"{padding(depth)}{LOGGER_PREFIX} window node requires enrichment")
     return gen_enrichment_node(
         window_node,
         join_keys=concept_to_relevant_joins(parent_concepts),

trilogy/core/processing/nodes/__init__.py

@@ -1,7 +1,7 @@
 from .filter_node import FilterNode
 from .group_node import GroupNode
 from .merge_node import MergeNode
-from .select_node_v2 import SelectNode, StaticSelectNode, ConstantNode
+from .select_node_v2 import SelectNode, ConstantNode
 from .window_node import WindowNode
 from .base_node import StrategyNode, NodeJoin
 from .unnest_node import UnnestNode
@@ -37,10 +37,12 @@ class History(BaseModel):
             raise ValueError(
                 f"Parent key {parent_key} is the same as the current key {key}"
             )
-        return self.history.get(
-            key,
-            False,
-        )
+        if key in self.history:
+            node = self.history[key]
+            if node:
+                return node.copy()
+            return node
+        return False
 
     def log_start(
         self,
@@ -125,7 +127,6 @@ __all__ = [
     "GroupNode",
     "MergeNode",
     "SelectNode",
-    "StaticSelectNode",
     "WindowNode",
     "StrategyNode",
     "NodeJoin",

trilogy/core/processing/nodes/base_node.py

@@ -17,6 +17,7 @@ from trilogy.core.models import (
 from trilogy.core.enums import Purpose, JoinType, PurposeLineage, Granularity
 from trilogy.utility import unique
 from dataclasses import dataclass
+from trilogy.constants import logger
 
 
 def concept_list_to_grain(
@@ -55,11 +56,18 @@ def resolve_concept_map(
         defaultdict(set)
     )
     full_addresses = {c.address for c in full_joins} if full_joins else set()
+    inherited = set([t.address for t in inherited_inputs])
     for input in inputs:
         for concept in input.output_concepts:
+            logger.info(concept.address)
             if concept.address not in input.non_partial_concept_addresses:
                 continue
-            if concept.address not in [t.address for t in inherited_inputs]:
+            if concept.address not in inherited:
+                continue
+            if (
+                isinstance(input, QueryDatasource)
+                and concept.address in input.hidden_concepts
+            ):
                 continue
             if concept.address in full_addresses:
                 concept_map[concept.address].add(input)
@@ -71,11 +79,16 @@ def resolve_concept_map(
         for concept in input.output_concepts:
             if concept.address not in [t.address for t in inherited_inputs]:
                 continue
+            if (
+                isinstance(input, QueryDatasource)
+                and concept.address in input.hidden_concepts
+            ):
+                continue
             if len(concept_map.get(concept.address, [])) == 0:
                 concept_map[concept.address].add(input)
     # this adds our new derived metrics, which are not created in this CTE
     for target in targets:
-        if target not in inherited_inputs:
+        if target.address not in inherited:
             # an empty source means it is defined in this CTE
             concept_map[target.address] = set()
     return concept_map
@@ -108,6 +121,8 @@ class StrategyNode:
         force_group: bool | None = None,
         grain: Optional[Grain] = None,
         hidden_concepts: List[Concept] | None = None,
+        existence_concepts: List[Concept] | None = None,
+        virtual_output_concepts: List[Concept] | None = None,
     ):
         self.input_concepts: List[Concept] = (
             unique(input_concepts, "address") if input_concepts else []
@@ -131,6 +146,8 @@ class StrategyNode:
         self.force_group = force_group
         self.tainted = False
         self.hidden_concepts = hidden_concepts or []
+        self.existence_concepts = existence_concepts or []
+        self.virtual_output_concepts = virtual_output_concepts or []
         for parent in self.parents:
             if not parent:
                 raise SyntaxError("Unresolvable parent")
@@ -162,12 +179,11 @@ class StrategyNode:
             p.resolve() for p in self.parents
         ]
 
-        # if conditional:
-        #     for condition in conditions[1:]:
-        #         conditional += condition
         grain = Grain(components=self.output_concepts)
         source_map = resolve_concept_map(
-            parent_sources, self.output_concepts, self.input_concepts
+            parent_sources,
+            self.output_concepts,
+            self.input_concepts + self.existence_concepts,
         )
         return QueryDatasource(
             input_concepts=self.input_concepts,
@@ -197,6 +213,24 @@ class StrategyNode:
         self.resolution_cache = qds
         return qds
 
+    def copy(self) -> "StrategyNode":
+        return self.__class__(
+            input_concepts=list(self.input_concepts),
+            output_concepts=list(self.output_concepts),
+            environment=self.environment,
+            g=self.g,
+            whole_grain=self.whole_grain,
+            parents=list(self.parents),
+            partial_concepts=list(self.partial_concepts),
+            depth=self.depth,
+            conditions=self.conditions,
+            force_group=self.force_group,
+            grain=self.grain,
+            hidden_concepts=list(self.hidden_concepts),
+            existence_concepts=list(self.existence_concepts),
+            virtual_output_concepts=list(self.virtual_output_concepts),
+        )
+
 
 @dataclass
 class NodeJoin:

trilogy/core/processing/nodes/filter_node.py

@@ -33,7 +33,7 @@ class FilterNode(StrategyNode):
         depth: int = 0,
         conditions: Conditional | Comparison | Parenthetical | None = None,
         partial_concepts: List[Concept] | None = None,
-        force_group: bool = False,
+        force_group: bool | None = False,
     ):
         super().__init__(
             output_concepts=output_concepts,
@@ -47,3 +47,17 @@ class FilterNode(StrategyNode):
             partial_concepts=partial_concepts,
             force_group=force_group,
         )
+
+    def copy(self) -> "FilterNode":
+        return FilterNode(
+            input_concepts=list(self.input_concepts),
+            output_concepts=list(self.output_concepts),
+            environment=self.environment,
+            g=self.g,
+            whole_grain=self.whole_grain,
+            parents=self.parents,
+            depth=self.depth,
+            conditions=self.conditions,
+            partial_concepts=list(self.partial_concepts),
+            force_group=self.force_group,
+        )

trilogy/core/processing/nodes/group_node.py

@@ -33,6 +33,7 @@ class GroupNode(StrategyNode):
         parents: List["StrategyNode"] | None = None,
         depth: int = 0,
         partial_concepts: Optional[List[Concept]] = None,
+        force_group: bool | None = None,
     ):
         super().__init__(
             input_concepts=input_concepts,
@@ -43,6 +44,7 @@ class GroupNode(StrategyNode):
             parents=parents,
             depth=depth,
             partial_concepts=partial_concepts,
+            force_group=force_group,
         )
 
     def _resolve(self) -> QueryDatasource:
@@ -57,7 +59,11 @@ class GroupNode(StrategyNode):
 
         # dynamically select if we need to group
         # because sometimes, we are already at required grain
-        if comp_grain == grain and self.output_lcl == self.input_lcl:
+        if (
+            comp_grain == grain
+            and self.output_lcl == self.input_lcl
+            and self.force_group is not True
+        ):
             # if there is no group by, and inputs equal outputs
             # return the parent
             logger.info(
@@ -111,3 +117,16 @@ class GroupNode(StrategyNode):
             partial_concepts=self.partial_concepts,
             condition=self.conditions,
         )
+
+    def copy(self) -> "GroupNode":
+        return GroupNode(
+            input_concepts=list(self.input_concepts),
+            output_concepts=list(self.output_concepts),
+            environment=self.environment,
+            g=self.g,
+            whole_grain=self.whole_grain,
+            parents=self.parents,
+            depth=self.depth,
+            partial_concepts=list(self.partial_concepts),
+            force_group=self.force_group,
+        )