pytrilogy 0.0.1.110__py3-none-any.whl → 0.0.1.112__py3-none-any.whl

trilogy/core/processing/node_generators/unnest_node.py

@@ -2,7 +2,11 @@ from typing import List
 
 
 from trilogy.core.models import Concept, Function
-from trilogy.core.processing.nodes import UnnestNode, History
+from trilogy.core.processing.nodes import SelectNode, UnnestNode, History, StrategyNode
+from trilogy.core.processing.utility import padding
+from trilogy.constants import logger
+
+LOGGER_PREFIX = "[GEN_ROWSET_NODE]"
 
 
 def gen_unnest_node(
@@ -13,7 +17,7 @@ def gen_unnest_node(
     depth: int,
     source_concepts,
     history: History | None = None,
-) -> UnnestNode | None:
+) -> StrategyNode | None:
     arguments = []
     if isinstance(concept.lineage, Function):
         arguments = concept.lineage.concept_arguments
@@ -26,8 +30,12 @@ def gen_unnest_node(
             history=history,
         )
         if not parent:
+            logger.info(
+                f"{padding(depth)}{LOGGER_PREFIX} could not find unnest node parents"
+            )
             return None
-    return UnnestNode(
+
+    base = UnnestNode(
         unnest_concept=concept,
         input_concepts=arguments + local_optional,
         output_concepts=[concept] + local_optional,
@@ -35,3 +43,16 @@ def gen_unnest_node(
         g=g,
         parents=([parent] if (arguments or local_optional) else []),
     )
+    # we need to sometimes nest an unnest node,
+    # as unnest operations are not valid in all situations
+    # TODO: inline this node when we can detect it's safe
+    new = SelectNode(
+        input_concepts=[concept] + local_optional,
+        output_concepts=[concept] + local_optional,
+        environment=environment,
+        g=g,
+        parents=[base],
+    )
+    qds = new.resolve()
+    assert qds.source_map[concept.address] == {base.resolve()}
+    return new

trilogy/core/processing/node_generators/window_node.py

@@ -59,19 +59,21 @@ def gen_window_node(
         parents=[
             parent_node,
         ],
+        depth=depth,
     )
     window_node = MergeNode(
         parents=[_window_node],
         environment=environment,
         g=g,
-        input_concepts=_window_node.input_concepts,
+        input_concepts=[concept] + _window_node.input_concepts,
         output_concepts=_window_node.output_concepts,
         grain=_window_node.grain,
         force_group=False,
+        depth=depth,
     )
     if not local_optional:
         return window_node
-    logger.info(f"{padding(depth)}{LOGGER_PREFIX} group node requires enrichment")
+    logger.info(f"{padding(depth)}{LOGGER_PREFIX} window node requires enrichment")
     return gen_enrichment_node(
         window_node,
         join_keys=concept_to_relevant_joins(parent_concepts),

trilogy/core/processing/nodes/__init__.py

@@ -1,7 +1,7 @@
 from .filter_node import FilterNode
 from .group_node import GroupNode
 from .merge_node import MergeNode
-from .select_node_v2 import SelectNode, StaticSelectNode, ConstantNode
+from .select_node_v2 import SelectNode, ConstantNode
 from .window_node import WindowNode
 from .base_node import StrategyNode, NodeJoin
 from .unnest_node import UnnestNode
@@ -37,10 +37,12 @@ class History(BaseModel):
             raise ValueError(
                 f"Parent key {parent_key} is the same as the current key {key}"
             )
-        return self.history.get(
-            key,
-            False,
-        )
+        if key in self.history:
+            node = self.history[key]
+            if node:
+                return node.copy()
+            return node
+        return False
 
     def log_start(
         self,
@@ -125,7 +127,6 @@ __all__ = [
     "GroupNode",
     "MergeNode",
     "SelectNode",
-    "StaticSelectNode",
     "WindowNode",
     "StrategyNode",
     "NodeJoin",

trilogy/core/processing/nodes/base_node.py

@@ -17,6 +17,7 @@ from trilogy.core.models import (
 from trilogy.core.enums import Purpose, JoinType, PurposeLineage, Granularity
 from trilogy.utility import unique
 from dataclasses import dataclass
+from trilogy.constants import logger
 
 
 def concept_list_to_grain(
@@ -55,11 +56,18 @@ def resolve_concept_map(
         defaultdict(set)
     )
     full_addresses = {c.address for c in full_joins} if full_joins else set()
+    inherited = set([t.address for t in inherited_inputs])
     for input in inputs:
         for concept in input.output_concepts:
+            logger.info(concept.address)
             if concept.address not in input.non_partial_concept_addresses:
                 continue
-            if concept.address not in [t.address for t in inherited_inputs]:
+            if concept.address not in inherited:
+                continue
+            if (
+                isinstance(input, QueryDatasource)
+                and concept.address in input.hidden_concepts
+            ):
                 continue
             if concept.address in full_addresses:
                 concept_map[concept.address].add(input)
@@ -71,11 +79,16 @@ def resolve_concept_map(
         for concept in input.output_concepts:
             if concept.address not in [t.address for t in inherited_inputs]:
                 continue
+            if (
+                isinstance(input, QueryDatasource)
+                and concept.address in input.hidden_concepts
+            ):
+                continue
             if len(concept_map.get(concept.address, [])) == 0:
                 concept_map[concept.address].add(input)
     # this adds our new derived metrics, which are not created in this CTE
     for target in targets:
-        if target not in inherited_inputs:
+        if target.address not in inherited:
             # an empty source means it is defined in this CTE
             concept_map[target.address] = set()
     return concept_map
@@ -108,6 +121,8 @@ class StrategyNode:
         force_group: bool | None = None,
         grain: Optional[Grain] = None,
         hidden_concepts: List[Concept] | None = None,
+        existence_concepts: List[Concept] | None = None,
+        virtual_output_concepts: List[Concept] | None = None,
     ):
         self.input_concepts: List[Concept] = (
             unique(input_concepts, "address") if input_concepts else []
@@ -131,6 +146,8 @@ class StrategyNode:
         self.force_group = force_group
         self.tainted = False
         self.hidden_concepts = hidden_concepts or []
+        self.existence_concepts = existence_concepts or []
+        self.virtual_output_concepts = virtual_output_concepts or []
         for parent in self.parents:
             if not parent:
                 raise SyntaxError("Unresolvable parent")
@@ -162,12 +179,11 @@ class StrategyNode:
             p.resolve() for p in self.parents
         ]
 
-        # if conditional:
-        #     for condition in conditions[1:]:
-        #         conditional += condition
         grain = Grain(components=self.output_concepts)
         source_map = resolve_concept_map(
-            parent_sources, self.output_concepts, self.input_concepts
+            parent_sources,
+            self.output_concepts,
+            self.input_concepts + self.existence_concepts,
         )
         return QueryDatasource(
             input_concepts=self.input_concepts,
@@ -197,6 +213,24 @@ class StrategyNode:
         self.resolution_cache = qds
         return qds
 
+    def copy(self) -> "StrategyNode":
+        return self.__class__(
+            input_concepts=list(self.input_concepts),
+            output_concepts=list(self.output_concepts),
+            environment=self.environment,
+            g=self.g,
+            whole_grain=self.whole_grain,
+            parents=list(self.parents),
+            partial_concepts=list(self.partial_concepts),
+            depth=self.depth,
+            conditions=self.conditions,
+            force_group=self.force_group,
+            grain=self.grain,
+            hidden_concepts=list(self.hidden_concepts),
+            existence_concepts=list(self.existence_concepts),
+            virtual_output_concepts=list(self.virtual_output_concepts),
+        )
+
 
 @dataclass
 class NodeJoin:

trilogy/core/processing/nodes/filter_node.py

@@ -33,7 +33,7 @@ class FilterNode(StrategyNode):
         depth: int = 0,
         conditions: Conditional | Comparison | Parenthetical | None = None,
         partial_concepts: List[Concept] | None = None,
-        force_group: bool = False,
+        force_group: bool | None = False,
     ):
         super().__init__(
             output_concepts=output_concepts,
@@ -47,3 +47,17 @@ class FilterNode(StrategyNode):
             partial_concepts=partial_concepts,
             force_group=force_group,
         )
+
+    def copy(self) -> "FilterNode":
+        return FilterNode(
+            input_concepts=list(self.input_concepts),
+            output_concepts=list(self.output_concepts),
+            environment=self.environment,
+            g=self.g,
+            whole_grain=self.whole_grain,
+            parents=self.parents,
+            depth=self.depth,
+            conditions=self.conditions,
+            partial_concepts=list(self.partial_concepts),
+            force_group=self.force_group,
+        )

trilogy/core/processing/nodes/group_node.py

@@ -33,6 +33,7 @@ class GroupNode(StrategyNode):
         parents: List["StrategyNode"] | None = None,
         depth: int = 0,
         partial_concepts: Optional[List[Concept]] = None,
+        force_group: bool | None = None,
     ):
         super().__init__(
             input_concepts=input_concepts,
@@ -43,6 +44,7 @@ class GroupNode(StrategyNode):
             parents=parents,
             depth=depth,
             partial_concepts=partial_concepts,
+            force_group=force_group,
         )
 
     def _resolve(self) -> QueryDatasource:
@@ -57,7 +59,11 @@ class GroupNode(StrategyNode):
 
         # dynamically select if we need to group
         # because sometimes, we are already at required grain
-        if comp_grain == grain and self.output_lcl == self.input_lcl:
+        if (
+            comp_grain == grain
+            and self.output_lcl == self.input_lcl
+            and self.force_group is not True
+        ):
             # if there is no group by, and inputs equal outputs
             # return the parent
             logger.info(
@@ -111,3 +117,16 @@ class GroupNode(StrategyNode):
             partial_concepts=self.partial_concepts,
             condition=self.conditions,
         )
+
+    def copy(self) -> "GroupNode":
+        return GroupNode(
+            input_concepts=list(self.input_concepts),
+            output_concepts=list(self.output_concepts),
+            environment=self.environment,
+            g=self.g,
+            whole_grain=self.whole_grain,
+            parents=self.parents,
+            depth=self.depth,
+            partial_concepts=list(self.partial_concepts),
+            force_group=self.force_group,
+        )

trilogy/core/processing/nodes/merge_node.py

@@ -12,6 +12,8 @@ from trilogy.core.models import (
     Concept,
     UnnestJoin,
     Conditional,
+    Comparison,
+    Parenthetical,
 )
 from trilogy.utility import unique
 from trilogy.core.processing.nodes.base_node import (
@@ -103,8 +105,9 @@ class MergeNode(StrategyNode):
         force_group: bool | None = None,
         depth: int = 0,
         grain: Grain | None = None,
-        conditions: Conditional | None = None,
+        conditions: Conditional | Comparison | Parenthetical | None = None,
         hidden_concepts: List[Concept] | None = None,
+        virtual_output_concepts: List[Concept] | None = None,
     ):
         super().__init__(
             input_concepts=input_concepts,
@@ -119,10 +122,12 @@ class MergeNode(StrategyNode):
             grain=grain,
             conditions=conditions,
             hidden_concepts=hidden_concepts,
+            virtual_output_concepts=virtual_output_concepts,
         )
         self.join_concepts = join_concepts
         self.force_join_type = force_join_type
         self.node_joins = node_joins
+
         final_joins = []
         if self.node_joins:
             for join in self.node_joins:
@@ -312,17 +317,21 @@ class MergeNode(StrategyNode):
             force_group = None
 
         qd_joins: List[BaseJoin | UnnestJoin] = [*joins]
+        source_map = resolve_concept_map(
+            parent_sources,
+            targets=self.output_concepts,
+            inherited_inputs=self.input_concepts + self.existence_concepts,
+            full_joins=full_join_concepts,
+        )
+        logger.info(
+            f"{self.logging_prefix}{LOGGER_PREFIX} source_map {str(source_map)}"
+        )
         qds = QueryDatasource(
             input_concepts=unique(self.input_concepts, "address"),
             output_concepts=unique(self.output_concepts, "address"),
             datasources=final_datasets,
             source_type=self.source_type,
-            source_map=resolve_concept_map(
-                parent_sources,
-                self.output_concepts,
-                self.input_concepts,
-                full_joins=full_join_concepts,
-            ),
+            source_map=source_map,
             joins=qd_joins,
             grain=grain,
             partial_concepts=self.partial_concepts,
@@ -331,3 +340,23 @@ class MergeNode(StrategyNode):
             hidden_concepts=self.hidden_concepts,
         )
         return qds
+
+    def copy(self) -> "MergeNode":
+        return MergeNode(
+            input_concepts=list(self.input_concepts),
+            output_concepts=list(self.output_concepts),
+            environment=self.environment,
+            g=self.g,
+            whole_grain=self.whole_grain,
+            parents=self.parents,
+            depth=self.depth,
+            partial_concepts=list(self.partial_concepts),
+            force_group=self.force_group,
+            grain=self.grain,
+            conditions=self.conditions,
+            hidden_concepts=list(self.hidden_concepts),
+            virtual_output_concepts=list(self.virtual_output_concepts),
+            node_joins=self.node_joins,
+            join_concepts=list(self.join_concepts) if self.join_concepts else None,
+            force_join_type=self.force_join_type,
+        )

trilogy/core/processing/nodes/select_node_v2.py

@@ -5,14 +5,14 @@ from trilogy.constants import logger
 from trilogy.core.constants import CONSTANT_DATASET
 from trilogy.core.enums import Purpose, PurposeLineage
 from trilogy.core.models import (
-    Datasource,
+    Function,
+    Grain,
     QueryDatasource,
     SourceType,
-    Environment,
     Concept,
-    Grain,
-    Function,
+    Environment,
     UnnestJoin,
+    Datasource,
 )
 from trilogy.utility import unique
 from trilogy.core.processing.nodes.base_node import StrategyNode
@@ -22,39 +22,6 @@ from trilogy.core.exceptions import NoDatasourceException
 LOGGER_PREFIX = "[CONCEPT DETAIL - SELECT NODE]"
 
 
-class StaticSelectNode(StrategyNode):
-    """Static select nodes."""
-
-    source_type = SourceType.SELECT
-
-    def __init__(
-        self,
-        input_concepts: List[Concept],
-        output_concepts: List[Concept],
-        environment: Environment,
-        g,
-        datasource: QueryDatasource,
-        depth: int = 0,
-        partial_concepts: List[Concept] | None = None,
-    ):
-        super().__init__(
-            input_concepts=input_concepts,
-            output_concepts=output_concepts,
-            environment=environment,
-            g=g,
-            whole_grain=True,
-            parents=[],
-            depth=depth,
-            partial_concepts=partial_concepts,
-        )
-        self.datasource = datasource
-
-    def _resolve(self):
-        if self.datasource.grain == Grain():
-            raise NotImplementedError
-        return self.datasource
-
-
 class SelectNode(StrategyNode):
     """Select nodes actually fetch raw data from a table
     Responsible for selecting the cheapest option from which to select.
@@ -75,7 +42,7 @@ class SelectNode(StrategyNode):
         partial_concepts: List[Concept] | None = None,
         accept_partial: bool = False,
         grain: Optional[Grain] = None,
-        force_group: bool = False,
+        force_group: bool | None = False,
     ):
         super().__init__(
             input_concepts=input_concepts,
@@ -119,6 +86,7 @@ class SelectNode(StrategyNode):
             if x.address not in source_map and x.derivation in (
                 PurposeLineage.MULTISELECT,
                 PurposeLineage.MERGE,
+                PurposeLineage.FILTER,
             ):
                 source_map[x.address] = set()
 
@@ -140,6 +108,7 @@ class SelectNode(StrategyNode):
                 c.concept for c in datasource.columns if not c.is_complete
             ],
             source_type=SourceType.DIRECT_SELECT,
+            condition=self.conditions,
             # select nodes should never group
             force_group=self.force_group,
         )
@@ -154,6 +123,7 @@ class SelectNode(StrategyNode):
             source_map={concept.address: set() for concept in self.all_concepts},
             datasources=[datasource],
             grain=datasource.grain,
+            condition=self.conditions,
             joins=[],
             partial_concepts=[],
             source_type=SourceType.CONSTANT,
@@ -191,8 +161,33 @@ class SelectNode(StrategyNode):
             f"Could not find any way to associate required concepts {required}"
         )
 
+    def copy(self) -> "SelectNode":
+        return SelectNode(
+            input_concepts=list(self.input_concepts),
+            output_concepts=list(self.output_concepts),
+            environment=self.environment,
+            g=self.g,
+            datasource=self.datasource,
+            depth=self.depth,
+            parents=self.parents,
+            whole_grain=self.whole_grain,
+            partial_concepts=list(self.partial_concepts),
+            accept_partial=self.accept_partial,
+            grain=self.grain,
+            force_group=self.force_group,
+        )
+
 
 class ConstantNode(SelectNode):
     """Represents a constant value."""
 
-    pass
+    def copy(self) -> "ConstantNode":
+        return ConstantNode(
+            input_concepts=list(self.input_concepts),
+            output_concepts=list(self.output_concepts),
+            environment=self.environment,
+            g=self.g,
+            datasource=self.datasource,
+            depth=self.depth,
+            partial_concepts=list(self.partial_concepts),
+        )

trilogy/core/processing/nodes/unnest_node.py

@@ -52,3 +52,15 @@ class UnnestNode(StrategyNode):
         base.source_map[self.unnest_concept.address] = {unnest}
         base.join_derived_concepts = [self.unnest_concept]
         return base
+
+    def copy(self) -> "UnnestNode":
+        return UnnestNode(
+            unnest_concept=self.unnest_concept,
+            input_concepts=list(self.input_concepts),
+            output_concepts=list(self.output_concepts),
+            environment=self.environment,
+            g=self.g,
+            whole_grain=self.whole_grain,
+            parents=self.parents,
+            depth=self.depth,
+        )

trilogy/core/processing/nodes/window_node.py

@@ -32,3 +32,14 @@ class WindowNode(StrategyNode):
         base = super()._resolve()
         base.grain = Grain(components=self.input_concepts)
         return base
+
+    def copy(self) -> "WindowNode":
+        return WindowNode(
+            input_concepts=list(self.input_concepts),
+            output_concepts=list(self.output_concepts),
+            environment=self.environment,
+            g=self.g,
+            whole_grain=self.whole_grain,
+            parents=self.parents,
+            depth=self.depth,
+        )

trilogy/core/processing/utility.py

@@ -232,20 +232,6 @@ def get_node_joins(
                 raise SyntaxError(
                     f"Could not find join for {x.identifier} with output {[c.address for c in x.output_concepts]}, all {[z.identifier for z in datasources]}"
                 )
-    single_row = [x for x in datasources if x.grain.abstract]
-    for x in single_row:
-        for join in final_joins:
-            found = False
-            for join in final_joins:
-                if (
-                    join.left_datasource.identifier == x.identifier
-                    or join.right_datasource.identifier == x.identifier
-                ):
-                    found = True
-            if not found:
-                raise SyntaxError(
-                    f"Could not find join for {x.identifier} with output {[c.address for c in x.output_concepts]}, all {[z.identifier for z in datasources]}"
-                )
     return final_joins