pytrilogy 0.0.2.8__py3-none-any.whl → 0.0.2.9__py3-none-any.whl

trilogy/core/processing/node_generators/filter_node.py

@@ -2,13 +2,14 @@ from typing import List
 
 
 from trilogy.core.enums import JoinType
-from trilogy.core.models import Concept, Environment, FilterItem, Grain
+from trilogy.core.models import Concept, Environment, FilterItem, Grain, WhereClause
 from trilogy.core.processing.nodes import (
     FilterNode,
     MergeNode,
     NodeJoin,
     History,
     StrategyNode,
+    SelectNode,
 )
 from trilogy.core.processing.node_generators.common import (
     resolve_filter_parent_concepts,
@@ -28,6 +29,7 @@ def gen_filter_node(
     depth: int,
     source_concepts,
     history: History | None = None,
+    conditions: WhereClause | None = None,
 ) -> StrategyNode | None:
     immediate_parent, parent_row_concepts, parent_existence_concepts = (
         resolve_filter_parent_concepts(concept)
@@ -37,10 +39,10 @@ def gen_filter_node(
     where = concept.lineage.where
 
     logger.info(
-        f"{padding(depth)}{LOGGER_PREFIX} fetching filter node row parents {[x.address for x in parent_row_concepts]}"
+        f"{padding(depth)}{LOGGER_PREFIX} filter {concept.address} derived from {immediate_parent.address} row parents {[x.address for x in parent_row_concepts]} and {[[y.address] for x  in parent_existence_concepts for y  in x]} existence parents"
     )
     core_parents = []
-    parent: StrategyNode = source_concepts(
+    row_parent: StrategyNode = source_concepts(
         mandatory_list=parent_row_concepts,
         environment=environment,
         g=g,
@@ -48,33 +50,7 @@ def gen_filter_node(
         history=history,
     )
 
-    if not parent:
-        logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} filter node row parents {[x.address for x in parent_row_concepts]} could not be found"
-        )
-        return None
-
-    if not local_optional and not parent_existence_concepts:
-        optimized_pushdown = True
-    else:
-        optimized_pushdown = False
-
-    if optimized_pushdown:
-        if parent.conditions:
-            parent.conditions = parent.conditions + where.conditional
-        else:
-            parent.conditions = where.conditional
-        parent.output_concepts = [concept]
-        parent.grain = Grain(components=[concept])
-        parent.rebuild_cache()
-
-        logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} returning optimized filter node with pushdown to parent with condition {where.conditional}"
-        )
-        return parent
-
-    core_parents.append(parent)
-
+    flattened_existence = [x for y in parent_existence_concepts for x in y]
     if parent_existence_concepts:
         for existence_tuple in parent_existence_concepts:
             if not existence_tuple:
@@ -95,26 +71,91 @@ def gen_filter_node(
                 )
                 return None
             core_parents.append(parent_existence)
-    flattened_existence = [x for y in parent_existence_concepts for x in y]
-    filter_node = FilterNode(
-        input_concepts=unique(
-            [immediate_parent] + parent_row_concepts + flattened_existence,
-            "address",
-        ),
-        output_concepts=[concept, immediate_parent] + parent_row_concepts,
-        environment=environment,
-        g=g,
-        parents=core_parents,
-        grain=Grain(
-            components=[immediate_parent] + parent_row_concepts,
-        ),
-    )
+    if not row_parent:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} filter node row parents {[x.address for x in parent_row_concepts]} could not be found"
+        )
+        return None
+
+    optimized_pushdown = False
+    if not local_optional:
+        optimized_pushdown = True
+    elif conditions and conditions == where:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} query conditions are the same as filter conditions, can optimize across all concepts"
+        )
+        optimized_pushdown = True
+
+    if optimized_pushdown:
+        if isinstance(row_parent, SelectNode):
+            parent = StrategyNode(
+                input_concepts=row_parent.output_concepts,
+                output_concepts=[concept] + row_parent.output_concepts,
+                environment=row_parent.environment,
+                g=row_parent.g,
+                parents=[row_parent] + core_parents,
+                depth=row_parent.depth,
+                partial_concepts=row_parent.partial_concepts,
+                force_group=False,
+                conditions=(
+                    row_parent.conditions + where.conditional
+                    if row_parent.conditions
+                    else where.conditional
+                ),
+                existence_concepts=row_parent.existence_concepts,
+            )
+        else:
+            parent = row_parent
+
+        expected_output = [concept] + [
+            x
+            for x in local_optional
+            if x.address in [y.address for y in parent.output_concepts]
+        ]
+        parent.add_parents(core_parents)
+        parent.add_condition(where.conditional)
+        parent.add_existence_concepts(flattened_existence)
+        parent.set_output_concepts(expected_output)
+        parent.grain = Grain(
+            components=(
+                list(immediate_parent.keys)
+                if immediate_parent.keys
+                else [immediate_parent]
+            )
+            + [
+                x
+                for x in local_optional
+                if x.address in [y.address for y in parent.output_concepts]
+            ]
+        )
+        parent.rebuild_cache()
+
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} returning optimized filter node with pushdown to parent with condition {where.conditional}"
+        )
+        filter_node = parent
+    else:
+        core_parents.append(row_parent)
+
+        filter_node = FilterNode(
+            input_concepts=unique(
+                [immediate_parent] + parent_row_concepts + flattened_existence,
+                "address",
+            ),
+            output_concepts=[concept, immediate_parent] + parent_row_concepts,
+            environment=environment,
+            g=g,
+            parents=core_parents,
+            grain=Grain(
+                components=[immediate_parent] + parent_row_concepts,
+            ),
+        )
 
-    assert filter_node.resolve().grain == Grain(
-        components=[immediate_parent] + parent_row_concepts,
-    )
     if not local_optional or all(
-        [x.address in [y.address for y in parent_row_concepts] for x in local_optional]
+        [
+            x.address in [y.address for y in filter_node.output_concepts]
+            for x in local_optional
+        ]
     ):
         outputs = [
             x

trilogy/core/processing/node_generators/group_node.py

@@ -1,4 +1,4 @@
-from trilogy.core.models import Concept, Environment, LooseConceptList
+from trilogy.core.models import Concept, Environment, LooseConceptList, WhereClause
 from trilogy.utility import unique
 from trilogy.core.processing.nodes import GroupNode, StrategyNode, History
 from typing import List
@@ -22,6 +22,7 @@ def gen_group_node(
     depth: int,
     source_concepts,
     history: History | None = None,
+    conditions: WhereClause | None = None,
 ):
     # aggregates MUST always group to the proper grain
     # except when the
@@ -53,6 +54,7 @@ def gen_group_node(
             g=g,
             depth=depth,
             history=history,
+            conditions=conditions,
         )
         if not parent:
             logger.info(

trilogy/core/processing/node_generators/rowset_node.py

@@ -12,7 +12,7 @@ from typing import List
 
 from trilogy.core.enums import JoinType, PurposeLineage
 from trilogy.constants import logger
-from trilogy.core.processing.utility import padding, unique
+from trilogy.core.processing.utility import padding
 from trilogy.core.processing.node_generators.common import concept_to_relevant_joins
 
 
@@ -28,6 +28,8 @@ def gen_rowset_node(
     source_concepts,
     history: History | None = None,
 ) -> StrategyNode | None:
+    from trilogy.core.query_processor import get_query_node
+
     if not isinstance(concept.lineage, RowsetItem):
         raise SyntaxError(
             f"Invalid lineage passed into rowset fetch, got {type(concept.lineage)}, expected {RowsetItem}"
@@ -35,54 +37,14 @@ def gen_rowset_node(
     lineage: RowsetItem = concept.lineage
     rowset: RowsetDerivationStatement = lineage.rowset
     select: SelectStatement | MultiSelectStatement = lineage.rowset.select
-    existence_parents: List[StrategyNode] = []
-    if where := select.where_clause:
-        targets = select.output_components + where.conditional.row_arguments
-        for sub_select in where.conditional.existence_arguments:
-            logger.info(
-                f"{padding(depth)}{LOGGER_PREFIX} generating parent existence node with {[x.address for x in sub_select]}"
-            )
-            parent_check = source_concepts(
-                mandatory_list=sub_select,
-                environment=environment,
-                g=g,
-                depth=depth + 1,
-                history=history,
-            )
-            if not parent_check:
-                logger.info(
-                    f"{padding(depth)}{LOGGER_PREFIX} Cannot generate parent existence node for rowset node for {concept}"
-                )
-                return None
-            existence_parents.append(parent_check)
-    else:
-        targets = select.output_components
-    node: StrategyNode = source_concepts(
-        mandatory_list=unique(targets, "address"),
-        environment=environment,
-        g=g,
-        depth=depth + 1,
-        history=history,
-    )
+    node = get_query_node(environment, select, graph=g, history=history)
 
     if not node:
         logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} Cannot generate rowset node for {concept}"
+            f"{padding(depth)}{LOGGER_PREFIX} Cannot generate parent rowset node for {concept}"
         )
         return None
-    # add our existence concepts in
-    if existence_parents:
-        node.parents += existence_parents
-        # we don't need to join to any existence parents
-        # if isinstance(node, MergeNode) and node.node_joins is None:
-        #     # set it explicitly to empty to avoid inference
-        #     node.node_joins = []
-        for parent in existence_parents:
-            for x in parent.output_concepts:
-                if x.address not in node.output_lcl:
-                    node.existence_concepts.append(x)
 
-    node.conditions = select.where_clause.conditional if select.where_clause else None
     enrichment = set([x.address for x in local_optional])
     rowset_relevant = [x for x in rowset.derived_concepts]
     select_hidden = set([x.address for x in select.hidden_components])
@@ -113,26 +75,23 @@ def gen_rowset_node(
     # but don't include anything aggregate at this point
     assert node.resolution_cache
 
-    node.resolution_cache.grain = concept_list_to_grain(
+    node.grain = concept_list_to_grain(
         node.output_concepts, parent_sources=node.resolution_cache.datasources
     )
 
+    node.rebuild_cache()
+
     possible_joins = concept_to_relevant_joins(additional_relevant)
-    if not local_optional:
+    if not local_optional or all(
+        x.address in [y.address for y in node.output_concepts] for x in local_optional
+    ):
         logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} no enriched required for rowset node; exiting early"
+            f"{padding(depth)}{LOGGER_PREFIX} no enrichment required for rowset node as all optional found or no optional; exiting early."
         )
         return node
     if not possible_joins:
         logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} no possible joins for rowset node; exiting early"
-        )
-        return node
-    if all(
-        [x.address in [y.address for y in node.output_concepts] for x in local_optional]
-    ):
-        logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} all enriched concepts returned from base rowset node; exiting early"
+            f"{padding(depth)}{LOGGER_PREFIX} no possible joins for rowset node to get {[x.address for x in local_optional]}; have {[x.address for x in node.output_concepts]}"
         )
         return node
     enrich_node: MergeNode = source_concepts(  # this fetches the parent + join keys

trilogy/core/processing/node_generators/select_node.py

@@ -7,6 +7,7 @@ from trilogy.core.models import (
     Grain,
     LooseConceptList,
     Datasource,
+    WhereClause,
 )
 from trilogy.core.processing.nodes import (
     StrategyNode,
@@ -67,17 +68,6 @@ def dm_to_strategy_node(
             f"{padding(depth)}{LOGGER_PREFIX} target grain is not subset of datasource grain {datasource.grain}, required to group"
         )
         force_group = True
-    # if isinstance(datasource, MergeDatasource):
-    #     # if we're within a namespace, don't find merge nodes
-    #     bcandidate: StrategyNode = gen_environment_merge_node(
-    #         all_concepts=dm.matched.concepts,
-    #         environment=environment,
-    #         g=g,
-    #         depth=depth,
-    #         datasource=datasource,
-    #         source_concepts=source_concepts,
-    #     )
-    # else:
     bcandidate: StrategyNode = SelectNode(
         input_concepts=[c.concept for c in datasource.columns],
         output_concepts=dm.matched.concepts,
@@ -116,6 +106,7 @@ def gen_select_nodes_from_tables_v2(
     target_grain: Grain,
     source_concepts: Callable,
     accept_partial: bool = False,
+    conditions: WhereClause | None = None,
 ) -> tuple[bool, list[Concept], list[StrategyNode]]:
     # if we have only constants
     # we don't need a table
@@ -252,6 +243,7 @@ def gen_select_node_from_table(
     target_grain: Grain,
     source_concepts,
     accept_partial: bool = False,
+    conditions: WhereClause | None = None,
 ) -> Optional[StrategyNode]:
     # if we have only constants
     # we don't need a table
@@ -372,6 +364,9 @@ def gen_select_node_from_table(
             grain=Grain(components=all_concepts),
             conditions=datasource.where.conditional if datasource.where else None,
         )
+        # if conditions:
+        #     for component in conditions.components:
+        #         if
         # we need to nest the group node one further
         if force_group is True:
             candidate: StrategyNode = GroupNode(
@@ -407,6 +402,7 @@ def gen_select_node(
     fail_if_not_found: bool = True,
     accept_partial_optional: bool = True,
     target_grain: Grain | None = None,
+    conditions: WhereClause | None = None,
 ) -> StrategyNode | None:
     all_concepts = [concept] + local_optional
     all_lcl = LooseConceptList(concepts=all_concepts)
@@ -445,6 +441,7 @@ def gen_select_node(
         accept_partial=accept_partial,
         target_grain=target_grain,
         source_concepts=source_concepts,
+        conditions=conditions,
     )
     if ds:
         logger.info(
@@ -461,6 +458,7 @@ def gen_select_node(
         target_grain=target_grain,
         accept_partial=accept_partial,
         source_concepts=source_concepts,
+        conditions=conditions,
     )
     if parents and (all_found or accept_partial_optional):
         all_partial = [
@@ -488,7 +486,6 @@ def gen_select_node(
         if len(parents) == 1:
             candidate = parents[0]
         else:
-
             candidate = MergeNode(
                 output_concepts=[concept] + found,
                 input_concepts=[concept] + found,
@@ -499,8 +496,8 @@ def gen_select_node(
                 partial_concepts=all_partial,
                 grain=inferred_grain,
             )
+
         candidate.depth += 1
-        # source_grain = candidate.grain
         if force_group:
             logger.info(
                 f"{padding(depth)}{LOGGER_PREFIX} datasource grain {inferred_grain} does not match target grain {target_grain} for select, adding group node"

trilogy/core/processing/node_generators/unnest_node.py

@@ -1,8 +1,8 @@
 from typing import List
 
 
-from trilogy.core.models import Concept, Function
-from trilogy.core.processing.nodes import SelectNode, UnnestNode, History, StrategyNode
+from trilogy.core.models import Concept, Function, WhereClause
+from trilogy.core.processing.nodes import UnnestNode, History, StrategyNode
 from trilogy.core.processing.utility import padding
 from trilogy.constants import logger
 
@@ -17,6 +17,7 @@ def gen_unnest_node(
     depth: int,
     source_concepts,
     history: History | None = None,
+    conditions: WhereClause | None = None,
 ) -> StrategyNode | None:
     arguments = []
     if isinstance(concept.lineage, Function):
@@ -28,6 +29,7 @@ def gen_unnest_node(
             g=g,
             depth=depth + 1,
             history=history,
+            conditions=conditions,
         )
         if not parent:
             logger.info(
@@ -46,7 +48,7 @@ def gen_unnest_node(
     # we need to sometimes nest an unnest node,
     # as unnest operations are not valid in all situations
     # TODO: inline this node when we can detect it's safe
-    new = SelectNode(
+    new = StrategyNode(
         input_concepts=[concept] + local_optional,
         output_concepts=[concept] + local_optional,
         environment=environment,

trilogy/core/processing/node_generators/window_node.py

@@ -1,7 +1,7 @@
 from typing import List
 
 
-from trilogy.core.models import Concept, WindowItem, Environment
+from trilogy.core.models import Concept, WindowItem, Environment, WhereClause
 from trilogy.utility import unique
 from trilogy.core.processing.nodes import (
     WindowNode,
@@ -38,19 +38,36 @@ def gen_window_node(
     depth: int,
     source_concepts,
     history: History | None = None,
+    conditions: WhereClause | None = None,
 ) -> WindowNode | MergeNode | None:
     parent_concepts = resolve_window_parent_concepts(concept)
-
     parent_node = source_concepts(
         mandatory_list=parent_concepts,
         environment=environment,
         g=g,
         depth=depth + 1,
         history=history,
+        conditions=conditions,
     )
     if not parent_node:
         logger.info(f"{padding(depth)}{LOGGER_PREFIX} window node parents unresolvable")
         return None
+    parent_node.resolve()
+    if not all(
+        [
+            x.address in [y.address for y in parent_node.output_concepts]
+            for x in parent_concepts
+        ]
+    ):
+        missing = [
+            x
+            for x in parent_concepts
+            if x.address not in [y.address for y in parent_node.output_concepts]
+        ]
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} window node parents unresolvable, missing {missing}"
+        )
+        raise SyntaxError
     _window_node = WindowNode(
         input_concepts=parent_concepts,
         output_concepts=[concept] + parent_concepts,
@@ -61,6 +78,8 @@ def gen_window_node(
         ],
         depth=depth,
     )
+    _window_node.rebuild_cache()
+    _window_node.resolve()
     window_node = MergeNode(
         parents=[_window_node],
         environment=environment,
@@ -71,6 +90,7 @@ def gen_window_node(
         force_group=False,
         depth=depth,
     )
+    window_node.resolve()
     if not local_optional:
         return window_node
     logger.info(f"{padding(depth)}{LOGGER_PREFIX} window node requires enrichment")
@@ -84,4 +104,5 @@ def gen_window_node(
         source_concepts=source_concepts,
         log_lambda=create_log_lambda(LOGGER_PREFIX, depth, logger),
         history=history,
+        conditions=conditions,
     )

trilogy/core/processing/nodes/__init__.py

@@ -6,7 +6,7 @@ from .window_node import WindowNode
 from .base_node import StrategyNode, NodeJoin
 from .unnest_node import UnnestNode
 from pydantic import BaseModel, Field, ConfigDict
-from trilogy.core.models import Concept, Environment
+from trilogy.core.models import Concept, Environment, WhereClause
 
 
 class History(BaseModel):
@@ -15,23 +15,42 @@ class History(BaseModel):
     started: set[str] = Field(default_factory=set)
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
-    def _concepts_to_lookup(self, search: list[Concept], accept_partial: bool) -> str:
+    def _concepts_to_lookup(
+        self,
+        search: list[Concept],
+        accept_partial: bool,
+        conditions: WhereClause | None = None,
+    ) -> str:
+        if conditions:
+            return (
+                "-".join([c.address for c in search])
+                + str(accept_partial)
+                + str(conditions)
+            )
         return "-".join([c.address for c in search]) + str(accept_partial)
 
     def search_to_history(
-        self, search: list[Concept], accept_partial: bool, output: StrategyNode | None
+        self,
+        search: list[Concept],
+        accept_partial: bool,
+        output: StrategyNode | None,
+        conditions: WhereClause | None = None,
     ):
-        self.history[self._concepts_to_lookup(search, accept_partial)] = output
+        self.history[
+            self._concepts_to_lookup(search, accept_partial, conditions=conditions)
+        ] = output
 
     def get_history(
         self,
         search: list[Concept],
+        conditions: WhereClause | None = None,
         accept_partial: bool = False,
         parent_key: str = "",
     ) -> StrategyNode | None | bool:
         key = self._concepts_to_lookup(
             search,
             accept_partial,
+            conditions,
         )
         if parent_key and parent_key == key:
             raise ValueError(
@@ -48,11 +67,13 @@ class History(BaseModel):
         self,
         search: list[Concept],
         accept_partial: bool = False,
+        conditions: WhereClause | None = None,
     ):
         self.started.add(
             self._concepts_to_lookup(
                 search,
-                accept_partial,
+                accept_partial=accept_partial,
+                conditions=conditions,
             )
         )
 
@@ -60,11 +81,13 @@ class History(BaseModel):
         self,
         search: list[Concept],
         accept_partial: bool = False,
+        conditions: WhereClause | None = None,
     ):
         return (
             self._concepts_to_lookup(
                 search,
                 accept_partial,
+                conditions=conditions,
             )
             in self.started
         )
@@ -76,6 +99,7 @@ class History(BaseModel):
         accept_partial: bool,
         fail_if_not_found: bool,
         accept_partial_optional: bool,
+        conditions: WhereClause | None = None,
     ) -> str:
         return (
             str(main.address)
@@ -84,6 +108,7 @@ class History(BaseModel):
             + str(accept_partial)
             + str(fail_if_not_found)
             + str(accept_partial_optional)
+            + str(conditions)
         )
 
     def gen_select_node(
@@ -97,6 +122,7 @@ class History(BaseModel):
         fail_if_not_found: bool = False,
         accept_partial: bool = False,
         accept_partial_optional: bool = False,
+        conditions: WhereClause | None = None,
     ) -> StrategyNode | None:
         from trilogy.core.processing.node_generators.select_node import gen_select_node
 
@@ -105,7 +131,8 @@ class History(BaseModel):
             local_optional,
             accept_partial,
             fail_if_not_found,
-            accept_partial_optional,
+            accept_partial_optional=accept_partial_optional,
+            conditions=conditions,
         )
         if fingerprint in self.select_history:
             return self.select_history[fingerprint]
@@ -119,6 +146,7 @@ class History(BaseModel):
             accept_partial=accept_partial,
             accept_partial_optional=accept_partial_optional,
             source_concepts=source_concepts,
+            conditions=conditions,
         )
         self.select_history[fingerprint] = gen
         return gen