pytrilogy 0.0.2.7__py3-none-any.whl → 0.0.2.9__py3-none-any.whl

trilogy/core/processing/node_generators/common.py

@@ -9,6 +9,7 @@ from trilogy.core.models import (
     FilterItem,
     Environment,
     LooseConceptList,
+    WhereClause,
 )
 from trilogy.utility import unique
 from trilogy.core.processing.nodes.base_node import StrategyNode
@@ -45,6 +46,17 @@ def resolve_function_parent_concepts(concept: Concept) -> List[Concept]:
     return unique(concept.lineage.concept_arguments, "address")
 
 
+def resolve_condition_parent_concepts(
+    condition: WhereClause,
+) -> Tuple[List[Concept], List[Tuple[Concept, ...]]]:
+    base_existence = []
+    base_rows = []
+    base_rows += condition.row_arguments
+    for ctuple in condition.existence_arguments:
+        base_existence.append(ctuple)
+    return unique(base_rows, "address"), base_existence
+
+
 def resolve_filter_parent_concepts(
     concept: Concept,
 ) -> Tuple[Concept, List[Concept], List[Tuple[Concept, ...]]]:
@@ -55,10 +67,11 @@ def resolve_filter_parent_concepts(
     direct_parent = concept.lineage.content
     base_existence = []
     base_rows = [direct_parent]
-    base_rows += concept.lineage.where.row_arguments
-    # TODO: pass tuple groups through
-    for ctuple in concept.lineage.where.existence_arguments:
-        base_existence.append(ctuple)
+    condition_rows, condition_existence = resolve_condition_parent_concepts(
+        concept.lineage.where
+    )
+    base_rows += condition_rows
+    base_existence += condition_existence
     if direct_parent.grain:
         base_rows += direct_parent.grain.components_copy
     if (
@@ -84,6 +97,7 @@ def gen_property_enrichment_node(
     depth: int,
     source_concepts,
     history: History | None = None,
+    conditions: WhereClause | None = None,
 ):
     required_keys: dict[str, set[str]] = defaultdict(set)
     for x in extra_properties:
@@ -102,6 +116,7 @@ def gen_property_enrichment_node(
             g=g,
             depth=depth + 1,
             history=history,
+            conditions=conditions,
         )
         final_nodes.append(enrich_node)
         node_joins.append(
@@ -147,11 +162,12 @@ def gen_enrichment_node(
     source_concepts,
     log_lambda,
     history: History | None = None,
+    conditions: WhereClause | None = None,
 ):
 
     local_opts = LooseConceptList(concepts=local_optional)
 
-    if local_opts.issubset(LooseConceptList(concepts=base_node.output_concepts)):
+    if local_opts.issubset(base_node.output_lcl):
         log_lambda(
             f"{str(type(base_node).__name__)} has all optional { base_node.output_lcl}, skipping enrichmennt"
         )
@@ -180,6 +196,7 @@ def gen_enrichment_node(
                 depth,
                 source_concepts,
                 history=history,
+                conditions=conditions,
             )
 
     enrich_node: StrategyNode = source_concepts(  # this fetches the parent + join keys
@@ -189,6 +206,7 @@ def gen_enrichment_node(
         g=g,
         depth=depth,
         history=history,
+        conditions=conditions,
     )
     if not enrich_node:
         log_lambda(
@@ -225,17 +243,31 @@ def gen_enrichment_node(
 
 
 def resolve_join_order(joins: List[NodeJoin]) -> List[NodeJoin]:
+    if not joins:
+        return []
     available_aliases: set[StrategyNode] = set()
     final_joins_pre = [*joins]
     final_joins = []
+    left = set()
+    right = set()
+    for join in joins:
+        left.add(join.left_node)
+        right.add(join.right_node)
+
+    potential_basis = left.difference(right)
+    base_candidates = [x for x in final_joins_pre if x.left_node in potential_basis]
+    if not base_candidates:
+        raise SyntaxError(
+            f"Unresolvable join dependencies, left requires {left} and right requires {right}"
+        )
+    base = base_candidates[0]
+    final_joins.append(base)
+    available_aliases.add(base.left_node)
+    available_aliases.add(base.right_node)
     while final_joins_pre:
         new_final_joins_pre: List[NodeJoin] = []
         for join in final_joins_pre:
-            if not available_aliases:
-                final_joins.append(join)
-                available_aliases.add(join.left_node)
-                available_aliases.add(join.right_node)
-            elif join.left_node in available_aliases:
+            if join.left_node in available_aliases:
                 # we don't need to join twice
                 # so whatever join we found first, works
                 if join.right_node in available_aliases:
@@ -245,11 +277,5 @@ def resolve_join_order(joins: List[NodeJoin]) -> List[NodeJoin]:
                 available_aliases.add(join.right_node)
             else:
                 new_final_joins_pre.append(join)
-        if len(new_final_joins_pre) == len(final_joins_pre):
-            remaining = [join.left_node for join in new_final_joins_pre]
-            remaining_right = [join.right_node for join in new_final_joins_pre]
-            raise SyntaxError(
-                f"did not find any new joins, available {available_aliases} remaining is {remaining + remaining_right} "
-            )
         final_joins_pre = new_final_joins_pre
     return final_joins

trilogy/core/processing/node_generators/filter_node.py

@@ -2,13 +2,14 @@ from typing import List
 
 
 from trilogy.core.enums import JoinType
-from trilogy.core.models import Concept, Environment, FilterItem, Grain
+from trilogy.core.models import Concept, Environment, FilterItem, Grain, WhereClause
 from trilogy.core.processing.nodes import (
     FilterNode,
     MergeNode,
     NodeJoin,
     History,
     StrategyNode,
+    SelectNode,
 )
 from trilogy.core.processing.node_generators.common import (
     resolve_filter_parent_concepts,
@@ -28,6 +29,7 @@ def gen_filter_node(
     depth: int,
     source_concepts,
     history: History | None = None,
+    conditions: WhereClause | None = None,
 ) -> StrategyNode | None:
     immediate_parent, parent_row_concepts, parent_existence_concepts = (
         resolve_filter_parent_concepts(concept)
@@ -37,10 +39,10 @@ def gen_filter_node(
     where = concept.lineage.where
 
     logger.info(
-        f"{padding(depth)}{LOGGER_PREFIX} fetching filter node row parents {[x.address for x in parent_row_concepts]}"
+        f"{padding(depth)}{LOGGER_PREFIX} filter {concept.address} derived from {immediate_parent.address} row parents {[x.address for x in parent_row_concepts]} and {[[y.address] for x  in parent_existence_concepts for y  in x]} existence parents"
     )
     core_parents = []
-    parent: StrategyNode = source_concepts(
+    row_parent: StrategyNode = source_concepts(
         mandatory_list=parent_row_concepts,
         environment=environment,
         g=g,
@@ -48,33 +50,7 @@ def gen_filter_node(
         history=history,
     )
 
-    if not parent:
-        logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} filter node row parents {[x.address for x in parent_row_concepts]} could not be found"
-        )
-        return None
-
-    if not local_optional and not parent_existence_concepts:
-        optimized_pushdown = True
-    else:
-        optimized_pushdown = False
-
-    if optimized_pushdown:
-        if parent.conditions:
-            parent.conditions = parent.conditions + where.conditional
-        else:
-            parent.conditions = where.conditional
-        parent.output_concepts = [concept]
-        parent.grain = Grain(components=[concept])
-        parent.rebuild_cache()
-
-        logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} returning optimized filter node with pushdown to parent with condition {where.conditional}"
-        )
-        return parent
-
-    core_parents.append(parent)
-
+    flattened_existence = [x for y in parent_existence_concepts for x in y]
     if parent_existence_concepts:
         for existence_tuple in parent_existence_concepts:
             if not existence_tuple:
@@ -95,26 +71,91 @@ def gen_filter_node(
                 )
                 return None
             core_parents.append(parent_existence)
-    flattened_existence = [x for y in parent_existence_concepts for x in y]
-    filter_node = FilterNode(
-        input_concepts=unique(
-            [immediate_parent] + parent_row_concepts + flattened_existence,
-            "address",
-        ),
-        output_concepts=[concept, immediate_parent] + parent_row_concepts,
-        environment=environment,
-        g=g,
-        parents=core_parents,
-        grain=Grain(
-            components=[immediate_parent] + parent_row_concepts,
-        ),
-    )
+    if not row_parent:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} filter node row parents {[x.address for x in parent_row_concepts]} could not be found"
+        )
+        return None
+
+    optimized_pushdown = False
+    if not local_optional:
+        optimized_pushdown = True
+    elif conditions and conditions == where:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} query conditions are the same as filter conditions, can optimize across all concepts"
+        )
+        optimized_pushdown = True
+
+    if optimized_pushdown:
+        if isinstance(row_parent, SelectNode):
+            parent = StrategyNode(
+                input_concepts=row_parent.output_concepts,
+                output_concepts=[concept] + row_parent.output_concepts,
+                environment=row_parent.environment,
+                g=row_parent.g,
+                parents=[row_parent] + core_parents,
+                depth=row_parent.depth,
+                partial_concepts=row_parent.partial_concepts,
+                force_group=False,
+                conditions=(
+                    row_parent.conditions + where.conditional
+                    if row_parent.conditions
+                    else where.conditional
+                ),
+                existence_concepts=row_parent.existence_concepts,
+            )
+        else:
+            parent = row_parent
+
+        expected_output = [concept] + [
+            x
+            for x in local_optional
+            if x.address in [y.address for y in parent.output_concepts]
+        ]
+        parent.add_parents(core_parents)
+        parent.add_condition(where.conditional)
+        parent.add_existence_concepts(flattened_existence)
+        parent.set_output_concepts(expected_output)
+        parent.grain = Grain(
+            components=(
+                list(immediate_parent.keys)
+                if immediate_parent.keys
+                else [immediate_parent]
+            )
+            + [
+                x
+                for x in local_optional
+                if x.address in [y.address for y in parent.output_concepts]
+            ]
+        )
+        parent.rebuild_cache()
+
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} returning optimized filter node with pushdown to parent with condition {where.conditional}"
+        )
+        filter_node = parent
+    else:
+        core_parents.append(row_parent)
+
+        filter_node = FilterNode(
+            input_concepts=unique(
+                [immediate_parent] + parent_row_concepts + flattened_existence,
+                "address",
+            ),
+            output_concepts=[concept, immediate_parent] + parent_row_concepts,
+            environment=environment,
+            g=g,
+            parents=core_parents,
+            grain=Grain(
+                components=[immediate_parent] + parent_row_concepts,
+            ),
+        )
 
-    assert filter_node.resolve().grain == Grain(
-        components=[immediate_parent] + parent_row_concepts,
-    )
     if not local_optional or all(
-        [x.address in [y.address for y in parent_row_concepts] for x in local_optional]
+        [
+            x.address in [y.address for y in filter_node.output_concepts]
+            for x in local_optional
+        ]
     ):
         outputs = [
             x

trilogy/core/processing/node_generators/group_node.py

@@ -1,4 +1,4 @@
-from trilogy.core.models import Concept, Environment, LooseConceptList
+from trilogy.core.models import Concept, Environment, LooseConceptList, WhereClause
 from trilogy.utility import unique
 from trilogy.core.processing.nodes import GroupNode, StrategyNode, History
 from typing import List
@@ -22,6 +22,7 @@ def gen_group_node(
     depth: int,
     source_concepts,
     history: History | None = None,
+    conditions: WhereClause | None = None,
 ):
     # aggregates MUST always group to the proper grain
     # except when the
@@ -53,6 +54,7 @@ def gen_group_node(
             g=g,
             depth=depth,
             history=history,
+            conditions=conditions,
         )
         if not parent:
             logger.info(

trilogy/core/processing/node_generators/rowset_node.py

@@ -12,7 +12,7 @@ from typing import List
 
 from trilogy.core.enums import JoinType, PurposeLineage
 from trilogy.constants import logger
-from trilogy.core.processing.utility import padding, unique
+from trilogy.core.processing.utility import padding
 from trilogy.core.processing.node_generators.common import concept_to_relevant_joins
 
 
@@ -28,6 +28,8 @@ def gen_rowset_node(
     source_concepts,
     history: History | None = None,
 ) -> StrategyNode | None:
+    from trilogy.core.query_processor import get_query_node
+
     if not isinstance(concept.lineage, RowsetItem):
         raise SyntaxError(
             f"Invalid lineage passed into rowset fetch, got {type(concept.lineage)}, expected {RowsetItem}"
@@ -35,54 +37,14 @@ def gen_rowset_node(
     lineage: RowsetItem = concept.lineage
     rowset: RowsetDerivationStatement = lineage.rowset
     select: SelectStatement | MultiSelectStatement = lineage.rowset.select
-    existence_parents: List[StrategyNode] = []
-    if where := select.where_clause:
-        targets = select.output_components + where.conditional.row_arguments
-        for sub_select in where.conditional.existence_arguments:
-            logger.info(
-                f"{padding(depth)}{LOGGER_PREFIX} generating parent existence node with {[x.address for x in sub_select]}"
-            )
-            parent_check = source_concepts(
-                mandatory_list=sub_select,
-                environment=environment,
-                g=g,
-                depth=depth + 1,
-                history=history,
-            )
-            if not parent_check:
-                logger.info(
-                    f"{padding(depth)}{LOGGER_PREFIX} Cannot generate parent existence node for rowset node for {concept}"
-                )
-                return None
-            existence_parents.append(parent_check)
-    else:
-        targets = select.output_components
-    node: StrategyNode = source_concepts(
-        mandatory_list=unique(targets, "address"),
-        environment=environment,
-        g=g,
-        depth=depth + 1,
-        history=history,
-    )
+    node = get_query_node(environment, select, graph=g, history=history)
 
     if not node:
         logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} Cannot generate rowset node for {concept}"
+            f"{padding(depth)}{LOGGER_PREFIX} Cannot generate parent rowset node for {concept}"
         )
         return None
-    # add our existence concepts in
-    if existence_parents:
-        node.parents += existence_parents
-        # we don't need to join to any existence parents
-        # if isinstance(node, MergeNode) and node.node_joins is None:
-        #     # set it explicitly to empty to avoid inference
-        #     node.node_joins = []
-        for parent in existence_parents:
-            for x in parent.output_concepts:
-                if x.address not in node.output_lcl:
-                    node.existence_concepts.append(x)
 
-    node.conditions = select.where_clause.conditional if select.where_clause else None
     enrichment = set([x.address for x in local_optional])
     rowset_relevant = [x for x in rowset.derived_concepts]
     select_hidden = set([x.address for x in select.hidden_components])
@@ -113,26 +75,23 @@ def gen_rowset_node(
     # but don't include anything aggregate at this point
     assert node.resolution_cache
 
-    node.resolution_cache.grain = concept_list_to_grain(
+    node.grain = concept_list_to_grain(
         node.output_concepts, parent_sources=node.resolution_cache.datasources
     )
 
+    node.rebuild_cache()
+
     possible_joins = concept_to_relevant_joins(additional_relevant)
-    if not local_optional:
+    if not local_optional or all(
+        x.address in [y.address for y in node.output_concepts] for x in local_optional
+    ):
         logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} no enriched required for rowset node; exiting early"
+            f"{padding(depth)}{LOGGER_PREFIX} no enrichment required for rowset node as all optional found or no optional; exiting early."
         )
         return node
     if not possible_joins:
         logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} no possible joins for rowset node; exiting early"
-        )
-        return node
-    if all(
-        [x.address in [y.address for y in node.output_concepts] for x in local_optional]
-    ):
-        logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} all enriched concepts returned from base rowset node; exiting early"
+            f"{padding(depth)}{LOGGER_PREFIX} no possible joins for rowset node to get {[x.address for x in local_optional]}; have {[x.address for x in node.output_concepts]}"
         )
         return node
     enrich_node: MergeNode = source_concepts(  # this fetches the parent + join keys

trilogy/core/processing/node_generators/select_node.py

@@ -7,6 +7,7 @@ from trilogy.core.models import (
     Grain,
     LooseConceptList,
     Datasource,
+    WhereClause,
 )
 from trilogy.core.processing.nodes import (
     StrategyNode,
@@ -67,17 +68,6 @@ def dm_to_strategy_node(
             f"{padding(depth)}{LOGGER_PREFIX} target grain is not subset of datasource grain {datasource.grain}, required to group"
         )
         force_group = True
-    # if isinstance(datasource, MergeDatasource):
-    #     # if we're within a namespace, don't find merge nodes
-    #     bcandidate: StrategyNode = gen_environment_merge_node(
-    #         all_concepts=dm.matched.concepts,
-    #         environment=environment,
-    #         g=g,
-    #         depth=depth,
-    #         datasource=datasource,
-    #         source_concepts=source_concepts,
-    #     )
-    # else:
     bcandidate: StrategyNode = SelectNode(
         input_concepts=[c.concept for c in datasource.columns],
         output_concepts=dm.matched.concepts,
@@ -116,6 +106,7 @@ def gen_select_nodes_from_tables_v2(
     target_grain: Grain,
     source_concepts: Callable,
     accept_partial: bool = False,
+    conditions: WhereClause | None = None,
 ) -> tuple[bool, list[Concept], list[StrategyNode]]:
     # if we have only constants
     # we don't need a table
@@ -252,6 +243,7 @@ def gen_select_node_from_table(
     target_grain: Grain,
     source_concepts,
     accept_partial: bool = False,
+    conditions: WhereClause | None = None,
 ) -> Optional[StrategyNode]:
     # if we have only constants
     # we don't need a table
@@ -372,6 +364,9 @@ def gen_select_node_from_table(
             grain=Grain(components=all_concepts),
             conditions=datasource.where.conditional if datasource.where else None,
         )
+        # if conditions:
+        #     for component in conditions.components:
+        #         if
         # we need to nest the group node one further
         if force_group is True:
             candidate: StrategyNode = GroupNode(
@@ -407,6 +402,7 @@ def gen_select_node(
     fail_if_not_found: bool = True,
     accept_partial_optional: bool = True,
     target_grain: Grain | None = None,
+    conditions: WhereClause | None = None,
 ) -> StrategyNode | None:
     all_concepts = [concept] + local_optional
     all_lcl = LooseConceptList(concepts=all_concepts)
@@ -445,6 +441,7 @@ def gen_select_node(
         accept_partial=accept_partial,
         target_grain=target_grain,
         source_concepts=source_concepts,
+        conditions=conditions,
     )
     if ds:
         logger.info(
@@ -461,6 +458,7 @@ def gen_select_node(
         target_grain=target_grain,
         accept_partial=accept_partial,
         source_concepts=source_concepts,
+        conditions=conditions,
     )
     if parents and (all_found or accept_partial_optional):
         all_partial = [
@@ -488,7 +486,6 @@ def gen_select_node(
         if len(parents) == 1:
             candidate = parents[0]
         else:
-
             candidate = MergeNode(
                 output_concepts=[concept] + found,
                 input_concepts=[concept] + found,
@@ -499,8 +496,8 @@ def gen_select_node(
                 partial_concepts=all_partial,
                 grain=inferred_grain,
             )
+
         candidate.depth += 1
-        # source_grain = candidate.grain
         if force_group:
             logger.info(
                 f"{padding(depth)}{LOGGER_PREFIX} datasource grain {inferred_grain} does not match target grain {target_grain} for select, adding group node"

trilogy/core/processing/node_generators/unnest_node.py

@@ -1,8 +1,8 @@
 from typing import List
 
 
-from trilogy.core.models import Concept, Function
-from trilogy.core.processing.nodes import SelectNode, UnnestNode, History, StrategyNode
+from trilogy.core.models import Concept, Function, WhereClause
+from trilogy.core.processing.nodes import UnnestNode, History, StrategyNode
 from trilogy.core.processing.utility import padding
 from trilogy.constants import logger
 
@@ -17,6 +17,7 @@ def gen_unnest_node(
     depth: int,
     source_concepts,
     history: History | None = None,
+    conditions: WhereClause | None = None,
 ) -> StrategyNode | None:
     arguments = []
     if isinstance(concept.lineage, Function):
@@ -28,6 +29,7 @@ def gen_unnest_node(
             g=g,
             depth=depth + 1,
             history=history,
+            conditions=conditions,
         )
         if not parent:
             logger.info(
@@ -46,7 +48,7 @@ def gen_unnest_node(
     # we need to sometimes nest an unnest node,
     # as unnest operations are not valid in all situations
     # TODO: inline this node when we can detect it's safe
-    new = SelectNode(
+    new = StrategyNode(
         input_concepts=[concept] + local_optional,
         output_concepts=[concept] + local_optional,
         environment=environment,

trilogy/core/processing/node_generators/window_node.py

@@ -1,7 +1,7 @@
 from typing import List
 
 
-from trilogy.core.models import Concept, WindowItem, Environment
+from trilogy.core.models import Concept, WindowItem, Environment, WhereClause
 from trilogy.utility import unique
 from trilogy.core.processing.nodes import (
     WindowNode,
@@ -38,19 +38,36 @@ def gen_window_node(
     depth: int,
     source_concepts,
     history: History | None = None,
+    conditions: WhereClause | None = None,
 ) -> WindowNode | MergeNode | None:
     parent_concepts = resolve_window_parent_concepts(concept)
-
     parent_node = source_concepts(
         mandatory_list=parent_concepts,
         environment=environment,
         g=g,
         depth=depth + 1,
         history=history,
+        conditions=conditions,
     )
     if not parent_node:
         logger.info(f"{padding(depth)}{LOGGER_PREFIX} window node parents unresolvable")
         return None
+    parent_node.resolve()
+    if not all(
+        [
+            x.address in [y.address for y in parent_node.output_concepts]
+            for x in parent_concepts
+        ]
+    ):
+        missing = [
+            x
+            for x in parent_concepts
+            if x.address not in [y.address for y in parent_node.output_concepts]
+        ]
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} window node parents unresolvable, missing {missing}"
+        )
+        raise SyntaxError
     _window_node = WindowNode(
         input_concepts=parent_concepts,
         output_concepts=[concept] + parent_concepts,
@@ -61,6 +78,8 @@ def gen_window_node(
         ],
         depth=depth,
     )
+    _window_node.rebuild_cache()
+    _window_node.resolve()
     window_node = MergeNode(
         parents=[_window_node],
         environment=environment,
@@ -71,6 +90,7 @@ def gen_window_node(
         force_group=False,
         depth=depth,
     )
+    window_node.resolve()
     if not local_optional:
         return window_node
     logger.info(f"{padding(depth)}{LOGGER_PREFIX} window node requires enrichment")
@@ -84,4 +104,5 @@ def gen_window_node(
         source_concepts=source_concepts,
         log_lambda=create_log_lambda(LOGGER_PREFIX, depth, logger),
         history=history,
+        conditions=conditions,
     )