pytrilogy 0.0.1.117__py3-none-any.whl → 0.0.2.1__py3-none-any.whl

trilogy/core/optimization.py

@@ -3,8 +3,6 @@ from trilogy.core.models import (
     SelectStatement,
     PersistStatement,
     MultiSelectStatement,
-    Conditional,
-    BooleanOperator,
 )
 from trilogy.core.enums import PurposeLineage
 from trilogy.constants import logger, CONFIG
@@ -60,8 +58,6 @@ def is_direct_return_eligible(
         if select.where_clause
         else set()
     )
-    if conditions and select.limit:
-        return False
     for x in derived_concepts:
         if x.derivation == PurposeLineage.WINDOW:
             return False
@@ -71,7 +67,7 @@ def is_direct_return_eligible(
             if x.address in conditions:
                 return False
     logger.info(
-        f"Upleveling output select to final CTE with derived_concepts {[x.address for x in derived_concepts]}"
+        f"[Optimization][EarlyReturn] Upleveling output select to final CTE with derived_concepts {[x.address for x in derived_concepts]}"
     )
     return eligible
 
@@ -93,39 +89,46 @@ def sort_select_output(cte: CTE, query: SelectStatement | MultiSelectStatement):
 def optimize_ctes(
     input: list[CTE], root_cte: CTE, select: SelectStatement | MultiSelectStatement
 ) -> list[CTE]:
-    complete = False
-    REGISTERED_RULES: list["OptimizationRule"] = []
+
     if CONFIG.optimizations.direct_return and is_direct_return_eligible(
         root_cte, select
     ):
         root_cte.order_by = select.order_by
         root_cte.limit = select.limit
-        if select.where_clause:
-
-            if root_cte.condition:
-                root_cte.condition = Conditional(
-                    left=root_cte.condition,
-                    operator=BooleanOperator.AND,
-                    right=select.where_clause.conditional,
-                )
-            else:
-                root_cte.condition = select.where_clause.conditional
+        # if select.where_clause:
+
+        #     if root_cte.condition:
+        #         root_cte.condition = Conditional(
+        #             left=root_cte.condition,
+        #             operator=BooleanOperator.AND,
+        #             right=select.where_clause.conditional,
+        #         )
+        #     else:
+        #         root_cte.condition = select.where_clause.conditional
         root_cte.requires_nesting = False
         sort_select_output(root_cte, select)
+
+    REGISTERED_RULES: list["OptimizationRule"] = []
+    if CONFIG.optimizations.constant_inlining:
+        REGISTERED_RULES.append(InlineConstant())
     if CONFIG.optimizations.datasource_inlining:
         REGISTERED_RULES.append(InlineDatasource())
     if CONFIG.optimizations.predicate_pushdown:
         REGISTERED_RULES.append(PredicatePushdown())
-    if CONFIG.optimizations.constant_inlining:
-        REGISTERED_RULES.append(InlineConstant())
-    loops = 0
-    while not complete and (loops <= MAX_OPTIMIZATION_LOOPS):
-        actions_taken = False
-        for rule in REGISTERED_RULES:
-            for cte in input:
-                inverse_map = gen_inverse_map(input)
-                actions_taken = actions_taken or rule.optimize(cte, inverse_map)
-        complete = not actions_taken
-        loops += 1
+
+    for rule in REGISTERED_RULES:
+        loops = 0
+        complete = False
+        while not complete and (loops <= MAX_OPTIMIZATION_LOOPS):
+            actions_taken = False
+            # assume we go through all CTEs once
+            look_at = [root_cte, *input]
+            inverse_map = gen_inverse_map(look_at)
+            for cte in look_at:
+                opt = rule.optimize(cte, inverse_map)
+                actions_taken = actions_taken or opt
+            complete = not actions_taken
+            loops += 1
+        logger.info(f"finished checking for {type(rule).__name__} in {loops} loops")
 
     return filter_irrelevant_ctes(input, root_cte)

trilogy/core/optimizations/inline_constant.py

@@ -21,9 +21,12 @@ class InlineConstant(OptimizationRule):
         if to_inline:
             inlined = False
             for c in to_inline:
-                self.log(f"Inlining constant {c.address} on  {cte.name}")
+                self.log(f"Attempting to inline constant {c.address} on {cte.name}")
                 test = cte.inline_constant(c)
                 if test:
+                    self.log(f"Successfully inlined constant to {cte.name}")
                     inlined = True
+                else:
+                    self.log(f"Could not inline constant to {cte.name}")
             return inlined
         return False

trilogy/core/optimizations/inline_datasource.py

@@ -4,15 +4,20 @@ from trilogy.core.models import (
 )
 
 from trilogy.core.optimizations.base_optimization import OptimizationRule
+from collections import defaultdict
 
 
 class InlineDatasource(OptimizationRule):
 
+    def __init__(self):
+        super().__init__()
+        self.candidates = defaultdict(lambda: set())
+        self.count = defaultdict(lambda: 0)
+
     def optimize(self, cte: CTE, inverse_map: dict[str, list[CTE]]) -> bool:
         if not cte.parent_ctes:
             return False
 
-        optimized = False
         self.log(
             f"Checking {cte.name} for consolidating inline tables with {len(cte.parent_ctes)} parents"
         )
@@ -25,6 +30,9 @@ class InlineDatasource(OptimizationRule):
             if parent_cte.parent_ctes:
                 self.log(f"parent {parent_cte.name} has parents")
                 continue
+            if parent_cte.condition:
+                self.log(f"parent {parent_cte.name} has condition, cannot be inlined")
+                continue
             raw_root = parent_cte.source.datasources[0]
             if not isinstance(raw_root, Datasource):
                 self.log(f"parent {parent_cte.name} is not datasource")
@@ -34,7 +42,8 @@ class InlineDatasource(OptimizationRule):
                 self.log(f"parent {parent_cte.name} datasource is not inlineable")
                 continue
             root_outputs = {x.address for x in root.output_concepts}
-            cte_outputs = {x.address for x in parent_cte.output_columns}
+            cte_outputs = {x.address for x in cte.output_columns}
+            # cte_inherited_outputs = {x.address for x in parent_cte.output_columns if parent_cte.source_map.get(x.address)}
             grain_components = {x.address for x in root.grain.components}
             if not cte_outputs.issubset(root_outputs):
                 self.log(f"Not all {parent_cte.name} outputs are found on datasource")
@@ -44,11 +53,23 @@ class InlineDatasource(OptimizationRule):
                 force_group = True
             to_inline.append(parent_cte)
 
+        optimized = False
         for replaceable in to_inline:
-
+            if replaceable.name not in self.candidates[cte.name]:
+                self.candidates[cte.name].add(replaceable.name)
+                self.count[replaceable.source.name] += 1
+                return True
+            if self.count[replaceable.source.name] > 1:
+                self.log(
+                    f"Skipping inlining raw datasource {replaceable.source.name} ({replaceable.name}) due to multiple references"
+                )
+                continue
             result = cte.inline_parent_datasource(replaceable, force_group=force_group)
             if result:
-                self.log(f"Inlined parent {replaceable.name}")
+                self.log(
+                    f"Inlined parent {replaceable.name} with {replaceable.source.name}"
+                )
+                optimized = True
             else:
                 self.log(f"Failed to inline {replaceable.name}")
         return optimized

trilogy/core/optimizations/predicate_pushdown.py

@@ -21,85 +21,125 @@ def decompose_condition(conditional: Conditional):
 
 
 def is_child_of(a, comparison):
+    base = comparison == a
+    if base:
+        return True
     if isinstance(comparison, Conditional):
         return (
             is_child_of(a, comparison.left) or is_child_of(a, comparison.right)
         ) and comparison.operator == BooleanOperator.AND
-    return comparison == a
+    return base
 
 
 class PredicatePushdown(OptimizationRule):
 
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.complete: dict[str, bool] = {}
+
+    def _check_parent(
+        self,
+        parent_cte: CTE,
+        candidate: Conditional,
+        inverse_map: dict[str, list[CTE]],
+    ):
+        conditions = {x.address for x in candidate.concept_arguments}
+        if is_child_of(candidate, parent_cte.condition):
+            return False
+
+        materialized = {k for k, v in parent_cte.source_map.items() if v != []}
+        if not conditions or not materialized:
+            return False
+        # if it's a root datasource, we can filter on _any_ of the output concepts
+        if parent_cte.is_root_datasource:
+            extra_check = {
+                x.address for x in parent_cte.source.datasources[0].output_concepts
+            }
+            if conditions.issubset(extra_check):
+                for x in conditions:
+                    if x not in materialized:
+                        materialized.add(x)
+                        parent_cte.source_map[x] = [
+                            parent_cte.source.datasources[0].name
+                        ]
+        if conditions.issubset(materialized):
+            children = inverse_map.get(parent_cte.name, [])
+            if all([is_child_of(candidate, child.condition) for child in children]):
+                self.log(
+                    f"All concepts are found on {parent_cte.name} with existing {parent_cte.condition} and all it's {len(children)} children include same filter; pushing up {candidate}"
+                )
+                if parent_cte.condition:
+                    parent_cte.condition = Conditional(
+                        left=parent_cte.condition,
+                        operator=BooleanOperator.AND,
+                        right=candidate,
+                    )
+                else:
+                    parent_cte.condition = candidate
+                return True
+        self.debug(
+            f"conditions {conditions} not subset of parent {parent_cte.name} parent has {materialized} "
+        )
+        return False
+
     def optimize(self, cte: CTE, inverse_map: dict[str, list[CTE]]) -> bool:
+        optimized = False
 
         if not cte.parent_ctes:
             self.debug(f"No parent CTEs for {cte.name}")
 
             return False
 
-        optimized = False
         if not cte.condition:
             self.debug(f"No CTE condition for {cte.name}")
             return False
-        self.log(
+        if all(
+            [
+                is_child_of(cte.condition, parent_cte.condition)
+                for parent_cte in cte.parent_ctes
+            ]
+        ) and not any([isinstance(x, Datasource) for x in cte.source.datasources]):
+            self.log(
+                f"All parents of {cte.name} have same filter, removing filter from {cte.name}"
+            )
+            cte.condition = None
+            return True
+        else:
+            mapping = {
+                parent.name: is_child_of(cte.condition, parent.condition)
+                for parent in cte.parent_ctes
+            }
+            self.log(
+                f"Could not remove filter from {cte.name}, as not all parents have the same filter: {mapping}"
+            )
+        if self.complete.get(cte.name):
+            self.debug("Have done this CTE before")
+            return False
+
+        self.debug(
             f"Checking {cte.name} for predicate pushdown with {len(cte.parent_ctes)} parents"
         )
         if isinstance(cte.condition, Conditional):
             candidates = cte.condition.decompose()
         else:
             candidates = [cte.condition]
-        self.log(f"Have {len(candidates)} candidates to try to push down")
+        self.debug(
+            f"Have {len(candidates)} candidates to try to push down from parent {type(cte.condition)}"
+        )
+        optimized = False
         for candidate in candidates:
-            conditions = {x.address for x in candidate.concept_arguments}
+            self.debug(f"Checking candidate {candidate}")
             for parent_cte in cte.parent_ctes:
-                if is_child_of(cte.condition, parent_cte.condition):
-                    continue
-                materialized = {k for k, v in parent_cte.source_map.items() if v != []}
-                # if it's a root datasource, we can filter on _any_ of the output concepts
-                if parent_cte.is_root_datasource:
-                    extra_check = {
-                        x.address
-                        for x in parent_cte.source.datasources[0].output_concepts
-                    }
-                    if conditions.issubset(extra_check):
-                        for x in conditions:
-                            if x not in materialized:
-                                materialized.add(x)
-                                parent_cte.source_map[x] = [
-                                    parent_cte.source.datasources[0].name
-                                ]
-                if conditions.issubset(materialized):
-                    if all(
-                        [
-                            is_child_of(candidate, child.condition)
-                            for child in inverse_map.get(parent_cte.name, [])
-                        ]
-                    ):
-                        self.log(
-                            f"All concepts are found on {parent_cte.name} and all it's children include same filter; pushing up filter"
-                        )
-                        if parent_cte.condition:
-                            parent_cte.condition = Conditional(
-                                left=parent_cte.condition,
-                                operator=BooleanOperator.AND,
-                                right=candidate,
-                            )
-                        else:
-                            parent_cte.condition = candidate
-                        optimized = True
-                else:
-                    self.log(
-                        f"conditions {conditions} not subset of parent {parent_cte.name} parent has {materialized} "
-                    )
-
-        if all(
-            [
-                is_child_of(cte.condition, parent_cte.condition)
-                for parent_cte in cte.parent_ctes
-            ]
-        ) and not any([isinstance(x, Datasource) for x in cte.source.datasources]):
-            self.log("All parents have same filter, removing filter")
-            cte.condition = None
-            optimized = True
+                local_pushdown = self._check_parent(
+                    parent_cte=parent_cte, candidate=candidate, inverse_map=inverse_map
+                )
+                optimized = optimized or local_pushdown
+                if local_pushdown:
+                    # taint a CTE again when something is pushed up to it.
+                    self.complete[parent_cte.name] = False
+                self.debug(
+                    f"Pushed down {candidate} from {cte.name} to {parent_cte.name}"
+                )
 
+        self.complete[cte.name] = True
         return optimized

trilogy/core/processing/concept_strategies_v3.py

@@ -27,7 +27,6 @@ from trilogy.core.processing.node_generators import (
     gen_group_to_node,
     gen_rowset_node,
     gen_multiselect_node,
-    gen_concept_merge_node,
 )
 
 from enum import Enum
@@ -68,7 +67,6 @@ def get_priority_concept(
     # sometimes we need to scan intermediate concepts to get merge keys, so fall back
     # to exhaustive search
     pass_two = [c for c in all_concepts if c.address not in attempted_addresses]
-
     for remaining_concept in (pass_one, pass_two):
         priority = (
             # find anything that needs no joins first, so we can exit early
@@ -79,9 +77,6 @@ def get_priority_concept(
                 and c.granularity == Granularity.SINGLE_ROW
             ]
             +
-            # anything that requires merging concept universes
-            [c for c in remaining_concept if c.derivation == PurposeLineage.MERGE]
-            +
             # then multiselects to remove them from scope
             [c for c in remaining_concept if c.derivation == PurposeLineage.MULTISELECT]
             +
@@ -174,7 +169,9 @@ def get_priority_concept(
 
 
 def generate_candidates_restrictive(
-    priority_concept: Concept, candidates: list[Concept], exhausted: set[str]
+    priority_concept: Concept,
+    candidates: list[Concept],
+    exhausted: set[str],
 ) -> List[List[Concept]]:
     # if it's single row, joins are irrelevant. Fetch without keys.
     if priority_concept.granularity == Granularity.SINGLE_ROW:
@@ -216,6 +213,7 @@ def generate_node(
         fail_if_not_found=False,
         accept_partial=accept_partial,
         accept_partial_optional=False,
+        source_concepts=source_concepts,
     )
 
     if candidate:
@@ -273,14 +271,6 @@ def generate_node(
         return gen_multiselect_node(
             concept, local_optional, environment, g, depth + 1, source_concepts, history
         )
-    elif concept.derivation == PurposeLineage.MERGE:
-        logger.info(
-            f"{depth_to_prefix(depth)}{LOGGER_PREFIX} for {concept.address}, generating multiselect node with optional {[x.address for x in local_optional]}"
-        )
-        node = gen_concept_merge_node(
-            concept, local_optional, environment, g, depth + 1, source_concepts, history
-        )
-        return node
     elif concept.derivation == PurposeLineage.CONSTANT:
         logger.info(
             f"{depth_to_prefix(depth)}{LOGGER_PREFIX} for {concept.address}, generating constant node"
@@ -331,17 +321,60 @@ def generate_node(
             fail_if_not_found=False,
             accept_partial=accept_partial,
             accept_partial_optional=True,
+            source_concepts=source_concepts,
         )
     else:
         raise ValueError(f"Unknown derivation {concept.derivation}")
 
 
+def validate_concept(
+    concept: Concept,
+    node: StrategyNode,
+    found_addresses: set[str],
+    non_partial_addresses: set[str],
+    partial_addresses: set[str],
+    virtual_addresses: set[str],
+    found_map: dict[str, set[Concept]],
+    accept_partial: bool,
+):
+    found_map[str(node)].add(concept)
+    if concept not in node.partial_concepts:
+
+        found_addresses.add(concept.address)
+        non_partial_addresses.add(concept.address)
+        # remove it from our partial tracking
+        if concept.address in partial_addresses:
+            partial_addresses.remove(concept.address)
+        if concept.address in virtual_addresses:
+            virtual_addresses.remove(concept.address)
+    if concept in node.partial_concepts:
+        if concept.address in non_partial_addresses:
+            return None
+        partial_addresses.add(concept.address)
+        if accept_partial:
+            found_addresses.add(concept.address)
+            found_map[str(node)].add(concept)
+    for _, v in concept.pseudonyms.items():
+        if v.address == concept.address:
+            return
+        validate_concept(
+            v,
+            node,
+            found_addresses,
+            non_partial_addresses,
+            partial_addresses,
+            virtual_addresses,
+            found_map,
+            accept_partial,
+        )
+
+
 def validate_stack(
     stack: List[StrategyNode],
     concepts: List[Concept],
     accept_partial: bool = False,
 ) -> tuple[ValidationResult, set[str], set[str], set[str], set[str]]:
-    found_map = defaultdict(set)
+    found_map: dict[str, set[Concept]] = defaultdict(set)
     found_addresses: set[str] = set()
     non_partial_addresses: set[str] = set()
     partial_addresses: set[str] = set()
@@ -349,27 +382,22 @@ def validate_stack(
     for node in stack:
         resolved = node.resolve()
         for concept in resolved.output_concepts:
-            found_map[str(node)].add(concept)
-            if concept not in node.partial_concepts:
-                found_addresses.add(concept.address)
-                non_partial_addresses.add(concept.address)
-                # remove it from our partial tracking
-                if concept.address in partial_addresses:
-                    partial_addresses.remove(concept.address)
-                if concept.address in virtual_addresses:
-                    virtual_addresses.remove(concept.address)
-            if concept in node.partial_concepts:
-                if concept.address in non_partial_addresses:
-                    continue
-                partial_addresses.add(concept.address)
-                if accept_partial:
-                    found_addresses.add(concept.address)
-                    found_map[str(node)].add(concept)
+            validate_concept(
+                concept,
+                node,
+                found_addresses,
+                non_partial_addresses,
+                partial_addresses,
+                virtual_addresses,
+                found_map,
+                accept_partial,
+            )
         for concept in node.virtual_output_concepts:
             if concept.address in non_partial_addresses:
                 continue
             found_addresses.add(concept.address)
             virtual_addresses.add(concept.address)
+
     # zip in those we know we found
     if not all([c.address in found_addresses for c in concepts]):
         return (
@@ -379,7 +407,8 @@ def validate_stack(
             partial_addresses,
             virtual_addresses,
         )
-    graph_count, graphs = get_disconnected_components(found_map)
+
+    graph_count, _ = get_disconnected_components(found_map)
     if graph_count in (0, 1):
         return (
             ValidationResult.COMPLETE,
@@ -415,7 +444,7 @@ def search_concepts(
     hist = history.get_history(mandatory_list, accept_partial)
     if hist is not False:
         logger.info(
-            f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Returning search node from history for {[c.address for c in mandatory_list]} with accept_partial {accept_partial}"
+            f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Returning search node from history ({'exists' if hist is not None else 'does not exist'}) for {[c.address for c in mandatory_list]} with accept_partial {accept_partial}"
         )
         assert not isinstance(hist, bool)
         return hist
@@ -445,6 +474,7 @@ def _search_concepts(
 ) -> StrategyNode | None:
 
     mandatory_list = unique(mandatory_list, "address")
+
     all_mandatory = set(c.address for c in mandatory_list)
     attempted: set[str] = set()
 
@@ -457,6 +487,7 @@ def _search_concepts(
         priority_concept = get_priority_concept(
             mandatory_list, attempted, found_concepts=found, depth=depth
         )
+
         logger.info(
             f"{depth_to_prefix(depth)}{LOGGER_PREFIX} priority concept is {str(priority_concept)}"
         )
@@ -467,16 +498,16 @@ def _search_concepts(
         candidate_lists = generate_candidates_restrictive(
             priority_concept, candidates, skip
         )
-        for list in candidate_lists:
+        for clist in candidate_lists:
             logger.info(
-                f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Beginning sourcing loop for {str(priority_concept)}, accept_partial {accept_partial} optional {[str(v) for v in list]}, exhausted {[str(c) for c in skip]}"
+                f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Beginning sourcing loop for {str(priority_concept)}, accept_partial {accept_partial}, optional {[str(v) for v in clist]}, exhausted {[str(c) for c in skip]}"
             )
             node = generate_node(
                 priority_concept,
-                list,
+                clist,
                 environment,
                 g,
-                depth + 1,
+                depth,
                 source_concepts=search_concepts,
                 accept_partial=accept_partial,
                 history=history,
@@ -494,7 +525,6 @@ def _search_concepts(
                     PurposeLineage.ROWSET,
                     PurposeLineage.BASIC,
                     PurposeLineage.MULTISELECT,
-                    PurposeLineage.MERGE,
                 ]:
                     skip.add(priority_concept.address)
                 break
@@ -504,7 +534,7 @@ def _search_concepts(
         )
 
         logger.info(
-            f"{depth_to_prefix(depth)}{LOGGER_PREFIX} finished concept loop for {priority_concept} flag for accepting partial addresses is "
+            f"{depth_to_prefix(depth)}{LOGGER_PREFIX} finished concept loop for {priority_concept} flag for accepting partial addresses is"
             f" {accept_partial} (complete: {complete}), have {found} from {[n for n in stack]} (missing {missing} partial {partial} virtual {virtual}), attempted {attempted}"
         )
         # early exit if we have a complete stack with one node

trilogy/core/processing/graph_utils.py

@@ -29,10 +29,10 @@ def extract_required_subgraphs(
 def extract_mandatory_subgraphs(paths: Dict[str, List[str]], g) -> List[List[Concept]]:
     final: list[list[str]] = []
     assocs: defaultdict[str, list] = defaultdict(list)
-    for path in paths:
-        extract_required_subgraphs(assocs, paths[path])
+    for path in paths.values():
+        extract_required_subgraphs(assocs, path)
 
-    for k, v in assocs.items():
+    for _, v in assocs.items():
         final.append(v)
     final_concepts = []
     for value in final:

trilogy/core/processing/node_generators/__init__.py

@@ -8,7 +8,6 @@ from .unnest_node import gen_unnest_node
 from .node_merge_node import gen_merge_node
 from .rowset_node import gen_rowset_node
 from .multiselect_node import gen_multiselect_node
-from .concept_merge_node import gen_concept_merge_node
 
 __all__ = [
     "gen_filter_node",
@@ -21,5 +20,4 @@ __all__ = [
     "gen_group_to_node",
     "gen_rowset_node",
     "gen_multiselect_node",
-    "gen_concept_merge_node",
 ]