pytrilogy 0.0.2.57__py3-none-any.whl → 0.0.3.0__py3-none-any.whl

trilogy/core/optimization.py

@@ -1,12 +1,9 @@
 from trilogy.constants import CONFIG, logger
-from trilogy.core.enums import BooleanOperator, PurposeLineage
-from trilogy.core.models import (
-    CTE,
-    Conditional,
-    MultiSelectStatement,
-    SelectStatement,
-    UnionCTE,
+from trilogy.core.enums import BooleanOperator, Derivation
+from trilogy.core.models.build import (
+    BuildConditional,
 )
+from trilogy.core.models.execute import CTE, UnionCTE
 from trilogy.core.optimizations import (
     InlineConstant,
     InlineDatasource,
@@ -15,6 +12,7 @@ from trilogy.core.optimizations import (
     PredicatePushdownRemove,
 )
 from trilogy.core.processing.utility import sort_select_output
+from trilogy.core.statements.author import MultiSelectStatement, SelectStatement
 
 MAX_OPTIMIZATION_LOOPS = 100
 
@@ -136,33 +134,29 @@ def is_direct_return_eligible(cte: CTE | UnionCTE) -> CTE | UnionCTE | None:
 
     assert isinstance(cte, CTE)
     derived_concepts = [
-        c
-        for c in cte.source.output_concepts + cte.source.hidden_concepts
-        if c not in cte.source.input_concepts
+        c for c in cte.source.output_concepts if c not in cte.source.input_concepts
     ]
 
     parent_derived_concepts = [
         c
         for c in direct_parent.source.output_concepts
-        + direct_parent.source.hidden_concepts
         if c not in direct_parent.source.input_concepts
     ]
     condition_arguments = cte.condition.row_arguments if cte.condition else []
     for x in derived_concepts:
-        if x.derivation == PurposeLineage.WINDOW:
+        if x.derivation == Derivation.WINDOW:
             return None
-        if x.derivation == PurposeLineage.UNNEST:
+        if x.derivation == Derivation.UNNEST:
             return None
-        if x.derivation == PurposeLineage.AGGREGATE:
+        if x.derivation == Derivation.AGGREGATE:
             return None
     for x in parent_derived_concepts:
         if x.address not in condition_arguments:
             continue
-        if x.derivation == PurposeLineage.UNNEST:
+        if x.derivation == Derivation.UNNEST:
             return None
-        if x.derivation == PurposeLineage.WINDOW:
+        if x.derivation == Derivation.WINDOW:
             return None
-
     logger.info(
         f"[Optimization][EarlyReturn] Removing redundant output CTE with derived_concepts {[x.address for x in derived_concepts]}"
     )
@@ -180,12 +174,12 @@ def optimize_ctes(
     ):
         direct_parent.order_by = root_cte.order_by
         direct_parent.limit = root_cte.limit
-        direct_parent.hidden_concepts = (
-            root_cte.hidden_concepts + direct_parent.hidden_concepts
+        direct_parent.hidden_concepts = root_cte.hidden_concepts.union(
+            direct_parent.hidden_concepts
         )
         if root_cte.condition:
             if direct_parent.condition:
-                direct_parent.condition = Conditional(
+                direct_parent.condition = BuildConditional(
                     left=direct_parent.condition,
                     operator=BooleanOperator.AND,
                     right=root_cte.condition,
@@ -197,14 +191,15 @@ def optimize_ctes(
         sort_select_output(root_cte, select)
 
     REGISTERED_RULES: list["OptimizationRule"] = []
-    if CONFIG.optimizations.constant_inlining:
-        REGISTERED_RULES.append(InlineConstant())
+
     if CONFIG.optimizations.datasource_inlining:
         REGISTERED_RULES.append(InlineDatasource())
     if CONFIG.optimizations.predicate_pushdown:
         REGISTERED_RULES.append(PredicatePushdown())
     if CONFIG.optimizations.predicate_pushdown:
         REGISTERED_RULES.append(PredicatePushdownRemove())
+    if CONFIG.optimizations.constant_inlining:
+        REGISTERED_RULES.append(InlineConstant())
     for rule in REGISTERED_RULES:
         loops = 0
         complete = False

trilogy/core/optimizations/base_optimization.py

@@ -1,7 +1,7 @@
 from abc import ABC
 
 from trilogy.constants import logger
-from trilogy.core.models import CTE, UnionCTE
+from trilogy.core.models.execute import CTE, UnionCTE
 
 
 class OptimizationRule(ABC):

trilogy/core/optimizations/inline_constant.py

@@ -1,7 +1,7 @@
-from trilogy.core.enums import PurposeLineage
-from trilogy.core.models import (
+from trilogy.core.enums import Derivation
+from trilogy.core.models.build import BuildConcept
+from trilogy.core.models.execute import (
     CTE,
-    Concept,
     UnionCTE,
 )
 from trilogy.core.optimizations.base_optimization import OptimizationRule
@@ -14,11 +14,11 @@ class InlineConstant(OptimizationRule):
         if isinstance(cte, UnionCTE):
             return any(self.optimize(x, inverse_map) for x in cte.internal_ctes)
 
-        to_inline: list[Concept] = []
+        to_inline: list[BuildConcept] = []
         for x in cte.source.input_concepts:
             if x.address not in cte.source_map:
                 continue
-            if x.derivation == PurposeLineage.CONSTANT:
+            if x.derivation == Derivation.CONSTANT:
                 self.log(f"Found constant {x.address} on {cte.name}")
                 to_inline.append(x)
         if to_inline:
@@ -27,7 +27,7 @@ class InlineConstant(OptimizationRule):
                 self.log(f"Attempting to inline constant {c.address} on {cte.name}")
                 test = cte.inline_constant(c)
                 if test:
-                    self.log(f"Successfully inlined constant to {cte.name}")
+                    self.log(f"Successfully inlined constant {c.address} to {cte.name}")
                     inlined = True
                 else:
                     self.log(f"Could not inline constant to {cte.name}")

trilogy/core/optimizations/inline_datasource.py

@@ -1,9 +1,11 @@
 from collections import defaultdict
 
 from trilogy.constants import CONFIG
-from trilogy.core.models import (
+
+# from trilogy.core.models.datasource import Datasource
+from trilogy.core.models.build import BuildDatasource
+from trilogy.core.models.execute import (
     CTE,
-    Datasource,
     UnionCTE,
 )
 from trilogy.core.optimizations.base_optimization import OptimizationRule
@@ -35,21 +37,25 @@ class InlineDatasource(OptimizationRule):
             if isinstance(parent_cte, UnionCTE):
                 continue
             if not parent_cte.is_root_datasource:
-                self.debug(f"parent {parent_cte.name} is not root")
+                self.debug(f"Cannot inline: parent {parent_cte.name} is not root")
                 continue
             if parent_cte.parent_ctes:
-                self.debug(f"parent {parent_cte.name} has parents")
+                self.debug(f"Cannot inline: parent {parent_cte.name} has parents")
                 continue
             if parent_cte.condition:
-                self.debug(f"parent {parent_cte.name} has condition, cannot be inlined")
+                self.debug(
+                    f"Cannot inline: parent {parent_cte.name} has condition, cannot be inlined"
+                )
                 continue
             raw_root = parent_cte.source.datasources[0]
-            if not isinstance(raw_root, Datasource):
-                self.debug(f"Parent {parent_cte.name} is not datasource")
+            if not isinstance(raw_root, BuildDatasource):
+                self.debug(f"Cannot inline: Parent {parent_cte.name} is not datasource")
                 continue
-            root: Datasource = raw_root
+            root: BuildDatasource = raw_root
             if not root.can_be_inlined:
-                self.debug(f"Parent {parent_cte.name} datasource is not inlineable")
+                self.debug(
+                    f"Cannot inline: Parent {parent_cte.name} datasource is not inlineable"
+                )
                 continue
             root_outputs = {x.address for x in root.output_concepts}
             inherited = {
@@ -58,12 +64,12 @@ class InlineDatasource(OptimizationRule):
             if not inherited.issubset(root_outputs):
                 cte_missing = inherited - root_outputs
                 self.log(
-                    f"Not all {parent_cte.name} require inputs are found on datasource, missing {cte_missing}"
+                    f"Cannot inline: Not all required inputs to {parent_cte.name} are found on datasource, missing {cte_missing}"
                 )
                 continue
             if not root.grain.issubset(parent_cte.grain):
                 self.log(
-                    f"{parent_cte.name} is at wrong grain to inline ({root.grain} vs {parent_cte.grain})"
+                    f"Cannot inline: {parent_cte.name} is at wrong grain to inline ({root.grain} vs {parent_cte.grain})"
                 )
                 continue
             to_inline.append(parent_cte)

trilogy/core/optimizations/predicate_pushdown.py

@@ -1,14 +1,15 @@
-from trilogy.core.models import (
-    CTE,
+from trilogy.core.enums import (
     BooleanOperator,
-    Comparison,
-    ConceptArgs,
-    Conditional,
-    Datasource,
-    Parenthetical,
-    UnionCTE,
-    WindowItem,
 )
+from trilogy.core.models.build import (
+    BuildComparison,
+    BuildConceptArgs,
+    BuildConditional,
+    BuildDatasource,
+    BuildParenthetical,
+    BuildWindowItem,
+)
+from trilogy.core.models.execute import CTE, UnionCTE
 from trilogy.core.optimizations.base_optimization import OptimizationRule
 from trilogy.core.processing.utility import is_scalar_condition
 from trilogy.utility import unique
@@ -18,7 +19,7 @@ def is_child_of(a, comparison):
     base = comparison == a
     if base:
         return True
-    if isinstance(comparison, Conditional):
+    if isinstance(comparison, BuildConditional):
         return (
             is_child_of(a, comparison.left) or is_child_of(a, comparison.right)
         ) and comparison.operator == BooleanOperator.AND
@@ -34,10 +35,10 @@ class PredicatePushdown(OptimizationRule):
         self,
         cte: CTE | UnionCTE,
         parent_cte: CTE | UnionCTE,
-        candidate: Conditional | Comparison | Parenthetical | None,
+        candidate: BuildConditional | BuildComparison | BuildParenthetical | None,
         inverse_map: dict[str, list[CTE | UnionCTE]],
     ):
-        if not isinstance(candidate, ConceptArgs):
+        if not isinstance(candidate, BuildConceptArgs):
             return False
         if not isinstance(parent_cte, CTE):
             return False
@@ -52,7 +53,7 @@ class PredicatePushdown(OptimizationRule):
         concrete = [
             x for x in parent_cte.output_columns if x.address in non_materialized
         ]
-        if any(isinstance(x.lineage, WindowItem) for x in concrete):
+        if any(isinstance(x.lineage, BuildWindowItem) for x in concrete):
             self.debug(
                 f"CTE {parent_cte.name} has window clause calculation, cannot push up to this without changing results"
             )
@@ -88,7 +89,7 @@ class PredicatePushdown(OptimizationRule):
                     self.log("Parent condition is not scalar, not safe to push up")
                     return False
                 if parent_cte.condition:
-                    parent_cte.condition = Conditional(
+                    parent_cte.condition = BuildConditional(
                         left=parent_cte.condition,
                         operator=BooleanOperator.AND,
                         right=candidate,
@@ -137,7 +138,7 @@ class PredicatePushdown(OptimizationRule):
         self.debug(
             f"Checking {cte.name} for predicate pushdown with {len(cte.parent_ctes)} parents"
         )
-        if isinstance(cte.condition, Conditional):
+        if isinstance(cte.condition, BuildConditional):
             candidates = cte.condition.decompose()
         else:
             candidates = [cte.condition]
@@ -216,7 +217,7 @@ class PredicatePushdownRemove(OptimizationRule):
                 for key, value in parent_filter_status.items()
                 if key not in existence_only
             ]
-        ) and not any([isinstance(x, Datasource) for x in cte.source.datasources]):
+        ) and not any([isinstance(x, BuildDatasource) for x in cte.source.datasources]):
             self.log(
                 f"All parents of {cte.name} have same filter or are existence only inputs, removing filter from {cte.name}"
             )