pytrilogy 0.0.1.108__tar.gz → 0.0.1.110__tar.gz

{pytrilogy-0.0.1.108/pytrilogy.egg-info → pytrilogy-0.0.1.110}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.1.108
+Version: 0.0.1.110
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.1.108 → pytrilogy-0.0.1.110/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.1.108
+Version: 0.0.1.110
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.1.108 → pytrilogy-0.0.1.110}/trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.1.108"
+__version__ = "0.0.1.110"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

{pytrilogy-0.0.1.108 → pytrilogy-0.0.1.110}/trilogy/constants.py

@@ -1,8 +1,8 @@
 from logging import getLogger
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from enum import Enum
 
-logger = getLogger("preql")
+logger = getLogger("trilogy")
 
 DEFAULT_NAMESPACE = "local"
 
@@ -18,12 +18,20 @@ class MagicConstants(Enum):
 NULL_VALUE = MagicConstants.NULL
 
 
+@dataclass
+class Optimizations:
+    predicate_pushdown: bool = True
+    datasource_inlining: bool = True
+    direct_return: bool = True
+
+
 # TODO: support loading from environments
 @dataclass
 class Config:
     strict_mode: bool = True
     human_identifiers: bool = True
     inline_datasources: bool = True
+    optimizations: Optimizations = field(default_factory=Optimizations)
 
 
 CONFIG = Config()

{pytrilogy-0.0.1.108 → pytrilogy-0.0.1.110}/trilogy/core/enums.py

@@ -263,6 +263,7 @@ class SourceType(Enum):
     WINDOW = "window"
     UNNEST = "unnest"
     CONSTANT = "constant"
+    ROWSET = "rowset"
 
 
 class ShowCategory(Enum):

{pytrilogy-0.0.1.108 → pytrilogy-0.0.1.110}/trilogy/core/models.py

@@ -496,13 +496,31 @@ class Concept(Namespaced, SelectGrain, BaseModel):
     @property
     def sources(self) -> List["Concept"]:
         if self.lineage:
-            output = []
-            for item in self.lineage.arguments:
-                if isinstance(item, Concept):
-                    if item.address == self.address:
-                        raise SyntaxError(f"Concept {self.address} references itself")
-                    output.append(item)
-                    output += item.sources
+            output: List[Concept] = []
+
+            def get_sources(
+                expr: Union[
+                    Function,
+                    WindowItem,
+                    FilterItem,
+                    AggregateWrapper,
+                    RowsetItem,
+                    MultiSelectStatement | MergeStatement,
+                ],
+                output: List[Concept],
+            ):
+                for item in expr.arguments:
+                    if isinstance(item, Concept):
+                        if item.address == self.address:
+                            raise SyntaxError(
+                                f"Concept {self.address} references itself"
+                            )
+                        output.append(item)
+                        output += item.sources
+                    elif isinstance(item, Function):
+                        get_sources(item, output)
+
+            get_sources(self.lineage, output)
             return output
         return []
 
@@ -1923,6 +1941,9 @@ class QueryDatasource(BaseModel):
             ),
             join_derived_concepts=self.join_derived_concepts,
             force_group=self.force_group,
+            hidden_concepts=unique(
+                self.hidden_concepts + other.hidden_concepts, "address"
+            ),
         )
 
         return qds
@@ -2083,6 +2104,9 @@ class CTE(BaseModel):
         self.source.output_concepts = unique(
             self.source.output_concepts + other.source.output_concepts, "address"
         )
+        self.hidden_concepts = unique(
+            self.hidden_concepts + other.hidden_concepts, "address"
+        )
         return self
 
     @property
@@ -2710,6 +2734,9 @@ class Comparison(ConceptArgs, Namespaced, SelectGrain, BaseModel):
     def __repr__(self):
         return f"{str(self.left)} {self.operator.value} {str(self.right)}"
 
+    def __str__(self):
+        return self.__repr__()
+
     def with_namespace(self, namespace: str):
         return self.__class__(
             left=(

{pytrilogy-0.0.1.108 → pytrilogy-0.0.1.110}/trilogy/core/optimization.py

@@ -4,24 +4,32 @@ from trilogy.core.models import (
     PersistStatement,
     Datasource,
     MultiSelectStatement,
+    Conditional,
+    BooleanOperator,
 )
 from trilogy.core.enums import PurposeLineage
-from trilogy.constants import logger
+from trilogy.constants import logger, CONFIG
 from abc import ABC
 
 
+REGISTERED_RULES: list["OptimizationRule"] = []
+
+
 class OptimizationRule(ABC):
 
-    def optimize(self, cte: CTE) -> bool:
+    def optimize(self, cte: CTE, inverse_map: dict[str, list[CTE]]) -> bool:
         raise NotImplementedError
 
     def log(self, message: str):
         logger.info(f"[Optimization][{self.__class__.__name__}] {message}")
 
+    def debug(self, message: str):
+        logger.debug(f"[Optimization][{self.__class__.__name__}] {message}")
+
 
 class InlineDatasource(OptimizationRule):
 
-    def optimize(self, cte: CTE) -> bool:
+    def optimize(self, cte: CTE, inverse_map: dict[str, list[CTE]]) -> bool:
         if not cte.parent_ctes:
             return False
 
@@ -60,10 +68,98 @@ class InlineDatasource(OptimizationRule):
         return optimized
 
 
-REGISTERED_RULES: list[OptimizationRule] = [InlineDatasource()]
+# This will be used in the future for more complex condition decomposition
+def decompose_condition(conditional: Conditional):
+    chunks = []
+    if conditional.operator == BooleanOperator.AND:
+        for val in [conditional.left, conditional.right]:
+            if isinstance(val, Conditional):
+                chunks.extend(decompose_condition(val))
+            else:
+                chunks.append(val)
+    else:
+        chunks.append(conditional)
+    return chunks
+
+
+def is_child_of(a, comparison):
+    if isinstance(comparison, Conditional):
+        return (
+            is_child_of(a, comparison.left) or is_child_of(a, comparison.right)
+        ) and comparison.operator == BooleanOperator.AND
+    return comparison == a
+
+
+class PredicatePushdown(OptimizationRule):
+
+    def optimize(self, cte: CTE, inverse_map: dict[str, list[CTE]]) -> bool:
+
+        if not cte.parent_ctes:
+            self.debug(f"No parent CTEs for {cte.name}")
+
+            return False
+
+        optimized = False
+        if not cte.condition:
+            self.debug(f"No CTE condition for {cte.name}")
+            return False
+        self.log(
+            f"Checking {cte.name} for predicate pushdown with {len(cte.parent_ctes)} parents"
+        )
+        if isinstance(cte.condition, Conditional):
+            candidates = decompose_condition(cte.condition)
+        else:
+            candidates = [cte.condition]
+        logger.info(f"Have {len(candidates)} candidates to try to push down")
+        for candidate in candidates:
+            conditions = {x.address for x in candidate.concept_arguments}
+            for parent_cte in cte.parent_ctes:
+                materialized = {k for k, v in parent_cte.source_map.items() if v != ""}
+                if conditions.issubset(materialized):
+                    if all(
+                        [
+                            is_child_of(candidate, child.condition)
+                            for child in inverse_map[parent_cte.name]
+                        ]
+                    ):
+                        self.log(
+                            f"All concepts are found on {parent_cte.name} and all it's children include same filter; pushing up filter"
+                        )
+                        if parent_cte.condition:
+                            parent_cte.condition = Conditional(
+                                left=parent_cte.condition,
+                                operator=BooleanOperator.AND,
+                                right=candidate,
+                            )
+                        else:
+                            parent_cte.condition = candidate
+                        optimized = True
+                else:
+                    logger.info("conditions not subset of parent materialized")
+
+        if all(
+            [
+                is_child_of(cte.condition, parent_cte.condition)
+                for parent_cte in cte.parent_ctes
+            ]
+        ):
+            self.log("All parents have same filter, removing filter")
+            cte.condition = None
+            optimized = True
+
+        return optimized
+
+
+if CONFIG.optimizations.datasource_inlining:
+    REGISTERED_RULES.append(InlineDatasource())
+if CONFIG.optimizations.predicate_pushdown:
+    REGISTERED_RULES.append(PredicatePushdown())
 
 
-def filter_irrelevant_ctes(input: list[CTE], root_cte: CTE):
+def filter_irrelevant_ctes(
+    input: list[CTE],
+    root_cte: CTE,
+):
     relevant_ctes = set()
 
     def recurse(cte: CTE):
@@ -75,6 +171,16 @@ def filter_irrelevant_ctes(input: list[CTE], root_cte: CTE):
     return [cte for cte in input if cte.name in relevant_ctes]
 
 
+def gen_inverse_map(input: list[CTE]) -> dict[str, list[CTE]]:
+    inverse_map: dict[str, list[CTE]] = {}
+    for cte in input:
+        for parent in cte.parent_ctes:
+            if parent.name not in inverse_map:
+                inverse_map[parent.name] = []
+            inverse_map[parent.name].append(cte)
+    return inverse_map
+
+
 def is_direct_return_eligible(
     cte: CTE, select: SelectStatement | PersistStatement | MultiSelectStatement
 ) -> bool:
@@ -126,7 +232,8 @@ def optimize_ctes(
         actions_taken = False
         for rule in REGISTERED_RULES:
             for cte in input:
-                actions_taken = rule.optimize(cte)
+                inverse_map = gen_inverse_map(input)
+                actions_taken = rule.optimize(cte, inverse_map)
         complete = not actions_taken
 
     if is_direct_return_eligible(root_cte, select):

{pytrilogy-0.0.1.108 → pytrilogy-0.0.1.110}/trilogy/core/processing/node_generators/common.py

@@ -196,6 +196,7 @@ def gen_enrichment_node(
     log_lambda(
         f"{str(type(base_node).__name__)} returning merge node with group node + enrichment node"
     )
+
     return MergeNode(
         input_concepts=unique(
             join_keys + extra_required + base_node.output_concepts, "address"

{pytrilogy-0.0.1.108 → pytrilogy-0.0.1.110}/trilogy/core/processing/node_generators/rowset_node.py

@@ -10,9 +10,9 @@ from trilogy.core.processing.nodes import MergeNode, NodeJoin, History, Strategy
 from trilogy.core.processing.nodes.base_node import concept_list_to_grain
 from typing import List
 
-from trilogy.core.enums import JoinType
+from trilogy.core.enums import JoinType, PurposeLineage
 from trilogy.constants import logger
-from trilogy.core.processing.utility import padding
+from trilogy.core.processing.utility import padding, unique
 from trilogy.core.processing.node_generators.common import concept_to_relevant_joins
 
 
@@ -40,7 +40,7 @@ def gen_rowset_node(
     else:
         targets = select.output_components
     node: StrategyNode = source_concepts(
-        mandatory_list=targets,
+        mandatory_list=unique(targets, "address"),
         environment=environment,
         g=g,
         depth=depth + 1,
@@ -52,14 +52,19 @@ def gen_rowset_node(
         )
         return None
     node.conditions = select.where_clause.conditional if select.where_clause else None
-    # rebuild any cached info with the new condition clause
-    node.rebuild_cache()
     enrichment = set([x.address for x in local_optional])
     rowset_relevant = [
         x
         for x in rowset.derived_concepts
         # if x.address == concept.address or x.address in enrichment
     ]
+    select_hidden = set([x.address for x in select.hidden_components])
+    rowset_hidden = [
+        x
+        for x in rowset.derived_concepts
+        if isinstance(x.lineage, RowsetItem)
+        and x.lineage.content.address in select_hidden
+    ]
     additional_relevant = [
         x for x in select.output_components if x.address in enrichment
     ]
@@ -71,9 +76,15 @@ def gen_rowset_node(
     if select.where_clause:
         for item in additional_relevant:
             node.partial_concepts.append(item)
-
+    node.hidden_concepts = rowset_hidden + [
+        x
+        for x in node.output_concepts
+        if x.address not in [y.address for y in local_optional + [concept]]
+        and x.derivation != PurposeLineage.ROWSET
+    ]
     # assume grain to be output of select
     # but don't include anything aggregate at this point
+    node.rebuild_cache()
     assert node.resolution_cache
     node.resolution_cache.grain = concept_list_to_grain(
         node.output_concepts, parent_sources=node.resolution_cache.datasources

{pytrilogy-0.0.1.108 → pytrilogy-0.0.1.110}/trilogy/core/processing/nodes/base_node.py

@@ -184,6 +184,7 @@ class StrategyNode:
 
     def rebuild_cache(self) -> QueryDatasource:
         self.tainted = True
+        self.output_lcl = LooseConceptList(concepts=self.output_concepts)
         if not self.resolution_cache:
             return self.resolve()
         self.resolution_cache = None

{pytrilogy-0.0.1.108 → pytrilogy-0.0.1.110}/trilogy/core/processing/nodes/merge_node.py

@@ -207,6 +207,7 @@ class MergeNode(StrategyNode):
             joins = self.translate_node_joins(final_joins)
         else:
             return []
+
         for join in joins:
             logger.info(
                 f"{self.logging_prefix}{LOGGER_PREFIX} final join {join.join_type} {[str(c) for c in join.concepts]}"
@@ -281,6 +282,7 @@ class MergeNode(StrategyNode):
                 if c.address in [x.address for x in self.output_concepts]
             ]
         )
+
         logger.info(
             f"{self.logging_prefix}{LOGGER_PREFIX} has pre grain {pregrain} and final merge node grain {grain}"
         )
@@ -306,9 +308,6 @@ class MergeNode(StrategyNode):
                 f"{self.logging_prefix}{LOGGER_PREFIX} no parents include full grain {grain} and pregrain {pregrain} does not match, assume must group to grain. Have {[str(d.grain) for d in final_datasets]}"
             )
             force_group = True
-            # Grain<returns.customer.id,returns.store.id,returns.item.id,returns.store_sales.ticket_number>
-            # Grain<returns.customer.id,returns.store.id,returns.return_date.id,returns.item.id,returns.store_sales.ticket_number>
-            # Grain<returns.customer.id,returns.store.id,returns.item.id,returns.store_sales.ticket_number>
         else:
             force_group = None
 

{pytrilogy-0.0.1.108 → pytrilogy-0.0.1.110}/trilogy/parsing/parse_engine.py

@@ -981,13 +981,13 @@ class ParseToObjects(Transformer):
 
     @v_args(meta=True)
     def merge_statement(self, meta: Meta, args) -> MergeStatement:
-
         parsed = [self.environment.concepts[x] for x in args]
         datatypes = {x.datatype for x in parsed}
-        if not len(datatypes) == 1:
+        if not len(datatypes) == 1 and self.environment.concepts.fail_on_missing:
+            type_dict = {x.address: x.datatype for x in parsed}
             raise SyntaxError(
-                f"Cannot merge concepts with different datatypes {datatypes}"
-                f"line: {meta.line} concepts: {[x.address for x in parsed]}"
+                f"Cannot merge concepts with different datatype"
+                f"line: {meta.line} concepts: {type_dict}"
             )
         merge = MergeStatement(concepts=parsed, datatype=datatypes.pop())
         new = merge.merge_concept