pytrilogy 0.0.1.109__py3-none-any.whl → 0.0.1.110__py3-none-any.whl

{pytrilogy-0.0.1.109.dist-info → pytrilogy-0.0.1.110.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.1.109
+Version: 0.0.1.110
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.1.109.dist-info → pytrilogy-0.0.1.110.dist-info}/RECORD

@@ -1,6 +1,6 @@
-trilogy/__init__.py,sha256=e2cU9lfgy43E_xTR0vpKBnQaJvz7-8qF7hjum9oQk-k,292
+trilogy/__init__.py,sha256=zz6RmV7fp8n1Ezl-E5jUW92Ns-EMfac6jy3fX5zQuzc,292
 trilogy/compiler.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/constants.py,sha256=LxiK2TiVQPEa6tXkxWk9DJHOR3zsGNSqgQuqtOf66cw,518
+trilogy/constants.py,sha256=C6mwa0BpVyENkcWi-dwqHorCP85ScTXDemfHFiSmLQ8,737
 trilogy/engine.py,sha256=R5ubIxYyrxRExz07aZCUfrTsoXCHQ8DKFTDsobXdWdA,1102
 trilogy/executor.py,sha256=xF6wzbhP6a3wz4nrxsRCKeKF7qytUQEL75oI3BGJ2hQ,8744
 trilogy/parser.py,sha256=UtuqSiGiCjpMAYgo1bvNq-b7NSzCA5hzbUW31RXaMII,281
@@ -8,7 +8,7 @@ trilogy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/utility.py,sha256=zM__8r29EsyDW7K9VOHz8yvZC2bXFzh7xKy3cL7GKsk,707
 trilogy/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/constants.py,sha256=LL8NLvxb3HRnAjvofyLRXqQJijLcYiXAQYQzGarVD-g,128
-trilogy/core/enums.py,sha256=KEZQTzJ8tlGIukuUwQUIG1FTHOP1B4i0EeCgFjfsbDw,5394
+trilogy/core/enums.py,sha256=XSvq2yPxn9oJ18nhn7UERgIV1IXZDRiSWaGpvtU34eE,5416
 trilogy/core/env_processor.py,sha256=SU-jpaGfoWLe9sGTeQYG1qjVnwGQ7TwctmnJRlfzluc,1459
 trilogy/core/environment_helpers.py,sha256=mzBDHhdF9ssZ_-LY8CcaM_ddfJavkpRYrFImUd3cjXI,5972
 trilogy/core/ergonomics.py,sha256=w3gwXdgrxNHCuaRdyKg73t6F36tj-wIjQf47WZkHmJk,1465
@@ -16,8 +16,8 @@ trilogy/core/exceptions.py,sha256=NvV_4qLOgKXbpotgRf7c8BANDEvHxlqRPaA53IThQ2o,56
 trilogy/core/functions.py,sha256=zkRReytiotOBAW-a3Ri5eoejZDYTt2-7Op80ZxZxUmw,9129
 trilogy/core/graph_models.py,sha256=oJUMSpmYhqXlavckHLpR07GJxuQ8dZ1VbB1fB0KaS8c,2036
 trilogy/core/internal.py,sha256=jNGFHKENnbMiMCtAgsnLZYVSENDK4b5ALecXFZpTDzQ,1075
-trilogy/core/models.py,sha256=AwVZNiDN1hM0BeEquEyrfTnuVBPDR8UuTnoFUhAaqUo,109648
-trilogy/core/optimization.py,sha256=chfzpLVJo9eg8H4e2hdnpRqWMDTQ3tJWPdDfGESa-EU,4510
+trilogy/core/models.py,sha256=FNAMbqJrHh-KfC8QbBe3anWVdNxP-uMjs--MrbJM8QM,109943
+trilogy/core/optimization.py,sha256=5n5HMGGdTGkDZf479lZPpv2angLSxklZJ0D4DBrFDeA,8411
 trilogy/core/query_processor.py,sha256=6BqLYPwyFkRtueTIRFZi3IcVFTpbpGRNowayhSn3_AY,11805
 trilogy/core/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/processing/concept_strategies_v3.py,sha256=27lZXFLgDEF3sh2MUR7HX_atVz7TC1fJB7z3oxa1TcY,22610
@@ -32,7 +32,7 @@ trilogy/core/processing/node_generators/group_node.py,sha256=xWI1xNIXEOj6jlRGD9h
 trilogy/core/processing/node_generators/group_to_node.py,sha256=BzPdYwzoo8gRMH7BDffTTXq4z-mjfCEzvfB5I-P0_nw,2941
 trilogy/core/processing/node_generators/multiselect_node.py,sha256=vP84dnLQy6dtypi6mUbt9sMAcmmrTgQ1Oz4GI6X1IEo,6421
 trilogy/core/processing/node_generators/node_merge_node.py,sha256=sQQ9jhw1oAJh649DBAJX6U7r_E_piFS95mxKvm7pxqQ,5818
-trilogy/core/processing/node_generators/rowset_node.py,sha256=zlSRd58V4fDqz1Km65cWblOrEFpXAT3jlSvv6NKC3pc,4909
+trilogy/core/processing/node_generators/rowset_node.py,sha256=BYTpXyiFJwoFp_n8kKE-HUdnwgaK_58n0rSFOz2jOVM,5141
 trilogy/core/processing/node_generators/select_node.py,sha256=xeCqIUEubrf3u_QQfbGdf1BG4fO0HYQ64hiFur8NUqY,20080
 trilogy/core/processing/node_generators/unnest_node.py,sha256=s1VXQZSf1LnX3ISeQ5JzmzmCKUw30-5OK_f0YTB9_48,1031
 trilogy/core/processing/node_generators/window_node.py,sha256=ekazi5eXxnShpcp-qukXNG4DHFdULoXrX-YWUWLNEpM,2527
@@ -40,7 +40,7 @@ trilogy/core/processing/nodes/__init__.py,sha256=gzKxGSduIQ5QwpMWrmwSYiE8sg2mWej
 trilogy/core/processing/nodes/base_node.py,sha256=Du7hRjVVOAiGb0okytzKIa_TQqhwTNYGU8PGNnrE1xs,9142
 trilogy/core/processing/nodes/filter_node.py,sha256=DqSRv8voEajPZqzeeiIsxuv4ubvsmeQcCW6x_v2CmOk,1359
 trilogy/core/processing/nodes/group_node.py,sha256=Y_NWB_AwFrE-YithjZ7lYYDN4e0el4su3ICq2EIr3HA,3837
-trilogy/core/processing/nodes/merge_node.py,sha256=baLDHCJiX5tk1dsVTm1KebJKPyy1w3WGMfN5wdm0BRw,12759
+trilogy/core/processing/nodes/merge_node.py,sha256=uo1AfLjA02EiIpJku8T0TBjN2D-IyP6NVq5m7BfiZbQ,12413
 trilogy/core/processing/nodes/select_node_v2.py,sha256=tAADeVruch-flFiedbY1zi7ukMG2RpWecvxxZ5aL3ZU,6354
 trilogy/core/processing/nodes/unnest_node.py,sha256=t4kY3a_dR3iXistPemStfdw0uJfnxwTcoQg1HiDa3xo,1501
 trilogy/core/processing/nodes/window_node.py,sha256=QjAWgqBZqFSRCPwc7JBmgQJobWW50rsHI0pjJe0Zzg0,926
@@ -69,9 +69,9 @@ trilogy/parsing/parse_engine.py,sha256=iOqKUCyLeHyFVwwAt-XTSJGHia4zzLUN6bYDuIfJ1
 trilogy/parsing/render.py,sha256=fxjpq2FZLgllw_d4cru-t_IXNPAz2DmYkT7v9ED0XRI,11540
 trilogy/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/scripts/trilogy.py,sha256=PHxvv6f2ODv0esyyhWxlARgra8dVhqQhYl0lTrSyVNo,3729
-pytrilogy-0.0.1.109.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
-pytrilogy-0.0.1.109.dist-info/METADATA,sha256=-m-LMyjvq4whCY7LpRM9zFqpn3hqH4-DuuP0JYOFfwQ,7882
-pytrilogy-0.0.1.109.dist-info/WHEEL,sha256=Z4pYXqR_rTB7OWNDYFOm1qRk0RX6GFP2o8LgvP453Hk,91
-pytrilogy-0.0.1.109.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
-pytrilogy-0.0.1.109.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
-pytrilogy-0.0.1.109.dist-info/RECORD,,
+pytrilogy-0.0.1.110.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
+pytrilogy-0.0.1.110.dist-info/METADATA,sha256=FbH2jc_Eg9QjfxlbLhTAqBzFoCxe4aM-6EbE4HDC4TM,7882
+pytrilogy-0.0.1.110.dist-info/WHEEL,sha256=Z4pYXqR_rTB7OWNDYFOm1qRk0RX6GFP2o8LgvP453Hk,91
+pytrilogy-0.0.1.110.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
+pytrilogy-0.0.1.110.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
+pytrilogy-0.0.1.110.dist-info/RECORD,,

trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.1.109"
+__version__ = "0.0.1.110"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

trilogy/constants.py

@@ -1,8 +1,8 @@
 from logging import getLogger
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from enum import Enum
 
-logger = getLogger("preql")
+logger = getLogger("trilogy")
 
 DEFAULT_NAMESPACE = "local"
 
@@ -18,12 +18,20 @@ class MagicConstants(Enum):
 NULL_VALUE = MagicConstants.NULL
 
 
+@dataclass
+class Optimizations:
+    predicate_pushdown: bool = True
+    datasource_inlining: bool = True
+    direct_return: bool = True
+
+
 # TODO: support loading from environments
 @dataclass
 class Config:
     strict_mode: bool = True
     human_identifiers: bool = True
     inline_datasources: bool = True
+    optimizations: Optimizations = field(default_factory=Optimizations)
 
 
 CONFIG = Config()

trilogy/core/enums.py

@@ -263,6 +263,7 @@ class SourceType(Enum):
     WINDOW = "window"
     UNNEST = "unnest"
     CONSTANT = "constant"
+    ROWSET = "rowset"
 
 
 class ShowCategory(Enum):

trilogy/core/models.py

@@ -1941,6 +1941,9 @@ class QueryDatasource(BaseModel):
             ),
             join_derived_concepts=self.join_derived_concepts,
             force_group=self.force_group,
+            hidden_concepts=unique(
+                self.hidden_concepts + other.hidden_concepts, "address"
+            ),
         )
 
         return qds
@@ -2101,6 +2104,9 @@ class CTE(BaseModel):
         self.source.output_concepts = unique(
             self.source.output_concepts + other.source.output_concepts, "address"
         )
+        self.hidden_concepts = unique(
+            self.hidden_concepts + other.hidden_concepts, "address"
+        )
         return self
 
     @property
@@ -2728,6 +2734,9 @@ class Comparison(ConceptArgs, Namespaced, SelectGrain, BaseModel):
     def __repr__(self):
         return f"{str(self.left)} {self.operator.value} {str(self.right)}"
 
+    def __str__(self):
+        return self.__repr__()
+
     def with_namespace(self, namespace: str):
         return self.__class__(
             left=(

trilogy/core/optimization.py

@@ -4,24 +4,32 @@ from trilogy.core.models import (
     PersistStatement,
     Datasource,
     MultiSelectStatement,
+    Conditional,
+    BooleanOperator,
 )
 from trilogy.core.enums import PurposeLineage
-from trilogy.constants import logger
+from trilogy.constants import logger, CONFIG
 from abc import ABC
 
 
+REGISTERED_RULES: list["OptimizationRule"] = []
+
+
 class OptimizationRule(ABC):
 
-    def optimize(self, cte: CTE) -> bool:
+    def optimize(self, cte: CTE, inverse_map: dict[str, list[CTE]]) -> bool:
         raise NotImplementedError
 
     def log(self, message: str):
         logger.info(f"[Optimization][{self.__class__.__name__}] {message}")
 
+    def debug(self, message: str):
+        logger.debug(f"[Optimization][{self.__class__.__name__}] {message}")
+
 
 class InlineDatasource(OptimizationRule):
 
-    def optimize(self, cte: CTE) -> bool:
+    def optimize(self, cte: CTE, inverse_map: dict[str, list[CTE]]) -> bool:
         if not cte.parent_ctes:
             return False
 
@@ -60,10 +68,98 @@ class InlineDatasource(OptimizationRule):
         return optimized
 
 
-REGISTERED_RULES: list[OptimizationRule] = [InlineDatasource()]
+# This will be used in the future for more complex condition decomposition
+def decompose_condition(conditional: Conditional):
+    chunks = []
+    if conditional.operator == BooleanOperator.AND:
+        for val in [conditional.left, conditional.right]:
+            if isinstance(val, Conditional):
+                chunks.extend(decompose_condition(val))
+            else:
+                chunks.append(val)
+    else:
+        chunks.append(conditional)
+    return chunks
+
+
+def is_child_of(a, comparison):
+    if isinstance(comparison, Conditional):
+        return (
+            is_child_of(a, comparison.left) or is_child_of(a, comparison.right)
+        ) and comparison.operator == BooleanOperator.AND
+    return comparison == a
+
+
+class PredicatePushdown(OptimizationRule):
+
+    def optimize(self, cte: CTE, inverse_map: dict[str, list[CTE]]) -> bool:
+
+        if not cte.parent_ctes:
+            self.debug(f"No parent CTEs for {cte.name}")
+
+            return False
+
+        optimized = False
+        if not cte.condition:
+            self.debug(f"No CTE condition for {cte.name}")
+            return False
+        self.log(
+            f"Checking {cte.name} for predicate pushdown with {len(cte.parent_ctes)} parents"
+        )
+        if isinstance(cte.condition, Conditional):
+            candidates = decompose_condition(cte.condition)
+        else:
+            candidates = [cte.condition]
+        logger.info(f"Have {len(candidates)} candidates to try to push down")
+        for candidate in candidates:
+            conditions = {x.address for x in candidate.concept_arguments}
+            for parent_cte in cte.parent_ctes:
+                materialized = {k for k, v in parent_cte.source_map.items() if v != ""}
+                if conditions.issubset(materialized):
+                    if all(
+                        [
+                            is_child_of(candidate, child.condition)
+                            for child in inverse_map[parent_cte.name]
+                        ]
+                    ):
+                        self.log(
+                            f"All concepts are found on {parent_cte.name} and all it's children include same filter; pushing up filter"
+                        )
+                        if parent_cte.condition:
+                            parent_cte.condition = Conditional(
+                                left=parent_cte.condition,
+                                operator=BooleanOperator.AND,
+                                right=candidate,
+                            )
+                        else:
+                            parent_cte.condition = candidate
+                        optimized = True
+                else:
+                    logger.info("conditions not subset of parent materialized")
+
+        if all(
+            [
+                is_child_of(cte.condition, parent_cte.condition)
+                for parent_cte in cte.parent_ctes
+            ]
+        ):
+            self.log("All parents have same filter, removing filter")
+            cte.condition = None
+            optimized = True
+
+        return optimized
+
+
+if CONFIG.optimizations.datasource_inlining:
+    REGISTERED_RULES.append(InlineDatasource())
+if CONFIG.optimizations.predicate_pushdown:
+    REGISTERED_RULES.append(PredicatePushdown())
 
 
-def filter_irrelevant_ctes(input: list[CTE], root_cte: CTE):
+def filter_irrelevant_ctes(
+    input: list[CTE],
+    root_cte: CTE,
+):
     relevant_ctes = set()
 
     def recurse(cte: CTE):
@@ -75,6 +171,16 @@ def filter_irrelevant_ctes(input: list[CTE], root_cte: CTE):
     return [cte for cte in input if cte.name in relevant_ctes]
 
 
+def gen_inverse_map(input: list[CTE]) -> dict[str, list[CTE]]:
+    inverse_map: dict[str, list[CTE]] = {}
+    for cte in input:
+        for parent in cte.parent_ctes:
+            if parent.name not in inverse_map:
+                inverse_map[parent.name] = []
+            inverse_map[parent.name].append(cte)
+    return inverse_map
+
+
 def is_direct_return_eligible(
     cte: CTE, select: SelectStatement | PersistStatement | MultiSelectStatement
 ) -> bool:
@@ -126,7 +232,8 @@ def optimize_ctes(
         actions_taken = False
         for rule in REGISTERED_RULES:
             for cte in input:
-                actions_taken = rule.optimize(cte)
+                inverse_map = gen_inverse_map(input)
+                actions_taken = rule.optimize(cte, inverse_map)
         complete = not actions_taken
 
     if is_direct_return_eligible(root_cte, select):

trilogy/core/processing/node_generators/rowset_node.py

@@ -12,7 +12,7 @@ from typing import List
 
 from trilogy.core.enums import JoinType, PurposeLineage
 from trilogy.constants import logger
-from trilogy.core.processing.utility import padding
+from trilogy.core.processing.utility import padding, unique
 from trilogy.core.processing.node_generators.common import concept_to_relevant_joins
 
 
@@ -40,7 +40,7 @@ def gen_rowset_node(
     else:
         targets = select.output_components
     node: StrategyNode = source_concepts(
-        mandatory_list=targets,
+        mandatory_list=unique(targets, "address"),
         environment=environment,
         g=g,
         depth=depth + 1,
@@ -52,14 +52,19 @@ def gen_rowset_node(
         )
         return None
     node.conditions = select.where_clause.conditional if select.where_clause else None
-    # rebuild any cached info with the new condition clause
-
     enrichment = set([x.address for x in local_optional])
     rowset_relevant = [
         x
         for x in rowset.derived_concepts
         # if x.address == concept.address or x.address in enrichment
     ]
+    select_hidden = set([x.address for x in select.hidden_components])
+    rowset_hidden = [
+        x
+        for x in rowset.derived_concepts
+        if isinstance(x.lineage, RowsetItem)
+        and x.lineage.content.address in select_hidden
+    ]
     additional_relevant = [
         x for x in select.output_components if x.address in enrichment
     ]
@@ -71,7 +76,7 @@ def gen_rowset_node(
     if select.where_clause:
         for item in additional_relevant:
             node.partial_concepts.append(item)
-    node.hidden_concepts = [
+    node.hidden_concepts = rowset_hidden + [
         x
         for x in node.output_concepts
         if x.address not in [y.address for y in local_optional + [concept]]

trilogy/core/processing/nodes/merge_node.py

@@ -282,6 +282,7 @@ class MergeNode(StrategyNode):
                 if c.address in [x.address for x in self.output_concepts]
             ]
         )
+
         logger.info(
             f"{self.logging_prefix}{LOGGER_PREFIX} has pre grain {pregrain} and final merge node grain {grain}"
         )
@@ -307,9 +308,6 @@ class MergeNode(StrategyNode):
                 f"{self.logging_prefix}{LOGGER_PREFIX} no parents include full grain {grain} and pregrain {pregrain} does not match, assume must group to grain. Have {[str(d.grain) for d in final_datasets]}"
             )
             force_group = True
-            # Grain<returns.customer.id,returns.store.id,returns.item.id,returns.store_sales.ticket_number>
-            # Grain<returns.customer.id,returns.store.id,returns.return_date.id,returns.item.id,returns.store_sales.ticket_number>
-            # Grain<returns.customer.id,returns.store.id,returns.item.id,returns.store_sales.ticket_number>
         else:
             force_group = None