pytrilogy 0.0.2.9__py3-none-any.whl → 0.0.2.10__py3-none-any.whl

{pytrilogy-0.0.2.9.dist-info → pytrilogy-0.0.2.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.9
+Version: 0.0.2.10
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.2.9.dist-info → pytrilogy-0.0.2.10.dist-info}/RECORD

@@ -1,6 +1,6 @@
-trilogy/__init__.py,sha256=aJ0glX-aTfIGPSjtNqSF-k7RYaH3odx9C8DPBaxv1_4,290
+trilogy/__init__.py,sha256=-fwTLWZDBATA0SPkOXcsRS3py_W_F1oq7wt2cVysbNg,291
 trilogy/compiler.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/constants.py,sha256=gdaaoXq08_F7UVbZpJcUOfU-AcpykOE2blby6dL1L8c,922
+trilogy/constants.py,sha256=HRQq4i3cpSEJCywt61QKEzRO1jd4tEPZNSBuxUA_7yg,922
 trilogy/engine.py,sha256=R5ubIxYyrxRExz07aZCUfrTsoXCHQ8DKFTDsobXdWdA,1102
 trilogy/executor.py,sha256=5cRbU4Rj7p1pNV76rfp1pz704Hx_0q8_O8HFURjgXxQ,11016
 trilogy/parser.py,sha256=UtuqSiGiCjpMAYgo1bvNq-b7NSzCA5hzbUW31RXaMII,281
@@ -16,9 +16,9 @@ trilogy/core/exceptions.py,sha256=NvV_4qLOgKXbpotgRf7c8BANDEvHxlqRPaA53IThQ2o,56
 trilogy/core/functions.py,sha256=ARJAyBjeS415-54k3G_bx807rkPZonEulMaLRxSP7vU,10371
 trilogy/core/graph_models.py,sha256=oJUMSpmYhqXlavckHLpR07GJxuQ8dZ1VbB1fB0KaS8c,2036
 trilogy/core/internal.py,sha256=jNGFHKENnbMiMCtAgsnLZYVSENDK4b5ALecXFZpTDzQ,1075
-trilogy/core/models.py,sha256=jkJnt9Os8dATWJDuS5W_RPlmVKu82OEaoel6icX8ReQ,142371
+trilogy/core/models.py,sha256=mjORKd_RFvNc_c7QqCHCM4GrJSb5VA50tMqo7fX5-Bw,142855
 trilogy/core/optimization.py,sha256=RJmDr2f9hyFePF-B6LpmHkj69-AzUm-nNtvO59899O8,6601
-trilogy/core/query_processor.py,sha256=AWMX7Qguo6iI_e3Cm6iHrkvTdC_c8kWi22uPR84HiFA,18561
+trilogy/core/query_processor.py,sha256=N5Jdbb8D8_mdeJufIch0uCbfuf2psSSkLxGXwC5CqCg,18663
 trilogy/core/optimizations/__init__.py,sha256=pxRzNzd2g8oRMy4f_ub5va6bNS2pd4hnyp9JBzTKc1E,300
 trilogy/core/optimizations/base_optimization.py,sha256=tWWT-xnTbnEU-mNi_isMNbywm8B9WTRsNFwGpeh3rqE,468
 trilogy/core/optimizations/inline_constant.py,sha256=kHNyc2UoaPVdYfVAPAFwnWuk4sJ_IF5faRtVcDOrBtw,1110
@@ -27,11 +27,11 @@ trilogy/core/optimizations/predicate_pushdown.py,sha256=4Y6zfJN3VVexkD6p9IYWN4BT
 trilogy/core/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/processing/concept_strategies_v3.py,sha256=yjCiGKzghjAc0E0QAcdJ1Xr5RXHAQL2zMPOz-dMhEzM,25431
 trilogy/core/processing/graph_utils.py,sha256=aq-kqk4Iado2HywDxWEejWc-7PGO6Oa-ZQLAM6XWPHw,1199
-trilogy/core/processing/utility.py,sha256=0Teib4_VZlURRy1TvjDB6VBBxsqfJ7zTm2sXfPQjgXc,13950
+trilogy/core/processing/utility.py,sha256=15Qi68ktmnQmMt5jxrpueQ34oG5fSSgst3kIEejwF8A,14519
 trilogy/core/processing/node_generators/__init__.py,sha256=-mzYkRsaRNa_dfTckYkKVFSR8h8a3ihEiPJDU_tAmDo,672
 trilogy/core/processing/node_generators/basic_node.py,sha256=4242PNGTCm2tklqMIkqVu5Iv4m_IeTnOYXxDveuCDZM,2856
 trilogy/core/processing/node_generators/common.py,sha256=lDBRq9X6dQ_xSwXxLLNDq2pW8D-XwAY-ylTJLMugkLw,9525
-trilogy/core/processing/node_generators/filter_node.py,sha256=T4qmS_XvNDrvzm7srjpmlZQ1IkKhifn5oP4Gkc7Sk7Y,7133
+trilogy/core/processing/node_generators/filter_node.py,sha256=N8PoPvj-_ZQbucTyNqpZ1EyPTDAy6oB7DuEYCQxQBOs,7411
 trilogy/core/processing/node_generators/group_node.py,sha256=JwT8qYeGbozrZNGAlpndc9aaIN7K9ROYkkc2rAS8Qz8,3079
 trilogy/core/processing/node_generators/group_to_node.py,sha256=nzITnhaALIT7FMonyo16nNo-kSrLfefa9sZBYecrvkU,2887
 trilogy/core/processing/node_generators/multiselect_node.py,sha256=vP84dnLQy6dtypi6mUbt9sMAcmmrTgQ1Oz4GI6X1IEo,6421
@@ -43,17 +43,17 @@ trilogy/core/processing/node_generators/window_node.py,sha256=lFfmEjX_mLB7MuOM6C
 trilogy/core/processing/nodes/__init__.py,sha256=jyduHk96j5fpju72sc8swOiBjR3Md866kt8JZGkp3ZU,4866
 trilogy/core/processing/nodes/base_node.py,sha256=KVLAPLlo5QQq9Cm6lTLsqLN7l8EyXGxQkKrZFHdyRp8,12871
 trilogy/core/processing/nodes/filter_node.py,sha256=DBOSGFfkiILrZa1BlLv2uxUSkgWtSIKiZplqyKXPjg8,2132
-trilogy/core/processing/nodes/group_node.py,sha256=zLBOM_nWxSG_jIlRyayFIBDt-8yP8TIPgjNSewUCGlc,6492
+trilogy/core/processing/nodes/group_node.py,sha256=RKqODYxbY8wmClvI8-k3UgXWL7qQv5Hxhz5AB-RVUu0,6344
 trilogy/core/processing/nodes/merge_node.py,sha256=ZUcFIm5v4hip_Ml_oJXqVMrevUkE62wvPrvCrO9FWAc,14272
 trilogy/core/processing/nodes/select_node_v2.py,sha256=QuXNcwgjTRYamOoIooGrp4ie6INcqA9whtC5LZWjD8s,7180
 trilogy/core/processing/nodes/unnest_node.py,sha256=JFtm90IVM-46aCYkTNIaJah6v9ApAfonjVhcVM1HmDE,1903
 trilogy/core/processing/nodes/window_node.py,sha256=X7qxLUKd3tekjUUsmH_4vz5b-U89gMnGd04VBxuu2Ns,1280
 trilogy/dialect/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/dialect/base.py,sha256=qYPs_d-5Qu0ZK3Y1zQfwzCNNFKbWv0VlEbfl0DWsSrE,30377
+trilogy/dialect/base.py,sha256=_WKy2YJyDZdelHVX4mcnwKzAp2J-56UjoBpS5hG2yrg,30819
 trilogy/dialect/bigquery.py,sha256=15KJ-cOpBlk9O7FPviPgmg8xIydJeKx7WfmL3SSsPE8,2953
 trilogy/dialect/common.py,sha256=HVNPL8dGyQjT2REruV5C2YPpBbVR7KlD0akVM7GcuPI,3329
 trilogy/dialect/config.py,sha256=tLVEMctaTDhUgARKXUNfHUcIolGaALkQ0RavUvXAY4w,2994
-trilogy/dialect/duckdb.py,sha256=2ERqiG2owb3uvmU90Bdo1e2fq4ICWC3we2RUQpHVPwg,3173
+trilogy/dialect/duckdb.py,sha256=u_gpL35kouWxoBLas1h0ABYY2QzlVtEh22hm5h0lCOM,3182
 trilogy/dialect/enums.py,sha256=4NdpsydBpDn6jnh0JzFz5VvQEtnShErWtWHVyT6TNpw,3948
 trilogy/dialect/postgres.py,sha256=ev1RJZsC8BB3vJSxJ4q-TTYqZ4Hk1NXUtuRkLrQEBX0,3254
 trilogy/dialect/presto.py,sha256=2Rs53UfPxKU0rJTcEbiS-Lxm-CDiqUGojh7yRpQgyRE,3416
@@ -69,14 +69,14 @@ trilogy/parsing/common.py,sha256=zNd5buKxK4z9WSszOk4zOI0GexDFukzZtdNfQxg3kVw,621
 trilogy/parsing/config.py,sha256=Z-DaefdKhPDmSXLgg5V4pebhSB0h590vI0_VtHnlukI,111
 trilogy/parsing/exceptions.py,sha256=92E5i2frv5hj9wxObJZsZqj5T6bglvPzvdvco_vW1Zk,38
 trilogy/parsing/helpers.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-trilogy/parsing/parse_engine.py,sha256=L-Mlb1rdFG4UPxx1YY1JdNUe2dkGuXU6fuQzM5w-pek,61859
+trilogy/parsing/parse_engine.py,sha256=TD7dr9f4ffRcNB0BpzgaACqc-gD3Hy4_pOyLA7rmK7A,62563
 trilogy/parsing/render.py,sha256=Gy_6wVYPwYLf35Iota08sbqveuWILtUhI8MYStcvtJM,12174
 trilogy/parsing/trilogy.lark,sha256=kj1DXkqRFAb20PXN4gGDtXtQI61nUv1U8nfPWSWCr1Y,11407
 trilogy/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/scripts/trilogy.py,sha256=PHxvv6f2ODv0esyyhWxlARgra8dVhqQhYl0lTrSyVNo,3729
-pytrilogy-0.0.2.9.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
-pytrilogy-0.0.2.9.dist-info/METADATA,sha256=DkZW90_Pp71fS8wh_B6UulpaM-J8RU5CT2zzLyGSHGk,7906
-pytrilogy-0.0.2.9.dist-info/WHEEL,sha256=cVxcB9AmuTcXqmwrtPhNK88dr7IR_b6qagTj0UvIEbY,91
-pytrilogy-0.0.2.9.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
-pytrilogy-0.0.2.9.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
-pytrilogy-0.0.2.9.dist-info/RECORD,,
+pytrilogy-0.0.2.10.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
+pytrilogy-0.0.2.10.dist-info/METADATA,sha256=sH_c8ZeuuG6nuSMIrKQvhLoHi67-4qWu3DZ5KM27FJQ,7907
+pytrilogy-0.0.2.10.dist-info/WHEEL,sha256=cVxcB9AmuTcXqmwrtPhNK88dr7IR_b6qagTj0UvIEbY,91
+pytrilogy-0.0.2.10.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
+pytrilogy-0.0.2.10.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
+pytrilogy-0.0.2.10.dist-info/RECORD,,

trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.2.9"
+__version__ = "0.0.2.10"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

trilogy/constants.py

@@ -24,7 +24,7 @@ class Optimizations:
     predicate_pushdown: bool = True
     datasource_inlining: bool = True
     constant_inlining: bool = True
-    constant_inline_cutoff: int = 2
+    constant_inline_cutoff: int = 3
     direct_return: bool = True
 
 

trilogy/core/models.py

@@ -70,7 +70,7 @@ from trilogy.utility import unique
 from collections import UserList, UserDict
 from functools import cached_property
 from abc import ABC
-
+from collections import defaultdict
 
 LOGGER_PREFIX = "[MODELS]"
 
@@ -2027,7 +2027,7 @@ class Datasource(Namespaced, BaseModel):
         return self.__repr__()
 
     def __hash__(self):
-        return (self.namespace + self.identifier).__hash__()
+        return self.full_name.__hash__()
 
     def with_namespace(self, namespace: str):
         new_namespace = (
@@ -2212,9 +2212,9 @@ class BaseJoin(BaseModel):
 class QueryDatasource(BaseModel):
     input_concepts: List[Concept]
     output_concepts: List[Concept]
+    datasources: List[Union[Datasource, "QueryDatasource"]]
     source_map: Dict[str, Set[Union[Datasource, "QueryDatasource", "UnnestJoin"]]]
 
-    datasources: List[Union[Datasource, "QueryDatasource"]]
     grain: Grain
     joins: List[BaseJoin | UnnestJoin]
     limit: Optional[int] = None
@@ -2266,7 +2266,7 @@ class QueryDatasource(BaseModel):
 
     @field_validator("source_map")
     @classmethod
-    def validate_source_map(cls, v, info: ValidationInfo):
+    def validate_source_map(cls, v: dict, info: ValidationInfo):
         values = info.data
         for key in ("input_concepts", "output_concepts"):
             if not values.get(key):
@@ -2344,11 +2344,23 @@ class QueryDatasource(BaseModel):
         )
 
         merged_datasources = {}
+
         for ds in [*self.datasources, *other.datasources]:
             if ds.full_name in merged_datasources:
                 merged_datasources[ds.full_name] = merged_datasources[ds.full_name] + ds
             else:
                 merged_datasources[ds.full_name] = ds
+
+        final_source_map = defaultdict(set)
+        for key in self.source_map:
+            final_source_map[key] = self.source_map[key].union(
+                other.source_map.get(key, set())
+            )
+        for key in other.source_map:
+            if key not in final_source_map:
+                final_source_map[key] = other.source_map[key]
+        for k, v in final_source_map.items():
+            final_source_map[k] = set(merged_datasources[x.full_name] for x in list(v))
         qds = QueryDatasource(
             input_concepts=unique(
                 self.input_concepts + other.input_concepts, "address"
@@ -2356,7 +2368,7 @@ class QueryDatasource(BaseModel):
             output_concepts=unique(
                 self.output_concepts + other.output_concepts, "address"
             ),
-            source_map={**self.source_map, **other.source_map},
+            source_map=final_source_map,
             datasources=list(merged_datasources.values()),
             grain=self.grain,
             joins=unique(self.joins + other.joins, "unique_id"),

trilogy/core/processing/node_generators/filter_node.py

@@ -17,6 +17,7 @@ from trilogy.core.processing.node_generators.common import (
 from trilogy.constants import logger
 from trilogy.core.processing.utility import padding, unique
 from trilogy.core.processing.node_generators.common import concept_to_relevant_joins
+from trilogy.core.processing.utility import is_scalar_condition
 
 LOGGER_PREFIX = "[GEN_FILTER_NODE]"
 
@@ -78,16 +79,20 @@ def gen_filter_node(
         return None
 
     optimized_pushdown = False
-    if not local_optional:
+    if not is_scalar_condition(where.conditional):
+        optimized_pushdown = False
+    elif not local_optional:
         optimized_pushdown = True
     elif conditions and conditions == where:
         logger.info(
             f"{padding(depth)}{LOGGER_PREFIX} query conditions are the same as filter conditions, can optimize across all concepts"
         )
         optimized_pushdown = True
-
     if optimized_pushdown:
         if isinstance(row_parent, SelectNode):
+            logger.info(
+                f"{padding(depth)}{LOGGER_PREFIX} nesting select node in strategy node"
+            )
             parent = StrategyNode(
                 input_concepts=row_parent.output_concepts,
                 output_concepts=[concept] + row_parent.output_concepts,

trilogy/core/processing/nodes/group_node.py

@@ -130,7 +130,7 @@ class GroupNode(StrategyNode):
                     if self.conditions
                     else self.output_concepts
                 ),
-                inherited_inputs=self.input_concepts,
+                inherited_inputs=self.input_concepts + self.existence_concepts,
             ),
             joins=[],
             grain=grain,
@@ -139,10 +139,6 @@ class GroupNode(StrategyNode):
         )
         # if there is a condition on a group node and it's not scalar
         # inject an additional CTE
-        if self.conditions:
-            logger.info("CONDITIONS")
-            logger.info(str(self.conditions))
-            logger.info(is_scalar_condition(self.conditions))
         if self.conditions and not is_scalar_condition(self.conditions):
             base.condition = None
             base.output_concepts = self.output_concepts + self.conditions.row_arguments

trilogy/core/processing/utility.py

@@ -355,27 +355,40 @@ def is_scalar_condition(
         | MagicConstants
         | DataType
     ),
+    materialized: set[str] | None = None,
 ) -> bool:
     if isinstance(element, Parenthetical):
-        return is_scalar_condition(element.content)
+        return is_scalar_condition(element.content, materialized)
     elif isinstance(element, SubselectComparison):
         return True
     elif isinstance(element, Comparison):
-        return is_scalar_condition(element.left) and is_scalar_condition(element.right)
+        return is_scalar_condition(element.left, materialized) and is_scalar_condition(
+            element.right, materialized
+        )
     elif isinstance(element, Function):
         if element.operator in FunctionClass.AGGREGATE_FUNCTIONS.value:
             return False
+    elif isinstance(element, Concept):
+        if materialized and element.address in materialized:
+            return True
+        if element.lineage and isinstance(element.lineage, AggregateWrapper):
+            return is_scalar_condition(element.lineage, materialized)
+        return True
     elif isinstance(element, AggregateWrapper):
-        return is_scalar_condition(element.function)
+        return is_scalar_condition(element.function, materialized)
     elif isinstance(element, Conditional):
-        return is_scalar_condition(element.left) and is_scalar_condition(element.right)
+        return is_scalar_condition(element.left, materialized) and is_scalar_condition(
+            element.right, materialized
+        )
     return True
 
 
 def decompose_condition(
-    conditional: Conditional,
+    conditional: Conditional | Comparison | Parenthetical,
 ) -> list[SubselectComparison | Comparison | Conditional | Parenthetical]:
     chunks: list[SubselectComparison | Comparison | Conditional | Parenthetical] = []
+    if not isinstance(conditional, Conditional):
+        return [conditional]
     if conditional.operator == BooleanOperator.AND:
         if not (
             isinstance(

trilogy/core/query_processor.py

@@ -261,7 +261,7 @@ def datasource_to_ctes(
 
     human_id = generate_cte_name(query_datasource.full_name, name_map)
     logger.info(
-        f"Finished building source map for {human_id} with {len(parents)} parents, have {source_map}, query_datasource had non-empty keys {[k for k, v in query_datasource.source_map.items() if v]} "
+        f"Finished building source map for {human_id} with {len(parents)} parents, have {source_map}, query_datasource had non-empty keys {[k for k, v in query_datasource.source_map.items() if v]} and existence had non-empty keys {[k for k, v in query_datasource.existence_source_map.items() if v]} "
     )
     final_joins = [
         x

trilogy/dialect/base.py

@@ -2,7 +2,7 @@ from typing import List, Union, Optional, Dict, Any, Sequence, Callable
 
 from jinja2 import Template
 
-from trilogy.core.processing.utility import is_scalar_condition
+from trilogy.core.processing.utility import is_scalar_condition, decompose_condition
 from trilogy.constants import CONFIG, logger, MagicConstants
 from trilogy.core.internal import DEFAULT_CONCEPTS
 from trilogy.core.enums import (
@@ -538,6 +538,20 @@ class BaseDialect:
             final_joins = []
         else:
             final_joins = cte.joins or []
+        where: Conditional | Parenthetical | Comparison | None = None
+        having: Conditional | Parenthetical | Comparison | None = None
+        materialized = {x for x, v in cte.source_map.items() if v}
+        if cte.condition:
+            if is_scalar_condition(cte.condition, materialized=materialized):
+                where = cte.condition
+            else:
+                components = decompose_condition(cte.condition)
+                for x in components:
+                    if is_scalar_condition(x, materialized=materialized):
+                        where = where + x if where else x
+                    else:
+                        having = having + x if having else x
+
         return CompiledCTE(
             name=cte.name,
             statement=self.SQL_TEMPLATE.render(
@@ -561,16 +575,8 @@ class BaseDialect:
                     ]
                     if j
                 ],
-                where=(
-                    self.render_expr(cte.condition, cte)
-                    if cte.condition and is_scalar_condition(cte.condition)
-                    else None
-                ),
-                having=(
-                    self.render_expr(cte.condition, cte)
-                    if cte.condition and not is_scalar_condition(cte.condition)
-                    else None
-                ),
+                where=(self.render_expr(where, cte) if where else None),
+                having=(self.render_expr(having, cte) if having else None),
                 order_by=(
                     [self.render_order_item(i, cte) for i in cte.order_by.items]
                     if cte.order_by

trilogy/dialect/duckdb.py

@@ -36,7 +36,7 @@ FUNCTION_MAP = {
 # we may return a static value
 FUNCTION_GRAIN_MATCH_MAP = {
     **FUNCTION_MAP,
-    FunctionType.COUNT: lambda args: "1",
+    FunctionType.COUNT: lambda args: f"{args[0]}",
     FunctionType.SUM: lambda args: f"{args[0]}",
     FunctionType.AVG: lambda args: f"{args[0]}",
 }

trilogy/parsing/parse_engine.py

@@ -1116,7 +1116,25 @@ class ParseToObjects(Transformer):
     def comparison(self, args) -> Comparison:
         if args[1] == ComparisonOperator.IN:
             raise SyntaxError
-        return Comparison(left=args[0], right=args[2], operator=args[1])
+        if isinstance(args[0], AggregateWrapper):
+            left = arbitrary_to_concept(
+                args[0],
+                namespace=self.environment.namespace,
+                name=f"{VIRTUAL_CONCEPT_PREFIX}_{string_to_hash(str(args[0]))}",
+            )
+            self.environment.add_concept(left)
+        else:
+            left = args[0]
+        if isinstance(args[2], AggregateWrapper):
+            right = arbitrary_to_concept(
+                args[2],
+                namespace=self.environment.namespace,
+                name=f"{VIRTUAL_CONCEPT_PREFIX}_{string_to_hash(str(args[2]))}",
+            )
+            self.environment.add_concept(right)
+        else:
+            right = args[2]
+        return Comparison(left=left, right=right, operator=args[1])
 
     def between_comparison(self, args) -> Conditional:
         left_bound = args[1]