pytrilogy 0.0.2.9__py3-none-any.whl → 0.0.2.11__py3-none-any.whl

{pytrilogy-0.0.2.9.dist-info → pytrilogy-0.0.2.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.9
+Version: 0.0.2.11
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.2.9.dist-info → pytrilogy-0.0.2.11.dist-info}/RECORD

@@ -1,6 +1,6 @@
-trilogy/__init__.py,sha256=aJ0glX-aTfIGPSjtNqSF-k7RYaH3odx9C8DPBaxv1_4,290
+trilogy/__init__.py,sha256=rSlG7hlvpZjtLdU_7j6ldvO5bmiafhgPB40sdcOu3j0,291
 trilogy/compiler.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/constants.py,sha256=gdaaoXq08_F7UVbZpJcUOfU-AcpykOE2blby6dL1L8c,922
+trilogy/constants.py,sha256=HRQq4i3cpSEJCywt61QKEzRO1jd4tEPZNSBuxUA_7yg,922
 trilogy/engine.py,sha256=R5ubIxYyrxRExz07aZCUfrTsoXCHQ8DKFTDsobXdWdA,1102
 trilogy/executor.py,sha256=5cRbU4Rj7p1pNV76rfp1pz704Hx_0q8_O8HFURjgXxQ,11016
 trilogy/parser.py,sha256=UtuqSiGiCjpMAYgo1bvNq-b7NSzCA5hzbUW31RXaMII,281
@@ -16,26 +16,26 @@ trilogy/core/exceptions.py,sha256=NvV_4qLOgKXbpotgRf7c8BANDEvHxlqRPaA53IThQ2o,56
 trilogy/core/functions.py,sha256=ARJAyBjeS415-54k3G_bx807rkPZonEulMaLRxSP7vU,10371
 trilogy/core/graph_models.py,sha256=oJUMSpmYhqXlavckHLpR07GJxuQ8dZ1VbB1fB0KaS8c,2036
 trilogy/core/internal.py,sha256=jNGFHKENnbMiMCtAgsnLZYVSENDK4b5ALecXFZpTDzQ,1075
-trilogy/core/models.py,sha256=jkJnt9Os8dATWJDuS5W_RPlmVKu82OEaoel6icX8ReQ,142371
+trilogy/core/models.py,sha256=rIeYw173UjfTfc7194y-jqv7u51Uy6LufwbiwKI2Cgs,143834
 trilogy/core/optimization.py,sha256=RJmDr2f9hyFePF-B6LpmHkj69-AzUm-nNtvO59899O8,6601
-trilogy/core/query_processor.py,sha256=AWMX7Qguo6iI_e3Cm6iHrkvTdC_c8kWi22uPR84HiFA,18561
+trilogy/core/query_processor.py,sha256=50FHJ9Rw89ZNBflZK8R9jn7O_WdDHplS1hZSNXCozN8,19352
 trilogy/core/optimizations/__init__.py,sha256=pxRzNzd2g8oRMy4f_ub5va6bNS2pd4hnyp9JBzTKc1E,300
 trilogy/core/optimizations/base_optimization.py,sha256=tWWT-xnTbnEU-mNi_isMNbywm8B9WTRsNFwGpeh3rqE,468
 trilogy/core/optimizations/inline_constant.py,sha256=kHNyc2UoaPVdYfVAPAFwnWuk4sJ_IF5faRtVcDOrBtw,1110
 trilogy/core/optimizations/inline_datasource.py,sha256=AATzQ6YrtW_1-aQFjQyTYqEYKBoMFhek7ADfBr4uUdQ,3634
 trilogy/core/optimizations/predicate_pushdown.py,sha256=4Y6zfJN3VVexkD6p9IYWN4BTL8RwW6hyNb0VQZ8ETdI,8020
 trilogy/core/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/core/processing/concept_strategies_v3.py,sha256=yjCiGKzghjAc0E0QAcdJ1Xr5RXHAQL2zMPOz-dMhEzM,25431
+trilogy/core/processing/concept_strategies_v3.py,sha256=5yZQRWlEyNR2sddq7kx12_PRUNhekEu6bWO2yIvjjUA,25511
 trilogy/core/processing/graph_utils.py,sha256=aq-kqk4Iado2HywDxWEejWc-7PGO6Oa-ZQLAM6XWPHw,1199
-trilogy/core/processing/utility.py,sha256=0Teib4_VZlURRy1TvjDB6VBBxsqfJ7zTm2sXfPQjgXc,13950
+trilogy/core/processing/utility.py,sha256=15Qi68ktmnQmMt5jxrpueQ34oG5fSSgst3kIEejwF8A,14519
 trilogy/core/processing/node_generators/__init__.py,sha256=-mzYkRsaRNa_dfTckYkKVFSR8h8a3ihEiPJDU_tAmDo,672
 trilogy/core/processing/node_generators/basic_node.py,sha256=4242PNGTCm2tklqMIkqVu5Iv4m_IeTnOYXxDveuCDZM,2856
 trilogy/core/processing/node_generators/common.py,sha256=lDBRq9X6dQ_xSwXxLLNDq2pW8D-XwAY-ylTJLMugkLw,9525
-trilogy/core/processing/node_generators/filter_node.py,sha256=T4qmS_XvNDrvzm7srjpmlZQ1IkKhifn5oP4Gkc7Sk7Y,7133
-trilogy/core/processing/node_generators/group_node.py,sha256=JwT8qYeGbozrZNGAlpndc9aaIN7K9ROYkkc2rAS8Qz8,3079
+trilogy/core/processing/node_generators/filter_node.py,sha256=Ij2WqyOsu-TFxhAcL50PLMGpghsSWXJnWEJ8yTqOwrY,8228
+trilogy/core/processing/node_generators/group_node.py,sha256=Du-9uFXD0M-aHq2MV7v5R3QCrAL0JZBFMW-YQwgb6Bw,3135
 trilogy/core/processing/node_generators/group_to_node.py,sha256=nzITnhaALIT7FMonyo16nNo-kSrLfefa9sZBYecrvkU,2887
 trilogy/core/processing/node_generators/multiselect_node.py,sha256=vP84dnLQy6dtypi6mUbt9sMAcmmrTgQ1Oz4GI6X1IEo,6421
-trilogy/core/processing/node_generators/node_merge_node.py,sha256=yZ75lsl8RtM8kr-XQxqZEU6fm-JN5DTJxNbneayu36c,12319
+trilogy/core/processing/node_generators/node_merge_node.py,sha256=qS8VKppi10NKfqNb7BXTKyLBBn4wJuMmFYhrtMUkYb8,12580
 trilogy/core/processing/node_generators/rowset_node.py,sha256=6KVnuk75mRzWJ-jIk7e8azN8BIPPuCn-VxPlxDqfPVE,4616
 trilogy/core/processing/node_generators/select_node.py,sha256=E8bKOAUpwLwZy1iiaFVD5sM4XK-eFpHgijdyIWLMyH4,18904
 trilogy/core/processing/node_generators/unnest_node.py,sha256=gHjurhr86JFkbq6vxTHDQGDKt95EdotSFHVKgrR_Z3Q,1860
@@ -43,17 +43,17 @@ trilogy/core/processing/node_generators/window_node.py,sha256=lFfmEjX_mLB7MuOM6C
 trilogy/core/processing/nodes/__init__.py,sha256=jyduHk96j5fpju72sc8swOiBjR3Md866kt8JZGkp3ZU,4866
 trilogy/core/processing/nodes/base_node.py,sha256=KVLAPLlo5QQq9Cm6lTLsqLN7l8EyXGxQkKrZFHdyRp8,12871
 trilogy/core/processing/nodes/filter_node.py,sha256=DBOSGFfkiILrZa1BlLv2uxUSkgWtSIKiZplqyKXPjg8,2132
-trilogy/core/processing/nodes/group_node.py,sha256=zLBOM_nWxSG_jIlRyayFIBDt-8yP8TIPgjNSewUCGlc,6492
+trilogy/core/processing/nodes/group_node.py,sha256=RKqODYxbY8wmClvI8-k3UgXWL7qQv5Hxhz5AB-RVUu0,6344
 trilogy/core/processing/nodes/merge_node.py,sha256=ZUcFIm5v4hip_Ml_oJXqVMrevUkE62wvPrvCrO9FWAc,14272
 trilogy/core/processing/nodes/select_node_v2.py,sha256=QuXNcwgjTRYamOoIooGrp4ie6INcqA9whtC5LZWjD8s,7180
 trilogy/core/processing/nodes/unnest_node.py,sha256=JFtm90IVM-46aCYkTNIaJah6v9ApAfonjVhcVM1HmDE,1903
 trilogy/core/processing/nodes/window_node.py,sha256=X7qxLUKd3tekjUUsmH_4vz5b-U89gMnGd04VBxuu2Ns,1280
 trilogy/dialect/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/dialect/base.py,sha256=qYPs_d-5Qu0ZK3Y1zQfwzCNNFKbWv0VlEbfl0DWsSrE,30377
+trilogy/dialect/base.py,sha256=xbZXcWvwdRadyb77J2qat67R_gRAmKeeeI4JRH4P928,30726
 trilogy/dialect/bigquery.py,sha256=15KJ-cOpBlk9O7FPviPgmg8xIydJeKx7WfmL3SSsPE8,2953
 trilogy/dialect/common.py,sha256=HVNPL8dGyQjT2REruV5C2YPpBbVR7KlD0akVM7GcuPI,3329
 trilogy/dialect/config.py,sha256=tLVEMctaTDhUgARKXUNfHUcIolGaALkQ0RavUvXAY4w,2994
-trilogy/dialect/duckdb.py,sha256=2ERqiG2owb3uvmU90Bdo1e2fq4ICWC3we2RUQpHVPwg,3173
+trilogy/dialect/duckdb.py,sha256=u_gpL35kouWxoBLas1h0ABYY2QzlVtEh22hm5h0lCOM,3182
 trilogy/dialect/enums.py,sha256=4NdpsydBpDn6jnh0JzFz5VvQEtnShErWtWHVyT6TNpw,3948
 trilogy/dialect/postgres.py,sha256=ev1RJZsC8BB3vJSxJ4q-TTYqZ4Hk1NXUtuRkLrQEBX0,3254
 trilogy/dialect/presto.py,sha256=2Rs53UfPxKU0rJTcEbiS-Lxm-CDiqUGojh7yRpQgyRE,3416
@@ -69,14 +69,14 @@ trilogy/parsing/common.py,sha256=zNd5buKxK4z9WSszOk4zOI0GexDFukzZtdNfQxg3kVw,621
 trilogy/parsing/config.py,sha256=Z-DaefdKhPDmSXLgg5V4pebhSB0h590vI0_VtHnlukI,111
 trilogy/parsing/exceptions.py,sha256=92E5i2frv5hj9wxObJZsZqj5T6bglvPzvdvco_vW1Zk,38
 trilogy/parsing/helpers.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-trilogy/parsing/parse_engine.py,sha256=L-Mlb1rdFG4UPxx1YY1JdNUe2dkGuXU6fuQzM5w-pek,61859
+trilogy/parsing/parse_engine.py,sha256=riXOhMqYGvzyy1aeTZTjPIz3jOszue2VTEH7hn59zD4,63808
 trilogy/parsing/render.py,sha256=Gy_6wVYPwYLf35Iota08sbqveuWILtUhI8MYStcvtJM,12174
-trilogy/parsing/trilogy.lark,sha256=kj1DXkqRFAb20PXN4gGDtXtQI61nUv1U8nfPWSWCr1Y,11407
+trilogy/parsing/trilogy.lark,sha256=QNJnExOdvJyKTrQA4ffh-SGIz7rYd93kf2Ccs0m3cn4,11498
 trilogy/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/scripts/trilogy.py,sha256=PHxvv6f2ODv0esyyhWxlARgra8dVhqQhYl0lTrSyVNo,3729
-pytrilogy-0.0.2.9.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
-pytrilogy-0.0.2.9.dist-info/METADATA,sha256=DkZW90_Pp71fS8wh_B6UulpaM-J8RU5CT2zzLyGSHGk,7906
-pytrilogy-0.0.2.9.dist-info/WHEEL,sha256=cVxcB9AmuTcXqmwrtPhNK88dr7IR_b6qagTj0UvIEbY,91
-pytrilogy-0.0.2.9.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
-pytrilogy-0.0.2.9.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
-pytrilogy-0.0.2.9.dist-info/RECORD,,
+pytrilogy-0.0.2.11.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
+pytrilogy-0.0.2.11.dist-info/METADATA,sha256=mZ6V2gwihh9nv-bIGkimK2-bKNvsY_3JIUSNzP_m2Wg,7907
+pytrilogy-0.0.2.11.dist-info/WHEEL,sha256=cVxcB9AmuTcXqmwrtPhNK88dr7IR_b6qagTj0UvIEbY,91
+pytrilogy-0.0.2.11.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
+pytrilogy-0.0.2.11.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
+pytrilogy-0.0.2.11.dist-info/RECORD,,

trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.2.9"
+__version__ = "0.0.2.11"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

trilogy/constants.py

@@ -24,7 +24,7 @@ class Optimizations:
     predicate_pushdown: bool = True
     datasource_inlining: bool = True
     constant_inlining: bool = True
-    constant_inline_cutoff: int = 2
+    constant_inline_cutoff: int = 3
     direct_return: bool = True
 
 

trilogy/core/models.py

@@ -70,7 +70,7 @@ from trilogy.utility import unique
 from collections import UserList, UserDict
 from functools import cached_property
 from abc import ABC
-
+from collections import defaultdict
 
 LOGGER_PREFIX = "[MODELS]"
 
@@ -801,15 +801,18 @@ class Concept(Mergeable, Namespaced, SelectContext, BaseModel):
     ) -> "Concept":
         from trilogy.utility import string_to_hash
 
-        name = string_to_hash(self.name + str(condition))
+        if self.lineage and isinstance(self.lineage, FilterItem):
+            if self.lineage.where.conditional == condition:
+                return self
+        hash = string_to_hash(self.name + str(condition))
         new = Concept(
-            name=f"{self.name}_{name}",
+            name=f"{self.name}_filter_{hash}",
             datatype=self.datatype,
             purpose=self.purpose,
             metadata=self.metadata,
             lineage=FilterItem(content=self, where=WhereClause(conditional=condition)),
-            keys=None,
-            grain=(self.grain if self.purpose == Purpose.PROPERTY else Grain()),
+            keys=(self.keys if self.purpose == Purpose.PROPERTY else None),
+            grain=self.grain if self.grain else Grain(components=[]),
             namespace=self.namespace,
             modifiers=self.modifiers,
             pseudonyms=self.pseudonyms,
@@ -842,6 +845,16 @@ class Grain(Mergeable, BaseModel):
         v2 = sorted(final, key=lambda x: x.name)
         return v2
 
+    def with_filter(
+        self,
+        condition: "Conditional | Comparison | Parenthetical",
+        environment: Environment | None = None,
+    ) -> "Grain":
+        return Grain(
+            components=[c.with_filter(condition, environment) for c in self.components],
+            nested=self.nested,
+        )
+
     @property
     def components_copy(self) -> List[Concept]:
         return [*self.components]
@@ -1680,6 +1693,9 @@ class SelectStatement(Mergeable, Namespaced, SelectTypeMixin, BaseModel):
                 )
             ):
                 output.append(item)
+        # TODO: explore implicit filtering more
+        # if self.where_clause.conditional and self.where_clause_category == SelectFiltering.IMPLICIT:
+        #     output =[x.with_filter(self.where_clause.conditional) for x in output]
         return Grain(
             components=unique(output, "address"), where_clause=self.where_clause
         )
@@ -2027,7 +2043,7 @@ class Datasource(Namespaced, BaseModel):
         return self.__repr__()
 
     def __hash__(self):
-        return (self.namespace + self.identifier).__hash__()
+        return self.full_name.__hash__()
 
     def with_namespace(self, namespace: str):
         new_namespace = (
@@ -2212,9 +2228,9 @@ class BaseJoin(BaseModel):
 class QueryDatasource(BaseModel):
     input_concepts: List[Concept]
     output_concepts: List[Concept]
+    datasources: List[Union[Datasource, "QueryDatasource"]]
     source_map: Dict[str, Set[Union[Datasource, "QueryDatasource", "UnnestJoin"]]]
 
-    datasources: List[Union[Datasource, "QueryDatasource"]]
     grain: Grain
     joins: List[BaseJoin | UnnestJoin]
     limit: Optional[int] = None
@@ -2266,7 +2282,7 @@ class QueryDatasource(BaseModel):
 
     @field_validator("source_map")
     @classmethod
-    def validate_source_map(cls, v, info: ValidationInfo):
+    def validate_source_map(cls, v: dict, info: ValidationInfo):
         values = info.data
         for key in ("input_concepts", "output_concepts"):
             if not values.get(key):
@@ -2344,11 +2360,23 @@ class QueryDatasource(BaseModel):
         )
 
         merged_datasources = {}
+
         for ds in [*self.datasources, *other.datasources]:
             if ds.full_name in merged_datasources:
                 merged_datasources[ds.full_name] = merged_datasources[ds.full_name] + ds
             else:
                 merged_datasources[ds.full_name] = ds
+
+        final_source_map = defaultdict(set)
+        for key in self.source_map:
+            final_source_map[key] = self.source_map[key].union(
+                other.source_map.get(key, set())
+            )
+        for key in other.source_map:
+            if key not in final_source_map:
+                final_source_map[key] = other.source_map[key]
+        for k, v in final_source_map.items():
+            final_source_map[k] = set(merged_datasources[x.full_name] for x in list(v))
         qds = QueryDatasource(
             input_concepts=unique(
                 self.input_concepts + other.input_concepts, "address"
@@ -2356,7 +2384,7 @@ class QueryDatasource(BaseModel):
             output_concepts=unique(
                 self.output_concepts + other.output_concepts, "address"
             ),
-            source_map={**self.source_map, **other.source_map},
+            source_map=final_source_map,
             datasources=list(merged_datasources.values()),
             grain=self.grain,
             joins=unique(self.joins + other.joins, "unique_id"),
@@ -4062,8 +4090,13 @@ class RowsetDerivationStatement(Namespaced, BaseModel):
         output: list[Concept] = []
         orig: dict[str, Concept] = {}
         for orig_concept in self.select.output_components:
+            name = orig_concept.name
+            if isinstance(orig_concept.lineage, FilterItem):
+                if orig_concept.lineage.where == self.select.where_clause:
+                    name = orig_concept.lineage.content.name
+
             new_concept = Concept(
-                name=orig_concept.name,
+                name=name,
                 datatype=orig_concept.datatype,
                 purpose=orig_concept.purpose,
                 lineage=RowsetItem(

trilogy/core/processing/concept_strategies_v3.py

@@ -191,6 +191,7 @@ def generate_candidates_restrictive(
         ):
             combos.append(local_candidates)
     combos.append(grain_check)
+    # combos.append(local_candidates)
     # append the empty set for sourcing concept by itself last
     combos.append([])
     return combos
@@ -645,6 +646,7 @@ def _search_concepts(
             depth=depth,
             source_concepts=search_concepts,
             history=history,
+            search_conditions=conditions,
         )
 
         if expanded:

trilogy/core/processing/node_generators/filter_node.py

@@ -17,6 +17,7 @@ from trilogy.core.processing.node_generators.common import (
 from trilogy.constants import logger
 from trilogy.core.processing.utility import padding, unique
 from trilogy.core.processing.node_generators.common import concept_to_relevant_joins
+from trilogy.core.processing.utility import is_scalar_condition
 
 LOGGER_PREFIX = "[GEN_FILTER_NODE]"
 
@@ -38,6 +39,15 @@ def gen_filter_node(
         raise SyntaxError('Filter node must have a lineage of type "FilterItem"')
     where = concept.lineage.where
 
+    optional_included: list[Concept] = []
+    for x in local_optional:
+        if isinstance(x.lineage, FilterItem):
+            if concept.lineage.where == where:
+                logger.info(
+                    f"{padding(depth)}{LOGGER_PREFIX} fetching {x.lineage.content.address} as optional parent with same filter conditions "
+                )
+                parent_row_concepts.append(x.lineage.content)
+                optional_included.append(x)
     logger.info(
         f"{padding(depth)}{LOGGER_PREFIX} filter {concept.address} derived from {immediate_parent.address} row parents {[x.address for x in parent_row_concepts]} and {[[y.address] for x  in parent_existence_concepts for y  in x]} existence parents"
     )
@@ -48,6 +58,7 @@ def gen_filter_node(
         g=g,
         depth=depth + 1,
         history=history,
+        conditions=conditions,
     )
 
     flattened_existence = [x for y in parent_existence_concepts for x in y]
@@ -78,16 +89,25 @@ def gen_filter_node(
         return None
 
     optimized_pushdown = False
-    if not local_optional:
+    if not is_scalar_condition(where.conditional):
+        optimized_pushdown = False
+    elif not local_optional:
         optimized_pushdown = True
     elif conditions and conditions == where:
         logger.info(
             f"{padding(depth)}{LOGGER_PREFIX} query conditions are the same as filter conditions, can optimize across all concepts"
         )
         optimized_pushdown = True
-
+    elif optional_included == local_optional:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} all optional concepts are included in the filter, can optimize across all concepts"
+        )
+        optimized_pushdown = True
     if optimized_pushdown:
         if isinstance(row_parent, SelectNode):
+            logger.info(
+                f"{padding(depth)}{LOGGER_PREFIX} nesting select node in strategy node"
+            )
             parent = StrategyNode(
                 input_concepts=row_parent.output_concepts,
                 output_concepts=[concept] + row_parent.output_concepts,
@@ -111,6 +131,7 @@ def gen_filter_node(
             x
             for x in local_optional
             if x.address in [y.address for y in parent.output_concepts]
+            or x.address in [y.address for y in optional_included]
         ]
         parent.add_parents(core_parents)
         parent.add_condition(where.conditional)
@@ -170,6 +191,7 @@ def gen_filter_node(
         ] + outputs
         filter_node.rebuild_cache()
         return filter_node
+
     enrich_node = source_concepts(  # this fetches the parent + join keys
         # to then connect to the rest of the query
         mandatory_list=[immediate_parent] + parent_row_concepts + local_optional,
@@ -177,10 +199,11 @@ def gen_filter_node(
         g=g,
         depth=depth + 1,
         history=history,
+        conditions=conditions,
     )
     if not enrich_node:
         return filter_node
-    x = MergeNode(
+    return MergeNode(
         input_concepts=[concept, immediate_parent] + local_optional,
         output_concepts=[
             concept,
@@ -201,8 +224,7 @@ def gen_filter_node(
                     [immediate_parent] + parent_row_concepts
                 ),
                 join_type=JoinType.LEFT_OUTER,
-                filter_to_mutual=False,
+                filter_to_mutual=True,
             )
         ],
     )
-    return x

trilogy/core/processing/node_generators/group_node.py

@@ -91,4 +91,6 @@ def gen_group_node(
         depth=depth,
         source_concepts=source_concepts,
         log_lambda=create_log_lambda(LOGGER_PREFIX, depth, logger),
+        history=history,
+        conditions=conditions,
     )

trilogy/core/processing/node_generators/node_merge_node.py

@@ -1,6 +1,6 @@
 from typing import List, Optional
 
-from trilogy.core.models import Concept, Environment, Conditional
+from trilogy.core.models import Concept, Environment, Conditional, WhereClause
 from trilogy.core.processing.nodes import MergeNode, History, StrategyNode
 import networkx as nx
 from trilogy.core.graph_models import concept_to_node
@@ -260,6 +260,7 @@ def subgraphs_to_merge_node(
     source_concepts,
     history,
     conditions,
+    search_conditions: WhereClause | None = None,
     enable_early_exit: bool = True,
 ):
     parents: List[StrategyNode] = []
@@ -277,6 +278,7 @@ def subgraphs_to_merge_node(
             g=g,
             depth=depth + 1,
             history=history,
+            conditions=search_conditions,
         )
         if not parent:
             logger.info(
@@ -315,6 +317,7 @@ def gen_merge_node(
     accept_partial: bool = False,
     history: History | None = None,
     conditions: Conditional | None = None,
+    search_conditions: WhereClause | None = None,
 ) -> Optional[MergeNode]:
 
     for filter_downstream in [True, False]:
@@ -339,6 +342,7 @@ def gen_merge_node(
                 source_concepts=source_concepts,
                 history=history,
                 conditions=conditions,
+                search_conditions=search_conditions,
             )
     # one concept handling may need to be kicked to alias
     if len(all_concepts) == 1:
@@ -354,6 +358,7 @@ def gen_merge_node(
                 history=history,
                 conditions=conditions,
                 enable_early_exit=False,
+                search_conditions=search_conditions,
             )
             if test:
                 return test

trilogy/core/processing/nodes/group_node.py

@@ -130,7 +130,7 @@ class GroupNode(StrategyNode):
                     if self.conditions
                     else self.output_concepts
                 ),
-                inherited_inputs=self.input_concepts,
+                inherited_inputs=self.input_concepts + self.existence_concepts,
             ),
             joins=[],
             grain=grain,
@@ -139,10 +139,6 @@ class GroupNode(StrategyNode):
         )
         # if there is a condition on a group node and it's not scalar
         # inject an additional CTE
-        if self.conditions:
-            logger.info("CONDITIONS")
-            logger.info(str(self.conditions))
-            logger.info(is_scalar_condition(self.conditions))
         if self.conditions and not is_scalar_condition(self.conditions):
             base.condition = None
             base.output_concepts = self.output_concepts + self.conditions.row_arguments

trilogy/core/processing/utility.py

@@ -355,27 +355,40 @@ def is_scalar_condition(
         | MagicConstants
         | DataType
     ),
+    materialized: set[str] | None = None,
 ) -> bool:
     if isinstance(element, Parenthetical):
-        return is_scalar_condition(element.content)
+        return is_scalar_condition(element.content, materialized)
     elif isinstance(element, SubselectComparison):
         return True
     elif isinstance(element, Comparison):
-        return is_scalar_condition(element.left) and is_scalar_condition(element.right)
+        return is_scalar_condition(element.left, materialized) and is_scalar_condition(
+            element.right, materialized
+        )
     elif isinstance(element, Function):
         if element.operator in FunctionClass.AGGREGATE_FUNCTIONS.value:
             return False
+    elif isinstance(element, Concept):
+        if materialized and element.address in materialized:
+            return True
+        if element.lineage and isinstance(element.lineage, AggregateWrapper):
+            return is_scalar_condition(element.lineage, materialized)
+        return True
     elif isinstance(element, AggregateWrapper):
-        return is_scalar_condition(element.function)
+        return is_scalar_condition(element.function, materialized)
     elif isinstance(element, Conditional):
-        return is_scalar_condition(element.left) and is_scalar_condition(element.right)
+        return is_scalar_condition(element.left, materialized) and is_scalar_condition(
+            element.right, materialized
+        )
     return True
 
 
 def decompose_condition(
-    conditional: Conditional,
+    conditional: Conditional | Comparison | Parenthetical,
 ) -> list[SubselectComparison | Comparison | Conditional | Parenthetical]:
     chunks: list[SubselectComparison | Comparison | Conditional | Parenthetical] = []
+    if not isinstance(conditional, Conditional):
+        return [conditional]
     if conditional.operator == BooleanOperator.AND:
         if not (
             isinstance(

trilogy/core/query_processor.py

@@ -145,7 +145,9 @@ def generate_source_map(
         names = set([x.name for x in ev])
         ematches = [cte.name for cte in all_new_ctes if cte.source.name in names]
         existence_source_map[ek] = ematches
-    return {k: [] if not v else v for k, v in source_map.items()}, existence_source_map
+    return {
+        k: [] if not v else list(set(v)) for k, v in source_map.items()
+    }, existence_source_map
 
 
 def datasource_to_query_datasource(datasource: Datasource) -> QueryDatasource:
@@ -191,6 +193,8 @@ def resolve_cte_base_name_and_alias_v2(
     raw_joins: List[Join | InstantiatedUnnestJoin],
 ) -> Tuple[str | None, str | None]:
     joins: List[Join] = [join for join in raw_joins if isinstance(join, Join)]
+    # INFO     trilogy:query_processor.py:263 Finished building source map for civet with 3 parents, have {'local.relevant_customers': ['fowl', 'fowl'],
+    # 'customer.demographics.gender': ['mandrill'], 'customer.id': ['mandrill'], 'customer.demographics.id': ['mandrill'], 'customer.id_9268029262289908': [], 'customer.demographics.gender_1513806568509111': []}, query_datasource had non-empty keys ['local.relevant_customers', 'customer.demographics.gender', 'customer.id', 'customer.demographics.id'] and existence had non-empty keys []
     if (
         len(source.datasources) == 1
         and isinstance(source.datasources[0], Datasource)
@@ -212,12 +216,16 @@ def resolve_cte_base_name_and_alias_v2(
 
     counts: dict[str, int] = defaultdict(lambda: 0)
     output_addresses = [x.address for x in source.output_concepts]
+    input_address = [x.address for x in source.input_concepts]
     for k, v in source_map.items():
         for vx in v:
             if k in output_addresses:
                 counts[vx] = counts[vx] + 1
-            else:
-                counts[vx] = counts[vx]
+
+            if k in input_address:
+                counts[vx] = counts[vx] + 1
+
+            counts[vx] = counts[vx]
     if counts:
         return max(counts, key=counts.get), max(counts, key=counts.get)  # type: ignore
     return None, None
@@ -261,7 +269,7 @@ def datasource_to_ctes(
 
     human_id = generate_cte_name(query_datasource.full_name, name_map)
     logger.info(
-        f"Finished building source map for {human_id} with {len(parents)} parents, have {source_map}, query_datasource had non-empty keys {[k for k, v in query_datasource.source_map.items() if v]} "
+        f"Finished building source map for {human_id} with {len(parents)} parents, have {source_map}, query_datasource had non-empty keys {[k for k, v in query_datasource.source_map.items() if v]} and existence had non-empty keys {[k for k, v in query_datasource.existence_source_map.items() if v]} "
     )
     final_joins = [
         x

trilogy/dialect/base.py

@@ -2,7 +2,7 @@ from typing import List, Union, Optional, Dict, Any, Sequence, Callable
 
 from jinja2 import Template
 
-from trilogy.core.processing.utility import is_scalar_condition
+from trilogy.core.processing.utility import is_scalar_condition, decompose_condition
 from trilogy.constants import CONFIG, logger, MagicConstants
 from trilogy.core.internal import DEFAULT_CONCEPTS
 from trilogy.core.enums import (
@@ -281,10 +281,7 @@ class BaseDialect:
                 rval = f"{self.WINDOW_FUNCTION_MAP[c.lineage.type](concept = self.render_concept_sql(c.lineage.content, cte=cte, alias=False), window=','.join(rendered_over_components), sort=','.join(rendered_order_components))}"  # noqa: E501
             elif isinstance(c.lineage, FilterItem):
                 # for cases when we've optimized this
-                if (
-                    len(cte.output_columns) == 1
-                    and cte.condition == c.lineage.where.conditional
-                ):
+                if cte.condition == c.lineage.where.conditional:
                     rval = self.render_expr(c.lineage.content, cte=cte)
                 else:
                     rval = f"CASE WHEN {self.render_expr(c.lineage.where.conditional, cte=cte)} THEN {self.render_concept_sql(c.lineage.content, cte=cte, alias=False)} ELSE NULL END"
@@ -538,6 +535,20 @@ class BaseDialect:
             final_joins = []
         else:
             final_joins = cte.joins or []
+        where: Conditional | Parenthetical | Comparison | None = None
+        having: Conditional | Parenthetical | Comparison | None = None
+        materialized = {x for x, v in cte.source_map.items() if v}
+        if cte.condition:
+            if is_scalar_condition(cte.condition, materialized=materialized):
+                where = cte.condition
+            else:
+                components = decompose_condition(cte.condition)
+                for x in components:
+                    if is_scalar_condition(x, materialized=materialized):
+                        where = where + x if where else x
+                    else:
+                        having = having + x if having else x
+
         return CompiledCTE(
             name=cte.name,
             statement=self.SQL_TEMPLATE.render(
@@ -561,16 +572,8 @@ class BaseDialect:
                     ]
                     if j
                 ],
-                where=(
-                    self.render_expr(cte.condition, cte)
-                    if cte.condition and is_scalar_condition(cte.condition)
-                    else None
-                ),
-                having=(
-                    self.render_expr(cte.condition, cte)
-                    if cte.condition and not is_scalar_condition(cte.condition)
-                    else None
-                ),
+                where=(self.render_expr(where, cte) if where else None),
+                having=(self.render_expr(having, cte) if having else None),
                 order_by=(
                     [self.render_order_item(i, cte) for i in cte.order_by.items]
                     if cte.order_by

trilogy/dialect/duckdb.py

@@ -36,7 +36,7 @@ FUNCTION_MAP = {
 # we may return a static value
 FUNCTION_GRAIN_MATCH_MAP = {
     **FUNCTION_MAP,
-    FunctionType.COUNT: lambda args: "1",
+    FunctionType.COUNT: lambda args: f"{args[0]}",
     FunctionType.SUM: lambda args: f"{args[0]}",
     FunctionType.AVG: lambda args: f"{args[0]}",
 }

trilogy/parsing/parse_engine.py

@@ -983,6 +983,7 @@ class ParseToObjects(Transformer):
             order_by=order_by,
             meta=Metadata(line_number=meta.line),
         )
+
         for item in select_items:
             # we don't know the grain of an aggregate at assignment time
             # so rebuild at this point in the tree
@@ -1000,21 +1001,43 @@ class ParseToObjects(Transformer):
                 )
                 self.environment.add_concept(new_concept, meta=meta)
                 item.content.output = new_concept
+            # TODO: revisit if we can push down every filter
+            # else:
+            #     item.content = (
+            #         item.content.with_filter(
+            #             output.where_clause.conditional, environment=self.environment
+            #         )
+            #         if output.where_clause
+            #         and output.where_clause_category == SelectFiltering.IMPLICIT
+            #         else item.content
+            #     )
+
         if order_by:
             for orderitem in order_by.items:
-                if (
-                    isinstance(orderitem.expr, Concept)
-                    and orderitem.expr.purpose == Purpose.METRIC
-                ):
-                    orderitem.expr = orderitem.expr.with_select_context(
-                        output.grain,
-                        conditional=(
-                            output.where_clause.conditional
-                            if output.where_clause
-                            and output.where_clause_category == SelectFiltering.IMPLICIT
-                            else None
-                        ),
-                    )
+                if isinstance(orderitem.expr, Concept):
+                    if orderitem.expr.purpose == Purpose.METRIC:
+                        orderitem.expr = orderitem.expr.with_select_context(
+                            output.grain,
+                            conditional=(
+                                output.where_clause.conditional
+                                if output.where_clause
+                                and output.where_clause_category
+                                == SelectFiltering.IMPLICIT
+                                else None
+                            ),
+                            environment=self.environment,
+                        )
+                    # TODO :push down every filter
+                    # else:
+                    #     orderitem.expr = (
+                    #         orderitem.expr.with_filter(
+                    #             output.where_clause.conditional,
+                    #             environment=self.environment,
+                    #         )
+                    #         if output.where_clause
+                    #         and output.where_clause_category == SelectFiltering.IMPLICIT
+                    #         else orderitem.expr
+                    #     )
         return output
 
     @v_args(meta=True)
@@ -1116,7 +1139,25 @@ class ParseToObjects(Transformer):
     def comparison(self, args) -> Comparison:
         if args[1] == ComparisonOperator.IN:
             raise SyntaxError
-        return Comparison(left=args[0], right=args[2], operator=args[1])
+        if isinstance(args[0], AggregateWrapper):
+            left = arbitrary_to_concept(
+                args[0],
+                namespace=self.environment.namespace,
+                name=f"{VIRTUAL_CONCEPT_PREFIX}_{string_to_hash(str(args[0]))}",
+            )
+            self.environment.add_concept(left)
+        else:
+            left = args[0]
+        if isinstance(args[2], AggregateWrapper):
+            right = arbitrary_to_concept(
+                args[2],
+                namespace=self.environment.namespace,
+                name=f"{VIRTUAL_CONCEPT_PREFIX}_{string_to_hash(str(args[2]))}",
+            )
+            self.environment.add_concept(right)
+        else:
+            right = args[2]
+        return Comparison(left=left, right=right, operator=args[1])
 
     def between_comparison(self, args) -> Conditional:
         left_bound = args[1]
@@ -1219,7 +1260,11 @@ class ParseToObjects(Transformer):
 
     def filter_item(self, args) -> FilterItem:
         where: WhereClause
-        string_concept, where = args
+        string_concept, raw = args
+        if isinstance(raw, WhereClause):
+            where = raw
+        else:
+            where = WhereClause(conditional=raw)
         concept = self.environment.concepts[string_concept]
         return FilterItem(content=concept, where=where)
 

trilogy/parsing/trilogy.lark

@@ -83,7 +83,11 @@
     raw_function: "bind" "sql"  IDENTIFIER  "("  function_binding_list ")" "->" data_type "as"i MULTILINE_STRING
     
     // user_id where state = Mexico
-    filter_item: "filter"i IDENTIFIER where
+    _filter_alt: IDENTIFIER "?" conditional
+    _filter_base: "filter"i IDENTIFIER where
+    filter_item: _filter_base | _filter_alt
+
+
 
     // rank/lag/lead
     WINDOW_TYPE: ("row_number"i|"rank"i|"lag"i|"lead"i | "sum"i)  /[\s]+/