pytrilogy 0.0.2.10__tar.gz → 0.0.2.11__tar.gz

{pytrilogy-0.0.2.10/pytrilogy.egg-info → pytrilogy-0.0.2.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.10
+Version: 0.0.2.11
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.2.10 → pytrilogy-0.0.2.11/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.10
+Version: 0.0.2.11
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.2.10 → pytrilogy-0.0.2.11}/tests/test_models.py

@@ -12,6 +12,7 @@ from trilogy.core.models import (
     Comparison,
     Join,
     JoinKey,
+    Concept,
 )
 
 
@@ -69,6 +70,23 @@ def test_concept(test_environment, test_environment_graph):
     )
 
 
+def test_concept_filter(test_environment, test_environment_graph):
+    test_concept: Concept = list(test_environment.concepts.values())[0]
+    new = test_concept.with_filter(
+        Comparison(left=1, right="abc", operator=ComparisonOperator.EQ)
+    )
+    new2 = test_concept.with_filter(
+        Comparison(left=1, right="abc", operator=ComparisonOperator.EQ)
+    )
+
+    assert new.name == new2.name != test_concept.name
+
+    new3 = new.with_filter(
+        Comparison(left=1, right="abc", operator=ComparisonOperator.EQ)
+    )
+    assert new3 == new
+
+
 def test_conditional(test_environment, test_environment_graph):
     test_concept = list(test_environment.concepts.values())[-1]
 

{pytrilogy-0.0.2.10 → pytrilogy-0.0.2.11}/tests/test_parsing.py

@@ -492,3 +492,27 @@ select x;
     results = Dialects.DUCK_DB.default_executor().generate_sql(text)[0]
 
     assert "abcdef as test" in results, results
+
+
+def test_filter_concise():
+
+    text = """
+key x int;
+key y int;
+
+datasource test (
+x:x,
+y:y)
+grain(x)
+address `abc:def`
+;
+
+auto filtered_test <- x ? y > 10;
+
+select filtered_test;
+"""
+    env, parsed = parse_text(text)
+
+    results = Dialects.DUCK_DB.default_executor().generate_sql(text)[0]
+
+    assert "filtered_test" in results, results

{pytrilogy-0.0.2.10 → pytrilogy-0.0.2.11}/trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.2.10"
+__version__ = "0.0.2.11"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

{pytrilogy-0.0.2.10 → pytrilogy-0.0.2.11}/trilogy/core/models.py

@@ -801,15 +801,18 @@ class Concept(Mergeable, Namespaced, SelectContext, BaseModel):
     ) -> "Concept":
         from trilogy.utility import string_to_hash
 
-        name = string_to_hash(self.name + str(condition))
+        if self.lineage and isinstance(self.lineage, FilterItem):
+            if self.lineage.where.conditional == condition:
+                return self
+        hash = string_to_hash(self.name + str(condition))
         new = Concept(
-            name=f"{self.name}_{name}",
+            name=f"{self.name}_filter_{hash}",
             datatype=self.datatype,
             purpose=self.purpose,
             metadata=self.metadata,
             lineage=FilterItem(content=self, where=WhereClause(conditional=condition)),
-            keys=None,
-            grain=(self.grain if self.purpose == Purpose.PROPERTY else Grain()),
+            keys=(self.keys if self.purpose == Purpose.PROPERTY else None),
+            grain=self.grain if self.grain else Grain(components=[]),
             namespace=self.namespace,
             modifiers=self.modifiers,
             pseudonyms=self.pseudonyms,
@@ -842,6 +845,16 @@ class Grain(Mergeable, BaseModel):
         v2 = sorted(final, key=lambda x: x.name)
         return v2
 
+    def with_filter(
+        self,
+        condition: "Conditional | Comparison | Parenthetical",
+        environment: Environment | None = None,
+    ) -> "Grain":
+        return Grain(
+            components=[c.with_filter(condition, environment) for c in self.components],
+            nested=self.nested,
+        )
+
     @property
     def components_copy(self) -> List[Concept]:
         return [*self.components]
@@ -1680,6 +1693,9 @@ class SelectStatement(Mergeable, Namespaced, SelectTypeMixin, BaseModel):
                 )
             ):
                 output.append(item)
+        # TODO: explore implicit filtering more
+        # if self.where_clause.conditional and self.where_clause_category == SelectFiltering.IMPLICIT:
+        #     output =[x.with_filter(self.where_clause.conditional) for x in output]
         return Grain(
             components=unique(output, "address"), where_clause=self.where_clause
         )
@@ -4074,8 +4090,13 @@ class RowsetDerivationStatement(Namespaced, BaseModel):
         output: list[Concept] = []
         orig: dict[str, Concept] = {}
         for orig_concept in self.select.output_components:
+            name = orig_concept.name
+            if isinstance(orig_concept.lineage, FilterItem):
+                if orig_concept.lineage.where == self.select.where_clause:
+                    name = orig_concept.lineage.content.name
+
             new_concept = Concept(
-                name=orig_concept.name,
+                name=name,
                 datatype=orig_concept.datatype,
                 purpose=orig_concept.purpose,
                 lineage=RowsetItem(

{pytrilogy-0.0.2.10 → pytrilogy-0.0.2.11}/trilogy/core/processing/concept_strategies_v3.py

@@ -191,6 +191,7 @@ def generate_candidates_restrictive(
         ):
             combos.append(local_candidates)
     combos.append(grain_check)
+    # combos.append(local_candidates)
     # append the empty set for sourcing concept by itself last
     combos.append([])
     return combos
@@ -645,6 +646,7 @@ def _search_concepts(
             depth=depth,
             source_concepts=search_concepts,
             history=history,
+            search_conditions=conditions,
         )
 
         if expanded:

{pytrilogy-0.0.2.10 → pytrilogy-0.0.2.11}/trilogy/core/processing/node_generators/filter_node.py

@@ -39,6 +39,15 @@ def gen_filter_node(
         raise SyntaxError('Filter node must have a lineage of type "FilterItem"')
     where = concept.lineage.where
 
+    optional_included: list[Concept] = []
+    for x in local_optional:
+        if isinstance(x.lineage, FilterItem):
+            if concept.lineage.where == where:
+                logger.info(
+                    f"{padding(depth)}{LOGGER_PREFIX} fetching {x.lineage.content.address} as optional parent with same filter conditions "
+                )
+                parent_row_concepts.append(x.lineage.content)
+                optional_included.append(x)
     logger.info(
         f"{padding(depth)}{LOGGER_PREFIX} filter {concept.address} derived from {immediate_parent.address} row parents {[x.address for x in parent_row_concepts]} and {[[y.address] for x  in parent_existence_concepts for y  in x]} existence parents"
     )
@@ -49,6 +58,7 @@ def gen_filter_node(
         g=g,
         depth=depth + 1,
         history=history,
+        conditions=conditions,
     )
 
     flattened_existence = [x for y in parent_existence_concepts for x in y]
@@ -88,6 +98,11 @@ def gen_filter_node(
             f"{padding(depth)}{LOGGER_PREFIX} query conditions are the same as filter conditions, can optimize across all concepts"
         )
         optimized_pushdown = True
+    elif optional_included == local_optional:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} all optional concepts are included in the filter, can optimize across all concepts"
+        )
+        optimized_pushdown = True
     if optimized_pushdown:
         if isinstance(row_parent, SelectNode):
             logger.info(
@@ -116,6 +131,7 @@ def gen_filter_node(
             x
             for x in local_optional
             if x.address in [y.address for y in parent.output_concepts]
+            or x.address in [y.address for y in optional_included]
         ]
         parent.add_parents(core_parents)
         parent.add_condition(where.conditional)
@@ -175,6 +191,7 @@ def gen_filter_node(
         ] + outputs
         filter_node.rebuild_cache()
         return filter_node
+
     enrich_node = source_concepts(  # this fetches the parent + join keys
         # to then connect to the rest of the query
         mandatory_list=[immediate_parent] + parent_row_concepts + local_optional,
@@ -182,10 +199,11 @@ def gen_filter_node(
         g=g,
         depth=depth + 1,
         history=history,
+        conditions=conditions,
     )
     if not enrich_node:
         return filter_node
-    x = MergeNode(
+    return MergeNode(
         input_concepts=[concept, immediate_parent] + local_optional,
         output_concepts=[
             concept,
@@ -206,8 +224,7 @@ def gen_filter_node(
                     [immediate_parent] + parent_row_concepts
                 ),
                 join_type=JoinType.LEFT_OUTER,
-                filter_to_mutual=False,
+                filter_to_mutual=True,
             )
         ],
     )
-    return x

{pytrilogy-0.0.2.10 → pytrilogy-0.0.2.11}/trilogy/core/processing/node_generators/group_node.py

@@ -91,4 +91,6 @@ def gen_group_node(
         depth=depth,
         source_concepts=source_concepts,
         log_lambda=create_log_lambda(LOGGER_PREFIX, depth, logger),
+        history=history,
+        conditions=conditions,
     )

{pytrilogy-0.0.2.10 → pytrilogy-0.0.2.11}/trilogy/core/processing/node_generators/node_merge_node.py

@@ -1,6 +1,6 @@
 from typing import List, Optional
 
-from trilogy.core.models import Concept, Environment, Conditional
+from trilogy.core.models import Concept, Environment, Conditional, WhereClause
 from trilogy.core.processing.nodes import MergeNode, History, StrategyNode
 import networkx as nx
 from trilogy.core.graph_models import concept_to_node
@@ -260,6 +260,7 @@ def subgraphs_to_merge_node(
     source_concepts,
     history,
     conditions,
+    search_conditions: WhereClause | None = None,
     enable_early_exit: bool = True,
 ):
     parents: List[StrategyNode] = []
@@ -277,6 +278,7 @@ def subgraphs_to_merge_node(
             g=g,
             depth=depth + 1,
             history=history,
+            conditions=search_conditions,
         )
         if not parent:
             logger.info(
@@ -315,6 +317,7 @@ def gen_merge_node(
     accept_partial: bool = False,
     history: History | None = None,
     conditions: Conditional | None = None,
+    search_conditions: WhereClause | None = None,
 ) -> Optional[MergeNode]:
 
     for filter_downstream in [True, False]:
@@ -339,6 +342,7 @@ def gen_merge_node(
                 source_concepts=source_concepts,
                 history=history,
                 conditions=conditions,
+                search_conditions=search_conditions,
             )
     # one concept handling may need to be kicked to alias
     if len(all_concepts) == 1:
@@ -354,6 +358,7 @@ def gen_merge_node(
                 history=history,
                 conditions=conditions,
                 enable_early_exit=False,
+                search_conditions=search_conditions,
             )
             if test:
                 return test

{pytrilogy-0.0.2.10 → pytrilogy-0.0.2.11}/trilogy/core/query_processor.py

@@ -145,7 +145,9 @@ def generate_source_map(
         names = set([x.name for x in ev])
         ematches = [cte.name for cte in all_new_ctes if cte.source.name in names]
         existence_source_map[ek] = ematches
-    return {k: [] if not v else v for k, v in source_map.items()}, existence_source_map
+    return {
+        k: [] if not v else list(set(v)) for k, v in source_map.items()
+    }, existence_source_map
 
 
 def datasource_to_query_datasource(datasource: Datasource) -> QueryDatasource:
@@ -191,6 +193,8 @@ def resolve_cte_base_name_and_alias_v2(
     raw_joins: List[Join | InstantiatedUnnestJoin],
 ) -> Tuple[str | None, str | None]:
     joins: List[Join] = [join for join in raw_joins if isinstance(join, Join)]
+    # INFO     trilogy:query_processor.py:263 Finished building source map for civet with 3 parents, have {'local.relevant_customers': ['fowl', 'fowl'],
+    # 'customer.demographics.gender': ['mandrill'], 'customer.id': ['mandrill'], 'customer.demographics.id': ['mandrill'], 'customer.id_9268029262289908': [], 'customer.demographics.gender_1513806568509111': []}, query_datasource had non-empty keys ['local.relevant_customers', 'customer.demographics.gender', 'customer.id', 'customer.demographics.id'] and existence had non-empty keys []
     if (
         len(source.datasources) == 1
         and isinstance(source.datasources[0], Datasource)
@@ -212,12 +216,16 @@ def resolve_cte_base_name_and_alias_v2(
 
     counts: dict[str, int] = defaultdict(lambda: 0)
     output_addresses = [x.address for x in source.output_concepts]
+    input_address = [x.address for x in source.input_concepts]
     for k, v in source_map.items():
         for vx in v:
             if k in output_addresses:
                 counts[vx] = counts[vx] + 1
-            else:
-                counts[vx] = counts[vx]
+
+            if k in input_address:
+                counts[vx] = counts[vx] + 1
+
+            counts[vx] = counts[vx]
     if counts:
         return max(counts, key=counts.get), max(counts, key=counts.get)  # type: ignore
     return None, None

{pytrilogy-0.0.2.10 → pytrilogy-0.0.2.11}/trilogy/dialect/base.py

@@ -281,10 +281,7 @@ class BaseDialect:
                 rval = f"{self.WINDOW_FUNCTION_MAP[c.lineage.type](concept = self.render_concept_sql(c.lineage.content, cte=cte, alias=False), window=','.join(rendered_over_components), sort=','.join(rendered_order_components))}"  # noqa: E501
             elif isinstance(c.lineage, FilterItem):
                 # for cases when we've optimized this
-                if (
-                    len(cte.output_columns) == 1
-                    and cte.condition == c.lineage.where.conditional
-                ):
+                if cte.condition == c.lineage.where.conditional:
                     rval = self.render_expr(c.lineage.content, cte=cte)
                 else:
                     rval = f"CASE WHEN {self.render_expr(c.lineage.where.conditional, cte=cte)} THEN {self.render_concept_sql(c.lineage.content, cte=cte, alias=False)} ELSE NULL END"

{pytrilogy-0.0.2.10 → pytrilogy-0.0.2.11}/trilogy/parsing/parse_engine.py

@@ -983,6 +983,7 @@ class ParseToObjects(Transformer):
             order_by=order_by,
             meta=Metadata(line_number=meta.line),
         )
+
         for item in select_items:
             # we don't know the grain of an aggregate at assignment time
             # so rebuild at this point in the tree
@@ -1000,21 +1001,43 @@ class ParseToObjects(Transformer):
                 )
                 self.environment.add_concept(new_concept, meta=meta)
                 item.content.output = new_concept
+            # TODO: revisit if we can push down every filter
+            # else:
+            #     item.content = (
+            #         item.content.with_filter(
+            #             output.where_clause.conditional, environment=self.environment
+            #         )
+            #         if output.where_clause
+            #         and output.where_clause_category == SelectFiltering.IMPLICIT
+            #         else item.content
+            #     )
+
         if order_by:
             for orderitem in order_by.items:
-                if (
-                    isinstance(orderitem.expr, Concept)
-                    and orderitem.expr.purpose == Purpose.METRIC
-                ):
-                    orderitem.expr = orderitem.expr.with_select_context(
-                        output.grain,
-                        conditional=(
-                            output.where_clause.conditional
-                            if output.where_clause
-                            and output.where_clause_category == SelectFiltering.IMPLICIT
-                            else None
-                        ),
-                    )
+                if isinstance(orderitem.expr, Concept):
+                    if orderitem.expr.purpose == Purpose.METRIC:
+                        orderitem.expr = orderitem.expr.with_select_context(
+                            output.grain,
+                            conditional=(
+                                output.where_clause.conditional
+                                if output.where_clause
+                                and output.where_clause_category
+                                == SelectFiltering.IMPLICIT
+                                else None
+                            ),
+                            environment=self.environment,
+                        )
+                    # TODO :push down every filter
+                    # else:
+                    #     orderitem.expr = (
+                    #         orderitem.expr.with_filter(
+                    #             output.where_clause.conditional,
+                    #             environment=self.environment,
+                    #         )
+                    #         if output.where_clause
+                    #         and output.where_clause_category == SelectFiltering.IMPLICIT
+                    #         else orderitem.expr
+                    #     )
         return output
 
     @v_args(meta=True)
@@ -1237,7 +1260,11 @@ class ParseToObjects(Transformer):
 
     def filter_item(self, args) -> FilterItem:
         where: WhereClause
-        string_concept, where = args
+        string_concept, raw = args
+        if isinstance(raw, WhereClause):
+            where = raw
+        else:
+            where = WhereClause(conditional=raw)
         concept = self.environment.concepts[string_concept]
         return FilterItem(content=concept, where=where)
 

{pytrilogy-0.0.2.10 → pytrilogy-0.0.2.11}/trilogy/parsing/trilogy.lark

@@ -83,7 +83,11 @@
     raw_function: "bind" "sql"  IDENTIFIER  "("  function_binding_list ")" "->" data_type "as"i MULTILINE_STRING
     
     // user_id where state = Mexico
-    filter_item: "filter"i IDENTIFIER where
+    _filter_alt: IDENTIFIER "?" conditional
+    _filter_base: "filter"i IDENTIFIER where
+    filter_item: _filter_base | _filter_alt
+
+
 
     // rank/lag/lead
     WINDOW_TYPE: ("row_number"i|"rank"i|"lag"i|"lead"i | "sum"i)  /[\s]+/