pytrilogy 0.0.2.17__py3-none-any.whl → 0.0.2.18__py3-none-any.whl

trilogy/core/query_processor.py

@@ -4,9 +4,9 @@ from trilogy.core.env_processor import generate_graph
 from trilogy.core.graph_models import ReferenceGraph
 from trilogy.core.constants import CONSTANT_DATASET
 from trilogy.core.processing.concept_strategies_v3 import source_query_concepts
-from trilogy.core.enums import SelectFiltering, BooleanOperator
+from trilogy.core.enums import BooleanOperator
 from trilogy.constants import CONFIG, DEFAULT_NAMESPACE
-from trilogy.core.processing.nodes import GroupNode, SelectNode, StrategyNode, History
+from trilogy.core.processing.nodes import SelectNode, StrategyNode, History
 from trilogy.core.models import (
     Concept,
     Environment,
@@ -193,8 +193,7 @@ def resolve_cte_base_name_and_alias_v2(
 ) -> Tuple[str | None, str | None]:
     joins: List[Join] = [join for join in raw_joins if isinstance(join, Join)]
     if (
-        len(source.datasources) == 1
-        and isinstance(source.datasources[0], Datasource)
+        isinstance(source.datasources[0], Datasource)
         and not source.datasources[0].name == CONSTANT_DATASET
     ):
         ds = source.datasources[0]
@@ -265,9 +264,7 @@ def datasource_to_ctes(
             existence_map = source_map
 
     human_id = generate_cte_name(query_datasource.full_name, name_map)
-    logger.info(
-        f"Finished building source map for {human_id} with {len(parents)} parents, have {source_map}, query_datasource had non-empty keys {[k for k, v in query_datasource.source_map.items() if v]} and existence had non-empty keys {[k for k, v in query_datasource.existence_source_map.items() if v]} "
-    )
+
     final_joins = [
         x
         for x in [base_join_to_join(join, parents) for join in query_datasource.joins]
@@ -317,29 +314,6 @@ def datasource_to_ctes(
     return output
 
 
-def append_existence_check(
-    node: StrategyNode,
-    environment: Environment,
-    graph: ReferenceGraph,
-    history: History | None = None,
-):
-    # we if we have a where clause doing an existence check
-    # treat that as separate subquery
-    if (where := node.conditions) and where.existence_arguments:
-        for subselect in where.existence_arguments:
-            if not subselect:
-                continue
-
-            eds = source_query_concepts(
-                [*subselect], environment=environment, g=graph, history=history
-            )
-            logger.info(
-                f"{LOGGER_PREFIX} fetching existence clause inputs {[str(c) for c in subselect]}"
-            )
-            node.add_parents([eds])
-            node.add_existence_concepts([*subselect])
-
-
 def get_query_node(
     environment: Environment,
     statement: SelectStatement | MultiSelectStatement,
@@ -354,16 +328,6 @@ def get_query_node(
         raise ValueError(f"Statement has no output components {statement}")
 
     search_concepts: list[Concept] = statement.output_components
-    nest_where = statement.where_clause_category == SelectFiltering.IMPLICIT
-
-    # if all are aggregates, we've pushed the filtering inside the aggregates anyway
-    all_aggregate = all([x.is_aggregate for x in search_concepts])
-
-    if nest_where and statement.where_clause and not all_aggregate:
-        search_concepts = unique(
-            statement.where_clause.row_arguments + search_concepts, "address"
-        )
-        nest_where = True
 
     ods: StrategyNode = source_query_concepts(
         search_concepts,
@@ -376,45 +340,24 @@ def get_query_node(
         raise ValueError(
             f"Could not find source query concepts for {[x.address for x in search_concepts]}"
         )
-    ds: StrategyNode
-    if nest_where and statement.where_clause:
-        if not all_aggregate:
-            ods.conditions = statement.where_clause.conditional
-        ods.set_output_concepts(statement.output_components)
-        append_existence_check(ods, environment, graph, history)
-        ds = GroupNode(
-            output_concepts=statement.output_components,
-            input_concepts=statement.output_components,
-            parents=[ods],
-            environment=ods.environment,
-            g=ods.g,
-            partial_concepts=ods.partial_concepts,
-        )
-        # we can still check existence here.
-
-    elif statement.where_clause:
-        ds = SelectNode(
-            output_concepts=statement.output_components,
-            input_concepts=ods.input_concepts,
-            parents=[ods],
-            environment=ods.environment,
-            g=ods.g,
-            partial_concepts=ods.partial_concepts,
-            conditions=statement.where_clause.conditional,
-        )
-        append_existence_check(ds, environment, graph)
-
-    else:
-        ds = ods
+    ds: StrategyNode = ods
     if statement.having_clause:
+        final = statement.having_clause.conditional
         if ds.conditions:
-            ds.conditions = Conditional(
+            final = Conditional(
                 left=ds.conditions,
                 right=statement.having_clause.conditional,
                 operator=BooleanOperator.AND,
             )
-        else:
-            ds.conditions = statement.having_clause.conditional
+        ds = SelectNode(
+            output_concepts=statement.output_components,
+            input_concepts=ds.output_concepts,
+            parents=[ds],
+            environment=ds.environment,
+            g=ds.g,
+            partial_concepts=ds.partial_concepts,
+            conditions=final,
+        )
     return ds
 
 
@@ -477,6 +420,7 @@ def process_query(
     hooks: List[BaseHook] | None = None,
 ) -> ProcessedQuery:
     hooks = hooks or []
+    statement.refresh_bindings(environment)
     graph = generate_graph(environment)
     root_datasource = get_query_datasources(
         environment=environment, graph=graph, statement=statement, hooks=hooks

trilogy/dialect/base.py

@@ -121,6 +121,7 @@ FUNCTION_MAP = {
     FunctionType.CASE: lambda x: render_case(x),
     FunctionType.SPLIT: lambda x: f"split({x[0]}, {x[1]})",
     FunctionType.IS_NULL: lambda x: f"isnull({x[0]})",
+    FunctionType.BOOL: lambda x: f"CASE WHEN {x[0]} THEN TRUE ELSE FALSE END",
     # complex
     FunctionType.INDEX_ACCESS: lambda x: f"{x[0]}[{x[1]}]",
     FunctionType.MAP_ACCESS: lambda x: f"{x[0]}[{x[1]}][1]",
@@ -176,8 +177,8 @@ FUNCTION_MAP = {
 
 FUNCTION_GRAIN_MATCH_MAP = {
     **FUNCTION_MAP,
-    FunctionType.COUNT_DISTINCT: lambda args: f"{args[0]}",
-    FunctionType.COUNT: lambda args: f"{args[0]}",
+    FunctionType.COUNT_DISTINCT: lambda args: f"CASE WHEN{args[0]} IS NOT NULL THEN 1 ELSE 0 END",
+    FunctionType.COUNT: lambda args: f"CASE WHEN {args[0]} IS NOT NULL THEN 1 ELSE 0 END",
     FunctionType.SUM: lambda args: f"{args[0]}",
     FunctionType.AVG: lambda args: f"{args[0]}",
     FunctionType.MAX: lambda args: f"{args[0]}",
@@ -582,8 +583,11 @@ class BaseDialect:
         having: Conditional | Parenthetical | Comparison | None = None
         materialized = {x for x, v in cte.source_map.items() if v}
         if cte.condition:
-            if is_scalar_condition(cte.condition, materialized=materialized):
+            if not cte.group_to_grain or is_scalar_condition(
+                cte.condition, materialized=materialized
+            ):
                 where = cte.condition
+
             else:
                 components = decompose_condition(cte.condition)
                 for x in components:

trilogy/dialect/duckdb.py

@@ -36,9 +36,12 @@ FUNCTION_MAP = {
 # we may return a static value
 FUNCTION_GRAIN_MATCH_MAP = {
     **FUNCTION_MAP,
-    FunctionType.COUNT: lambda args: f"{args[0]}",
+    FunctionType.COUNT_DISTINCT: lambda args: f"CASE WHEN{args[0]} IS NOT NULL THEN 1 ELSE 0 END",
+    FunctionType.COUNT: lambda args: f"CASE WHEN {args[0]} IS NOT NULL THEN 1 ELSE 0 END",
     FunctionType.SUM: lambda args: f"{args[0]}",
     FunctionType.AVG: lambda args: f"{args[0]}",
+    FunctionType.MAX: lambda args: f"{args[0]}",
+    FunctionType.MIN: lambda args: f"{args[0]}",
 }
 
 DUCKDB_TEMPLATE = Template(

trilogy/dialect/sql_server.py

@@ -41,9 +41,9 @@ TSQL_TEMPLATE = Template(
     """{%- if ctes %}
 WITH {% for cte in ctes %}
 {{cte.name}} as ({{cte.statement}}){% if not loop.last %},{% endif %}{% endfor %}{% endif %}
-{%- if full_select -%}
-{{full_select}}
-{%- else -%}
+{%- if full_select -%}{{full_select}}
+{%- else -%}{%- if comment %}
+-- {{ comment }}{% endif %}
 SELECT
 {%- if limit is not none %}
 TOP {{ limit }}{% endif %}

trilogy/hooks/query_debugger.py

@@ -31,6 +31,9 @@ def print_recursive_resolved(
             extra.append("filter")
     if input.group_required:
         extra.append("group")
+    output = [c.address for c in input.output_concepts[:3]]
+    if len(input.output_concepts) > 3:
+        output.append("...")
     display = [
         (
             "  " * depth,
@@ -40,7 +43,7 @@ def print_recursive_resolved(
             ">",
             # [c.address for c in input.input_concepts],
             "->",
-            [c.address for c in input.output_concepts],
+            output,
         )
     ]
     if isinstance(input, QueryDatasource):
@@ -86,7 +89,6 @@ def print_recursive_ctes(input: CTE, depth: int = 0, max_depth: int | None = Non
     sql = renderer.render_cte(input).statement
     for line in sql.split("\n"):
         logger.debug("  " * (depth) + line)
-    print("-----")
     if isinstance(input, CTE):
         for child in input.parent_ctes:
             print_recursive_ctes(child, depth + 1)
@@ -130,5 +132,5 @@ class DebuggingHook(BaseHook):
         if self.process_nodes != PrintMode.OFF:
             printed = print_recursive_nodes(node, mode=self.process_nodes)
             for row in printed:
-                logger.info("".join([str(v) for v in row]))
+                # logger.info("".join([str(v) for v in row]))
                 print("".join([str(v) for v in row]))

trilogy/parsing/parse_engine.py

@@ -30,7 +30,7 @@ from trilogy.core.enums import (
     WindowType,
     DatePart,
     ShowCategory,
-    SelectFiltering,
+    FunctionClass,
 )
 from trilogy.core.exceptions import InvalidSyntaxException, UndefinedConceptException
 from trilogy.core.functions import (
@@ -50,6 +50,7 @@ from trilogy.core.functions import (
     CurrentDate,
     CurrentDatetime,
     IsNull,
+    Bool,
     SubString,
     StrPos,
 )
@@ -840,6 +841,7 @@ class ParseToObjects(Transformer):
             grain: Grain | None = args[3]
         else:
             grain = None
+
         new_datasource = select.to_datasource(
             namespace=(
                 self.environment.namespace
@@ -930,7 +932,8 @@ class ParseToObjects(Transformer):
             order_by=order_by,
             meta=Metadata(line_number=meta.line),
         )
-
+        locally_derived: set[str] = set()
+        all_in_output: set[str] = set()
         for item in select_items:
             # we don't know the grain of an aggregate at assignment time
             # so rebuild at this point in the tree
@@ -938,59 +941,79 @@ class ParseToObjects(Transformer):
             if isinstance(item.content, ConceptTransform):
                 new_concept = item.content.output.with_select_context(
                     output.grain,
-                    conditional=(
-                        output.where_clause.conditional
-                        if output.where_clause
-                        and output.where_clause_category == SelectFiltering.IMPLICIT
-                        else None
-                    ),
+                    conditional=None,
+                    # conditional=(
+                    #     output.where_clause.conditional
+                    #     if output.where_clause
+                    #     and output.where_clause_category == SelectFiltering.IMPLICIT
+                    #     else None
+                    # ),
                     environment=self.environment,
                 )
                 self.environment.add_concept(new_concept, meta=meta)
                 item.content.output = new_concept
+                locally_derived.add(new_concept.address)
+                all_in_output.add(new_concept.address)
             elif isinstance(item.content, Concept):
                 # Sometimes cached values here don't have the latest info
-                # bug we can't just use environment, as it might not have the right grain.
+                # but we can't just use environment, as it might not have the right grain.
                 item.content = self.environment.concepts[
                     item.content.address
                 ].with_grain(item.content.grain)
-            # TODO: revisit if we can push down every filter
-            # else:
-            #     item.content = (
-            #         item.content.with_filter(
-            #             output.where_clause.conditional, environment=self.environment
-            #         )
-            #         if output.where_clause
-            #         and output.where_clause_category == SelectFiltering.IMPLICIT
-            #         else item.content
-            #     )
-
+                all_in_output.add(item.content.address)
         if order_by:
             for orderitem in order_by.items:
                 if isinstance(orderitem.expr, Concept):
                     if orderitem.expr.purpose == Purpose.METRIC:
                         orderitem.expr = orderitem.expr.with_select_context(
                             output.grain,
-                            conditional=(
-                                output.where_clause.conditional
-                                if output.where_clause
-                                and output.where_clause_category
-                                == SelectFiltering.IMPLICIT
-                                else None
-                            ),
+                            conditional=None,
+                            # conditional=(
+                            #     output.where_clause.conditional
+                            #     if output.where_clause
+                            #     and output.where_clause_category
+                            #     == SelectFiltering.IMPLICIT
+                            #     else None
+                            # ),
                             environment=self.environment,
                         )
-                    # TODO :push down every filter
-                    # else:
-                    #     orderitem.expr = (
-                    #         orderitem.expr.with_filter(
-                    #             output.where_clause.conditional,
-                    #             environment=self.environment,
-                    #         )
-                    #         if output.where_clause
-                    #         and output.where_clause_category == SelectFiltering.IMPLICIT
-                    #         else orderitem.expr
-                    #     )
+        if output.where_clause:
+            for concept in output.where_clause.concept_arguments:
+
+                if (
+                    concept.lineage
+                    and isinstance(concept.lineage, Function)
+                    and concept.lineage.operator
+                    in FunctionClass.AGGREGATE_FUNCTIONS.value
+                ):
+                    if concept.address in locally_derived:
+                        raise SyntaxError(
+                            f"Cannot reference an aggregate derived in the select ({concept.address}) in the same statement where clause; move to the HAVING clause instead; Line: {meta.line}"
+                        )
+
+                if (
+                    concept.lineage
+                    and isinstance(concept.lineage, AggregateWrapper)
+                    and concept.lineage.function.operator
+                    in FunctionClass.AGGREGATE_FUNCTIONS.value
+                ):
+                    if concept.address in locally_derived:
+                        raise SyntaxError(
+                            f"Cannot reference an aggregate derived in the select ({concept.address}) in the same statement where clause; move to the HAVING clause instead; Line: {meta.line}"
+                        )
+        if output.having_clause:
+            for concept in output.having_clause.concept_arguments:
+                if concept.address not in all_in_output:
+                    raise SyntaxError(
+                        f"Cannot reference a column ({concept.address}) that is not in the select projection in the HAVING clause, move to WHERE;  Line: {meta.line}"
+                    )
+        if output.order_by:
+            for concept in output.order_by.concept_arguments:
+                if concept.address not in all_in_output:
+                    raise SyntaxError(
+                        f"Cannot order by a column that is not in the output projection; {meta.line}"
+                    )
+
         return output
 
     @v_args(meta=True)
@@ -1832,6 +1855,11 @@ class ParseToObjects(Transformer):
         args = process_function_args(args, meta=meta, environment=self.environment)
         return IsNull(args)
 
+    @v_args(meta=True)
+    def fbool(self, meta, args):
+        args = process_function_args(args, meta=meta, environment=self.environment)
+        return Bool(args)
+
 
 def unpack_visit_error(e: VisitError):
     """This is required to get exceptions from imports, which would

trilogy/parsing/trilogy.lark

@@ -187,8 +187,9 @@
     fcase: "CASE"i (fcase_when)* (fcase_else)? "END"i
     len: "len"i "(" expr ")"
     fnot: "NOT"i expr
+    fbool: "bool"i "(" expr ")"
 
-    _generic_functions: fcast | concat | fcoalesce | fcase | len | fnot
+    _generic_functions: fcast | concat | fcoalesce | fcase | len | fnot | fbool
 
     //constant
     CURRENT_DATE.1: /current_date\(\)/