pytrilogy 0.0.2.8__py3-none-any.whl → 0.0.2.9__py3-none-any.whl

trilogy/core/processing/nodes/base_node.py

@@ -17,6 +17,7 @@ from trilogy.core.models import (
 from trilogy.core.enums import Purpose, JoinType, PurposeLineage, Granularity
 from trilogy.utility import unique
 from dataclasses import dataclass
+from trilogy.core.enums import BooleanOperator
 
 
 def concept_list_to_grain(
@@ -71,7 +72,6 @@ def resolve_concept_map(
                 concept_map[concept.address].add(input)
             elif concept.address not in concept_map:
                 concept_map[concept.address].add(input)
-
     # second loop, include partials
     for input in inputs:
         for concept in input.output_concepts:
@@ -92,14 +92,31 @@ def resolve_concept_map(
     return concept_map
 
 
-def get_all_parent_partial(all_concepts: List[Concept], parents: List["StrategyNode"]):
-    return [
-        c
-        for c in all_concepts
-        if len([c.address in [x.address for x in p.partial_concepts] for p in parents])
-        >= 1
-        and all([c.address in [x.address for x in p.partial_concepts] for p in parents])
-    ]
+def get_all_parent_partial(
+    all_concepts: List[Concept], parents: List["StrategyNode"]
+) -> List[Concept]:
+    return unique(
+        [
+            c
+            for c in all_concepts
+            if len(
+                [
+                    p
+                    for p in parents
+                    if c.address in [x.address for x in p.partial_concepts]
+                ]
+            )
+            >= 1
+            and all(
+                [
+                    c.address in p.partial_lcl
+                    for p in parents
+                    if c.address in p.output_lcl
+                ]
+            )
+        ],
+        "address",
+    )
 
 
 class StrategyNode:
@@ -137,7 +154,7 @@ class StrategyNode:
         self.partial_concepts = partial_concepts or get_all_parent_partial(
             self.output_concepts, self.parents
         )
-        self.partial_lcl = LooseConceptList(concepts=self.partial_concepts)
+
         self.depth = depth
         self.conditions = conditions
         self.grain = grain
@@ -146,13 +163,50 @@ class StrategyNode:
         self.hidden_concepts = hidden_concepts or []
         self.existence_concepts = existence_concepts or []
         self.virtual_output_concepts = virtual_output_concepts or []
+        self.validate_parents()
+
+    def add_parents(self, parents: list["StrategyNode"]):
+        self.parents += parents
+        self.validate_parents()
+
+    def add_condition(self, condition: Conditional | Comparison | Parenthetical):
+        if self.conditions:
+            self.conditions = Conditional(
+                left=self.conditions, right=condition, operator=BooleanOperator.AND
+            )
+        else:
+            self.conditions = condition
+
+    def validate_parents(self):
+        # validate parents exist
+        # assign partial values where needed
         for parent in self.parents:
             if not parent:
                 raise SyntaxError("Unresolvable parent")
 
+        # TODO: make this accurate
+        if self.parents:
+            self.partial_concepts = get_all_parent_partial(
+                self.output_concepts, self.parents
+            )
+
+        self.partial_lcl = LooseConceptList(concepts=self.partial_concepts)
+
     def add_output_concepts(self, concepts: List[Concept]):
         for concept in concepts:
-            self.output_concepts.append(concept)
+            if concept.address not in self.output_lcl.addresses:
+                self.output_concepts.append(concept)
+        self.output_lcl = LooseConceptList(concepts=self.output_concepts)
+        self.rebuild_cache()
+
+    def add_existence_concepts(self, concepts: List[Concept]):
+        for concept in concepts:
+            if concept.address not in [x.address for x in self.output_concepts]:
+                self.existence_concepts.append(concept)
+        self.rebuild_cache()
+
+    def set_output_concepts(self, concepts: List[Concept]):
+        self.output_concepts = concepts
         self.output_lcl = LooseConceptList(concepts=self.output_concepts)
         self.rebuild_cache()
 
@@ -195,8 +249,8 @@ class StrategyNode:
         grain = self.grain if self.grain else Grain(components=self.output_concepts)
         source_map = resolve_concept_map(
             parent_sources,
-            self.output_concepts,
-            self.input_concepts + self.existence_concepts,
+            targets=self.output_concepts,
+            inherited_inputs=self.input_concepts + self.existence_concepts,
         )
         return QueryDatasource(
             input_concepts=self.input_concepts,

trilogy/core/processing/nodes/filter_node.py

@@ -36,6 +36,7 @@ class FilterNode(StrategyNode):
         partial_concepts: List[Concept] | None = None,
         force_group: bool | None = False,
         grain: Grain | None = None,
+        existence_concepts: List[Concept] | None = None,
     ):
         super().__init__(
             output_concepts=output_concepts,
@@ -49,6 +50,7 @@ class FilterNode(StrategyNode):
             partial_concepts=partial_concepts,
             force_group=force_group,
             grain=grain,
+            existence_concepts=existence_concepts,
         )
 
     def copy(self) -> "FilterNode":
@@ -64,4 +66,5 @@ class FilterNode(StrategyNode):
             partial_concepts=list(self.partial_concepts),
             force_group=self.force_group,
             grain=self.grain,
+            existence_concepts=list(self.existence_concepts),
         )

trilogy/core/processing/nodes/group_node.py

@@ -39,6 +39,7 @@ class GroupNode(StrategyNode):
         partial_concepts: Optional[List[Concept]] = None,
         force_group: bool | None = None,
         conditions: Conditional | Comparison | Parenthetical | None = None,
+        existence_concepts: List[Concept] | None = None,
     ):
         super().__init__(
             input_concepts=input_concepts,
@@ -51,6 +52,7 @@ class GroupNode(StrategyNode):
             partial_concepts=partial_concepts,
             force_group=force_group,
             conditions=conditions,
+            existence_concepts=existence_concepts,
         )
 
     def _resolve(self) -> QueryDatasource:
@@ -173,4 +175,5 @@ class GroupNode(StrategyNode):
             partial_concepts=list(self.partial_concepts),
             force_group=self.force_group,
             conditions=self.conditions,
+            existence_concepts=list(self.existence_concepts),
         )

trilogy/core/processing/nodes/merge_node.py

@@ -138,16 +138,6 @@ class MergeNode(StrategyNode):
                     continue
                 final_joins.append(join)
             self.node_joins = final_joins
-        partial_lookup: list[Concept] = []
-        non_partial: List[Concept] = []
-        for node in parents or []:
-            partial_lookup += node.partial_concepts
-            non_partial += [
-                x for x in node.output_concepts if x not in node.partial_concepts
-            ]
-
-        final_partial = [x for x in partial_lookup if x not in non_partial]
-        self.partial_concepts = final_partial
 
     def translate_node_joins(self, node_joins: List[NodeJoin]) -> List[BaseJoin]:
         joins = []
@@ -265,7 +255,6 @@ class MergeNode(StrategyNode):
             for x in final_datasets
             if all([y in self.existence_concepts for y in x.output_concepts])
         ]
-
         if len(merged.keys()) == 1:
             final: QueryDatasource | Datasource = list(merged.values())[0]
             if (
@@ -378,4 +367,5 @@ class MergeNode(StrategyNode):
             node_joins=self.node_joins,
             join_concepts=list(self.join_concepts) if self.join_concepts else None,
             force_join_type=self.force_join_type,
+            existence_concepts=list(self.existence_concepts),
         )

trilogy/core/processing/nodes/select_node_v2.py

@@ -166,6 +166,7 @@ class SelectNode(StrategyNode):
             resolution = self.resolve_from_provided_datasource()
             if resolution:
                 return resolution
+
         required = [c.address for c in self.all_concepts]
         raise NoDatasourceException(
             f"Could not find any way to resolve datasources for required concepts {required} with derivation {[x.derivation for x in self.all_concepts]}"

trilogy/core/processing/utility.py

@@ -20,7 +20,7 @@ from trilogy.core.models import (
     DataType,
 )
 
-from trilogy.core.enums import Purpose, Granularity
+from trilogy.core.enums import Purpose, Granularity, BooleanOperator
 from trilogy.core.constants import CONSTANT_DATASET
 from enum import Enum
 from trilogy.utility import unique
@@ -62,7 +62,7 @@ def create_log_lambda(prefix: str, depth: int, logger: Logger):
     pad = padding(depth)
 
     def log_lambda(msg: str):
-        logger.info(f"{pad} {prefix} {msg}")
+        logger.info(f"{pad}{prefix} {msg}")
 
     return log_lambda
 
@@ -328,14 +328,6 @@ def get_disconnected_components(
     for datasource, concepts in concept_map.items():
         graph.add_node(datasource, type=NodeType.NODE)
         for concept in concepts:
-            # TODO: determine if this is the right way to handle things
-            # if concept.derivation in (PurposeLineage.FILTER, PurposeLineage.WINDOW):
-            #     if isinstance(concept.lineage, FilterItem):
-            #         graph.add_node(concept.lineage.content.address, type=NodeType.CONCEPT)
-            #         graph.add_edge(datasource, concept.lineage.content.address)
-            #     if isinstance(concept.lineage, WindowItem):
-            #         graph.add_node(concept.lineage.content.address, type=NodeType.CONCEPT)
-            #         graph.add_edge(datasource, concept.lineage.content.address)
             graph.add_node(concept.address, type=NodeType.CONCEPT)
             graph.add_edge(datasource, concept.address)
             all_concepts.add(concept)
@@ -378,3 +370,30 @@ def is_scalar_condition(
     elif isinstance(element, Conditional):
         return is_scalar_condition(element.left) and is_scalar_condition(element.right)
     return True
+
+
+def decompose_condition(
+    conditional: Conditional,
+) -> list[SubselectComparison | Comparison | Conditional | Parenthetical]:
+    chunks: list[SubselectComparison | Comparison | Conditional | Parenthetical] = []
+    if conditional.operator == BooleanOperator.AND:
+        if not (
+            isinstance(
+                conditional.left,
+                (SubselectComparison, Comparison, Conditional, Parenthetical),
+            )
+            and isinstance(
+                conditional.right,
+                (SubselectComparison, Comparison, Conditional, Parenthetical),
+            )
+        ):
+            chunks.append(conditional)
+        else:
+            for val in [conditional.left, conditional.right]:
+                if isinstance(val, Conditional):
+                    chunks.extend(decompose_condition(val))
+                else:
+                    chunks.append(val)
+    else:
+        chunks.append(conditional)
+    return chunks

trilogy/core/query_processor.py

@@ -4,9 +4,9 @@ from trilogy.core.env_processor import generate_graph
 from trilogy.core.graph_models import ReferenceGraph
 from trilogy.core.constants import CONSTANT_DATASET
 from trilogy.core.processing.concept_strategies_v3 import source_query_concepts
-from trilogy.core.enums import SelectFiltering
+from trilogy.core.enums import SelectFiltering, BooleanOperator
 from trilogy.constants import CONFIG, DEFAULT_NAMESPACE
-from trilogy.core.processing.nodes import GroupNode, SelectNode, StrategyNode
+from trilogy.core.processing.nodes import GroupNode, SelectNode, StrategyNode, History
 from trilogy.core.models import (
     Concept,
     Environment,
@@ -24,6 +24,7 @@ from trilogy.core.models import (
     Datasource,
     BaseJoin,
     InstantiatedUnnestJoin,
+    Conditional,
 )
 
 from trilogy.utility import unique
@@ -307,7 +308,10 @@ def datasource_to_ctes(
 
 
 def append_existence_check(
-    node: StrategyNode, environment: Environment, graph: ReferenceGraph
+    node: StrategyNode,
+    environment: Environment,
+    graph: ReferenceGraph,
+    history: History | None = None,
 ):
     # we if we have a where clause doing an existence check
     # treat that as separate subquery
@@ -318,25 +322,22 @@ def append_existence_check(
             logger.info(
                 f"{LOGGER_PREFIX} fetching existance clause inputs {[str(c) for c in subselect]}"
             )
-            eds = source_query_concepts([*subselect], environment=environment, g=graph)
-
-            final_eds = eds.resolve()
-            first_parent = node.resolve()
-            first_parent.datasources.append(final_eds)
-            for x in final_eds.output_concepts:
-                if x.address not in first_parent.existence_source_map:
-                    first_parent.existence_source_map[x.address] = {final_eds}
+            eds = source_query_concepts(
+                [*subselect], environment=environment, g=graph, history=history
+            )
+            node.add_parents([eds])
+            node.add_existence_concepts([*subselect])
 
 
-def get_query_datasources(
+def get_query_node(
     environment: Environment,
     statement: SelectStatement | MultiSelectStatement,
     graph: Optional[ReferenceGraph] = None,
-    hooks: Optional[List[BaseHook]] = None,
-) -> QueryDatasource:
+    history: History | None = None,
+) -> StrategyNode:
     graph = graph or generate_graph(environment)
     logger.info(
-        f"{LOGGER_PREFIX} getting source datasource for query with output {[str(c) for c in statement.output_components]}"
+        f"{LOGGER_PREFIX} getting source datasource for query with filtering {statement.where_clause_category} and output {[str(c) for c in statement.output_components]}"
     )
     if not statement.output_components:
         raise ValueError(f"Statement has no output components {statement}")
@@ -353,22 +354,28 @@ def get_query_datasources(
         )
         nest_where = True
 
-    ods = source_query_concepts(
+    ods: StrategyNode = source_query_concepts(
         search_concepts,
         environment=environment,
         g=graph,
+        conditions=(statement.where_clause if statement.where_clause else None),
+        history=history,
     )
-    ds: GroupNode | SelectNode
+    if not ods:
+        raise ValueError(
+            f"Could not find source query concepts for {[x.address for x in search_concepts]}"
+        )
+    ds: StrategyNode
     if nest_where and statement.where_clause:
         if not all_aggregate:
             ods.conditions = statement.where_clause.conditional
-        ods.output_concepts = search_concepts
+        ods.output_concepts = statement.output_components
         # ods.hidden_concepts = where_delta
         ods.rebuild_cache()
-        append_existence_check(ods, environment, graph)
+        append_existence_check(ods, environment, graph, history)
         ds = GroupNode(
             output_concepts=statement.output_components,
-            input_concepts=search_concepts,
+            input_concepts=statement.output_components,
             parents=[ods],
             environment=ods.environment,
             g=ods.g,
@@ -390,7 +397,26 @@ def get_query_datasources(
 
     else:
         ds = ods
+    if statement.having_clause:
+        if ds.conditions:
+            ds.conditions = Conditional(
+                left=ds.conditions,
+                right=statement.having_clause.conditional,
+                operator=BooleanOperator.AND,
+            )
+        else:
+            ds.conditions = statement.having_clause.conditional
+    return ds
+
+
+def get_query_datasources(
+    environment: Environment,
+    statement: SelectStatement | MultiSelectStatement,
+    graph: Optional[ReferenceGraph] = None,
+    hooks: Optional[List[BaseHook]] = None,
+) -> QueryDatasource:
 
+    ds = get_query_node(environment, statement, graph)
     final_qds = ds.resolve()
     if hooks:
         for hook in hooks:
@@ -475,6 +501,7 @@ def process_query(
         grain=statement.grain,
         limit=statement.limit,
         where_clause=statement.where_clause,
+        having_clause=statement.having_clause,
         output_columns=statement.output_components,
         ctes=final_ctes,
         base=root_cte,

trilogy/dialect/base.py

@@ -386,14 +386,19 @@ class BaseDialect:
                         e.right.address,
                         [
                             INVALID_REFERENCE_STRING(
-                                f"Missing source reference to {e.right.name}"
+                                f"Missing source reference to {e.right.address}"
                             )
                         ],
                     )
                 else:
                     lookup = lookup_cte.existence_source_map[e.right.address]
-
-                return f"{self.render_expr(e.left, cte=cte, cte_map=cte_map)} {e.operator.value} (select {lookup[0]}.{self.QUOTE_CHARACTER}{e.right.safe_address}{self.QUOTE_CHARACTER} from {lookup[0]} where {lookup[0]}.{self.QUOTE_CHARACTER}{e.right.safe_address}{self.QUOTE_CHARACTER} is not null)"
+                if len(lookup) > 0:
+                    target = lookup[0]
+                else:
+                    target = INVALID_REFERENCE_STRING(
+                        f"Missing source CTE for {e.right.address}"
+                    )
+                return f"{self.render_expr(e.left, cte=cte, cte_map=cte_map)} {e.operator.value} (select {target}.{self.QUOTE_CHARACTER}{e.right.safe_address}{self.QUOTE_CHARACTER} from {target} where {target}.{self.QUOTE_CHARACTER}{e.right.safe_address}{self.QUOTE_CHARACTER} is not null)"
             elif isinstance(e.right, (ListWrapper, Parenthetical, list)):
                 return f"{self.render_expr(e.left, cte=cte, cte_map=cte_map)} {e.operator.value} {self.render_expr(e.right, cte=cte, cte_map=cte_map)}"
 
@@ -514,7 +519,9 @@ class BaseDialect:
                 ):
                     source = f"{render_unnest(self.UNNEST_MODE, self.QUOTE_CHARACTER, cte.join_derived_concepts[0], self.render_concept_sql, cte)}"
                 # direct - eg DUCK DB - can be directly selected inline
-                elif cte.join_derived_concepts and self.UNNEST_MODE == UnnestMode.DIRECT:
+                elif (
+                    cte.join_derived_concepts and self.UNNEST_MODE == UnnestMode.DIRECT
+                ):
                     source = None
                 else:
                     raise SyntaxError("CTE has joins but no from clause")

trilogy/parsing/parse_engine.py

@@ -108,6 +108,7 @@ from trilogy.core.models import (
     list_to_wrapper,
     dict_to_map_wrapper,
     NumericType,
+    HavingClause,
 )
 from trilogy.parsing.exceptions import ParseError
 from trilogy.utility import string_to_hash
@@ -960,6 +961,7 @@ class ParseToObjects(Transformer):
         limit = None
         order_by = None
         where = None
+        having = None
         for arg in args:
             if isinstance(arg, List):
                 select_items = arg
@@ -967,13 +969,16 @@ class ParseToObjects(Transformer):
                 limit = arg.count
             elif isinstance(arg, OrderBy):
                 order_by = arg
-            elif isinstance(arg, WhereClause):
+            elif isinstance(arg, WhereClause) and not isinstance(arg, HavingClause):
                 where = arg
+            elif isinstance(arg, HavingClause):
+                having = arg
         if not select_items:
             raise ValueError("Malformed select, missing select items")
         output = SelectStatement(
             selection=select_items,
             where_clause=where,
+            having_clause=having,
             limit=limit,
             order_by=order_by,
             meta=Metadata(line_number=meta.line),
@@ -991,6 +996,7 @@ class ParseToObjects(Transformer):
                         and output.where_clause_category == SelectFiltering.IMPLICIT
                         else None
                     ),
+                    environment=self.environment,
                 )
                 self.environment.add_concept(new_concept, meta=meta)
                 item.content.output = new_concept
@@ -1032,6 +1038,19 @@ class ParseToObjects(Transformer):
                 )
         return WhereClause(conditional=root)
 
+    def having(self, args):
+        root = args[0]
+        if not isinstance(root, (Comparison, Conditional, Parenthetical)):
+            if arg_to_datatype(root) == DataType.BOOL:
+                root = Comparison(left=root, right=True, operator=ComparisonOperator.EQ)
+            else:
+                root = Comparison(
+                    left=root,
+                    right=MagicConstants.NULL,
+                    operator=ComparisonOperator.IS_NOT,
+                )
+        return HavingClause(conditional=root)
+
     @v_args(meta=True)
     def function_binding_list(self, meta: Meta, args) -> Concept:
         return args

trilogy/parsing/trilogy.lark

@@ -61,7 +61,7 @@
     persist_statement: "persist"i IDENTIFIER "into"i IDENTIFIER "from"i select_statement grain_clause?
 
     // select statement
-    select_statement: "select"i select_list  where?  order_by?  limit? 
+    select_statement: where? "select"i select_list where?  having?  order_by?  limit? 
 
     // multiple_selects
     multi_select_statement: select_statement ("merge" select_statement)+ "align"i align_clause  where? order_by?  limit? 
@@ -134,6 +134,8 @@
     
     where: "WHERE"i conditional
     
+    having: "HAVING"i conditional
+
     !array_comparison: ( ("NOT"i "IN"i) | "IN"i)
 
     COMPARISON_OPERATOR: (/is[\s]+not/ | "is" |"=" | ">" | "<" | ">=" | "<=" | "!=")