pytrilogy 0.0.2.9__tar.gz → 0.0.2.10__tar.gz

{pytrilogy-0.0.2.9/pytrilogy.egg-info → pytrilogy-0.0.2.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.9
+Version: 0.0.2.10
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.2.9 → pytrilogy-0.0.2.10/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.9
+Version: 0.0.2.10
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.2.9 → pytrilogy-0.0.2.10}/trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.2.9"
+__version__ = "0.0.2.10"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

{pytrilogy-0.0.2.9 → pytrilogy-0.0.2.10}/trilogy/constants.py

@@ -24,7 +24,7 @@ class Optimizations:
     predicate_pushdown: bool = True
     datasource_inlining: bool = True
     constant_inlining: bool = True
-    constant_inline_cutoff: int = 2
+    constant_inline_cutoff: int = 3
     direct_return: bool = True
 
 

{pytrilogy-0.0.2.9 → pytrilogy-0.0.2.10}/trilogy/core/models.py

@@ -70,7 +70,7 @@ from trilogy.utility import unique
 from collections import UserList, UserDict
 from functools import cached_property
 from abc import ABC
-
+from collections import defaultdict
 
 LOGGER_PREFIX = "[MODELS]"
 
@@ -2027,7 +2027,7 @@ class Datasource(Namespaced, BaseModel):
         return self.__repr__()
 
     def __hash__(self):
-        return (self.namespace + self.identifier).__hash__()
+        return self.full_name.__hash__()
 
     def with_namespace(self, namespace: str):
         new_namespace = (
@@ -2212,9 +2212,9 @@ class BaseJoin(BaseModel):
 class QueryDatasource(BaseModel):
     input_concepts: List[Concept]
     output_concepts: List[Concept]
+    datasources: List[Union[Datasource, "QueryDatasource"]]
     source_map: Dict[str, Set[Union[Datasource, "QueryDatasource", "UnnestJoin"]]]
 
-    datasources: List[Union[Datasource, "QueryDatasource"]]
     grain: Grain
     joins: List[BaseJoin | UnnestJoin]
     limit: Optional[int] = None
@@ -2266,7 +2266,7 @@ class QueryDatasource(BaseModel):
 
     @field_validator("source_map")
     @classmethod
-    def validate_source_map(cls, v, info: ValidationInfo):
+    def validate_source_map(cls, v: dict, info: ValidationInfo):
         values = info.data
         for key in ("input_concepts", "output_concepts"):
             if not values.get(key):
@@ -2344,11 +2344,23 @@ class QueryDatasource(BaseModel):
         )
 
         merged_datasources = {}
+
         for ds in [*self.datasources, *other.datasources]:
             if ds.full_name in merged_datasources:
                 merged_datasources[ds.full_name] = merged_datasources[ds.full_name] + ds
             else:
                 merged_datasources[ds.full_name] = ds
+
+        final_source_map = defaultdict(set)
+        for key in self.source_map:
+            final_source_map[key] = self.source_map[key].union(
+                other.source_map.get(key, set())
+            )
+        for key in other.source_map:
+            if key not in final_source_map:
+                final_source_map[key] = other.source_map[key]
+        for k, v in final_source_map.items():
+            final_source_map[k] = set(merged_datasources[x.full_name] for x in list(v))
         qds = QueryDatasource(
             input_concepts=unique(
                 self.input_concepts + other.input_concepts, "address"
@@ -2356,7 +2368,7 @@ class QueryDatasource(BaseModel):
             output_concepts=unique(
                 self.output_concepts + other.output_concepts, "address"
             ),
-            source_map={**self.source_map, **other.source_map},
+            source_map=final_source_map,
             datasources=list(merged_datasources.values()),
             grain=self.grain,
             joins=unique(self.joins + other.joins, "unique_id"),

{pytrilogy-0.0.2.9 → pytrilogy-0.0.2.10}/trilogy/core/processing/node_generators/filter_node.py

@@ -17,6 +17,7 @@ from trilogy.core.processing.node_generators.common import (
 from trilogy.constants import logger
 from trilogy.core.processing.utility import padding, unique
 from trilogy.core.processing.node_generators.common import concept_to_relevant_joins
+from trilogy.core.processing.utility import is_scalar_condition
 
 LOGGER_PREFIX = "[GEN_FILTER_NODE]"
 
@@ -78,16 +79,20 @@ def gen_filter_node(
         return None
 
     optimized_pushdown = False
-    if not local_optional:
+    if not is_scalar_condition(where.conditional):
+        optimized_pushdown = False
+    elif not local_optional:
         optimized_pushdown = True
     elif conditions and conditions == where:
         logger.info(
             f"{padding(depth)}{LOGGER_PREFIX} query conditions are the same as filter conditions, can optimize across all concepts"
         )
         optimized_pushdown = True
-
     if optimized_pushdown:
         if isinstance(row_parent, SelectNode):
+            logger.info(
+                f"{padding(depth)}{LOGGER_PREFIX} nesting select node in strategy node"
+            )
             parent = StrategyNode(
                 input_concepts=row_parent.output_concepts,
                 output_concepts=[concept] + row_parent.output_concepts,

{pytrilogy-0.0.2.9 → pytrilogy-0.0.2.10}/trilogy/core/processing/nodes/group_node.py

@@ -130,7 +130,7 @@ class GroupNode(StrategyNode):
                     if self.conditions
                     else self.output_concepts
                 ),
-                inherited_inputs=self.input_concepts,
+                inherited_inputs=self.input_concepts + self.existence_concepts,
             ),
             joins=[],
             grain=grain,
@@ -139,10 +139,6 @@ class GroupNode(StrategyNode):
         )
         # if there is a condition on a group node and it's not scalar
         # inject an additional CTE
-        if self.conditions:
-            logger.info("CONDITIONS")
-            logger.info(str(self.conditions))
-            logger.info(is_scalar_condition(self.conditions))
         if self.conditions and not is_scalar_condition(self.conditions):
             base.condition = None
             base.output_concepts = self.output_concepts + self.conditions.row_arguments

{pytrilogy-0.0.2.9 → pytrilogy-0.0.2.10}/trilogy/core/processing/utility.py

@@ -355,27 +355,40 @@ def is_scalar_condition(
         | MagicConstants
         | DataType
     ),
+    materialized: set[str] | None = None,
 ) -> bool:
     if isinstance(element, Parenthetical):
-        return is_scalar_condition(element.content)
+        return is_scalar_condition(element.content, materialized)
     elif isinstance(element, SubselectComparison):
         return True
     elif isinstance(element, Comparison):
-        return is_scalar_condition(element.left) and is_scalar_condition(element.right)
+        return is_scalar_condition(element.left, materialized) and is_scalar_condition(
+            element.right, materialized
+        )
     elif isinstance(element, Function):
         if element.operator in FunctionClass.AGGREGATE_FUNCTIONS.value:
             return False
+    elif isinstance(element, Concept):
+        if materialized and element.address in materialized:
+            return True
+        if element.lineage and isinstance(element.lineage, AggregateWrapper):
+            return is_scalar_condition(element.lineage, materialized)
+        return True
     elif isinstance(element, AggregateWrapper):
-        return is_scalar_condition(element.function)
+        return is_scalar_condition(element.function, materialized)
     elif isinstance(element, Conditional):
-        return is_scalar_condition(element.left) and is_scalar_condition(element.right)
+        return is_scalar_condition(element.left, materialized) and is_scalar_condition(
+            element.right, materialized
+        )
     return True
 
 
 def decompose_condition(
-    conditional: Conditional,
+    conditional: Conditional | Comparison | Parenthetical,
 ) -> list[SubselectComparison | Comparison | Conditional | Parenthetical]:
     chunks: list[SubselectComparison | Comparison | Conditional | Parenthetical] = []
+    if not isinstance(conditional, Conditional):
+        return [conditional]
     if conditional.operator == BooleanOperator.AND:
         if not (
             isinstance(

{pytrilogy-0.0.2.9 → pytrilogy-0.0.2.10}/trilogy/core/query_processor.py

@@ -261,7 +261,7 @@ def datasource_to_ctes(
 
     human_id = generate_cte_name(query_datasource.full_name, name_map)
     logger.info(
-        f"Finished building source map for {human_id} with {len(parents)} parents, have {source_map}, query_datasource had non-empty keys {[k for k, v in query_datasource.source_map.items() if v]} "
+        f"Finished building source map for {human_id} with {len(parents)} parents, have {source_map}, query_datasource had non-empty keys {[k for k, v in query_datasource.source_map.items() if v]} and existence had non-empty keys {[k for k, v in query_datasource.existence_source_map.items() if v]} "
     )
     final_joins = [
         x

{pytrilogy-0.0.2.9 → pytrilogy-0.0.2.10}/trilogy/dialect/base.py

@@ -2,7 +2,7 @@ from typing import List, Union, Optional, Dict, Any, Sequence, Callable
 
 from jinja2 import Template
 
-from trilogy.core.processing.utility import is_scalar_condition
+from trilogy.core.processing.utility import is_scalar_condition, decompose_condition
 from trilogy.constants import CONFIG, logger, MagicConstants
 from trilogy.core.internal import DEFAULT_CONCEPTS
 from trilogy.core.enums import (
@@ -538,6 +538,20 @@ class BaseDialect:
             final_joins = []
         else:
             final_joins = cte.joins or []
+        where: Conditional | Parenthetical | Comparison | None = None
+        having: Conditional | Parenthetical | Comparison | None = None
+        materialized = {x for x, v in cte.source_map.items() if v}
+        if cte.condition:
+            if is_scalar_condition(cte.condition, materialized=materialized):
+                where = cte.condition
+            else:
+                components = decompose_condition(cte.condition)
+                for x in components:
+                    if is_scalar_condition(x, materialized=materialized):
+                        where = where + x if where else x
+                    else:
+                        having = having + x if having else x
+
         return CompiledCTE(
             name=cte.name,
             statement=self.SQL_TEMPLATE.render(
@@ -561,16 +575,8 @@ class BaseDialect:
                     ]
                     if j
                 ],
-                where=(
-                    self.render_expr(cte.condition, cte)
-                    if cte.condition and is_scalar_condition(cte.condition)
-                    else None
-                ),
-                having=(
-                    self.render_expr(cte.condition, cte)
-                    if cte.condition and not is_scalar_condition(cte.condition)
-                    else None
-                ),
+                where=(self.render_expr(where, cte) if where else None),
+                having=(self.render_expr(having, cte) if having else None),
                 order_by=(
                     [self.render_order_item(i, cte) for i in cte.order_by.items]
                     if cte.order_by

{pytrilogy-0.0.2.9 → pytrilogy-0.0.2.10}/trilogy/dialect/duckdb.py

@@ -36,7 +36,7 @@ FUNCTION_MAP = {
 # we may return a static value
 FUNCTION_GRAIN_MATCH_MAP = {
     **FUNCTION_MAP,
-    FunctionType.COUNT: lambda args: "1",
+    FunctionType.COUNT: lambda args: f"{args[0]}",
     FunctionType.SUM: lambda args: f"{args[0]}",
     FunctionType.AVG: lambda args: f"{args[0]}",
 }

{pytrilogy-0.0.2.9 → pytrilogy-0.0.2.10}/trilogy/parsing/parse_engine.py

@@ -1116,7 +1116,25 @@ class ParseToObjects(Transformer):
     def comparison(self, args) -> Comparison:
         if args[1] == ComparisonOperator.IN:
             raise SyntaxError
-        return Comparison(left=args[0], right=args[2], operator=args[1])
+        if isinstance(args[0], AggregateWrapper):
+            left = arbitrary_to_concept(
+                args[0],
+                namespace=self.environment.namespace,
+                name=f"{VIRTUAL_CONCEPT_PREFIX}_{string_to_hash(str(args[0]))}",
+            )
+            self.environment.add_concept(left)
+        else:
+            left = args[0]
+        if isinstance(args[2], AggregateWrapper):
+            right = arbitrary_to_concept(
+                args[2],
+                namespace=self.environment.namespace,
+                name=f"{VIRTUAL_CONCEPT_PREFIX}_{string_to_hash(str(args[2]))}",
+            )
+            self.environment.add_concept(right)
+        else:
+            right = args[2]
+        return Comparison(left=left, right=right, operator=args[1])
 
     def between_comparison(self, args) -> Conditional:
         left_bound = args[1]