pytrilogy 0.0.2.3__tar.gz → 0.0.2.5__tar.gz

{pytrilogy-0.0.2.3/pytrilogy.egg-info → pytrilogy-0.0.2.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.3
+Version: 0.0.2.5
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.2.3 → pytrilogy-0.0.2.5/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.3
+Version: 0.0.2.5
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.2.3 → pytrilogy-0.0.2.5}/tests/test_parsing.py

@@ -407,3 +407,44 @@ property id.labels map<string, int>;
     )
     assert env.concepts["labels"].datatype.key_type == DataType.STRING
     assert env.concepts["labels"].datatype.value_type == DataType.INTEGER
+
+
+def test_map_string_access():
+    env, parsed = parse_text(
+        """
+const labels <- { 'a': 1, 'b': 2, 'c': 3 };
+
+
+select
+    labels['a'] as label_a,
+;
+
+"""
+    )
+    assert env.concepts["labels"].datatype.key_type == DataType.STRING
+    assert env.concepts["labels"].datatype.value_type == DataType.INTEGER
+
+
+def test_struct_attr_access():
+
+    text = """
+const labels <- struct(a=1, b=2, c=3);
+
+
+select
+    labels,
+    labels.a as label_a
+;
+
+"""
+    env, parsed = parse_text(text)
+    assert env.concepts["labels"].datatype.fields_map["a"] == 1
+
+    assert env.concepts["labels.a"].concept_arguments[0].name == "labels"
+
+    results = Dialects.DUCK_DB.default_executor().execute_text(text)[0]
+
+    assert results.fetchall()[0] == (
+        {"a": 1, "b": 2, "c": 3},
+        1,
+    )

{pytrilogy-0.0.2.3 → pytrilogy-0.0.2.5}/tests/test_where_clause.py

@@ -3,6 +3,7 @@ from trilogy.core.models import SelectStatement, Grain, Parenthetical
 from trilogy.core.query_processor import process_query
 from trilogy.dialect.base import BaseDialect
 from trilogy.parser import parse
+from trilogy.core.processing.utility import is_scalar_condition
 
 
 def test_select_where(test_environment):
@@ -44,6 +45,8 @@ where
 
 
 def test_select_where_agg(test_environment):
+    from trilogy.hooks.query_debugger import DebuggingHook
+
     declarations = """
 property my_favorite_order_revenue <- filter revenue where order_id in (1,2,3);
 
@@ -54,10 +57,15 @@ select
 
 
     """
-    env, parsed = parse(declarations, environment=test_environment)
+    env, parsed = parse(
+        declarations,
+        environment=test_environment,
+    )
     select: SelectStatement = parsed[-1]
 
-    BaseDialect().compile_statement(process_query(test_environment, select))
+    BaseDialect().compile_statement(
+        process_query(test_environment, select, hooks=[DebuggingHook()])
+    )
 
 
 def test_select_where_joins(test_environment):
@@ -205,3 +213,48 @@ where
 
     """
     env, parsed = parse(declarations, environment=test_environment)
+
+
+def test_where_scalar(test_environment):
+    declarations = """
+select
+    category_name
+where
+    count(order_id) > 1
+;
+"""
+    env, parsed = parse(declarations, environment=test_environment)
+    select: SelectStatement = parsed[-1]
+
+    assert is_scalar_condition(select.where_clause.conditional) is False
+    _ = BaseDialect().compile_statement(process_query(test_environment, select))
+
+
+def test_case_where(test_environment):
+    from trilogy.hooks.query_debugger import DebuggingHook
+
+    declarations = """property order_id_even_name <- CASE
+    when order_id %2 = 0 then 'even'
+    else 'odd'
+    END;
+
+const test <- 1;
+    
+auto order_even_class_filter <- filter category_id where order_id_even_name = 'even' and 1= test; 
+
+select
+    category_id,
+    category_name
+where
+    category_name like '%abc%' and category_id not in order_even_class_filter
+    and category_id = test
+;"""
+    env, parsed = parse(declarations, environment=test_environment)
+    select: SelectStatement = parsed[-1]
+
+    query = BaseDialect().compile_statement(
+        process_query(test_environment, select, hooks=[DebuggingHook()])
+    )
+
+    # check to make sure our subselect is well-formed
+    assert "`category_id` not in (select" in query, query

{pytrilogy-0.0.2.3 → pytrilogy-0.0.2.5}/trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.2.3"
+__version__ = "0.0.2.5"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

{pytrilogy-0.0.2.3 → pytrilogy-0.0.2.5}/trilogy/core/enums.py

@@ -121,6 +121,7 @@ class FunctionType(Enum):
     INDEX_ACCESS = "index_access"
     MAP_ACCESS = "map_access"
     ATTR_ACCESS = "attr_access"
+    STRUCT = "struct"
 
     # TEXT AND MAYBE MORE
     SPLIT = "split"

{pytrilogy-0.0.2.3 → pytrilogy-0.0.2.5}/trilogy/core/functions.py

@@ -251,11 +251,19 @@ def MapAccess(args: list[Concept]):
     )
 
 
+def get_attr_datatype(
+    arg: Concept, lookup
+) -> DataType | ListType | StructType | MapType | NumericType:
+    if isinstance(arg.datatype, StructType):
+        return arg_to_datatype(arg.datatype.fields_map[lookup])
+    return arg.datatype
+
+
 def AttrAccess(args: list[Concept]):
     return Function(
         operator=FunctionType.ATTR_ACCESS,
         arguments=args,
-        output_datatype=args[0].field_map[args[1]].datatype,  # type: ignore
+        output_datatype=get_attr_datatype(args[0], args[1]),  # type: ignore
         output_purpose=Purpose.PROPERTY,
         valid_inputs=[
             {DataType.STRUCT},

{pytrilogy-0.0.2.3 → pytrilogy-0.0.2.5}/trilogy/core/models.py

@@ -296,7 +296,7 @@ class MapType(BaseModel):
 
 class StructType(BaseModel):
     fields: List[ALL_TYPES]
-    fields_map: Dict[str, Concept] = Field(default_factory=dict)
+    fields_map: Dict[str, Concept | int | float | str] = Field(default_factory=dict)
 
     @property
     def data_type(self):
@@ -419,6 +419,23 @@ class Concept(Mergeable, Namespaced, SelectContext, BaseModel):
     def __hash__(self):
         return hash(str(self))
 
+    @property
+    def is_aggregate(self):
+        if (
+            self.lineage
+            and isinstance(self.lineage, Function)
+            and self.lineage.operator in FunctionClass.AGGREGATE_FUNCTIONS.value
+        ):
+            return True
+        if (
+            self.lineage
+            and isinstance(self.lineage, AggregateWrapper)
+            and self.lineage.function.operator
+            in FunctionClass.AGGREGATE_FUNCTIONS.value
+        ):
+            return True
+        return False
+
     def with_merge(self, source: Concept, target: Concept, modifiers: List[Modifier]):
         if self.address == source.address:
             new = target.with_grain(self.grain.with_merge(source, target, modifiers))
@@ -2416,7 +2433,13 @@ class CTE(BaseModel):
         # if we've entirely removed the need to join to someplace to get the concept
         # drop the join as well.
         for removed_cte in removed:
-            still_required = any([removed_cte in x for x in self.source_map.values()])
+            still_required = any(
+                [
+                    removed_cte in x
+                    for x in self.source_map.values()
+                    or self.existence_source_map.values()
+                ]
+            )
             if not still_required:
                 self.joins = [
                     join
@@ -2434,6 +2457,7 @@ class CTE(BaseModel):
                     candidates = [x.name for x in self.parent_ctes]
                     self.base_name_override = candidates[0] if candidates else None
                     self.base_alias_override = candidates[0] if candidates else None
+        return True
 
     def inline_parent_datasource(self, parent: CTE, force_group: bool = False) -> bool:
         qds_being_inlined = parent.source
@@ -3298,7 +3322,7 @@ class Comparison(
             and self.operator == other.operator
         )
 
-    def inline_constant(self, constant: Concept) -> "Comparison":
+    def inline_constant(self, constant: Concept):
         assert isinstance(constant.lineage, Function)
         new_val = constant.lineage.arguments[0]
         if isinstance(self.left, ConstantInlineable):
@@ -3315,10 +3339,7 @@ class Comparison(
         else:
             new_right = self.right
 
-        if self.right == constant:
-            new_right = new_val
-
-        return Comparison(
+        return self.__class__(
             left=new_left,
             right=new_right,
             operator=self.operator,

{pytrilogy-0.0.2.3 → pytrilogy-0.0.2.5}/trilogy/core/optimizations/inline_datasource.py

@@ -43,10 +43,14 @@ class InlineDatasource(OptimizationRule):
                 continue
             root_outputs = {x.address for x in root.output_concepts}
             cte_outputs = {x.address for x in cte.output_columns}
+            inherited = {x for x, v in cte.source_map.items() if v}
             # cte_inherited_outputs = {x.address for x in parent_cte.output_columns if parent_cte.source_map.get(x.address)}
             grain_components = {x.address for x in root.grain.components}
-            if not cte_outputs.issubset(root_outputs):
-                self.log(f"Not all {parent_cte.name} outputs are found on datasource")
+            if not inherited.issubset(root_outputs):
+                cte_missing = inherited - root_outputs
+                self.log(
+                    f"Not all {parent_cte.name} require inputs are found on datasource, missing {cte_missing}"
+                )
                 continue
             if not grain_components.issubset(cte_outputs):
                 self.log("Not all datasource components in cte outputs, forcing group")

{pytrilogy-0.0.2.3 → pytrilogy-0.0.2.5}/trilogy/core/optimizations/predicate_pushdown.py

@@ -6,16 +6,9 @@ from trilogy.core.models import (
     SubselectComparison,
     Comparison,
     Parenthetical,
-    Function,
-    FilterItem,
-    MagicConstants,
-    Concept,
-    WindowItem,
-    AggregateWrapper,
-    DataType,
 )
 from trilogy.core.optimizations.base_optimization import OptimizationRule
-from trilogy.core.enums import FunctionClass
+from trilogy.core.processing.utility import is_scalar_condition
 
 
 def decompose_condition(
@@ -56,40 +49,6 @@ def is_child_of(a, comparison):
     return base
 
 
-def is_basic(
-    element: (
-        int
-        | str
-        | float
-        | list
-        | WindowItem
-        | FilterItem
-        | Concept
-        | Comparison
-        | Conditional
-        | Parenthetical
-        | Function
-        | AggregateWrapper
-        | MagicConstants
-        | DataType
-    ),
-) -> bool:
-    if isinstance(element, Parenthetical):
-        return is_basic(element.content)
-    elif isinstance(element, SubselectComparison):
-        return True
-    elif isinstance(element, Comparison):
-        return is_basic(element.left) and is_basic(element.right)
-    elif isinstance(element, Function):
-        if element.operator in FunctionClass.AGGREGATE_FUNCTIONS.value:
-            return False
-    elif isinstance(element, AggregateWrapper):
-        return is_basic(element.function)
-    elif isinstance(element, Conditional):
-        return is_basic(element.left) and is_basic(element.right)
-    return True
-
-
 class PredicatePushdown(OptimizationRule):
 
     def __init__(self, *args, **kwargs) -> None:
@@ -187,13 +146,13 @@ class PredicatePushdown(OptimizationRule):
         )
         optimized = False
         for candidate in candidates:
-            if not is_basic(candidate):
+            if not is_scalar_condition(candidate):
                 self.debug(
                     f"Skipping {candidate} as not a basic [no aggregate, etc] condition"
                 )
                 continue
             self.log(
-                f"Checking candidate {candidate}, {type(candidate)}, {is_basic(candidate)}"
+                f"Checking candidate {candidate}, {type(candidate)}, scalar: {is_scalar_condition(candidate)}"
             )
             for parent_cte in cte.parent_ctes:
                 local_pushdown = self._check_parent(

{pytrilogy-0.0.2.3 → pytrilogy-0.0.2.5}/trilogy/core/processing/concept_strategies_v3.py

@@ -489,6 +489,7 @@ def _search_concepts(
 
     found: set[str] = set()
     skip: set[str] = set()
+    virtual: set[str] = set()
     stack: List[StrategyNode] = []
     complete = ValidationResult.INCOMPLETE
 

{pytrilogy-0.0.2.3 → pytrilogy-0.0.2.5}/trilogy/core/processing/node_generators/common.py

@@ -47,7 +47,7 @@ def resolve_function_parent_concepts(concept: Concept) -> List[Concept]:
 
 def resolve_filter_parent_concepts(
     concept: Concept,
-) -> Tuple[Concept, List[Concept], List[Concept]]:
+) -> Tuple[Concept, List[Concept], List[Tuple[Concept, ...]]]:
     if not isinstance(concept.lineage, FilterItem):
         raise ValueError(
             f"Concept {concept} lineage is not filter item, is {type(concept.lineage)}"
@@ -58,7 +58,7 @@ def resolve_filter_parent_concepts(
     base_rows += concept.lineage.where.row_arguments
     # TODO: pass tuple groups through
     for ctuple in concept.lineage.where.existence_arguments:
-        base_existence += list(ctuple)
+        base_existence.append(ctuple)
     if direct_parent.grain:
         base_rows += direct_parent.grain.components_copy
     if (
@@ -71,7 +71,7 @@ def resolve_filter_parent_concepts(
         return (
             concept.lineage.content,
             unique(base_rows, "address"),
-            unique(base_existence, "address"),
+            base_existence,
         )
     return concept.lineage.content, unique(base_rows, "address"), []
 

{pytrilogy-0.0.2.3 → pytrilogy-0.0.2.5}/trilogy/core/processing/node_generators/filter_node.py

@@ -74,27 +74,31 @@ def gen_filter_node(
         return parent
 
     core_parents.append(parent)
+
     if parent_existence_concepts:
-        logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} fetching filter node existence parents {[x.address for x in parent_existence_concepts]}"
-        )
-        parent_existence = source_concepts(
-            mandatory_list=parent_existence_concepts,
-            environment=environment,
-            g=g,
-            depth=depth + 1,
-            history=history,
-        )
-        if not parent_existence:
+        for existence_tuple in parent_existence_concepts:
+            if not existence_tuple:
+                continue
             logger.info(
-                f"{padding(depth)}{LOGGER_PREFIX} filter existence node parents could not be found"
+                f"{padding(depth)}{LOGGER_PREFIX} fetching filter node existence parents {[x.address for x in existence_tuple]}"
             )
-            return None
-        core_parents.append(parent_existence)
-
+            parent_existence = source_concepts(
+                mandatory_list=list(existence_tuple),
+                environment=environment,
+                g=g,
+                depth=depth + 1,
+                history=history,
+            )
+            if not parent_existence:
+                logger.info(
+                    f"{padding(depth)}{LOGGER_PREFIX} filter existence node parents could not be found"
+                )
+                return None
+            core_parents.append(parent_existence)
+    flattened_existence = [x for y in parent_existence_concepts for x in y]
     filter_node = FilterNode(
         input_concepts=unique(
-            [immediate_parent] + parent_row_concepts + parent_existence_concepts,
+            [immediate_parent] + parent_row_concepts + flattened_existence,
             "address",
         ),
         output_concepts=[concept, immediate_parent] + parent_row_concepts,

{pytrilogy-0.0.2.3 → pytrilogy-0.0.2.5}/trilogy/core/processing/nodes/group_node.py

@@ -19,7 +19,7 @@ from trilogy.core.processing.nodes.base_node import (
     concept_list_to_grain,
 )
 from trilogy.utility import unique
-
+from trilogy.core.processing.utility import is_scalar_condition
 
 LOGGER_PREFIX = "[CONCEPT DETAIL - GROUP NODE]"
 
@@ -111,7 +111,8 @@ class GroupNode(StrategyNode):
                     f" {parent.grain}"
                 )
             source_type = SourceType.GROUP
-        return QueryDatasource(
+
+        base = QueryDatasource(
             input_concepts=self.input_concepts,
             output_concepts=self.output_concepts,
             datasources=parent_sources,
@@ -134,6 +135,31 @@ class GroupNode(StrategyNode):
             partial_concepts=self.partial_concepts,
             condition=self.conditions,
         )
+        # if there is a condition on a group node and it's not scalar
+        # inject an additional CTE
+        if self.conditions:
+            logger.info("CONDITIONS")
+            logger.info(str(self.conditions))
+            logger.info(is_scalar_condition(self.conditions))
+        if self.conditions and not is_scalar_condition(self.conditions):
+            base.condition = None
+            base.output_concepts = self.output_concepts + self.conditions.row_arguments
+            return QueryDatasource(
+                input_concepts=base.output_concepts,
+                output_concepts=self.output_concepts,
+                datasources=[base],
+                source_type=SourceType.SELECT,
+                source_map=resolve_concept_map(
+                    [base],
+                    targets=self.output_concepts,
+                    inherited_inputs=base.output_concepts,
+                ),
+                joins=[],
+                grain=grain,
+                partial_concepts=self.partial_concepts,
+                condition=self.conditions,
+            )
+        return base
 
     def copy(self) -> "GroupNode":
         return GroupNode(

{pytrilogy-0.0.2.3 → pytrilogy-0.0.2.5}/trilogy/core/processing/utility.py

@@ -8,6 +8,16 @@ from trilogy.core.models import (
     QueryDatasource,
     LooseConceptList,
     Environment,
+    Conditional,
+    SubselectComparison,
+    Comparison,
+    Parenthetical,
+    Function,
+    FilterItem,
+    MagicConstants,
+    WindowItem,
+    AggregateWrapper,
+    DataType,
 )
 
 from trilogy.core.enums import Purpose, Granularity
@@ -18,6 +28,8 @@ from collections import defaultdict
 from logging import Logger
 from pydantic import BaseModel
 
+from trilogy.core.enums import FunctionClass
+
 
 class NodeType(Enum):
     CONCEPT = 1
@@ -332,3 +344,37 @@ def get_disconnected_components(
         x for x in sub_graphs if calculate_graph_relevance(graph, x, all_concepts) > 0
     ]
     return len(sub_graphs), sub_graphs
+
+
+def is_scalar_condition(
+    element: (
+        int
+        | str
+        | float
+        | list
+        | WindowItem
+        | FilterItem
+        | Concept
+        | Comparison
+        | Conditional
+        | Parenthetical
+        | Function
+        | AggregateWrapper
+        | MagicConstants
+        | DataType
+    ),
+) -> bool:
+    if isinstance(element, Parenthetical):
+        return is_scalar_condition(element.content)
+    elif isinstance(element, SubselectComparison):
+        return True
+    elif isinstance(element, Comparison):
+        return is_scalar_condition(element.left) and is_scalar_condition(element.right)
+    elif isinstance(element, Function):
+        if element.operator in FunctionClass.AGGREGATE_FUNCTIONS.value:
+            return False
+    elif isinstance(element, AggregateWrapper):
+        return is_scalar_condition(element.function)
+    elif isinstance(element, Conditional):
+        return is_scalar_condition(element.left) and is_scalar_condition(element.right)
+    return True

{pytrilogy-0.0.2.3 → pytrilogy-0.0.2.5}/trilogy/core/query_processor.py

@@ -350,7 +350,11 @@ def get_query_datasources(
 
     search_concepts: list[Concept] = statement.output_components
     nest_where = statement.where_clause_category == SelectFiltering.IMPLICIT
-    if nest_where and statement.where_clause:
+
+    # if all are aggregates, we've pushed the filtering inside the aggregates anyway
+    all_aggregate = all([x.is_aggregate for x in search_concepts])
+
+    if nest_where and statement.where_clause and not all_aggregate:
         search_concepts = unique(
             statement.where_clause.row_arguments + search_concepts, "address"
         )
@@ -363,7 +367,8 @@ def get_query_datasources(
     )
     ds: GroupNode | SelectNode
     if nest_where and statement.where_clause:
-        ods.conditions = statement.where_clause.conditional
+        if not all_aggregate:
+            ods.conditions = statement.where_clause.conditional
         ods.output_concepts = search_concepts
         # ods.hidden_concepts = where_delta
         ods.rebuild_cache()