pytrilogy 0.0.3.42__tar.gz → 0.0.3.44__tar.gz

{pytrilogy-0.0.3.42/pytrilogy.egg-info → pytrilogy-0.0.3.44}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.42
+Version: 0.0.3.44
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.3.42 → pytrilogy-0.0.3.44/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.42
+Version: 0.0.3.44
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.3.42 → pytrilogy-0.0.3.44}/pytrilogy.egg-info/SOURCES.txt

@@ -23,6 +23,7 @@ tests/test_models.py
 tests/test_multi_join_assignments.py
 tests/test_parse_engine.py
 tests/test_parsing.py
+tests/test_parsing_failures.py
 tests/test_partial_handling.py
 tests/test_query_processing.py
 tests/test_query_render.py

{pytrilogy-0.0.3.42 → pytrilogy-0.0.3.44}/tests/test_models.py

@@ -213,9 +213,6 @@ def test_comparison():
     with raises(SyntaxError):
         Comparison(left=1, right="abc", operator=ComparisonOperator.EQ)
 
-    with raises(SyntaxError):
-        Comparison(left=1, right=1, operator=ComparisonOperator.IN)
-
     # this should not error
     Comparison(left=1, right=[1, 2, 3], operator=ComparisonOperator.IN)
 

{pytrilogy-0.0.3.42 → pytrilogy-0.0.3.44}/tests/test_parsing.py

@@ -1,9 +1,6 @@
-from pytest import raises
-
 from trilogy import Dialects
-from trilogy.constants import MagicConstants, Parsing
+from trilogy.constants import MagicConstants
 from trilogy.core.enums import BooleanOperator, ComparisonOperator, Purpose
-from trilogy.core.exceptions import InvalidSyntaxException
 from trilogy.core.functions import argument_to_purpose, function_args_to_output_purpose
 from trilogy.core.models.author import Comparison
 from trilogy.core.models.core import (
@@ -20,7 +17,6 @@ from trilogy.core.statements.author import SelectStatement, ShowStatement
 from trilogy.core.statements.execute import ProcessedQuery
 from trilogy.dialect.base import BaseDialect
 from trilogy.parsing.parse_engine import (
-    ParseError,
     arg_to_datatype,
     parse_text,
 )
@@ -668,99 +664,3 @@ select x % 10 -> x_mod_10;
             
 """
     )
-
-
-def test_import_shows_source():
-
-    env = Environment(
-        config=EnvironmentOptions(
-            import_resolver=DictImportResolver(
-                content={
-                    "test": """
-import test_dep as test_dep;
-key x int;
-datasource test (
-x: x)
-grain(x)
-query '''
-select 1 as x
-union all
-select 11 as x
-''' TYPO
-""",
-                    "test_dep": """
-key x int;
-""",
-                }
-            )
-        )
-    )
-    assert isinstance(env.config.import_resolver, DictImportResolver)
-
-    with raises(Exception, match="Unable to import 'test', parsing error") as e:
-        env.parse(
-            """
-        import test;
-                
-    select x % 10 -> x_mod_10;
-                
-                
-    """
-        )
-        assert "TYPO" in str(e.value)
-        assert 1 == 0
-
-
-def test_concept_shadow_warning():
-    x = """
-key scalar int;    
-property scalar.int_array list<int>;
-
-key split <- unnest(int_array);
-
-datasource avalues (
-    int_array: int_array,
-	scalar: scalar
-    ) 
-grain (scalar) 
-query '''(
-select [1,2,3,4] as int_array, 2 as scalar
-union all
-select [5,6,7,8] as int_array, 4 as scalar
-)''';
-
-SELECT
-    int_array,
-    1+2->scalar
-;
-"""
-    with raises(ParseError):
-        env, parsed = parse_text(
-            x, parse_config=Parsing(strict_name_shadow_enforcement=True)
-        )
-    x = """
-key scalar int;    
-property scalar.int_array list<int>;
-
-key split <- unnest(int_array);
-
-datasource avalues (
-    int_array: int_array,
-	scalar: scalar
-    ) 
-grain (scalar) 
-query '''(
-select [1,2,3,4] as int_array, 2 as scalar
-union all
-select [5,6,7,8] as int_array, 4 as scalar
-)''';
-
-SELECT
-    int_array,
-    sum(scalar)->scalar
-;
-"""
-    with raises(InvalidSyntaxException):
-        env, parsed = parse_text(
-            x, parse_config=Parsing(strict_name_shadow_enforcement=True)
-        )

pytrilogy-0.0.3.44/tests/test_parsing_failures.py

@@ -0,0 +1,143 @@
+from pytest import raises
+
+from trilogy.constants import Parsing
+from trilogy.core.exceptions import InvalidSyntaxException
+from trilogy.core.models.environment import (
+    DictImportResolver,
+    Environment,
+    EnvironmentOptions,
+)
+from trilogy.parsing.parse_engine import (
+    ParseError,
+    parse_text,
+)
+
+
+def test_import_shows_source():
+
+    env = Environment(
+        config=EnvironmentOptions(
+            import_resolver=DictImportResolver(
+                content={
+                    "test": """
+import test_dep as test_dep;
+key x int;
+datasource test (
+x: x)
+grain(x)
+query '''
+select 1 as x
+union all
+select 11 as x
+''' TYPO
+""",
+                    "test_dep": """
+key x int;
+""",
+                }
+            )
+        )
+    )
+    assert isinstance(env.config.import_resolver, DictImportResolver)
+
+    with raises(Exception, match="Unable to import 'test', parsing error") as e:
+        env.parse(
+            """
+        import test;
+                
+    select x % 10 -> x_mod_10;
+                
+                
+    """
+        )
+        assert "TYPO" in str(e.value)
+        assert 1 == 0
+
+
+def test_concept_shadow_warning():
+    x = """
+key scalar int;    
+property scalar.int_array list<int>;
+
+key split <- unnest(int_array);
+
+datasource avalues (
+    int_array: int_array,
+	scalar: scalar
+    ) 
+grain (scalar) 
+query '''(
+select [1,2,3,4] as int_array, 2 as scalar
+union all
+select [5,6,7,8] as int_array, 4 as scalar
+)''';
+
+SELECT
+    int_array,
+    1+2->scalar
+;
+"""
+    with raises(ParseError) as e:
+        env, parsed = parse_text(
+            x, parse_config=Parsing(strict_name_shadow_enforcement=True)
+        )
+        assert "abc" in str(e)
+    x = """
+key scalar int;    
+property scalar.int_array list<int>;
+
+key split <- unnest(int_array);
+
+datasource avalues (
+    int_array: int_array,
+	scalar: scalar
+    ) 
+grain (scalar) 
+query '''(
+select [1,2,3,4] as int_array, 2 as scalar
+union all
+select [5,6,7,8] as int_array, 4 as scalar
+)''';
+
+SELECT
+    int_array,
+    sum(scalar)->scalar
+;
+"""
+    with raises(InvalidSyntaxException):
+        env, parsed = parse_text(
+            x, parse_config=Parsing(strict_name_shadow_enforcement=True)
+        )
+
+
+def test_parsing_bad_order():
+    x = """
+key scalar int;    
+property scalar.int_array list<int>;
+
+key split <- unnest(int_array);
+
+datasource avalues (
+    int_array: int_array,
+	scalar: scalar
+    ) 
+grain (scalar) 
+query '''(
+select [1,2,3,4] as int_array, 2 as scalar
+union all
+select [5,6,7,8] as int_array, 4 as scalar
+)''';
+
+SELECT
+    int_array,
+    sum(scalar)->scalar
+order 
+    int_array asc
+;
+"""
+    with raises(InvalidSyntaxException) as e:
+
+        env, parsed = parse_text(
+            x, parse_config=Parsing(strict_name_shadow_enforcement=True)
+        )
+        assert "^" in str(e)

{pytrilogy-0.0.3.42 → pytrilogy-0.0.3.44}/trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.dialect.enums import Dialects
 from trilogy.executor import Executor
 from trilogy.parser import parse
 
-__version__ = "0.0.3.42"
+__version__ = "0.0.3.44"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

{pytrilogy-0.0.3.42 → pytrilogy-0.0.3.44}/trilogy/core/models/author.py

@@ -629,18 +629,8 @@ class Comparison(ConceptArgs, Mergeable, DataTyped, Namespaced, BaseModel):
                 )
         elif self.operator in (ComparisonOperator.IN, ComparisonOperator.NOT_IN):
             right_type = arg_to_datatype(self.right)
-            if not any(
-                [
-                    isinstance(self.right, ConceptRef),
-                    right_type in (DataType.LIST,),
-                    isinstance(right_type, (ListType, ListWrapper, TupleWrapper)),
-                ]
-            ):
-                raise SyntaxError(
-                    f"Cannot use {self.operator.value} with non-list, non-tuple, non-concept object {self.right} in {str(self)}"
-                )
 
-            elif isinstance(right_type, ListType) and not is_compatible_datatype(
+            if isinstance(right_type, ListType) and not is_compatible_datatype(
                 arg_to_datatype(self.left), right_type.value_data_type
             ):
                 raise SyntaxError(
@@ -1916,7 +1906,7 @@ class AggregateWrapper(Mergeable, DataTyped, ConceptArgs, Namespaced, BaseModel)
 
 
 class FilterItem(DataTyped, Namespaced, ConceptArgs, BaseModel):
-    content: ConceptRef
+    content: Expr
     where: "WhereClause"
 
     @field_validator("content", mode="before")
@@ -1932,13 +1922,21 @@ class FilterItem(DataTyped, Namespaced, ConceptArgs, BaseModel):
         self, source: Concept, target: Concept, modifiers: List[Modifier]
     ) -> "FilterItem":
         return FilterItem.model_construct(
-            content=self.content.with_merge(source, target, modifiers),
+            content=(
+                self.content.with_merge(source, target, modifiers)
+                if isinstance(self.content, Mergeable)
+                else self.content
+            ),
             where=self.where.with_merge(source, target, modifiers),
         )
 
     def with_namespace(self, namespace: str) -> "FilterItem":
         return FilterItem.model_construct(
-            content=self.content.with_namespace(namespace),
+            content=(
+                self.content.with_namespace(namespace)
+                if isinstance(self.content, Namespaced)
+                else self.content
+            ),
             where=self.where.with_namespace(namespace),
         )
 

{pytrilogy-0.0.3.42 → pytrilogy-0.0.3.44}/trilogy/core/models/build.py

@@ -1496,22 +1496,43 @@ class Factory:
     ):
         return base
 
+    def instantiate_concept(
+        self,
+        arg: (
+            AggregateWrapper
+            | FunctionCallWrapper
+            | WindowItem
+            | FilterItem
+            | Function
+            | ListWrapper[Any]
+            | MapWrapper[Any, Any]
+            | int
+            | float
+            | str
+        ),
+    ) -> tuple[Concept, BuildConcept]:
+        from trilogy.parsing.common import arbitrary_to_concept
+
+        new = arbitrary_to_concept(
+            arg,
+            environment=self.environment,
+        )
+        built = self.build(new)
+        self.local_concepts[new.address] = built
+        return new, built
+
     @build.register
     def _(self, base: None) -> None:
         return base
 
     @build.register
     def _(self, base: Function) -> BuildFunction | BuildAggregateWrapper:
-        from trilogy.parsing.common import arbitrary_to_concept
 
         raw_args: list[Concept | FuncArgs] = []
         for arg in base.arguments:
             # to do proper discovery, we need to inject virtual intermediate ocncepts
             if isinstance(arg, (AggregateWrapper, FilterItem, WindowItem)):
-                narg = arbitrary_to_concept(
-                    arg,
-                    environment=self.environment,
-                )
+                narg, _ = self.instantiate_concept(arg)
                 raw_args.append(narg)
             else:
                 raw_args.append(arg)
@@ -1532,24 +1553,16 @@ class Factory:
                 for x in arguments:
                     if isinstance(x, (ConceptRef, Concept)):
                         final_args.append(x)
-                    elif isinstance(x, (AggregateWrapper, FilterItem, WindowItem)):
-                        newx = arbitrary_to_concept(
-                            x,
-                            environment=self.environment,
-                        )
-                        final_args.append(newx)
                     else:
                         # constants, etc, can be ignored for group
                         continue
-                group_base = arbitrary_to_concept(
+                _, rval = self.instantiate_concept(
                     AggregateWrapper(
                         function=group_base.lineage.function,
                         by=final_args,
-                    ),
-                    environment=self.environment,
+                    )
                 )
 
-                rval = self.build(group_base)
                 return BuildFunction.model_construct(
                     operator=base.operator,
                     arguments=[rval, *[self.build(c) for c in raw_args[1:]]],
@@ -1580,20 +1593,13 @@ class Factory:
 
     @build.register
     def _(self, base: CaseWhen) -> BuildCaseWhen:
-        from trilogy.parsing.common import arbitrary_to_concept
 
         comparison = base.comparison
         if isinstance(comparison, (AggregateWrapper, FilterItem, WindowItem)):
-            comparison = arbitrary_to_concept(
-                comparison,
-                environment=self.environment,
-            )
+            comparison, _ = self.instantiate_concept(comparison)
         expr: Concept | FuncArgs = base.expr
         if isinstance(expr, (AggregateWrapper, FilterItem, WindowItem)):
-            expr = arbitrary_to_concept(
-                expr,
-                environment=self.environment,
-            )
+            expr, _ = self.instantiate_concept(expr)
         return BuildCaseWhen.model_construct(
             comparison=self.build(comparison),
             expr=self.build(expr),
@@ -1615,6 +1621,7 @@ class Factory:
         else:
             build_lineage = None
         derivation = Concept.calculate_derivation(build_lineage, base.purpose)
+
         granularity = Concept.calculate_granularity(
             derivation, final_grain, build_lineage
         )
@@ -1675,16 +1682,12 @@ class Factory:
 
     @build.register
     def _(self, base: OrderItem) -> BuildOrderItem:
-        from trilogy.parsing.common import arbitrary_to_concept
 
         bexpr: Any
         if isinstance(base.expr, (AggregateWrapper, WindowItem, FilterItem)) or (
             isinstance(base.expr, Function) and base.expr.operator == FunctionType.GROUP
         ):
-            bexpr = arbitrary_to_concept(
-                base.expr,
-                environment=self.environment,
-            )
+            bexpr, _ = self.instantiate_concept(base.expr)
         else:
             bexpr = base.expr
         return BuildOrderItem.model_construct(
@@ -1707,15 +1710,10 @@ class Factory:
 
     @build.register
     def _(self, base: WindowItem) -> BuildWindowItem:
-        # to do proper discovery, we need to inject virtual intermediate ocncepts
-        from trilogy.parsing.common import arbitrary_to_concept
 
         content: Concept | FuncArgs = base.content
         if isinstance(content, (AggregateWrapper, FilterItem, WindowItem)):
-            content = arbitrary_to_concept(
-                content,
-                environment=self.environment,
-            )
+            content, _ = self.instantiate_concept(content)
         final_by = []
         for x in base.order_by:
             if (
@@ -1743,30 +1741,26 @@ class Factory:
 
     @build.register
     def _(self, base: SubselectComparison) -> BuildSubselectComparison:
+        right: Any = base.right
+        if isinstance(base.right, (AggregateWrapper, WindowItem, FilterItem, Function)):
+            right_c, _ = self.instantiate_concept(base.right)
+            right = right_c
         return BuildSubselectComparison.model_construct(
-            left=(self.build(base.left)),
-            right=(self.build(base.right)),
+            left=self.build(base.left),
+            right=self.build(right),
             operator=base.operator,
         )
 
     @build.register
     def _(self, base: Comparison) -> BuildComparison:
-        from trilogy.parsing.common import arbitrary_to_concept
 
         left = base.left
         if isinstance(left, (AggregateWrapper, WindowItem, FilterItem)):
-            left_c = arbitrary_to_concept(
-                left,
-                environment=self.environment,
-            )
+            left_c, _ = self.instantiate_concept(left)
             left = left_c  # type: ignore
         right = base.right
         if isinstance(right, (AggregateWrapper, WindowItem, FilterItem)):
-            right_c = arbitrary_to_concept(
-                right,
-                environment=self.environment,
-            )
-
+            right_c, _ = self.instantiate_concept(right)
             right = right_c  # type: ignore
         return BuildComparison.model_construct(
             left=(self.build(left)),
@@ -1826,6 +1820,13 @@ class Factory:
 
     @build.register
     def _(self, base: FilterItem) -> BuildFilterItem:
+        if isinstance(
+            base.content, (Function, AggregateWrapper, WindowItem, FilterItem)
+        ):
+            _, built = self.instantiate_concept(base.content)
+            return BuildFilterItem.model_construct(
+                content=built, where=self.build(base.where)
+            )
         return BuildFilterItem.model_construct(
             content=self.build(base.content), where=self.build(base.where)
         )
@@ -1969,6 +1970,10 @@ class Factory:
             new.datasources[k] = self.build(d)
         for k, a in base.alias_origin_lookup.items():
             new.alias_origin_lookup[k] = self.build(a)
+        # add in anything that was built as a side-effect
+        for bk, bv in self.local_concepts.items():
+            if bk not in new.concepts:
+                new.concepts[bk] = bv
         new.gen_concept_list_caches()
         return new
 

{pytrilogy-0.0.3.42 → pytrilogy-0.0.3.44}/trilogy/core/processing/node_generators/common.py

@@ -178,7 +178,7 @@ def gen_enrichment_node(
             for x in extra_required
         ):
             log_lambda(
-                f"{str(type(base_node).__name__)} returning property optimized enrichment node"
+                f"{str(type(base_node).__name__)} returning property optimized enrichment node for {extra_required[0].keys}"
             )
             return gen_property_enrichment_node(
                 base_node,

{pytrilogy-0.0.3.42 → pytrilogy-0.0.3.44}/trilogy/core/processing/node_generators/window_node.py

@@ -32,7 +32,9 @@ def resolve_window_parent_concepts(
     if concept.lineage.order_by:
         for item in concept.lineage.order_by:
             base += item.concept_arguments
-
+    if concept.grain:
+        for gitem in concept.grain.components:
+            base.append(environment.concepts[gitem])
     return concept.lineage.content, unique(base, "address")
 
 
@@ -61,6 +63,9 @@ def gen_window_node(
     # append in keys to get the right grain
     if concept.keys:
         for item in concept.keys:
+            logger.info(
+                f"{padding(depth)}{LOGGER_PREFIX} appending search for key {item}"
+            )
             targets.append(environment.concepts[item])
     additional_outputs = []
     if equivalent_optional:
@@ -72,10 +77,15 @@ def gen_window_node(
             additional_outputs.append(x)
 
     grain_equivalents = [
-        x for x in local_optional if x.keys and all([key in targets for key in x.keys])
+        x
+        for x in local_optional
+        if x.keys
+        and all([key in targets for key in x.keys])
+        and x.grain == concept.grain
     ]
 
     for x in grain_equivalents:
+        logger.info("Appending grain equivalent %s", x)
         targets.append(x)
 
     # finally, the ones we'll need to enrich
@@ -123,20 +133,24 @@ def gen_window_node(
     )
     _window_node.rebuild_cache()
     _window_node.resolve()
+
     window_node = StrategyNode(
         input_concepts=[concept] + additional_outputs + parent_concepts + targets,
         output_concepts=[concept] + additional_outputs + parent_concepts + targets,
         environment=environment,
         parents=[_window_node],
         preexisting_conditions=conditions.conditional if conditions else None,
-        # hidden_concepts=[
-        #     x.address for x in parent_concepts if x.address not in local_optional
-        # ],
+        grain=BuildGrain.from_concepts(
+            concepts=[concept] + additional_outputs + parent_concepts + targets,
+            environment=environment,
+        ),
     )
     if not non_equivalent_optional:
         logger.info(
             f"{padding(depth)}{LOGGER_PREFIX} no optional concepts, returning window node"
         )
+        # prune outputs if we don't need join keys
+        window_node.set_output_concepts([concept] + additional_outputs + targets)
         return window_node
 
     missing_optional = [

{pytrilogy-0.0.3.42 → pytrilogy-0.0.3.44}/trilogy/dialect/bigquery.py

@@ -34,7 +34,7 @@ FUNCTION_MAP = {
 
 FUNCTION_GRAIN_MATCH_MAP = {
     **FUNCTION_MAP,
-    FunctionType.COUNT: lambda args: f"{args[0]}",
+    FunctionType.COUNT: lambda args: "1",
     FunctionType.SUM: lambda args: f"{args[0]}",
     FunctionType.AVG: lambda args: f"{args[0]}",
 }