pytrilogy 0.0.3.42__py3-none-any.whl → 0.0.3.44__py3-none-any.whl

{pytrilogy-0.0.3.42.dist-info → pytrilogy-0.0.3.44.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.42
+Version: 0.0.3.44
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.3.42.dist-info → pytrilogy-0.0.3.44.dist-info}/RECORD

@@ -1,5 +1,5 @@
-pytrilogy-0.0.3.42.dist-info/licenses/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
-trilogy/__init__.py,sha256=6Oo1hj7rRiJ3WTlh1NH1PAfbHWNk1Zgr5f5DpIefPy8,303
+pytrilogy-0.0.3.44.dist-info/licenses/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
+trilogy/__init__.py,sha256=_DLr4-giprU_EZXs7LGyoeoIt8LaSs4wYqbdAajaVr4,303
 trilogy/compiler.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/constants.py,sha256=5eQxk1A0pv-TQk3CCvgZCFA9_K-6nxrOm7E5Lxd7KIY,1652
 trilogy/engine.py,sha256=OK2RuqCIUId6yZ5hfF8J1nxGP0AJqHRZiafcowmW0xc,1728
@@ -22,8 +22,8 @@ trilogy/core/internal.py,sha256=iicDBlC6nM8d7e7jqzf_ZOmpUsW8yrr2AA8AqEiLx-s,1577
 trilogy/core/optimization.py,sha256=aihzx4-2-mSjx5td1TDTYGvc7e9Zvy-_xEyhPqLS-Ig,8314
 trilogy/core/query_processor.py,sha256=Vl-u0F0rbqI2liv82yJgiZCB255Kx_KiuzZVHL6aeTM,19459
 trilogy/core/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/core/models/author.py,sha256=hS1caD8y7XWRBlHfwgZOrBcz3TisDPba8joFaiEXxX0,77072
-trilogy/core/models/build.py,sha256=NdDRcgvNoa2CHoXcPvWAUkCA2TRUKYm0zIl5NBx0sfo,61796
+trilogy/core/models/author.py,sha256=7lPUVm1uCXYsyV85p34AvtLLzHqDGCeesTUT8ZHCwo4,76859
+trilogy/core/models/build.py,sha256=EsI7BLmFXdxj1an3NnKR_Qm79tcjlFKjmLjmt3_v2eA,61829
 trilogy/core/models/build_environment.py,sha256=s_C9xAHuD3yZ26T15pWVBvoqvlp2LdZ8yjsv2_HdXLk,5363
 trilogy/core/models/core.py,sha256=wx6hJcFECMG-Ij972ADNkr-3nFXkYESr82ObPiC46_U,10875
 trilogy/core/models/datasource.py,sha256=6RjJUd2u4nYmEwFBpJlM9LbHVYDv8iHJxqiBMZqUrwI,9422
@@ -40,7 +40,7 @@ trilogy/core/processing/graph_utils.py,sha256=8QUVrkE9j-9C1AyrCb1nQEh8daCe0u1HuX
 trilogy/core/processing/utility.py,sha256=rfzdgl-vWkCyhLzXNNuWgPLK59eiYypQb6TdZKymUqk,21469
 trilogy/core/processing/node_generators/__init__.py,sha256=o8rOFHPSo-s_59hREwXMW6gjUJCsiXumdbJNozHUf-Y,800
 trilogy/core/processing/node_generators/basic_node.py,sha256=UVsXMn6jTjm_ofVFt218jAS11s4RV4zD781vP4im-GI,3371
-trilogy/core/processing/node_generators/common.py,sha256=ZsDzThjm_mAtdQpKAg8QIJiPVZ4KuUkKyilj4eOhSDs,9439
+trilogy/core/processing/node_generators/common.py,sha256=nVeH_AdO58ygtNSO0wNgMR7_h2D0dFSGM_rh1fJd4Yc,9468
 trilogy/core/processing/node_generators/filter_node.py,sha256=lT167yBgy3P9sDBM1Cjj0PKSXro8dvGtBmc8nwsUjig,8366
 trilogy/core/processing/node_generators/group_node.py,sha256=kO-ersxIL04rZwX5-vFIFQQnp357PFo_7ZKXoGq3wyc,5989
 trilogy/core/processing/node_generators/group_to_node.py,sha256=jKcNCDOY6fNblrdZwaRU0sbUSr9H0moQbAxrGgX6iGA,3832
@@ -52,7 +52,7 @@ trilogy/core/processing/node_generators/select_node.py,sha256=Y-zO0AFkTrpi2Lyebj
 trilogy/core/processing/node_generators/synonym_node.py,sha256=9LHK2XHDjbyTLjmDQieskG8fqbiSpRnFOkfrutDnOTE,2258
 trilogy/core/processing/node_generators/union_node.py,sha256=VNo6Oey4p8etU9xrOh2oTT2lIOTvY6PULUPRvVa2uxU,2877
 trilogy/core/processing/node_generators/unnest_node.py,sha256=cOEKnMRzXUW3bwmiOlgn3E1-B38osng0dh2pDykwITY,2410
-trilogy/core/processing/node_generators/window_node.py,sha256=MjLmFKUiS-_p-Ak_9mr3becGde9eu5frxmqI7plIETY,5808
+trilogy/core/processing/node_generators/window_node.py,sha256=RUHgpYovQObFod1xRIMWtDzMcxwlm4-1Fdrf_Cuw5W4,6346
 trilogy/core/processing/node_generators/select_helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/processing/node_generators/select_helpers/datasource_injection.py,sha256=GMW07bb6hXurhF0hZLYoMAKSIS65tat5hwBjvqqPeSA,6516
 trilogy/core/processing/nodes/__init__.py,sha256=Lxr3rs_bqOAtMtn3DHIkY058ZzjyLM5mSfGMKW2z0NY,5555
@@ -71,7 +71,7 @@ trilogy/core/statements/common.py,sha256=KxEmz2ySySyZ6CTPzn0fJl5NX2KOk1RPyuUSwWh
 trilogy/core/statements/execute.py,sha256=cSlvpHFOqpiZ89pPZ5GDp9Hu6j6uj-5_h21FWm_L-KM,1248
 trilogy/dialect/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/dialect/base.py,sha256=vp6_9fUkblAWVpCXGBIcoAx6N7vof9M7s9t6-b_waUY,41409
-trilogy/dialect/bigquery.py,sha256=j5PQvwMUMcLHaxZgbqe6P-v-pwhHDQ38z8uK6ecxzR0,3359
+trilogy/dialect/bigquery.py,sha256=7LcgPLDkeNBk6YTfaE-RBBi7SjWFV-jjuvZM1VMIXqk,3350
 trilogy/dialect/common.py,sha256=XjHkP8Dqezjkd2JU5xoAlMRS_6HNyXQCF4CykLK3C8o,5011
 trilogy/dialect/config.py,sha256=olnyeVU5W5T6b9-dMeNAnvxuPlyc2uefb7FRME094Ec,3834
 trilogy/dialect/dataframe.py,sha256=RUbNgReEa9g3pL6H7fP9lPTrAij5pkqedpZ99D8_5AE,1522
@@ -87,13 +87,13 @@ trilogy/hooks/graph_hook.py,sha256=c-vC-IXoJ_jDmKQjxQyIxyXPOuUcLIURB573gCsAfzQ,2
 trilogy/hooks/query_debugger.py,sha256=1npRjww94sPV5RRBBlLqMJRaFkH9vhEY6o828MeoEcw,5583
 trilogy/metadata/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/parsing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/parsing/common.py,sha256=2KwR86ZNH04tcs500l02jLPkFaStEYc9wtR1NVO6cYo,26411
+trilogy/parsing/common.py,sha256=U9RNi1GyPTQaitZGwXy1QftdC5PWYArP7V8t-v3H8Po,27157
 trilogy/parsing/config.py,sha256=Z-DaefdKhPDmSXLgg5V4pebhSB0h590vI0_VtHnlukI,111
 trilogy/parsing/exceptions.py,sha256=92E5i2frv5hj9wxObJZsZqj5T6bglvPzvdvco_vW1Zk,38
 trilogy/parsing/helpers.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-trilogy/parsing/parse_engine.py,sha256=GTraw3-Dbdo36yhprizifZY-wbm_ub8ktI1nMEziee0,64723
+trilogy/parsing/parse_engine.py,sha256=9SO2q8m5MlZo_Eho-_r6hmTSm5VH38k47C2iHTtYwjU,68224
 trilogy/parsing/render.py,sha256=hI4y-xjXrEXvHslY2l2TQ8ic0zAOpN41ADH37J2_FZY,19047
-trilogy/parsing/trilogy.lark,sha256=2Noe58vGYteilKd6w-np3fb4lzWI-G9Gt0AMyOMVw3k,13735
+trilogy/parsing/trilogy.lark,sha256=zbDAIG7gpsImxBtteD8E2pKwcJCGpM-rEQDRqpgzoSQ,13717
 trilogy/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/scripts/trilogy.py,sha256=1L0XrH4mVHRt1C9T1HnaDv2_kYEfbWTb5_-cBBke79w,3774
 trilogy/std/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -102,8 +102,8 @@ trilogy/std/display.preql,sha256=2BbhvqR4rcltyAbOXAUo7SZ_yGFYZgFnurglHMbjW2g,40
 trilogy/std/geography.preql,sha256=-fqAGnBL6tR-UtT8DbSek3iMFg66ECR_B_41pODxv-k,504
 trilogy/std/money.preql,sha256=ZHW-csTX-kYbOLmKSO-TcGGgQ-_DMrUXy0BjfuJSFxM,80
 trilogy/std/report.preql,sha256=LbV-XlHdfw0jgnQ8pV7acG95xrd1-p65fVpiIc-S7W4,202
-pytrilogy-0.0.3.42.dist-info/METADATA,sha256=L_128KNMAQLKLe33FsvUWvD5NRHCKqzkvmHT1Xx6o4c,9100
-pytrilogy-0.0.3.42.dist-info/WHEEL,sha256=ck4Vq1_RXyvS4Jt6SI0Vz6fyVs4GWg7AINwpsaGEgPE,91
-pytrilogy-0.0.3.42.dist-info/entry_points.txt,sha256=ewBPU2vLnVexZVnB-NrVj-p3E-4vukg83Zk8A55Wp2w,56
-pytrilogy-0.0.3.42.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
-pytrilogy-0.0.3.42.dist-info/RECORD,,
+pytrilogy-0.0.3.44.dist-info/METADATA,sha256=-TUyQi3fENhsnYhqsy2HiD-thLyAz6S8TAmgdVuaCZo,9100
+pytrilogy-0.0.3.44.dist-info/WHEEL,sha256=GHB6lJx2juba1wDgXDNlMTyM13ckjBMKf-OnwgKOCtA,91
+pytrilogy-0.0.3.44.dist-info/entry_points.txt,sha256=ewBPU2vLnVexZVnB-NrVj-p3E-4vukg83Zk8A55Wp2w,56
+pytrilogy-0.0.3.44.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
+pytrilogy-0.0.3.44.dist-info/RECORD,,

{pytrilogy-0.0.3.42.dist-info → pytrilogy-0.0.3.44.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.0.0)
+Generator: setuptools (80.3.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.dialect.enums import Dialects
 from trilogy.executor import Executor
 from trilogy.parser import parse
 
-__version__ = "0.0.3.42"
+__version__ = "0.0.3.44"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

trilogy/core/models/author.py

@@ -629,18 +629,8 @@ class Comparison(ConceptArgs, Mergeable, DataTyped, Namespaced, BaseModel):
                 )
         elif self.operator in (ComparisonOperator.IN, ComparisonOperator.NOT_IN):
             right_type = arg_to_datatype(self.right)
-            if not any(
-                [
-                    isinstance(self.right, ConceptRef),
-                    right_type in (DataType.LIST,),
-                    isinstance(right_type, (ListType, ListWrapper, TupleWrapper)),
-                ]
-            ):
-                raise SyntaxError(
-                    f"Cannot use {self.operator.value} with non-list, non-tuple, non-concept object {self.right} in {str(self)}"
-                )
 
-            elif isinstance(right_type, ListType) and not is_compatible_datatype(
+            if isinstance(right_type, ListType) and not is_compatible_datatype(
                 arg_to_datatype(self.left), right_type.value_data_type
             ):
                 raise SyntaxError(
@@ -1916,7 +1906,7 @@ class AggregateWrapper(Mergeable, DataTyped, ConceptArgs, Namespaced, BaseModel)
 
 
 class FilterItem(DataTyped, Namespaced, ConceptArgs, BaseModel):
-    content: ConceptRef
+    content: Expr
     where: "WhereClause"
 
     @field_validator("content", mode="before")
@@ -1932,13 +1922,21 @@ class FilterItem(DataTyped, Namespaced, ConceptArgs, BaseModel):
         self, source: Concept, target: Concept, modifiers: List[Modifier]
     ) -> "FilterItem":
         return FilterItem.model_construct(
-            content=self.content.with_merge(source, target, modifiers),
+            content=(
+                self.content.with_merge(source, target, modifiers)
+                if isinstance(self.content, Mergeable)
+                else self.content
+            ),
             where=self.where.with_merge(source, target, modifiers),
         )
 
     def with_namespace(self, namespace: str) -> "FilterItem":
         return FilterItem.model_construct(
-            content=self.content.with_namespace(namespace),
+            content=(
+                self.content.with_namespace(namespace)
+                if isinstance(self.content, Namespaced)
+                else self.content
+            ),
             where=self.where.with_namespace(namespace),
         )
 

trilogy/core/models/build.py

@@ -1496,22 +1496,43 @@ class Factory:
     ):
         return base
 
+    def instantiate_concept(
+        self,
+        arg: (
+            AggregateWrapper
+            | FunctionCallWrapper
+            | WindowItem
+            | FilterItem
+            | Function
+            | ListWrapper[Any]
+            | MapWrapper[Any, Any]
+            | int
+            | float
+            | str
+        ),
+    ) -> tuple[Concept, BuildConcept]:
+        from trilogy.parsing.common import arbitrary_to_concept
+
+        new = arbitrary_to_concept(
+            arg,
+            environment=self.environment,
+        )
+        built = self.build(new)
+        self.local_concepts[new.address] = built
+        return new, built
+
     @build.register
     def _(self, base: None) -> None:
         return base
 
     @build.register
     def _(self, base: Function) -> BuildFunction | BuildAggregateWrapper:
-        from trilogy.parsing.common import arbitrary_to_concept
 
         raw_args: list[Concept | FuncArgs] = []
         for arg in base.arguments:
             # to do proper discovery, we need to inject virtual intermediate ocncepts
             if isinstance(arg, (AggregateWrapper, FilterItem, WindowItem)):
-                narg = arbitrary_to_concept(
-                    arg,
-                    environment=self.environment,
-                )
+                narg, _ = self.instantiate_concept(arg)
                 raw_args.append(narg)
             else:
                 raw_args.append(arg)
@@ -1532,24 +1553,16 @@ class Factory:
                 for x in arguments:
                     if isinstance(x, (ConceptRef, Concept)):
                         final_args.append(x)
-                    elif isinstance(x, (AggregateWrapper, FilterItem, WindowItem)):
-                        newx = arbitrary_to_concept(
-                            x,
-                            environment=self.environment,
-                        )
-                        final_args.append(newx)
                     else:
                         # constants, etc, can be ignored for group
                         continue
-                group_base = arbitrary_to_concept(
+                _, rval = self.instantiate_concept(
                     AggregateWrapper(
                         function=group_base.lineage.function,
                         by=final_args,
-                    ),
-                    environment=self.environment,
+                    )
                 )
 
-                rval = self.build(group_base)
                 return BuildFunction.model_construct(
                     operator=base.operator,
                     arguments=[rval, *[self.build(c) for c in raw_args[1:]]],
@@ -1580,20 +1593,13 @@ class Factory:
 
     @build.register
     def _(self, base: CaseWhen) -> BuildCaseWhen:
-        from trilogy.parsing.common import arbitrary_to_concept
 
         comparison = base.comparison
         if isinstance(comparison, (AggregateWrapper, FilterItem, WindowItem)):
-            comparison = arbitrary_to_concept(
-                comparison,
-                environment=self.environment,
-            )
+            comparison, _ = self.instantiate_concept(comparison)
         expr: Concept | FuncArgs = base.expr
         if isinstance(expr, (AggregateWrapper, FilterItem, WindowItem)):
-            expr = arbitrary_to_concept(
-                expr,
-                environment=self.environment,
-            )
+            expr, _ = self.instantiate_concept(expr)
         return BuildCaseWhen.model_construct(
             comparison=self.build(comparison),
             expr=self.build(expr),
@@ -1615,6 +1621,7 @@ class Factory:
         else:
             build_lineage = None
         derivation = Concept.calculate_derivation(build_lineage, base.purpose)
+
         granularity = Concept.calculate_granularity(
             derivation, final_grain, build_lineage
         )
@@ -1675,16 +1682,12 @@ class Factory:
 
     @build.register
     def _(self, base: OrderItem) -> BuildOrderItem:
-        from trilogy.parsing.common import arbitrary_to_concept
 
         bexpr: Any
         if isinstance(base.expr, (AggregateWrapper, WindowItem, FilterItem)) or (
             isinstance(base.expr, Function) and base.expr.operator == FunctionType.GROUP
         ):
-            bexpr = arbitrary_to_concept(
-                base.expr,
-                environment=self.environment,
-            )
+            bexpr, _ = self.instantiate_concept(base.expr)
         else:
             bexpr = base.expr
         return BuildOrderItem.model_construct(
@@ -1707,15 +1710,10 @@ class Factory:
 
     @build.register
     def _(self, base: WindowItem) -> BuildWindowItem:
-        # to do proper discovery, we need to inject virtual intermediate ocncepts
-        from trilogy.parsing.common import arbitrary_to_concept
 
         content: Concept | FuncArgs = base.content
         if isinstance(content, (AggregateWrapper, FilterItem, WindowItem)):
-            content = arbitrary_to_concept(
-                content,
-                environment=self.environment,
-            )
+            content, _ = self.instantiate_concept(content)
         final_by = []
         for x in base.order_by:
             if (
@@ -1743,30 +1741,26 @@ class Factory:
 
     @build.register
     def _(self, base: SubselectComparison) -> BuildSubselectComparison:
+        right: Any = base.right
+        if isinstance(base.right, (AggregateWrapper, WindowItem, FilterItem, Function)):
+            right_c, _ = self.instantiate_concept(base.right)
+            right = right_c
         return BuildSubselectComparison.model_construct(
-            left=(self.build(base.left)),
-            right=(self.build(base.right)),
+            left=self.build(base.left),
+            right=self.build(right),
             operator=base.operator,
         )
 
     @build.register
     def _(self, base: Comparison) -> BuildComparison:
-        from trilogy.parsing.common import arbitrary_to_concept
 
         left = base.left
         if isinstance(left, (AggregateWrapper, WindowItem, FilterItem)):
-            left_c = arbitrary_to_concept(
-                left,
-                environment=self.environment,
-            )
+            left_c, _ = self.instantiate_concept(left)
             left = left_c  # type: ignore
         right = base.right
         if isinstance(right, (AggregateWrapper, WindowItem, FilterItem)):
-            right_c = arbitrary_to_concept(
-                right,
-                environment=self.environment,
-            )
-
+            right_c, _ = self.instantiate_concept(right)
             right = right_c  # type: ignore
         return BuildComparison.model_construct(
             left=(self.build(left)),
@@ -1826,6 +1820,13 @@ class Factory:
 
     @build.register
     def _(self, base: FilterItem) -> BuildFilterItem:
+        if isinstance(
+            base.content, (Function, AggregateWrapper, WindowItem, FilterItem)
+        ):
+            _, built = self.instantiate_concept(base.content)
+            return BuildFilterItem.model_construct(
+                content=built, where=self.build(base.where)
+            )
         return BuildFilterItem.model_construct(
             content=self.build(base.content), where=self.build(base.where)
         )
@@ -1969,6 +1970,10 @@ class Factory:
             new.datasources[k] = self.build(d)
         for k, a in base.alias_origin_lookup.items():
             new.alias_origin_lookup[k] = self.build(a)
+        # add in anything that was built as a side-effect
+        for bk, bv in self.local_concepts.items():
+            if bk not in new.concepts:
+                new.concepts[bk] = bv
         new.gen_concept_list_caches()
         return new
 

trilogy/core/processing/node_generators/common.py

@@ -178,7 +178,7 @@ def gen_enrichment_node(
             for x in extra_required
         ):
             log_lambda(
-                f"{str(type(base_node).__name__)} returning property optimized enrichment node"
+                f"{str(type(base_node).__name__)} returning property optimized enrichment node for {extra_required[0].keys}"
             )
             return gen_property_enrichment_node(
                 base_node,

trilogy/core/processing/node_generators/window_node.py

@@ -32,7 +32,9 @@ def resolve_window_parent_concepts(
     if concept.lineage.order_by:
         for item in concept.lineage.order_by:
             base += item.concept_arguments
-
+    if concept.grain:
+        for gitem in concept.grain.components:
+            base.append(environment.concepts[gitem])
     return concept.lineage.content, unique(base, "address")
 
 
@@ -61,6 +63,9 @@ def gen_window_node(
     # append in keys to get the right grain
     if concept.keys:
         for item in concept.keys:
+            logger.info(
+                f"{padding(depth)}{LOGGER_PREFIX} appending search for key {item}"
+            )
             targets.append(environment.concepts[item])
     additional_outputs = []
     if equivalent_optional:
@@ -72,10 +77,15 @@ def gen_window_node(
             additional_outputs.append(x)
 
     grain_equivalents = [
-        x for x in local_optional if x.keys and all([key in targets for key in x.keys])
+        x
+        for x in local_optional
+        if x.keys
+        and all([key in targets for key in x.keys])
+        and x.grain == concept.grain
     ]
 
     for x in grain_equivalents:
+        logger.info("Appending grain equivalent %s", x)
         targets.append(x)
 
     # finally, the ones we'll need to enrich
@@ -123,20 +133,24 @@ def gen_window_node(
     )
     _window_node.rebuild_cache()
     _window_node.resolve()
+
     window_node = StrategyNode(
         input_concepts=[concept] + additional_outputs + parent_concepts + targets,
         output_concepts=[concept] + additional_outputs + parent_concepts + targets,
         environment=environment,
         parents=[_window_node],
         preexisting_conditions=conditions.conditional if conditions else None,
-        # hidden_concepts=[
-        #     x.address for x in parent_concepts if x.address not in local_optional
-        # ],
+        grain=BuildGrain.from_concepts(
+            concepts=[concept] + additional_outputs + parent_concepts + targets,
+            environment=environment,
+        ),
     )
     if not non_equivalent_optional:
         logger.info(
             f"{padding(depth)}{LOGGER_PREFIX} no optional concepts, returning window node"
         )
+        # prune outputs if we don't need join keys
+        window_node.set_output_concepts([concept] + additional_outputs + targets)
         return window_node
 
     missing_optional = [

trilogy/dialect/bigquery.py

@@ -34,7 +34,7 @@ FUNCTION_MAP = {
 
 FUNCTION_GRAIN_MATCH_MAP = {
     **FUNCTION_MAP,
-    FunctionType.COUNT: lambda args: f"{args[0]}",
+    FunctionType.COUNT: lambda args: "1",
     FunctionType.SUM: lambda args: f"{args[0]}",
     FunctionType.AVG: lambda args: f"{args[0]}",
 }

trilogy/parsing/common.py

@@ -458,7 +458,25 @@ def filter_item_to_concept(
     metadata: Metadata | None = None,
 ) -> Concept:
     fmetadata = metadata or Metadata()
-    cparent = environment.concepts[parent.content.address]
+    if isinstance(parent.content, ConceptRef):
+        cparent = environment.concepts[parent.content.address]
+    elif isinstance(
+        parent.content,
+        (
+            FilterItem,
+            AggregateWrapper,
+            FunctionCallWrapper,
+            WindowItem,
+            Function,
+            ListWrapper,
+            MapWrapper,
+        ),
+    ):
+        cparent = arbitrary_to_concept(parent.content, environment, namespace=namespace)
+    else:
+        raise NotImplementedError(
+            f"Filter item with non ref content {parent.content} not yet supported"
+        )
     modifiers = get_upstream_modifiers(
         cparent.concept_arguments, environment=environment
     )
@@ -494,24 +512,6 @@ def window_item_to_concept(
     metadata: Metadata | None = None,
 ) -> Concept:
     fmetadata = metadata or Metadata()
-    # if isinstance(
-    #     parent.content,
-    #     (
-    #         AggregateWrapper
-    #         | FunctionCallWrapper
-    #         | WindowItem
-    #         | FilterItem
-    #         | Function
-    #         | ListWrapper
-    #         | MapWrapper
-    #     ),
-    # ):
-    #     new_parent = arbitrary_to_concept(
-    #         parent.content, environment=environment, namespace=namespace
-    #     )
-    #     environment.add_concept(new_parent)
-    #     parent = parent.model_copy(update={"content": new_parent.reference})
-
     if not isinstance(parent.content, ConceptRef):
         raise NotImplementedError(
             f"Window function wiht non ref content {parent.content} not yet supported"
@@ -523,16 +523,26 @@ def window_item_to_concept(
         local_purpose, keys = get_purpose_and_keys(None, (bcontent,), environment)
     else:
         local_purpose = Purpose.PROPERTY
-        keys = {
-            bcontent.address,
-        }
+        keys = Grain.from_concepts(
+            [bcontent.address] + [y.address for y in parent.over], environment
+        ).components
 
+    # when including the order by in discovery grain
     if parent.order_by:
         grain_components = parent.over + [bcontent.output]
         for item in parent.order_by:
-            grain_components += item.concept_arguments
+            # confirm that it's not just an aggregate at the grain of the stuff we're already keying of of
+            # in which case we can ignore contributions
+            if (
+                isinstance(item.expr, AggregateWrapper)
+                and set([x.address for x in item.expr.by]) == keys
+            ):
+                continue
+            else:
+                grain_components += item.concept_arguments
     else:
         grain_components = parent.over + [bcontent.output]
+
     final_grain = Grain.from_concepts(grain_components, environment)
     modifiers = get_upstream_modifiers(bcontent.concept_arguments, environment)
     datatype = parent.content.datatype
@@ -651,7 +661,9 @@ def rowset_concept(
     orig_concept = environment.concepts[orig_address.address]
     name = orig_concept.name
     if isinstance(orig_concept.lineage, FilterItem):
-        if orig_concept.lineage.where == rowset.select.where_clause:
+        if orig_concept.lineage.where == rowset.select.where_clause and isinstance(
+            orig_concept.lineage.content, (ConceptRef, Concept)
+        ):
             name = environment.concepts[orig_concept.lineage.content.address].name
     base_namespace = (
         f"{rowset.name}.{orig_concept.namespace}"
@@ -761,7 +773,10 @@ def arbitrary_to_concept(
         )
     elif isinstance(parent, FilterItem):
         if not name:
-            name = f"{VIRTUAL_CONCEPT_PREFIX}_filter_{parent.content.name}_{string_to_hash(str(parent))}"
+            if isinstance(parent.content, ConceptRef):
+                name = f"{VIRTUAL_CONCEPT_PREFIX}_filter_{parent.content.name}_{string_to_hash(str(parent))}"
+            else:
+                name = f"{VIRTUAL_CONCEPT_PREFIX}_filter_{string_to_hash(str(parent))}"
         return filter_item_to_concept(
             parent,
             name,

trilogy/parsing/parse_engine.py

@@ -495,7 +495,7 @@ class ParseToObjects(Transformer):
         return ComparisonOperator([x.value.lower() for x in args])
 
     def COMPARISON_OPERATOR(self, args) -> ComparisonOperator:
-        return ComparisonOperator(args)
+        return ComparisonOperator(args.strip())
 
     def LOGICAL_OPERATOR(self, args) -> BooleanOperator:
         return BooleanOperator(args.lower())
@@ -682,8 +682,7 @@ class ParseToObjects(Transformer):
             return ConceptDerivationStatement(concept=concept)
 
         raise SyntaxError(
-            f"Received invalid type {type(args[2])} {args[2]} as input to select"
-            " transform"
+            f"Received invalid type {type(args[2])} {args[2]} as input to concept derivation: `{self.text_lookup[self.token_address][meta.start_pos:meta.end_pos]}`"
         )
 
     @v_args(meta=True)
@@ -1255,7 +1254,7 @@ class ParseToObjects(Transformer):
             intersection = base.locally_derived.intersection(pre_keys)
             if intersection:
                 raise ParseError(
-                    f"Select statement {base} has derived concepts {list(intersection)} that shadow existing environment concepts, which may cause unexpected behavior. Rename these."
+                    f"Select statement {base} creates new derived concepts {list(intersection)} from transformations with identical name(s) to existing concept(s). Do you mean to drop the calculation and directly use the existing concept? If not, alias these concept(s) under new names."
                 )
         return base
 
@@ -1365,11 +1364,78 @@ class ParseToObjects(Transformer):
     def literal(self, args):
         return args[0]
 
+    def product_operator(self, args) -> Function | Any:
+        if len(args) == 1:
+            return args[0]
+        result = args[0]
+        for i in range(1, len(args), 2):
+            new_result = None
+            op = args[i]
+            right = args[i + 1]
+            if op == "*":
+                new_result = self.function_factory.create_function(
+                    [result, right], operator=FunctionType.MULTIPLY
+                )
+            elif op == "/":
+                new_result = self.function_factory.create_function(
+                    [result, right], operator=FunctionType.DIVIDE
+                )
+            elif op == "%":
+                new_result = self.function_factory.create_function(
+                    [result, right], operator=FunctionType.MOD
+                )
+            else:
+                raise ValueError(f"Unknown operator: {op}")
+            result = new_result
+        return new_result
+
+    def PLUS_OR_MINUS(self, args) -> str:
+        return args.value
+
+    def MULTIPLY_DIVIDE_PERCENT(self, args) -> str:
+        return args[0]
+
+    @v_args(meta=True)
+    def sum_operator(self, meta: Meta, args) -> Function | Any:
+        if len(args) == 1:
+            return args[0]
+        result = args[0]
+        for i in range(1, len(args), 2):
+            new_result = None
+            op = args[i]
+            right = args[i + 1]
+            if op == "+":
+                new_result = self.function_factory.create_function(
+                    [result, right], operator=FunctionType.ADD, meta=meta
+                )
+            elif op == "-":
+                new_result = self.function_factory.create_function(
+                    [result, right], operator=FunctionType.SUBTRACT, meta=meta
+                )
+            elif op == "||":
+                new_result = self.function_factory.create_function(
+                    [result, right], operator=FunctionType.CONCAT, meta=meta
+                )
+            elif op == "like":
+                new_result = self.function_factory.create_function(
+                    [result, right], operator=FunctionType.LIKE, meta=meta
+                )
+            else:
+                raise ValueError(f"Unknown operator: {op}")
+            result = new_result
+        return result
+
     def comparison(self, args) -> Comparison:
-        if args[1] == ComparisonOperator.IN:
-            raise SyntaxError
+        if len(args) == 1:
+            return args[0]
         left = args[0]
         right = args[2]
+        if args[1] in (ComparisonOperator.IN, ComparisonOperator.NOT_IN):
+            return SubselectComparison(
+                left=left,
+                right=right,
+                operator=args[1],
+            )
         return Comparison(left=left, right=right, operator=args[1])
 
     def between_comparison(self, args) -> Conditional:
@@ -1464,7 +1530,7 @@ class ParseToObjects(Transformer):
         return DatePart(args.value)
 
     @v_args(meta=True)
-    def window_item(self, meta, args) -> WindowItem:
+    def window_item(self, meta: Meta, args) -> WindowItem:
         type: WindowType = args[0]
         order_by = []
         over = []
@@ -1486,7 +1552,10 @@ class ParseToObjects(Transformer):
             else:
                 concept = arbitrary_to_concept(item, environment=self.environment)
                 self.environment.add_concept(concept, meta=meta)
-        assert concept
+        if not concept:
+            raise ParseError(
+                f"Window statements must be on fields, not constants - error in: `{self.text_lookup[self.parse_address][meta.start_pos:meta.end_pos]}`"
+            )
         return WindowItem(
             type=type,
             content=concept.reference,
@@ -1497,13 +1566,14 @@ class ParseToObjects(Transformer):
 
     def filter_item(self, args) -> FilterItem:
         where: WhereClause
-        string_concept, raw = args
+        expr, raw = args
         if isinstance(raw, WhereClause):
             where = raw
         else:
             where = WhereClause(conditional=raw)
-        concept = self.environment.concepts[string_concept].reference
-        return FilterItem(content=concept, where=where)
+        if isinstance(expr, str):
+            expr = self.environment.concepts[expr].reference
+        return FilterItem(content=expr, where=where)
 
     # BEGIN FUNCTIONS
     @v_args(meta=True)
@@ -1722,10 +1792,7 @@ class ParseToObjects(Transformer):
     def fyear(self, meta, args):
         return self.function_factory.create_function(args, FunctionType.YEAR, meta)
 
-    # utility functions
-    @v_args(meta=True)
-    def fcast(self, meta, args) -> Function:
-        # if it's casting a constant, we'll process that directly
+    def internal_fcast(self, meta, args):
         args = process_function_args(args, meta=meta, environment=self.environment)
         if isinstance(args[0], str):
             processed: date | datetime | int | float | bool | str
@@ -1750,9 +1817,15 @@ class ParseToObjects(Transformer):
             )
         return self.function_factory.create_function(args, FunctionType.CAST, meta)
 
+    # utility functions
+    @v_args(meta=True)
+    def fcast(self, meta, args) -> Function:
+        return self.internal_fcast(meta, args)
+
     # math functions
     @v_args(meta=True)
     def fadd(self, meta, args) -> Function:
+
         return self.function_factory.create_function(args, FunctionType.ADD, meta)
 
     @v_args(meta=True)
@@ -1815,16 +1888,24 @@ class ParseToObjects(Transformer):
         return self.function_factory.create_function(args, FunctionType.BOOL, meta)
 
 
-def unpack_visit_error(e: VisitError):
+def unpack_visit_error(e: VisitError, text: str | None = None):
     """This is required to get exceptions from imports, which would
     raise nested VisitErrors"""
     if isinstance(e.orig_exc, VisitError):
-        unpack_visit_error(e.orig_exc)
+        unpack_visit_error(e.orig_exc, text)
     elif isinstance(e.orig_exc, (UndefinedConceptException, ImportError)):
         raise e.orig_exc
     elif isinstance(e.orig_exc, (SyntaxError, TypeError)):
         if isinstance(e.obj, Tree):
-            raise InvalidSyntaxException(
+            if text:
+                extract = text[e.obj.meta.start_pos - 5 : e.obj.meta.end_pos + 5]
+                raise InvalidSyntaxException(
+                    str(e.orig_exc)
+                    + " in "
+                    + str(e.rule)
+                    + f" Line: {e.obj.meta.line} ({extract})"
+                )
+            InvalidSyntaxException(
                 str(e.orig_exc) + " in " + str(e.rule) + f" Line: {e.obj.meta.line}"
             )
         raise InvalidSyntaxException(str(e.orig_exc)).with_traceback(
@@ -1875,7 +1956,7 @@ def parse_text(
             f"Parse time: {end - start} for {len(text)} characters, {len(output)} objects"
         )
     except VisitError as e:
-        unpack_visit_error(e)
+        unpack_visit_error(e, text)
         # this will never be reached
         raise e
     except (
@@ -1886,6 +1967,12 @@ def parse_text(
         ValidationError,
         TypeError,
     ) as e:
+        if isinstance(
+            e, (UnexpectedCharacters, UnexpectedEOF, UnexpectedInput, UnexpectedToken)
+        ):
+            raise InvalidSyntaxException(
+                str(e) + "\nContext:\n" + e.get_context(text.replace("\n", " "), 20)
+            )
         raise InvalidSyntaxException(str(e))
 
     return environment, output

trilogy/parsing/trilogy.lark

@@ -52,7 +52,7 @@
     
     //column_assignment
     //figure out if we want static
-    column_assignment: (raw_column_assignment | IDENTIFIER | QUOTED_IDENTIFIER |  _static_functions  ) ":" concept_assignment
+    column_assignment: (raw_column_assignment | IDENTIFIER | QUOTED_IDENTIFIER |  expr  ) ":" concept_assignment
     
     RAW_ENTRY.1: /raw\s*\(/s
 
@@ -102,7 +102,7 @@
     type_declaration: "type" IDENTIFIER data_type 
 
     // user_id where state = Mexico
-    _filter_alt: IDENTIFIER "?" conditional
+    _filter_alt: (IDENTIFIER | "(" expr ")") "?" conditional
     _filter_base: "filter"i IDENTIFIER where
     filter_item: _filter_base | _filter_alt
 
@@ -129,12 +129,6 @@
     
     limit: "LIMIT"i /[0-9]+/
     
-    !window_order: /TOP|BOTTOM/i
-    
-    window: window_order /[0-9]+/
-    
-    window_order_by: "BY"i column_list
-    
     order_list: expr ordering (","  expr ordering)* ","?
     
     over_list: concept_lit (","  concept_lit )* ","?
@@ -166,11 +160,9 @@
 
     !array_comparison: ( ("NOT"i "IN"i) | "IN"i)
 
-    COMPARISON_OPERATOR: /(is\s+not|is|=|>=|<=|!=|>|<)/i
+    COMPARISON_OPERATOR: /(\s+is\s+not\s|\s+is\s|\s+in\s|\s+not\s+in\s|=|>=|<=|!=|>|<)/i
     
-    comparison: expr COMPARISON_OPERATOR expr
-
-    between_comparison: expr "between"i expr "and"i expr
+    between_comparison: "between"i expr "and"i expr
 
     subselect_comparison: expr array_comparison (literal | _constant_functions | _string_functions | concept_lit | filter_item | window_item | unnest | fgroup |  expr_tuple | parenthetical )
     
@@ -187,31 +179,49 @@
     union: _UNION   (expr ",")* expr ")"
 
     //indexing into an expression is a function
-    index_access: expr "[" int_lit "]"
-    map_key_access: expr "[" string_lit "]"
-    attr_access: expr  "." string_lit 
+    index_access: atom "[" int_lit "]"
+    map_key_access: atom "[" string_lit "]"
+    attr_access: atom  "." string_lit 
 
+    ?expr: comparison_root | between_root
 
-    expr: _basic_expr | _functional_expr | _operation_expr |  _access_expr
+    ?comparison_root: sum_chain (COMPARISON_OPERATOR sum_chain)? -> comparison
+    ?between_root: sum_chain "between"i sum_chain "and"i sum_chain -> between_comparison
 
-   # Most common/basic expressions
-   _basic_expr: literal | concept_lit | parenthetical | expr_tuple
+    PLUS_OR_MINUS: ("+" | /-(?!>)/ | "||" | "like" ) 
 
-   # Operations and comparisons 
-   _operation_expr: comparison | alt_like | between_comparison | subselect_comparison
+    ?sum_chain: product_chain (PLUS_OR_MINUS product_chain)* -> sum_operator
+    
+    MULTIPLY_DIVIDE_PERCENT: ("*" | "/" | "%")
+
+    ?product_chain: atom ( MULTIPLY_DIVIDE_PERCENT atom)* -> product_operator
+
+    ?atom: literal | concept_lit | parenthetical
+     | expr_tuple
+     | custom_function
+     | _constant_functions
+     | _static_functions
+     | _generic_functions
+     | _date_functions
+     | aggregate_functions
+     | window_item
+     | unnest
+     | union
+     | fgroup
+     | filter_item
+     | _access_expr
+     | aggregate_by
 
-   # Function-like expressions
-   _functional_expr:  _constant_functions | _static_functions  | filter_item | aggregate_functions | window_item | custom_function | fgroup | unnest | union |  aggregate_by
 
    # Access patterns
    _access_expr: index_access | map_key_access | attr_access
     // functions
     
-    fadd: (/add\(/ expr "," expr ")" ) | ( expr "+" expr )
-    fsub: ("subtract"i "(" expr "," expr ")" ) | ( expr "-" expr )
-    fmul: ("multiply"i "(" expr "," expr ")" ) | ( expr "*" expr )
-    fdiv: ( "divide"i "(" expr "," expr ")") | ( expr "/" expr )
-    fmod: ( "mod"i "(" expr "," (int_lit | concept_lit ) ")") | ( expr "%" (int_lit | concept_lit ) )
+    fadd: (/add\(/ expr "," expr ")" ) 
+    fsub: ("subtract"i "(" expr "," expr ")" ) 
+    fmul: ("multiply"i "(" expr "," expr ")" ) 
+    fdiv: ( "divide"i "(" expr "," expr ")") 
+    fmod: ( "mod"i "(" expr "," (int_lit | concept_lit ) ")")  
     _ROUND.1: "round"i "("
     fround: _ROUND expr "," expr ")"
     fabs: "abs"i "(" expr ")"
@@ -224,9 +234,9 @@
     
     //generic
     _fcast_primary: "cast"i "(" expr "as"i data_type ")"
-    _fcast_alt: expr "::" data_type
+    _fcast_alt: atom "::" data_type
     fcast: _fcast_primary | _fcast_alt
-    concat: ("concat"i "(" (expr ",")* expr ")") | (expr "||" expr)
+    concat: ("concat"i "(" (expr ",")* expr ")") 
     fcoalesce: "coalesce"i "(" (expr ",")* expr ")"
     fcase_when: "WHEN"i conditional "THEN"i expr
     fcase_else: "ELSE"i expr
@@ -235,7 +245,7 @@
     fnot: "NOT"i expr
     fbool: "bool"i "(" expr ")"
 
-    _generic_functions: fcast | concat | fcoalesce | fcase | len | fnot | fbool
+    _generic_functions: fcast |  concat  | fcoalesce | fcase  | len | fnot | fbool
 
     //constant
     CURRENT_DATE.1: /current_date\(\)/
@@ -333,12 +343,12 @@
     
     _date_functions: fdate | fdate_add | fdate_sub | fdate_diff | fdatetime | ftimestamp | fsecond | fminute | fhour | fday | fday_of_week | fweek | fmonth | fquarter | fyear | fdate_part | fdate_trunc
     
-    _static_functions:  _string_functions | _math_functions | _generic_functions |  _date_functions
+    _static_functions:  _string_functions | _math_functions 
 
     custom_function:  "@" IDENTIFIER  "(" (expr ",")* expr ")"
 
     // base language constructs
-    concept_lit: MINUS? IDENTIFIER
+    concept_lit:  IDENTIFIER
     IDENTIFIER: /[a-zA-Z\_][a-zA-Z0-9\_\.]*/
     WILDCARD_IDENTIFIER: /[a-zA-Z\_][a-zA-Z0-9\_\-\.\*]*/
     QUOTED_IDENTIFIER:  /`[a-zA-Z\_][a-zA-Z0-9\_\.\-\*\:\s]*`/
@@ -353,9 +363,9 @@
     _double_quote: "\"" ( DOUBLE_STRING_CHARS )* "\"" 
     string_lit: _single_quote | _double_quote | MULTILINE_STRING
 
-    MINUS: "-"
 
-    int_lit: MINUS? /[0-9]+/
+
+    int_lit:  /\-?[0-9]+/
     
     float_lit: /[0-9]*\.[0-9]+/