pytrilogy 0.0.3.37__py3-none-any.whl → 0.0.3.38__py3-none-any.whl

{pytrilogy-0.0.3.37.dist-info → pytrilogy-0.0.3.38.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.37
+Version: 0.0.3.38
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.3.37.dist-info → pytrilogy-0.0.3.38.dist-info}/RECORD

@@ -1,5 +1,5 @@
-pytrilogy-0.0.3.37.dist-info/licenses/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
-trilogy/__init__.py,sha256=5M7wcjmeEthyEiIJM3OK2LvRgaZK8UTv_Zcf0SXA8t0,303
+pytrilogy-0.0.3.38.dist-info/licenses/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
+trilogy/__init__.py,sha256=qMCD2lyNbU_pQxwX1NltGMuBiXL8s_weV7CjLANmsDM,303
 trilogy/compiler.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/constants.py,sha256=5eQxk1A0pv-TQk3CCvgZCFA9_K-6nxrOm7E5Lxd7KIY,1652
 trilogy/engine.py,sha256=OK2RuqCIUId6yZ5hfF8J1nxGP0AJqHRZiafcowmW0xc,1728
@@ -16,14 +16,14 @@ trilogy/core/env_processor.py,sha256=pFsxnluKIusGKx1z7tTnfsd_xZcPy9pZDungkjkyvI0
 trilogy/core/environment_helpers.py,sha256=80XJrhSBd3CO7aWUc7r_oxXh6CeiR9cv0zaPyfK_uDM,8540
 trilogy/core/ergonomics.py,sha256=e-7gE29vPLFdg0_A1smQ7eOrUwKl5VYdxRSTddHweRA,1631
 trilogy/core/exceptions.py,sha256=JPYyBcit3T_pRtlHdtKSeVJkIyWUTozW2aaut25A2xI,673
-trilogy/core/functions.py,sha256=bgmvWdO9x2fTZ0zjekspUb4YQ4xbVfpKYIE1wu8ftPQ,27567
+trilogy/core/functions.py,sha256=rU9Wc2HaIZEQC9L1iDKSAPeshheCOH0eEpBK1QR6JwE,27644
 trilogy/core/graph_models.py,sha256=z17EoO8oky2QOuO6E2aMWoVNKEVJFhLdsQZOhC4fNLU,2079
 trilogy/core/internal.py,sha256=iicDBlC6nM8d7e7jqzf_ZOmpUsW8yrr2AA8AqEiLx-s,1577
 trilogy/core/optimization.py,sha256=aihzx4-2-mSjx5td1TDTYGvc7e9Zvy-_xEyhPqLS-Ig,8314
-trilogy/core/query_processor.py,sha256=Do8YpdPBdsbKtl9n37hobzk8SORMGqH-e_zNNxd-BE4,19456
+trilogy/core/query_processor.py,sha256=Vl-u0F0rbqI2liv82yJgiZCB255Kx_KiuzZVHL6aeTM,19459
 trilogy/core/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/core/models/author.py,sha256=rb-OVaCVBlaE08JhWNFK2sMi9r7yGBT2QK97JE0oKJg,76950
-trilogy/core/models/build.py,sha256=PgTLergGdBnq2NTDqoxnGtZctw3LEOmILPzp8ldf9ws,59569
+trilogy/core/models/author.py,sha256=xV4l0mmX1HskrioE3LEuJbY6wKjvetUVepL447EZ6xQ,77142
+trilogy/core/models/build.py,sha256=flDLLFCR0XJ0fpUhkGg-bpDOkH915tDdXaKmrAzNIcg,61828
 trilogy/core/models/build_environment.py,sha256=s_C9xAHuD3yZ26T15pWVBvoqvlp2LdZ8yjsv2_HdXLk,5363
 trilogy/core/models/core.py,sha256=wx6hJcFECMG-Ij972ADNkr-3nFXkYESr82ObPiC46_U,10875
 trilogy/core/models/datasource.py,sha256=6RjJUd2u4nYmEwFBpJlM9LbHVYDv8iHJxqiBMZqUrwI,9422
@@ -35,42 +35,42 @@ trilogy/core/optimizations/inline_constant.py,sha256=lvNTIXaLNkw3HseJyXyDNk5R52d
 trilogy/core/optimizations/inline_datasource.py,sha256=AHuTGh2x0GQ8usOe0NiFncfTFQ_KogdgDl4uucmhIbI,4241
 trilogy/core/optimizations/predicate_pushdown.py,sha256=g4AYE8Aw_iMlAh68TjNXGP754NTurrDduFECkUjoBnc,9399
 trilogy/core/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/core/processing/concept_strategies_v3.py,sha256=m24X5FOOcLTCvY1MY1yUK5qFcWwokMWZ5cAFI2YN9G8,43352
+trilogy/core/processing/concept_strategies_v3.py,sha256=reXqgktFKy5qhm4k6abrs4J16Sovxd4naHrsEcNarEk,43957
 trilogy/core/processing/graph_utils.py,sha256=8QUVrkE9j-9C1AyrCb1nQEh8daCe0u1HuXl-Te85lag,1205
-trilogy/core/processing/utility.py,sha256=3tC__aT76EzcnIexZfOqCH-3WzvPiCAYWs9TBoMvGjc,20903
+trilogy/core/processing/utility.py,sha256=rfzdgl-vWkCyhLzXNNuWgPLK59eiYypQb6TdZKymUqk,21469
 trilogy/core/processing/node_generators/__init__.py,sha256=o8rOFHPSo-s_59hREwXMW6gjUJCsiXumdbJNozHUf-Y,800
 trilogy/core/processing/node_generators/basic_node.py,sha256=UVsXMn6jTjm_ofVFt218jAS11s4RV4zD781vP4im-GI,3371
 trilogy/core/processing/node_generators/common.py,sha256=ZsDzThjm_mAtdQpKAg8QIJiPVZ4KuUkKyilj4eOhSDs,9439
-trilogy/core/processing/node_generators/filter_node.py,sha256=rlY7TbgjJlGhahYgdCIJpJbaSREAGVJEsyUIGaA38O0,8271
+trilogy/core/processing/node_generators/filter_node.py,sha256=lT167yBgy3P9sDBM1Cjj0PKSXro8dvGtBmc8nwsUjig,8366
 trilogy/core/processing/node_generators/group_node.py,sha256=kO-ersxIL04rZwX5-vFIFQQnp357PFo_7ZKXoGq3wyc,5989
-trilogy/core/processing/node_generators/group_to_node.py,sha256=E5bEjovSx422d_MlAUCDFdY4P2WJVp61BmWwltkhzA8,3095
+trilogy/core/processing/node_generators/group_to_node.py,sha256=v2Xl5VjNO2hKJAdmqCbmAOqaRPvxRUbtfhhQXFMifWk,3197
 trilogy/core/processing/node_generators/multiselect_node.py,sha256=GWV5yLmKTe1yyPhN60RG1Rnrn4ktfn9lYYXi_FVU4UI,7061
 trilogy/core/processing/node_generators/node_merge_node.py,sha256=sv55oynfqgpHEpo1OEtVDri-5fywzPhDlR85qaWikvY,16195
 trilogy/core/processing/node_generators/rowset_node.py,sha256=YmBs6ZQ7azLXRFEmeoecpGjK4pMHsUCovuBxfb3UKZI,6848
 trilogy/core/processing/node_generators/select_merge_node.py,sha256=lxXhMhDKGbu67QFNbbAT-BO8gbWppIvjn_hAXpLEPe0,19953
 trilogy/core/processing/node_generators/select_node.py,sha256=Y-zO0AFkTrpi2LyebjpyHU7WWANr7nKZSS9rY7DH4Wo,1888
 trilogy/core/processing/node_generators/synonym_node.py,sha256=9LHK2XHDjbyTLjmDQieskG8fqbiSpRnFOkfrutDnOTE,2258
-trilogy/core/processing/node_generators/union_node.py,sha256=zuMSmgF170vzlp2BBQEhKbqUMjVl2xQDqUB82Dhv-VU,2536
+trilogy/core/processing/node_generators/union_node.py,sha256=VNo6Oey4p8etU9xrOh2oTT2lIOTvY6PULUPRvVa2uxU,2877
 trilogy/core/processing/node_generators/unnest_node.py,sha256=cOEKnMRzXUW3bwmiOlgn3E1-B38osng0dh2pDykwITY,2410
 trilogy/core/processing/node_generators/window_node.py,sha256=MjLmFKUiS-_p-Ak_9mr3becGde9eu5frxmqI7plIETY,5808
 trilogy/core/processing/node_generators/select_helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/processing/node_generators/select_helpers/datasource_injection.py,sha256=GMW07bb6hXurhF0hZLYoMAKSIS65tat5hwBjvqqPeSA,6516
-trilogy/core/processing/nodes/__init__.py,sha256=DqPG3Y8vl5-UTeox6hn1EE6iwPIJpsM-XeZALHSgLZQ,5058
+trilogy/core/processing/nodes/__init__.py,sha256=Lxr3rs_bqOAtMtn3DHIkY058ZzjyLM5mSfGMKW2z0NY,5555
 trilogy/core/processing/nodes/base_node.py,sha256=FHrY8GsTKPuMJklOjILbhGqCt5s1nmlj62Z-molARDA,16835
 trilogy/core/processing/nodes/filter_node.py,sha256=5VtRfKbCORx0dV-vQfgy3gOEkmmscL9f31ExvlODwvY,2461
 trilogy/core/processing/nodes/group_node.py,sha256=MUvcOg9U5J6TnWBel8eht9PdI9BfAKjUxmfjP_ZXx9o,10484
-trilogy/core/processing/nodes/merge_node.py,sha256=djFBJ8Sq1_crP0g7REGCbMFl4so7zPzabkPgAA1TUl4,15720
+trilogy/core/processing/nodes/merge_node.py,sha256=02oWRca0ba41U6PSAB14jwnWWxoyrvxRPLwkli259SY,15865
 trilogy/core/processing/nodes/select_node_v2.py,sha256=Xyfq8lU7rP7JTAd8VV0ATDNal64n4xIBgWQsOuMe_Ak,8824
 trilogy/core/processing/nodes/union_node.py,sha256=fDFzLAUh5876X6_NM7nkhoMvHEdGJ_LpvPokpZKOhx4,1425
 trilogy/core/processing/nodes/unnest_node.py,sha256=oLKMMNMx6PLDPlt2V5neFMFrFWxET8r6XZElAhSNkO0,2181
 trilogy/core/processing/nodes/window_node.py,sha256=JXJ0iVRlSEM2IBr1TANym2RaUf_p5E_l2sNykRzXWDo,1710
 trilogy/core/statements/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/core/statements/author.py,sha256=dOYX2fJ9VHZgP9RhgK7Ssaq4W5vYtMJBkeiZlwqAoDM,14726
+trilogy/core/statements/author.py,sha256=rYDf9rCQ4YKEO9br1OWmOZd-51AiaDkaWYegvteJa8M,14728
 trilogy/core/statements/build.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/statements/common.py,sha256=KxEmz2ySySyZ6CTPzn0fJl5NX2KOk1RPyuUSwWhnK1g,759
 trilogy/core/statements/execute.py,sha256=cSlvpHFOqpiZ89pPZ5GDp9Hu6j6uj-5_h21FWm_L-KM,1248
 trilogy/dialect/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/dialect/base.py,sha256=zeuSrurbKxM22uAZBLd6__a6a27bl8zUFk3zqTV8GO0,41361
+trilogy/dialect/base.py,sha256=vp6_9fUkblAWVpCXGBIcoAx6N7vof9M7s9t6-b_waUY,41409
 trilogy/dialect/bigquery.py,sha256=j5PQvwMUMcLHaxZgbqe6P-v-pwhHDQ38z8uK6ecxzR0,3359
 trilogy/dialect/common.py,sha256=XjHkP8Dqezjkd2JU5xoAlMRS_6HNyXQCF4CykLK3C8o,5011
 trilogy/dialect/config.py,sha256=olnyeVU5W5T6b9-dMeNAnvxuPlyc2uefb7FRME094Ec,3834
@@ -87,13 +87,13 @@ trilogy/hooks/graph_hook.py,sha256=c-vC-IXoJ_jDmKQjxQyIxyXPOuUcLIURB573gCsAfzQ,2
 trilogy/hooks/query_debugger.py,sha256=1npRjww94sPV5RRBBlLqMJRaFkH9vhEY6o828MeoEcw,5583
 trilogy/metadata/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/parsing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/parsing/common.py,sha256=0fRM9_9k3Y24oanSTgHHjVI6LSTAaxTvkaq5ESVWR5o,23541
+trilogy/parsing/common.py,sha256=ksmz25YM0NoRfGe6vd5aIY5jb6ehVxAeWrPqeV1byvs,25851
 trilogy/parsing/config.py,sha256=Z-DaefdKhPDmSXLgg5V4pebhSB0h590vI0_VtHnlukI,111
 trilogy/parsing/exceptions.py,sha256=92E5i2frv5hj9wxObJZsZqj5T6bglvPzvdvco_vW1Zk,38
 trilogy/parsing/helpers.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-trilogy/parsing/parse_engine.py,sha256=Ol4nGl-TnOshE9Gjbo_tis-l-x9a-6c11ZrJ3RuYjBc,63873
+trilogy/parsing/parse_engine.py,sha256=1UAz-F_yDNbkJUCl7yw7lrf_7dYdgE0pkLO5IoxKuWE,64090
 trilogy/parsing/render.py,sha256=hI4y-xjXrEXvHslY2l2TQ8ic0zAOpN41ADH37J2_FZY,19047
-trilogy/parsing/trilogy.lark,sha256=VqGM8rS157hVZpOqA-1aHsrb0jS-iVlTvqrCbPyqMSY,13276
+trilogy/parsing/trilogy.lark,sha256=doS_ADa58Aypqwhmq9xXzJYZhhTXduOpeiaAsIS3QkA,13367
 trilogy/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/scripts/trilogy.py,sha256=1L0XrH4mVHRt1C9T1HnaDv2_kYEfbWTb5_-cBBke79w,3774
 trilogy/std/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -102,8 +102,8 @@ trilogy/std/display.preql,sha256=2BbhvqR4rcltyAbOXAUo7SZ_yGFYZgFnurglHMbjW2g,40
 trilogy/std/geography.preql,sha256=-fqAGnBL6tR-UtT8DbSek3iMFg66ECR_B_41pODxv-k,504
 trilogy/std/money.preql,sha256=ZHW-csTX-kYbOLmKSO-TcGGgQ-_DMrUXy0BjfuJSFxM,80
 trilogy/std/report.preql,sha256=LbV-XlHdfw0jgnQ8pV7acG95xrd1-p65fVpiIc-S7W4,202
-pytrilogy-0.0.3.37.dist-info/METADATA,sha256=QJwr5jfw6iF9Njsd4mSH9WwvcLR9YykKvzlf9OmWuLg,9100
-pytrilogy-0.0.3.37.dist-info/WHEEL,sha256=pxyMxgL8-pra_rKaQ4drOZAegBVuX-G_4nRHjjgWbmo,91
-pytrilogy-0.0.3.37.dist-info/entry_points.txt,sha256=ewBPU2vLnVexZVnB-NrVj-p3E-4vukg83Zk8A55Wp2w,56
-pytrilogy-0.0.3.37.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
-pytrilogy-0.0.3.37.dist-info/RECORD,,
+pytrilogy-0.0.3.38.dist-info/METADATA,sha256=7N_2ajwk3_pT4c2GSBO9CNkHsNVGXMIYRVIf5hRxq8g,9100
+pytrilogy-0.0.3.38.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
+pytrilogy-0.0.3.38.dist-info/entry_points.txt,sha256=ewBPU2vLnVexZVnB-NrVj-p3E-4vukg83Zk8A55Wp2w,56
+pytrilogy-0.0.3.38.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
+pytrilogy-0.0.3.38.dist-info/RECORD,,

{pytrilogy-0.0.3.37.dist-info → pytrilogy-0.0.3.38.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (79.0.0)
+Generator: setuptools (79.0.1)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.dialect.enums import Dialects
 from trilogy.executor import Executor
 from trilogy.parser import parse
 
-__version__ = "0.0.3.37"
+__version__ = "0.0.3.38"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

trilogy/core/functions.py

@@ -191,6 +191,7 @@ FUNCTION_REGISTRY: dict[FunctionType, FunctionConfig] = {
     ),
     FunctionType.GROUP: FunctionConfig(
         arg_count=-1,
+        output_type_function=lambda args: get_output_type_at_index(args, 0),
     ),
     FunctionType.COUNT: FunctionConfig(
         output_purpose=Purpose.METRIC,

trilogy/core/models/author.py

@@ -180,8 +180,14 @@ class UndefinedConcept(ConceptRef):
 
 def address_with_namespace(address: str, namespace: str) -> str:
     existing_ns = address.split(".", 1)[0]
+    if "." in address:
+        existing_name = address.split(".", 1)[1]
+    else:
+        existing_name = address
+    if existing_name == ALL_ROWS_CONCEPT:
+        return address
     if existing_ns == DEFAULT_NAMESPACE:
-        return f"{namespace}.{address.split('.',1)[1]}"
+        return f"{namespace}.{existing_name}"
     return f"{namespace}.{address}"
 
 
@@ -1129,7 +1135,7 @@ class Concept(Addressable, DataTyped, ConceptArgs, Mergeable, Namespaced, BaseMo
         return self.lineage.concept_arguments if self.lineage else []
 
     @classmethod
-    def calculate_derivation(self, lineage, purpose):
+    def calculate_derivation(self, lineage, purpose: Purpose) -> Derivation:
         from trilogy.core.models.build import (
             BuildAggregateWrapper,
             BuildFilterItem,

trilogy/core/models/build.py

@@ -65,6 +65,7 @@ from trilogy.core.models.author import (
     RowsetLineage,
     SelectLineage,
     SubselectComparison,
+    UndefinedConcept,
     WhereClause,
     WindowItem,
 )
@@ -800,9 +801,6 @@ class BuildConcept(Addressable, BuildConceptArgs, DataTyped, BaseModel):
     modifiers: List[Modifier] = Field(default_factory=list)  # type: ignore
     pseudonyms: set[str] = Field(default_factory=set)
 
-    def with_select_context(self, *args, **kwargs):
-        return self
-
     @property
     def is_aggregate(self) -> bool:
         return self.build_is_aggregate
@@ -1503,7 +1501,7 @@ class Factory:
         return base
 
     @build.register
-    def _(self, base: Function) -> BuildFunction:
+    def _(self, base: Function) -> BuildFunction | BuildAggregateWrapper:
         from trilogy.parsing.common import arbitrary_to_concept
 
         raw_args: list[Concept | FuncArgs] = []
@@ -1517,6 +1515,54 @@ class Factory:
                 raw_args.append(narg)
             else:
                 raw_args.append(arg)
+        if base.operator == FunctionType.GROUP:
+            group_base = raw_args[0]
+            final_args: List[Concept | ConceptRef] = []
+            if isinstance(group_base, ConceptRef):
+                if group_base.address in self.environment.concepts and not isinstance(
+                    self.environment.concepts[group_base.address], UndefinedConcept
+                ):
+                    group_base = self.environment.concepts[group_base.address]
+            if (
+                isinstance(group_base, Concept)
+                and isinstance(group_base.lineage, AggregateWrapper)
+                and not group_base.lineage.by
+            ):
+                arguments = raw_args[1:]
+                for x in arguments:
+                    if isinstance(x, (ConceptRef, Concept)):
+                        final_args.append(x)
+                    elif isinstance(x, (AggregateWrapper, FilterItem, WindowItem)):
+                        newx = arbitrary_to_concept(
+                            x,
+                            environment=self.environment,
+                        )
+                        final_args.append(newx)
+                    else:
+                        # constants, etc, can be ignored for group
+                        continue
+                group_base = group_base.model_copy(
+                    deep=True,
+                    update={
+                        "lineage": AggregateWrapper(
+                            function=group_base.lineage.function,
+                            by=final_args,
+                        )
+                    },
+                )
+                group_base = group_base.with_grain(
+                    Grain.from_concepts(final_args, environment=self.environment)
+                )
+                rval = self.build(group_base)
+                return BuildFunction.model_construct(
+                    operator=base.operator,
+                    arguments=[rval, *[self.build(c) for c in raw_args[1:]]],
+                    output_datatype=base.output_datatype,
+                    output_purpose=base.output_purpose,
+                    valid_inputs=base.valid_inputs,
+                    arg_count=base.arg_count,
+                )
+
         new = BuildFunction.model_construct(
             operator=base.operator,
             arguments=[self.build(c) for c in raw_args],
@@ -1568,7 +1614,10 @@ class Factory:
         new_lineage, final_grain, _ = base.get_select_grain_and_keys(
             self.grain, self.environment
         )
-        build_lineage = self.build(new_lineage)
+        if new_lineage:
+            build_lineage = self.build(new_lineage)
+        else:
+            build_lineage = None
         derivation = Concept.calculate_derivation(build_lineage, base.purpose)
         granularity = Concept.calculate_granularity(
             derivation, final_grain, build_lineage

trilogy/core/processing/concept_strategies_v3.py

@@ -110,11 +110,14 @@ def get_priority_concept(
     depth: int,
 ) -> BuildConcept:
     # optimized search for missing concepts
-    pass_one = [
-        c
-        for c in all_concepts
-        if c.address not in attempted_addresses and c.address not in found_concepts
-    ]
+    pass_one = sorted(
+        [
+            c
+            for c in all_concepts
+            if c.address not in attempted_addresses and c.address not in found_concepts
+        ],
+        key=lambda x: x.address,
+    )
     # sometimes we need to scan intermediate concepts to get merge keys or filter keys,
     # so do an exhaustive search
     # pass_two = [c for c in all_concepts+filter_only if c.address not in attempted_addresses]
@@ -388,6 +391,9 @@ def generate_node(
                     # conditions=conditions,
                 )
             else:
+                logger.info(
+                    f"{depth_to_prefix(depth)}{LOGGER_PREFIX} skipping search, already in a recursion fot these concepts"
+                )
                 return None
         return ConstantNode(
             input_concepts=[],
@@ -453,9 +459,10 @@ def generate_node(
                 f"{depth_to_prefix(depth)}{LOGGER_PREFIX} including any filters, there are non-root concepts we should expand first: {non_root}. Recursing with all of these as mandatory"
             )
 
-            if not history.check_started(
-                root_targets, accept_partial=accept_partial, conditions=conditions
-            ):
+            # if not history.check_started(
+            #     root_targets, accept_partial=accept_partial, conditions=conditions
+            # ) or 1==1:
+            if True:
                 history.log_start(
                     root_targets, accept_partial=accept_partial, conditions=conditions
                 )
@@ -470,7 +477,10 @@ def generate_node(
                     # which we do whenever we hit a root node
                     # conditions=conditions,
                 )
-
+            else:
+                logger.info(
+                    f"{depth_to_prefix(depth)}{LOGGER_PREFIX} skipping root search, already in a recursion for these concepts"
+                )
         check = history.gen_select_node(
             concept,
             local_optional,
@@ -546,7 +556,10 @@ def generate_node(
                         f"{depth_to_prefix(depth)}{LOGGER_PREFIX} resolved concepts through synonyms"
                     )
                     return resolved
-
+            else:
+                logger.info(
+                    f"{depth_to_prefix(depth)}{LOGGER_PREFIX} skipping synonym search, already in a recursion for these concepts"
+                )
             return None
     else:
         raise ValueError(f"Unknown derivation {concept.derivation} on {concept}")
@@ -1041,7 +1054,6 @@ def _search_concepts(
                 environment=environment,
                 depth=depth,
             )
-            logger.info(f"gcheck result is {result}")
             if result.required:
                 logger.info(
                     f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Adding group node"
@@ -1104,7 +1116,6 @@ def source_query_concepts(
         history=history,
         conditions=conditions,
     )
-
     if not root:
         error_strings = [
             f"{c.address}<{c.purpose}>{c.derivation}>" for c in output_concepts
@@ -1130,5 +1141,4 @@ def source_query_concepts(
         )
     else:
         candidate = root
-
     return candidate

trilogy/core/processing/node_generators/filter_node.py

@@ -48,9 +48,10 @@ def gen_filter_node(
         if isinstance(x.lineage, FILTER_TYPES):
             if concept.lineage.where == where:
                 logger.info(
-                    f"{padding(depth)}{LOGGER_PREFIX} fetching {x.lineage.content.address} as optional parent with same filter conditions "
+                    f"{padding(depth)}{LOGGER_PREFIX} fetching {x.lineage.content.address} as optional parent from optional {x} with same filter conditions "
                 )
-                parent_row_concepts.append(x.lineage.content)
+                if x.lineage.content.address not in parent_row_concepts:
+                    parent_row_concepts.append(x.lineage.content)
                 optional_included.append(x)
                 continue
         if conditions and conditions == where:

trilogy/core/processing/node_generators/group_to_node.py

@@ -3,6 +3,8 @@ from typing import List
 from trilogy.constants import logger
 from trilogy.core.models.build import BuildConcept, BuildFunction, BuildWhereClause
 from trilogy.core.models.build_environment import BuildEnvironment
+
+# C:\Users\ethan\coding_projects\pytrilogy\trilogy\core\processing\node_generators\group_to_node.py
 from trilogy.core.processing.nodes import (
     GroupNode,
     History,
@@ -30,6 +32,7 @@ def gen_group_to_node(
             f"Group to should have function lineage, is {type(concept.lineage)}"
         )
     group_arg = concept.lineage.arguments[0]
+
     parent_concepts: List[BuildConcept] = concept.lineage.concept_arguments
     logger.info(
         f"{padding(depth)}{LOGGER_PREFIX} group by node has required parents {[x.address for x in parent_concepts]}"

trilogy/core/processing/node_generators/union_node.py

@@ -1,7 +1,7 @@
 from typing import List
 
 from trilogy.constants import logger
-from trilogy.core.enums import FunctionType, Purpose
+from trilogy.core.enums import FunctionType
 from trilogy.core.models.build import BuildConcept, BuildFunction, BuildWhereClause
 from trilogy.core.processing.nodes import History, StrategyNode, UnionNode
 from trilogy.core.processing.utility import padding
@@ -16,6 +16,38 @@ def is_union(c: BuildConcept):
     )
 
 
+def build_layers(
+    concepts: list[BuildConcept],
+) -> tuple[list[list[BuildConcept]], list[BuildConcept]]:
+    sources = {
+        x.address: x.lineage.concept_arguments if x.lineage else [] for x in concepts
+    }
+    root = concepts[0]
+
+    built_layers = []
+    layers = root.lineage.concept_arguments if root.lineage else []
+    sourced = set()
+    while layers:
+        layer = []
+        current = layers.pop()
+        sourced.add(current.address)
+        layer.append(current)
+        for key, values in sources.items():
+            if key == current.address:
+                continue
+            for value in values:
+                if value.address in (current.keys or []) or current.address in (
+                    value.keys or []
+                ):
+                    layer.append(value)
+                    sourced.add(value.address)
+        built_layers.append(layer)
+    complete = [
+        x for x in concepts if all([x.address in sourced for x in sources[x.address]])
+    ]
+    return built_layers, complete
+
+
 def gen_union_node(
     concept: BuildConcept,
     local_optional: List[BuildConcept],
@@ -27,27 +59,15 @@ def gen_union_node(
     conditions: BuildWhereClause | None = None,
 ) -> StrategyNode | None:
     all_unions = [x for x in local_optional if is_union(x)] + [concept]
-
-    parents = []
-    keys = [x for x in all_unions if x.purpose == Purpose.KEY]
-    base = keys.pop()
-    remaining = [x for x in all_unions if x.address != base.address]
-    arguments = []
-    if isinstance(base.lineage, BuildFunction):
-        arguments = base.lineage.concept_arguments
-    for arg in arguments:
-        relevant_parents: list[BuildConcept] = []
-        for other_union in remaining:
-            assert other_union.lineage
-            potential_parents = [z for z in other_union.lineage.concept_arguments]
-            relevant_parents += [
-                x for x in potential_parents if x.keys and arg.address in x.keys
-            ]
+    logger.info(f"{padding(depth)}{LOGGER_PREFIX} found unions {all_unions}")
+    parent_nodes = []
+    layers, resolved = build_layers(all_unions)
+    for layer in layers:
         logger.info(
-            f"For parent arg {arg.address}, including additional union inputs {[c.address for c in relevant_parents]}"
+            f"{padding(depth)}{LOGGER_PREFIX} fetching layer {layer} with resolved {resolved}"
         )
         parent: StrategyNode = source_concepts(
-            mandatory_list=[arg] + relevant_parents,
+            mandatory_list=layer,
             environment=environment,
             g=g,
             depth=depth + 1,
@@ -55,12 +75,8 @@ def gen_union_node(
             conditions=conditions,
         )
         parent.hide_output_concepts(parent.output_concepts)
-        # parent.remove_output_concepts(parent.output_concepts)
-        parent.add_output_concept(concept)
-        for x in remaining:
-            parent.add_output_concept(x)
-
-        parents.append(parent)
+        parent.add_output_concepts(resolved)
+        parent_nodes.append(parent)
         if not parent:
             logger.info(
                 f"{padding(depth)}{LOGGER_PREFIX} could not find union node parents"
@@ -68,8 +84,8 @@ def gen_union_node(
             return None
 
     return UnionNode(
-        input_concepts=[concept] + local_optional,
-        output_concepts=[concept] + local_optional,
+        input_concepts=resolved,
+        output_concepts=resolved,
         environment=environment,
-        parents=parents,
+        parents=parent_nodes,
     )

trilogy/core/processing/nodes/__init__.py

@@ -45,6 +45,11 @@ class History(BaseModel):
         self.history[
             self._concepts_to_lookup(search, accept_partial, conditions=conditions)
         ] = output
+        self.log_end(
+            search,
+            accept_partial=accept_partial,
+            conditions=conditions,
+        )
 
     def get_history(
         self,
@@ -83,6 +88,20 @@ class History(BaseModel):
             )
         )
 
+    def log_end(
+        self,
+        search: list[BuildConcept],
+        accept_partial: bool = False,
+        conditions: BuildWhereClause | None = None,
+    ):
+        self.started.discard(
+            self._concepts_to_lookup(
+                search,
+                accept_partial=accept_partial,
+                conditions=conditions,
+            )
+        )
+
     def check_started(
         self,
         search: list[BuildConcept],

trilogy/core/processing/nodes/merge_node.py

@@ -272,6 +272,8 @@ class MergeNode(StrategyNode):
                     f"{self.logging_prefix}{LOGGER_PREFIX} Merge node has only one parent with the same"
                     " outputs as this merge node, dropping merge node "
                 )
+                # push up any conditions we need
+                final.ordering = self.ordering
                 return final
 
         # if we have multiple candidates, see if one is good enough
@@ -293,6 +295,7 @@ class MergeNode(StrategyNode):
                     f" has all required output properties with partial {[c.address for c in dataset.partial_concepts]}"
                     f" and self has no conditions ({self.conditions})"
                 )
+                dataset.ordering = self.ordering
                 return dataset
 
         pregrain = BuildGrain()

trilogy/core/processing/utility.py

@@ -603,15 +603,29 @@ def find_nullable_concepts(
 def sort_select_output_processed(
     cte: CTE | UnionCTE, query: ProcessedQuery
 ) -> CTE | UnionCTE:
-    output_addresses = [
-        c.address for c in query.output_columns if c.address not in query.hidden_columns
-    ]
+    output_addresses = [c.address for c in query.output_columns]
 
     mapping = {x.address: x for x in cte.output_columns}
 
-    new_output = []
+    new_output: list[BuildConcept] = []
     for x in output_addresses:
         new_output.append(mapping[x])
+
+    for oc in cte.output_columns:
+        # add hidden back
+        if oc.address not in output_addresses:
+            new_output.append(oc)
+
+    cte.hidden_concepts = set(
+        [
+            c.address
+            for c in cte.output_columns
+            if (
+                c.address not in query.output_columns
+                or c.address in query.hidden_columns
+            )
+        ]
+    )
     cte.output_columns = new_output
     return cte
 
@@ -619,18 +633,28 @@ def sort_select_output_processed(
 def sort_select_output(
     cte: CTE | UnionCTE, query: SelectStatement | MultiSelectStatement | ProcessedQuery
 ) -> CTE | UnionCTE:
+
     if isinstance(query, ProcessedQuery):
         return sort_select_output_processed(cte, query)
+
     output_addresses = [
         c.address
         for c in query.output_components
-        if c.address not in query.hidden_components
+        # if c.address not in query.hidden_components
     ]
 
     mapping = {x.address: x for x in cte.output_columns}
 
-    new_output = []
+    new_output: list[BuildConcept] = []
     for x in output_addresses:
         new_output.append(mapping[x])
     cte.output_columns = new_output
+    cte.hidden_concepts = set(
+        [
+            c.address
+            for c in query.output_components
+            if c.address in query.hidden_components
+        ]
+    )
+
     return cte

trilogy/core/query_processor.py

@@ -431,7 +431,9 @@ def get_query_datasources(
     hooks: Optional[List[BaseHook]] = None,
 ) -> QueryDatasource:
     ds = get_query_node(environment, statement.as_lineage(environment))
+
     final_qds = ds.resolve()
+
     if hooks:
         for hook in hooks:
             hook.process_root_strategy_node(ds)
@@ -510,6 +512,7 @@ def process_query(
         hook.process_root_datasource(root_datasource)
     # this should always return 1 - TODO, refactor
     root_cte = datasource_to_cte(root_datasource, environment.cte_name_map)
+
     for hook in hooks:
         hook.process_root_cte(root_cte)
     raw_ctes: List[CTE | UnionCTE] = list(reversed(flatten_ctes(root_cte)))

trilogy/core/statements/author.py

@@ -135,6 +135,7 @@ class SelectStatement(HasUUID, SelectTypeMixin, BaseModel):
         )
 
         output.grain = output.calculate_grain(environment)
+
         for x in selection:
             if x.is_undefined and environment.concepts.fail_on_missing:
                 environment.concepts.raise_undefined(
@@ -166,6 +167,7 @@ class SelectStatement(HasUUID, SelectTypeMixin, BaseModel):
         targets = []
         for x in self.selection:
             targets.append(x.concept)
+
         result = Grain.from_concepts(
             targets, where_clause=self.where_clause, environment=environment
         )

trilogy/dialect/base.py

@@ -171,6 +171,7 @@ FUNCTION_MAP = {
     FunctionType.DATETIME_LITERAL: lambda x: f"datetime '{x}'",
     # math
     FunctionType.ADD: lambda x: " + ".join(x),
+    FunctionType.ABS: lambda x: f"abs({x[0]})",
     FunctionType.SUBTRACT: lambda x: " - ".join(x),
     FunctionType.DIVIDE: lambda x: " / ".join(x),
     FunctionType.MULTIPLY: lambda x: " * ".join(x),

trilogy/parsing/common.py

@@ -220,7 +220,6 @@ def concept_is_relevant(
         if all([c in others for c in concept.grain.components]):
             return False
     if concept.derivation in (Derivation.BASIC,):
-
         return any(
             concept_is_relevant(c, others, environment)
             for c in concept.concept_arguments
@@ -235,6 +234,7 @@ def concepts_to_grain_concepts(
 ) -> list[Concept]:
     pconcepts: list[Concept] = []
     for c in concepts:
+
         if isinstance(c, Concept):
             pconcepts.append(c)
         elif isinstance(c, ConceptRef) and environment:
@@ -281,6 +281,76 @@ def get_relevant_parent_concepts(arg):
     return results
 
 
+def group_function_to_concept(
+    parent: Function,
+    name: str,
+    environment: Environment,
+    namespace: str | None = None,
+    metadata: Metadata | None = None,
+):
+    pkeys: List[Concept] = []
+    namespace = namespace or environment.namespace
+    is_metric = False
+    ref_args, is_metric = get_relevant_parent_concepts(parent)
+    concrete_args = [environment.concepts[c.address] for c in ref_args]
+    pkeys += [x for x in concrete_args if not x.derivation == Derivation.CONSTANT]
+    modifiers = get_upstream_modifiers(pkeys, environment)
+    key_grain: list[str] = []
+    for x in pkeys:
+        # for a group to, if we have a dynamic metric, ignore it
+        # it will end up with the group target grain
+        if x.purpose == Purpose.METRIC and not x.keys:
+            continue
+        # metrics will group to keys, so do no do key traversal
+        elif is_metric:
+            key_grain.append(x.address)
+        else:
+            key_grain.append(x.address)
+    keys = set(key_grain)
+
+    grain = Grain.from_concepts(keys, environment)
+    if is_metric:
+        purpose = Purpose.METRIC
+    elif not pkeys:
+        purpose = Purpose.CONSTANT
+    else:
+        purpose = parent.output_purpose
+    fmetadata = metadata or Metadata()
+    granularity = Granularity.MULTI_ROW
+
+    if grain is not None:
+        # deduplicte
+        grain = Grain.from_concepts(grain.components, environment)
+
+        r = Concept(
+            name=name,
+            datatype=parent.output_datatype,
+            purpose=purpose,
+            lineage=parent,
+            namespace=namespace,
+            keys=keys,
+            modifiers=modifiers,
+            grain=grain,
+            metadata=fmetadata,
+            derivation=Derivation.BASIC,
+            granularity=granularity,
+        )
+        return r
+
+    return Concept(
+        name=name,
+        datatype=parent.output_datatype,
+        purpose=purpose,
+        lineage=parent,
+        namespace=namespace,
+        keys=keys,
+        modifiers=modifiers,
+        metadata=fmetadata,
+        derivation=Derivation.BASIC,
+        granularity=granularity,
+    )
+
+
 def function_to_concept(
     parent: Function,
     name: str,
@@ -347,7 +417,6 @@ def function_to_concept(
     else:
         derivation = Derivation.BASIC
         granularity = Granularity.MULTI_ROW
-    # granularity = Concept.calculate_granularity(derivation, grain, parent)
 
     if grain is not None:
         r = Concept(
@@ -695,6 +764,14 @@ def arbitrary_to_concept(
     elif isinstance(parent, Function):
         if not name:
             name = f"{VIRTUAL_CONCEPT_PREFIX}_func_{parent.operator.value}_{string_to_hash(str(parent))}"
+        if parent.operator == FunctionType.GROUP:
+            return group_function_to_concept(
+                parent,
+                name,
+                environment=environment,
+                namespace=namespace,
+                metadata=metadata,
+            )
         return function_to_concept(
             parent,
             name,

trilogy/parsing/parse_engine.py

@@ -754,6 +754,11 @@ class ParseToObjects(Transformer):
             components=set([self.environment.concepts[a].address for a in args[0]])
         )
 
+    @v_args(meta=True)
+    def aggregate_by(self, meta: Meta, args):
+        args = [self.environment.concepts[a] for a in args]
+        return self.function_factory.create_function(args, FunctionType.GROUP, meta)
+
     def whole_grain_clause(self, args) -> WholeGrainWrapper:
         return WholeGrainWrapper(where=args[0])
 
@@ -1436,12 +1441,14 @@ class ParseToObjects(Transformer):
         return args[0]
 
     def window(self, args):
+
         return Window(count=args[1].value, window_order=args[0])
 
     def WINDOW_TYPE(self, args):
         return WindowType(args.strip())
 
     def window_item_over(self, args):
+
         return WindowItemOver(contents=args[0])
 
     def window_item_order(self, args):

trilogy/parsing/trilogy.lark

@@ -108,12 +108,12 @@
 
     // rank/lag/lead
     WINDOW_TYPE: ("row_number"i|"rank"i|"lag"i|"lead"i | "sum"i | "avg"i | "max"i | "min"i )  /[\s]+/
-    
-    window_item: WINDOW_TYPE int_lit? expr  window_item_over? window_item_order?
-    
+
     window_item_over: ("OVER"i over_list)
-    
+   
     window_item_order: ("ORDER"i? "BY"i order_list)
+
+    window_item: WINDOW_TYPE int_lit? expr window_item_over? window_item_order?
     
     select_hide_modifier: "--"
     select_partial_modifier: "~"
@@ -185,7 +185,7 @@
     attr_access: expr  "." string_lit 
 
 
-    expr:  _constant_functions | window_item | filter_item | subselect_comparison | between_comparison |  fgroup |  aggregate_functions | unnest | union | _static_functions |  literal |  concept_lit  | index_access |  map_key_access | attr_access | parenthetical | expr_tuple | comparison |  alt_like | custom_function 
+    expr:   _constant_functions | window_item | filter_item | subselect_comparison | between_comparison |  fgroup |  aggregate_functions | unnest | union | _static_functions |  literal |  concept_lit  | index_access |  map_key_access | attr_access | parenthetical | expr_tuple | comparison |  alt_like |  aggregate_by | custom_function 
     
     // functions
     
@@ -250,6 +250,10 @@
     // special aggregate
     _GROUP.1: "group("i
     fgroup: _GROUP expr ")" aggregate_over?
+
+    //by:
+    aggregate_by: "group" IDENTIFIER "BY"i (IDENTIFIER ",")* IDENTIFIER
+
     //aggregates
     _COUNT.1: "count("i
     count:   _COUNT  expr ")"