pytrilogy 0.0.3.33__py3-none-any.whl → 0.0.3.35__py3-none-any.whl

{pytrilogy-0.0.3.33.dist-info → pytrilogy-0.0.3.35.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.33
+Version: 0.0.3.35
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.3.33.dist-info → pytrilogy-0.0.3.35.dist-info}/RECORD

@@ -1,5 +1,5 @@
-pytrilogy-0.0.3.33.dist-info/licenses/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
-trilogy/__init__.py,sha256=d8QFTlSiA9H5vsTbRyWcF6hZW9Dih9-6ZorFRaWkb_Q,303
+pytrilogy-0.0.3.35.dist-info/licenses/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
+trilogy/__init__.py,sha256=XKMFtr6LPZcLEaFqOMZ_Bx_NXbihLoadMcLr5wNv0KM,303
 trilogy/compiler.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/constants.py,sha256=5eQxk1A0pv-TQk3CCvgZCFA9_K-6nxrOm7E5Lxd7KIY,1652
 trilogy/engine.py,sha256=OK2RuqCIUId6yZ5hfF8J1nxGP0AJqHRZiafcowmW0xc,1728
@@ -8,7 +8,7 @@ trilogy/parser.py,sha256=o4cfk3j3yhUFoiDKq9ZX_GjBF3dKhDjXEwb63rcBkBM,293
 trilogy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/render.py,sha256=qQWwduymauOlB517UtM-VGbVe8Cswa4UJub5aGbSO6c,1512
 trilogy/utility.py,sha256=euQccZLKoYBz0LNg5tzLlvv2YHvXh9HArnYp1V3uXsM,763
-trilogy/authoring/__init__.py,sha256=ohkYA3_LGYZh3fwzEYKTN6ofACDI5GYl3VCbGxVvlzY,2233
+trilogy/authoring/__init__.py,sha256=v9PRuZs4fTnxhpXAnwTxCDwlLasUax6g2FONidcujR4,2369
 trilogy/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/constants.py,sha256=7XaCpZn5mQmjTobbeBn56SzPWq9eMNDfzfsRU-fP0VE,171
 trilogy/core/enums.py,sha256=fWexUZtssfvP5TiD7eQ66Q_tPUCNCCTGNSzLbVXrnqQ,7358
@@ -16,28 +16,28 @@ trilogy/core/env_processor.py,sha256=pFsxnluKIusGKx1z7tTnfsd_xZcPy9pZDungkjkyvI0
 trilogy/core/environment_helpers.py,sha256=5ayyhf4CGBlg_LssPu3DbS_H9H1Kq6Qog5TgR8pwyMk,8518
 trilogy/core/ergonomics.py,sha256=e-7gE29vPLFdg0_A1smQ7eOrUwKl5VYdxRSTddHweRA,1631
 trilogy/core/exceptions.py,sha256=JPYyBcit3T_pRtlHdtKSeVJkIyWUTozW2aaut25A2xI,673
-trilogy/core/functions.py,sha256=z5uyC5sAS_vnFBJcky7TjA0QAy-xI4Z9uD_9vK_XP1s,27134
+trilogy/core/functions.py,sha256=YjPbZtfQauY6u7puehFLmPsauB5IuGa5xGETKDAgHrU,27396
 trilogy/core/graph_models.py,sha256=z17EoO8oky2QOuO6E2aMWoVNKEVJFhLdsQZOhC4fNLU,2079
 trilogy/core/internal.py,sha256=iicDBlC6nM8d7e7jqzf_ZOmpUsW8yrr2AA8AqEiLx-s,1577
 trilogy/core/optimization.py,sha256=aihzx4-2-mSjx5td1TDTYGvc7e9Zvy-_xEyhPqLS-Ig,8314
 trilogy/core/query_processor.py,sha256=Do8YpdPBdsbKtl9n37hobzk8SORMGqH-e_zNNxd-BE4,19456
 trilogy/core/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/core/models/author.py,sha256=qEN8dJCnOMtB2Uu58GcbiYz6t5LiluCAdDUCUfl6BMk,76572
-trilogy/core/models/build.py,sha256=N3Zr47iN4y9gFWS72uYjecmiQfcm8Fl9wT3QS8xKBAA,58185
-trilogy/core/models/build_environment.py,sha256=8UggvlPU708GZWYPJMc_ou2r7M3TY2g69eqGvz03YX0,5528
+trilogy/core/models/author.py,sha256=rb-OVaCVBlaE08JhWNFK2sMi9r7yGBT2QK97JE0oKJg,76950
+trilogy/core/models/build.py,sha256=6Mjq4NoBqk5U7IWg8Fbb6AAOdMHazVymfYXOm-wcWv4,59346
+trilogy/core/models/build_environment.py,sha256=s_C9xAHuD3yZ26T15pWVBvoqvlp2LdZ8yjsv2_HdXLk,5363
 trilogy/core/models/core.py,sha256=wx6hJcFECMG-Ij972ADNkr-3nFXkYESr82ObPiC46_U,10875
 trilogy/core/models/datasource.py,sha256=6RjJUd2u4nYmEwFBpJlM9LbHVYDv8iHJxqiBMZqUrwI,9422
-trilogy/core/models/environment.py,sha256=yd19vDdC-IiKPV5YbNZcDUOEr1BC_ZxE_5fQDIipa3s,26910
-trilogy/core/models/execute.py,sha256=cmDPkVng3LHw87Q4WiOxJvDQJCoPhEiVMeMqxSxRj7Q,34261
+trilogy/core/models/environment.py,sha256=axgk7W3STy5EIrG8fUwl2oh6WCqeBAr7PWy6EOe-_Dc,27002
+trilogy/core/models/execute.py,sha256=mQm5Gydo2Ph0W7w9wm5dQEarS04PC-IKAgNVsdqOZsQ,34524
 trilogy/core/optimizations/__init__.py,sha256=EBanqTXEzf1ZEYjAneIWoIcxtMDite5-n2dQ5xcfUtg,356
 trilogy/core/optimizations/base_optimization.py,sha256=gzDOKImoFn36k7XBD3ysEYDnbnb6vdVIztUfFQZsGnM,513
 trilogy/core/optimizations/inline_constant.py,sha256=lvNTIXaLNkw3HseJyXyDNk5R52doLU9sIg3pmU2_S08,1332
 trilogy/core/optimizations/inline_datasource.py,sha256=AHuTGh2x0GQ8usOe0NiFncfTFQ_KogdgDl4uucmhIbI,4241
 trilogy/core/optimizations/predicate_pushdown.py,sha256=g4AYE8Aw_iMlAh68TjNXGP754NTurrDduFECkUjoBnc,9399
 trilogy/core/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/core/processing/concept_strategies_v3.py,sha256=MhSCS1M6etHaonfrPNwe6Gk-QuDTVGHkPjt1_9_gqko,42536
+trilogy/core/processing/concept_strategies_v3.py,sha256=m24X5FOOcLTCvY1MY1yUK5qFcWwokMWZ5cAFI2YN9G8,43352
 trilogy/core/processing/graph_utils.py,sha256=8QUVrkE9j-9C1AyrCb1nQEh8daCe0u1HuXl-Te85lag,1205
-trilogy/core/processing/utility.py,sha256=mnN_pewdpDgRou4QJ1JLcqYHyZdp8DrcsGsqW3QmA3o,20591
+trilogy/core/processing/utility.py,sha256=3tC__aT76EzcnIexZfOqCH-3WzvPiCAYWs9TBoMvGjc,20903
 trilogy/core/processing/node_generators/__init__.py,sha256=o8rOFHPSo-s_59hREwXMW6gjUJCsiXumdbJNozHUf-Y,800
 trilogy/core/processing/node_generators/basic_node.py,sha256=UVsXMn6jTjm_ofVFt218jAS11s4RV4zD781vP4im-GI,3371
 trilogy/core/processing/node_generators/common.py,sha256=ZsDzThjm_mAtdQpKAg8QIJiPVZ4KuUkKyilj4eOhSDs,9439
@@ -47,7 +47,7 @@ trilogy/core/processing/node_generators/group_to_node.py,sha256=E5bEjovSx422d_Ml
 trilogy/core/processing/node_generators/multiselect_node.py,sha256=GWV5yLmKTe1yyPhN60RG1Rnrn4ktfn9lYYXi_FVU4UI,7061
 trilogy/core/processing/node_generators/node_merge_node.py,sha256=sv55oynfqgpHEpo1OEtVDri-5fywzPhDlR85qaWikvY,16195
 trilogy/core/processing/node_generators/rowset_node.py,sha256=YmBs6ZQ7azLXRFEmeoecpGjK4pMHsUCovuBxfb3UKZI,6848
-trilogy/core/processing/node_generators/select_merge_node.py,sha256=pIsHfXWA30RkKSMBDKPtDmCriJtHoNKRMJC0THSDxpI,19951
+trilogy/core/processing/node_generators/select_merge_node.py,sha256=lxXhMhDKGbu67QFNbbAT-BO8gbWppIvjn_hAXpLEPe0,19953
 trilogy/core/processing/node_generators/select_node.py,sha256=Y-zO0AFkTrpi2LyebjpyHU7WWANr7nKZSS9rY7DH4Wo,1888
 trilogy/core/processing/node_generators/synonym_node.py,sha256=9LHK2XHDjbyTLjmDQieskG8fqbiSpRnFOkfrutDnOTE,2258
 trilogy/core/processing/node_generators/union_node.py,sha256=zuMSmgF170vzlp2BBQEhKbqUMjVl2xQDqUB82Dhv-VU,2536
@@ -65,7 +65,7 @@ trilogy/core/processing/nodes/union_node.py,sha256=fDFzLAUh5876X6_NM7nkhoMvHEdGJ
 trilogy/core/processing/nodes/unnest_node.py,sha256=oLKMMNMx6PLDPlt2V5neFMFrFWxET8r6XZElAhSNkO0,2181
 trilogy/core/processing/nodes/window_node.py,sha256=JXJ0iVRlSEM2IBr1TANym2RaUf_p5E_l2sNykRzXWDo,1710
 trilogy/core/statements/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/core/statements/author.py,sha256=75ZIMOfQFAahtVLpv3qYJre2SAIdf0vrGBsTR5qweDg,14638
+trilogy/core/statements/author.py,sha256=dOYX2fJ9VHZgP9RhgK7Ssaq4W5vYtMJBkeiZlwqAoDM,14726
 trilogy/core/statements/build.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/statements/common.py,sha256=KxEmz2ySySyZ6CTPzn0fJl5NX2KOk1RPyuUSwWhnK1g,759
 trilogy/core/statements/execute.py,sha256=cSlvpHFOqpiZ89pPZ5GDp9Hu6j6uj-5_h21FWm_L-KM,1248
@@ -87,23 +87,23 @@ trilogy/hooks/graph_hook.py,sha256=c-vC-IXoJ_jDmKQjxQyIxyXPOuUcLIURB573gCsAfzQ,2
 trilogy/hooks/query_debugger.py,sha256=1npRjww94sPV5RRBBlLqMJRaFkH9vhEY6o828MeoEcw,5583
 trilogy/metadata/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/parsing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/parsing/common.py,sha256=bhZ_f520D2diRcXC3YeEToPSXBbUBBNq42zII_smdYc,23518
+trilogy/parsing/common.py,sha256=0fRM9_9k3Y24oanSTgHHjVI6LSTAaxTvkaq5ESVWR5o,23541
 trilogy/parsing/config.py,sha256=Z-DaefdKhPDmSXLgg5V4pebhSB0h590vI0_VtHnlukI,111
 trilogy/parsing/exceptions.py,sha256=92E5i2frv5hj9wxObJZsZqj5T6bglvPzvdvco_vW1Zk,38
 trilogy/parsing/helpers.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-trilogy/parsing/parse_engine.py,sha256=2BeNdsYoyn5UktTdafEKTZ8H7-ELEahDJeqcwMvPtO8,62567
-trilogy/parsing/render.py,sha256=ElpmCWUhGs8h489S7cdlbI8bilJlnBgHZ8KMR8y1FrM,18840
-trilogy/parsing/trilogy.lark,sha256=SeuOZrpIUSqQwCj5mTSmTSkeSLTPwBgtm2AdpcGAy9A,13023
+trilogy/parsing/parse_engine.py,sha256=2zUqtpEyTaxNUz4L3IkA9RKHEFif7VDjxF0Y7sFigts,63506
+trilogy/parsing/render.py,sha256=hI4y-xjXrEXvHslY2l2TQ8ic0zAOpN41ADH37J2_FZY,19047
+trilogy/parsing/trilogy.lark,sha256=ErSKUy2sqpmc3OnflRbQnzTeE81y3ISfUJBD_wvypFM,13159
 trilogy/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/scripts/trilogy.py,sha256=1L0XrH4mVHRt1C9T1HnaDv2_kYEfbWTb5_-cBBke79w,3774
 trilogy/std/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/std/dashboard.preql,sha256=eJJTs3xPjYKmp-U5SUxCXAAbp55NlmmC3kECaNg4ya4,128
 trilogy/std/date.preql,sha256=0MHeGLp2mG4QBKtmozcBZ7qVjOAdWOtrliIKn6hz1Pc,95
 trilogy/std/display.preql,sha256=2BbhvqR4rcltyAbOXAUo7SZ_yGFYZgFnurglHMbjW2g,40
 trilogy/std/geography.preql,sha256=-fqAGnBL6tR-UtT8DbSek3iMFg66ECR_B_41pODxv-k,504
 trilogy/std/money.preql,sha256=ZHW-csTX-kYbOLmKSO-TcGGgQ-_DMrUXy0BjfuJSFxM,80
-pytrilogy-0.0.3.33.dist-info/METADATA,sha256=nCv6QIF6oA0nSwlsLa_znFnm6RetCf4zyfBdBszleNY,9100
-pytrilogy-0.0.3.33.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-pytrilogy-0.0.3.33.dist-info/entry_points.txt,sha256=ewBPU2vLnVexZVnB-NrVj-p3E-4vukg83Zk8A55Wp2w,56
-pytrilogy-0.0.3.33.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
-pytrilogy-0.0.3.33.dist-info/RECORD,,
+trilogy/std/report.preql,sha256=LbV-XlHdfw0jgnQ8pV7acG95xrd1-p65fVpiIc-S7W4,202
+pytrilogy-0.0.3.35.dist-info/METADATA,sha256=0nFaVh5d7D6qAUyxTGqdKlRI93zNMvCtVM-9OVb5ACo,9100
+pytrilogy-0.0.3.35.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+pytrilogy-0.0.3.35.dist-info/entry_points.txt,sha256=ewBPU2vLnVexZVnB-NrVj-p3E-4vukg83Zk8A55Wp2w,56
+pytrilogy-0.0.3.35.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
+pytrilogy-0.0.3.35.dist-info/RECORD,,

trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.dialect.enums import Dialects
 from trilogy.executor import Executor
 from trilogy.parser import parse
 
-__version__ = "0.0.3.33"
+__version__ = "0.0.3.35"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

trilogy/authoring/__init__.py

@@ -19,12 +19,15 @@ from trilogy.core.models.author import (
     Conditional,
     FilterItem,
     Function,
+    FunctionCallWrapper,
     HavingClause,
     MagicConstants,
     Metadata,
+    MultiSelectLineage,
     OrderBy,
     OrderItem,
     Parenthetical,
+    RowsetItem,
     SubselectComparison,
     WhereClause,
     WindowItem,
@@ -103,4 +106,7 @@ __all__ = [
     "RawSQLStatement",
     "Datasource",
     "DatasourceMetadata",
+    "MultiSelectLineage",
+    "RowsetItem",
+    "FunctionCallWrapper",
 ]

trilogy/core/functions.py

@@ -111,11 +111,14 @@ def validate_case_output(
         if output_datatype != DataType.NULL:
             datatypes.add(output_datatype.data_type)
         mapz[str(arg.expr)] = output_datatype
-    if not len(datatypes) == 1:
+    known = [x for x in datatypes if x != DataType.UNKNOWN]
+    if len(known) == 0:
+        return DataType.UNKNOWN
+    if not len(known) == 1:
         raise SyntaxError(
             f"All case expressions must have the same output datatype, got {datatypes} from {mapz}"
         )
-    return datatypes.pop()
+    return known.pop()
 
 
 def create_struct_output(
@@ -672,12 +675,22 @@ FUNCTION_REGISTRY: dict[FunctionType, FunctionConfig] = {
         arg_count=1,
     ),
     FunctionType.SUM: FunctionConfig(
-        valid_inputs={DataType.INTEGER, DataType.FLOAT, DataType.NUMBER},
+        valid_inputs={
+            DataType.INTEGER,
+            DataType.FLOAT,
+            DataType.NUMBER,
+            DataType.NUMERIC,
+        },
         output_purpose=Purpose.METRIC,
         arg_count=1,
     ),
     FunctionType.AVG: FunctionConfig(
-        valid_inputs={DataType.INTEGER, DataType.FLOAT, DataType.NUMBER},
+        valid_inputs={
+            DataType.INTEGER,
+            DataType.FLOAT,
+            DataType.NUMBER,
+            DataType.NUMERIC,
+        },
         output_purpose=Purpose.METRIC,
         arg_count=1,
     ),

trilogy/core/models/author.py

@@ -107,6 +107,10 @@ class ConceptRef(Addressable, Namespaced, DataTyped, Mergeable, BaseModel):
     ) = DataType.UNKNOWN
     metadata: Optional["Metadata"] = None
 
+    @property
+    def reference(self):
+        return self
+
     @property
     def line_no(self) -> int | None:
         if self.metadata:
@@ -1089,7 +1093,7 @@ class Concept(Addressable, DataTyped, ConceptArgs, Mergeable, Namespaced, BaseMo
             pseudonyms=self.pseudonyms,
         )
 
-    @property
+    @cached_property
     def sources(self) -> List["ConceptRef"]:
         if self.lineage:
             output: List[ConceptRef] = []
@@ -1426,7 +1430,7 @@ def get_basic_type(
     return type
 
 
-class CaseWhen(Namespaced, ConceptArgs, Mergeable, BaseModel):
+class CaseWhen(Namespaced, DataTyped, ConceptArgs, Mergeable, BaseModel):
     comparison: Conditional | SubselectComparison | Comparison
     expr: "Expr"
 
@@ -1436,6 +1440,10 @@ class CaseWhen(Namespaced, ConceptArgs, Mergeable, BaseModel):
             return v.reference
         return v
 
+    @property
+    def output_datatype(self):
+        return arg_to_datatype(self.expr)
+
     def __str__(self):
         return self.__repr__()
 
@@ -1488,7 +1496,7 @@ class CaseWhen(Namespaced, ConceptArgs, Mergeable, BaseModel):
         )
 
 
-class CaseElse(Namespaced, ConceptArgs, Mergeable, BaseModel):
+class CaseElse(Namespaced, ConceptArgs, DataTyped, Mergeable, BaseModel):
     expr: "Expr"
     # this ensures that it's easily differentiable from CaseWhen
     discriminant: ComparisonOperator = ComparisonOperator.ELSE
@@ -1499,6 +1507,10 @@ class CaseElse(Namespaced, ConceptArgs, Mergeable, BaseModel):
     def __repr__(self):
         return f"ELSE {str(self.expr)}"
 
+    @property
+    def output_datatype(self):
+        return arg_to_datatype(self.expr)
+
     @field_validator("expr", mode="before")
     def enforce_expr(cls, v):
         if isinstance(v, Concept):
@@ -1837,7 +1849,7 @@ class FunctionCallWrapper(
 
 class AggregateWrapper(Mergeable, DataTyped, ConceptArgs, Namespaced, BaseModel):
     function: Function
-    by: List[ConceptRef] = Field(default_factory=list)
+    by: List[ConceptRef | Concept] = Field(default_factory=list)
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
@@ -1863,7 +1875,7 @@ class AggregateWrapper(Mergeable, DataTyped, ConceptArgs, Namespaced, BaseModel)
 
     @property
     def concept_arguments(self) -> List[ConceptRef]:
-        return self.function.concept_arguments + self.by
+        return self.function.concept_arguments + [x.reference for x in self.by]
 
     @property
     def output_datatype(self):
@@ -2392,4 +2404,8 @@ FuncArgs = (
     | NumericType
     | list
     | ListWrapper[Any]
+    | TupleWrapper[Any]
+    | Comparison
+    | Conditional
+    | MagicConstants
 )

trilogy/core/models/build.py

@@ -1538,9 +1538,23 @@ class Factory:
 
     @build.register
     def _(self, base: CaseWhen) -> BuildCaseWhen:
+        from trilogy.parsing.common import arbitrary_to_concept
+
+        comparison = base.comparison
+        if isinstance(comparison, (AggregateWrapper, FilterItem, WindowItem)):
+            comparison = arbitrary_to_concept(
+                comparison,
+                environment=self.environment,
+            )
+        expr: Concept | FuncArgs = base.expr
+        if isinstance(expr, (AggregateWrapper, FilterItem, WindowItem)):
+            expr = arbitrary_to_concept(
+                expr,
+                environment=self.environment,
+            )
         return BuildCaseWhen.model_construct(
-            comparison=self.build(base.comparison),
-            expr=(self.build(base.expr)),
+            comparison=self.build(comparison),
+            expr=self.build(expr),
         )
 
     @build.register
@@ -1647,10 +1661,27 @@ class Factory:
     @build.register
     def _(self, base: WindowItem) -> BuildWindowItem:
         # to do proper discovery, we need to inject virtual intermediate ocncepts
+        from trilogy.parsing.common import arbitrary_to_concept
+
+        content: Concept | FuncArgs = base.content
+        if isinstance(content, (AggregateWrapper, FilterItem, WindowItem)):
+            content = arbitrary_to_concept(
+                content,
+                environment=self.environment,
+            )
+        final_by = []
+        for x in base.order_by:
+            if (
+                isinstance(x.expr, AggregateWrapper)
+                and not x.expr.by
+                and isinstance(content, (ConceptRef, Concept))
+            ):
+                x.expr.by = [content]
+            final_by.append(x)
         return BuildWindowItem.model_construct(
             type=base.type,
-            content=self.build(base.content),
-            order_by=[self.build(x) for x in base.order_by],
+            content=self.build(content),
+            order_by=[self.build(x) for x in final_by],
             over=[self.build(x) for x in base.over],
             index=base.index,
         )

trilogy/core/models/build_environment.py

@@ -31,19 +31,12 @@ class BuildEnvironmentConceptDict(dict):
     def raise_undefined(
         self, key: str, line_no: int | None = None, file: Path | str | None = None
     ) -> Never:
-
-        matches = self._find_similar_concepts(key)
-        message = f"Undefined concept: {key}."
-        if matches:
-            message += f" Suggestions: {matches}"
-
-        if line_no:
-            if file:
-                raise UndefinedConceptException(
-                    f"{file}: {line_no}: " + message, matches
-                )
-            raise UndefinedConceptException(f"line: {line_no}: " + message, matches)
-        raise UndefinedConceptException(message, matches)
+        # build environment should never check for missing values.
+        if line_no is not None:
+            message = f"Concept '{key}' not found in environment at line {line_no}."
+        else:
+            message = f"Concept '{key}' not found in environment."
+        raise UndefinedConceptException(message, [])
 
     def __getitem__(
         self, key: str, line_no: int | None = None, file: Path | None = None

trilogy/core/models/environment.py

@@ -686,7 +686,8 @@ class Environment(BaseModel):
                 replacements[k] = target
             # we need to update keys and grains of all concepts
             else:
-                replacements[k] = v.with_merge(source, target, modifiers)
+                if source.address in v.sources or source.address in v.grain.components:
+                    replacements[k] = v.with_merge(source, target, modifiers)
         self.concepts.update(replacements)
         for k, ds in self.datasources.items():
             if source.address in ds.output_lcl:

trilogy/core/models/execute.py

@@ -282,6 +282,7 @@ class CTE(BaseModel):
             **self.existence_source_map,
             **other.existence_source_map,
         }
+
         return self
 
     @property
@@ -764,8 +765,15 @@ class QueryDatasource(BaseModel):
     def identifier(self) -> str:
         filters = abs(hash(str(self.condition))) if self.condition else ""
         grain = "_".join([str(c).replace(".", "_") for c in self.grain.components])
+        group = ""
+        if self.source_type == SourceType.GROUP:
+            keys = [
+                x.address for x in self.output_concepts if x.purpose != Purpose.METRIC
+            ]
+            group = "_grouped_by_" + "_".join(keys)
         return (
             "_join_".join([d.identifier for d in self.datasources])
+            + group
             + (f"_at_{grain}" if grain else "_at_abstract")
             + (f"_filtered_by_{filters}" if filters else "")
         )

trilogy/core/processing/concept_strategies_v3.py

@@ -860,7 +860,7 @@ def _search_concepts(
             and priority_concept.address not in conditions.row_arguments
         ):
             logger.info(
-                f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Force including conditions to push filtering above complex condition that is not condition member or parent"
+                f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Force including conditions in {priority_concept.address} to push filtering above complex condition that is not condition member or parent"
             )
             local_conditions = conditions
 
@@ -936,7 +936,23 @@ def _search_concepts(
         if complete == ValidationResult.COMPLETE and (
             not accept_partial or (accept_partial and not partial)
         ):
+            logger.info(
+                f"{depth_to_prefix(depth)}{LOGGER_PREFIX} breaking loop, complete"
+            )
             break
+        elif complete == ValidationResult.COMPLETE and accept_partial and partial:
+            if len(attempted) == len(mandatory_list):
+                logger.info(
+                    f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Breaking as we have attempted all nodes"
+                )
+                break
+            logger.info(
+                f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Found complete stack with partials {partial}, continuing search, attempted {attempted} all {len(mandatory_list)}"
+            )
+        else:
+            logger.info(
+                f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Not complete, continuing search"
+            )
         # if we have attempted on root node, we've tried them all.
         # inject in another search with filter concepts
         if priority_concept.derivation == Derivation.ROOT:

trilogy/core/processing/node_generators/select_merge_node.py

@@ -344,12 +344,14 @@ def create_datasource_node(
         for c in datasource.columns
         if not c.is_complete and c.concept.address in all_concepts
     ]
+
     partial_lcl = LooseBuildConceptList(concepts=partial_concepts)
     nullable_concepts = [
         c.concept
         for c in datasource.columns
         if c.is_nullable and c.concept.address in all_concepts
     ]
+
     nullable_lcl = LooseBuildConceptList(concepts=nullable_concepts)
     partial_is_full = conditions and (conditions == datasource.non_partial_for)
 

trilogy/core/processing/utility.py

@@ -81,7 +81,7 @@ class JoinOrderOutput:
 
 
 def resolve_join_order_v2(
-    g: nx.Graph, partials: dict[str, list[str]]
+    g: nx.Graph, partials: dict[str, list[str]], nullables: dict[str, list[str]]
 ) -> list[JoinOrderOutput]:
     datasources = [x for x in g.nodes if x.startswith("ds~")]
     concepts = [x for x in g.nodes if x.startswith("c~")]
@@ -108,7 +108,7 @@ def resolve_join_order_v2(
             root = next_pivots[0]
             pivots = [x for x in pivots if x != root]
         else:
-            root = pivots.pop()
+            root = pivots.pop(0)
 
         # sort so less partials is last and eligible lefts are
         def score_key(x: str) -> tuple[int, int, str]:
@@ -119,6 +119,8 @@ def resolve_join_order_v2(
             # if it has the concept as a partial, lower weight
             if root in partials.get(x, []):
                 base -= 1
+            if root in nullables.get(x, []):
+                base -= 1
             return (base, len(x), x)
 
         # get remainig un-joined datasets
@@ -159,9 +161,11 @@ def resolve_join_order_v2(
                 )
                 right_is_partial = any(key in partials.get(right, []) for key in common)
                 # we don't care if left is nullable for join type (just keys), but if we did
-                # ex: left_is_nullable = any(key in partials.get(left_candidate, [])
+                # left_is_nullable = any(
+                #     key in nullables.get(left_candidate, []) for key in common
+                # )
                 right_is_nullable = any(
-                    key in partials.get(right, []) for key in common
+                    key in nullables.get(right, []) for key in common
                 )
                 if left_is_partial:
                     join_type = JoinType.FULL
@@ -356,6 +360,7 @@ def get_node_joins(
 ) -> List[BaseJoin]:
     graph = nx.Graph()
     partials: dict[str, list[str]] = {}
+    nullables: dict[str, list[str]] = {}
     ds_node_map: dict[str, QueryDatasource | BuildDatasource] = {}
     concept_map: dict[str, BuildConcept] = {}
     for datasource in datasources:
@@ -363,6 +368,7 @@ def get_node_joins(
         ds_node_map[ds_node] = datasource
         graph.add_node(ds_node, type=NodeType.NODE)
         partials[ds_node] = [f"c~{c.address}" for c in datasource.partial_concepts]
+        nullables[ds_node] = [f"c~{c.address}" for c in datasource.nullable_concepts]
         for concept in datasource.output_concepts:
             if concept.address in datasource.hidden_concepts:
                 continue
@@ -374,7 +380,7 @@ def get_node_joins(
                 environment=environment,
             )
 
-    joins = resolve_join_order_v2(graph, partials=partials)
+    joins = resolve_join_order_v2(graph, partials=partials, nullables=nullables)
     return [
         BaseJoin(
             left_datasource=ds_node_map[j.left] if j.left else None,

trilogy/core/statements/author.py

@@ -135,7 +135,6 @@ class SelectStatement(HasUUID, SelectTypeMixin, BaseModel):
         )
 
         output.grain = output.calculate_grain(environment)
-
         for x in selection:
             if x.is_undefined and environment.concepts.fail_on_missing:
                 environment.concepts.raise_undefined(
@@ -384,6 +383,11 @@ class MergeStatementV2(HasUUID, BaseModel):
     modifiers: List[Modifier] = Field(default_factory=list)
 
 
+class KeyMergeStatement(HasUUID, BaseModel):
+    keys: set[str]
+    target: ConceptRef
+
+
 class ImportStatement(HasUUID, BaseModel):
     # import abc.def as bar
     # the bit after 'as', eg bar

trilogy/parsing/common.py

@@ -270,7 +270,7 @@ def _get_relevant_parent_concepts(arg) -> tuple[list[ConceptRef], bool]:
     elif isinstance(arg, AggregateWrapper) and not arg.by:
         return [], True
     elif isinstance(arg, AggregateWrapper) and arg.by:
-        return arg.by, True
+        return [x.reference for x in arg.by], True
     elif isinstance(arg, FunctionCallWrapper):
         return get_relevant_parent_concepts(arg.content)
     return get_concept_arguments(arg), False

trilogy/parsing/parse_engine.py

@@ -116,6 +116,7 @@ from trilogy.core.statements.author import (
     CopyStatement,
     FunctionDeclaration,
     ImportStatement,
+    KeyMergeStatement,
     Limit,
     MergeStatementV2,
     MultiSelectStatement,
@@ -890,6 +891,29 @@ class ParseToObjects(Transformer):
     def over_list(self, args):
         return [x for x in args]
 
+    @v_args(meta=True)
+    def key_merge_statement(self, meta: Meta, args) -> KeyMergeStatement | None:
+        key_inputs = args[:-1]
+        target = args[-1]
+        keys = [self.environment.concepts[grain] for grain in key_inputs]
+        target_c = self.environment.concepts[target]
+        new = KeyMergeStatement(
+            keys=set([x.address for x in keys]),
+            target=target_c.reference,
+        )
+        internal = Concept(
+            name="_" + target_c.address.replace(".", "_"),
+            namespace=self.environment.namespace,
+            purpose=Purpose.PROPERTY,
+            keys=set([x.address for x in keys]),
+            datatype=target_c.datatype,
+            grain=Grain(components={x.address for x in keys}),
+        )
+        self.environment.add_concept(internal)
+        # always a full merge
+        self.environment.merge_concept(target_c, internal, [])
+        return new
+
     @v_args(meta=True)
     def merge_statement(self, meta: Meta, args) -> MergeStatementV2 | None:
         modifiers = []

trilogy/parsing/render.py

@@ -51,6 +51,7 @@ from trilogy.core.statements.author import (
     CopyStatement,
     FunctionDeclaration,
     ImportStatement,
+    KeyMergeStatement,
     MergeStatementV2,
     MultiSelectStatement,
     PersistStatement,
@@ -525,6 +526,11 @@ class Renderer:
             return f"MERGE {self.to_string(arg.sources[0])} into {''.join([self.to_string(modifier) for modifier in arg.modifiers])}{self.to_string(arg.targets[arg.sources[0].address])};"
         return f"MERGE {arg.source_wildcard}.* into {''.join([self.to_string(modifier) for modifier in arg.modifiers])}{arg.target_wildcard}.*;"
 
+    @to_string.register
+    def _(self, arg: KeyMergeStatement):
+        keys = ", ".join(sorted(list(arg.keys)))
+        return f"MERGE PROPERTY <{keys}> from {arg.target.address};"
+
     @to_string.register
     def _(self, arg: Modifier):
         if arg == Modifier.PARTIAL:

trilogy/parsing/trilogy.lark

@@ -10,6 +10,7 @@
     | rowset_derivation_statement
     | import_statement
     | copy_statement
+    | key_merge_statement
     | merge_statement
     | rawsql_statement
     
@@ -76,6 +77,8 @@
 
     align_clause: align_item ("AND"i align_item)*  "AND"i?
 
+    key_merge_statement: "merge"i "property"i "<" IDENTIFIER ("," IDENTIFIER )* ","? ">" "from"i IDENTIFIER 
+
     merge_statement: "merge"i  WILDCARD_IDENTIFIER "into"i SHORTHAND_MODIFIER? WILDCARD_IDENTIFIER
 
     // raw sql statement

trilogy/std/report.preql

@@ -0,0 +1,5 @@
+
+
+
+def top_x_by_metric(val, metric, x=10, default='other') -> case when rank val by metric desc <x+1 then val else default end; # Get the top X of a value by a certain metric, bucket the rest in other
+

trilogy/std/dashboard.preql

@@ -1,5 +0,0 @@
-
-
-
-# def top_x_by_metric(val, metric, x=10) -> case when rank val by metric order by metric desc <x then val else 'Other' end;
-