pytrilogy 0.0.3.99__py3-none-any.whl → 0.0.3.101__py3-none-any.whl

{pytrilogy-0.0.3.99.dist-info → pytrilogy-0.0.3.101.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.99
+Version: 0.0.3.101
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 
@@ -123,15 +123,12 @@ Versus SQL, Trilogy aims to:
 **Improve:**
 - Simplicity
 - Refactoring/maintainability
-- Reusability
+- Reusability/composability
+- Expressivness
 
 **Maintain:**
 - Acceptable performance
 
-**Remove:**
-- Lower-level procedural features
-- Transactional optimizations/non-analytics features
-
 ## Backend Support
 
 | Backend | Status | Notes |
@@ -161,6 +158,7 @@ property sentence_id.word_three string::word; # a description to it
 # comments in other places are just comments
 
 # define our datasource to bind the model to data
+# for most work, you can import something already defined
 # testing using query fixtures is a common pattern
 datasource word_one(
     sentence: sentence_id,
@@ -323,7 +321,7 @@ from pytrilogy import Executor, Dialect
 
 ### Authoring Imports
 
-Are also stable, and should be used for cases which programatically generate Trilogy statements without a base text format 
+Are also stable, and should be used for cases which programatically generate Trilogy statements without text inputs
 or need to process/transform parsed code in more complicated ways.
 
 ```python
@@ -391,6 +389,16 @@ datasource <name>(
 )
 grain(<concept>, <concept>)
 address <table>;
+
+datasource orders(
+    order_id,
+    order_date,
+    total_rev: point_of_sale_rev,
+    customomer_id: customer.id
+)
+grain orders
+address orders;
+
 ```
 
 ### Queries

{pytrilogy-0.0.3.99.dist-info → pytrilogy-0.0.3.101.dist-info}/RECORD

@@ -1,6 +1,6 @@
-pytrilogy-0.0.3.99.dist-info/licenses/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
-trilogy/__init__.py,sha256=_mu-EP3oMYeyuDb0ESiaTOvq4vYDFOgPukUj0Nk4pwI,303
-trilogy/constants.py,sha256=SSsRMg9HTou259nMKAw-rJNBgzkWjQ3QIQXcrq9i5Kk,1717
+pytrilogy-0.0.3.101.dist-info/licenses/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
+trilogy/__init__.py,sha256=NMiEE_jE99ZiREk8IPjfT2M-jxAwtmd2vyCWVD3kT28,304
+trilogy/constants.py,sha256=ohmro6so7PPNp2ruWQKVc0ijjXYPOyRrxB9LI8dr3TU,1746
 trilogy/engine.py,sha256=3MiADf5MKcmxqiHBuRqiYdsXiLj7oitDfVvXvHrfjkA,2178
 trilogy/executor.py,sha256=KgCAQhHPT-j0rPkBbALX0f84W9-Q-bkjHayGuavg99w,16490
 trilogy/parser.py,sha256=o4cfk3j3yhUFoiDKq9ZX_GjBF3dKhDjXEwb63rcBkBM,293
@@ -14,8 +14,8 @@ trilogy/core/enums.py,sha256=H8I2Dz4POHZ4ixYCGzNs4c3KDqxLQklGLVfmje1DSMo,8877
 trilogy/core/env_processor.py,sha256=H-rr2ALj31l5oh3FqeI47Qju6OOfiXBacXNJGNZ92zQ,4521
 trilogy/core/environment_helpers.py,sha256=TRlqVctqIRBxzfjRBmpQsAVoiCcsEKBhG1B6PUE0l1M,12743
 trilogy/core/ergonomics.py,sha256=e-7gE29vPLFdg0_A1smQ7eOrUwKl5VYdxRSTddHweRA,1631
-trilogy/core/exceptions.py,sha256=fI16oTNCVMMAJFSn2AFzZVapzsF5M9WbdN5e5UixwXc,2807
-trilogy/core/functions.py,sha256=oY-F0hsA9vp1ZipGTyx4QVtz_x83Ekk-lkHv6mMkHVQ,33095
+trilogy/core/exceptions.py,sha256=axkVXYJYQXCCwMHwlyDA232g4tCOwdCZUt7eHeUMDMg,2829
+trilogy/core/functions.py,sha256=sdV6Z3NUVfwL1d18eNcaAXllVNqzLez23McsJ6xIp7M,33182
 trilogy/core/graph_models.py,sha256=4EWFTHGfYd72zvS2HYoV6hm7nMC_VEd7vWr6txY-ig0,3400
 trilogy/core/internal.py,sha256=r9QagDB2GvpqlyD_I7VrsfbVfIk5mnok2znEbv72Aa4,2681
 trilogy/core/optimization.py,sha256=ojpn-p79lr03SSVQbbw74iPCyoYpDYBmj1dbZ3oXCjI,8860
@@ -25,48 +25,47 @@ trilogy/core/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSu
 trilogy/core/models/author.py,sha256=ZSKEJ6Vg4otpI_m7_JuGyrFM8dZV1HaxBwprvDSwUzo,81149
 trilogy/core/models/build.py,sha256=ZwJJGyp4rVsISvL8Er_AxQdVJrafYc4fesSj4MNgoxU,70615
 trilogy/core/models/build_environment.py,sha256=mpx7MKGc60fnZLVdeLi2YSREy7eQbQYycCrP4zF-rHU,5258
-trilogy/core/models/core.py,sha256=EofJ8-kltNr_7oFhyCPqauVX1bSJzJI5xOp0eMP_vlA,12892
+trilogy/core/models/core.py,sha256=iT9WdZoiXeglmUHWn6bZyXCTBpkApTGPKtNm_Mhbu_g,12987
 trilogy/core/models/datasource.py,sha256=wogTevZ-9CyUW2a8gjzqMCieircxi-J5lkI7EOAZnck,9596
 trilogy/core/models/environment.py,sha256=hwTIRnJgaHUdCYof7U5A9NPitGZ2s9yxqiW5O2SaJ9Y,28759
-trilogy/core/models/execute.py,sha256=lQTpiuNhBT4In-oQ76ImgIoTdUbs4mmyd0J0iTOZOdw,42105
+trilogy/core/models/execute.py,sha256=lsNzNjS3nZvoW5CHjYwxDTwBe502NZyytpK1eq8CwW4,42357
 trilogy/core/optimizations/__init__.py,sha256=YH2-mGXZnVDnBcWVi8vTbrdw7Qs5TivG4h38rH3js_I,290
 trilogy/core/optimizations/base_optimization.py,sha256=gzDOKImoFn36k7XBD3ysEYDnbnb6vdVIztUfFQZsGnM,513
 trilogy/core/optimizations/inline_datasource.py,sha256=2sWNRpoRInnTgo9wExVT_r9RfLAQHI57reEV5cGHUcg,4329
 trilogy/core/optimizations/predicate_pushdown.py,sha256=g4AYE8Aw_iMlAh68TjNXGP754NTurrDduFECkUjoBnc,9399
 trilogy/core/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/core/processing/concept_strategies_v3.py,sha256=Izo7yfR6sGHTaD17lN7ZzGYCtXA5AkXAmIp_OBjHH58,23161
-trilogy/core/processing/discovery_loop.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+trilogy/core/processing/concept_strategies_v3.py,sha256=tvSN_aiqb1H7LkTl96vj7YK_DKcq_1nDdRJ69wZCLc8,22158
 trilogy/core/processing/discovery_node_factory.py,sha256=5QVYUsci_h6iYWhS0GCoDow2tSAipiBW1OyTRX-g_L8,15581
-trilogy/core/processing/discovery_utility.py,sha256=eY4n7_r6_R-cx_Sm8FiouMXh78v2iO2SGhi0aI5jvDg,4549
+trilogy/core/processing/discovery_utility.py,sha256=Xntgug6VnEF96uw5Zwen1qMEUwKjqrm_ZDUr4i4tc1U,5595
 trilogy/core/processing/discovery_validation.py,sha256=eZ4HfHMpqZLI8MGG2jez8arS8THs6ceuVrQFIY6gXrU,5364
 trilogy/core/processing/graph_utils.py,sha256=8QUVrkE9j-9C1AyrCb1nQEh8daCe0u1HuXl-Te85lag,1205
 trilogy/core/processing/utility.py,sha256=PGQKZgX58kI3gG4nprY8HRGayc2D8fF5RmbvBhQj8ok,23319
 trilogy/core/processing/node_generators/__init__.py,sha256=iVJ-crowPxYeut-hFjyEjfibKIDq7PfB4LEuDAUCjGY,943
-trilogy/core/processing/node_generators/basic_node.py,sha256=TLZCv4WS196a-0g5xgKuJGthnGP8Ugm46iz85_3NIY4,5626
+trilogy/core/processing/node_generators/basic_node.py,sha256=0Uhnf07056SBbRkt-wYLw4DZqsFR6jztGLUaE9ebPZs,5577
 trilogy/core/processing/node_generators/common.py,sha256=PdysdroW9DUADP7f5Wv_GKPUyCTROZV1g3L45fawxi8,9443
 trilogy/core/processing/node_generators/constant_node.py,sha256=LfpDq2WrBRZ3tGsLxw77LuigKfhbteWWh9L8BGdMGwk,1146
 trilogy/core/processing/node_generators/filter_node.py,sha256=ArBsQJl-4fWBJWCE28CRQ7UT7ErnFfbcseoQQZrBodY,11220
-trilogy/core/processing/node_generators/group_node.py,sha256=8HJ1lkOvIXfX3xoS2IMbM_wCu_mT0J_hQ7xnTaxsVlo,6611
+trilogy/core/processing/node_generators/group_node.py,sha256=yqOWl5TCV4PrdJua4OJkPUIHkljaLoSW2Y8eRAmVddQ,6733
 trilogy/core/processing/node_generators/group_to_node.py,sha256=jKcNCDOY6fNblrdZwaRU0sbUSr9H0moQbAxrGgX6iGA,3832
 trilogy/core/processing/node_generators/multiselect_node.py,sha256=GWV5yLmKTe1yyPhN60RG1Rnrn4ktfn9lYYXi_FVU4UI,7061
-trilogy/core/processing/node_generators/node_merge_node.py,sha256=DfivU4xAo3iOFvuZxmztlD5AlusM_Nq0IOOgvPKMH_M,22964
+trilogy/core/processing/node_generators/node_merge_node.py,sha256=1joMV7XpQ9Gpe-d5y7JUMBHIqakV5wFJi3Mtvs4UcL4,23415
 trilogy/core/processing/node_generators/recursive_node.py,sha256=l5zdh0dURKwmAy8kK4OpMtZfyUEQRk6N-PwSWIyBpSM,2468
 trilogy/core/processing/node_generators/rowset_node.py,sha256=5L5u6xz1In8EaHQdcYgR2si-tz9WB9YLXURo4AkUT9A,6630
 trilogy/core/processing/node_generators/select_merge_node.py,sha256=KQvGoNT5ZBWQ_caEomRTtG1PKZC7OPT4PKfY0QmwMGE,22270
 trilogy/core/processing/node_generators/select_node.py,sha256=Ta1G39V94gjX_AgyZDz9OqnwLz4BjY3D6Drx9YpziMQ,3555
 trilogy/core/processing/node_generators/synonym_node.py,sha256=AnAsa_Wj50NJ_IK0HSgab_7klYmKVrv0WI1uUe-GvEY,3766
 trilogy/core/processing/node_generators/union_node.py,sha256=VNo6Oey4p8etU9xrOh2oTT2lIOTvY6PULUPRvVa2uxU,2877
-trilogy/core/processing/node_generators/unnest_node.py,sha256=ueOQtoTf2iJHO09RzWHDFQ5iKZq2fVhGf2KAF2U2kU8,2677
+trilogy/core/processing/node_generators/unnest_node.py,sha256=w9vhPzASz53QPASLqFcLDdR9eY132tgVUcp3QolD5Jw,3726
 trilogy/core/processing/node_generators/window_node.py,sha256=A90linr4pkZtTNfn9k2YNLqrJ_SFII3lbHxB-BC6mI8,6688
 trilogy/core/processing/node_generators/select_helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/processing/node_generators/select_helpers/datasource_injection.py,sha256=m2YQ4OmG0N2O61a7NEq1ZzbTa7JsCC00lxB2ymjcYRI,8224
 trilogy/core/processing/nodes/__init__.py,sha256=zTge1EzwzEydlcMliIFO_TT7h7lS8l37lyZuQDir1h0,5487
 trilogy/core/processing/nodes/base_node.py,sha256=C_CjlOzlGMXckyV0b_PJZerpopNesRCKfambMq7Asvc,18221
 trilogy/core/processing/nodes/filter_node.py,sha256=5VtRfKbCORx0dV-vQfgy3gOEkmmscL9f31ExvlODwvY,2461
-trilogy/core/processing/nodes/group_node.py,sha256=ZJ9LRF1sDOyel5v0MHXHORn6DYdn771nX0-KdHdt3-4,10517
-trilogy/core/processing/nodes/merge_node.py,sha256=02oWRca0ba41U6PSAB14jwnWWxoyrvxRPLwkli259SY,15865
+trilogy/core/processing/nodes/group_node.py,sha256=njz-5T7OJ3-kaBC7EhdtPra3G77HnI7apjUwMGhUeXo,10569
+trilogy/core/processing/nodes/merge_node.py,sha256=daJywBxh44Gqk-7eTiXbYtY7xo6O6fNvqX-DagTOTmE,16231
 trilogy/core/processing/nodes/recursive_node.py,sha256=k0rizxR8KE64ievfHx_GPfQmU8QAP118Laeyq5BLUOk,1526
-trilogy/core/processing/nodes/select_node_v2.py,sha256=Xyfq8lU7rP7JTAd8VV0ATDNal64n4xIBgWQsOuMe_Ak,8824
+trilogy/core/processing/nodes/select_node_v2.py,sha256=IWyKyNgFlV8A2S1FUTPdIaogg6PzaHh-HmQo6v24sbg,8862
 trilogy/core/processing/nodes/union_node.py,sha256=hLAXXVWqEgMWi7dlgSHfCF59fon64av14-uPgJzoKzM,1870
 trilogy/core/processing/nodes/unnest_node.py,sha256=oLKMMNMx6PLDPlt2V5neFMFrFWxET8r6XZElAhSNkO0,2181
 trilogy/core/processing/nodes/window_node.py,sha256=JXJ0iVRlSEM2IBr1TANym2RaUf_p5E_l2sNykRzXWDo,1710
@@ -78,11 +77,11 @@ trilogy/core/statements/execute.py,sha256=kiwJcVeMa4wZR-xLfM2oYOJ9DeyJkP8An38WFy
 trilogy/core/validation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/validation/common.py,sha256=Sd-towAX1uSDe3dK51FcVtIwVrMhayEwdHqhzeJHro0,4776
 trilogy/core/validation/concept.py,sha256=PM2BxBxLvuBScSWZMPsDZVcOblDil5pNT0pHLcLhdPA,5242
-trilogy/core/validation/datasource.py,sha256=d9AQNcukIRgN2spItPsXFiNtlZva-lDnfei3i06yQCE,6489
+trilogy/core/validation/datasource.py,sha256=nJeEFyb6iMBwlEVdYVy1vLzAbdRZwOsUjGxgWKgY8oM,7636
 trilogy/core/validation/environment.py,sha256=ymvhQyt7jLK641JAAIQkqjQaAmr9C5022ILzYvDgPP0,2835
 trilogy/core/validation/fix.py,sha256=Z818UFNLxndMTLiyhB3doLxIfnOZ-16QGvVFWuD7UsA,3750
 trilogy/dialect/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/dialect/base.py,sha256=0QVHv4F0t3_gRQrZ0woFoUNKu7vaXGo-BG1l47CZUKc,49698
+trilogy/dialect/base.py,sha256=d2gXfa5Jh3uyN9H9MxG53JT-xQQgntq2X7EprobJYUc,49698
 trilogy/dialect/bigquery.py,sha256=XS3hpybeowgfrOrkycAigAF3NX2YUzTzfgE6f__2fT4,4316
 trilogy/dialect/common.py,sha256=_MarnMWRBn3VcNt3k5VUdFrwH6oHzGdNQquSpHNLq4o,5644
 trilogy/dialect/config.py,sha256=olnyeVU5W5T6b9-dMeNAnvxuPlyc2uefb7FRME094Ec,3834
@@ -105,8 +104,8 @@ trilogy/parsing/config.py,sha256=Z-DaefdKhPDmSXLgg5V4pebhSB0h590vI0_VtHnlukI,111
 trilogy/parsing/exceptions.py,sha256=Xwwsv2C9kSNv2q-HrrKC1f60JNHShXcCMzstTSEbiCw,154
 trilogy/parsing/helpers.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 trilogy/parsing/parse_engine.py,sha256=2k1TvnBYE_CW5zCmNfVbf1aWBuMDm5Wz4QfKKgGnE5k,81824
-trilogy/parsing/render.py,sha256=tqB3GlGk3bX6AbkJjvADad2QH6n63nw1kgrpjzLX2tI,20520
-trilogy/parsing/trilogy.lark,sha256=rM4WleeyGhoRgU-FOGcaeHOzZcYVxN4f13e_3B4OeLQ,16389
+trilogy/parsing/render.py,sha256=E8-R0zO40QoeTeVX9OYdi5e9YgRYtuRrezDRj7VOgds,20614
+trilogy/parsing/trilogy.lark,sha256=2-jguxgJQnNLbODjTijqrXXzFZ_UlivTdiYhec2YWuc,16451
 trilogy/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/scripts/trilogy.py,sha256=1L0XrH4mVHRt1C9T1HnaDv2_kYEfbWTb5_-cBBke79w,3774
 trilogy/std/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -118,8 +117,8 @@ trilogy/std/money.preql,sha256=XWwvAV3WxBsHX9zfptoYRnBigcfYwrYtBHXTME0xJuQ,2082
 trilogy/std/net.preql,sha256=WZCuvH87_rZntZiuGJMmBDMVKkdhTtxeHOkrXNwJ1EE,416
 trilogy/std/ranking.preql,sha256=LDoZrYyz4g3xsII9XwXfmstZD-_92i1Eox1UqkBIfi8,83
 trilogy/std/report.preql,sha256=LbV-XlHdfw0jgnQ8pV7acG95xrd1-p65fVpiIc-S7W4,202
-pytrilogy-0.0.3.99.dist-info/METADATA,sha256=Symb6C1gs-2By5qBIt51posuNDI3_en-b84YtlWKTiU,11683
-pytrilogy-0.0.3.99.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-pytrilogy-0.0.3.99.dist-info/entry_points.txt,sha256=ewBPU2vLnVexZVnB-NrVj-p3E-4vukg83Zk8A55Wp2w,56
-pytrilogy-0.0.3.99.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
-pytrilogy-0.0.3.99.dist-info/RECORD,,
+pytrilogy-0.0.3.101.dist-info/METADATA,sha256=dkvyYmeCXSZl2uHkPpoy-R7HdKb2w7pLGFrDu1tRGEU,11811
+pytrilogy-0.0.3.101.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+pytrilogy-0.0.3.101.dist-info/entry_points.txt,sha256=ewBPU2vLnVexZVnB-NrVj-p3E-4vukg83Zk8A55Wp2w,56
+pytrilogy-0.0.3.101.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
+pytrilogy-0.0.3.101.dist-info/RECORD,,

trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.dialect.enums import Dialects
 from trilogy.executor import Executor
 from trilogy.parser import parse
 
-__version__ = "0.0.3.99"
+__version__ = "0.0.3.101"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

trilogy/constants.py

@@ -40,6 +40,7 @@ class Comments:
     joins: bool = True
     nullable: bool = True
     partial: bool = True
+    source_map: bool = False
 
 
 @dataclass

trilogy/core/exceptions.py

@@ -69,7 +69,7 @@ class DatasourceColumnBindingData:
     actual_modifiers: List[Modifier]
 
     def format_failure(self):
-        return f"Concept {self.address} value '{self.value}' with type {self.value_modifiers} does not conform to expected type {str(self.actual_type)} with modifiers {self.actual_modifiers}"
+        return f"Concept {self.address} value '{self.value}' with type {self.value_type} and {self.value_modifiers} does not conform to expected type {str(self.actual_type)} with modifiers {self.actual_modifiers}"
 
     def is_modifier_issue(self) -> bool:
         return len(self.value_modifiers) > 0 and any(

trilogy/core/functions.py

@@ -18,6 +18,7 @@ from trilogy.core.models.author import (
     AggregateWrapper,
     Concept,
     ConceptRef,
+    Conditional,
     Function,
     Parenthetical,
     UndefinedConcept,
@@ -129,8 +130,8 @@ def validate_case_output(
 def create_struct_output(
     args: list[Any],
 ) -> StructType:
-    zipped = dict(zip(args[::2], args[1::2]))
-    types = [arg_to_datatype(x) for x in args[1::2]]
+    zipped = dict(zip(args[1::2], args[::2]))
+    types = [arg_to_datatype(x) for x in args[::2]]
     return StructType(fields=types, fields_map=zipped)
 
 
@@ -997,6 +998,8 @@ def argument_to_purpose(arg) -> Purpose:
         return argument_to_purpose(arg.content)
     elif isinstance(arg, WindowItem):
         return Purpose.PROPERTY
+    elif isinstance(arg, Conditional):
+        return Purpose.PROPERTY
     elif isinstance(arg, Concept):
         base = arg.purpose
         if (

trilogy/core/models/core.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 from abc import ABC
 from collections import UserDict, UserList
 from datetime import date, datetime
+from decimal import Decimal
 from enum import Enum
 from typing import (
     Any,
@@ -448,6 +449,8 @@ def arg_to_datatype(arg) -> CONCRETE_TYPES:
         return DataType.STRING
     elif isinstance(arg, float):
         return DataType.FLOAT
+    elif isinstance(arg, Decimal):
+        return DataType.NUMERIC
     elif isinstance(arg, DataType):
         return arg
     elif isinstance(arg, NumericType):

trilogy/core/models/execute.py

@@ -118,19 +118,20 @@ class CTE(BaseModel):
         base += f" Source: {self.source.source_type}."
         if self.parent_ctes:
             base += f" References: {', '.join([x.name for x in self.parent_ctes])}."
-        if self.joins:
+        if self.joins and CONFIG.comments.joins:
             base += f"\n-- Joins: {', '.join([str(x) for x in self.joins])}."
-        if self.partial_concepts:
+        if self.partial_concepts and CONFIG.comments.partial:
             base += (
                 f"\n-- Partials: {', '.join([str(x) for x in self.partial_concepts])}."
             )
-        base += f"\n-- Source Map: {self.source_map}."
+        if CONFIG.comments.source_map:
+            base += f"\n-- Source Map: {self.source_map}."
         base += f"\n-- Output: {', '.join([str(x) for x in self.output_columns])}."
         if self.source.input_concepts:
             base += f"\n-- Inputs: {', '.join([str(x) for x in self.source.input_concepts])}."
         if self.hidden_concepts:
             base += f"\n-- Hidden: {', '.join([str(x) for x in self.hidden_concepts])}."
-        if self.nullable_concepts:
+        if self.nullable_concepts and CONFIG.comments.nullable:
             base += (
                 f"\n-- Nullable: {', '.join([str(x) for x in self.nullable_concepts])}."
             )
@@ -368,6 +369,7 @@ class CTE(BaseModel):
     @property
     def group_concepts(self) -> List[BuildConcept]:
         def check_is_not_in_group(c: BuildConcept):
+
             if len(self.source_map.get(c.address, [])) > 0:
                 return False
             if c.derivation == Derivation.ROWSET:
@@ -381,8 +383,6 @@ class CTE(BaseModel):
                 and c.lineage.operator in FunctionClass.AGGREGATE_FUNCTIONS.value
             ):
                 return True
-            if c.purpose == Purpose.METRIC:
-                return True
 
             if c.derivation == Derivation.BASIC and c.lineage:
                 if all([check_is_not_in_group(x) for x in c.lineage.concept_arguments]):
@@ -392,6 +392,10 @@ class CTE(BaseModel):
                     and c.lineage.operator == FunctionType.GROUP
                 ):
                     return check_is_not_in_group(c.lineage.concept_arguments[0])
+                return False
+            if c.purpose == Purpose.METRIC:
+                return True
+
             return False
 
         return (
@@ -707,6 +711,8 @@ class QueryDatasource(BaseModel):
             f" {[c.address for c in self.output_concepts]} concepts and"
             f" {other.name} with {[c.address for c in other.output_concepts]} concepts"
         )
+        logger.info(self.source_map)
+        logger.info(other.source_map)
 
         merged_datasources: dict[str, Union[BuildDatasource, "QueryDatasource"]] = {}
 
@@ -770,6 +776,7 @@ class QueryDatasource(BaseModel):
         logger.debug(
             f"[Query Datasource] merged with {[c.address for c in qds.output_concepts]} concepts"
         )
+        logger.debug(qds.source_map)
         return qds
 
     @property
@@ -777,7 +784,7 @@ class QueryDatasource(BaseModel):
         filters = abs(hash(str(self.condition))) if self.condition else ""
         grain = "_".join([str(c).replace(".", "_") for c in self.grain.components])
         group = ""
-        if self.source_type == SourceType.GROUP:
+        if self.group_required:
             keys = [
                 x.address for x in self.output_concepts if x.purpose != Purpose.METRIC
             ]

trilogy/core/processing/concept_strategies_v3.py

@@ -19,13 +19,13 @@ from trilogy.core.processing.discovery_utility import (
     LOGGER_PREFIX,
     depth_to_prefix,
     get_priority_concept,
+    group_if_required,
 )
 from trilogy.core.processing.discovery_validation import (
     ValidationResult,
     validate_stack,
 )
 from trilogy.core.processing.nodes import (
-    GroupNode,
     History,
     MergeNode,
     StrategyNode,
@@ -218,6 +218,7 @@ def initialize_loop_context(
                 f"{depth_to_prefix(depth)}{LOGGER_PREFIX} derived condition row inputs {[x.address for x in required_filters]} present in mandatory list, forcing condition evaluation at this level. "
             )
             mandatory_list = completion_mandatory
+            all_mandatory = set(c.address for c in completion_mandatory)
             must_evaluate_condition_on_this_level_not_push_down = True
         else:
             logger.info(
@@ -263,7 +264,7 @@ def evaluate_loop_conditions(
             ]
         ) and not any(
             [
-                x.derivation not in ROOT_DERIVATIONS
+                x.derivation not in ROOT_DERIVATIONS + [Derivation.BASIC]
                 for x in context.mandatory_list
                 if x.address not in context.conditions.row_arguments
             ]
@@ -282,7 +283,7 @@ def evaluate_loop_conditions(
     # to ensure filtering happens before something like a SUM
     if (
         context.conditions
-        and priority_concept.derivation not in ROOT_DERIVATIONS
+        and priority_concept.derivation not in ROOT_DERIVATIONS + [Derivation.BASIC]
         and priority_concept.address not in context.conditions.row_arguments
     ):
         logger.info(
@@ -419,26 +420,9 @@ def generate_loop_completion(context: LoopContext, virtual: set[str]) -> Strateg
         logger.info(
             f"{depth_to_prefix(context.depth)}{LOGGER_PREFIX} Conditions {context.conditions} were injected, checking if we need a group to restore grain"
         )
-
-        result = GroupNode.check_if_required(
-            downstream_concepts=output.usable_outputs,
-            parents=[output.resolve()],
-            environment=context.environment,
-            depth=context.depth,
+        return group_if_required(
+            output, context.original_mandatory, context.environment
         )
-        if result.required:
-            logger.info(
-                f"{depth_to_prefix(context.depth)}{LOGGER_PREFIX} Adding group node with outputs {[x.address for x in context.original_mandatory]}"
-            )
-            return GroupNode(
-                output_concepts=context.original_mandatory,
-                input_concepts=output.usable_outputs,
-                environment=context.environment,
-                parents=[output],
-                partial_concepts=output.partial_concepts,
-                preexisting_conditions=context.conditions.conditional,
-                depth=context.depth,
-            )
     return output
 
 
@@ -604,18 +588,4 @@ def source_query_concepts(
     logger.info(
         f"{depth_to_prefix(0)}{LOGGER_PREFIX} final concepts are {[x.address for x in final]}"
     )
-    if GroupNode.check_if_required(
-        downstream_concepts=final,
-        parents=[root.resolve()],
-        environment=environment,
-    ).required:
-        candidate: StrategyNode = GroupNode(
-            output_concepts=final,
-            input_concepts=final,
-            environment=environment,
-            parents=[root],
-            partial_concepts=root.partial_concepts,
-        )
-    else:
-        candidate = root
-    return candidate
+    return group_if_required(root, output_concepts, environment)

trilogy/core/processing/discovery_utility.py

@@ -6,6 +6,8 @@ from trilogy.core.models.build import (
     BuildConcept,
     BuildRowsetItem,
 )
+from trilogy.core.models.build_environment import BuildEnvironment
+from trilogy.core.processing.nodes import GroupNode, MergeNode, StrategyNode
 
 
 def depth_to_prefix(depth: int) -> str:
@@ -15,6 +17,34 @@ def depth_to_prefix(depth: int) -> str:
 LOGGER_PREFIX = "[DISCOVERY LOOP]"
 
 
+def group_if_required(
+    root: StrategyNode, final: List[BuildConcept], environment: BuildEnvironment
+):
+    if isinstance(root, MergeNode) and root.force_group is True:
+        return root
+    elif isinstance(root, GroupNode):
+        return root
+    elif GroupNode.check_if_required(
+        downstream_concepts=final,
+        parents=[root.resolve()],
+        environment=environment,
+    ).required:
+        if isinstance(root, MergeNode):
+            root.force_group = True
+            root.set_output_concepts(final, rebuild=False)
+            root.rebuild_cache()
+            return root
+        return GroupNode(
+            output_concepts=final,
+            input_concepts=final,
+            environment=environment,
+            parents=[root],
+            partial_concepts=root.partial_concepts,
+            preexisting_conditions=root.preexisting_conditions,
+        )
+    return root
+
+
 def get_upstream_concepts(base: BuildConcept, nested: bool = False) -> set[str]:
     upstream = set()
     if nested:

trilogy/core/processing/node_generators/basic_node.py

@@ -120,8 +120,8 @@ def gen_basic_node(
             f"{depth_prefix}{LOGGER_PREFIX} No basic node could be generated for {concept}"
         )
         return None
-
-    parent_node.source_type = SourceType.BASIC
+    if parent_node.source_type != SourceType.CONSTANT:
+        parent_node.source_type = SourceType.BASIC
     parent_node.add_output_concept(concept)
     for x in equivalent_optional:
         parent_node.add_output_concept(x)
@@ -129,24 +129,18 @@ def gen_basic_node(
     logger.info(
         f"{depth_prefix}{LOGGER_PREFIX} Returning basic select for {concept}: output {[x.address for x in parent_node.output_concepts]}"
     )
+    # if it's a constant, don't prune outputs
+    if parent_node.source_type == SourceType.CONSTANT:
+        return parent_node
     targets = [concept] + local_optional + equivalent_optional
-    should_hide = [
-        x
-        for x in parent_node.output_concepts
-        if (
-            x.address not in targets
-            and not any(x.address in y.pseudonyms for y in targets)
-        )
-    ]
-    parent_node.hide_output_concepts(should_hide)
-    should_not_hide = [
-        x
-        for x in parent_node.output_concepts
-        if x.address in targets or any(x.address in y.pseudonyms for y in targets)
-    ]
-    parent_node.unhide_output_concepts(should_not_hide)
+    targets = [
+        s
+        for s in parent_node.output_concepts
+        if any(s.address in y.pseudonyms for y in targets)
+    ] + targets
+    parent_node.set_output_concepts(targets)
 
     logger.info(
-        f"{depth_prefix}{LOGGER_PREFIX} Returning basic select for {concept}: output {[x.address for x in parent_node.output_concepts]} hidden {[x for x in parent_node.hidden_concepts]}"
+        f"{depth_prefix}{LOGGER_PREFIX} Returning basic select for {concept}: input: {[x.address for x in parent_node.input_concepts]} output {[x.address for x in parent_node.output_concepts]} hidden {[x for x in parent_node.hidden_concepts]}"
     )
     return parent_node

trilogy/core/processing/node_generators/group_node.py

@@ -28,6 +28,7 @@ def get_aggregate_grain(
     parent_concepts: List[BuildConcept] = unique(
         resolve_function_parent_concepts(concept, environment=environment), "address"
     )
+
     if (
         concept.grain
         and len(concept.grain.components) > 0
@@ -108,13 +109,14 @@ def gen_group_node(
                         f"{padding(depth)}{LOGGER_PREFIX} cannot include optional agg {possible_agg.address}; it has mismatched parent grain {comp_grain } vs local parent {build_grain_parents}"
                     )
     if parent_concepts:
+        target_grain = BuildGrain.from_concepts(parent_concepts)
         logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} fetching group node parents {LooseBuildConceptList(concepts=parent_concepts)}"
+            f"{padding(depth)}{LOGGER_PREFIX} fetching group node parents {LooseBuildConceptList(concepts=parent_concepts)} with expected grain {target_grain}"
         )
         parent_concepts = unique(
             [x for x in parent_concepts if not x.name == ALL_ROWS_CONCEPT], "address"
         )
-        parent = source_concepts(
+        parent: StrategyNode | None = source_concepts(
             mandatory_list=parent_concepts,
             environment=environment,
             g=g,

trilogy/core/processing/node_generators/node_merge_node.py

@@ -20,6 +20,9 @@ from trilogy.core.models.build import (
     BuildWhereClause,
 )
 from trilogy.core.models.build_environment import BuildEnvironment
+from trilogy.core.processing.discovery_utility import (
+    group_if_required,
+)
 from trilogy.core.processing.nodes import History, MergeNode, StrategyNode
 from trilogy.core.processing.utility import padding
 from trilogy.utility import unique
@@ -161,9 +164,6 @@ def reinject_common_join_keys_v2(
             reduced = BuildGrain.from_concepts(concrete_concepts).components
             existing_addresses = set()
             for concrete in concrete_concepts:
-                logger.debug(
-                    f"looking at column {concrete.address} with pseudonyms {concrete.pseudonyms}"
-                )
                 cnode = concept_to_node(concrete.with_default_grain())
                 if cnode in final.nodes:
                     existing_addresses.add(concrete.address)
@@ -510,6 +510,7 @@ def subgraphs_to_merge_node(
     search_conditions: BuildWhereClause | None = None,
     enable_early_exit: bool = True,
 ):
+    target_grain = BuildGrain.from_concepts(output_concepts, environment=environment)
     parents: List[StrategyNode] = []
     logger.info(
         f"{padding(depth)}{LOGGER_PREFIX} fetching subgraphs {[[c.address for c in subgraph] for subgraph in concept_subgraphs]}"
@@ -549,16 +550,20 @@ def subgraphs_to_merge_node(
                 output_c.append(y)
 
     if len(parents) == 1 and enable_early_exit:
+
         logger.info(
             f"{padding(depth)}{LOGGER_PREFIX} only one parent node, exiting early w/ {[c.address for c in parents[0].output_concepts]}"
         )
-        return parents[0]
+        parent = parents[0]
+        return group_if_required(parent, output_concepts, environment)
+
     rval = MergeNode(
         input_concepts=unique(input_c, "address"),
-        output_concepts=output_c,
+        output_concepts=output_concepts,
         environment=environment,
         parents=parents,
         depth=depth,
+        grain=target_grain,
         # hidden_concepts=[]
         # conditions=conditions,
         # conditions=search_conditions.conditional,
@@ -579,6 +584,12 @@ def gen_merge_node(
     conditions: BuildConditional | None = None,
     search_conditions: BuildWhereClause | None = None,
 ) -> Optional[MergeNode]:
+
+    # we do not actually APPLY these conditions anywhere
+    # though we could look at doing that as an optimization
+    # it's important to include them so the base discovery loop that was generating
+    # the merge node can then add them automatically
+    # so we should not return a node with preexisting conditions
     if search_conditions:
         all_search_concepts = unique(
             all_concepts + list(search_conditions.row_arguments), "address"

trilogy/core/processing/node_generators/unnest_node.py

@@ -33,6 +33,8 @@ def gen_unnest_node(
     logger.info(
         f"{depth_prefix}{LOGGER_PREFIX} unnest node for {concept} with lineage {concept.lineage} has parents + optional {all_parents} and equivalent optional {equivalent_optional}"
     )
+    local_conditions = False
+    expected_outputs = [concept] + local_optional
     if arguments or local_optional:
         parent = source_concepts(
             mandatory_list=all_parents,
@@ -47,24 +49,49 @@ def gen_unnest_node(
                 f"{padding(depth)}{LOGGER_PREFIX} could not find unnest node parents"
             )
             return None
+    elif conditions:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} unnest node has no parents but conditions inputs {conditions.row_arguments} vs expected output {expected_outputs}"
+        )
+        if all([x.address in expected_outputs for x in conditions.row_arguments]):
+            local_conditions = True
+        else:
+            parent = source_concepts(
+                mandatory_list=conditions.conditional.row_arguments,
+                environment=environment,
+                g=g,
+                depth=depth + 1,
+                history=history,
+                conditions=conditions,
+            )
+            if not parent:
+                logger.info(
+                    f"{padding(depth)}{LOGGER_PREFIX} could not find unnest node condition inputs with no parents"
+                )
+                return None
+    else:
+        parent = None
 
     base = UnnestNode(
         unnest_concepts=[concept] + equivalent_optional,
         input_concepts=arguments + non_equivalent_optional,
         output_concepts=[concept] + local_optional,
         environment=environment,
-        parents=([parent] if (arguments or local_optional) else []),
+        parents=([parent] if parent else []),
     )
     # we need to sometimes nest an unnest node,
     # as unnest operations are not valid in all situations
     # TODO: inline this node when we can detect it's safe
+    conditional = conditions.conditional if conditions else None
     new = StrategyNode(
         input_concepts=base.output_concepts,
         output_concepts=base.output_concepts,
         environment=environment,
         parents=[base],
-        # conditions=conditions.conditional if conditions else None,
-        preexisting_conditions=conditions.conditional if conditions else None,
+        conditions=conditional if local_conditions is True else None,
+        preexisting_conditions=(
+            conditional if conditional and local_conditions is False else None
+        ),
     )
     qds = new.resolve()
     assert qds.source_map[concept.address] == {base.resolve()}

trilogy/core/processing/nodes/group_node.py

@@ -113,16 +113,18 @@ class GroupNode(StrategyNode):
             environment.concepts[c] for c in (comp_grain - target_grain).components
         ]
         logger.info(
-            f"{padding}{LOGGER_PREFIX} Group requirement check: {comp_grain}, {target_grain}, difference {difference}"
+            f"{padding}{LOGGER_PREFIX} Group requirement check: {comp_grain}, {target_grain}, difference {[x.address for x in difference]}"
         )
 
         # if the difference is all unique properties whose keys are in the source grain
         # we can also suppress the group
         if all(
             [
-                x.purpose == Purpose.UNIQUE_PROPERTY
-                and x.keys
-                and all(z in comp_grain.components for z in x.keys)
+                x.keys
+                and all(
+                    environment.concepts[z].address in comp_grain.components
+                    for z in x.keys
+                )
                 for x in difference
             ]
         ):

trilogy/core/processing/nodes/merge_node.py

@@ -308,7 +308,13 @@ class MergeNode(StrategyNode):
                     f"{self.logging_prefix}{LOGGER_PREFIX} skipping existence only source with {source.output_concepts} from grain accumulation"
                 )
                 continue
+            logger.info(
+                f"{self.logging_prefix}{LOGGER_PREFIX} adding source grain {source.grain} from source {source.identifier} to pregrain"
+            )
             pregrain += source.grain
+            logger.info(
+                f"{self.logging_prefix}{LOGGER_PREFIX} pregrain is now {pregrain}"
+            )
 
         pregrain = BuildGrain.from_concepts(
             pregrain.components, environment=self.environment
@@ -334,7 +340,9 @@ class MergeNode(StrategyNode):
             if isinstance(join, BaseJoin) and join.join_type == JoinType.FULL:
                 full_join_concepts += join.input_concepts
 
-        if self.whole_grain:
+        if self.force_group is True:
+            force_group = True
+        elif self.whole_grain:
             force_group = False
         elif self.force_group is False:
             force_group = False

trilogy/core/processing/nodes/select_node_v2.py

@@ -221,6 +221,7 @@ class SelectNode(StrategyNode):
 
 
 class ConstantNode(SelectNode):
+    source_type = SourceType.CONSTANT
     """Represents a constant value."""
 
     def copy(self) -> "ConstantNode":

trilogy/core/validation/datasource.py

@@ -36,31 +36,51 @@ def type_check(
 ) -> bool:
     if input is None and nullable:
         return True
+
     target_type = expected_type
     while isinstance(target_type, TraitDataType):
         return type_check(input, target_type.data_type, nullable)
+
     if target_type == DataType.STRING:
         return isinstance(input, str)
     if target_type == DataType.INTEGER:
         return isinstance(input, int)
+    if target_type == DataType.BIGINT:
+        return isinstance(input, int)  # or check for larger int if needed
     if target_type == DataType.FLOAT or isinstance(target_type, NumericType):
         return (
             isinstance(input, float)
             or isinstance(input, int)
             or isinstance(input, Decimal)
         )
+    if target_type == DataType.NUMBER:
+        return isinstance(input, (int, float, Decimal))
+    if target_type == DataType.NUMERIC:
+        return isinstance(input, (int, float, Decimal))
     if target_type == DataType.BOOL:
         return isinstance(input, bool)
     if target_type == DataType.DATE:
-        return isinstance(input, date)
+        return isinstance(input, date) and not isinstance(input, datetime)
     if target_type == DataType.DATETIME:
         return isinstance(input, datetime)
+    if target_type == DataType.TIMESTAMP:
+        return isinstance(input, datetime)  # or timestamp type if you have one
+    if target_type == DataType.UNIX_SECONDS:
+        return isinstance(input, (int, float))  # Unix timestamps are numeric
+    if target_type == DataType.DATE_PART:
+        return isinstance(
+            input, str
+        )  # assuming date parts are strings like "year", "month"
     if target_type == DataType.ARRAY or isinstance(target_type, ArrayType):
         return isinstance(input, list)
     if target_type == DataType.MAP or isinstance(target_type, MapType):
         return isinstance(input, dict)
     if target_type == DataType.STRUCT or isinstance(target_type, StructType):
         return isinstance(input, dict)
+    if target_type == DataType.NULL:
+        return input is None
+    if target_type == DataType.UNKNOWN:
+        return True
     return False
 
 
@@ -125,15 +145,19 @@ def validate_datasource(
             rval = row[actual_address]
             passed = type_check(rval, col.concept.datatype, col.is_nullable)
             if not passed:
+                value_type = (
+                    arg_to_datatype(rval) if rval is not None else col.concept.datatype
+                )
+                traits = None
+                if isinstance(col.concept.datatype, TraitDataType):
+                    traits = col.concept.datatype.traits
+                if traits and not isinstance(value_type, TraitDataType):
+                    value_type = TraitDataType(type=value_type, traits=traits)
                 failures.append(
                     DatasourceColumnBindingData(
                         address=col.concept.address,
                         value=rval,
-                        value_type=(
-                            arg_to_datatype(rval)
-                            if rval is not None
-                            else col.concept.datatype
-                        ),
+                        value_type=value_type,
                         value_modifiers=[Modifier.NULLABLE] if rval is None else [],
                         actual_type=col.concept.datatype,
                         actual_modifiers=col.concept.modifiers,

trilogy/dialect/base.py

@@ -163,7 +163,7 @@ def render_case(args):
 
 
 def struct_arg(args):
-    return [f"{x[0]}: {x[1]}" for x in zip(args[::2], args[1::2])]
+    return [f"{x[1]}: {x[0]}" for x in zip(args[::2], args[1::2])]
 
 
 FUNCTION_MAP = {

trilogy/parsing/render.py

@@ -349,7 +349,8 @@ class Renderer:
         else:
             output = f"{concept.purpose.value} {namespace}{concept.name} <- {self.to_string(concept.lineage)};"
         if base_description:
-            output += f" #{base_description}"
+            lines = "\n#".join(base_description.split("\n"))
+            output += f" #{lines}"
         return output
 
     @to_string.register
@@ -439,7 +440,7 @@ class Renderer:
 
     @to_string.register
     def _(self, arg: "Conditional"):
-        return f"({self.to_string(arg.left)} {arg.operator.value} {self.to_string(arg.right)})"
+        return f"{self.to_string(arg.left)} {arg.operator.value} {self.to_string(arg.right)}"
 
     @to_string.register
     def _(self, arg: "SubselectComparison"):
@@ -451,7 +452,8 @@ class Renderer:
 
     @to_string.register
     def _(self, arg: "Comment"):
-        return f"{arg.text}"
+        lines = "\n#".join(arg.text.split("\n"))
+        return f"{lines}"
 
     @to_string.register
     def _(self, arg: "WindowItem"):

trilogy/parsing/trilogy.lark

@@ -134,10 +134,12 @@
     metadata: "metadata" "(" IDENTIFIER "=" string_lit ")"
     
     limit: "LIMIT"i /[0-9]+/
+
+    _order_atom: expr ordering
     
-    order_list: expr ordering (","  expr ordering)* ","?
+    order_list: _order_atom (","  _order_atom)* ","?
     
-    over_list: concept_lit (","  concept_lit )* ","?
+    over_list: (concept_lit ",")*  concept_lit ","?
     
     ORDERING_DIRECTION: /ASC|DESC/i
 
@@ -433,7 +435,8 @@
     map_lit: "{" (literal ":" literal ",")* literal ":" literal ","? "}"
 
     _STRUCT.1: "struct("i
-    struct_lit: _STRUCT (IDENTIFIER "->" expr ",")* IDENTIFIER "->" expr ","? ")"
+    _BINDING.1: "->"
+    struct_lit: _STRUCT expr _BINDING IDENTIFIER   ( "," expr _BINDING  IDENTIFIER  )* ","? ")"
     
     !bool_lit: "True"i | "False"i