pytrilogy 0.0.3.103__py3-none-any.whl → 0.0.3.104__py3-none-any.whl

{pytrilogy-0.0.3.103.dist-info → pytrilogy-0.0.3.104.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.103
+Version: 0.0.3.104
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.3.103.dist-info → pytrilogy-0.0.3.104.dist-info}/RECORD

@@ -1,6 +1,6 @@
-pytrilogy-0.0.3.103.dist-info/licenses/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
-trilogy/__init__.py,sha256=lBanLP2CsDaPdJvJ3K68ncf8sO3sMfgsapECGG5J4fk,304
-trilogy/constants.py,sha256=ohmro6so7PPNp2ruWQKVc0ijjXYPOyRrxB9LI8dr3TU,1746
+pytrilogy-0.0.3.104.dist-info/licenses/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
+trilogy/__init__.py,sha256=HyZF9WId40s9G3BjFS2OBMeHI7XNeE-YU1cFLvNOSWk,304
+trilogy/constants.py,sha256=g_zkVCNjGop6coZ1kM8eXXAzCnUN22ldx3TYFz0E9sc,1747
 trilogy/engine.py,sha256=3MiADf5MKcmxqiHBuRqiYdsXiLj7oitDfVvXvHrfjkA,2178
 trilogy/executor.py,sha256=KgCAQhHPT-j0rPkBbALX0f84W9-Q-bkjHayGuavg99w,16490
 trilogy/parser.py,sha256=o4cfk3j3yhUFoiDKq9ZX_GjBF3dKhDjXEwb63rcBkBM,293
@@ -28,30 +28,30 @@ trilogy/core/models/build_environment.py,sha256=mpx7MKGc60fnZLVdeLi2YSREy7eQbQYy
 trilogy/core/models/core.py,sha256=iT9WdZoiXeglmUHWn6bZyXCTBpkApTGPKtNm_Mhbu_g,12987
 trilogy/core/models/datasource.py,sha256=wogTevZ-9CyUW2a8gjzqMCieircxi-J5lkI7EOAZnck,9596
 trilogy/core/models/environment.py,sha256=hwTIRnJgaHUdCYof7U5A9NPitGZ2s9yxqiW5O2SaJ9Y,28759
-trilogy/core/models/execute.py,sha256=lsNzNjS3nZvoW5CHjYwxDTwBe502NZyytpK1eq8CwW4,42357
+trilogy/core/models/execute.py,sha256=pdL3voYB4dCQR_KMHwFaofP3ZpRbALRC2ELHueWyTko,42191
 trilogy/core/optimizations/__init__.py,sha256=yspWc25M5SgAuvXYoSt5J8atyPbDlOfsKjIo5yGD9s4,368
 trilogy/core/optimizations/base_optimization.py,sha256=gzDOKImoFn36k7XBD3ysEYDnbnb6vdVIztUfFQZsGnM,513
 trilogy/core/optimizations/hide_unused_concept.py,sha256=DbsP8NqQOxmPv9omDOoFNPUGObUkqsRRNrr5d1xDxx4,1962
 trilogy/core/optimizations/inline_datasource.py,sha256=2sWNRpoRInnTgo9wExVT_r9RfLAQHI57reEV5cGHUcg,4329
-trilogy/core/optimizations/predicate_pushdown.py,sha256=g4AYE8Aw_iMlAh68TjNXGP754NTurrDduFECkUjoBnc,9399
+trilogy/core/optimizations/predicate_pushdown.py,sha256=5ubatgq1IwWQ4L2FDt4--y168YLuGP-vwqH0m8IeTIw,9786
 trilogy/core/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/core/processing/concept_strategies_v3.py,sha256=tvSN_aiqb1H7LkTl96vj7YK_DKcq_1nDdRJ69wZCLc8,22158
-trilogy/core/processing/discovery_node_factory.py,sha256=r1JAnVhnB9YHEB1TW3racNH9mJvXjKRPZjzZrXsuiqg,15348
-trilogy/core/processing/discovery_utility.py,sha256=Xntgug6VnEF96uw5Zwen1qMEUwKjqrm_ZDUr4i4tc1U,5595
+trilogy/core/processing/concept_strategies_v3.py,sha256=AcMU1d5uCo8I1PFCkBtmcC6iFmM9vN6xSdKxSVMGfpA,23080
+trilogy/core/processing/discovery_node_factory.py,sha256=p23jiiHyhrW-Q8ndbnRlqMHJKT8ZqPOA89SzE4xaFFo,15445
+trilogy/core/processing/discovery_utility.py,sha256=wIuLsE6yuVykeYZdIqRSagivDNU3-ooiS7z6in4yqho,11518
 trilogy/core/processing/discovery_validation.py,sha256=eZ4HfHMpqZLI8MGG2jez8arS8THs6ceuVrQFIY6gXrU,5364
 trilogy/core/processing/graph_utils.py,sha256=8QUVrkE9j-9C1AyrCb1nQEh8daCe0u1HuXl-Te85lag,1205
-trilogy/core/processing/utility.py,sha256=PGQKZgX58kI3gG4nprY8HRGayc2D8fF5RmbvBhQj8ok,23319
+trilogy/core/processing/utility.py,sha256=1_oNnk6lWiy-D7LKYr07kU_v7iAM4i6ITUAS4bIiCr4,23444
 trilogy/core/processing/node_generators/__init__.py,sha256=iVJ-crowPxYeut-hFjyEjfibKIDq7PfB4LEuDAUCjGY,943
-trilogy/core/processing/node_generators/basic_node.py,sha256=0Uhnf07056SBbRkt-wYLw4DZqsFR6jztGLUaE9ebPZs,5577
-trilogy/core/processing/node_generators/common.py,sha256=PdysdroW9DUADP7f5Wv_GKPUyCTROZV1g3L45fawxi8,9443
+trilogy/core/processing/node_generators/basic_node.py,sha256=74LoVZXLinRvSzk2LmI1kwza96TnuH3ELoYRIbHB29A,5578
+trilogy/core/processing/node_generators/common.py,sha256=xF32Kf6B08dZgKs2SOow1HomptSiSC057GCUCHFlS5s,9464
 trilogy/core/processing/node_generators/constant_node.py,sha256=LfpDq2WrBRZ3tGsLxw77LuigKfhbteWWh9L8BGdMGwk,1146
-trilogy/core/processing/node_generators/filter_node.py,sha256=ArBsQJl-4fWBJWCE28CRQ7UT7ErnFfbcseoQQZrBodY,11220
-trilogy/core/processing/node_generators/group_node.py,sha256=yqOWl5TCV4PrdJua4OJkPUIHkljaLoSW2Y8eRAmVddQ,6733
+trilogy/core/processing/node_generators/filter_node.py,sha256=ndPznkcFu_cdCNgaRpgot8oqnzdHv4KAIfjeUIzrE2w,10816
+trilogy/core/processing/node_generators/group_node.py,sha256=NdK1rl6Ze94XFWtgeC2dlRiL4pS3lh1ArKGPEltLtnw,8525
 trilogy/core/processing/node_generators/group_to_node.py,sha256=jKcNCDOY6fNblrdZwaRU0sbUSr9H0moQbAxrGgX6iGA,3832
-trilogy/core/processing/node_generators/multiselect_node.py,sha256=GWV5yLmKTe1yyPhN60RG1Rnrn4ktfn9lYYXi_FVU4UI,7061
-trilogy/core/processing/node_generators/node_merge_node.py,sha256=1joMV7XpQ9Gpe-d5y7JUMBHIqakV5wFJi3Mtvs4UcL4,23415
+trilogy/core/processing/node_generators/multiselect_node.py,sha256=a505AEixjsjp5jI8Ng3H5KF_AaehkS6HfRfTef64l_o,7063
+trilogy/core/processing/node_generators/node_merge_node.py,sha256=hNcZxnDLTZyYJWfojg769zH9HB9PfZfESmpN1lcHWXg,23172
 trilogy/core/processing/node_generators/recursive_node.py,sha256=l5zdh0dURKwmAy8kK4OpMtZfyUEQRk6N-PwSWIyBpSM,2468
-trilogy/core/processing/node_generators/rowset_node.py,sha256=T11Rqj-tsfubjFvBO0rzIVxtv9tOwwKXjGyut0r9xIY,5919
+trilogy/core/processing/node_generators/rowset_node.py,sha256=MuVNIexXhqGONho_mewqMOwaYXNUnjjvyPvk_RDGNYE,5943
 trilogy/core/processing/node_generators/select_merge_node.py,sha256=KQvGoNT5ZBWQ_caEomRTtG1PKZC7OPT4PKfY0QmwMGE,22270
 trilogy/core/processing/node_generators/select_node.py,sha256=Ta1G39V94gjX_AgyZDz9OqnwLz4BjY3D6Drx9YpziMQ,3555
 trilogy/core/processing/node_generators/synonym_node.py,sha256=AnAsa_Wj50NJ_IK0HSgab_7klYmKVrv0WI1uUe-GvEY,3766
@@ -61,10 +61,10 @@ trilogy/core/processing/node_generators/window_node.py,sha256=A90linr4pkZtTNfn9k
 trilogy/core/processing/node_generators/select_helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/processing/node_generators/select_helpers/datasource_injection.py,sha256=m2YQ4OmG0N2O61a7NEq1ZzbTa7JsCC00lxB2ymjcYRI,8224
 trilogy/core/processing/nodes/__init__.py,sha256=zTge1EzwzEydlcMliIFO_TT7h7lS8l37lyZuQDir1h0,5487
-trilogy/core/processing/nodes/base_node.py,sha256=TQZLEz_xfXpdVyFa9R5BwvikH1OqzJUioOPw8vTETWc,18144
+trilogy/core/processing/nodes/base_node.py,sha256=6LPQ5zP_dZJ6-k_dmX9ZSLsHaQMHgqiR5DEylpHYGZA,18478
 trilogy/core/processing/nodes/filter_node.py,sha256=5VtRfKbCORx0dV-vQfgy3gOEkmmscL9f31ExvlODwvY,2461
-trilogy/core/processing/nodes/group_node.py,sha256=njz-5T7OJ3-kaBC7EhdtPra3G77HnI7apjUwMGhUeXo,10569
-trilogy/core/processing/nodes/merge_node.py,sha256=daJywBxh44Gqk-7eTiXbYtY7xo6O6fNvqX-DagTOTmE,16231
+trilogy/core/processing/nodes/group_node.py,sha256=sKsRP_BWEKg6z63T1X5ZlkJF2IMif0IEbVWTk-cdOH8,7100
+trilogy/core/processing/nodes/merge_node.py,sha256=uc0tlz30Yt9SnCwLhMcWuPVbXLzm3dzy0XqbyirqqTo,16521
 trilogy/core/processing/nodes/recursive_node.py,sha256=k0rizxR8KE64ievfHx_GPfQmU8QAP118Laeyq5BLUOk,1526
 trilogy/core/processing/nodes/select_node_v2.py,sha256=IWyKyNgFlV8A2S1FUTPdIaogg6PzaHh-HmQo6v24sbg,8862
 trilogy/core/processing/nodes/union_node.py,sha256=hLAXXVWqEgMWi7dlgSHfCF59fon64av14-uPgJzoKzM,1870
@@ -82,7 +82,7 @@ trilogy/core/validation/datasource.py,sha256=nJeEFyb6iMBwlEVdYVy1vLzAbdRZwOsUjGx
 trilogy/core/validation/environment.py,sha256=ymvhQyt7jLK641JAAIQkqjQaAmr9C5022ILzYvDgPP0,2835
 trilogy/core/validation/fix.py,sha256=Z818UFNLxndMTLiyhB3doLxIfnOZ-16QGvVFWuD7UsA,3750
 trilogy/dialect/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/dialect/base.py,sha256=mgARj-aldkFAqdwps_25da03NLIAxU6Xg9Jq_VcOtp0,50181
+trilogy/dialect/base.py,sha256=hFX0_3N-m3ZRTCyv1S650a8OPlx9qjp5Zh8wzTBx6E8,50338
 trilogy/dialect/bigquery.py,sha256=XS3hpybeowgfrOrkycAigAF3NX2YUzTzfgE6f__2fT4,4316
 trilogy/dialect/common.py,sha256=cUI7JMmpG_A5KcaxRI-GoyqwLMD6jTf0JJhgcOdwQK4,5833
 trilogy/dialect/config.py,sha256=olnyeVU5W5T6b9-dMeNAnvxuPlyc2uefb7FRME094Ec,3834
@@ -110,16 +110,17 @@ trilogy/parsing/trilogy.lark,sha256=6eBDD6d4D9N1Nnn4CtmaoB-NpOpjHrEn5oi0JykAlbE,
 trilogy/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/scripts/trilogy.py,sha256=1L0XrH4mVHRt1C9T1HnaDv2_kYEfbWTb5_-cBBke79w,3774
 trilogy/std/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+trilogy/std/color.preql,sha256=sS9AXLDkECuDbNGnBMi2KnUuJukyVZVThKI9mP-ZOZI,50
 trilogy/std/date.preql,sha256=HWZm4t4HWyxr5geWRsY05RnHBVDMci8z8YA2cu0-OOw,188
-trilogy/std/display.preql,sha256=S20HW8qbShBc4OZPcHYiRlLdcaBp9dwruozWBoXKscs,293
+trilogy/std/display.preql,sha256=ZJ08crsZnC3kaWwNUrMB1ZH5j6DUUbz8RaUgihA8sm4,299
 trilogy/std/geography.preql,sha256=1A9Sq5PPMBnEPPf7f-rPVYxJfsnWpQ8oV_k4Fm3H2dU,675
 trilogy/std/metric.preql,sha256=DRECGhkMyqfit5Fl4Ut9zbWrJuSMI1iO9HikuyoBpE0,421
 trilogy/std/money.preql,sha256=XWwvAV3WxBsHX9zfptoYRnBigcfYwrYtBHXTME0xJuQ,2082
 trilogy/std/net.preql,sha256=WZCuvH87_rZntZiuGJMmBDMVKkdhTtxeHOkrXNwJ1EE,416
 trilogy/std/ranking.preql,sha256=LDoZrYyz4g3xsII9XwXfmstZD-_92i1Eox1UqkBIfi8,83
 trilogy/std/report.preql,sha256=LbV-XlHdfw0jgnQ8pV7acG95xrd1-p65fVpiIc-S7W4,202
-pytrilogy-0.0.3.103.dist-info/METADATA,sha256=RnMfz8EH2sCtqHEDAraYhAb_V7oPbovtuI3PsL2F3Ms,11839
-pytrilogy-0.0.3.103.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-pytrilogy-0.0.3.103.dist-info/entry_points.txt,sha256=ewBPU2vLnVexZVnB-NrVj-p3E-4vukg83Zk8A55Wp2w,56
-pytrilogy-0.0.3.103.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
-pytrilogy-0.0.3.103.dist-info/RECORD,,
+pytrilogy-0.0.3.104.dist-info/METADATA,sha256=IJmkrwnxe7gz3s89ZYVrDe6SkRY2cf6xNpmj5GTXkSE,11839
+pytrilogy-0.0.3.104.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+pytrilogy-0.0.3.104.dist-info/entry_points.txt,sha256=ewBPU2vLnVexZVnB-NrVj-p3E-4vukg83Zk8A55Wp2w,56
+pytrilogy-0.0.3.104.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
+pytrilogy-0.0.3.104.dist-info/RECORD,,

trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.dialect.enums import Dialects
 from trilogy.executor import Executor
 from trilogy.parser import parse
 
-__version__ = "0.0.3.103"
+__version__ = "0.0.3.104"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

trilogy/constants.py

@@ -37,7 +37,7 @@ class Comments:
 
     show: bool = False
     basic: bool = True
-    joins: bool = True
+    joins: bool = False
     nullable: bool = True
     partial: bool = True
     source_map: bool = False

trilogy/core/models/execute.py

@@ -711,8 +711,6 @@ class QueryDatasource(BaseModel):
             f" {[c.address for c in self.output_concepts]} concepts and"
             f" {other.name} with {[c.address for c in other.output_concepts]} concepts"
         )
-        logger.info(self.source_map)
-        logger.info(other.source_map)
 
         merged_datasources: dict[str, Union[BuildDatasource, "QueryDatasource"]] = {}
 
@@ -816,10 +814,7 @@ class QueryDatasource(BaseModel):
                     use_raw_name,
                     force_alias=force_alias,
                 )
-            except ValueError as e:
-                from trilogy.constants import logger
-
-                logger.debug(e)
+            except ValueError:
                 continue
         existing = [c.with_grain(self.grain) for c in self.output_concepts]
         if concept in existing:

trilogy/core/optimizations/predicate_pushdown.py

@@ -1,5 +1,6 @@
 from trilogy.core.enums import (
     BooleanOperator,
+    SourceType,
 )
 from trilogy.core.models.build import (
     BuildComparison,
@@ -59,12 +60,19 @@ class PredicatePushdown(OptimizationRule):
             )
             return False
         materialized = {k for k, v in parent_cte.source_map.items() if v != []}
+
         if not row_conditions or not materialized:
             return False
         output_addresses = {x.address for x in parent_cte.output_columns}
         # if any of the existence conditions are created on the asset, we can't push up to it
         if existence_conditions and existence_conditions.intersection(output_addresses):
             return False
+        if existence_conditions:
+            self.log(
+                f"Not pushing up existence {candidate} to {parent_cte.name} as it is a filter node"
+            )
+            if parent_cte.source.source_type == SourceType.FILTER:
+                return False
         # if it's a root datasource, we can filter on _any_ of the output concepts
         if parent_cte.is_root_datasource:
             extra_check = {
@@ -81,7 +89,7 @@ class PredicatePushdown(OptimizationRule):
             children = inverse_map.get(parent_cte.name, [])
             if all([is_child_of(candidate, child.condition) for child in children]):
                 self.log(
-                    f"All concepts are found on {parent_cte.name} with existing {parent_cte.condition} and all it's {len(children)} children include same filter; pushing up {candidate}"
+                    f"All concepts [{row_conditions}] and existence conditions [{existence_conditions}] not block pushup of [{output_addresses}]found on {parent_cte.name} with existing {parent_cte.condition} and all it's {len(children)} children include same filter; pushing up {candidate}"
                 )
                 if parent_cte.condition and not is_scalar_condition(
                     parent_cte.condition

trilogy/core/processing/concept_strategies_v3.py

@@ -19,7 +19,7 @@ from trilogy.core.processing.discovery_utility import (
     LOGGER_PREFIX,
     depth_to_prefix,
     get_priority_concept,
-    group_if_required,
+    group_if_required_v2,
 )
 from trilogy.core.processing.discovery_validation import (
     ValidationResult,
@@ -66,7 +66,19 @@ def generate_candidates_restrictive(
 
     # if it's single row, joins are irrelevant. Fetch without keys.
     if priority_concept.granularity == Granularity.SINGLE_ROW:
-        return [], conditions
+        logger.info("Have single row concept, including only other single row optional")
+        optional = (
+            [
+                x
+                for x in candidates
+                if x.granularity == Granularity.SINGLE_ROW
+                and x.address not in priority_concept.pseudonyms
+                and priority_concept.address not in x.pseudonyms
+            ]
+            if priority_concept.derivation == Derivation.AGGREGATE
+            else []
+        )
+        return optional, conditions
 
     if conditions and priority_concept.derivation in ROOT_DERIVATIONS:
         logger.info(
@@ -374,15 +386,21 @@ def generate_loop_completion(context: LoopContext, virtual: set[str]) -> Strateg
             logger.info(
                 f"{depth_to_prefix(context.depth)}{LOGGER_PREFIX} Found different non-virtual output concepts ({non_virtual_difference_values}), removing condition injected values by setting outputs to {[x.address for x in output.output_concepts if x.address in non_virtual_output]}"
             )
-            output.set_output_concepts(
-                [
-                    x
-                    for x in output.output_concepts
-                    if x.address in non_virtual_output
-                    or any(c in non_virtual_output for c in x.pseudonyms)
-                ],
-                rebuild=False,
-            )
+            # output.set_output_concepts(
+            #     [
+            #         x
+            #         for x in output.output_concepts
+            #         if x.address not in non_virtual_difference_values
+            #         or any(c in non_virtual_output for c in x.pseudonyms)
+            #     ],
+            #     rebuild=True,
+            #     change_visibility=False
+            # )
+            # output.set_output_concepts(context.original_mandatory)
+
+            # if isinstance(output, MergeNode):
+            #     output.force_group = True
+            #     output.rebuild_cache()
 
         logger.info(
             f"{depth_to_prefix(context.depth)}{LOGGER_PREFIX} Source stack has single node, returning that {type(output)}"
@@ -416,14 +434,17 @@ def generate_loop_completion(context: LoopContext, virtual: set[str]) -> Strateg
     logger.info(
         f"{depth_to_prefix(context.depth)}{LOGGER_PREFIX} Graph is connected, returning {type(output)} node output {[x.address for x in output.usable_outputs]} partial {[c.address for c in output.partial_concepts or []]} with {context.conditions}"
     )
+    from trilogy.core.processing.discovery_utility import group_if_required_v2
+
     if condition_required and context.conditions and non_virtual_different:
         logger.info(
             f"{depth_to_prefix(context.depth)}{LOGGER_PREFIX} Conditions {context.conditions} were injected, checking if we need a group to restore grain"
         )
-        return group_if_required(
+        return group_if_required_v2(
             output, context.original_mandatory, context.environment
         )
-    return output
+
+    return group_if_required_v2(output, context.original_mandatory, context.environment)
 
 
 def _search_concepts(
@@ -588,4 +609,4 @@ def source_query_concepts(
     logger.info(
         f"{depth_to_prefix(0)}{LOGGER_PREFIX} final concepts are {[x.address for x in final]}"
     )
-    return group_if_required(root, output_concepts, environment)
+    return group_if_required_v2(root, output_concepts, environment)

trilogy/core/processing/discovery_node_factory.py

@@ -177,7 +177,12 @@ def _generate_union_node(ctx: NodeGenerationContext) -> StrategyNode | None:
 def _generate_aggregate_node(ctx: NodeGenerationContext) -> StrategyNode | None:
     # Filter out constants to avoid multiplication issues
     agg_optional = [
-        x for x in ctx.local_optional if x.granularity != Granularity.SINGLE_ROW
+        x
+        for x in ctx.local_optional
+        if not (
+            x.granularity == Granularity.SINGLE_ROW
+            and x.derivation != Derivation.AGGREGATE
+        )
     ]
 
     logger.info(

trilogy/core/processing/discovery_utility.py

@@ -1,13 +1,18 @@
 from typing import List
 
 from trilogy.constants import logger
-from trilogy.core.enums import Derivation
+from trilogy.core.enums import Derivation, Purpose
 from trilogy.core.models.build import (
     BuildConcept,
+    BuildDatasource,
+    BuildFilterItem,
+    BuildGrain,
     BuildRowsetItem,
 )
 from trilogy.core.models.build_environment import BuildEnvironment
+from trilogy.core.models.execute import QueryDatasource, UnnestJoin
 from trilogy.core.processing.nodes import GroupNode, MergeNode, StrategyNode
+from trilogy.core.processing.utility import GroupRequiredResponse
 
 
 def depth_to_prefix(depth: int) -> str:
@@ -17,31 +22,175 @@ def depth_to_prefix(depth: int) -> str:
 LOGGER_PREFIX = "[DISCOVERY LOOP]"
 
 
-def group_if_required(
+def calculate_effective_parent_grain(
+    node: QueryDatasource | BuildDatasource,
+) -> BuildGrain:
+    # calculate the effective grain of the parent node
+    # this is the union of all parent grains
+    if isinstance(node, MergeNode):
+        grain = BuildGrain()
+        qds = node.resolve()
+        if not qds.joins:
+            return qds.datasources[0].grain
+        for join in qds.joins:
+            if isinstance(join, UnnestJoin):
+                continue
+            pairs = join.concept_pairs or []
+            for key in pairs:
+                left = key.existing_datasource
+                grain += left.grain
+            keys = [key.right for key in pairs]
+            join_grain = BuildGrain.from_concepts(keys)
+            if join_grain == join.right_datasource.grain:
+                logger.info(f"irrelevant right join {join}, does not change grain")
+            else:
+                logger.info(
+                    f"join changes grain, adding {join.right_datasource.grain} to {grain}"
+                )
+                grain += join.right_datasource.grain
+        return grain
+    else:
+        return node.grain or BuildGrain()
+
+
+def check_if_group_required(
+    downstream_concepts: List[BuildConcept],
+    parents: list[QueryDatasource | BuildDatasource],
+    environment: BuildEnvironment,
+    depth: int = 0,
+) -> GroupRequiredResponse:
+    padding = "\t" * depth
+    target_grain = BuildGrain.from_concepts(
+        downstream_concepts,
+        environment=environment,
+    )
+
+    comp_grain = BuildGrain()
+    for source in parents:
+        # comp_grain += source.grain
+        comp_grain += calculate_effective_parent_grain(source)
+
+    # dynamically select if we need to group
+    # we must avoid grouping if we are already at grain
+    if comp_grain.issubset(target_grain):
+
+        logger.info(
+            f"{padding}{LOGGER_PREFIX} Group requirement check: {comp_grain}, target: {target_grain}, grain is subset of target, no group node required"
+        )
+        return GroupRequiredResponse(target_grain, comp_grain, False)
+    # find out what extra is in the comp grain vs target grain
+    difference = [
+        environment.concepts[c] for c in (comp_grain - target_grain).components
+    ]
+    logger.info(
+        f"{padding}{LOGGER_PREFIX} Group requirement check: upstream grain: {comp_grain}, desired grain: {target_grain} from , difference {[x.address for x in difference]}"
+    )
+    for x in difference:
+        logger.info(
+            f"{padding}{LOGGER_PREFIX} Difference concept {x.address} purpose {x.purpose} keys {x.keys}"
+        )
+
+    # if the difference is all unique properties whose keys are in the source grain
+    # we can also suppress the group
+    if all(
+        [
+            x.keys
+            and all(
+                environment.concepts[z].address in comp_grain.components for z in x.keys
+            )
+            for x in difference
+        ]
+    ):
+        logger.info(
+            f"{padding}{LOGGER_PREFIX} Group requirement check: skipped due to unique property validation"
+        )
+        return GroupRequiredResponse(target_grain, comp_grain, False)
+    if all([x.purpose == Purpose.KEY for x in difference]):
+        logger.info(
+            f"{padding}{LOGGER_PREFIX} checking if downstream is unique properties of key"
+        )
+        replaced_grain_raw: list[set[str]] = [
+            (
+                x.keys or set()
+                if x.purpose == Purpose.UNIQUE_PROPERTY
+                else set([x.address])
+            )
+            for x in downstream_concepts
+            if x.address in target_grain.components
+        ]
+        # flatten the list of lists
+        replaced_grain = [item for sublist in replaced_grain_raw for item in sublist]
+        # if the replaced grain is a subset of the comp grain, we can skip the group
+        unique_grain_comp = BuildGrain.from_concepts(
+            replaced_grain, environment=environment
+        )
+        if comp_grain.issubset(unique_grain_comp):
+            logger.info(
+                f"{padding}{LOGGER_PREFIX} Group requirement check: skipped due to unique property validation"
+            )
+            return GroupRequiredResponse(target_grain, comp_grain, False)
+    logger.info(
+        f"{padding}{LOGGER_PREFIX} Checking for grain equivalence for filters and rowsets"
+    )
+    ngrain = []
+    for con in target_grain.components:
+        full = environment.concepts[con]
+        if full.derivation == Derivation.ROWSET:
+            ngrain.append(full.address.split(".", 1)[1])
+        elif full.derivation == Derivation.FILTER:
+            assert isinstance(full.lineage, BuildFilterItem)
+            if isinstance(full.lineage.content, BuildConcept):
+                ngrain.append(full.lineage.content.address)
+        else:
+            ngrain.append(full.address)
+    target_grain2 = BuildGrain.from_concepts(
+        ngrain,
+        environment=environment,
+    )
+    if comp_grain.issubset(target_grain2):
+        logger.info(
+            f"{padding}{LOGGER_PREFIX} Group requirement check: {comp_grain}, {target_grain2}, pre rowset grain is subset of target, no group node required"
+        )
+        return GroupRequiredResponse(target_grain2, comp_grain, False)
+
+    logger.info(f"{padding}{LOGGER_PREFIX} Group requirement check: group required")
+    return GroupRequiredResponse(target_grain, comp_grain, True)
+
+
+def group_if_required_v2(
     root: StrategyNode, final: List[BuildConcept], environment: BuildEnvironment
 ):
-    if isinstance(root, MergeNode) and root.force_group is True:
-        return root
-    elif isinstance(root, GroupNode):
-        return root
-    elif GroupNode.check_if_required(
-        downstream_concepts=final,
-        parents=[root.resolve()],
-        environment=environment,
-    ).required:
+    required = check_if_group_required(
+        downstream_concepts=final, parents=[root.resolve()], environment=environment
+    )
+    targets = [
+        x
+        for x in root.output_concepts
+        if x.address in final or any(c in final for c in x.pseudonyms)
+    ]
+    if required.required:
         if isinstance(root, MergeNode):
             root.force_group = True
-            root.set_output_concepts(final, rebuild=False)
+            root.set_output_concepts(targets, rebuild=False, change_visibility=False)
             root.rebuild_cache()
             return root
+        elif isinstance(root, GroupNode):
+            # root.set_output_concepts(final, rebuild=False)
+            # root.rebuild_cache()
+            return root
         return GroupNode(
-            output_concepts=final,
-            input_concepts=final,
+            output_concepts=targets,
+            input_concepts=targets,
             environment=environment,
             parents=[root],
             partial_concepts=root.partial_concepts,
             preexisting_conditions=root.preexisting_conditions,
         )
+    elif isinstance(root, GroupNode):
+
+        return root
+    else:
+        root.set_output_concepts(targets, rebuild=False, change_visibility=False)
     return root
 
 

trilogy/core/processing/node_generators/basic_node.py

@@ -143,4 +143,5 @@ def gen_basic_node(
     logger.info(
         f"{depth_prefix}{LOGGER_PREFIX} Returning basic select for {concept}: input: {[x.address for x in parent_node.input_concepts]} output {[x.address for x in parent_node.output_concepts]} hidden {[x for x in parent_node.hidden_concepts]}"
     )
+
     return parent_node

trilogy/core/processing/node_generators/common.py

@@ -222,6 +222,7 @@ def gen_enrichment_node(
         parents=[enrich_node, base_node],
         force_group=False,
         preexisting_conditions=conditions.conditional if conditions else None,
+        depth=depth,
     )
 
 

trilogy/core/processing/node_generators/filter_node.py

@@ -246,13 +246,6 @@ def gen_filter_node(
         filter_node = parent
     else:
         core_parent_nodes.append(row_parent)
-        filters = [concept] + same_filter_optional
-        parents_for_grain = [
-            x.lineage.content
-            for x in filters
-            if isinstance(x.lineage, BuildFilterItem)
-            and isinstance(x.lineage.content, BuildConcept)
-        ]
         filter_node = FilterNode(
             input_concepts=unique(
                 parent_row_concepts + flattened_existence,
@@ -261,9 +254,6 @@ def gen_filter_node(
             output_concepts=[concept] + same_filter_optional + parent_row_concepts,
             environment=environment,
             parents=core_parent_nodes,
-            grain=BuildGrain.from_concepts(
-                parents_for_grain + parent_row_concepts, environment=environment
-            ),
             preexisting_conditions=conditions.conditional if conditions else None,
         )
 

trilogy/core/processing/node_generators/group_node.py

@@ -108,6 +108,42 @@ def gen_group_node(
                     logger.info(
                         f"{padding(depth)}{LOGGER_PREFIX} cannot include optional agg {possible_agg.address}; it has mismatched parent grain {comp_grain } vs local parent {build_grain_parents}"
                     )
+    elif concept.grain.abstract:
+        for possible_agg in local_optional:
+            if not isinstance(
+                possible_agg.lineage,
+                (BuildAggregateWrapper, BuildFunction),
+            ):
+
+                continue
+            logger.info(
+                f"{padding(depth)}{LOGGER_PREFIX} considering optional agg {possible_agg.address} for {concept.address}"
+            )
+            agg_parents = resolve_function_parent_concepts(
+                possible_agg,
+                environment=environment,
+            )
+            comp_grain = get_aggregate_grain(possible_agg, environment)
+            if not possible_agg.grain.abstract:
+                continue
+            if set([x.address for x in agg_parents]).issubset(
+                set([x.address for x in parent_concepts])
+            ):
+                output_concepts.append(possible_agg)
+                logger.info(
+                    f"{padding(depth)}{LOGGER_PREFIX} found equivalent group by optional concept {possible_agg.address} for {concept.address}"
+                )
+            elif comp_grain == get_aggregate_grain(concept, environment):
+                extra = [x for x in agg_parents if x.address not in parent_concepts]
+                parent_concepts += extra
+                output_concepts.append(possible_agg)
+                logger.info(
+                    f"{padding(depth)}{LOGGER_PREFIX} found equivalent group by optional concept {possible_agg.address} for {concept.address}"
+                )
+            else:
+                logger.info(
+                    f"{padding(depth)}{LOGGER_PREFIX} cannot include optional agg {possible_agg.address}; it has mismatched parent grain {comp_grain } vs local parent {get_aggregate_grain(concept, environment)}"
+                )
     if parent_concepts:
         target_grain = BuildGrain.from_concepts(parent_concepts)
         logger.info(

trilogy/core/processing/node_generators/multiselect_node.py

@@ -156,7 +156,7 @@ def gen_multiselect_node(
     possible_joins = concept_to_relevant_joins(additional_relevant)
     if not local_optional:
         logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} no enriched required for rowset node; exiting early"
+            f"{padding(depth)}{LOGGER_PREFIX} no enrichment required for rowset node; exiting early"
         )
         return node
     if not possible_joins:

trilogy/core/processing/node_generators/node_merge_node.py

@@ -20,9 +20,6 @@ from trilogy.core.models.build import (
     BuildWhereClause,
 )
 from trilogy.core.models.build_environment import BuildEnvironment
-from trilogy.core.processing.discovery_utility import (
-    group_if_required,
-)
 from trilogy.core.processing.nodes import History, MergeNode, StrategyNode
 from trilogy.core.processing.utility import padding
 from trilogy.utility import unique
@@ -510,7 +507,7 @@ def subgraphs_to_merge_node(
     search_conditions: BuildWhereClause | None = None,
     enable_early_exit: bool = True,
 ):
-    target_grain = BuildGrain.from_concepts(output_concepts, environment=environment)
+
     parents: List[StrategyNode] = []
     logger.info(
         f"{padding(depth)}{LOGGER_PREFIX} fetching subgraphs {[[c.address for c in subgraph] for subgraph in concept_subgraphs]}"
@@ -555,7 +552,7 @@ def subgraphs_to_merge_node(
             f"{padding(depth)}{LOGGER_PREFIX} only one parent node, exiting early w/ {[c.address for c in parents[0].output_concepts]}"
         )
         parent = parents[0]
-        return group_if_required(parent, output_concepts, environment)
+        return parent
 
     rval = MergeNode(
         input_concepts=unique(input_c, "address"),
@@ -563,7 +560,6 @@ def subgraphs_to_merge_node(
         environment=environment,
         parents=parents,
         depth=depth,
-        grain=target_grain,
         # hidden_concepts=[]
         # conditions=conditions,
         # conditions=search_conditions.conditional,

trilogy/core/processing/node_generators/rowset_node.py

@@ -92,7 +92,7 @@ def gen_rowset_node(
 
     node.rebuild_cache()
     logger.info(
-        f"{padding(depth)}{LOGGER_PREFIX} final output is {[x.address for x in node.output_concepts]}"
+        f"{padding(depth)}{LOGGER_PREFIX} final output is {[x.address for x in node.output_concepts]} with grain {node.grain}"
     )
     if not local_optional or all(
         (

trilogy/core/processing/nodes/base_node.py

@@ -285,12 +285,23 @@ class StrategyNode:
             self.rebuild_cache()
         return self
 
-    def set_output_concepts(self, concepts: List[BuildConcept], rebuild: bool = True):
+    def set_visible_concepts(self, concepts: List[BuildConcept]):
+        for x in self.output_concepts:
+            if x.address not in [c.address for c in concepts]:
+                self.hidden_concepts.add(x.address)
+        return self
+
+    def set_output_concepts(
+        self,
+        concepts: List[BuildConcept],
+        rebuild: bool = True,
+        change_visibility: bool = True,
+    ):
         # exit if no changes
         if self.output_concepts == concepts:
             return self
         self.output_concepts = concepts
-        if self.hidden_concepts:
+        if self.hidden_concepts and change_visibility:
             self.hidden_concepts = set(
                 x for x in self.hidden_concepts if x not in concepts
             )

trilogy/core/processing/nodes/group_node.py

@@ -1,14 +1,12 @@
-from dataclasses import dataclass
 from typing import List, Optional
 
 from trilogy.constants import logger
-from trilogy.core.enums import Purpose, SourceType
+from trilogy.core.enums import SourceType
 from trilogy.core.models.build import (
     BuildComparison,
     BuildConcept,
     BuildConditional,
     BuildDatasource,
-    BuildGrain,
     BuildOrderBy,
     BuildParenthetical,
 )
@@ -18,19 +16,16 @@ from trilogy.core.processing.nodes.base_node import (
     StrategyNode,
     resolve_concept_map,
 )
-from trilogy.core.processing.utility import find_nullable_concepts, is_scalar_condition
+from trilogy.core.processing.utility import (
+    GroupRequiredResponse,
+    find_nullable_concepts,
+    is_scalar_condition,
+)
 from trilogy.utility import unique
 
 LOGGER_PREFIX = "[CONCEPT DETAIL - GROUP NODE]"
 
 
-@dataclass
-class GroupRequiredResponse:
-    target: BuildGrain
-    upstream: BuildGrain
-    required: bool
-
-
 class GroupNode(StrategyNode):
     source_type = SourceType.GROUP
 
@@ -80,92 +75,15 @@ class GroupNode(StrategyNode):
         environment: BuildEnvironment,
         depth: int = 0,
     ) -> GroupRequiredResponse:
-        padding = "\t" * depth
-        target_grain = BuildGrain.from_concepts(
-            downstream_concepts,
-            environment=environment,
-        )
-
-        # the concepts of the souce grain might not exist in the output environment
-        # so we need to construct a new
-        concept_map: dict[str, BuildConcept] = {}
-        comp_grain = BuildGrain()
-        for source in parents:
-            comp_grain += source.grain
-            for x in source.output_concepts:
-                concept_map[x.address] = x
-        lookups: list[BuildConcept | str] = [
-            concept_map[x] if x in concept_map else x for x in comp_grain.components
-        ]
-
-        comp_grain = BuildGrain.from_concepts(lookups, environment=environment)
-
-        # dynamically select if we need to group
-        # because sometimes, we are already at required grain
-        if comp_grain.issubset(target_grain):
-
-            logger.info(
-                f"{padding}{LOGGER_PREFIX} Group requirement check: {comp_grain}, {target_grain}, grain is subset of target, no group node required"
-            )
-            return GroupRequiredResponse(target_grain, comp_grain, False)
-        # find out what extra is in the comp grain vs target grain
-        difference = [
-            environment.concepts[c] for c in (comp_grain - target_grain).components
-        ]
-        logger.info(
-            f"{padding}{LOGGER_PREFIX} Group requirement check: {comp_grain}, {target_grain}, difference {[x.address for x in difference]}"
-        )
+        from trilogy.core.processing.discovery_utility import check_if_group_required
 
-        # if the difference is all unique properties whose keys are in the source grain
-        # we can also suppress the group
-        if all(
-            [
-                x.keys
-                and all(
-                    environment.concepts[z].address in comp_grain.components
-                    for z in x.keys
-                )
-                for x in difference
-            ]
-        ):
-            logger.info(
-                f"{padding}{LOGGER_PREFIX} Group requirement check: skipped due to unique property validation"
-            )
-            return GroupRequiredResponse(target_grain, comp_grain, False)
-        if all([x.purpose == Purpose.KEY for x in difference]):
-            logger.info(
-                f"{padding}{LOGGER_PREFIX} checking if downstream is unique properties of key"
-            )
-            replaced_grain_raw: list[set[str]] = [
-                (
-                    x.keys or set()
-                    if x.purpose == Purpose.UNIQUE_PROPERTY
-                    else set([x.address])
-                )
-                for x in downstream_concepts
-                if x.address in target_grain.components
-            ]
-            # flatten the list of lists
-            replaced_grain = [
-                item for sublist in replaced_grain_raw for item in sublist
-            ]
-            # if the replaced grain is a subset of the comp grain, we can skip the group
-            unique_grain_comp = BuildGrain.from_concepts(
-                replaced_grain, environment=environment
-            )
-            if comp_grain.issubset(unique_grain_comp):
-                logger.info(
-                    f"{padding}{LOGGER_PREFIX} Group requirement check: skipped due to unique property validation"
-                )
-                return GroupRequiredResponse(target_grain, comp_grain, False)
-
-        logger.info(f"{padding}{LOGGER_PREFIX} Group requirement check: group required")
-        return GroupRequiredResponse(target_grain, comp_grain, True)
+        return check_if_group_required(downstream_concepts, parents, environment, depth)
 
     def _resolve(self) -> QueryDatasource:
         parent_sources: List[QueryDatasource | BuildDatasource] = [
             p.resolve() for p in self.parents
         ]
+
         grains = self.check_if_required(
             self.output_concepts, parent_sources, self.environment, self.depth
         )

trilogy/core/processing/nodes/merge_node.py

@@ -341,6 +341,7 @@ class MergeNode(StrategyNode):
                 full_join_concepts += join.input_concepts
 
         if self.force_group is True:
+
             force_group = True
         elif self.whole_grain:
             force_group = False
@@ -367,6 +368,14 @@ class MergeNode(StrategyNode):
         nullable_concepts = find_nullable_concepts(
             source_map=source_map, joins=joins, datasources=final_datasets
         )
+        if force_group:
+
+            grain = BuildGrain.from_concepts(
+                self.output_concepts, environment=self.environment
+            )
+            logger.info(
+                f"{self.logging_prefix}{LOGGER_PREFIX} forcing group by to achieve grain {grain}"
+            )
         qds = QueryDatasource(
             input_concepts=unique(self.input_concepts, "address"),
             output_concepts=unique(self.output_concepts, "address"),

trilogy/core/processing/utility.py

@@ -27,6 +27,7 @@ from trilogy.core.models.build import (
     BuildDatasource,
     BuildFilterItem,
     BuildFunction,
+    BuildGrain,
     BuildParenthetical,
     BuildSubselectComparison,
     BuildWindowItem,
@@ -82,6 +83,13 @@ class JoinOrderOutput:
         return set(self.keys.keys())
 
 
+@dataclass
+class GroupRequiredResponse:
+    target: BuildGrain
+    upstream: BuildGrain
+    required: bool
+
+
 def resolve_join_order_v2(
     g: nx.Graph, partials: dict[str, list[str]], nullables: dict[str, list[str]]
 ) -> list[JoinOrderOutput]:

trilogy/dialect/base.py

@@ -786,6 +786,8 @@ class BaseDialect:
             return str(e.value)
         elif isinstance(e, ArrayType):
             return f"{self.COMPLEX_DATATYPE_MAP[DataType.ARRAY](self.render_expr(e.value_data_type, cte=cte, cte_map=cte_map))}"
+        elif isinstance(e, list):
+            return f"{self.FUNCTION_MAP[FunctionType.ARRAY]([self.render_expr(x, cte=cte, cte_map=cte_map) for x in e])}"
         elif isinstance(e, BuildParamaterizedConceptReference):
             if self.rendering.parameters:
                 if e.concept.namespace == DEFAULT_NAMESPACE:
@@ -794,6 +796,7 @@ class BaseDialect:
             elif e.concept.lineage:
                 return self.render_expr(e.concept.lineage, cte=cte, cte_map=cte_map)
             return f"{self.QUOTE_CHARACTER}{e.concept.address}{self.QUOTE_CHARACTER}"
+
         else:
             raise ValueError(f"Unable to render type {type(e)} {e}")
 

trilogy/std/color.preql

@@ -0,0 +1,3 @@
+
+
+type hex string; # Hex color value, e.g. #FF5733

trilogy/std/display.preql

@@ -3,7 +3,7 @@
 type percent float; # Percentage value
 
 def calc_percent(a, b, digits=-1) -> case when digits =-1 then 
-case when b = 0 then 0.0::float::percent else
-(a/b)::float::percent end
+case when b = 0 then 0.0::numeric::percent else
+(a/b)::numeric::percent end
 else round((case when b = 0 then 0.0::float::percent else
-(a/b)::float::percent end):: float::percent, digits) end;
+(a/b)::float::percent end):: numeric::percent, digits) end;