pytrilogy 0.0.3.102__py3-none-any.whl → 0.0.3.104__py3-none-any.whl

{pytrilogy-0.0.3.102.dist-info → pytrilogy-0.0.3.104.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.102
+Version: 0.0.3.104
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 
@@ -19,6 +19,7 @@ Requires-Dist: sqlalchemy<2.0.0
 Requires-Dist: networkx
 Requires-Dist: pyodbc
 Requires-Dist: pydantic
+Requires-Dist: duckdb<1.4.0
 Requires-Dist: duckdb-engine
 Requires-Dist: click
 Provides-Extra: postgres

{pytrilogy-0.0.3.102.dist-info → pytrilogy-0.0.3.104.dist-info}/RECORD

@@ -1,6 +1,6 @@
-pytrilogy-0.0.3.102.dist-info/licenses/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
-trilogy/__init__.py,sha256=WZdbHlLqyuYo0xjcYkV5QDokunZLDlhGeibgoay48uc,304
-trilogy/constants.py,sha256=ohmro6so7PPNp2ruWQKVc0ijjXYPOyRrxB9LI8dr3TU,1746
+pytrilogy-0.0.3.104.dist-info/licenses/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
+trilogy/__init__.py,sha256=HyZF9WId40s9G3BjFS2OBMeHI7XNeE-YU1cFLvNOSWk,304
+trilogy/constants.py,sha256=g_zkVCNjGop6coZ1kM8eXXAzCnUN22ldx3TYFz0E9sc,1747
 trilogy/engine.py,sha256=3MiADf5MKcmxqiHBuRqiYdsXiLj7oitDfVvXvHrfjkA,2178
 trilogy/executor.py,sha256=KgCAQhHPT-j0rPkBbALX0f84W9-Q-bkjHayGuavg99w,16490
 trilogy/parser.py,sha256=o4cfk3j3yhUFoiDKq9ZX_GjBF3dKhDjXEwb63rcBkBM,293
@@ -18,7 +18,7 @@ trilogy/core/exceptions.py,sha256=axkVXYJYQXCCwMHwlyDA232g4tCOwdCZUt7eHeUMDMg,28
 trilogy/core/functions.py,sha256=sdV6Z3NUVfwL1d18eNcaAXllVNqzLez23McsJ6xIp7M,33182
 trilogy/core/graph_models.py,sha256=4EWFTHGfYd72zvS2HYoV6hm7nMC_VEd7vWr6txY-ig0,3400
 trilogy/core/internal.py,sha256=r9QagDB2GvpqlyD_I7VrsfbVfIk5mnok2znEbv72Aa4,2681
-trilogy/core/optimization.py,sha256=ojpn-p79lr03SSVQbbw74iPCyoYpDYBmj1dbZ3oXCjI,8860
+trilogy/core/optimization.py,sha256=Km0ITEx9n6Iv5ReX6tm4uXO5uniSv_ooahycNNiET3g,9212
 trilogy/core/query_processor.py,sha256=uqygDJqkjIH4vLP-lbGRgTN7rRcYEkr3KGqNimNw_80,20345
 trilogy/core/utility.py,sha256=3VC13uSQWcZNghgt7Ot0ZTeEmNqs__cx122abVq9qhM,410
 trilogy/core/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -28,42 +28,43 @@ trilogy/core/models/build_environment.py,sha256=mpx7MKGc60fnZLVdeLi2YSREy7eQbQYy
 trilogy/core/models/core.py,sha256=iT9WdZoiXeglmUHWn6bZyXCTBpkApTGPKtNm_Mhbu_g,12987
 trilogy/core/models/datasource.py,sha256=wogTevZ-9CyUW2a8gjzqMCieircxi-J5lkI7EOAZnck,9596
 trilogy/core/models/environment.py,sha256=hwTIRnJgaHUdCYof7U5A9NPitGZ2s9yxqiW5O2SaJ9Y,28759
-trilogy/core/models/execute.py,sha256=lsNzNjS3nZvoW5CHjYwxDTwBe502NZyytpK1eq8CwW4,42357
-trilogy/core/optimizations/__init__.py,sha256=YH2-mGXZnVDnBcWVi8vTbrdw7Qs5TivG4h38rH3js_I,290
+trilogy/core/models/execute.py,sha256=pdL3voYB4dCQR_KMHwFaofP3ZpRbALRC2ELHueWyTko,42191
+trilogy/core/optimizations/__init__.py,sha256=yspWc25M5SgAuvXYoSt5J8atyPbDlOfsKjIo5yGD9s4,368
 trilogy/core/optimizations/base_optimization.py,sha256=gzDOKImoFn36k7XBD3ysEYDnbnb6vdVIztUfFQZsGnM,513
+trilogy/core/optimizations/hide_unused_concept.py,sha256=DbsP8NqQOxmPv9omDOoFNPUGObUkqsRRNrr5d1xDxx4,1962
 trilogy/core/optimizations/inline_datasource.py,sha256=2sWNRpoRInnTgo9wExVT_r9RfLAQHI57reEV5cGHUcg,4329
-trilogy/core/optimizations/predicate_pushdown.py,sha256=g4AYE8Aw_iMlAh68TjNXGP754NTurrDduFECkUjoBnc,9399
+trilogy/core/optimizations/predicate_pushdown.py,sha256=5ubatgq1IwWQ4L2FDt4--y168YLuGP-vwqH0m8IeTIw,9786
 trilogy/core/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/core/processing/concept_strategies_v3.py,sha256=tvSN_aiqb1H7LkTl96vj7YK_DKcq_1nDdRJ69wZCLc8,22158
-trilogy/core/processing/discovery_node_factory.py,sha256=5QVYUsci_h6iYWhS0GCoDow2tSAipiBW1OyTRX-g_L8,15581
-trilogy/core/processing/discovery_utility.py,sha256=Xntgug6VnEF96uw5Zwen1qMEUwKjqrm_ZDUr4i4tc1U,5595
+trilogy/core/processing/concept_strategies_v3.py,sha256=AcMU1d5uCo8I1PFCkBtmcC6iFmM9vN6xSdKxSVMGfpA,23080
+trilogy/core/processing/discovery_node_factory.py,sha256=p23jiiHyhrW-Q8ndbnRlqMHJKT8ZqPOA89SzE4xaFFo,15445
+trilogy/core/processing/discovery_utility.py,sha256=wIuLsE6yuVykeYZdIqRSagivDNU3-ooiS7z6in4yqho,11518
 trilogy/core/processing/discovery_validation.py,sha256=eZ4HfHMpqZLI8MGG2jez8arS8THs6ceuVrQFIY6gXrU,5364
 trilogy/core/processing/graph_utils.py,sha256=8QUVrkE9j-9C1AyrCb1nQEh8daCe0u1HuXl-Te85lag,1205
-trilogy/core/processing/utility.py,sha256=PGQKZgX58kI3gG4nprY8HRGayc2D8fF5RmbvBhQj8ok,23319
+trilogy/core/processing/utility.py,sha256=1_oNnk6lWiy-D7LKYr07kU_v7iAM4i6ITUAS4bIiCr4,23444
 trilogy/core/processing/node_generators/__init__.py,sha256=iVJ-crowPxYeut-hFjyEjfibKIDq7PfB4LEuDAUCjGY,943
-trilogy/core/processing/node_generators/basic_node.py,sha256=0Uhnf07056SBbRkt-wYLw4DZqsFR6jztGLUaE9ebPZs,5577
-trilogy/core/processing/node_generators/common.py,sha256=PdysdroW9DUADP7f5Wv_GKPUyCTROZV1g3L45fawxi8,9443
+trilogy/core/processing/node_generators/basic_node.py,sha256=74LoVZXLinRvSzk2LmI1kwza96TnuH3ELoYRIbHB29A,5578
+trilogy/core/processing/node_generators/common.py,sha256=xF32Kf6B08dZgKs2SOow1HomptSiSC057GCUCHFlS5s,9464
 trilogy/core/processing/node_generators/constant_node.py,sha256=LfpDq2WrBRZ3tGsLxw77LuigKfhbteWWh9L8BGdMGwk,1146
-trilogy/core/processing/node_generators/filter_node.py,sha256=ArBsQJl-4fWBJWCE28CRQ7UT7ErnFfbcseoQQZrBodY,11220
-trilogy/core/processing/node_generators/group_node.py,sha256=yqOWl5TCV4PrdJua4OJkPUIHkljaLoSW2Y8eRAmVddQ,6733
+trilogy/core/processing/node_generators/filter_node.py,sha256=ndPznkcFu_cdCNgaRpgot8oqnzdHv4KAIfjeUIzrE2w,10816
+trilogy/core/processing/node_generators/group_node.py,sha256=NdK1rl6Ze94XFWtgeC2dlRiL4pS3lh1ArKGPEltLtnw,8525
 trilogy/core/processing/node_generators/group_to_node.py,sha256=jKcNCDOY6fNblrdZwaRU0sbUSr9H0moQbAxrGgX6iGA,3832
-trilogy/core/processing/node_generators/multiselect_node.py,sha256=GWV5yLmKTe1yyPhN60RG1Rnrn4ktfn9lYYXi_FVU4UI,7061
-trilogy/core/processing/node_generators/node_merge_node.py,sha256=1joMV7XpQ9Gpe-d5y7JUMBHIqakV5wFJi3Mtvs4UcL4,23415
+trilogy/core/processing/node_generators/multiselect_node.py,sha256=a505AEixjsjp5jI8Ng3H5KF_AaehkS6HfRfTef64l_o,7063
+trilogy/core/processing/node_generators/node_merge_node.py,sha256=hNcZxnDLTZyYJWfojg769zH9HB9PfZfESmpN1lcHWXg,23172
 trilogy/core/processing/node_generators/recursive_node.py,sha256=l5zdh0dURKwmAy8kK4OpMtZfyUEQRk6N-PwSWIyBpSM,2468
-trilogy/core/processing/node_generators/rowset_node.py,sha256=5L5u6xz1In8EaHQdcYgR2si-tz9WB9YLXURo4AkUT9A,6630
+trilogy/core/processing/node_generators/rowset_node.py,sha256=MuVNIexXhqGONho_mewqMOwaYXNUnjjvyPvk_RDGNYE,5943
 trilogy/core/processing/node_generators/select_merge_node.py,sha256=KQvGoNT5ZBWQ_caEomRTtG1PKZC7OPT4PKfY0QmwMGE,22270
 trilogy/core/processing/node_generators/select_node.py,sha256=Ta1G39V94gjX_AgyZDz9OqnwLz4BjY3D6Drx9YpziMQ,3555
 trilogy/core/processing/node_generators/synonym_node.py,sha256=AnAsa_Wj50NJ_IK0HSgab_7klYmKVrv0WI1uUe-GvEY,3766
-trilogy/core/processing/node_generators/union_node.py,sha256=VNo6Oey4p8etU9xrOh2oTT2lIOTvY6PULUPRvVa2uxU,2877
-trilogy/core/processing/node_generators/unnest_node.py,sha256=w9vhPzASz53QPASLqFcLDdR9eY132tgVUcp3QolD5Jw,3726
+trilogy/core/processing/node_generators/union_node.py,sha256=NxQbnRRoYMI4WjMeph41yk4E6yipj53qdGuNt-Mozxw,2818
+trilogy/core/processing/node_generators/unnest_node.py,sha256=7uOZzBidEEKeZE0VW_XlgHGhEYf_snEHtV8UgJ_ZjyY,4048
 trilogy/core/processing/node_generators/window_node.py,sha256=A90linr4pkZtTNfn9k2YNLqrJ_SFII3lbHxB-BC6mI8,6688
 trilogy/core/processing/node_generators/select_helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/processing/node_generators/select_helpers/datasource_injection.py,sha256=m2YQ4OmG0N2O61a7NEq1ZzbTa7JsCC00lxB2ymjcYRI,8224
 trilogy/core/processing/nodes/__init__.py,sha256=zTge1EzwzEydlcMliIFO_TT7h7lS8l37lyZuQDir1h0,5487
-trilogy/core/processing/nodes/base_node.py,sha256=C_CjlOzlGMXckyV0b_PJZerpopNesRCKfambMq7Asvc,18221
+trilogy/core/processing/nodes/base_node.py,sha256=6LPQ5zP_dZJ6-k_dmX9ZSLsHaQMHgqiR5DEylpHYGZA,18478
 trilogy/core/processing/nodes/filter_node.py,sha256=5VtRfKbCORx0dV-vQfgy3gOEkmmscL9f31ExvlODwvY,2461
-trilogy/core/processing/nodes/group_node.py,sha256=njz-5T7OJ3-kaBC7EhdtPra3G77HnI7apjUwMGhUeXo,10569
-trilogy/core/processing/nodes/merge_node.py,sha256=daJywBxh44Gqk-7eTiXbYtY7xo6O6fNvqX-DagTOTmE,16231
+trilogy/core/processing/nodes/group_node.py,sha256=sKsRP_BWEKg6z63T1X5ZlkJF2IMif0IEbVWTk-cdOH8,7100
+trilogy/core/processing/nodes/merge_node.py,sha256=uc0tlz30Yt9SnCwLhMcWuPVbXLzm3dzy0XqbyirqqTo,16521
 trilogy/core/processing/nodes/recursive_node.py,sha256=k0rizxR8KE64ievfHx_GPfQmU8QAP118Laeyq5BLUOk,1526
 trilogy/core/processing/nodes/select_node_v2.py,sha256=IWyKyNgFlV8A2S1FUTPdIaogg6PzaHh-HmQo6v24sbg,8862
 trilogy/core/processing/nodes/union_node.py,sha256=hLAXXVWqEgMWi7dlgSHfCF59fon64av14-uPgJzoKzM,1870
@@ -81,9 +82,9 @@ trilogy/core/validation/datasource.py,sha256=nJeEFyb6iMBwlEVdYVy1vLzAbdRZwOsUjGx
 trilogy/core/validation/environment.py,sha256=ymvhQyt7jLK641JAAIQkqjQaAmr9C5022ILzYvDgPP0,2835
 trilogy/core/validation/fix.py,sha256=Z818UFNLxndMTLiyhB3doLxIfnOZ-16QGvVFWuD7UsA,3750
 trilogy/dialect/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/dialect/base.py,sha256=d2gXfa5Jh3uyN9H9MxG53JT-xQQgntq2X7EprobJYUc,49698
+trilogy/dialect/base.py,sha256=hFX0_3N-m3ZRTCyv1S650a8OPlx9qjp5Zh8wzTBx6E8,50338
 trilogy/dialect/bigquery.py,sha256=XS3hpybeowgfrOrkycAigAF3NX2YUzTzfgE6f__2fT4,4316
-trilogy/dialect/common.py,sha256=_MarnMWRBn3VcNt3k5VUdFrwH6oHzGdNQquSpHNLq4o,5644
+trilogy/dialect/common.py,sha256=cUI7JMmpG_A5KcaxRI-GoyqwLMD6jTf0JJhgcOdwQK4,5833
 trilogy/dialect/config.py,sha256=olnyeVU5W5T6b9-dMeNAnvxuPlyc2uefb7FRME094Ec,3834
 trilogy/dialect/dataframe.py,sha256=RUbNgReEa9g3pL6H7fP9lPTrAij5pkqedpZ99D8_5AE,1522
 trilogy/dialect/duckdb.py,sha256=JoUvQ19WvgxoaJkGLM7DPXOd1H0394k3vBiblksQzOI,5631
@@ -109,16 +110,17 @@ trilogy/parsing/trilogy.lark,sha256=6eBDD6d4D9N1Nnn4CtmaoB-NpOpjHrEn5oi0JykAlbE,
 trilogy/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/scripts/trilogy.py,sha256=1L0XrH4mVHRt1C9T1HnaDv2_kYEfbWTb5_-cBBke79w,3774
 trilogy/std/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+trilogy/std/color.preql,sha256=sS9AXLDkECuDbNGnBMi2KnUuJukyVZVThKI9mP-ZOZI,50
 trilogy/std/date.preql,sha256=HWZm4t4HWyxr5geWRsY05RnHBVDMci8z8YA2cu0-OOw,188
-trilogy/std/display.preql,sha256=S20HW8qbShBc4OZPcHYiRlLdcaBp9dwruozWBoXKscs,293
+trilogy/std/display.preql,sha256=ZJ08crsZnC3kaWwNUrMB1ZH5j6DUUbz8RaUgihA8sm4,299
 trilogy/std/geography.preql,sha256=1A9Sq5PPMBnEPPf7f-rPVYxJfsnWpQ8oV_k4Fm3H2dU,675
 trilogy/std/metric.preql,sha256=DRECGhkMyqfit5Fl4Ut9zbWrJuSMI1iO9HikuyoBpE0,421
 trilogy/std/money.preql,sha256=XWwvAV3WxBsHX9zfptoYRnBigcfYwrYtBHXTME0xJuQ,2082
 trilogy/std/net.preql,sha256=WZCuvH87_rZntZiuGJMmBDMVKkdhTtxeHOkrXNwJ1EE,416
 trilogy/std/ranking.preql,sha256=LDoZrYyz4g3xsII9XwXfmstZD-_92i1Eox1UqkBIfi8,83
 trilogy/std/report.preql,sha256=LbV-XlHdfw0jgnQ8pV7acG95xrd1-p65fVpiIc-S7W4,202
-pytrilogy-0.0.3.102.dist-info/METADATA,sha256=fQKKWHDkY9Nhofow6RO22oMSXp91H-vOD5d3kk3S-V8,11811
-pytrilogy-0.0.3.102.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-pytrilogy-0.0.3.102.dist-info/entry_points.txt,sha256=ewBPU2vLnVexZVnB-NrVj-p3E-4vukg83Zk8A55Wp2w,56
-pytrilogy-0.0.3.102.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
-pytrilogy-0.0.3.102.dist-info/RECORD,,
+pytrilogy-0.0.3.104.dist-info/METADATA,sha256=IJmkrwnxe7gz3s89ZYVrDe6SkRY2cf6xNpmj5GTXkSE,11839
+pytrilogy-0.0.3.104.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+pytrilogy-0.0.3.104.dist-info/entry_points.txt,sha256=ewBPU2vLnVexZVnB-NrVj-p3E-4vukg83Zk8A55Wp2w,56
+pytrilogy-0.0.3.104.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
+pytrilogy-0.0.3.104.dist-info/RECORD,,

trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.dialect.enums import Dialects
 from trilogy.executor import Executor
 from trilogy.parser import parse
 
-__version__ = "0.0.3.102"
+__version__ = "0.0.3.104"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

trilogy/constants.py

@@ -37,7 +37,7 @@ class Comments:
 
     show: bool = False
     basic: bool = True
-    joins: bool = True
+    joins: bool = False
     nullable: bool = True
     partial: bool = True
     source_map: bool = False

trilogy/core/models/execute.py

@@ -711,8 +711,6 @@ class QueryDatasource(BaseModel):
             f" {[c.address for c in self.output_concepts]} concepts and"
             f" {other.name} with {[c.address for c in other.output_concepts]} concepts"
         )
-        logger.info(self.source_map)
-        logger.info(other.source_map)
 
         merged_datasources: dict[str, Union[BuildDatasource, "QueryDatasource"]] = {}
 
@@ -816,10 +814,7 @@ class QueryDatasource(BaseModel):
                     use_raw_name,
                     force_alias=force_alias,
                 )
-            except ValueError as e:
-                from trilogy.constants import logger
-
-                logger.debug(e)
+            except ValueError:
                 continue
         existing = [c.with_grain(self.grain) for c in self.output_concepts]
         if concept in existing:

trilogy/core/optimization.py

@@ -5,6 +5,7 @@ from trilogy.core.models.build import (
 )
 from trilogy.core.models.execute import CTE, RecursiveCTE, UnionCTE
 from trilogy.core.optimizations import (
+    HideUnusedConcepts,
     InlineDatasource,
     OptimizationRule,
     PredicatePushdown,
@@ -84,11 +85,18 @@ def filter_irrelevant_ctes(
         #                 child.existence_source_map[x2].append(parent.name)
         # else:
         relevant_ctes.add(cte.name)
-        for cte in cte.parent_ctes:
-            recurse(cte, inverse_map)
+
+        for parent in cte.parent_ctes:
+            if parent.name in relevant_ctes:
+                logger.info(
+                    f"[Optimization][Irrelevent CTE filtering] Already visited {parent.name} when visting {cte.name}, potential recursive dag"
+                )
+                continue
+
+            recurse(parent, inverse_map)
         if isinstance(cte, UnionCTE):
-            for cte in cte.internal_ctes:
-                recurse(cte, inverse_map)
+            for internal in cte.internal_ctes:
+                recurse(internal, inverse_map)
 
     inverse_map = gen_inverse_map(input)
     recurse(root_cte, inverse_map)
@@ -220,6 +228,7 @@ def optimize_ctes(
         REGISTERED_RULES.append(PredicatePushdown())
     if CONFIG.optimizations.predicate_pushdown:
         REGISTERED_RULES.append(PredicatePushdownRemove())
+    REGISTERED_RULES.append(HideUnusedConcepts())
     for rule in REGISTERED_RULES:
         loops = 0
         complete = False

trilogy/core/optimizations/__init__.py

@@ -1,4 +1,5 @@
 from .base_optimization import OptimizationRule
+from .hide_unused_concept import HideUnusedConcepts
 from .inline_datasource import InlineDatasource
 from .predicate_pushdown import PredicatePushdown, PredicatePushdownRemove
 
@@ -7,4 +8,5 @@ __all__ = [
     "InlineDatasource",
     "PredicatePushdown",
     "PredicatePushdownRemove",
+    "HideUnusedConcepts",
 ]

trilogy/core/optimizations/hide_unused_concept.py

@@ -0,0 +1,51 @@
+from trilogy.core.models.build import (
+    BuildConcept,
+)
+from trilogy.core.models.execute import CTE, UnionCTE
+from trilogy.core.optimizations.base_optimization import OptimizationRule
+
+
+class HideUnusedConcepts(OptimizationRule):
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+
+    def optimize(
+        self, cte: CTE | UnionCTE, inverse_map: dict[str, list[CTE | UnionCTE]]
+    ) -> bool:
+        used = set()
+        from trilogy.dialect.base import BaseDialect
+
+        renderer = BaseDialect()
+        children = inverse_map.get(cte.name, [])
+        if not children:
+            return False
+        for v in children:
+            self.log(f"Analyzing usage of {cte.name} in {v.name}")
+            renderer.render_cte(v)
+        used = renderer.used_map.get(cte.name, set())
+        self.log(f"Used concepts for {cte.name}: {used} from {renderer.used_map}")
+        add_to_hidden: list[BuildConcept] = []
+        for concept in cte.output_columns:
+            if concept.address not in used:
+                add_to_hidden.append(concept)
+        newly_hidden = [
+            x.address for x in add_to_hidden if x.address not in cte.hidden_concepts
+        ]
+        non_hidden = [
+            x for x in cte.output_columns if x.address not in cte.hidden_concepts
+        ]
+        if not newly_hidden or len(non_hidden) <= 1:
+            return False
+        self.log(
+            f"Hiding unused concepts {[x.address for x in add_to_hidden]} from {cte.name} (used: {used}, all: {[x.address for x in cte.output_columns]})"
+        )
+        candidates = [
+            x.address
+            for x in cte.output_columns
+            if x.address not in used and x.address not in cte.hidden_concepts
+        ]
+        if len(candidates) == len(set([x.address for x in cte.output_columns])):
+            # pop one out
+            candidates.pop()
+        cte.hidden_concepts = set(candidates)
+        return True

trilogy/core/optimizations/predicate_pushdown.py

@@ -1,5 +1,6 @@
 from trilogy.core.enums import (
     BooleanOperator,
+    SourceType,
 )
 from trilogy.core.models.build import (
     BuildComparison,
@@ -59,12 +60,19 @@ class PredicatePushdown(OptimizationRule):
             )
             return False
         materialized = {k for k, v in parent_cte.source_map.items() if v != []}
+
         if not row_conditions or not materialized:
             return False
         output_addresses = {x.address for x in parent_cte.output_columns}
         # if any of the existence conditions are created on the asset, we can't push up to it
         if existence_conditions and existence_conditions.intersection(output_addresses):
             return False
+        if existence_conditions:
+            self.log(
+                f"Not pushing up existence {candidate} to {parent_cte.name} as it is a filter node"
+            )
+            if parent_cte.source.source_type == SourceType.FILTER:
+                return False
         # if it's a root datasource, we can filter on _any_ of the output concepts
         if parent_cte.is_root_datasource:
             extra_check = {
@@ -81,7 +89,7 @@ class PredicatePushdown(OptimizationRule):
             children = inverse_map.get(parent_cte.name, [])
             if all([is_child_of(candidate, child.condition) for child in children]):
                 self.log(
-                    f"All concepts are found on {parent_cte.name} with existing {parent_cte.condition} and all it's {len(children)} children include same filter; pushing up {candidate}"
+                    f"All concepts [{row_conditions}] and existence conditions [{existence_conditions}] not block pushup of [{output_addresses}]found on {parent_cte.name} with existing {parent_cte.condition} and all it's {len(children)} children include same filter; pushing up {candidate}"
                 )
                 if parent_cte.condition and not is_scalar_condition(
                     parent_cte.condition

trilogy/core/processing/concept_strategies_v3.py

@@ -19,7 +19,7 @@ from trilogy.core.processing.discovery_utility import (
     LOGGER_PREFIX,
     depth_to_prefix,
     get_priority_concept,
-    group_if_required,
+    group_if_required_v2,
 )
 from trilogy.core.processing.discovery_validation import (
     ValidationResult,
@@ -66,7 +66,19 @@ def generate_candidates_restrictive(
 
     # if it's single row, joins are irrelevant. Fetch without keys.
     if priority_concept.granularity == Granularity.SINGLE_ROW:
-        return [], conditions
+        logger.info("Have single row concept, including only other single row optional")
+        optional = (
+            [
+                x
+                for x in candidates
+                if x.granularity == Granularity.SINGLE_ROW
+                and x.address not in priority_concept.pseudonyms
+                and priority_concept.address not in x.pseudonyms
+            ]
+            if priority_concept.derivation == Derivation.AGGREGATE
+            else []
+        )
+        return optional, conditions
 
     if conditions and priority_concept.derivation in ROOT_DERIVATIONS:
         logger.info(
@@ -374,15 +386,21 @@ def generate_loop_completion(context: LoopContext, virtual: set[str]) -> Strateg
             logger.info(
                 f"{depth_to_prefix(context.depth)}{LOGGER_PREFIX} Found different non-virtual output concepts ({non_virtual_difference_values}), removing condition injected values by setting outputs to {[x.address for x in output.output_concepts if x.address in non_virtual_output]}"
             )
-            output.set_output_concepts(
-                [
-                    x
-                    for x in output.output_concepts
-                    if x.address in non_virtual_output
-                    or any(c in non_virtual_output for c in x.pseudonyms)
-                ],
-                rebuild=False,
-            )
+            # output.set_output_concepts(
+            #     [
+            #         x
+            #         for x in output.output_concepts
+            #         if x.address not in non_virtual_difference_values
+            #         or any(c in non_virtual_output for c in x.pseudonyms)
+            #     ],
+            #     rebuild=True,
+            #     change_visibility=False
+            # )
+            # output.set_output_concepts(context.original_mandatory)
+
+            # if isinstance(output, MergeNode):
+            #     output.force_group = True
+            #     output.rebuild_cache()
 
         logger.info(
             f"{depth_to_prefix(context.depth)}{LOGGER_PREFIX} Source stack has single node, returning that {type(output)}"
@@ -416,14 +434,17 @@ def generate_loop_completion(context: LoopContext, virtual: set[str]) -> Strateg
     logger.info(
         f"{depth_to_prefix(context.depth)}{LOGGER_PREFIX} Graph is connected, returning {type(output)} node output {[x.address for x in output.usable_outputs]} partial {[c.address for c in output.partial_concepts or []]} with {context.conditions}"
     )
+    from trilogy.core.processing.discovery_utility import group_if_required_v2
+
     if condition_required and context.conditions and non_virtual_different:
         logger.info(
             f"{depth_to_prefix(context.depth)}{LOGGER_PREFIX} Conditions {context.conditions} were injected, checking if we need a group to restore grain"
         )
-        return group_if_required(
+        return group_if_required_v2(
             output, context.original_mandatory, context.environment
         )
-    return output
+
+    return group_if_required_v2(output, context.original_mandatory, context.environment)
 
 
 def _search_concepts(
@@ -588,4 +609,4 @@ def source_query_concepts(
     logger.info(
         f"{depth_to_prefix(0)}{LOGGER_PREFIX} final concepts are {[x.address for x in final]}"
     )
-    return group_if_required(root, output_concepts, environment)
+    return group_if_required_v2(root, output_concepts, environment)

trilogy/core/processing/discovery_node_factory.py

@@ -177,7 +177,12 @@ def _generate_union_node(ctx: NodeGenerationContext) -> StrategyNode | None:
 def _generate_aggregate_node(ctx: NodeGenerationContext) -> StrategyNode | None:
     # Filter out constants to avoid multiplication issues
     agg_optional = [
-        x for x in ctx.local_optional if x.granularity != Granularity.SINGLE_ROW
+        x
+        for x in ctx.local_optional
+        if not (
+            x.granularity == Granularity.SINGLE_ROW
+            and x.derivation != Derivation.AGGREGATE
+        )
     ]
 
     logger.info(
@@ -376,11 +381,6 @@ class RootNodeHandler:
 
         if pseudonyms:
             expanded.add_output_concepts(pseudonyms)
-            logger.info(
-                f"{depth_to_prefix(self.ctx.depth)}{LOGGER_PREFIX} "
-                f"Hiding pseudonyms {[c.address for c in pseudonyms]}"
-            )
-            expanded.hide_output_concepts(pseudonyms)
 
         logger.info(
             f"{depth_to_prefix(self.ctx.depth)}{LOGGER_PREFIX} "

trilogy/core/processing/discovery_utility.py

@@ -1,13 +1,18 @@
 from typing import List
 
 from trilogy.constants import logger
-from trilogy.core.enums import Derivation
+from trilogy.core.enums import Derivation, Purpose
 from trilogy.core.models.build import (
     BuildConcept,
+    BuildDatasource,
+    BuildFilterItem,
+    BuildGrain,
     BuildRowsetItem,
 )
 from trilogy.core.models.build_environment import BuildEnvironment
+from trilogy.core.models.execute import QueryDatasource, UnnestJoin
 from trilogy.core.processing.nodes import GroupNode, MergeNode, StrategyNode
+from trilogy.core.processing.utility import GroupRequiredResponse
 
 
 def depth_to_prefix(depth: int) -> str:
@@ -17,31 +22,175 @@ def depth_to_prefix(depth: int) -> str:
 LOGGER_PREFIX = "[DISCOVERY LOOP]"
 
 
-def group_if_required(
+def calculate_effective_parent_grain(
+    node: QueryDatasource | BuildDatasource,
+) -> BuildGrain:
+    # calculate the effective grain of the parent node
+    # this is the union of all parent grains
+    if isinstance(node, MergeNode):
+        grain = BuildGrain()
+        qds = node.resolve()
+        if not qds.joins:
+            return qds.datasources[0].grain
+        for join in qds.joins:
+            if isinstance(join, UnnestJoin):
+                continue
+            pairs = join.concept_pairs or []
+            for key in pairs:
+                left = key.existing_datasource
+                grain += left.grain
+            keys = [key.right for key in pairs]
+            join_grain = BuildGrain.from_concepts(keys)
+            if join_grain == join.right_datasource.grain:
+                logger.info(f"irrelevant right join {join}, does not change grain")
+            else:
+                logger.info(
+                    f"join changes grain, adding {join.right_datasource.grain} to {grain}"
+                )
+                grain += join.right_datasource.grain
+        return grain
+    else:
+        return node.grain or BuildGrain()
+
+
+def check_if_group_required(
+    downstream_concepts: List[BuildConcept],
+    parents: list[QueryDatasource | BuildDatasource],
+    environment: BuildEnvironment,
+    depth: int = 0,
+) -> GroupRequiredResponse:
+    padding = "\t" * depth
+    target_grain = BuildGrain.from_concepts(
+        downstream_concepts,
+        environment=environment,
+    )
+
+    comp_grain = BuildGrain()
+    for source in parents:
+        # comp_grain += source.grain
+        comp_grain += calculate_effective_parent_grain(source)
+
+    # dynamically select if we need to group
+    # we must avoid grouping if we are already at grain
+    if comp_grain.issubset(target_grain):
+
+        logger.info(
+            f"{padding}{LOGGER_PREFIX} Group requirement check: {comp_grain}, target: {target_grain}, grain is subset of target, no group node required"
+        )
+        return GroupRequiredResponse(target_grain, comp_grain, False)
+    # find out what extra is in the comp grain vs target grain
+    difference = [
+        environment.concepts[c] for c in (comp_grain - target_grain).components
+    ]
+    logger.info(
+        f"{padding}{LOGGER_PREFIX} Group requirement check: upstream grain: {comp_grain}, desired grain: {target_grain} from , difference {[x.address for x in difference]}"
+    )
+    for x in difference:
+        logger.info(
+            f"{padding}{LOGGER_PREFIX} Difference concept {x.address} purpose {x.purpose} keys {x.keys}"
+        )
+
+    # if the difference is all unique properties whose keys are in the source grain
+    # we can also suppress the group
+    if all(
+        [
+            x.keys
+            and all(
+                environment.concepts[z].address in comp_grain.components for z in x.keys
+            )
+            for x in difference
+        ]
+    ):
+        logger.info(
+            f"{padding}{LOGGER_PREFIX} Group requirement check: skipped due to unique property validation"
+        )
+        return GroupRequiredResponse(target_grain, comp_grain, False)
+    if all([x.purpose == Purpose.KEY for x in difference]):
+        logger.info(
+            f"{padding}{LOGGER_PREFIX} checking if downstream is unique properties of key"
+        )
+        replaced_grain_raw: list[set[str]] = [
+            (
+                x.keys or set()
+                if x.purpose == Purpose.UNIQUE_PROPERTY
+                else set([x.address])
+            )
+            for x in downstream_concepts
+            if x.address in target_grain.components
+        ]
+        # flatten the list of lists
+        replaced_grain = [item for sublist in replaced_grain_raw for item in sublist]
+        # if the replaced grain is a subset of the comp grain, we can skip the group
+        unique_grain_comp = BuildGrain.from_concepts(
+            replaced_grain, environment=environment
+        )
+        if comp_grain.issubset(unique_grain_comp):
+            logger.info(
+                f"{padding}{LOGGER_PREFIX} Group requirement check: skipped due to unique property validation"
+            )
+            return GroupRequiredResponse(target_grain, comp_grain, False)
+    logger.info(
+        f"{padding}{LOGGER_PREFIX} Checking for grain equivalence for filters and rowsets"
+    )
+    ngrain = []
+    for con in target_grain.components:
+        full = environment.concepts[con]
+        if full.derivation == Derivation.ROWSET:
+            ngrain.append(full.address.split(".", 1)[1])
+        elif full.derivation == Derivation.FILTER:
+            assert isinstance(full.lineage, BuildFilterItem)
+            if isinstance(full.lineage.content, BuildConcept):
+                ngrain.append(full.lineage.content.address)
+        else:
+            ngrain.append(full.address)
+    target_grain2 = BuildGrain.from_concepts(
+        ngrain,
+        environment=environment,
+    )
+    if comp_grain.issubset(target_grain2):
+        logger.info(
+            f"{padding}{LOGGER_PREFIX} Group requirement check: {comp_grain}, {target_grain2}, pre rowset grain is subset of target, no group node required"
+        )
+        return GroupRequiredResponse(target_grain2, comp_grain, False)
+
+    logger.info(f"{padding}{LOGGER_PREFIX} Group requirement check: group required")
+    return GroupRequiredResponse(target_grain, comp_grain, True)
+
+
+def group_if_required_v2(
     root: StrategyNode, final: List[BuildConcept], environment: BuildEnvironment
 ):
-    if isinstance(root, MergeNode) and root.force_group is True:
-        return root
-    elif isinstance(root, GroupNode):
-        return root
-    elif GroupNode.check_if_required(
-        downstream_concepts=final,
-        parents=[root.resolve()],
-        environment=environment,
-    ).required:
+    required = check_if_group_required(
+        downstream_concepts=final, parents=[root.resolve()], environment=environment
+    )
+    targets = [
+        x
+        for x in root.output_concepts
+        if x.address in final or any(c in final for c in x.pseudonyms)
+    ]
+    if required.required:
         if isinstance(root, MergeNode):
             root.force_group = True
-            root.set_output_concepts(final, rebuild=False)
+            root.set_output_concepts(targets, rebuild=False, change_visibility=False)
             root.rebuild_cache()
             return root
+        elif isinstance(root, GroupNode):
+            # root.set_output_concepts(final, rebuild=False)
+            # root.rebuild_cache()
+            return root
         return GroupNode(
-            output_concepts=final,
-            input_concepts=final,
+            output_concepts=targets,
+            input_concepts=targets,
             environment=environment,
             parents=[root],
             partial_concepts=root.partial_concepts,
             preexisting_conditions=root.preexisting_conditions,
         )
+    elif isinstance(root, GroupNode):
+
+        return root
+    else:
+        root.set_output_concepts(targets, rebuild=False, change_visibility=False)
     return root
 
 

trilogy/core/processing/node_generators/basic_node.py

@@ -143,4 +143,5 @@ def gen_basic_node(
     logger.info(
         f"{depth_prefix}{LOGGER_PREFIX} Returning basic select for {concept}: input: {[x.address for x in parent_node.input_concepts]} output {[x.address for x in parent_node.output_concepts]} hidden {[x for x in parent_node.hidden_concepts]}"
     )
+
     return parent_node

trilogy/core/processing/node_generators/common.py

@@ -222,6 +222,7 @@ def gen_enrichment_node(
         parents=[enrich_node, base_node],
         force_group=False,
         preexisting_conditions=conditions.conditional if conditions else None,
+        depth=depth,
     )