pytrilogy 0.0.3.101__py3-none-any.whl → 0.0.3.103__py3-none-any.whl

{pytrilogy-0.0.3.101.dist-info → pytrilogy-0.0.3.103.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.101
+Version: 0.0.3.103
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 
@@ -19,6 +19,7 @@ Requires-Dist: sqlalchemy<2.0.0
 Requires-Dist: networkx
 Requires-Dist: pyodbc
 Requires-Dist: pydantic
+Requires-Dist: duckdb<1.4.0
 Requires-Dist: duckdb-engine
 Requires-Dist: click
 Provides-Extra: postgres

{pytrilogy-0.0.3.101.dist-info → pytrilogy-0.0.3.103.dist-info}/RECORD

@@ -1,5 +1,5 @@
-pytrilogy-0.0.3.101.dist-info/licenses/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
-trilogy/__init__.py,sha256=NMiEE_jE99ZiREk8IPjfT2M-jxAwtmd2vyCWVD3kT28,304
+pytrilogy-0.0.3.103.dist-info/licenses/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
+trilogy/__init__.py,sha256=lBanLP2CsDaPdJvJ3K68ncf8sO3sMfgsapECGG5J4fk,304
 trilogy/constants.py,sha256=ohmro6so7PPNp2ruWQKVc0ijjXYPOyRrxB9LI8dr3TU,1746
 trilogy/engine.py,sha256=3MiADf5MKcmxqiHBuRqiYdsXiLj7oitDfVvXvHrfjkA,2178
 trilogy/executor.py,sha256=KgCAQhHPT-j0rPkBbALX0f84W9-Q-bkjHayGuavg99w,16490
@@ -18,24 +18,25 @@ trilogy/core/exceptions.py,sha256=axkVXYJYQXCCwMHwlyDA232g4tCOwdCZUt7eHeUMDMg,28
 trilogy/core/functions.py,sha256=sdV6Z3NUVfwL1d18eNcaAXllVNqzLez23McsJ6xIp7M,33182
 trilogy/core/graph_models.py,sha256=4EWFTHGfYd72zvS2HYoV6hm7nMC_VEd7vWr6txY-ig0,3400
 trilogy/core/internal.py,sha256=r9QagDB2GvpqlyD_I7VrsfbVfIk5mnok2znEbv72Aa4,2681
-trilogy/core/optimization.py,sha256=ojpn-p79lr03SSVQbbw74iPCyoYpDYBmj1dbZ3oXCjI,8860
+trilogy/core/optimization.py,sha256=Km0ITEx9n6Iv5ReX6tm4uXO5uniSv_ooahycNNiET3g,9212
 trilogy/core/query_processor.py,sha256=uqygDJqkjIH4vLP-lbGRgTN7rRcYEkr3KGqNimNw_80,20345
 trilogy/core/utility.py,sha256=3VC13uSQWcZNghgt7Ot0ZTeEmNqs__cx122abVq9qhM,410
 trilogy/core/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/core/models/author.py,sha256=ZSKEJ6Vg4otpI_m7_JuGyrFM8dZV1HaxBwprvDSwUzo,81149
-trilogy/core/models/build.py,sha256=ZwJJGyp4rVsISvL8Er_AxQdVJrafYc4fesSj4MNgoxU,70615
+trilogy/core/models/author.py,sha256=3I7PFpJgoQT9RPOT3DfiqAjEtkcQPJnScs60I2UoyWo,81461
+trilogy/core/models/build.py,sha256=iqk_-3plxX1BdxvUCTebqE9F3x62f40neKGf6Ld4VVU,70858
 trilogy/core/models/build_environment.py,sha256=mpx7MKGc60fnZLVdeLi2YSREy7eQbQYycCrP4zF-rHU,5258
 trilogy/core/models/core.py,sha256=iT9WdZoiXeglmUHWn6bZyXCTBpkApTGPKtNm_Mhbu_g,12987
 trilogy/core/models/datasource.py,sha256=wogTevZ-9CyUW2a8gjzqMCieircxi-J5lkI7EOAZnck,9596
 trilogy/core/models/environment.py,sha256=hwTIRnJgaHUdCYof7U5A9NPitGZ2s9yxqiW5O2SaJ9Y,28759
 trilogy/core/models/execute.py,sha256=lsNzNjS3nZvoW5CHjYwxDTwBe502NZyytpK1eq8CwW4,42357
-trilogy/core/optimizations/__init__.py,sha256=YH2-mGXZnVDnBcWVi8vTbrdw7Qs5TivG4h38rH3js_I,290
+trilogy/core/optimizations/__init__.py,sha256=yspWc25M5SgAuvXYoSt5J8atyPbDlOfsKjIo5yGD9s4,368
 trilogy/core/optimizations/base_optimization.py,sha256=gzDOKImoFn36k7XBD3ysEYDnbnb6vdVIztUfFQZsGnM,513
+trilogy/core/optimizations/hide_unused_concept.py,sha256=DbsP8NqQOxmPv9omDOoFNPUGObUkqsRRNrr5d1xDxx4,1962
 trilogy/core/optimizations/inline_datasource.py,sha256=2sWNRpoRInnTgo9wExVT_r9RfLAQHI57reEV5cGHUcg,4329
 trilogy/core/optimizations/predicate_pushdown.py,sha256=g4AYE8Aw_iMlAh68TjNXGP754NTurrDduFECkUjoBnc,9399
 trilogy/core/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/processing/concept_strategies_v3.py,sha256=tvSN_aiqb1H7LkTl96vj7YK_DKcq_1nDdRJ69wZCLc8,22158
-trilogy/core/processing/discovery_node_factory.py,sha256=5QVYUsci_h6iYWhS0GCoDow2tSAipiBW1OyTRX-g_L8,15581
+trilogy/core/processing/discovery_node_factory.py,sha256=r1JAnVhnB9YHEB1TW3racNH9mJvXjKRPZjzZrXsuiqg,15348
 trilogy/core/processing/discovery_utility.py,sha256=Xntgug6VnEF96uw5Zwen1qMEUwKjqrm_ZDUr4i4tc1U,5595
 trilogy/core/processing/discovery_validation.py,sha256=eZ4HfHMpqZLI8MGG2jez8arS8THs6ceuVrQFIY6gXrU,5364
 trilogy/core/processing/graph_utils.py,sha256=8QUVrkE9j-9C1AyrCb1nQEh8daCe0u1HuXl-Te85lag,1205
@@ -50,17 +51,17 @@ trilogy/core/processing/node_generators/group_to_node.py,sha256=jKcNCDOY6fNblrdZ
 trilogy/core/processing/node_generators/multiselect_node.py,sha256=GWV5yLmKTe1yyPhN60RG1Rnrn4ktfn9lYYXi_FVU4UI,7061
 trilogy/core/processing/node_generators/node_merge_node.py,sha256=1joMV7XpQ9Gpe-d5y7JUMBHIqakV5wFJi3Mtvs4UcL4,23415
 trilogy/core/processing/node_generators/recursive_node.py,sha256=l5zdh0dURKwmAy8kK4OpMtZfyUEQRk6N-PwSWIyBpSM,2468
-trilogy/core/processing/node_generators/rowset_node.py,sha256=5L5u6xz1In8EaHQdcYgR2si-tz9WB9YLXURo4AkUT9A,6630
+trilogy/core/processing/node_generators/rowset_node.py,sha256=T11Rqj-tsfubjFvBO0rzIVxtv9tOwwKXjGyut0r9xIY,5919
 trilogy/core/processing/node_generators/select_merge_node.py,sha256=KQvGoNT5ZBWQ_caEomRTtG1PKZC7OPT4PKfY0QmwMGE,22270
 trilogy/core/processing/node_generators/select_node.py,sha256=Ta1G39V94gjX_AgyZDz9OqnwLz4BjY3D6Drx9YpziMQ,3555
 trilogy/core/processing/node_generators/synonym_node.py,sha256=AnAsa_Wj50NJ_IK0HSgab_7klYmKVrv0WI1uUe-GvEY,3766
-trilogy/core/processing/node_generators/union_node.py,sha256=VNo6Oey4p8etU9xrOh2oTT2lIOTvY6PULUPRvVa2uxU,2877
-trilogy/core/processing/node_generators/unnest_node.py,sha256=w9vhPzASz53QPASLqFcLDdR9eY132tgVUcp3QolD5Jw,3726
+trilogy/core/processing/node_generators/union_node.py,sha256=NxQbnRRoYMI4WjMeph41yk4E6yipj53qdGuNt-Mozxw,2818
+trilogy/core/processing/node_generators/unnest_node.py,sha256=7uOZzBidEEKeZE0VW_XlgHGhEYf_snEHtV8UgJ_ZjyY,4048
 trilogy/core/processing/node_generators/window_node.py,sha256=A90linr4pkZtTNfn9k2YNLqrJ_SFII3lbHxB-BC6mI8,6688
 trilogy/core/processing/node_generators/select_helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/processing/node_generators/select_helpers/datasource_injection.py,sha256=m2YQ4OmG0N2O61a7NEq1ZzbTa7JsCC00lxB2ymjcYRI,8224
 trilogy/core/processing/nodes/__init__.py,sha256=zTge1EzwzEydlcMliIFO_TT7h7lS8l37lyZuQDir1h0,5487
-trilogy/core/processing/nodes/base_node.py,sha256=C_CjlOzlGMXckyV0b_PJZerpopNesRCKfambMq7Asvc,18221
+trilogy/core/processing/nodes/base_node.py,sha256=TQZLEz_xfXpdVyFa9R5BwvikH1OqzJUioOPw8vTETWc,18144
 trilogy/core/processing/nodes/filter_node.py,sha256=5VtRfKbCORx0dV-vQfgy3gOEkmmscL9f31ExvlODwvY,2461
 trilogy/core/processing/nodes/group_node.py,sha256=njz-5T7OJ3-kaBC7EhdtPra3G77HnI7apjUwMGhUeXo,10569
 trilogy/core/processing/nodes/merge_node.py,sha256=daJywBxh44Gqk-7eTiXbYtY7xo6O6fNvqX-DagTOTmE,16231
@@ -81,9 +82,9 @@ trilogy/core/validation/datasource.py,sha256=nJeEFyb6iMBwlEVdYVy1vLzAbdRZwOsUjGx
 trilogy/core/validation/environment.py,sha256=ymvhQyt7jLK641JAAIQkqjQaAmr9C5022ILzYvDgPP0,2835
 trilogy/core/validation/fix.py,sha256=Z818UFNLxndMTLiyhB3doLxIfnOZ-16QGvVFWuD7UsA,3750
 trilogy/dialect/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/dialect/base.py,sha256=d2gXfa5Jh3uyN9H9MxG53JT-xQQgntq2X7EprobJYUc,49698
+trilogy/dialect/base.py,sha256=mgARj-aldkFAqdwps_25da03NLIAxU6Xg9Jq_VcOtp0,50181
 trilogy/dialect/bigquery.py,sha256=XS3hpybeowgfrOrkycAigAF3NX2YUzTzfgE6f__2fT4,4316
-trilogy/dialect/common.py,sha256=_MarnMWRBn3VcNt3k5VUdFrwH6oHzGdNQquSpHNLq4o,5644
+trilogy/dialect/common.py,sha256=cUI7JMmpG_A5KcaxRI-GoyqwLMD6jTf0JJhgcOdwQK4,5833
 trilogy/dialect/config.py,sha256=olnyeVU5W5T6b9-dMeNAnvxuPlyc2uefb7FRME094Ec,3834
 trilogy/dialect/dataframe.py,sha256=RUbNgReEa9g3pL6H7fP9lPTrAij5pkqedpZ99D8_5AE,1522
 trilogy/dialect/duckdb.py,sha256=JoUvQ19WvgxoaJkGLM7DPXOd1H0394k3vBiblksQzOI,5631
@@ -99,26 +100,26 @@ trilogy/hooks/graph_hook.py,sha256=5BfR7Dt0bgEsCLgwjowgCsVkboGYfVJGOz8g9mqpnos,4
 trilogy/hooks/query_debugger.py,sha256=1npRjww94sPV5RRBBlLqMJRaFkH9vhEY6o828MeoEcw,5583
 trilogy/metadata/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/parsing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/parsing/common.py,sha256=550-L0444GUuBFdiDWkOg_DxnMXtcJFUMES2R5zlwik,31026
+trilogy/parsing/common.py,sha256=NJLm31J3W9BLWq1ClhNvYE43jrF950698KJ3o0UfSCo,31340
 trilogy/parsing/config.py,sha256=Z-DaefdKhPDmSXLgg5V4pebhSB0h590vI0_VtHnlukI,111
 trilogy/parsing/exceptions.py,sha256=Xwwsv2C9kSNv2q-HrrKC1f60JNHShXcCMzstTSEbiCw,154
 trilogy/parsing/helpers.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-trilogy/parsing/parse_engine.py,sha256=2k1TvnBYE_CW5zCmNfVbf1aWBuMDm5Wz4QfKKgGnE5k,81824
-trilogy/parsing/render.py,sha256=E8-R0zO40QoeTeVX9OYdi5e9YgRYtuRrezDRj7VOgds,20614
-trilogy/parsing/trilogy.lark,sha256=2-jguxgJQnNLbODjTijqrXXzFZ_UlivTdiYhec2YWuc,16451
+trilogy/parsing/parse_engine.py,sha256=T-3Q4UH256IB6cfX85crScZwZ6gAwslgv0fy3WKBdjc,81930
+trilogy/parsing/render.py,sha256=IklKMdXiqQEB6D28PrU1BewlDwD88Hnmqn1xjA9h720,23863
+trilogy/parsing/trilogy.lark,sha256=6eBDD6d4D9N1Nnn4CtmaoB-NpOpjHrEn5oi0JykAlbE,16509
 trilogy/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/scripts/trilogy.py,sha256=1L0XrH4mVHRt1C9T1HnaDv2_kYEfbWTb5_-cBBke79w,3774
 trilogy/std/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/std/date.preql,sha256=HWZm4t4HWyxr5geWRsY05RnHBVDMci8z8YA2cu0-OOw,188
-trilogy/std/display.preql,sha256=nm7lox87Xf6lBvXCVCS6x2HskguMKzndEBucJ5pktzk,175
+trilogy/std/display.preql,sha256=S20HW8qbShBc4OZPcHYiRlLdcaBp9dwruozWBoXKscs,293
 trilogy/std/geography.preql,sha256=1A9Sq5PPMBnEPPf7f-rPVYxJfsnWpQ8oV_k4Fm3H2dU,675
 trilogy/std/metric.preql,sha256=DRECGhkMyqfit5Fl4Ut9zbWrJuSMI1iO9HikuyoBpE0,421
 trilogy/std/money.preql,sha256=XWwvAV3WxBsHX9zfptoYRnBigcfYwrYtBHXTME0xJuQ,2082
 trilogy/std/net.preql,sha256=WZCuvH87_rZntZiuGJMmBDMVKkdhTtxeHOkrXNwJ1EE,416
 trilogy/std/ranking.preql,sha256=LDoZrYyz4g3xsII9XwXfmstZD-_92i1Eox1UqkBIfi8,83
 trilogy/std/report.preql,sha256=LbV-XlHdfw0jgnQ8pV7acG95xrd1-p65fVpiIc-S7W4,202
-pytrilogy-0.0.3.101.dist-info/METADATA,sha256=dkvyYmeCXSZl2uHkPpoy-R7HdKb2w7pLGFrDu1tRGEU,11811
-pytrilogy-0.0.3.101.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-pytrilogy-0.0.3.101.dist-info/entry_points.txt,sha256=ewBPU2vLnVexZVnB-NrVj-p3E-4vukg83Zk8A55Wp2w,56
-pytrilogy-0.0.3.101.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
-pytrilogy-0.0.3.101.dist-info/RECORD,,
+pytrilogy-0.0.3.103.dist-info/METADATA,sha256=RnMfz8EH2sCtqHEDAraYhAb_V7oPbovtuI3PsL2F3Ms,11839
+pytrilogy-0.0.3.103.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+pytrilogy-0.0.3.103.dist-info/entry_points.txt,sha256=ewBPU2vLnVexZVnB-NrVj-p3E-4vukg83Zk8A55Wp2w,56
+pytrilogy-0.0.3.103.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
+pytrilogy-0.0.3.103.dist-info/RECORD,,

trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.dialect.enums import Dialects
 from trilogy.executor import Executor
 from trilogy.parser import parse
 
-__version__ = "0.0.3.101"
+__version__ = "0.0.3.103"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

trilogy/core/models/author.py

@@ -259,6 +259,15 @@ class Parenthetical(
             )
         )
 
+    def with_reference_replacement(self, source, target):
+        return Parenthetical.model_construct(
+            content=(
+                self.content.with_reference_replacement(source, target)
+                if isinstance(self.content, Mergeable)
+                else self.content
+            )
+        )
+
     @property
     def concept_arguments(self) -> Sequence[ConceptRef]:
         base: List[ConceptRef] = []

trilogy/core/models/build.py

@@ -1511,7 +1511,10 @@ def requires_concept_nesting(
 ) -> AggregateWrapper | WindowItem | FilterItem | Function | None:
     if isinstance(expr, (AggregateWrapper, WindowItem, FilterItem)):
         return expr
-    if isinstance(expr, Function) and expr.operator == FunctionType.GROUP:
+    if isinstance(expr, Function) and expr.operator in (
+        FunctionType.GROUP,
+        FunctionType.PARENTHETICAL,
+    ):
         # group by requires nesting
         return expr
     return None
@@ -1696,13 +1699,12 @@ class Factory:
         return self._build_case_when(base)
 
     def _build_case_when(self, base: CaseWhen) -> BuildCaseWhen:
-        comparison = base.comparison
         expr: Concept | FuncArgs = base.expr
         validation = requires_concept_nesting(expr)
         if validation:
             expr, _ = self.instantiate_concept(validation)
         return BuildCaseWhen(
-            comparison=self.build(comparison),
+            comparison=self.build(base.comparison),
             expr=self.build(expr),
         )
 
@@ -2019,7 +2021,12 @@ class Factory:
         return self._build_parenthetical(base)
 
     def _build_parenthetical(self, base: Parenthetical) -> BuildParenthetical:
-        return BuildParenthetical(content=(self.build(base.content)))
+        validate = requires_concept_nesting(base.content)
+        if validate:
+            content, _ = self.instantiate_concept(validate)
+            return BuildParenthetical(content=self.build(content))
+        else:
+            return BuildParenthetical(content=self.build(base.content))
 
     @build.register
     def _(self, base: SelectLineage) -> BuildSelectLineage:

trilogy/core/optimization.py

@@ -5,6 +5,7 @@ from trilogy.core.models.build import (
 )
 from trilogy.core.models.execute import CTE, RecursiveCTE, UnionCTE
 from trilogy.core.optimizations import (
+    HideUnusedConcepts,
     InlineDatasource,
     OptimizationRule,
     PredicatePushdown,
@@ -84,11 +85,18 @@ def filter_irrelevant_ctes(
         #                 child.existence_source_map[x2].append(parent.name)
         # else:
         relevant_ctes.add(cte.name)
-        for cte in cte.parent_ctes:
-            recurse(cte, inverse_map)
+
+        for parent in cte.parent_ctes:
+            if parent.name in relevant_ctes:
+                logger.info(
+                    f"[Optimization][Irrelevent CTE filtering] Already visited {parent.name} when visting {cte.name}, potential recursive dag"
+                )
+                continue
+
+            recurse(parent, inverse_map)
         if isinstance(cte, UnionCTE):
-            for cte in cte.internal_ctes:
-                recurse(cte, inverse_map)
+            for internal in cte.internal_ctes:
+                recurse(internal, inverse_map)
 
     inverse_map = gen_inverse_map(input)
     recurse(root_cte, inverse_map)
@@ -220,6 +228,7 @@ def optimize_ctes(
         REGISTERED_RULES.append(PredicatePushdown())
     if CONFIG.optimizations.predicate_pushdown:
         REGISTERED_RULES.append(PredicatePushdownRemove())
+    REGISTERED_RULES.append(HideUnusedConcepts())
     for rule in REGISTERED_RULES:
         loops = 0
         complete = False

trilogy/core/optimizations/__init__.py

@@ -1,4 +1,5 @@
 from .base_optimization import OptimizationRule
+from .hide_unused_concept import HideUnusedConcepts
 from .inline_datasource import InlineDatasource
 from .predicate_pushdown import PredicatePushdown, PredicatePushdownRemove
 
@@ -7,4 +8,5 @@ __all__ = [
     "InlineDatasource",
     "PredicatePushdown",
     "PredicatePushdownRemove",
+    "HideUnusedConcepts",
 ]

trilogy/core/optimizations/hide_unused_concept.py

@@ -0,0 +1,51 @@
+from trilogy.core.models.build import (
+    BuildConcept,
+)
+from trilogy.core.models.execute import CTE, UnionCTE
+from trilogy.core.optimizations.base_optimization import OptimizationRule
+
+
+class HideUnusedConcepts(OptimizationRule):
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+
+    def optimize(
+        self, cte: CTE | UnionCTE, inverse_map: dict[str, list[CTE | UnionCTE]]
+    ) -> bool:
+        used = set()
+        from trilogy.dialect.base import BaseDialect
+
+        renderer = BaseDialect()
+        children = inverse_map.get(cte.name, [])
+        if not children:
+            return False
+        for v in children:
+            self.log(f"Analyzing usage of {cte.name} in {v.name}")
+            renderer.render_cte(v)
+        used = renderer.used_map.get(cte.name, set())
+        self.log(f"Used concepts for {cte.name}: {used} from {renderer.used_map}")
+        add_to_hidden: list[BuildConcept] = []
+        for concept in cte.output_columns:
+            if concept.address not in used:
+                add_to_hidden.append(concept)
+        newly_hidden = [
+            x.address for x in add_to_hidden if x.address not in cte.hidden_concepts
+        ]
+        non_hidden = [
+            x for x in cte.output_columns if x.address not in cte.hidden_concepts
+        ]
+        if not newly_hidden or len(non_hidden) <= 1:
+            return False
+        self.log(
+            f"Hiding unused concepts {[x.address for x in add_to_hidden]} from {cte.name} (used: {used}, all: {[x.address for x in cte.output_columns]})"
+        )
+        candidates = [
+            x.address
+            for x in cte.output_columns
+            if x.address not in used and x.address not in cte.hidden_concepts
+        ]
+        if len(candidates) == len(set([x.address for x in cte.output_columns])):
+            # pop one out
+            candidates.pop()
+        cte.hidden_concepts = set(candidates)
+        return True

trilogy/core/processing/discovery_node_factory.py

@@ -376,11 +376,6 @@ class RootNodeHandler:
 
         if pseudonyms:
             expanded.add_output_concepts(pseudonyms)
-            logger.info(
-                f"{depth_to_prefix(self.ctx.depth)}{LOGGER_PREFIX} "
-                f"Hiding pseudonyms {[c.address for c in pseudonyms]}"
-            )
-            expanded.hide_output_concepts(pseudonyms)
 
         logger.info(
             f"{depth_to_prefix(self.ctx.depth)}{LOGGER_PREFIX} "

trilogy/core/processing/node_generators/rowset_node.py

@@ -64,14 +64,6 @@ def gen_rowset_node(
         v for v in concept_pool if v.address in rowset_outputs
     ]
 
-    select_hidden = node.hidden_concepts
-    rowset_hidden = [
-        x
-        for x in rowset_relevant
-        if x.address in lineage.rowset.derived_concepts
-        and isinstance(x.lineage, BuildRowsetItem)
-        and x.lineage.content.address in select_hidden
-    ]
     additional_relevant = [
         factory.build(x) for x in select.output_components if x.address in enrichment
     ]
@@ -84,18 +76,6 @@ def gen_rowset_node(
             )
             node.partial_concepts.append(item)
 
-    final_hidden = rowset_hidden + [
-        x
-        for x in node.output_concepts
-        if x.address not in local_optional + [concept]
-        and x.derivation != Derivation.ROWSET
-        and not any(z in lineage.rowset.derived_concepts for z in x.pseudonyms)
-    ]
-    logger.info(
-        f"{padding(depth)}{LOGGER_PREFIX} hiding {final_hidden} local optional {local_optional}"
-    )
-    node.hide_output_concepts(final_hidden)
-
     node.grain = BuildGrain.from_concepts(
         [
             x

trilogy/core/processing/node_generators/union_node.py

@@ -74,7 +74,7 @@ def gen_union_node(
             history=history,
             conditions=conditions,
         )
-        parent.hide_output_concepts(parent.output_concepts)
+
         parent.add_output_concepts(resolved)
         parent_nodes.append(parent)
         if not parent:

trilogy/core/processing/node_generators/unnest_node.py

@@ -1,9 +1,19 @@
 from typing import List
 
 from trilogy.constants import logger
-from trilogy.core.models.build import BuildConcept, BuildFunction, BuildWhereClause
+from trilogy.core.models.build import (
+    BuildConcept,
+    BuildFunction,
+    BuildGrain,
+    BuildWhereClause,
+)
 from trilogy.core.models.build_environment import BuildEnvironment
-from trilogy.core.processing.nodes import History, StrategyNode, UnnestNode
+from trilogy.core.processing.nodes import (
+    History,
+    StrategyNode,
+    UnnestNode,
+    WhereSafetyNode,
+)
 from trilogy.core.processing.utility import padding
 
 LOGGER_PREFIX = "[GEN_UNNEST_NODE]"
@@ -71,7 +81,9 @@ def gen_unnest_node(
                 return None
     else:
         parent = None
-
+    logger.info(
+        f"{depth_prefix}{LOGGER_PREFIX} unnest node for {concept} got parent {parent}"
+    )
     base = UnnestNode(
         unnest_concepts=[concept] + equivalent_optional,
         input_concepts=arguments + non_equivalent_optional,
@@ -83,7 +95,7 @@ def gen_unnest_node(
     # as unnest operations are not valid in all situations
     # TODO: inline this node when we can detect it's safe
     conditional = conditions.conditional if conditions else None
-    new = StrategyNode(
+    new = WhereSafetyNode(
         input_concepts=base.output_concepts,
         output_concepts=base.output_concepts,
         environment=environment,
@@ -92,9 +104,13 @@ def gen_unnest_node(
         preexisting_conditions=(
             conditional if conditional and local_conditions is False else None
         ),
+        grain=BuildGrain.from_concepts(
+            concepts=base.output_concepts,
+            environment=environment,
+        ),
     )
-    qds = new.resolve()
-    assert qds.source_map[concept.address] == {base.resolve()}
-    for x in equivalent_optional:
-        assert qds.source_map[x.address] == {base.resolve()}
+    # qds = new.resolve()
+    # assert qds.source_map[concept.address] == {base.resolve()}
+    # for x in equivalent_optional:
+    #     assert qds.source_map[x.address] == {base.resolve()}
     return new

trilogy/core/processing/nodes/base_node.py

@@ -489,7 +489,6 @@ class WhereSafetyNode(StrategyNode):
             parent = parent.copy()
             # avoid performance hit by not rebuilding until end
             parent.set_output_concepts(self.output_concepts, rebuild=False)
-            parent.hide_output_concepts(self.hidden_concepts, rebuild=False)
 
             # these conditions
             if self.preexisting_conditions:

trilogy/dialect/base.py

@@ -1,3 +1,4 @@
+from collections import defaultdict
 from datetime import date, datetime
 from typing import Any, Callable, Dict, List, Optional, Sequence, Union
 
@@ -311,6 +312,7 @@ def safe_get_cte_value(
     c: BuildConcept,
     quote_char: str,
     render_expr: Callable,
+    use_map: dict[str, set[str]],
 ) -> Optional[str]:
     address = c.address
     raw = cte.source_map.get(address, None)
@@ -319,13 +321,17 @@ def safe_get_cte_value(
         return None
     if isinstance(raw, str):
         rendered = cte.get_alias(c, raw)
+        use_map[raw].add(c.address)
         return f"{quote_char}{raw}{quote_char}.{safe_quote(rendered, quote_char)}"
     if isinstance(raw, list) and len(raw) == 1:
         rendered = cte.get_alias(c, raw[0])
         if isinstance(rendered, FUNCTION_ITEMS):
             # if it's a function, we need to render it as a function
             return f"{render_expr(rendered, cte=cte, raise_invalid=True)}"
+        use_map[raw[0]].add(c.address)
         return f"{quote_char}{raw[0]}{quote_char}.{safe_quote(rendered, quote_char)}"
+    for x in raw:
+        use_map[x].add(c.address)
     return coalesce(
         sorted(
             [
@@ -350,13 +356,12 @@ class BaseDialect:
 
     def __init__(self, rendering: Rendering | None = None):
         self.rendering = rendering or CONFIG.rendering
+        self.used_map: dict[str, set[str]] = defaultdict(set)
 
     def render_order_item(
         self,
         order_item: BuildOrderItem,
         cte: CTE | UnionCTE,
-        final: bool = False,
-        alias: bool = True,
     ) -> str:
         # if final:
         #     if not alias:
@@ -527,6 +532,9 @@ class BaseDialect:
             )
 
             raw_content = cte.get_alias(c)
+            parent = cte.source_map.get(c.address, None)
+            if parent:
+                self.used_map[parent[0]].add(c.address)
             if isinstance(raw_content, RawColumnExpr):
                 rval = raw_content.text
             elif isinstance(raw_content, FUNCTION_ITEMS):
@@ -540,6 +548,7 @@ class BaseDialect:
                     c,
                     self.QUOTE_CHARACTER,
                     self.render_expr,
+                    self.used_map,
                 )
                 if not rval:
                     # unions won't have a specific source mapped; just use a generic column reference
@@ -615,6 +624,7 @@ class BaseDialect:
                 lookup_cte = cte
                 if cte_map and not lookup_cte:
                     lookup_cte = cte_map.get(e.right.address)
+
                 assert lookup_cte, "Subselects must be rendered with a CTE in context"
                 if e.right.address not in lookup_cte.existence_source_map:
                     lookup = lookup_cte.source_map.get(
@@ -634,6 +644,7 @@ class BaseDialect:
                         f"Missing source CTE for {e.right.address}"
                     )
                 assert cte, "CTE must be provided for inlined CTEs"
+                self.used_map[target].add(e.right.address)
                 if target in cte.inlined_ctes:
                     info = cte.inlined_ctes[target]
                     return f"{self.render_expr(e.left, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid)} {e.operator.value} (select {target}.{self.QUOTE_CHARACTER}{e.right.safe_address}{self.QUOTE_CHARACTER} from {info.new_base} as {target} where {target}.{self.QUOTE_CHARACTER}{e.right.safe_address}{self.QUOTE_CHARACTER} is not null)"
@@ -738,6 +749,7 @@ class BaseDialect:
                     raise_invalid=raise_invalid,
                 )
             elif cte_map:
+                self.used_map[cte_map[e.address].name].add(e.address)
                 return f"{cte_map[e.address].name}.{self.QUOTE_CHARACTER}{e.safe_address}{self.QUOTE_CHARACTER}"
             return f"{self.QUOTE_CHARACTER}{e.safe_address}{self.QUOTE_CHARACTER}"
         elif isinstance(e, bool):
@@ -822,10 +834,7 @@ class BaseDialect:
             )
             if cte.order_by:
 
-                ordering = [
-                    self.render_order_item(i, cte, final=True, alias=False)
-                    for i in cte.order_by.items
-                ]
+                ordering = [self.render_order_item(i, cte) for i in cte.order_by.items]
                 base_statement += "\nORDER BY " + ",".join(ordering)
             return CompiledCTE(name=cte.name, statement=base_statement)
         elif isinstance(cte, RecursiveCTE):
@@ -950,7 +959,8 @@ class BaseDialect:
                             self.QUOTE_CHARACTER,
                             self.render_expr,
                             cte,
-                            self.UNNEST_MODE,
+                            use_map=self.used_map,
+                            unnest_mode=self.UNNEST_MODE,
                         )
                         for join in final_joins
                     ]

trilogy/dialect/common.py

@@ -62,10 +62,12 @@ def render_join_concept(
     concept: BuildConcept,
     render_expr,
     inlined_ctes: set[str],
+    use_map: dict[str, set[str]],
 ):
     if cte.name in inlined_ctes:
         base = render_expr(concept, cte)
         return base
+    use_map[name].add(concept.address)
     return f"{quote_character}{name}{quote_character}.{quote_character}{concept.safe_address}{quote_character}"
 
 
@@ -85,6 +87,7 @@ def render_join(
         str,
     ],
     cte: CTE,
+    use_map: dict[str, set[str]],
     unnest_mode: UnnestMode = UnnestMode.CROSS_APPLY,
 ) -> str | None:
     # {% for key in join.joinkeys %}{{ key.inner }} = {{ key.outer}}{% endfor %}
@@ -121,6 +124,7 @@ def render_join(
                         pair.left,
                         render_expr_func,
                         join.inlined_ctes,
+                        use_map=use_map,
                     ),
                     render_join_concept(
                         right_name,
@@ -129,6 +133,7 @@ def render_join(
                         pair.right,
                         render_expr_func,
                         join.inlined_ctes,
+                        use_map=use_map,
                     ),
                     modifiers=pair.modifiers
                     + (pair.left.modifiers or [])

trilogy/parsing/common.py

@@ -249,11 +249,14 @@ def atom_is_relevant(
         return atom_is_relevant(atom.left, others, environment) or atom_is_relevant(
             atom.right, others, environment
         )
+    elif isinstance(atom, Parenthetical):
+        return atom_is_relevant(atom.content, others, environment)
     elif isinstance(atom, ConceptArgs):
         # use atom is relevant here to trigger the early exit behavior for concepts in set
         return any(
             [atom_is_relevant(x, others, environment) for x in atom.concept_arguments]
         )
+
     return False
 
 
@@ -294,12 +297,18 @@ def concept_is_relevant(
         if all([c in others for c in concept.grain.components]):
             return False
     if concept.derivation in (Derivation.BASIC,) and isinstance(
-        concept.lineage, Function
+        concept.lineage, (Function, CaseWhen)
     ):
         relevant = False
         for arg in concept.lineage.arguments:
             relevant = atom_is_relevant(arg, others, environment) or relevant
+
         return relevant
+    if concept.derivation in (Derivation.BASIC,) and isinstance(
+        concept.lineage, Parenthetical
+    ):
+        return atom_is_relevant(concept.lineage.content, others, environment)
+
     if concept.granularity == Granularity.SINGLE_ROW:
         return False
     return True
@@ -346,6 +355,7 @@ def concepts_to_grain_concepts(
         if sub.address in seen:
             continue
         if not concept_is_relevant(sub, pconcepts, environment):  # type: ignore
+
             continue
         seen.add(sub.address)
 

trilogy/parsing/parse_engine.py

@@ -992,6 +992,9 @@ class ParseToObjects(Transformer):
     def order_by(self, args):
         return OrderBy(items=args[0])
 
+    def over_component(self, args):
+        return ConceptRef(address=args[0].value.lstrip(",").strip())
+
     def over_list(self, args):
         return [x for x in args]
 

trilogy/parsing/render.py

@@ -1,4 +1,6 @@
 from collections import defaultdict
+from contextlib import contextmanager
+from dataclasses import dataclass
 from datetime import date, datetime
 from functools import singledispatchmethod
 from typing import Any
@@ -23,6 +25,7 @@ from trilogy.core.models.author import (
     FunctionCallWrapper,
     Grain,
     OrderBy,
+    Ordering,
     OrderItem,
     Parenthetical,
     SubselectComparison,
@@ -67,23 +70,72 @@ from trilogy.core.statements.author import (
 
 QUERY_TEMPLATE = Template(
     """{% if where %}WHERE
-    {{ where }}
+{{ where }}
 {% endif %}SELECT{%- for select in select_columns %}
-    {{ select }},{% endfor %}{% if having %}
+{{ select }},{% endfor %}{% if having %}
 HAVING
-    {{ having }}
+{{ having }}
 {% endif %}{%- if order_by %}
 ORDER BY{% for order in order_by %}
-    {{ order }}{% if not loop.last %},{% endif %}{% endfor %}{% endif %}{%- if limit is not none %}
+{{ order }}{% if not loop.last %},{% endif %}{% endfor %}{% endif %}{%- if limit is not none %}
 LIMIT {{ limit }}{% endif %}
 ;"""
 )
 
 
+@dataclass
+class IndentationContext:
+    """Tracks indentation state during rendering"""
+
+    depth: int = 0
+    indent_string: str = "    "  # 4 spaces by default
+
+    @property
+    def current_indent(self) -> str:
+        return self.indent_string * self.depth
+
+    def increase_depth(self, extra_levels: int = 1) -> "IndentationContext":
+        return IndentationContext(
+            depth=self.depth + extra_levels, indent_string=self.indent_string
+        )
+
+
 class Renderer:
 
-    def __init__(self, environment: Environment | None = None):
+    def __init__(
+        self, environment: Environment | None = None, indent_string: str = "    "
+    ):
         self.environment = environment
+        self.indent_context = IndentationContext(indent_string=indent_string)
+
+    @contextmanager
+    def indented(self, levels: int = 1):
+        """Context manager for temporarily increasing indentation"""
+        old_context = self.indent_context
+        self.indent_context = self.indent_context.increase_depth(levels)
+        try:
+            yield
+        finally:
+            self.indent_context = old_context
+
+    def indent_lines(self, text: str, extra_levels: int = 0) -> str:
+        """Apply current indentation to all lines in text"""
+        if not text:
+            return text
+
+        indent = self.indent_context.indent_string * (
+            self.indent_context.depth + extra_levels
+        )
+        lines = text.split("\n")
+        indented_lines = []
+
+        for line in lines:
+            if line.strip():  # Only indent non-empty lines
+                indented_lines.append(indent + line)
+            else:
+                indented_lines.append(line)  # Keep empty lines as-is
+
+        return "\n".join(indented_lines)
 
     def render_statement_string(self, list_of_statements: list[Any]) -> str:
         new = []
@@ -98,7 +150,7 @@ class Renderer:
                 new.append("\n\n")
             else:
                 new.append("\n")
-            new.append(Renderer().to_string(stmt))
+            new.append(self.to_string(stmt))
             last_statement_type = stmt_type
         return "".join(new)
 
@@ -192,14 +244,19 @@ class Renderer:
 
     @to_string.register
     def _(self, arg: Datasource):
-        assignments = ",\n    ".join([self.to_string(x) for x in arg.columns])
+        with self.indented():
+            assignments = ",\n".join(
+                [self.indent_lines(self.to_string(x)) for x in arg.columns]
+            )
+
         if arg.non_partial_for:
             non_partial = f"\ncomplete where {self.to_string(arg.non_partial_for)}"
         else:
             non_partial = ""
+
         base = f"""datasource {arg.name} (
-    {assignments}
-    )
+{assignments}
+)
 {self.to_string(arg.grain) if arg.grain.components else ''}{non_partial}
 {self.to_string(arg.address)}"""
 
@@ -390,26 +447,45 @@ class Renderer:
 
     @to_string.register
     def _(self, arg: SelectStatement):
+        with self.indented():
+            select_columns = [
+                self.indent_lines(self.to_string(c)) for c in arg.selection
+            ]
+            where_clause = None
+            if arg.where_clause:
+                where_clause = self.indent_lines(self.to_string(arg.where_clause))
+            having_clause = None
+            if arg.having_clause:
+                having_clause = self.indent_lines(self.to_string(arg.having_clause))
+            order_by = None
+            if arg.order_by:
+                order_by = [
+                    self.indent_lines(self.to_string(c)) for c in arg.order_by.items
+                ]
+
         return QUERY_TEMPLATE.render(
-            select_columns=[self.to_string(c) for c in arg.selection],
-            where=self.to_string(arg.where_clause) if arg.where_clause else None,
-            having=self.to_string(arg.having_clause) if arg.having_clause else None,
-            order_by=(
-                [self.to_string(c) for c in arg.order_by.items]
-                if arg.order_by
-                else None
-            ),
+            select_columns=select_columns,
+            where=where_clause,
+            having=having_clause,
+            order_by=order_by,
             limit=arg.limit,
         )
 
     @to_string.register
     def _(self, arg: MultiSelectStatement):
-        base = "\nMERGE\n".join([self.to_string(select)[:-2] for select in arg.selects])
+        # Each select gets its own indentation
+        select_parts = []
+        for select in arg.selects:
+            select_parts.append(
+                self.to_string(select)[:-2]
+            )  # Remove the trailing ";\n"
+
+        base = "\nMERGE\n".join(select_parts)
         base += self.to_string(arg.align)
         if arg.where_clause:
             base += f"\nWHERE\n{self.to_string(arg.where_clause)}"
         if arg.order_by:
-            base += f"\nORDER BY\n\t{self.to_string(arg.order_by)}"
+            base += f"\nORDER BY\n{self.to_string(arg.order_by)}"
         if arg.limit:
             base += f"\nLIMIT {arg.limit}"
         base += "\n;"
@@ -421,7 +497,9 @@ class Renderer:
 
     @to_string.register
     def _(self, arg: AlignClause):
-        return "\nALIGN\n\t" + ",\n\t".join([self.to_string(c) for c in arg.items])
+        with self.indented():
+            align_items = [self.indent_lines(self.to_string(c)) for c in arg.items]
+        return "\nALIGN\n" + ",\n".join(align_items)
 
     @to_string.register
     def _(self, arg: AlignItem):
@@ -429,7 +507,13 @@ class Renderer:
 
     @to_string.register
     def _(self, arg: OrderBy):
-        return ",\n".join([self.to_string(c) for c in arg.items])
+        with self.indented():
+            order_items = [self.indent_lines(self.to_string(c)) for c in arg.items]
+        return ",\n".join(order_items)
+
+    @to_string.register
+    def _(self, arg: Ordering):
+        return arg.value
 
     @to_string.register
     def _(self, arg: "WhereClause"):
@@ -469,7 +553,6 @@ class Renderer:
 
     @to_string.register
     def _(self, arg: "FilterItem"):
-
         return f"filter {self.to_string(arg.content)} where {self.to_string(arg.where)}"
 
     @to_string.register
@@ -538,18 +621,34 @@ class Renderer:
             if len(args) == 1:
                 return f"group({args[0]})"
             return f"group({args[0]}) by {arg_string}"
-        inputs = ",".join(args)
 
         if arg.operator == FunctionType.CONSTANT:
-            return f"{inputs}"
+            return f"{', '.join(args)}"
         if arg.operator == FunctionType.CAST:
             return f"CAST({self.to_string(arg.arguments[0])} AS {self.to_string(arg.arguments[1])})"
         if arg.operator == FunctionType.INDEX_ACCESS:
             return f"{self.to_string(arg.arguments[0])}[{self.to_string(arg.arguments[1])}]"
 
         if arg.operator == FunctionType.CASE:
-            inputs = "\n\t".join(args)
-            return f"CASE\n\t{inputs}\nEND"
+            with self.indented():
+                indented_args = [
+                    self.indent_lines(self.to_string(a)) for a in arg.arguments
+                ]
+            inputs = "\n".join(indented_args)
+            return f"CASE\n{inputs}\n{self.indent_context.current_indent}END"
+
+        if arg.operator == FunctionType.STRUCT:
+            # zip arguments to pairs
+            input_pairs = zip(arg.arguments[0::2], arg.arguments[1::2])
+            with self.indented():
+                pair_strings = []
+                for k, v in input_pairs:
+                    pair_line = f"{self.to_string(k)}-> {v}"
+                    pair_strings.append(self.indent_lines(pair_line))
+            inputs = ",\n".join(pair_strings)
+            return f"struct(\n{inputs}\n{self.indent_context.current_indent})"
+
+        inputs = ",".join(args)
         return f"{arg.operator.value}({inputs})"
 
     @to_string.register

trilogy/parsing/trilogy.lark

@@ -139,7 +139,9 @@
     
     order_list: _order_atom (","  _order_atom)* ","?
     
-    over_list: (concept_lit ",")*  concept_lit ","?
+    over_component: /,\s*[a-zA-Z\_][a-zA-Z0-9\_\.]*/ "END"?
+    
+    over_list: concept_lit  over_component* 
     
     ORDERING_DIRECTION: /ASC|DESC/i
 

trilogy/std/display.preql

@@ -2,5 +2,8 @@
 
 type percent float; # Percentage value
 
-def calc_percent(a, b, digits=-1) -> case when digits =-1 then (a/b):: float::percent
-else round((a/b):: float::percent, digits) end;
+def calc_percent(a, b, digits=-1) -> case when digits =-1 then 
+case when b = 0 then 0.0::float::percent else
+(a/b)::float::percent end
+else round((case when b = 0 then 0.0::float::percent else
+(a/b)::float::percent end):: float::percent, digits) end;