pytrilogy 0.0.1.107__py3-none-any.whl → 0.0.1.108__py3-none-any.whl

{pytrilogy-0.0.1.107.dist-info → pytrilogy-0.0.1.108.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.1.107
+Version: 0.0.1.108
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.1.107.dist-info → pytrilogy-0.0.1.108.dist-info}/RECORD

@@ -1,4 +1,4 @@
-trilogy/__init__.py,sha256=ouq-RNu0DVYw8n1C2ekRmcAJ_SL_PFzbRBzo2O814TM,292
+trilogy/__init__.py,sha256=XUWHIoJWL4bt0ApBe3huE8vvIiYJxxfm4krEBMhpLUc,292
 trilogy/compiler.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/constants.py,sha256=LxiK2TiVQPEa6tXkxWk9DJHOR3zsGNSqgQuqtOf66cw,518
 trilogy/engine.py,sha256=R5ubIxYyrxRExz07aZCUfrTsoXCHQ8DKFTDsobXdWdA,1102
@@ -9,53 +9,52 @@ trilogy/utility.py,sha256=zM__8r29EsyDW7K9VOHz8yvZC2bXFzh7xKy3cL7GKsk,707
 trilogy/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/core/constants.py,sha256=LL8NLvxb3HRnAjvofyLRXqQJijLcYiXAQYQzGarVD-g,128
 trilogy/core/enums.py,sha256=KEZQTzJ8tlGIukuUwQUIG1FTHOP1B4i0EeCgFjfsbDw,5394
-trilogy/core/env_processor.py,sha256=dfiUp5QxcrgG5YI3Py4xP4OTDRZqltWJGQp8PwukkfY,1401
+trilogy/core/env_processor.py,sha256=SU-jpaGfoWLe9sGTeQYG1qjVnwGQ7TwctmnJRlfzluc,1459
 trilogy/core/environment_helpers.py,sha256=mzBDHhdF9ssZ_-LY8CcaM_ddfJavkpRYrFImUd3cjXI,5972
 trilogy/core/ergonomics.py,sha256=w3gwXdgrxNHCuaRdyKg73t6F36tj-wIjQf47WZkHmJk,1465
 trilogy/core/exceptions.py,sha256=NvV_4qLOgKXbpotgRf7c8BANDEvHxlqRPaA53IThQ2o,561
 trilogy/core/functions.py,sha256=zkRReytiotOBAW-a3Ri5eoejZDYTt2-7Op80ZxZxUmw,9129
 trilogy/core/graph_models.py,sha256=oJUMSpmYhqXlavckHLpR07GJxuQ8dZ1VbB1fB0KaS8c,2036
 trilogy/core/internal.py,sha256=jNGFHKENnbMiMCtAgsnLZYVSENDK4b5ALecXFZpTDzQ,1075
-trilogy/core/models.py,sha256=WH7GHgn1a3xegVo12_NTP7V_ptN-_ObY7s4ZgkPI0D4,108548
-trilogy/core/optimization.py,sha256=SpWRQL1biAUvMCijk2I-FCQY2KzXd4eiu3ZlxzVE-uQ,4505
-trilogy/core/query_processor.py,sha256=w_CS2TEmSk8Bhk6ukxLavQgQyA9UwcgoPChouLREujQ,11747
+trilogy/core/models.py,sha256=JvqqYCQorhHRa7haVRBYCGmmjYq4jWKjOgPqbf2eTNI,109050
+trilogy/core/optimization.py,sha256=chfzpLVJo9eg8H4e2hdnpRqWMDTQ3tJWPdDfGESa-EU,4510
+trilogy/core/query_processor.py,sha256=6BqLYPwyFkRtueTIRFZi3IcVFTpbpGRNowayhSn3_AY,11805
 trilogy/core/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/core/processing/concept_strategies_v3.py,sha256=6Z1ODKbvxaWFMRHa3vX87oSBW9XKtC7S7pUxburCVkA,22369
+trilogy/core/processing/concept_strategies_v3.py,sha256=27lZXFLgDEF3sh2MUR7HX_atVz7TC1fJB7z3oxa1TcY,22610
 trilogy/core/processing/graph_utils.py,sha256=ulCJ4hYAISbUxLD6VM2fah9RBPGIXSEHEPeRBSFl0Rs,1197
 trilogy/core/processing/utility.py,sha256=Gk35HgyIG2SSUyI5OHZcB0bw1PZUVC_aNc9Sre6xPQU,10535
-trilogy/core/processing/node_generators/__init__.py,sha256=1L1TWnGkrSQlLe9ZuMG8DMGfS755v0fjCdz_W1ofCJQ,747
+trilogy/core/processing/node_generators/__init__.py,sha256=LIs6uBEum8LDc-26zjyAwjxa-ay2ok9tKtPjDNvbVkE,757
 trilogy/core/processing/node_generators/basic_node.py,sha256=tVPmg0r0kDdABkmn6z4sxsk1hKy9yTT_Xvl1eVN2Zck,2162
 trilogy/core/processing/node_generators/common.py,sha256=RWaynnlCeF8bhGXVLYWpUFrdngMF95TJwCFue73bXZo,8881
-trilogy/core/processing/node_generators/concept_merge.py,sha256=oKxRuc7m6wnsrxAaLg3q2-dM2dyFgAvaXCp6nmG--jU,5113
+trilogy/core/processing/node_generators/concept_merge_node.py,sha256=TRbOIjLWfLB0Nl6YmMV1ao0qhPP6OQDd9M3UViWkCBU,6621
 trilogy/core/processing/node_generators/filter_node.py,sha256=CGALiTzKhPAvXPFAguIQfjf6I3pjlafY0uaaM9MTkIE,3414
 trilogy/core/processing/node_generators/group_node.py,sha256=xWI1xNIXEOj6jlRGD9hcv2_vVNvY6lpzJl6pQ8HuFBE,2988
 trilogy/core/processing/node_generators/group_to_node.py,sha256=BzPdYwzoo8gRMH7BDffTTXq4z-mjfCEzvfB5I-P0_nw,2941
-trilogy/core/processing/node_generators/merge_node.py,sha256=sQQ9jhw1oAJh649DBAJX6U7r_E_piFS95mxKvm7pxqQ,5818
 trilogy/core/processing/node_generators/multiselect_node.py,sha256=vP84dnLQy6dtypi6mUbt9sMAcmmrTgQ1Oz4GI6X1IEo,6421
+trilogy/core/processing/node_generators/node_merge_node.py,sha256=sQQ9jhw1oAJh649DBAJX6U7r_E_piFS95mxKvm7pxqQ,5818
 trilogy/core/processing/node_generators/rowset_node.py,sha256=K-aoLi0OSfNADXR5_vxDcNv2dJeFy30XNp_IaaWWJ6o,4684
 trilogy/core/processing/node_generators/select_node.py,sha256=xeCqIUEubrf3u_QQfbGdf1BG4fO0HYQ64hiFur8NUqY,20080
 trilogy/core/processing/node_generators/unnest_node.py,sha256=s1VXQZSf1LnX3ISeQ5JzmzmCKUw30-5OK_f0YTB9_48,1031
 trilogy/core/processing/node_generators/window_node.py,sha256=ekazi5eXxnShpcp-qukXNG4DHFdULoXrX-YWUWLNEpM,2527
-trilogy/core/processing/nodes/__init__.py,sha256=ZkDGQksvsM5uNia5rhXFCUJcpTRhoYYFdyfJw-Eiu8s,3674
-trilogy/core/processing/nodes/base_node.py,sha256=VaK4rWV8PQMyTPTgStmCbPbV0mmTHyTwBhz0C0N2KG0,8961
+trilogy/core/processing/nodes/__init__.py,sha256=gzKxGSduIQ5QwpMWrmwSYiE8sg2mWejwVn0VvjYc6s0,3879
+trilogy/core/processing/nodes/base_node.py,sha256=075SP0fAqCZGxxE9Rn5-Qfd2mWd9J7NannGFkNfTkuY,9068
 trilogy/core/processing/nodes/filter_node.py,sha256=DqSRv8voEajPZqzeeiIsxuv4ubvsmeQcCW6x_v2CmOk,1359
 trilogy/core/processing/nodes/group_node.py,sha256=Y_NWB_AwFrE-YithjZ7lYYDN4e0el4su3ICq2EIr3HA,3837
-trilogy/core/processing/nodes/merge_node.py,sha256=pAtRTqkpsfE_pmqPSdeV0rHwFPzclJ3WItRLX8AuuJw,12609
+trilogy/core/processing/nodes/merge_node.py,sha256=a9JkWNSb5ikRnr33DJrlIpZ_OXVK7XDF4FFnqNab4wI,12758
 trilogy/core/processing/nodes/select_node_v2.py,sha256=tAADeVruch-flFiedbY1zi7ukMG2RpWecvxxZ5aL3ZU,6354
 trilogy/core/processing/nodes/unnest_node.py,sha256=t4kY3a_dR3iXistPemStfdw0uJfnxwTcoQg1HiDa3xo,1501
 trilogy/core/processing/nodes/window_node.py,sha256=QjAWgqBZqFSRCPwc7JBmgQJobWW50rsHI0pjJe0Zzg0,926
 trilogy/dialect/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/dialect/base.py,sha256=nYrm7Z-GnVVhr5vWHjMghWkGoq2r7ogzoGVasAGTxGo,29223
+trilogy/dialect/base.py,sha256=xPB5mh6471VJLHxNdXrYvi7q7vJC_tioVR1LrLcoZc0,29394
 trilogy/dialect/bigquery.py,sha256=9vxQn2BMv_oTGQSWQpoN5ho_OgqMWaHH9e-5vQVf44c,2906
 trilogy/dialect/common.py,sha256=zWrYmvevlXznocw9uGHmY5Ws1rp_kICm9zA_ulTe4eg,2165
-trilogy/dialect/config.py,sha256=JdGIiHf2EVoFNTYzqQUy1bMmzqZiFTjcnYglzAMa4dM,3351
+trilogy/dialect/config.py,sha256=tLVEMctaTDhUgARKXUNfHUcIolGaALkQ0RavUvXAY4w,2994
 trilogy/dialect/duckdb.py,sha256=Ddyt68sr8IL2HnZMenyytoD65FXwY_O2pz1McyS0bis,3075
 trilogy/dialect/enums.py,sha256=4NdpsydBpDn6jnh0JzFz5VvQEtnShErWtWHVyT6TNpw,3948
 trilogy/dialect/postgres.py,sha256=r47xbCA7nfEYENofiVfLZ-SnReNfDmUmW4OSHVkkP4E,3206
-trilogy/dialect/presto.py,sha256=rOr-ftb0S5AeEncQgp6jj1iCWlCEkzZuQC3TGMUcemg,2790
+trilogy/dialect/presto.py,sha256=8zjRn8AeYXZQGuUi-afyBWLet8o-LSt6gm5IH7bTdiw,2987
 trilogy/dialect/snowflake.py,sha256=N3HknYgN-fjD7BLX1Ucj-ss_ku2Ox8DgLsF3BIHutHo,2941
 trilogy/dialect/sql_server.py,sha256=UrLeA9bxiFJ4qpGsqVJqBybQCyJhetMebe8IzQW1q9s,2900
-trilogy/docs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/hooks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/hooks/base_hook.py,sha256=Xkb-A2qCHozYjum0A36zOy5PwTVwrP3NLDF0U2GpgHo,1100
 trilogy/hooks/graph_hook.py,sha256=i-Tv9sxZU0sMc-God8bLLz-nAg4-wYafogZtHaU8LXw,801
@@ -70,9 +69,9 @@ trilogy/parsing/parse_engine.py,sha256=TLy56pDatDfzfwbJkrJ-XXB05s_VW9_iRrkwtKR0G
 trilogy/parsing/render.py,sha256=fxjpq2FZLgllw_d4cru-t_IXNPAz2DmYkT7v9ED0XRI,11540
 trilogy/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/scripts/trilogy.py,sha256=PHxvv6f2ODv0esyyhWxlARgra8dVhqQhYl0lTrSyVNo,3729
-pytrilogy-0.0.1.107.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
-pytrilogy-0.0.1.107.dist-info/METADATA,sha256=KF68qL9kNj855oUEtUVCCMd5hei-LRfABfoaEWlTa8g,7882
-pytrilogy-0.0.1.107.dist-info/WHEEL,sha256=Z4pYXqR_rTB7OWNDYFOm1qRk0RX6GFP2o8LgvP453Hk,91
-pytrilogy-0.0.1.107.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
-pytrilogy-0.0.1.107.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
-pytrilogy-0.0.1.107.dist-info/RECORD,,
+pytrilogy-0.0.1.108.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
+pytrilogy-0.0.1.108.dist-info/METADATA,sha256=HUP7pQQZBmSTKuUUioE53ys3fCS76Xjwj2CF5SykMOs,7882
+pytrilogy-0.0.1.108.dist-info/WHEEL,sha256=Z4pYXqR_rTB7OWNDYFOm1qRk0RX6GFP2o8LgvP453Hk,91
+pytrilogy-0.0.1.108.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
+pytrilogy-0.0.1.108.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
+pytrilogy-0.0.1.108.dist-info/RECORD,,

trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.1.107"
+__version__ = "0.0.1.108"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

trilogy/core/env_processor.py

@@ -21,6 +21,8 @@ def generate_graph(
             for source in concept.sources:
                 generic = source.with_default_grain()
                 g.add_edge(generic, node_name)
+
+                # link the concept to the generic source
                 if concept.derivation == PurposeLineage.MERGE:
                     g.add_edge(node_name, generic)
     for _, dataset in environment.datasources.items():

trilogy/core/models.py

@@ -1512,7 +1512,7 @@ class MergeStatement(Namespaced, BaseModel):
                 if z.address == x.address:
                     return z
         raise SyntaxError(
-            f"Could not find upstream map for multiselect {str(concept)} on cte ({cte})"
+            f"Could not find upstream map for multiselect {str(concept)} on cte ({cte.name})"
         )
 
     def with_namespace(self, namespace: str) -> "MergeStatement":
@@ -1779,6 +1779,7 @@ class QueryDatasource(BaseModel):
     source_type: SourceType = SourceType.SELECT
     partial_concepts: List[Concept] = Field(default_factory=list)
     join_derived_concepts: List[Concept] = Field(default_factory=list)
+    hidden_concepts: List[Concept] = Field(default_factory=list)
     force_group: bool | None = None
 
     @property
@@ -1997,6 +1998,7 @@ class CTE(BaseModel):
     condition: Optional[Union["Conditional", "Comparison", "Parenthetical"]] = None
     partial_concepts: List[Concept] = Field(default_factory=list)
     join_derived_concepts: List[Concept] = Field(default_factory=list)
+    hidden_concepts: List[Concept] = Field(default_factory=list)
     order_by: Optional[OrderBy] = None
     limit: Optional[int] = None
     requires_nesting: bool = True
@@ -2433,6 +2435,7 @@ class Environment(BaseModel):
     cte_name_map: Dict[str, str] = Field(default_factory=dict)
 
     materialized_concepts: List[Concept] = Field(default_factory=list)
+    merged_concepts: Dict[str, Concept] = Field(default_factory=dict)
     _parse_count: int = 0
 
     @classmethod
@@ -2460,7 +2463,7 @@ class Environment(BaseModel):
             f.write(self.model_dump_json())
         return ppath
 
-    def gen_materialized_concepts(self) -> None:
+    def gen_concept_list_caches(self) -> None:
         concrete_addresses = set()
         for datasource in self.datasources.values():
             for concept in datasource.output_concepts:
@@ -2468,6 +2471,12 @@ class Environment(BaseModel):
         self.materialized_concepts = [
             c for c in self.concepts.values() if c.address in concrete_addresses
         ]
+        for concept in self.concepts.values():
+            if concept.derivation == PurposeLineage.MERGE:
+                ms = concept.lineage
+                assert isinstance(ms, MergeStatement)
+                for parent in ms.concepts:
+                    self.merged_concepts[parent.address] = concept
 
     def validate_concept(self, lookup: str, meta: Meta | None = None):
         existing: Concept = self.concepts.get(lookup)  # type: ignore
@@ -2503,7 +2512,7 @@ class Environment(BaseModel):
             self.concepts[f"{alias}.{key}"] = concept.with_namespace(alias)
         for key, datasource in environment.datasources.items():
             self.datasources[f"{alias}.{key}"] = datasource.with_namespace(alias)
-        self.gen_materialized_concepts()
+        self.gen_concept_list_caches()
         return self
 
     def add_file_import(self, path: str, alias: str, env: Environment | None = None):
@@ -2602,7 +2611,7 @@ class Environment(BaseModel):
             from trilogy.core.environment_helpers import generate_related_concepts
 
             generate_related_concepts(concept, self)
-        self.gen_materialized_concepts()
+        self.gen_concept_list_caches()
         return concept
 
     def add_datasource(
@@ -2612,12 +2621,12 @@ class Environment(BaseModel):
     ):
         if not datasource.namespace or datasource.namespace == DEFAULT_NAMESPACE:
             self.datasources[datasource.name] = datasource
-            self.gen_materialized_concepts()
+            self.gen_concept_list_caches()
             return datasource
         self.datasources[datasource.namespace + "." + datasource.identifier] = (
             datasource
         )
-        self.gen_materialized_concepts()
+        self.gen_concept_list_caches()
         return datasource
 
 

trilogy/core/optimization.py

@@ -136,6 +136,6 @@ def optimize_ctes(
             select.where_clause.conditional if select.where_clause else None
         )
         root_cte.requires_nesting = False
-        sort_select_output(cte, select)
+        sort_select_output(root_cte, select)
 
     return filter_irrelevant_ctes(input, root_cte)

trilogy/core/processing/concept_strategies_v3.py

@@ -337,7 +337,9 @@ def generate_node(
 
 
 def validate_stack(
-    stack: List[StrategyNode], concepts: List[Concept], accept_partial: bool = False
+    stack: List[StrategyNode],
+    concepts: List[Concept],
+    accept_partial: bool = False,
 ) -> tuple[ValidationResult, set[str], set[str], set[str]]:
     found_map = defaultdict(set)
     found_addresses: set[str] = set()
@@ -357,6 +359,7 @@ def validate_stack(
                 if accept_partial:
                     found_addresses.add(concept.address)
                     found_map[str(node)].add(concept)
+    # zip in those we know we found
     if not all([c.address in found_addresses for c in concepts]):
         return (
             ValidationResult.INCOMPLETE,
@@ -388,7 +391,7 @@ def search_concepts(
     hist = history.get_history(mandatory_list, accept_partial)
     if hist is not False:
         logger.info(
-            f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Returning search node from history"
+            f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Returning search node from history for {[c.address for c in mandatory_list]} with accept_partial {accept_partial}"
         )
         assert not isinstance(hist, bool)
         return hist
@@ -454,7 +457,6 @@ def _search_concepts(
             if node:
                 stack.append(node)
                 node.resolve()
-                found.add(priority_concept.address)
                 # these concepts should not be attempted to be sourced again
                 # as fetching them requires operating on a subset of concepts
                 if priority_concept.derivation in [
@@ -473,11 +475,12 @@ def _search_concepts(
         complete, found, missing, partial = validate_stack(
             stack, mandatory_list, accept_partial
         )
-        # early exit if we have a complete stack with one node
+
         logger.info(
             f"{depth_to_prefix(depth)}{LOGGER_PREFIX} finished concept loop for {priority_concept} flag for accepting partial addresses is "
             f" {accept_partial} (complete: {complete}), have {found} from {[n for n in stack]} (missing {missing} partial {partial}), attempted {attempted}"
         )
+        # early exit if we have a complete stack with one node
         # we can only early exit if we have a complete stack
         # and we are not looking for more non-partial sources
         if complete == ValidationResult.COMPLETE and (
@@ -514,6 +517,10 @@ def _search_concepts(
             parents=stack,
             depth=depth,
             partial_concepts=all_partial,
+            # always hide merge concepts
+            hidden_concepts=[
+                x for x in mandatory_list if x.derivation == PurposeLineage.MERGE
+            ],
         )
 
         # ensure we can resolve our final merge

trilogy/core/processing/node_generators/__init__.py

@@ -5,10 +5,10 @@ from .group_to_node import gen_group_to_node
 from .basic_node import gen_basic_node
 from .select_node import gen_select_node
 from .unnest_node import gen_unnest_node
-from .merge_node import gen_merge_node
+from .node_merge_node import gen_merge_node
 from .rowset_node import gen_rowset_node
 from .multiselect_node import gen_multiselect_node
-from .concept_merge import gen_concept_merge_node
+from .concept_merge_node import gen_concept_merge_node
 
 __all__ = [
     "gen_filter_node",

trilogy/core/processing/node_generators/{concept_merge.py → concept_merge_node.py}

@@ -7,27 +7,30 @@ from trilogy.core.processing.nodes import MergeNode, NodeJoin, History
 from trilogy.core.processing.nodes.base_node import concept_list_to_grain, StrategyNode
 from typing import List
 
-from trilogy.core.enums import JoinType
+from trilogy.core.enums import JoinType, PurposeLineage
 from trilogy.constants import logger
 from trilogy.core.processing.utility import padding
 from trilogy.core.processing.node_generators.common import concept_to_relevant_joins
 from itertools import combinations
 from trilogy.core.processing.node_generators.common import resolve_join_order
+from trilogy.utility import unique
+
 
 LOGGER_PREFIX = "[GEN_CONCEPT_MERGE_NODE]"
 
 
-def merge_joins(base: MergeStatement, parents: List[StrategyNode]) -> List[NodeJoin]:
+def merge_joins(
+    parents: List[StrategyNode], merge_concepts: List[Concept]
+) -> List[NodeJoin]:
     output = []
     for left, right in combinations(parents, 2):
         output.append(
             NodeJoin(
                 left_node=left,
                 right_node=right,
-                concepts=[
-                    base.merge_concept,
-                ],
+                concepts=merge_concepts,
                 join_type=JoinType.FULL,
+                filter_to_mutual=True,
             )
         )
     return resolve_join_order(output)
@@ -50,37 +53,70 @@ def gen_concept_merge_node(
     lineage: MergeStatement = concept.lineage
 
     base_parents: List[StrategyNode] = []
+
+    # get additional concepts that should be merged across the environments
+    additional_merge: List[Concept] = [*lineage.concepts]
+    for x in local_optional:
+        if x.address in environment.merged_concepts:
+            ms = environment.merged_concepts[x.address].lineage
+            assert isinstance(ms, MergeStatement)
+            additional_merge += ms.concepts
+
     for select in lineage.concepts:
         # if it's a merge concept, filter it out of the optional
         sub_optional = [
             x
             for x in local_optional
-            if x.address not in lineage.concepts_lcl and x.namespace == select.namespace
+            if x.address not in environment.merged_concepts
+            and x.namespace == select.namespace
+        ]
+
+        sub_additional_merge = [
+            x for x in additional_merge if x.namespace == select.namespace
         ]
+        sub_optional += sub_additional_merge
+        final: List[Concept] = unique([select] + sub_optional, "address")
         snode: StrategyNode = source_concepts(
-            mandatory_list=[select] + sub_optional,
+            mandatory_list=final,
             environment=environment,
             g=g,
             depth=depth + 1,
             history=history,
         )
+
         if not snode:
             logger.info(
                 f"{padding(depth)}{LOGGER_PREFIX} Cannot generate merge node for {concept}"
             )
             return None
-        snode.add_output_concept(lineage.merge_concept)
+        for x in sub_additional_merge:
+            snode.add_output_concept(environment.merged_concepts[x.address])
         base_parents.append(snode)
 
-    node_joins = merge_joins(lineage, base_parents)
+    node_joins = merge_joins(
+        base_parents,
+        unique(
+            [environment.merged_concepts[x.address] for x in additional_merge],
+            "address",
+        ),
+    )
 
     enrichment = set([x.address for x in local_optional])
-    outputs = [x for y in base_parents for x in y.output_concepts]
+    outputs = [
+        x
+        for y in base_parents
+        for x in y.output_concepts
+        if x.derivation != PurposeLineage.MERGE
+    ]
 
     additional_relevant = [x for x in outputs if x.address in enrichment]
+    final_outputs = outputs + additional_relevant + [concept]
     node = MergeNode(
         input_concepts=[x for y in base_parents for x in y.output_concepts],
-        output_concepts=outputs + additional_relevant + [concept],
+        output_concepts=[x for x in final_outputs],
+        hidden_concepts=[
+            x for x in final_outputs if x.derivation == PurposeLineage.MERGE
+        ],
         environment=environment,
         g=g,
         depth=depth,
@@ -127,9 +163,19 @@ def gen_concept_merge_node(
         )
         return node
 
+    # we still need the hidden concepts to be returned to the search
+    # since they must be on the final node
+    # to avoid further recursion
+    # TODO: let the downstream search know they were found
     return MergeNode(
         input_concepts=enrich_node.output_concepts + node.output_concepts,
+        # also filter out the
         output_concepts=node.output_concepts + local_optional,
+        hidden_concepts=[
+            x
+            for x in node.output_concepts + local_optional
+            if x.derivation == PurposeLineage.MERGE
+        ],
         environment=environment,
         g=g,
         depth=depth,

trilogy/core/processing/nodes/__init__.py

@@ -27,12 +27,18 @@ class History(BaseModel):
         self,
         search: list[Concept],
         accept_partial: bool = False,
+        parent_key: str = "",
     ) -> StrategyNode | None | bool:
+        key = self._concepts_to_lookup(
+            search,
+            accept_partial,
+        )
+        if parent_key and parent_key == key:
+            raise ValueError(
+                f"Parent key {parent_key} is the same as the current key {key}"
+            )
         return self.history.get(
-            self._concepts_to_lookup(
-                search,
-                accept_partial,
-            ),
+            key,
             False,
         )
 

trilogy/core/processing/nodes/base_node.py

@@ -107,6 +107,7 @@ class StrategyNode:
         conditions: Conditional | Comparison | Parenthetical | None = None,
         force_group: bool | None = None,
         grain: Optional[Grain] = None,
+        hidden_concepts: List[Concept] | None = None,
     ):
         self.input_concepts: List[Concept] = (
             unique(input_concepts, "address") if input_concepts else []
@@ -129,6 +130,7 @@ class StrategyNode:
         self.grain = grain
         self.force_group = force_group
         self.tainted = False
+        self.hidden_concepts = hidden_concepts or []
         for parent in self.parents:
             if not parent:
                 raise SyntaxError("Unresolvable parent")

trilogy/core/processing/nodes/merge_node.py

@@ -104,6 +104,7 @@ class MergeNode(StrategyNode):
         depth: int = 0,
         grain: Grain | None = None,
         conditions: Conditional | None = None,
+        hidden_concepts: List[Concept] | None = None,
     ):
         super().__init__(
             input_concepts=input_concepts,
@@ -117,6 +118,7 @@ class MergeNode(StrategyNode):
             force_group=force_group,
             grain=grain,
             conditions=conditions,
+            hidden_concepts=hidden_concepts,
         )
         self.join_concepts = join_concepts
         self.force_join_type = force_join_type
@@ -327,5 +329,6 @@ class MergeNode(StrategyNode):
             partial_concepts=self.partial_concepts,
             force_group=force_group,
             condition=self.conditions,
+            hidden_concepts=self.hidden_concepts,
         )
         return qds

trilogy/core/query_processor.py

@@ -216,6 +216,7 @@ def datasource_to_ctes(
         condition=query_datasource.condition,
         partial_concepts=query_datasource.partial_concepts,
         join_derived_concepts=query_datasource.join_derived_concepts,
+        hidden_concepts=query_datasource.hidden_concepts,
     )
     if cte.grain != query_datasource.grain:
         raise ValueError("Grain was corrupted in CTE generation")

trilogy/dialect/base.py

@@ -437,6 +437,7 @@ class BaseDialect:
                 self.render_concept_sql(c, cte)
                 for c in cte.output_columns
                 if c.address not in [y.address for y in cte.join_derived_concepts]
+                and c.address not in [y.address for y in cte.hidden_concepts]
             ] + [
                 f"{self.QUOTE_CHARACTER}{c.safe_address}{self.QUOTE_CHARACTER}"
                 for c in cte.join_derived_concepts
@@ -444,7 +445,9 @@ class BaseDialect:
         else:
             # otherwse, assume we are unnesting directly in the select
             select_columns = [
-                self.render_concept_sql(c, cte) for c in cte.output_columns
+                self.render_concept_sql(c, cte)
+                for c in cte.output_columns
+                if c.address not in [y.address for y in cte.hidden_concepts]
             ]
         return CompiledCTE(
             name=cte.name,

trilogy/dialect/config.py

@@ -100,22 +100,7 @@ class PrestoConfig(DialectConfig):
         return f"presto://{self.username}:{self.password}@{self.host}:{self.port}/{self.catalog}"
 
 
-class TrinoConfig(DialectConfig):
-    def __init__(
-        self,
-        host: str,
-        port: int,
-        username: str,
-        password: str,
-        catalog: str,
-        schema: str | None = None,
-    ):
-        self.host = host
-        self.port = port
-        self.username = username
-        self.password = password
-        self.catalog = catalog
-        self.schema = schema
+class TrinoConfig(PrestoConfig):
 
     def connection_string(self) -> str:
         if self.schema:

trilogy/dialect/presto.py

@@ -27,6 +27,9 @@ FUNCTION_MAP = {
     FunctionType.QUARTER: lambda x: f"EXTRACT(QUARTER from {x[0]})",
     # math
     FunctionType.DIVIDE: lambda x: f"SAFE_DIVIDE({x[0]},{x[1]})",
+    FunctionType.DATE_ADD: lambda x: f"DATE_ADD('{x[1]}', {x[2]}, {x[0]})",
+    FunctionType.CURRENT_DATE: lambda x: "CURRENT_DATE",
+    FunctionType.CURRENT_DATETIME: lambda x: "CURRENT_TIMESTAMP",
 }
 
 FUNCTION_GRAIN_MATCH_MAP = {
@@ -42,11 +45,10 @@ CREATE OR REPLACE TABLE {{ output.address }} AS
 {% endif %}{%- if ctes %}
 WITH {% for cte in ctes %}
 {{cte.name}} as ({{cte.statement}}){% if not loop.last %},{% endif %}{% endfor %}{% endif %}
-
-SELECT
 {%- if full_select -%}
 {{full_select}}
-{%- else -%}
+{%- else %}
+SELECT
 {%- for select in select_columns %}
     {{ select }}{% if not loop.last %},{% endif %}{% endfor %}
 {% if base %}FROM