pytrilogy 0.0.2.6__py3-none-any.whl → 0.0.2.7__py3-none-any.whl

{pytrilogy-0.0.2.6.dist-info → pytrilogy-0.0.2.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.6
+Version: 0.0.2.7
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.2.6.dist-info → pytrilogy-0.0.2.7.dist-info}/RECORD

@@ -1,6 +1,6 @@
-trilogy/__init__.py,sha256=P77DaUk5fO6LhD9i2bg5poc0-ZbDHKxy6bYyR7Zxsoc,290
+trilogy/__init__.py,sha256=Tmie80hiyGWAbZBCPHnj73ajJrgy-wgEl2dB5EZ5_Is,290
 trilogy/compiler.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/constants.py,sha256=KIvi-cgU4R9urNgDdGiCsRkCrzjAfM4xGHhZb1SVy2w,881
+trilogy/constants.py,sha256=3YKp-xvS7yB5z9t9YDcGofkrdZ6v13iePDnn4O5mwTE,886
 trilogy/engine.py,sha256=R5ubIxYyrxRExz07aZCUfrTsoXCHQ8DKFTDsobXdWdA,1102
 trilogy/executor.py,sha256=5cRbU4Rj7p1pNV76rfp1pz704Hx_0q8_O8HFURjgXxQ,11016
 trilogy/parser.py,sha256=UtuqSiGiCjpMAYgo1bvNq-b7NSzCA5hzbUW31RXaMII,281
@@ -16,44 +16,44 @@ trilogy/core/exceptions.py,sha256=NvV_4qLOgKXbpotgRf7c8BANDEvHxlqRPaA53IThQ2o,56
 trilogy/core/functions.py,sha256=ARJAyBjeS415-54k3G_bx807rkPZonEulMaLRxSP7vU,10371
 trilogy/core/graph_models.py,sha256=oJUMSpmYhqXlavckHLpR07GJxuQ8dZ1VbB1fB0KaS8c,2036
 trilogy/core/internal.py,sha256=jNGFHKENnbMiMCtAgsnLZYVSENDK4b5ALecXFZpTDzQ,1075
-trilogy/core/models.py,sha256=atXBfSP_-xgY1ixIG98K2p0TLxmhh7bc6pqI9tPTZSY,137562
+trilogy/core/models.py,sha256=xhEpN8MIF5HW41E5aBB70xeeH0sbwuc-xFbDH9frGDk,139559
 trilogy/core/optimization.py,sha256=A8S9C9H5RcQcFSQLYtEEBnm-r1CW_e9GEWlLK7q3MqA,4930
-trilogy/core/query_processor.py,sha256=6JiX6YqsTgnsEkIxcmWFxJuM6kRJbXbPGRCQgi8g-B8,17941
+trilogy/core/query_processor.py,sha256=fIRJWLBfMeKZ_o1-QB2rsQ8jZwsd9zxYkZ5QQzkGBus,17678
 trilogy/core/optimizations/__init__.py,sha256=pxRzNzd2g8oRMy4f_ub5va6bNS2pd4hnyp9JBzTKc1E,300
 trilogy/core/optimizations/base_optimization.py,sha256=tWWT-xnTbnEU-mNi_isMNbywm8B9WTRsNFwGpeh3rqE,468
 trilogy/core/optimizations/inline_constant.py,sha256=kHNyc2UoaPVdYfVAPAFwnWuk4sJ_IF5faRtVcDOrBtw,1110
-trilogy/core/optimizations/inline_datasource.py,sha256=9_jTjKZrRG_mPeQlvuKK2grSiZMMD2pqG4E9PAL9J4U,3383
+trilogy/core/optimizations/inline_datasource.py,sha256=PyThSsQeZ6fi5sM2mUprC8kXXKo1sP8qLVwaGqpFDJI,3428
 trilogy/core/optimizations/predicate_pushdown.py,sha256=iVZV_BVaL3I6wlELPOuJAiuF0heOM7bCUnNqyCxxRVw,6370
 trilogy/core/processing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/core/processing/concept_strategies_v3.py,sha256=xCPfg1ySHP6C-u777kqnV8eqIK1X68hqwvRcw61q5X0,23968
+trilogy/core/processing/concept_strategies_v3.py,sha256=A0-RVwoqD06gUapGRD4SANj3g9hniKPWs7LvqyeHtbg,24445
 trilogy/core/processing/graph_utils.py,sha256=aq-kqk4Iado2HywDxWEejWc-7PGO6Oa-ZQLAM6XWPHw,1199
 trilogy/core/processing/utility.py,sha256=rhnkn3KgB1MW17mZYSEZ8XFEbC__B7PPmZgpDql4Myg,13614
 trilogy/core/processing/node_generators/__init__.py,sha256=-mzYkRsaRNa_dfTckYkKVFSR8h8a3ihEiPJDU_tAmDo,672
 trilogy/core/processing/node_generators/basic_node.py,sha256=4242PNGTCm2tklqMIkqVu5Iv4m_IeTnOYXxDveuCDZM,2856
-trilogy/core/processing/node_generators/common.py,sha256=uLrqbEsuYREswiTfuTQuxldO9SsS6ylZEQT_q9iB2uY,8948
-trilogy/core/processing/node_generators/filter_node.py,sha256=kVsK9O9QWUgknui3ESHJJCnFY9dVMCHlGpHXn1QtXkU,4824
-trilogy/core/processing/node_generators/group_node.py,sha256=xWI1xNIXEOj6jlRGD9hcv2_vVNvY6lpzJl6pQ8HuFBE,2988
-trilogy/core/processing/node_generators/group_to_node.py,sha256=BzPdYwzoo8gRMH7BDffTTXq4z-mjfCEzvfB5I-P0_nw,2941
+trilogy/core/processing/node_generators/common.py,sha256=Z3R5Ucb0s09pzs3lkz-t9sI2A5R-SIum1UdwGvqKpXA,8975
+trilogy/core/processing/node_generators/filter_node.py,sha256=0zey11fmXhI-ihkv-cK43tUKHlwG10cHYogByOnGbiw,5433
+trilogy/core/processing/node_generators/group_node.py,sha256=u9X0gAEY7AAWwYsK4g3jPgxcTwSSlI1uDQqNIY6vfG8,2988
+trilogy/core/processing/node_generators/group_to_node.py,sha256=nzITnhaALIT7FMonyo16nNo-kSrLfefa9sZBYecrvkU,2887
 trilogy/core/processing/node_generators/multiselect_node.py,sha256=vP84dnLQy6dtypi6mUbt9sMAcmmrTgQ1Oz4GI6X1IEo,6421
-trilogy/core/processing/node_generators/node_merge_node.py,sha256=JDQEIs43BJCgz0yofztcjd7J20Ybk4W40IJG_6blQJI,12168
-trilogy/core/processing/node_generators/rowset_node.py,sha256=2ROd2c1_o4h5sbnDdaiL1o9UiiwyYw-B6OKyfHYNe0A,6188
+trilogy/core/processing/node_generators/node_merge_node.py,sha256=yZ75lsl8RtM8kr-XQxqZEU6fm-JN5DTJxNbneayu36c,12319
+trilogy/core/processing/node_generators/rowset_node.py,sha256=wFkzQtXfudvAGifySvobgc9Qo1usTEEKpSbjkzchNy0,6282
 trilogy/core/processing/node_generators/select_node.py,sha256=XGcz74XsWv5POWhV2gGC_jbi7T9g7sY-VtP3sjlppFc,19057
 trilogy/core/processing/node_generators/unnest_node.py,sha256=6CH66eGwpadNX7TzUhWZ8aqIisOtQeHINbLV6X3QBUk,1779
 trilogy/core/processing/node_generators/window_node.py,sha256=9nXUXUgQrNczU1gaOqhOZPNzCUxw-lkxt0R7HORI6ss,2582
 trilogy/core/processing/nodes/__init__.py,sha256=-8ONG7Nqa6sNLpst5wNNyMigyaHRfUKGdE3xce8E3mM,3998
-trilogy/core/processing/nodes/base_node.py,sha256=q1G4FO3Q4OuboWL8EaJgJ4jcnnqM9QsothMIcF2ot1g,11161
-trilogy/core/processing/nodes/filter_node.py,sha256=ImUHYj5G5eHbOtki-ZMR31Op38wKOjR1xu_ETiE_D6I,1932
+trilogy/core/processing/nodes/base_node.py,sha256=7AeDm9WK3uiDGUL6QefeZ8Ok9OcWhbh5h1f9K0hr8IQ,11191
+trilogy/core/processing/nodes/filter_node.py,sha256=stY0tkppSdfCD43ZKRfCxfX9XxdLJ9gEtDA2_wPDcGE,1962
 trilogy/core/processing/nodes/group_node.py,sha256=j3vtFQIn_t4EfCke8dGULlwVkG2KKPYfgGwo2rvgcww,6322
-trilogy/core/processing/nodes/merge_node.py,sha256=W7ryL9rceylCQGNgcuGY-dSiNoIz8qrbpJXKzg_Niwo,14056
+trilogy/core/processing/nodes/merge_node.py,sha256=RAuBPHPIQcGHqyU_R4HT6Kq9E40fWjQoOUCIngLEGQI,14629
 trilogy/core/processing/nodes/select_node_v2.py,sha256=COu-WPuyabGAc3HTkJB-_7eNzR4OgxIL4YDuZIRrST0,7179
 trilogy/core/processing/nodes/unnest_node.py,sha256=JFtm90IVM-46aCYkTNIaJah6v9ApAfonjVhcVM1HmDE,1903
 trilogy/core/processing/nodes/window_node.py,sha256=X7qxLUKd3tekjUUsmH_4vz5b-U89gMnGd04VBxuu2Ns,1280
 trilogy/dialect/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/dialect/base.py,sha256=QQBUhR53KvwW1Xac1cH4NAvZNnNR2HOoo-lOOUar3G4,28889
+trilogy/dialect/base.py,sha256=Ry5uGD-r1ht9e9tZRJ9LciZ1Qz2ZCMcli9p-xv1XDHE,28960
 trilogy/dialect/bigquery.py,sha256=15KJ-cOpBlk9O7FPviPgmg8xIydJeKx7WfmL3SSsPE8,2953
 trilogy/dialect/common.py,sha256=1oBJ44P4Dw1bMeek-wmZVwcflVYHbAiKb1ByN-iFXVk,2988
 trilogy/dialect/config.py,sha256=tLVEMctaTDhUgARKXUNfHUcIolGaALkQ0RavUvXAY4w,2994
-trilogy/dialect/duckdb.py,sha256=PHDNekefyUgdtpdJS-MVZQgfaich5pSec0e50IVPMMw,3124
+trilogy/dialect/duckdb.py,sha256=2ERqiG2owb3uvmU90Bdo1e2fq4ICWC3we2RUQpHVPwg,3173
 trilogy/dialect/enums.py,sha256=4NdpsydBpDn6jnh0JzFz5VvQEtnShErWtWHVyT6TNpw,3948
 trilogy/dialect/postgres.py,sha256=ev1RJZsC8BB3vJSxJ4q-TTYqZ4Hk1NXUtuRkLrQEBX0,3254
 trilogy/dialect/presto.py,sha256=ohWpG7-AWEUGQEpmnJbqVeGRDISX1u18DI-6GjYjwRA,3350
@@ -65,7 +65,7 @@ trilogy/hooks/graph_hook.py,sha256=onHvMQPwj_KOS3HOTpRFiy7QLLKAiycq2MzJ_Q0Oh5Y,2
 trilogy/hooks/query_debugger.py,sha256=NDChfkPmmW-KINa4TaQmDe_adGiwsKFdGLDSYpbodeU,4282
 trilogy/metadata/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/parsing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-trilogy/parsing/common.py,sha256=aYmLtsEk8VJj5yYASzUcTf0i0lXVsZBbZJwphwokxK4,6092
+trilogy/parsing/common.py,sha256=zNd5buKxK4z9WSszOk4zOI0GexDFukzZtdNfQxg3kVw,6218
 trilogy/parsing/config.py,sha256=Z-DaefdKhPDmSXLgg5V4pebhSB0h590vI0_VtHnlukI,111
 trilogy/parsing/exceptions.py,sha256=92E5i2frv5hj9wxObJZsZqj5T6bglvPzvdvco_vW1Zk,38
 trilogy/parsing/helpers.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
@@ -74,9 +74,9 @@ trilogy/parsing/render.py,sha256=Gy_6wVYPwYLf35Iota08sbqveuWILtUhI8MYStcvtJM,121
 trilogy/parsing/trilogy.lark,sha256=cUcwxUTlxU7jKFzYEXYARLTsPHG5cVLk-Xhltw6m2lY,11357
 trilogy/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 trilogy/scripts/trilogy.py,sha256=PHxvv6f2ODv0esyyhWxlARgra8dVhqQhYl0lTrSyVNo,3729
-pytrilogy-0.0.2.6.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
-pytrilogy-0.0.2.6.dist-info/METADATA,sha256=W8sJtJY77xavjIKHsL75YdHtxiXFJljB_z6a6xeWjH0,7906
-pytrilogy-0.0.2.6.dist-info/WHEEL,sha256=UvcQYKBHoFqaQd6LKyqHw9fxEolWLQnlzP0h_LgJAfI,91
-pytrilogy-0.0.2.6.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
-pytrilogy-0.0.2.6.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
-pytrilogy-0.0.2.6.dist-info/RECORD,,
+pytrilogy-0.0.2.7.dist-info/LICENSE.md,sha256=5ZRvtTyCCFwz1THxDTjAu3Lidds9WjPvvzgVwPSYNDo,1042
+pytrilogy-0.0.2.7.dist-info/METADATA,sha256=iloiI0JW-NDtGOmQU-hJ5jd1XhDMRD_-QkDymPuYKNA,7906
+pytrilogy-0.0.2.7.dist-info/WHEEL,sha256=ixB2d4u7mugx_bCBycvM9OzZ5yD7NmPXFRtKlORZS2Y,91
+pytrilogy-0.0.2.7.dist-info/entry_points.txt,sha256=0petKryjvvtEfTlbZC1AuMFumH_WQ9v8A19LvoS6G6c,54
+pytrilogy-0.0.2.7.dist-info/top_level.txt,sha256=cAy__NW_eMAa_yT9UnUNlZLFfxcg6eimUAZ184cdNiE,8
+pytrilogy-0.0.2.7.dist-info/RECORD,,

{pytrilogy-0.0.2.6.dist-info → pytrilogy-0.0.2.7.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (74.0.0)
+Generator: setuptools (74.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.2.6"
+__version__ = "0.0.2.7"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

trilogy/constants.py

@@ -33,6 +33,7 @@ class Config:
     strict_mode: bool = True
     human_identifiers: bool = True
     validate_missing: bool = True
+    show_comments: bool = False
     optimizations: Optimizations = field(default_factory=Optimizations)
 
     def set_random_seed(self, seed: int):
@@ -42,5 +43,3 @@ class Config:
 CONFIG = Config()
 
 CONFIG.set_random_seed(42)
-
-CONFIG.strict_mode = True

trilogy/core/models.py

@@ -945,12 +945,14 @@ class ColumnAssignment(BaseModel):
         )
 
     def with_merge(
-        self, concept: Concept, modifiers: List[Modifier]
+        self, source: Concept, target: Concept, modifiers: List[Modifier]
     ) -> "ColumnAssignment":
         return ColumnAssignment(
             alias=self.alias,
-            concept=concept,
-            modifiers=modifiers,
+            concept=self.concept.with_merge(source, target, modifiers),
+            modifiers=(
+                modifiers if self.concept.address == source.address else self.modifiers
+            ),
         )
 
 
@@ -1890,20 +1892,22 @@ class Datasource(Namespaced, BaseModel):
         self, source: Concept, target: Concept, modifiers: List[Modifier]
     ):
         original = [c for c in self.columns if c.concept.address == source.address]
+        if len(original) != 1:
+            raise ValueError(
+                f"Expected exactly one column to merge, got {len(original)} for {source.address}, {[x.alias for x in original]}"
+            )
         # map to the alias with the modifier, and the original
         self.columns = [
-            (
-                c.with_merge(target, modifiers)
-                if c.concept.address == source.address
-                else c
-            )
+            c.with_merge(source, target, modifiers)
             for c in self.columns
+            if c.concept.address != source.address
         ] + original
         self.grain = self.grain.with_merge(source, target, modifiers)
         self.where = (
             self.where.with_merge(source, target, modifiers) if self.where else None
         )
-        del self.output_lcl
+
+        self.add_column(target, original[0].alias, modifiers)
 
     @property
     def env_label(self) -> str:
@@ -1915,7 +1919,7 @@ class Datasource(Namespaced, BaseModel):
     def condition(self):
         return None
 
-    @cached_property
+    @property
     def output_lcl(self) -> LooseConceptList:
         return LooseConceptList(concepts=self.output_concepts)
 
@@ -1923,9 +1927,9 @@ class Datasource(Namespaced, BaseModel):
     def can_be_inlined(self) -> bool:
         if isinstance(self.address, Address) and self.address.is_query:
             return False
-        for x in self.columns:
-            if not isinstance(x.alias, str):
-                return False
+        # for x in self.columns:
+        #     if not isinstance(x.alias, str):
+        #         return False
         return True
 
     @property
@@ -1960,12 +1964,15 @@ class Datasource(Namespaced, BaseModel):
             )
         return grain
 
-    def add_column(self, concept: Concept, alias: str, modifiers=None):
+    def add_column(
+        self,
+        concept: Concept,
+        alias: str | RawColumnExpr | Function,
+        modifiers: List[Modifier] | None = None,
+    ):
         self.columns.append(
-            ColumnAssignment(alias=alias, concept=concept, modifiers=modifiers)
+            ColumnAssignment(alias=alias, concept=concept, modifiers=modifiers or [])
         )
-        # force refresh
-        del self.output_lcl
 
     def __add__(self, other):
         if not other == self:
@@ -1998,7 +2005,7 @@ class Datasource(Namespaced, BaseModel):
             where=self.where.with_namespace(namespace) if self.where else None,
         )
 
-    @cached_property
+    @property
     def concepts(self) -> List[Concept]:
         return [c.concept for c in self.columns]
 
@@ -2149,6 +2156,12 @@ class BaseJoin(BaseModel):
         )
 
     def __str__(self):
+        if self.concept_pairs:
+            return (
+                f"{self.join_type.value} JOIN {self.left_datasource.identifier} and"
+                f" {self.right_datasource.identifier} on"
+                f" {','.join([str(k[0])+'='+str(k[1]) for k in self.concept_pairs])}"
+            )
         return (
             f"{self.join_type.value} JOIN {self.left_datasource.identifier} and"
             f" {self.right_datasource.identifier} on"
@@ -2460,6 +2473,19 @@ class CTE(BaseModel):
                     self.base_alias_override = candidates[0] if candidates else None
         return True
 
+    @property
+    def comment(self) -> str:
+        base = f"Target: {str(self.grain)}."
+        if self.parent_ctes:
+            base += f" References: {', '.join([x.name for x in self.parent_ctes])}."
+        if self.joins:
+            base += f"\n-- Joins: {', '.join([str(x) for x in self.joins])}."
+        if self.partial_concepts:
+            base += (
+                f"\n-- Partials: {', '.join([str(x) for x in self.partial_concepts])}."
+            )
+        return base
+
     def inline_parent_datasource(self, parent: CTE, force_group: bool = False) -> bool:
         qds_being_inlined = parent.source
         ds_being_inlined = qds_being_inlined.datasources[0]
@@ -2550,6 +2576,10 @@ class CTE(BaseModel):
         self.hidden_concepts = unique(
             self.hidden_concepts + other.hidden_concepts, "address"
         )
+        self.existence_source_map = {
+            **self.existence_source_map,
+            **other.existence_source_map,
+        }
         return self
 
     @property
@@ -2741,6 +2771,12 @@ class Join(BaseModel):
         return self.left_name + self.right_name + self.jointype.value
 
     def __str__(self):
+        if self.joinkey_pairs:
+            return (
+                f"{self.jointype.value} JOIN {self.left_name} and"
+                f" {self.right_name} on"
+                f" {','.join([str(k[0])+'='+str(k[1]) for k in self.joinkey_pairs])}"
+            )
         return (
             f"{self.jointype.value} JOIN {self.left_name} and"
             f" {self.right_name} on {','.join([str(k) for k in self.joinkeys])}"
@@ -3002,6 +3038,7 @@ class Environment(BaseModel):
 
     materialized_concepts: List[Concept] = Field(default_factory=list)
     alias_origin_lookup: Dict[str, Concept] = Field(default_factory=dict)
+    canonical_map: Dict[str, str] = Field(default_factory=dict)
     _parse_count: int = 0
 
     @classmethod
@@ -3050,7 +3087,7 @@ class Environment(BaseModel):
             if x.address not in current_mat
         ]
         if new:
-            logger.info(f"Environment added new materialized concepts {new}")
+            logger.debug(f"Environment added new materialized concepts {new}")
 
     def validate_concept(self, lookup: str, meta: Meta | None = None):
         existing: Concept = self.concepts.get(lookup)  # type: ignore
@@ -3213,13 +3250,22 @@ class Environment(BaseModel):
         self, source: Concept, target: Concept, modifiers: List[Modifier]
     ):
         replacements = {}
+        # exit early if we've run this
+        if source.address in self.alias_origin_lookup:
+            if self.concepts[source.address] == target:
+                return
         self.alias_origin_lookup[source.address] = source
         for k, v in self.concepts.items():
+
             if v.address == target.address:
                 v.pseudonyms[source.address] = source
             if v.address == source.address:
                 replacements[k] = target
+                self.canonical_map[k] = target.address
                 v.pseudonyms[target.address] = target
+            # we need to update keys and grains of all concepts
+            else:
+                replacements[k] = v.with_merge(source, target, modifiers)
         self.concepts.update(replacements)
 
         for k, ds in self.datasources.items():

trilogy/core/optimizations/inline_datasource.py

@@ -42,19 +42,18 @@ class InlineDatasource(OptimizationRule):
                 self.log(f"parent {parent_cte.name} datasource is not inlineable")
                 continue
             root_outputs = {x.address for x in root.output_concepts}
-            cte_outputs = {x.address for x in cte.output_columns}
-            inherited = {x for x, v in cte.source_map.items() if v}
-            # cte_inherited_outputs = {x.address for x in parent_cte.output_columns if parent_cte.source_map.get(x.address)}
-            grain_components = {x.address for x in root.grain.components}
+            inherited = {
+                x for x, v in cte.source_map.items() if v and parent_cte.name in v
+            }
             if not inherited.issubset(root_outputs):
                 cte_missing = inherited - root_outputs
                 self.log(
                     f"Not all {parent_cte.name} require inputs are found on datasource, missing {cte_missing}"
                 )
                 continue
-            if not grain_components.issubset(cte_outputs):
-                self.log("Not all datasource components in cte outputs, forcing group")
-                force_group = True
+            if not root.grain.issubset(parent_cte.grain):
+                self.log(f"Not all {parent_cte.name} is at wrong grain to inline")
+                continue
             to_inline.append(parent_cte)
 
         optimized = False
@@ -68,6 +67,11 @@ class InlineDatasource(OptimizationRule):
                     f"Skipping inlining raw datasource {replaceable.source.name} ({replaceable.name}) due to multiple references"
                 )
                 continue
+            if not replaceable.source.datasources[0].grain.issubset(replaceable.grain):
+                self.log(
+                    f"Forcing group ({parent_cte.grain} being replaced by inlined source {root.grain})"
+                )
+                force_group = True
             result = cte.inline_parent_datasource(replaceable, force_group=force_group)
             if result:
                 self.log(

trilogy/core/processing/concept_strategies_v3.py

@@ -612,9 +612,19 @@ def _search_concepts(
         )
 
         if expanded:
-            expanded.resolve()
+            # we don't need to return the entire list; just the ones we needed pre-expansion
+            ex_resolve = expanded.resolve()
+            extra = [
+                x
+                for x in ex_resolve.output_concepts
+                if x.address not in [y.address for y in mandatory_list]
+                and x not in ex_resolve.grain.components
+            ]
+            expanded.output_concepts = mandatory_list
+            expanded.rebuild_cache()
+
             logger.info(
-                f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Found connections for {[c.address for c in mandatory_list]} via concept addition;"
+                f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Found connections for {[c.address for c in mandatory_list]} via concept addition; removing extra {[c.address for c in extra]}"
             )
             return expanded
     # if we can't find it after expanding to a merge, then

trilogy/core/processing/node_generators/common.py

@@ -209,6 +209,7 @@ def gen_enrichment_node(
         environment=environment,
         g=g,
         parents=[enrich_node, base_node],
+        force_group=False,
         node_joins=[
             NodeJoin(
                 left_node=enrich_node,

trilogy/core/processing/node_generators/filter_node.py

@@ -105,10 +105,29 @@ def gen_filter_node(
         environment=environment,
         g=g,
         parents=core_parents,
+        grain=Grain(
+            components=[immediate_parent] + parent_row_concepts,
+        ),
+    )
+
+    assert filter_node.resolve().grain == Grain(
+        components=[immediate_parent] + parent_row_concepts,
     )
     if not local_optional or all(
         [x.address in [y.address for y in parent_row_concepts] for x in local_optional]
     ):
+        outputs = [
+            x
+            for x in filter_node.output_concepts
+            if x.address in [y.address for y in local_optional]
+        ]
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} no extra enrichment needed for filter node"
+        )
+        filter_node.output_concepts = [
+            concept,
+        ] + outputs
+        filter_node.rebuild_cache()
         return filter_node
     enrich_node = source_concepts(  # this fetches the parent + join keys
         # to then connect to the rest of the query

trilogy/core/processing/node_generators/group_node.py

@@ -29,7 +29,7 @@ def gen_group_node(
         resolve_function_parent_concepts(concept), "address"
     )
     logger.info(
-        f"{padding(depth)}{LOGGER_PREFIX} parent_concepts are {[x.address for x in parent_concepts]} from group grain {concept.grain}"
+        f"{padding(depth)}{LOGGER_PREFIX} parent concepts are {[x.address for x in parent_concepts]} from group grain {concept.grain}"
     )
 
     # if the aggregation has a grain, we need to ensure these are the ONLY optional in the output of the select

trilogy/core/processing/node_generators/group_to_node.py

@@ -95,5 +95,4 @@ def gen_group_to_node(
         ],
         whole_grain=True,
         depth=depth,
-        partial_concepts=group_node.partial_concepts,
     )

trilogy/core/processing/node_generators/node_merge_node.py

@@ -260,6 +260,7 @@ def subgraphs_to_merge_node(
     source_concepts,
     history,
     conditions,
+    enable_early_exit: bool = True,
 ):
     parents: List[StrategyNode] = []
     logger.info(
@@ -290,6 +291,8 @@ def subgraphs_to_merge_node(
     for x in parents:
         for y in x.output_concepts:
             input_c.append(y)
+    if len(parents) == 1 and enable_early_exit:
+        return parents[0]
 
     return MergeNode(
         input_concepts=unique(input_c, "address"),
@@ -350,6 +353,7 @@ def gen_merge_node(
                 source_concepts=source_concepts,
                 history=history,
                 conditions=conditions,
+                enable_early_exit=False,
             )
             if test:
                 return test

trilogy/core/processing/node_generators/rowset_node.py

@@ -74,8 +74,9 @@ def gen_rowset_node(
     if existence_parents:
         node.parents += existence_parents
         # we don't need to join to any existence parents
-        if isinstance(node, MergeNode):
-            node.node_joins = []
+        # if isinstance(node, MergeNode) and node.node_joins is None:
+        #     # set it explicitly to empty to avoid inference
+        #     node.node_joins = []
         for parent in existence_parents:
             for x in parent.output_concepts:
                 if x.address not in node.output_lcl:

trilogy/core/processing/nodes/base_node.py

@@ -192,7 +192,7 @@ class StrategyNode:
             p.resolve() for p in self.parents
         ]
 
-        grain = Grain(components=self.output_concepts)
+        grain = self.grain if self.grain else Grain(components=self.output_concepts)
         source_map = resolve_concept_map(
             parent_sources,
             self.output_concepts,

trilogy/core/processing/nodes/filter_node.py

@@ -63,4 +63,5 @@ class FilterNode(StrategyNode):
             conditions=self.conditions,
             partial_concepts=list(self.partial_concepts),
             force_group=self.force_group,
+            grain=self.grain,
         )

trilogy/core/processing/nodes/merge_node.py

@@ -78,7 +78,7 @@ def deduplicate_nodes_and_joins(
         duplicates = False
         duplicates, merged, removed = deduplicate_nodes(merged, logging_prefix)
         # filter out any removed joins
-        if joins:
+        if joins is not None:
             joins = [
                 j
                 for j in joins
@@ -138,6 +138,16 @@ class MergeNode(StrategyNode):
                     continue
                 final_joins.append(join)
             self.node_joins = final_joins
+        partial_lookup: list[Concept] = []
+        non_partial: List[Concept] = []
+        for node in parents or []:
+            partial_lookup += node.partial_concepts
+            non_partial += [
+                x for x in node.output_concepts if x not in node.partial_concepts
+            ]
+
+        final_partial = [x for x in partial_lookup if x not in non_partial]
+        self.partial_concepts = final_partial
 
     def translate_node_joins(self, node_joins: List[NodeJoin]) -> List[BaseJoin]:
         joins = []
@@ -219,12 +229,13 @@ class MergeNode(StrategyNode):
             )
             joins = self.translate_node_joins(final_joins)
         else:
+            logger.info(
+                f"{self.logging_prefix}{LOGGER_PREFIX} Final joins is not null {final_joins} but is empty, skipping join generation"
+            )
             return []
 
         for join in joins:
-            logger.info(
-                f"{self.logging_prefix}{LOGGER_PREFIX} final join {join.join_type} {[str(c) for c in join.concepts]}"
-            )
+            logger.info(f"{self.logging_prefix}{LOGGER_PREFIX} final join {str(join)}")
         return joins
 
     def _resolve(self) -> QueryDatasource:
@@ -249,6 +260,12 @@ class MergeNode(StrategyNode):
         # early exit if we can just return the parent
         final_datasets: List[QueryDatasource | Datasource] = list(merged.values())
 
+        existence_final = [
+            x
+            for x in final_datasets
+            if all([y in self.existence_concepts for y in x.output_concepts])
+        ]
+
         if len(merged.keys()) == 1:
             final: QueryDatasource | Datasource = list(merged.values())[0]
             if (
@@ -288,34 +305,25 @@ class MergeNode(StrategyNode):
         for source in final_datasets:
             pregrain += source.grain
 
-        grain = (
-            self.grain
-            if self.grain
-            else Grain(
-                components=[
-                    c
-                    for c in pregrain.components
-                    if c.address in [x.address for x in self.output_concepts]
-                ]
-            )
-        )
+        grain = self.grain if self.grain else pregrain
 
         logger.info(
             f"{self.logging_prefix}{LOGGER_PREFIX} has pre grain {pregrain} and final merge node grain {grain}"
         )
-
-        if len(final_datasets) > 1:
+        join_candidates = [x for x in final_datasets if x not in existence_final]
+        if len(join_candidates) > 1:
             joins = self.generate_joins(
-                final_datasets, final_joins, pregrain, grain, self.environment
+                join_candidates, final_joins, pregrain, grain, self.environment
             )
         else:
             joins = []
-
+        logger.info(
+            f"{self.logging_prefix}{LOGGER_PREFIX} Final join count for CTE parent count {len(join_candidates)} is {len(joins)}"
+        )
         full_join_concepts = []
         for join in joins:
             if join.join_type == JoinType.FULL:
                 full_join_concepts += join.concepts
-
         if self.whole_grain:
             force_group = False
         elif self.force_group is False:
@@ -337,9 +345,6 @@ class MergeNode(StrategyNode):
             inherited_inputs=self.input_concepts + self.existence_concepts,
             full_joins=full_join_concepts,
         )
-        logger.info(
-            f"{self.logging_prefix}{LOGGER_PREFIX} source_map {str(source_map)}"
-        )
         qds = QueryDatasource(
             input_concepts=unique(self.input_concepts, "address"),
             output_concepts=unique(self.output_concepts, "address"),

trilogy/core/query_processor.py

@@ -183,49 +183,42 @@ def generate_cte_name(full_name: str, name_map: dict[str, str]) -> str:
         return full_name.replace("<", "").replace(">", "").replace(",", "_")
 
 
-def resolve_cte_base_name_and_alias(
+def resolve_cte_base_name_and_alias_v2(
     name: str,
     source: QueryDatasource,
-    parents: List[CTE],
-    joins: List[Join | InstantiatedUnnestJoin],
+    source_map: Dict[str, list[str]],
+    raw_joins: List[Join | InstantiatedUnnestJoin],
 ) -> Tuple[str | None, str | None]:
-
-    valid_joins: List[Join] = [join for join in joins if isinstance(join, Join)]
-    relevant_parent_sources = set()
-    for k, v in source.source_map.items():
-        if v:
-            relevant_parent_sources.update(v)
-    eligible = [x for x in source.datasources if x in relevant_parent_sources]
+    joins: List[Join] = [join for join in raw_joins if isinstance(join, Join)]
     if (
-        len(eligible) == 1
-        and isinstance(eligible[0], Datasource)
-        and not eligible[0].name == CONSTANT_DATASET
+        len(source.datasources) == 1
+        and isinstance(source.datasources[0], Datasource)
+        and not source.datasources[0].name == CONSTANT_DATASET
     ):
-        ds = eligible[0]
+        ds = source.datasources[0]
         return ds.safe_location, ds.identifier
 
-    # if we have multiple joined CTEs, pick the base
-    # as the root
-    elif len(eligible) == 1 and len(parents) == 1:
-        return parents[0].name, parents[0].name
-    elif valid_joins and len(valid_joins) > 0:
-        candidates = [x.left_cte.name for x in valid_joins]
-        disallowed = [x.right_cte.name for x in valid_joins]
+    if joins and len(joins) > 0:
+        candidates = [x.left_cte.name for x in joins]
+        disallowed = [x.right_cte.name for x in joins]
         try:
             cte = [y for y in candidates if y not in disallowed][0]
             return cte, cte
         except IndexError:
             raise SyntaxError(
-                f"Invalid join configuration {candidates} {disallowed} with all parents {[x.base_name for x in parents]}"
+                f"Invalid join configuration {candidates} {disallowed} for {name}",
             )
-    elif eligible:
-        matched = [x for x in parents if x.source.name == eligible[0].name]
-        if matched:
-            return matched[0].name, matched[0].name
 
-    logger.info(
-        f"Could not determine CTE base name for {name} with relevant sources {relevant_parent_sources}"
-    )
+    counts: dict[str, int] = defaultdict(lambda: 0)
+    output_addresses = [x.address for x in source.output_concepts]
+    for k, v in source_map.items():
+        for vx in v:
+            if k in output_addresses:
+                counts[vx] = counts[vx] + 1
+            else:
+                counts[vx] = counts[vx]
+    if counts:
+        return max(counts, key=counts.get), max(counts, key=counts.get)  # type: ignore
     return None, None
 
 
@@ -274,8 +267,8 @@ def datasource_to_ctes(
         for x in [base_join_to_join(join, parents) for join in query_datasource.joins]
         if x
     ]
-    base_name, base_alias = resolve_cte_base_name_and_alias(
-        human_id, query_datasource, parents, final_joins
+    base_name, base_alias = resolve_cte_base_name_and_alias_v2(
+        human_id, query_datasource, source_map, final_joins
     )
     cte = CTE(
         name=human_id,

trilogy/dialect/base.py

@@ -513,6 +513,7 @@ class BaseDialect:
                 grain=cte.grain,
                 limit=cte.limit,
                 # some joins may not need to be rendered
+                comment=cte.comment if CONFIG.show_comments else None,
                 joins=[
                     j
                     for j in [

trilogy/dialect/duckdb.py

@@ -47,8 +47,9 @@ CREATE OR REPLACE TABLE {{ output.address.location }} AS
 {% endif %}{%- if ctes %}
 WITH {% for cte in ctes %}
 {{cte.name}} as ({{cte.statement}}){% if not loop.last %},{% endif %}{% endfor %}{% endif %}
-{% if full_select -%}{{full_select}}
-{% else -%}
+{%- if full_select -%}{{full_select}}
+{%- else -%}{%- if comment %}
+-- {{ comment }}{% endif %}
 SELECT
 {%- for select in select_columns %}
     {{ select }}{% if not loop.last %},{% endif %}{% endfor %}
@@ -56,7 +57,8 @@ SELECT
     {{ base }}{% endif %}{% if joins %}
 {%- for join in joins %}
     {{ join }}{% endfor %}{% endif %}
-{% if where %}WHERE
+{%- if where %}
+WHERE
     {{ where }}
 {% endif -%}{%- if group_by %}
 GROUP BY {% for group in group_by %}

trilogy/parsing/common.py

@@ -112,12 +112,16 @@ def filter_item_to_concept(
     return Concept(
         name=name,
         datatype=parent.content.datatype,
-        purpose=parent.content.purpose,
+        purpose=Purpose.PROPERTY,
         lineage=parent,
         metadata=fmetadata,
         namespace=namespace,
         # filtered copies cannot inherit keys
-        keys=None,
+        keys=(
+            parent.content.keys
+            if parent.content.purpose == Purpose.PROPERTY
+            else (parent.content,)
+        ),
         grain=(
             parent.content.grain
             if parent.content.purpose == Purpose.PROPERTY