pytrilogy 0.0.2.6__tar.gz → 0.0.2.8__tar.gz

{pytrilogy-0.0.2.6/pytrilogy.egg-info → pytrilogy-0.0.2.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.6
+Version: 0.0.2.8
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.2.6 → pytrilogy-0.0.2.8/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pytrilogy
-Version: 0.0.2.6
+Version: 0.0.2.8
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.2.6 → pytrilogy-0.0.2.8}/tests/test_environment.py

@@ -59,6 +59,13 @@ key  order_id int;
 
     assert env1.concepts["order_id"] == env1.concepts["replacements.order_id"]
 
-    order_id = env1.datasources["replacements.replacements"].columns[0]
-    assert order_id.concept == env1.concepts["order_id"]
-    assert order_id.modifiers == [Modifier.PARTIAL]
+    found = False
+    for x in env1.datasources["replacements.replacements"].columns:
+        if (
+            x.alias == "order_id"
+            and x.concept.address == env1.concepts["order_id"].address
+        ):
+            assert x.concept == env1.concepts["order_id"]
+            assert x.modifiers == [Modifier.PARTIAL]
+            found = True
+    assert found

{pytrilogy-0.0.2.6 → pytrilogy-0.0.2.8}/tests/test_models.py

@@ -10,6 +10,8 @@ from trilogy.core.models import (
     UndefinedConcept,
     BaseJoin,
     Comparison,
+    Join,
+    JoinKey,
 )
 
 
@@ -54,6 +56,8 @@ def test_cte_merge(test_environment, test_environment_graph):
     merged = a + b
     assert merged.output_columns == outputs
 
+    assert "Target: Grain<Abstract>." in merged.comment
+
 
 def test_concept(test_environment, test_environment_graph):
     test_concept = list(test_environment.concepts.values())[0]
@@ -191,3 +195,54 @@ def test_comparison():
         Comparison(left=1, right="abc", operator=ComparisonOperator.EQ)
     except Exception as exc:
         assert isinstance(exc, SyntaxError)
+
+
+def test_join(test_environment: Environment):
+    datasource = list(test_environment.datasources.values())[0]
+    outputs = [c.concept for c in datasource.columns]
+    output_map = {
+        c.address: {
+            datasource,
+        }
+        for c in outputs
+    }
+    a = CTE(
+        name="test",
+        output_columns=[outputs[0]],
+        grain=Grain(),
+        source=QueryDatasource(
+            input_concepts=[outputs[0]],
+            output_concepts=[outputs[0]],
+            datasources=[datasource],
+            grain=Grain(),
+            joins=[],
+            source_map={outputs[0].address: {datasource}},
+        ),
+        source_map={c.address: [datasource.identifier] for c in outputs},
+    )
+
+    b = CTE(
+        name="testb",
+        output_columns=outputs,
+        grain=Grain(),
+        source=QueryDatasource(
+            input_concepts=outputs,
+            output_concepts=outputs,
+            datasources=[datasource],
+            grain=Grain(),
+            joins=[],
+            source_map=output_map,
+        ),
+        source_map={c.address: [datasource.identifier] for c in outputs},
+    )
+    test = Join(
+        left_cte=a,
+        right_cte=b,
+        joinkeys=[JoinKey(concept=x) for x in outputs],
+        jointype=JoinType.RIGHT_OUTER,
+    )
+
+    assert (
+        str(test)
+        == "right outer JOIN test and testb on local.product_id<local.product_id>,local.category_id<local.category_id>"
+    ), str(test)

{pytrilogy-0.0.2.6 → pytrilogy-0.0.2.8}/trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.executor import Executor
 from trilogy.parser import parse
 from trilogy.constants import CONFIG
 
-__version__ = "0.0.2.6"
+__version__ = "0.0.2.8"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

{pytrilogy-0.0.2.6 → pytrilogy-0.0.2.8}/trilogy/constants.py

@@ -33,6 +33,7 @@ class Config:
     strict_mode: bool = True
     human_identifiers: bool = True
     validate_missing: bool = True
+    show_comments: bool = False
     optimizations: Optimizations = field(default_factory=Optimizations)
 
     def set_random_seed(self, seed: int):
@@ -42,5 +43,3 @@ class Config:
 CONFIG = Config()
 
 CONFIG.set_random_seed(42)
-
-CONFIG.strict_mode = True

{pytrilogy-0.0.2.6 → pytrilogy-0.0.2.8}/trilogy/core/enums.py

@@ -123,6 +123,7 @@ class FunctionType(Enum):
     MAP_ACCESS = "map_access"
     ATTR_ACCESS = "attr_access"
     STRUCT = "struct"
+    ARRAY = "array"
 
     # TEXT AND MAYBE MORE
     SPLIT = "split"

{pytrilogy-0.0.2.6 → pytrilogy-0.0.2.8}/trilogy/core/models.py

@@ -945,12 +945,14 @@ class ColumnAssignment(BaseModel):
         )
 
     def with_merge(
-        self, concept: Concept, modifiers: List[Modifier]
+        self, source: Concept, target: Concept, modifiers: List[Modifier]
     ) -> "ColumnAssignment":
         return ColumnAssignment(
             alias=self.alias,
-            concept=concept,
-            modifiers=modifiers,
+            concept=self.concept.with_merge(source, target, modifiers),
+            modifiers=(
+                modifiers if self.concept.address == source.address else self.modifiers
+            ),
         )
 
 
@@ -1890,20 +1892,22 @@ class Datasource(Namespaced, BaseModel):
         self, source: Concept, target: Concept, modifiers: List[Modifier]
     ):
         original = [c for c in self.columns if c.concept.address == source.address]
+        if len(original) != 1:
+            raise ValueError(
+                f"Expected exactly one column to merge, got {len(original)} for {source.address}, {[x.alias for x in original]}"
+            )
         # map to the alias with the modifier, and the original
         self.columns = [
-            (
-                c.with_merge(target, modifiers)
-                if c.concept.address == source.address
-                else c
-            )
+            c.with_merge(source, target, modifiers)
             for c in self.columns
+            if c.concept.address != source.address
         ] + original
         self.grain = self.grain.with_merge(source, target, modifiers)
         self.where = (
             self.where.with_merge(source, target, modifiers) if self.where else None
         )
-        del self.output_lcl
+
+        self.add_column(target, original[0].alias, modifiers)
 
     @property
     def env_label(self) -> str:
@@ -1915,7 +1919,7 @@ class Datasource(Namespaced, BaseModel):
     def condition(self):
         return None
 
-    @cached_property
+    @property
     def output_lcl(self) -> LooseConceptList:
         return LooseConceptList(concepts=self.output_concepts)
 
@@ -1923,9 +1927,9 @@ class Datasource(Namespaced, BaseModel):
     def can_be_inlined(self) -> bool:
         if isinstance(self.address, Address) and self.address.is_query:
             return False
-        for x in self.columns:
-            if not isinstance(x.alias, str):
-                return False
+        # for x in self.columns:
+        #     if not isinstance(x.alias, str):
+        #         return False
         return True
 
     @property
@@ -1960,12 +1964,15 @@ class Datasource(Namespaced, BaseModel):
             )
         return grain
 
-    def add_column(self, concept: Concept, alias: str, modifiers=None):
+    def add_column(
+        self,
+        concept: Concept,
+        alias: str | RawColumnExpr | Function,
+        modifiers: List[Modifier] | None = None,
+    ):
         self.columns.append(
-            ColumnAssignment(alias=alias, concept=concept, modifiers=modifiers)
+            ColumnAssignment(alias=alias, concept=concept, modifiers=modifiers or [])
         )
-        # force refresh
-        del self.output_lcl
 
     def __add__(self, other):
         if not other == self:
@@ -1998,7 +2005,7 @@ class Datasource(Namespaced, BaseModel):
             where=self.where.with_namespace(namespace) if self.where else None,
         )
 
-    @cached_property
+    @property
     def concepts(self) -> List[Concept]:
         return [c.concept for c in self.columns]
 
@@ -2059,6 +2066,7 @@ class Datasource(Namespaced, BaseModel):
 class UnnestJoin(BaseModel):
     concept: Concept
     alias: str = "unnest"
+    rendering_required: bool = True
 
     def __hash__(self):
         return (self.alias + self.concept.address).__hash__()
@@ -2149,6 +2157,12 @@ class BaseJoin(BaseModel):
         )
 
     def __str__(self):
+        if self.concept_pairs:
+            return (
+                f"{self.join_type.value} JOIN {self.left_datasource.identifier} and"
+                f" {self.right_datasource.identifier} on"
+                f" {','.join([str(k[0])+'='+str(k[1]) for k in self.concept_pairs])}"
+            )
         return (
             f"{self.join_type.value} JOIN {self.left_datasource.identifier} and"
             f" {self.right_datasource.identifier} on"
@@ -2447,10 +2461,17 @@ class CTE(BaseModel):
                     for join in self.joins
                     if not isinstance(join, Join)
                     or (
-                        join.right_cte.name != removed_cte
-                        and join.left_cte.name != removed_cte
+                        isinstance(join, Join)
+                        and (
+                            join.right_cte.name != removed_cte
+                            and join.left_cte.name != removed_cte
+                        )
                     )
                 ]
+                for join in self.joins:
+                    if isinstance(join, UnnestJoin) and join.concept == concept:
+                        join.rendering_required = False
+
                 self.parent_ctes = [
                     x for x in self.parent_ctes if x.name != removed_cte
                 ]
@@ -2460,6 +2481,19 @@ class CTE(BaseModel):
                     self.base_alias_override = candidates[0] if candidates else None
         return True
 
+    @property
+    def comment(self) -> str:
+        base = f"Target: {str(self.grain)}."
+        if self.parent_ctes:
+            base += f" References: {', '.join([x.name for x in self.parent_ctes])}."
+        if self.joins:
+            base += f"\n-- Joins: {', '.join([str(x) for x in self.joins])}."
+        if self.partial_concepts:
+            base += (
+                f"\n-- Partials: {', '.join([str(x) for x in self.partial_concepts])}."
+            )
+        return base
+
     def inline_parent_datasource(self, parent: CTE, force_group: bool = False) -> bool:
         qds_being_inlined = parent.source
         ds_being_inlined = qds_being_inlined.datasources[0]
@@ -2550,6 +2584,10 @@ class CTE(BaseModel):
         self.hidden_concepts = unique(
             self.hidden_concepts + other.hidden_concepts, "address"
         )
+        self.existence_source_map = {
+            **self.existence_source_map,
+            **other.existence_source_map,
+        }
         return self
 
     @property
@@ -2741,6 +2779,12 @@ class Join(BaseModel):
         return self.left_name + self.right_name + self.jointype.value
 
     def __str__(self):
+        if self.joinkey_pairs:
+            return (
+                f"{self.jointype.value} JOIN {self.left_name} and"
+                f" {self.right_name} on"
+                f" {','.join([str(k[0])+'='+str(k[1]) for k in self.joinkey_pairs])}"
+            )
         return (
             f"{self.jointype.value} JOIN {self.left_name} and"
             f" {self.right_name} on {','.join([str(k) for k in self.joinkeys])}"
@@ -3002,6 +3046,7 @@ class Environment(BaseModel):
 
     materialized_concepts: List[Concept] = Field(default_factory=list)
     alias_origin_lookup: Dict[str, Concept] = Field(default_factory=dict)
+    canonical_map: Dict[str, str] = Field(default_factory=dict)
     _parse_count: int = 0
 
     @classmethod
@@ -3050,7 +3095,7 @@ class Environment(BaseModel):
             if x.address not in current_mat
         ]
         if new:
-            logger.info(f"Environment added new materialized concepts {new}")
+            logger.debug(f"Environment added new materialized concepts {new}")
 
     def validate_concept(self, lookup: str, meta: Meta | None = None):
         existing: Concept = self.concepts.get(lookup)  # type: ignore
@@ -3213,13 +3258,22 @@ class Environment(BaseModel):
         self, source: Concept, target: Concept, modifiers: List[Modifier]
     ):
         replacements = {}
+        # exit early if we've run this
+        if source.address in self.alias_origin_lookup:
+            if self.concepts[source.address] == target:
+                return
         self.alias_origin_lookup[source.address] = source
         for k, v in self.concepts.items():
+
             if v.address == target.address:
                 v.pseudonyms[source.address] = source
             if v.address == source.address:
                 replacements[k] = target
+                self.canonical_map[k] = target.address
                 v.pseudonyms[target.address] = target
+            # we need to update keys and grains of all concepts
+            else:
+                replacements[k] = v.with_merge(source, target, modifiers)
         self.concepts.update(replacements)
 
         for k, ds in self.datasources.items():

{pytrilogy-0.0.2.6 → pytrilogy-0.0.2.8}/trilogy/core/optimizations/inline_datasource.py

@@ -42,19 +42,18 @@ class InlineDatasource(OptimizationRule):
                 self.log(f"parent {parent_cte.name} datasource is not inlineable")
                 continue
             root_outputs = {x.address for x in root.output_concepts}
-            cte_outputs = {x.address for x in cte.output_columns}
-            inherited = {x for x, v in cte.source_map.items() if v}
-            # cte_inherited_outputs = {x.address for x in parent_cte.output_columns if parent_cte.source_map.get(x.address)}
-            grain_components = {x.address for x in root.grain.components}
+            inherited = {
+                x for x, v in cte.source_map.items() if v and parent_cte.name in v
+            }
             if not inherited.issubset(root_outputs):
                 cte_missing = inherited - root_outputs
                 self.log(
                     f"Not all {parent_cte.name} require inputs are found on datasource, missing {cte_missing}"
                 )
                 continue
-            if not grain_components.issubset(cte_outputs):
-                self.log("Not all datasource components in cte outputs, forcing group")
-                force_group = True
+            if not root.grain.issubset(parent_cte.grain):
+                self.log(f"Not all {parent_cte.name} is at wrong grain to inline")
+                continue
             to_inline.append(parent_cte)
 
         optimized = False
@@ -68,6 +67,11 @@ class InlineDatasource(OptimizationRule):
                     f"Skipping inlining raw datasource {replaceable.source.name} ({replaceable.name}) due to multiple references"
                 )
                 continue
+            if not replaceable.source.datasources[0].grain.issubset(replaceable.grain):
+                self.log(
+                    f"Forcing group ({parent_cte.grain} being replaced by inlined source {root.grain})"
+                )
+                force_group = True
             result = cte.inline_parent_datasource(replaceable, force_group=force_group)
             if result:
                 self.log(

{pytrilogy-0.0.2.6 → pytrilogy-0.0.2.8}/trilogy/core/processing/concept_strategies_v3.py

@@ -612,9 +612,19 @@ def _search_concepts(
         )
 
         if expanded:
-            expanded.resolve()
+            # we don't need to return the entire list; just the ones we needed pre-expansion
+            ex_resolve = expanded.resolve()
+            extra = [
+                x
+                for x in ex_resolve.output_concepts
+                if x.address not in [y.address for y in mandatory_list]
+                and x not in ex_resolve.grain.components
+            ]
+            expanded.output_concepts = mandatory_list
+            expanded.rebuild_cache()
+
             logger.info(
-                f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Found connections for {[c.address for c in mandatory_list]} via concept addition;"
+                f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Found connections for {[c.address for c in mandatory_list]} via concept addition; removing extra {[c.address for c in extra]}"
             )
             return expanded
     # if we can't find it after expanding to a merge, then

{pytrilogy-0.0.2.6 → pytrilogy-0.0.2.8}/trilogy/core/processing/node_generators/common.py

@@ -209,6 +209,7 @@ def gen_enrichment_node(
         environment=environment,
         g=g,
         parents=[enrich_node, base_node],
+        force_group=False,
         node_joins=[
             NodeJoin(
                 left_node=enrich_node,

{pytrilogy-0.0.2.6 → pytrilogy-0.0.2.8}/trilogy/core/processing/node_generators/filter_node.py

@@ -105,10 +105,29 @@ def gen_filter_node(
         environment=environment,
         g=g,
         parents=core_parents,
+        grain=Grain(
+            components=[immediate_parent] + parent_row_concepts,
+        ),
+    )
+
+    assert filter_node.resolve().grain == Grain(
+        components=[immediate_parent] + parent_row_concepts,
     )
     if not local_optional or all(
         [x.address in [y.address for y in parent_row_concepts] for x in local_optional]
     ):
+        outputs = [
+            x
+            for x in filter_node.output_concepts
+            if x.address in [y.address for y in local_optional]
+        ]
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} no extra enrichment needed for filter node"
+        )
+        filter_node.output_concepts = [
+            concept,
+        ] + outputs
+        filter_node.rebuild_cache()
         return filter_node
     enrich_node = source_concepts(  # this fetches the parent + join keys
         # to then connect to the rest of the query

{pytrilogy-0.0.2.6 → pytrilogy-0.0.2.8}/trilogy/core/processing/node_generators/group_node.py

@@ -29,7 +29,7 @@ def gen_group_node(
         resolve_function_parent_concepts(concept), "address"
     )
     logger.info(
-        f"{padding(depth)}{LOGGER_PREFIX} parent_concepts are {[x.address for x in parent_concepts]} from group grain {concept.grain}"
+        f"{padding(depth)}{LOGGER_PREFIX} parent concepts are {[x.address for x in parent_concepts]} from group grain {concept.grain}"
     )
 
     # if the aggregation has a grain, we need to ensure these are the ONLY optional in the output of the select

{pytrilogy-0.0.2.6 → pytrilogy-0.0.2.8}/trilogy/core/processing/node_generators/group_to_node.py

@@ -95,5 +95,4 @@ def gen_group_to_node(
         ],
         whole_grain=True,
         depth=depth,
-        partial_concepts=group_node.partial_concepts,
     )

{pytrilogy-0.0.2.6 → pytrilogy-0.0.2.8}/trilogy/core/processing/node_generators/node_merge_node.py

@@ -260,6 +260,7 @@ def subgraphs_to_merge_node(
     source_concepts,
     history,
     conditions,
+    enable_early_exit: bool = True,
 ):
     parents: List[StrategyNode] = []
     logger.info(
@@ -290,6 +291,8 @@ def subgraphs_to_merge_node(
     for x in parents:
         for y in x.output_concepts:
             input_c.append(y)
+    if len(parents) == 1 and enable_early_exit:
+        return parents[0]
 
     return MergeNode(
         input_concepts=unique(input_c, "address"),
@@ -350,6 +353,7 @@ def gen_merge_node(
                 source_concepts=source_concepts,
                 history=history,
                 conditions=conditions,
+                enable_early_exit=False,
             )
             if test:
                 return test

{pytrilogy-0.0.2.6 → pytrilogy-0.0.2.8}/trilogy/core/processing/node_generators/rowset_node.py

@@ -74,8 +74,9 @@ def gen_rowset_node(
     if existence_parents:
         node.parents += existence_parents
         # we don't need to join to any existence parents
-        if isinstance(node, MergeNode):
-            node.node_joins = []
+        # if isinstance(node, MergeNode) and node.node_joins is None:
+        #     # set it explicitly to empty to avoid inference
+        #     node.node_joins = []
         for parent in existence_parents:
             for x in parent.output_concepts:
                 if x.address not in node.output_lcl:

{pytrilogy-0.0.2.6 → pytrilogy-0.0.2.8}/trilogy/core/processing/nodes/base_node.py

@@ -192,7 +192,7 @@ class StrategyNode:
             p.resolve() for p in self.parents
         ]
 
-        grain = Grain(components=self.output_concepts)
+        grain = self.grain if self.grain else Grain(components=self.output_concepts)
         source_map = resolve_concept_map(
             parent_sources,
             self.output_concepts,

{pytrilogy-0.0.2.6 → pytrilogy-0.0.2.8}/trilogy/core/processing/nodes/filter_node.py

@@ -63,4 +63,5 @@ class FilterNode(StrategyNode):
             conditions=self.conditions,
             partial_concepts=list(self.partial_concepts),
             force_group=self.force_group,
+            grain=self.grain,
         )

{pytrilogy-0.0.2.6 → pytrilogy-0.0.2.8}/trilogy/core/processing/nodes/merge_node.py

@@ -78,7 +78,7 @@ def deduplicate_nodes_and_joins(
         duplicates = False
         duplicates, merged, removed = deduplicate_nodes(merged, logging_prefix)
         # filter out any removed joins
-        if joins:
+        if joins is not None:
             joins = [
                 j
                 for j in joins
@@ -138,6 +138,16 @@ class MergeNode(StrategyNode):
                     continue
                 final_joins.append(join)
             self.node_joins = final_joins
+        partial_lookup: list[Concept] = []
+        non_partial: List[Concept] = []
+        for node in parents or []:
+            partial_lookup += node.partial_concepts
+            non_partial += [
+                x for x in node.output_concepts if x not in node.partial_concepts
+            ]
+
+        final_partial = [x for x in partial_lookup if x not in non_partial]
+        self.partial_concepts = final_partial
 
     def translate_node_joins(self, node_joins: List[NodeJoin]) -> List[BaseJoin]:
         joins = []
@@ -219,12 +229,13 @@ class MergeNode(StrategyNode):
             )
             joins = self.translate_node_joins(final_joins)
         else:
+            logger.info(
+                f"{self.logging_prefix}{LOGGER_PREFIX} Final joins is not null {final_joins} but is empty, skipping join generation"
+            )
             return []
 
         for join in joins:
-            logger.info(
-                f"{self.logging_prefix}{LOGGER_PREFIX} final join {join.join_type} {[str(c) for c in join.concepts]}"
-            )
+            logger.info(f"{self.logging_prefix}{LOGGER_PREFIX} final join {str(join)}")
         return joins
 
     def _resolve(self) -> QueryDatasource:
@@ -249,6 +260,12 @@ class MergeNode(StrategyNode):
         # early exit if we can just return the parent
         final_datasets: List[QueryDatasource | Datasource] = list(merged.values())
 
+        existence_final = [
+            x
+            for x in final_datasets
+            if all([y in self.existence_concepts for y in x.output_concepts])
+        ]
+
         if len(merged.keys()) == 1:
             final: QueryDatasource | Datasource = list(merged.values())[0]
             if (
@@ -288,34 +305,25 @@ class MergeNode(StrategyNode):
         for source in final_datasets:
             pregrain += source.grain
 
-        grain = (
-            self.grain
-            if self.grain
-            else Grain(
-                components=[
-                    c
-                    for c in pregrain.components
-                    if c.address in [x.address for x in self.output_concepts]
-                ]
-            )
-        )
+        grain = self.grain if self.grain else pregrain
 
         logger.info(
             f"{self.logging_prefix}{LOGGER_PREFIX} has pre grain {pregrain} and final merge node grain {grain}"
         )
-
-        if len(final_datasets) > 1:
+        join_candidates = [x for x in final_datasets if x not in existence_final]
+        if len(join_candidates) > 1:
             joins = self.generate_joins(
-                final_datasets, final_joins, pregrain, grain, self.environment
+                join_candidates, final_joins, pregrain, grain, self.environment
             )
         else:
             joins = []
-
+        logger.info(
+            f"{self.logging_prefix}{LOGGER_PREFIX} Final join count for CTE parent count {len(join_candidates)} is {len(joins)}"
+        )
         full_join_concepts = []
         for join in joins:
             if join.join_type == JoinType.FULL:
                 full_join_concepts += join.concepts
-
         if self.whole_grain:
             force_group = False
         elif self.force_group is False:
@@ -337,9 +345,6 @@ class MergeNode(StrategyNode):
             inherited_inputs=self.input_concepts + self.existence_concepts,
             full_joins=full_join_concepts,
         )
-        logger.info(
-            f"{self.logging_prefix}{LOGGER_PREFIX} source_map {str(source_map)}"
-        )
         qds = QueryDatasource(
             input_concepts=unique(self.input_concepts, "address"),
             output_concepts=unique(self.output_concepts, "address"),

{pytrilogy-0.0.2.6 → pytrilogy-0.0.2.8}/trilogy/core/query_processor.py

@@ -183,49 +183,42 @@ def generate_cte_name(full_name: str, name_map: dict[str, str]) -> str:
         return full_name.replace("<", "").replace(">", "").replace(",", "_")
 
 
-def resolve_cte_base_name_and_alias(
+def resolve_cte_base_name_and_alias_v2(
     name: str,
     source: QueryDatasource,
-    parents: List[CTE],
-    joins: List[Join | InstantiatedUnnestJoin],
+    source_map: Dict[str, list[str]],
+    raw_joins: List[Join | InstantiatedUnnestJoin],
 ) -> Tuple[str | None, str | None]:
-
-    valid_joins: List[Join] = [join for join in joins if isinstance(join, Join)]
-    relevant_parent_sources = set()
-    for k, v in source.source_map.items():
-        if v:
-            relevant_parent_sources.update(v)
-    eligible = [x for x in source.datasources if x in relevant_parent_sources]
+    joins: List[Join] = [join for join in raw_joins if isinstance(join, Join)]
     if (
-        len(eligible) == 1
-        and isinstance(eligible[0], Datasource)
-        and not eligible[0].name == CONSTANT_DATASET
+        len(source.datasources) == 1
+        and isinstance(source.datasources[0], Datasource)
+        and not source.datasources[0].name == CONSTANT_DATASET
     ):
-        ds = eligible[0]
+        ds = source.datasources[0]
         return ds.safe_location, ds.identifier
 
-    # if we have multiple joined CTEs, pick the base
-    # as the root
-    elif len(eligible) == 1 and len(parents) == 1:
-        return parents[0].name, parents[0].name
-    elif valid_joins and len(valid_joins) > 0:
-        candidates = [x.left_cte.name for x in valid_joins]
-        disallowed = [x.right_cte.name for x in valid_joins]
+    if joins and len(joins) > 0:
+        candidates = [x.left_cte.name for x in joins]
+        disallowed = [x.right_cte.name for x in joins]
         try:
             cte = [y for y in candidates if y not in disallowed][0]
             return cte, cte
         except IndexError:
             raise SyntaxError(
-                f"Invalid join configuration {candidates} {disallowed} with all parents {[x.base_name for x in parents]}"
+                f"Invalid join configuration {candidates} {disallowed} for {name}",
             )
-    elif eligible:
-        matched = [x for x in parents if x.source.name == eligible[0].name]
-        if matched:
-            return matched[0].name, matched[0].name
 
-    logger.info(
-        f"Could not determine CTE base name for {name} with relevant sources {relevant_parent_sources}"
-    )
+    counts: dict[str, int] = defaultdict(lambda: 0)
+    output_addresses = [x.address for x in source.output_concepts]
+    for k, v in source_map.items():
+        for vx in v:
+            if k in output_addresses:
+                counts[vx] = counts[vx] + 1
+            else:
+                counts[vx] = counts[vx]
+    if counts:
+        return max(counts, key=counts.get), max(counts, key=counts.get)  # type: ignore
     return None, None
 
 
@@ -274,8 +267,8 @@ def datasource_to_ctes(
         for x in [base_join_to_join(join, parents) for join in query_datasource.joins]
         if x
     ]
-    base_name, base_alias = resolve_cte_base_name_and_alias(
-        human_id, query_datasource, parents, final_joins
+    base_name, base_alias = resolve_cte_base_name_and_alias_v2(
+        human_id, query_datasource, source_map, final_joins
     )
     cte = CTE(
         name=human_id,

{pytrilogy-0.0.2.6 → pytrilogy-0.0.2.8}/trilogy/dialect/base.py

@@ -51,7 +51,7 @@ from trilogy.core.models import (
     MergeStatementV2,
 )
 from trilogy.core.query_processor import process_query, process_persist
-from trilogy.dialect.common import render_join
+from trilogy.dialect.common import render_join, render_unnest
 from trilogy.hooks.base_hook import BaseHook
 from trilogy.core.enums import UnnestMode
 
@@ -128,6 +128,7 @@ FUNCTION_MAP = {
     FunctionType.UNNEST: lambda x: f"unnest({x[0]})",
     FunctionType.ATTR_ACCESS: lambda x: f"""{x[0]}.{x[1].replace("'", "")}""",
     FunctionType.STRUCT: lambda x: f"{{{', '.join(struct_arg(x))}}}",
+    FunctionType.ARRAY: lambda x: f"[{', '.join(x)}]",
     # math
     FunctionType.ADD: lambda x: f"{x[0]} + {x[1]}",
     FunctionType.SUBTRACT: lambda x: f"{x[0]} - {x[1]}",
@@ -466,7 +467,7 @@ class BaseDialect:
         elif isinstance(e, MapWrapper):
             return f"MAP {{{','.join([f'{self.render_expr(k, cte=cte, cte_map=cte_map)}:{self.render_expr(v, cte=cte, cte_map=cte_map)}' for k, v in e.items()])}}}"
         elif isinstance(e, list):
-            return f"[{','.join([self.render_expr(x, cte=cte, cte_map=cte_map) for x in e])}]"
+            return f"{self.FUNCTION_MAP[FunctionType.ARRAY]([self.render_expr(x, cte=cte, cte_map=cte_map) for x in e])}"
         elif isinstance(e, DataType):
             return str(e.value)
         elif isinstance(e, DatePart):
@@ -480,8 +481,12 @@ class BaseDialect:
             raise ValueError(f"Unable to render type {type(e)} {e}")
 
     def render_cte(self, cte: CTE):
-        if self.UNNEST_MODE in (UnnestMode.CROSS_APPLY, UnnestMode.CROSS_JOIN):
-            # for a cross apply, derviation happens in the join
+        if self.UNNEST_MODE in (
+            UnnestMode.CROSS_APPLY,
+            UnnestMode.CROSS_JOIN,
+            UnnestMode.CROSS_JOIN_ALIAS,
+        ):
+            # for a cross apply, derivation happens in the join
             # so we only use the alias to select
             select_columns = [
                 self.render_concept_sql(c, cte)
@@ -499,20 +504,42 @@ class BaseDialect:
                 for c in cte.output_columns
                 if c.address not in [y.address for y in cte.hidden_concepts]
             ]
-        if cte.quote_address:
-            source = f"{self.QUOTE_CHARACTER}{cte.base_name}{self.QUOTE_CHARACTER}"
+        source: str | None = cte.base_name
+        if not cte.render_from_clause:
+            if len(cte.joins) > 0:
+                if cte.join_derived_concepts and self.UNNEST_MODE in (
+                    UnnestMode.CROSS_JOIN_ALIAS,
+                    UnnestMode.CROSS_JOIN,
+                    UnnestMode.CROSS_APPLY,
+                ):
+                    source = f"{render_unnest(self.UNNEST_MODE, self.QUOTE_CHARACTER, cte.join_derived_concepts[0], self.render_concept_sql, cte)}"
+                # direct - eg DUCK DB - can be directly selected inline
+                elif cte.join_derived_concepts and self.UNNEST_MODE == UnnestMode.DIRECT:
+                    source = None
+                else:
+                    raise SyntaxError("CTE has joins but no from clause")
+            else:
+                source = None
+        else:
+            if cte.quote_address:
+                source = f"{self.QUOTE_CHARACTER}{cte.base_name}{self.QUOTE_CHARACTER}"
+            else:
+                source = cte.base_name
+            if cte.base_name != cte.base_alias:
+                source = f"{source} as {cte.base_alias}"
+        if not cte.render_from_clause:
+            final_joins = []
         else:
-            source = cte.base_name
-        if cte.base_name != cte.base_alias:
-            source = f"{source} as {cte.base_alias}"
+            final_joins = cte.joins or []
         return CompiledCTE(
             name=cte.name,
             statement=self.SQL_TEMPLATE.render(
                 select_columns=select_columns,
-                base=(f"{source}" if cte.render_from_clause else None),
+                base=f"{source}" if source else None,
                 grain=cte.grain,
                 limit=cte.limit,
                 # some joins may not need to be rendered
+                comment=cte.comment if CONFIG.show_comments else None,
                 joins=[
                     j
                     for j in [
@@ -523,7 +550,7 @@ class BaseDialect:
                             cte,
                             self.UNNEST_MODE,
                         )
-                        for join in (cte.joins or [])
+                        for join in final_joins
                     ]
                     if j
                 ],

{pytrilogy-0.0.2.6 → pytrilogy-0.0.2.8}/trilogy/dialect/common.py

@@ -9,6 +9,18 @@ def null_wrapper(lval: str, rval: str, concept: Concept) -> str:
     return f"{lval} = {rval}"
 
 
+def render_unnest(
+    unnest_mode: UnnestMode,
+    quote_character: str,
+    concept: Concept,
+    render_func: Callable[[Concept, CTE, bool], str],
+    cte: CTE,
+):
+    if unnest_mode == UnnestMode.CROSS_JOIN:
+        return f"{render_func(concept, cte, False)} as {quote_character}{concept.safe_address}{quote_character}"
+    return f"{render_func(concept, cte, False)} as unnest_wrapper ({quote_character}{concept.safe_address}{quote_character})"
+
+
 def render_join(
     join: Join | InstantiatedUnnestJoin,
     quote_character: str,
@@ -25,10 +37,10 @@ def render_join(
         if not cte:
             raise ValueError("must provide a cte to build an unnest joins")
         if unnest_mode == UnnestMode.CROSS_JOIN:
-            return f"CROSS JOIN {render_func(join.concept, cte, False)} as {quote_character}{join.concept.safe_address}{quote_character}"
+            return f"CROSS JOIN {render_unnest(unnest_mode, quote_character, join.concept, render_func, cte)}"
         if unnest_mode == UnnestMode.CROSS_JOIN_ALIAS:
-            return f"CROSS JOIN {render_func(join.concept, cte, False)} as array_unnest ({quote_character}{join.concept.safe_address}{quote_character})"
-        return f"FULL JOIN {render_func(join.concept, cte, False)} as unnest_wrapper({quote_character}{join.concept.safe_address}{quote_character})"
+            return f"CROSS JOIN {render_unnest(unnest_mode, quote_character, join.concept, render_func, cte)}"
+        return f"FULL JOIN {render_unnest(unnest_mode, quote_character, join.concept, render_func, cte)}"
     left_name = join.left_name
     right_name = join.right_name
     right_base = join.right_ref

{pytrilogy-0.0.2.6 → pytrilogy-0.0.2.8}/trilogy/dialect/duckdb.py

@@ -47,8 +47,9 @@ CREATE OR REPLACE TABLE {{ output.address.location }} AS
 {% endif %}{%- if ctes %}
 WITH {% for cte in ctes %}
 {{cte.name}} as ({{cte.statement}}){% if not loop.last %},{% endif %}{% endfor %}{% endif %}
-{% if full_select -%}{{full_select}}
-{% else -%}
+{%- if full_select -%}{{full_select}}
+{%- else -%}{%- if comment %}
+-- {{ comment }}{% endif %}
 SELECT
 {%- for select in select_columns %}
     {{ select }}{% if not loop.last %},{% endif %}{% endfor %}
@@ -56,7 +57,8 @@ SELECT
     {{ base }}{% endif %}{% if joins %}
 {%- for join in joins %}
     {{ join }}{% endfor %}{% endif %}
-{% if where %}WHERE
+{%- if where %}
+WHERE
     {{ where }}
 {% endif -%}{%- if group_by %}
 GROUP BY {% for group in group_by %}

{pytrilogy-0.0.2.6 → pytrilogy-0.0.2.8}/trilogy/dialect/presto.py

@@ -33,6 +33,7 @@ FUNCTION_MAP = {
     FunctionType.DATE_ADD: lambda x: f"DATE_ADD('{x[1]}', {x[2]}, {x[0]})",
     FunctionType.CURRENT_DATE: lambda x: "CURRENT_DATE",
     FunctionType.CURRENT_DATETIME: lambda x: "CURRENT_TIMESTAMP",
+    FunctionType.ARRAY: lambda x: f"ARRAY[{', '.join(x)}]",
 }
 
 FUNCTION_GRAIN_MATCH_MAP = {
@@ -86,7 +87,7 @@ class PrestoDialect(BaseDialect):
     QUOTE_CHARACTER = '"'
     SQL_TEMPLATE = SQL_TEMPLATE
     DATATYPE_MAP = {**BaseDialect.DATATYPE_MAP, DataType.NUMERIC: "DECIMAL"}
-    UNNEST_MODE = UnnestMode.CROSS_JOIN
+    UNNEST_MODE = UnnestMode.CROSS_JOIN_ALIAS
 
 
 class TrinoDialect(PrestoDialect):

{pytrilogy-0.0.2.6 → pytrilogy-0.0.2.8}/trilogy/parsing/common.py

@@ -112,12 +112,16 @@ def filter_item_to_concept(
     return Concept(
         name=name,
         datatype=parent.content.datatype,
-        purpose=parent.content.purpose,
+        purpose=Purpose.PROPERTY,
         lineage=parent,
         metadata=fmetadata,
         namespace=namespace,
         # filtered copies cannot inherit keys
-        keys=None,
+        keys=(
+            parent.content.keys
+            if parent.content.purpose == Purpose.PROPERTY
+            else (parent.content,)
+        ),
         grain=(
             parent.content.grain
             if parent.content.purpose == Purpose.PROPERTY