pytrilogy 0.0.2.11__py3-none-any.whl → 0.0.2.12__py3-none-any.whl

trilogy/core/processing/node_generators/unnest_node.py

@@ -22,9 +22,14 @@ def gen_unnest_node(
     arguments = []
     if isinstance(concept.lineage, Function):
         arguments = concept.lineage.concept_arguments
+
+    equivalent_optional = [x for x in local_optional if x.lineage == concept.lineage]
+    non_equivalent_optional = [
+        x for x in local_optional if x not in equivalent_optional
+    ]
     if arguments or local_optional:
         parent = source_concepts(
-            mandatory_list=arguments + local_optional,
+            mandatory_list=arguments + non_equivalent_optional,
             environment=environment,
             g=g,
             depth=depth + 1,
@@ -38,8 +43,8 @@ def gen_unnest_node(
             return None
 
     base = UnnestNode(
-        unnest_concept=concept,
-        input_concepts=arguments + local_optional,
+        unnest_concepts=[concept] + equivalent_optional,
+        input_concepts=arguments + non_equivalent_optional,
         output_concepts=[concept] + local_optional,
         environment=environment,
         g=g,
@@ -57,4 +62,6 @@ def gen_unnest_node(
     )
     qds = new.resolve()
     assert qds.source_map[concept.address] == {base.resolve()}
+    for x in equivalent_optional:
+        assert qds.source_map[x.address] == {base.resolve()}
     return new

trilogy/core/processing/nodes/base_node.py

@@ -61,17 +61,22 @@ def resolve_concept_map(
         for concept in input.output_concepts:
             if concept.address not in input.non_partial_concept_addresses:
                 continue
-            if concept.address not in inherited:
-                continue
+
             if (
                 isinstance(input, QueryDatasource)
                 and concept.address in input.hidden_concepts
             ):
                 continue
             if concept.address in full_addresses:
+
                 concept_map[concept.address].add(input)
             elif concept.address not in concept_map:
+                # equi_targets = [x for x in targets if concept.address in x.pseudonyms or x.address in concept.pseudonyms]
+                # if equi_targets:
+                #     for equi in equi_targets:
+                #         concept_map[equi.address] = set()
                 concept_map[concept.address].add(input)
+
     # second loop, include partials
     for input in inputs:
         for concept in input.output_concepts:

trilogy/core/processing/nodes/group_node.py

@@ -121,7 +121,6 @@ class GroupNode(StrategyNode):
             source_type=source_type,
             source_map=resolve_concept_map(
                 parent_sources,
-                # targets = self.output_concepts,
                 targets=(
                     unique(
                         self.output_concepts + self.conditions.concept_arguments,

trilogy/core/processing/nodes/merge_node.py

@@ -28,14 +28,18 @@ LOGGER_PREFIX = "[CONCEPT DETAIL - MERGE NODE]"
 
 
 def deduplicate_nodes(
-    merged: dict[str, QueryDatasource | Datasource], logging_prefix: str
+    merged: dict[str, QueryDatasource | Datasource],
+    logging_prefix: str,
+    environment: Environment,
 ) -> tuple[bool, dict[str, QueryDatasource | Datasource], set[str]]:
     duplicates = False
     removed: set[str] = set()
     set_map: dict[str, set[str]] = {}
     for k, v in merged.items():
         unique_outputs = [
-            x.address for x in v.output_concepts if x not in v.partial_concepts
+            environment.concepts[x.address].address
+            for x in v.output_concepts
+            if x not in v.partial_concepts
         ]
         set_map[k] = set(unique_outputs)
     for k1, v1 in set_map.items():
@@ -71,12 +75,15 @@ def deduplicate_nodes_and_joins(
     joins: List[NodeJoin] | None,
     merged: dict[str, QueryDatasource | Datasource],
     logging_prefix: str,
+    environment: Environment,
 ) -> Tuple[List[NodeJoin] | None, dict[str, QueryDatasource | Datasource]]:
     # it's possible that we have more sources than we need
     duplicates = True
     while duplicates:
         duplicates = False
-        duplicates, merged, removed = deduplicate_nodes(merged, logging_prefix)
+        duplicates, merged, removed = deduplicate_nodes(
+            merged, logging_prefix, environment=environment
+        )
         # filter out any removed joins
         if joins is not None:
             joins = [
@@ -245,7 +252,7 @@ class MergeNode(StrategyNode):
 
         # it's possible that we have more sources than we need
         final_joins, merged = deduplicate_nodes_and_joins(
-            final_joins, merged, self.logging_prefix
+            final_joins, merged, self.logging_prefix, self.environment
         )
         # early exit if we can just return the parent
         final_datasets: List[QueryDatasource | Datasource] = list(merged.values())

trilogy/core/processing/nodes/unnest_node.py

@@ -6,6 +6,7 @@ from trilogy.core.models import (
     SourceType,
     Concept,
     UnnestJoin,
+    Function,
 )
 from trilogy.core.processing.nodes.base_node import StrategyNode
 
@@ -19,7 +20,7 @@ class UnnestNode(StrategyNode):
 
     def __init__(
         self,
-        unnest_concept: Concept,
+        unnest_concepts: List[Concept],
         input_concepts: List[Concept],
         output_concepts: List[Concept],
         environment,
@@ -37,25 +38,28 @@ class UnnestNode(StrategyNode):
             parents=parents,
             depth=depth,
         )
-        self.unnest_concept = unnest_concept
+        self.unnest_concepts = unnest_concepts
 
     def _resolve(self) -> QueryDatasource:
         """We need to ensure that any filtered values are removed from the output to avoid inappropriate references"""
         base = super()._resolve()
-
+        lineage = self.unnest_concepts[0].lineage
+        assert isinstance(lineage, Function)
+        final = "_".join(set([c.address for c in self.unnest_concepts]))
         unnest = UnnestJoin(
-            concept=self.unnest_concept,
-            alias=f'unnest_{self.unnest_concept.address.replace(".", "_")}',
+            concepts=self.unnest_concepts,
+            parent=lineage,
+            alias=f'unnest_{final.replace(".", "_")}',
         )
         base.joins.append(unnest)
-
-        base.source_map[self.unnest_concept.address] = {unnest}
-        base.join_derived_concepts = [self.unnest_concept]
+        for unnest_concept in self.unnest_concepts:
+            base.source_map[unnest_concept.address] = {unnest}
+            base.join_derived_concepts = [unnest_concept]
         return base
 
     def copy(self) -> "UnnestNode":
         return UnnestNode(
-            unnest_concept=self.unnest_concept,
+            unnest_concepts=self.unnest_concepts,
             input_concepts=list(self.input_concepts),
             output_concepts=list(self.output_concepts),
             environment=self.environment,

trilogy/core/processing/utility.py

@@ -285,7 +285,9 @@ def get_node_joins(
                     raise SyntaxError(
                         f"Could not find {joinc.address} in {right_datasource.identifier} output {[c.address for c in right_datasource.output_concepts]}"
                     )
-            join_tuples.append((left_arg, right_arg))
+            narg = (left_arg, right_arg)
+            if narg not in join_tuples:
+                join_tuples.append((left_arg, right_arg))
         final_joins_pre.append(
             BaseJoin(
                 left_datasource=identifier_map[left],

trilogy/core/query_processor.py

@@ -46,7 +46,10 @@ def base_join_to_join(
     """This function converts joins at the datasource level
     to joins at the CTE level"""
     if isinstance(base_join, UnnestJoin):
-        return InstantiatedUnnestJoin(concept=base_join.concept, alias=base_join.alias)
+        return InstantiatedUnnestJoin(
+            concept_to_unnest=base_join.parent.concept_arguments[0],
+            alias=base_join.alias,
+        )
     if base_join.left_datasource.identifier == base_join.right_datasource.identifier:
         raise ValueError(f"Joining on same datasource {base_join}")
     left_ctes = [
@@ -306,7 +309,11 @@ def datasource_to_ctes(
     if cte.grain != query_datasource.grain:
         raise ValueError("Grain was corrupted in CTE generation")
     for x in cte.output_columns:
-        if x.address not in cte.source_map and CONFIG.validate_missing:
+        if (
+            x.address not in cte.source_map
+            and not any(y in cte.source_map for y in x.pseudonyms)
+            and CONFIG.validate_missing
+        ):
             raise ValueError(
                 f"Missing {x.address} in {cte.source_map}, source map {cte.source.source_map.keys()} "
             )

trilogy/dialect/base.py

@@ -9,7 +9,6 @@ from trilogy.core.enums import (
     FunctionType,
     WindowType,
     DatePart,
-    ComparisonOperator,
 )
 from trilogy.core.models import (
     ListType,
@@ -227,13 +226,7 @@ def safe_get_cte_value(coalesce, cte: CTE, c: Concept, quote_char: str):
     raw = cte.source_map.get(address, None)
 
     if not raw:
-        for k, v in c.pseudonyms.items():
-            if cte.source_map.get(k):
-                c = v
-                raw = cte.source_map[k]
-                break
-        if not raw:
-            return INVALID_REFERENCE_STRING("Missing source reference")
+        return None
     if isinstance(raw, str):
         rendered = cte.get_alias(c, raw)
         return f"{raw}.{safe_quote(rendered, quote_char)}"
@@ -260,35 +253,64 @@ class BaseDialect:
 
         return f"{self.render_concept_sql(order_item.expr, cte=cte, alias=False)} {order_item.order.value}"
 
-    def render_concept_sql(self, c: Concept, cte: CTE, alias: bool = True) -> str:
+    def render_concept_sql(
+        self, c: Concept, cte: CTE, alias: bool = True, raise_invalid: bool = False
+    ) -> str:
+        result = None
+        if c.pseudonyms:
+            for candidate in [c] + list(c.pseudonyms.values()):
+                try:
+                    logger.debug(
+                        f"{LOGGER_PREFIX} [{c.address}] Attempting rendering w/ candidate {candidate.address}"
+                    )
+                    result = self._render_concept_sql(
+                        candidate, cte, raise_invalid=True
+                    )
+                    if result:
+                        break
+                except ValueError:
+                    continue
+        if not result:
+            result = self._render_concept_sql(c, cte, raise_invalid=raise_invalid)
+        if alias:
+            return f"{result} as {self.QUOTE_CHARACTER}{c.safe_address}{self.QUOTE_CHARACTER}"
+        return result
+
+    def _render_concept_sql(
+        self, c: Concept, cte: CTE, raise_invalid: bool = False
+    ) -> str:
         # only recurse while it's in sources of the current cte
         logger.debug(
             f"{LOGGER_PREFIX} [{c.address}] Starting rendering loop on cte: {cte.name}"
         )
 
+        # check if it's not inherited AND no pseudonyms are inherited
         if c.lineage and cte.source_map.get(c.address, []) == []:
             logger.debug(
-                f"{LOGGER_PREFIX} [{c.address}] rendering concept with lineage that is not already existing"
+                f"{LOGGER_PREFIX} [{c.address}] rendering concept with lineage that is not already existing, have {cte.source_map}"
             )
             if isinstance(c.lineage, WindowItem):
                 rendered_order_components = [
-                    f"{self.render_concept_sql(x.expr, cte, alias=False)} {x.order.value}"
+                    f"{self.render_concept_sql(x.expr, cte, alias=False, raise_invalid=raise_invalid)} {x.order.value}"
                     for x in c.lineage.order_by
                 ]
                 rendered_over_components = [
-                    self.render_concept_sql(x, cte, alias=False) for x in c.lineage.over
+                    self.render_concept_sql(
+                        x, cte, alias=False, raise_invalid=raise_invalid
+                    )
+                    for x in c.lineage.over
                 ]
-                rval = f"{self.WINDOW_FUNCTION_MAP[c.lineage.type](concept = self.render_concept_sql(c.lineage.content, cte=cte, alias=False), window=','.join(rendered_over_components), sort=','.join(rendered_order_components))}"  # noqa: E501
+                rval = f"{self.WINDOW_FUNCTION_MAP[c.lineage.type](concept = self.render_concept_sql(c.lineage.content, cte=cte, alias=False, raise_invalid=raise_invalid), window=','.join(rendered_over_components), sort=','.join(rendered_order_components))}"  # noqa: E501
             elif isinstance(c.lineage, FilterItem):
                 # for cases when we've optimized this
                 if cte.condition == c.lineage.where.conditional:
                     rval = self.render_expr(c.lineage.content, cte=cte)
                 else:
-                    rval = f"CASE WHEN {self.render_expr(c.lineage.where.conditional, cte=cte)} THEN {self.render_concept_sql(c.lineage.content, cte=cte, alias=False)} ELSE NULL END"
+                    rval = f"CASE WHEN {self.render_expr(c.lineage.where.conditional, cte=cte)} THEN {self.render_concept_sql(c.lineage.content, cte=cte, alias=False, raise_invalid=raise_invalid)} ELSE NULL END"
             elif isinstance(c.lineage, RowsetItem):
-                rval = f"{self.render_concept_sql(c.lineage.content, cte=cte, alias=False)}"
+                rval = f"{self.render_concept_sql(c.lineage.content, cte=cte, alias=False, raise_invalid=raise_invalid)}"
             elif isinstance(c.lineage, MultiSelectStatement):
-                rval = f"{self.render_concept_sql(c.lineage.find_source(c, cte), cte=cte, alias=False)}"
+                rval = f"{self.render_concept_sql(c.lineage.find_source(c, cte), cte=cte, alias=False, raise_invalid=raise_invalid)}"
             elif isinstance(c.lineage, AggregateWrapper):
                 args = [
                     self.render_expr(v, cte)  # , alias=False)
@@ -304,16 +326,15 @@ class BaseDialect:
                     rval = f"{self.FUNCTION_GRAIN_MATCH_MAP[c.lineage.function.operator](args)}"
             else:
                 args = [
-                    self.render_expr(v, cte)  # , alias=False)
+                    self.render_expr(
+                        v, cte=cte, raise_invalid=raise_invalid
+                    )  # , alias=False)
                     for v in c.lineage.arguments
                 ]
 
                 if cte.group_to_grain:
                     rval = f"{self.FUNCTION_MAP[c.lineage.operator](args)}"
                 else:
-                    logger.debug(
-                        f"{LOGGER_PREFIX} [{c.address}] ignoring optimazable aggregate function, at grain so optimizing"
-                    )
                     rval = f"{self.FUNCTION_GRAIN_MATCH_MAP[c.lineage.operator](args)}"
         else:
             logger.debug(
@@ -324,14 +345,24 @@ class BaseDialect:
             if isinstance(raw_content, RawColumnExpr):
                 rval = raw_content.text
             elif isinstance(raw_content, Function):
-                rval = self.render_expr(raw_content, cte=cte)
+                rval = self.render_expr(
+                    raw_content, cte=cte, raise_invalid=raise_invalid
+                )
             else:
-                rval = f"{safe_get_cte_value(self.FUNCTION_MAP[FunctionType.COALESCE], cte, c, self.QUOTE_CHARACTER)}"
-        if alias:
-            return (
-                f"{rval} as"
-                f" {self.QUOTE_CHARACTER}{c.safe_address}{self.QUOTE_CHARACTER}"
-            )
+                rval = safe_get_cte_value(
+                    self.FUNCTION_MAP[FunctionType.COALESCE],
+                    cte,
+                    c,
+                    self.QUOTE_CHARACTER,
+                )
+                if not rval:
+                    if raise_invalid:
+                        raise ValueError(
+                            f"Invalid reference string found in query: {rval}, this should never occur. Please report this issue."
+                        )
+                    rval = INVALID_REFERENCE_STRING(
+                        f"Missing source reference to {c.address}"
+                    )
         return rval
 
     def render_expr(
@@ -367,6 +398,7 @@ class BaseDialect:
         ],
         cte: Optional[CTE] = None,
         cte_map: Optional[Dict[str, CTE]] = None,
+        raise_invalid: bool = False,
     ) -> str:
 
         if isinstance(e, SubselectComparison):
@@ -395,9 +427,9 @@ class BaseDialect:
                     target = INVALID_REFERENCE_STRING(
                         f"Missing source CTE for {e.right.address}"
                     )
-                return f"{self.render_expr(e.left, cte=cte, cte_map=cte_map)} {e.operator.value} (select {target}.{self.QUOTE_CHARACTER}{e.right.safe_address}{self.QUOTE_CHARACTER} from {target} where {target}.{self.QUOTE_CHARACTER}{e.right.safe_address}{self.QUOTE_CHARACTER} is not null)"
+                return f"{self.render_expr(e.left, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid)} {e.operator.value} (select {target}.{self.QUOTE_CHARACTER}{e.right.safe_address}{self.QUOTE_CHARACTER} from {target} where {target}.{self.QUOTE_CHARACTER}{e.right.safe_address}{self.QUOTE_CHARACTER} is not null)"
             elif isinstance(e.right, (ListWrapper, Parenthetical, list)):
-                return f"{self.render_expr(e.left, cte=cte, cte_map=cte_map)} {e.operator.value} {self.render_expr(e.right, cte=cte, cte_map=cte_map)}"
+                return f"{self.render_expr(e.left, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid)} {e.operator.value} {self.render_expr(e.right, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid)}"
 
             elif isinstance(
                 e.right,
@@ -408,53 +440,64 @@ class BaseDialect:
                     float,
                 ),
             ):
-                return f"{self.render_expr(e.left, cte=cte, cte_map=cte_map)} {e.operator.value} ({self.render_expr(e.right, cte=cte, cte_map=cte_map)})"
+                return f"{self.render_expr(e.left, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid)} {e.operator.value} ({self.render_expr(e.right, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid)})"
             else:
-                return f"{self.render_expr(e.left, cte=cte, cte_map=cte_map)} {e.operator.value} {self.render_expr(e.right, cte=cte, cte_map=cte_map)}"
+                return f"{self.render_expr(e.left, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid)} {e.operator.value} {self.render_expr(e.right, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid)}"
         elif isinstance(e, Comparison):
-            if e.operator == ComparisonOperator.BETWEEN:
-                right_comp = e.right
-                assert isinstance(right_comp, Conditional)
-                return f"{self.render_expr(e.left, cte=cte, cte_map=cte_map)} {e.operator.value} {self.render_expr(right_comp.left, cte=cte, cte_map=cte_map) and self.render_expr(right_comp.right, cte=cte, cte_map=cte_map)}"
-            return f"{self.render_expr(e.left, cte=cte, cte_map=cte_map)} {e.operator.value} {self.render_expr(e.right, cte=cte, cte_map=cte_map)}"
+            return f"{self.render_expr(e.left, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid)} {e.operator.value} {self.render_expr(e.right, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid)}"
         elif isinstance(e, Conditional):
             # conditions need to be nested in parentheses
-            return f"{self.render_expr(e.left, cte=cte, cte_map=cte_map)} {e.operator.value} {self.render_expr(e.right, cte=cte, cte_map=cte_map)}"
+            return f"{self.render_expr(e.left, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid)} {e.operator.value} {self.render_expr(e.right, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid)}"
         elif isinstance(e, WindowItem):
             rendered_order_components = [
-                f"{self.render_expr(x.expr, cte, cte_map=cte_map)} {x.order.value}"
+                f"{self.render_expr(x.expr, cte, cte_map=cte_map, raise_invalid=raise_invalid)} {x.order.value}"
                 for x in e.order_by
             ]
             rendered_over_components = [
-                self.render_expr(x, cte, cte_map=cte_map) for x in e.over
+                self.render_expr(x, cte, cte_map=cte_map, raise_invalid=raise_invalid)
+                for x in e.over
             ]
-            return f"{self.WINDOW_FUNCTION_MAP[e.type](concept = self.render_expr(e.content, cte=cte, cte_map=cte_map), window=','.join(rendered_over_components), sort=','.join(rendered_order_components))}"  # noqa: E501
+            return f"{self.WINDOW_FUNCTION_MAP[e.type](concept = self.render_expr(e.content, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid), window=','.join(rendered_over_components), sort=','.join(rendered_order_components))}"  # noqa: E501
         elif isinstance(e, Parenthetical):
             # conditions need to be nested in parentheses
             if isinstance(e.content, list):
-                return f"( {','.join([self.render_expr(x, cte=cte, cte_map=cte_map) for x in e.content])} )"
-            return f"( {self.render_expr(e.content, cte=cte, cte_map=cte_map)} )"
+                return f"( {','.join([self.render_expr(x, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid) for x in e.content])} )"
+            return f"( {self.render_expr(e.content, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid)} )"
         elif isinstance(e, CaseWhen):
-            return f"WHEN {self.render_expr(e.comparison, cte=cte, cte_map=cte_map) } THEN {self.render_expr(e.expr, cte=cte, cte_map=cte_map) }"
+            return f"WHEN {self.render_expr(e.comparison, cte=cte, cte_map=cte_map) } THEN {self.render_expr(e.expr, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid) }"
         elif isinstance(e, CaseElse):
-            return f"ELSE {self.render_expr(e.expr, cte=cte, cte_map=cte_map) }"
+            return f"ELSE {self.render_expr(e.expr, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid) }"
         elif isinstance(e, Function):
 
             if cte and cte.group_to_grain:
                 return self.FUNCTION_MAP[e.operator](
-                    [self.render_expr(z, cte=cte, cte_map=cte_map) for z in e.arguments]
+                    [
+                        self.render_expr(
+                            z, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid
+                        )
+                        for z in e.arguments
+                    ]
                 )
 
             return self.FUNCTION_GRAIN_MATCH_MAP[e.operator](
-                [self.render_expr(z, cte=cte, cte_map=cte_map) for z in e.arguments]
+                [
+                    self.render_expr(
+                        z, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid
+                    )
+                    for z in e.arguments
+                ]
             )
         elif isinstance(e, AggregateWrapper):
-            return self.render_expr(e.function, cte, cte_map=cte_map)
+            return self.render_expr(
+                e.function, cte, cte_map=cte_map, raise_invalid=raise_invalid
+            )
         elif isinstance(e, FilterItem):
-            return f"CASE WHEN {self.render_expr(e.where.conditional,cte=cte, cte_map=cte_map)} THEN {self.render_expr(e.content, cte, cte_map=cte_map)} ELSE NULL END"
+            return f"CASE WHEN {self.render_expr(e.where.conditional,cte=cte, cte_map=cte_map, raise_invalid=raise_invalid)} THEN {self.render_expr(e.content, cte, cte_map=cte_map, raise_invalid=raise_invalid)} ELSE NULL END"
         elif isinstance(e, Concept):
             if cte:
-                return self.render_concept_sql(e, cte, alias=False)
+                return self.render_concept_sql(
+                    e, cte, alias=False, raise_invalid=raise_invalid
+                )
             elif cte_map:
                 return f"{cte_map[e.address].name}.{self.QUOTE_CHARACTER}{e.safe_address}{self.QUOTE_CHARACTER}"
             return f"{self.QUOTE_CHARACTER}{e.safe_address}{self.QUOTE_CHARACTER}"
@@ -465,11 +508,11 @@ class BaseDialect:
         elif isinstance(e, (int, float)):
             return str(e)
         elif isinstance(e, ListWrapper):
-            return f"[{','.join([self.render_expr(x, cte=cte, cte_map=cte_map) for x in e])}]"
+            return f"[{','.join([self.render_expr(x, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid) for x in e])}]"
         elif isinstance(e, MapWrapper):
-            return f"MAP {{{','.join([f'{self.render_expr(k, cte=cte, cte_map=cte_map)}:{self.render_expr(v, cte=cte, cte_map=cte_map)}' for k, v in e.items()])}}}"
+            return f"MAP {{{','.join([f'{self.render_expr(k, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid)}:{self.render_expr(v, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid)}' for k, v in e.items()])}}}"
         elif isinstance(e, list):
-            return f"{self.FUNCTION_MAP[FunctionType.ARRAY]([self.render_expr(x, cte=cte, cte_map=cte_map) for x in e])}"
+            return f"{self.FUNCTION_MAP[FunctionType.ARRAY]([self.render_expr(x, cte=cte, cte_map=cte_map, raise_invalid=raise_invalid) for x in e])}"
         elif isinstance(e, DataType):
             return str(e.value)
         elif isinstance(e, DatePart):

trilogy/dialect/common.py

@@ -37,10 +37,10 @@ def render_join(
         if not cte:
             raise ValueError("must provide a cte to build an unnest joins")
         if unnest_mode == UnnestMode.CROSS_JOIN:
-            return f"CROSS JOIN {render_unnest(unnest_mode, quote_character, join.concept, render_func, cte)}"
+            return f"CROSS JOIN {render_unnest(unnest_mode, quote_character, join.concept_to_unnest, render_func, cte)}"
         if unnest_mode == UnnestMode.CROSS_JOIN_ALIAS:
-            return f"CROSS JOIN {render_unnest(unnest_mode, quote_character, join.concept, render_func, cte)}"
-        return f"FULL JOIN {render_unnest(unnest_mode, quote_character, join.concept, render_func, cte)}"
+            return f"CROSS JOIN {render_unnest(unnest_mode, quote_character, join.concept_to_unnest, render_func, cte)}"
+        return f"FULL JOIN {render_unnest(unnest_mode, quote_character, join.concept_to_unnest, render_func, cte)}"
     left_name = join.left_name
     right_name = join.right_name
     right_base = join.right_ref

trilogy/parsing/common.py

@@ -9,6 +9,10 @@ from trilogy.core.models import (
     ListWrapper,
     MapWrapper,
     WindowItem,
+    Meta,
+    Parenthetical,
+    FunctionClass,
+    Environment,
 )
 from typing import List, Tuple
 from trilogy.core.functions import (
@@ -16,8 +20,61 @@ from trilogy.core.functions import (
     FunctionType,
     arg_to_datatype,
 )
-from trilogy.utility import unique
+from trilogy.utility import unique, string_to_hash
 from trilogy.core.enums import PurposeLineage
+from trilogy.constants import (
+    VIRTUAL_CONCEPT_PREFIX,
+)
+
+
+def process_function_args(
+    args,
+    meta: Meta | None,
+    environment: Environment,
+):
+    final: List[Concept | Function] = []
+    for arg in args:
+        # if a function has an anonymous function argument
+        # create an implicit concept
+        while isinstance(arg, Parenthetical):
+            arg = arg.content
+        if isinstance(arg, Function):
+            # if it's not an aggregate function, we can skip the virtual concepts
+            # to simplify anonymous function handling
+            if (
+                arg.operator not in FunctionClass.AGGREGATE_FUNCTIONS.value
+                and arg.operator != FunctionType.UNNEST
+            ):
+                final.append(arg)
+                continue
+            id_hash = string_to_hash(str(arg))
+            concept = function_to_concept(
+                arg,
+                name=f"{VIRTUAL_CONCEPT_PREFIX}_{id_hash}",
+                namespace=environment.namespace,
+            )
+            # to satisfy mypy, concept will always have metadata
+            if concept.metadata and meta:
+                concept.metadata.line_number = meta.line
+            environment.add_concept(concept, meta=meta)
+            final.append(concept)
+        elif isinstance(
+            arg, (FilterItem, WindowItem, AggregateWrapper, ListWrapper, MapWrapper)
+        ):
+            id_hash = string_to_hash(str(arg))
+            concept = arbitrary_to_concept(
+                arg,
+                name=f"{VIRTUAL_CONCEPT_PREFIX}_{id_hash}",
+                namespace=environment.namespace,
+            )
+            if concept.metadata and meta:
+                concept.metadata.line_number = meta.line
+            environment.add_concept(concept, meta=meta)
+            final.append(concept)
+
+        else:
+            final.append(arg)
+    return final
 
 
 def get_purpose_and_keys(