pytrilogy 0.0.3.113__py3-none-any.whl → 0.0.3.116__py3-none-any.whl

trilogy/core/models/build.py

@@ -48,6 +48,8 @@ from trilogy.core.models.author import (
     Concept,
     ConceptRef,
     Conditional,
+    DeriveClause,
+    DeriveItem,
     FilterItem,
     FuncArgs,
     Function,
@@ -134,8 +136,9 @@ def concept_is_relevant(
     if concept.purpose in (Purpose.METRIC,):
         if all([c in others for c in concept.grain.components]):
             return False
+    if concept.derivation in (Derivation.UNNEST,):
+        return True
     if concept.derivation in (Derivation.BASIC,):
-
         return any(concept_is_relevant(c, others) for c in concept.concept_arguments)
     if concept.granularity == Granularity.SINGLE_ROW:
         return False
@@ -246,7 +249,7 @@ class BuildParamaterizedConceptReference(DataTyped):
     concept: BuildConcept
 
     def __str__(self):
-        return f":{self.concept.address}"
+        return f":{self.concept.address.replace('.', '_')}"
 
     @property
     def safe_address(self) -> str:
@@ -1267,6 +1270,22 @@ class BuildAlignClause:
     items: List[BuildAlignItem]
 
 
+@dataclass
+class BuildDeriveClause:
+    items: List[BuildDeriveItem]
+
+
+@dataclass
+class BuildDeriveItem:
+    expr: BuildExpr
+    name: str
+    namespace: str = field(default=DEFAULT_NAMESPACE)
+
+    @property
+    def address(self) -> str:
+        return f"{self.namespace}.{self.name}"
+
+
 @dataclass
 class BuildSelectLineage:
     selection: List[BuildConcept]
@@ -1298,12 +1317,16 @@ class BuildMultiSelectLineage(BuildConceptArgs):
     limit: Optional[int] = None
     where_clause: Union["BuildWhereClause", None] = field(default=None)
     having_clause: Union["BuildHavingClause", None] = field(default=None)
+    derive: BuildDeriveClause | None = None
 
     @property
     def derived_concepts(self) -> set[str]:
         output = set()
         for item in self.align.items:
             output.add(item.aligned_concept)
+        if self.derive:
+            for ditem in self.derive.items:
+                output.add(ditem.address)
         return output
 
     @property
@@ -1311,10 +1334,12 @@ class BuildMultiSelectLineage(BuildConceptArgs):
         return self.build_output_components
 
     @property
-    def derived_concept(self) -> set[str]:
-        output = set()
-        for item in self.align.items:
-            output.add(item.aligned_concept)
+    def calculated_derivations(self) -> set[str]:
+        output: set[str] = set()
+        if not self.derive:
+            return output
+        for item in self.derive.items:
+            output.add(item.address)
         return output
 
     @property
@@ -1334,6 +1359,7 @@ class BuildMultiSelectLineage(BuildConceptArgs):
                 for c in x.concepts:
                     if c.address in cte.output_lcl:
                         return c
+
         raise SyntaxError(
             f"Could not find upstream map for multiselect {str(concept)} on cte ({cte})"
         )
@@ -1668,7 +1694,6 @@ class Factory:
                     valid_inputs=base.valid_inputs,
                     arg_count=base.arg_count,
                 )
-
         new = BuildFunction(
             operator=base.operator,
             arguments=[self.handle_constant(self.build(c)) for c in raw_args],
@@ -1724,6 +1749,14 @@ class Factory:
         return self._build_concept(base)
 
     def _build_concept(self, base: Concept) -> BuildConcept:
+        try:
+            return self.__build_concept(base)
+        except RecursionError as e:
+            raise RecursionError(
+                f"Recursion error building concept {base.address}. This is likely due to a circular reference."
+            ) from e
+
+    def __build_concept(self, base: Concept) -> BuildConcept:
         # TODO: if we are using parameters, wrap it in a new model and use that in rendering
         if base.address in self.local_concepts:
             return self.local_concepts[base.address]
@@ -1953,6 +1986,28 @@ class Factory:
     def _build_align_clause(self, base: AlignClause) -> BuildAlignClause:
         return BuildAlignClause(items=[self._build_align_item(x) for x in base.items])
 
+    @build.register
+    def _(self, base: DeriveItem) -> BuildDeriveItem:
+        return self._build_derive_item(base)
+
+    def _build_derive_item(self, base: DeriveItem) -> BuildDeriveItem:
+        expr: Concept | FuncArgs = base.expr
+        validation = requires_concept_nesting(expr)
+        if validation:
+            expr, _ = self.instantiate_concept(validation)
+        return BuildDeriveItem(
+            expr=self.build(expr),
+            name=base.name,
+            namespace=base.namespace,
+        )
+
+    @build.register
+    def _(self, base: DeriveClause) -> BuildDeriveClause:
+        return self._build_derive_clause(base)
+
+    def _build_derive_clause(self, base: DeriveClause) -> BuildDeriveClause:
+        return BuildDeriveClause(items=[self.build(x) for x in base.items])
+
     @build.register
     def _(self, base: RowsetItem) -> BuildRowsetItem:
         return self._build_rowset_item(base)
@@ -2002,6 +2057,13 @@ class Factory:
     def _build_tuple_wrapper(self, base: TupleWrapper) -> TupleWrapper:
         return TupleWrapper(val=[self.build(x) for x in base.val], type=base.type)
 
+    @build.register
+    def _(self, base: ListWrapper) -> ListWrapper:
+        return self._build_list_wrapper(base)
+
+    def _build_list_wrapper(self, base: ListWrapper) -> ListWrapper:
+        return ListWrapper([self.build(x) for x in base], type=base.type)
+
     @build.register
     def _(self, base: FilterItem) -> BuildFilterItem:
         return self._build_filter_item(base)
@@ -2147,6 +2209,7 @@ class Factory:
             selects=base.selects,
             grain=final_grain,
             align=factory.build(base.align),
+            derive=factory.build(base.derive) if base.derive else None,
             # self.align.with_select_context(
             #     local_build_cache, self.grain, environment
             # ),

trilogy/core/models/environment.py

@@ -413,7 +413,8 @@ class Environment(BaseModel):
             self.imports[alias].append(imp_stm)
         # we can't exit early
         # as there may be new concepts
-        for k, concept in source.concepts.items():
+        iteration: list[tuple[str, Concept]] = list(source.concepts.items())
+        for k, concept in iteration:
             # skip internal namespace
             if INTERNAL_NAMESPACE in concept.address:
                 continue

trilogy/core/optimization.py

@@ -228,7 +228,8 @@ def optimize_ctes(
         REGISTERED_RULES.append(PredicatePushdown())
     if CONFIG.optimizations.predicate_pushdown:
         REGISTERED_RULES.append(PredicatePushdownRemove())
-    REGISTERED_RULES.append(HideUnusedConcepts())
+    if CONFIG.optimizations.hide_unused_concepts:
+        REGISTERED_RULES.append(HideUnusedConcepts())
     for rule in REGISTERED_RULES:
         loops = 0
         complete = False
@@ -242,7 +243,7 @@ def optimize_ctes(
                 actions_taken = actions_taken or opt
             complete = not actions_taken
             loops += 1
-            input = reorder_ctes(filter_irrelevant_ctes(input, root_cte))
+        input = reorder_ctes(filter_irrelevant_ctes(input, root_cte))
         logger.info(
             f"[Optimization] Finished checking for {type(rule).__name__} after {loops} loop(s)"
         )

trilogy/core/optimizations/hide_unused_concept.py

@@ -39,11 +39,7 @@ class HideUnusedConcepts(OptimizationRule):
         self.log(
             f"Hiding unused concepts {[x.address for x in add_to_hidden]} from {cte.name} (used: {used}, all: {[x.address for x in cte.output_columns]})"
         )
-        candidates = [
-            x.address
-            for x in cte.output_columns
-            if x.address not in used and x.address not in cte.hidden_concepts
-        ]
+        candidates = [x.address for x in cte.output_columns if x.address not in used]
         if len(candidates) == len(set([x.address for x in cte.output_columns])):
             # pop one out
             candidates.pop()

trilogy/core/processing/concept_strategies_v3.py

@@ -306,7 +306,12 @@ def evaluate_loop_conditions(
 
 
 def check_for_early_exit(
-    complete, partial, missing, context: LoopContext, priority_concept: BuildConcept
+    complete: ValidationResult,
+    found: set[str],
+    partial: set[str],
+    missing: set[str],
+    context: LoopContext,
+    priority_concept: BuildConcept,
 ) -> bool:
     if complete == ValidationResult.INCOMPLETE_CONDITION:
         cond_dict = {str(node): node.preexisting_conditions for node in context.stack}
@@ -331,8 +336,18 @@ def check_for_early_exit(
                 f"{depth_to_prefix(context.depth)}{LOGGER_PREFIX} Breaking as we have attempted all nodes"
             )
             return True
+        elif all(
+            [
+                x.address in found and x.address not in partial
+                for x in context.mandatory_list
+            ]
+        ):
+            logger.info(
+                f"{depth_to_prefix(context.depth)}{LOGGER_PREFIX} Breaking as we have found all mandatory nodes without partials"
+            )
+            return True
         logger.info(
-            f"{depth_to_prefix(context.depth)}{LOGGER_PREFIX} Found complete stack with partials {partial}, continuing search, attempted {context.attempted} all {len(context.mandatory_list)}"
+            f"{depth_to_prefix(context.depth)}{LOGGER_PREFIX} Found complete stack with partials {partial}, continuing search, attempted {context.attempted} of total {len(context.mandatory_list)}."
         )
     else:
         logger.info(
@@ -436,6 +451,7 @@ def generate_loop_completion(context: LoopContext, virtual: set[str]) -> Strateg
             context.original_mandatory,
             context.environment,
             non_virtual_difference_values,
+            depth=context.depth,
         )
 
     return group_if_required_v2(
@@ -443,6 +459,7 @@ def generate_loop_completion(context: LoopContext, virtual: set[str]) -> Strateg
         context.original_mandatory,
         context.environment,
         non_virtual_difference_values,
+        depth=context.depth,
     )
 
 
@@ -466,6 +483,7 @@ def _search_concepts(
         conditions=conditions,
     )
 
+    # if we get a can
     if candidate:
         return candidate
     context = initialize_loop_context(
@@ -477,13 +495,16 @@ def _search_concepts(
         accept_partial=accept_partial,
         conditions=conditions,
     )
-
+    partial: set[str] = set()
+    virtual: set[str] = set()
+    complete = ValidationResult.INCOMPLETE
     while context.incomplete:
 
         priority_concept = get_priority_concept(
             context.mandatory_list,
             context.attempted,
             found_concepts=context.found,
+            partial_concepts=partial,
             depth=depth,
         )
 
@@ -538,7 +559,7 @@ def _search_concepts(
         # assign
         context.found = found_c
         early_exit = check_for_early_exit(
-            complete, partial, missing_c, context, priority_concept
+            complete, found_c, partial, missing_c, context, priority_concept
         )
         if early_exit:
             break
@@ -608,4 +629,4 @@ def source_query_concepts(
     logger.info(
         f"{depth_to_prefix(0)}{LOGGER_PREFIX} final concepts are {[x.address for x in final]}"
     )
-    return group_if_required_v2(root, output_concepts, environment)
+    return group_if_required_v2(root, output_concepts, environment, depth=0)

trilogy/core/processing/discovery_node_factory.py

@@ -187,7 +187,7 @@ def _generate_aggregate_node(ctx: NodeGenerationContext) -> StrategyNode | None:
 
     logger.info(
         f"{depth_to_prefix(ctx.depth)}{LOGGER_PREFIX} "
-        f"for {ctx.concept.address}, generating aggregate node with {agg_optional}"
+        f"for {ctx.concept.address}, generating aggregate node with optional {agg_optional}"
     )
 
     return gen_group_node(
@@ -441,7 +441,7 @@ def generate_node(
     depth: int,
     source_concepts: SearchConceptsType,
     history: History,
-    accept_partial: bool = False,
+    accept_partial: bool,
     conditions: BuildWhereClause | None = None,
 ) -> StrategyNode | None:
 

trilogy/core/processing/discovery_utility.py

@@ -184,10 +184,14 @@ def group_if_required_v2(
     final: List[BuildConcept],
     environment: BuildEnvironment,
     where_injected: set[str] | None = None,
+    depth: int = 0,
 ):
     where_injected = where_injected or set()
     required = check_if_group_required(
-        downstream_concepts=final, parents=[root.resolve()], environment=environment
+        downstream_concepts=final,
+        parents=[root.resolve()],
+        environment=environment,
+        depth=depth,
     )
     targets = [
         x
@@ -258,6 +262,7 @@ def get_priority_concept(
     all_concepts: List[BuildConcept],
     attempted_addresses: set[str],
     found_concepts: set[str],
+    partial_concepts: set[str],
     depth: int,
 ) -> BuildConcept:
     # optimized search for missing concepts
@@ -265,13 +270,15 @@ def get_priority_concept(
         [
             c
             for c in all_concepts
-            if c.address not in attempted_addresses and c.address not in found_concepts
+            if c.address not in attempted_addresses
+            and (c.address not in found_concepts or c.address in partial_concepts)
         ],
         key=lambda x: x.address,
     )
     # sometimes we need to scan intermediate concepts to get merge keys or filter keys,
     # so do an exhaustive search
-    # pass_two = [c for c in all_concepts+filter_only if c.address not in attempted_addresses]
+    # pass_two = [c for c in all_concepts if c.address not in attempted_addresses]
+
     for remaining_concept in (pass_one,):
         priority = (
             # then multiselects to remove them from scope
@@ -333,5 +340,5 @@ def get_priority_concept(
         if final:
             return final[0]
     raise ValueError(
-        f"Cannot resolve query. No remaining priority concepts, have attempted {attempted_addresses}"
+        f"Cannot resolve query. No remaining priority concepts, have attempted {attempted_addresses} out of {all_concepts} with found {found_concepts}"
     )

trilogy/core/processing/node_generators/basic_node.py

@@ -7,7 +7,7 @@ from trilogy.core.models.build_environment import BuildEnvironment
 from trilogy.core.processing.node_generators.common import (
     resolve_function_parent_concepts,
 )
-from trilogy.core.processing.nodes import History, StrategyNode
+from trilogy.core.processing.nodes import ConstantNode, History, StrategyNode
 from trilogy.utility import unique
 
 LOGGER_PREFIX = "[GEN_BASIC_NODE]"
@@ -51,11 +51,14 @@ def gen_basic_node(
     )
     synonyms: list[BuildConcept] = []
     ignored_optional: set[str] = set()
-    assert isinstance(concept.lineage, BuildFunction)
+
     # when we are getting an attribute, if there is anything else
     # that is an attribute of the same struct in local optional
     # select that value for discovery as well
-    if concept.lineage.operator == FunctionType.ATTR_ACCESS:
+    if (
+        isinstance(concept.lineage, BuildFunction)
+        and concept.lineage.operator == FunctionType.ATTR_ACCESS
+    ):
         logger.info(
             f"{depth_prefix}{LOGGER_PREFIX} checking for synonyms for attribute access"
         )
@@ -106,20 +109,28 @@ def gen_basic_node(
     logger.info(
         f"{depth_prefix}{LOGGER_PREFIX} Fetching parents {[x.address for x in all_parents]}"
     )
-    parent_node: StrategyNode | None = source_concepts(
-        mandatory_list=all_parents,
-        environment=environment,
-        g=g,
-        depth=depth + 1,
-        history=history,
-        conditions=conditions,
-    )
+    if all_parents:
+        parent_node: StrategyNode | None = source_concepts(
+            mandatory_list=all_parents,
+            environment=environment,
+            g=g,
+            depth=depth + 1,
+            history=history,
+            conditions=conditions,
+        )
 
-    if not parent_node:
-        logger.info(
-            f"{depth_prefix}{LOGGER_PREFIX} No basic node could be generated for {concept}"
+        if not parent_node:
+            logger.info(
+                f"{depth_prefix}{LOGGER_PREFIX} No basic node could be generated for {concept}"
+            )
+            return None
+    else:
+        return ConstantNode(
+            input_concepts=[],
+            output_concepts=[concept],
+            environment=environment,
+            depth=depth,
         )
-        return None
     if parent_node.source_type != SourceType.CONSTANT:
         parent_node.source_type = SourceType.BASIC
     parent_node.add_output_concept(concept)

trilogy/core/processing/node_generators/common.py

@@ -4,6 +4,7 @@ from typing import Callable, List, Tuple
 from trilogy.core.enums import Derivation, Purpose
 from trilogy.core.models.build import (
     BuildAggregateWrapper,
+    BuildComparison,
     BuildConcept,
     BuildFilterItem,
     BuildFunction,
@@ -26,7 +27,9 @@ FUNCTION_TYPES = (BuildFunction,)
 def resolve_function_parent_concepts(
     concept: BuildConcept, environment: BuildEnvironment
 ) -> List[BuildConcept]:
-    if not isinstance(concept.lineage, (*FUNCTION_TYPES, *AGGREGATE_TYPES)):
+    if not isinstance(
+        concept.lineage, (*FUNCTION_TYPES, *AGGREGATE_TYPES, BuildComparison)
+    ):
         raise ValueError(
             f"Concept {concept} lineage is not function or aggregate, is {type(concept.lineage)}"
         )

trilogy/core/processing/node_generators/filter_node.py

@@ -96,6 +96,8 @@ def build_parent_concepts(
                 continue
         elif global_filter_is_local_filter:
             same_filter_optional.append(x)
+            # also append it to the parent row concepts
+            parent_row_concepts.append(x)
 
     # sometimes, it's okay to include other local optional above the filter
     # in case it is, prep our list
@@ -204,11 +206,16 @@ def gen_filter_node(
             f"{padding(depth)}{LOGGER_PREFIX} filter node row parents {[x.address for x in parent_row_concepts]} could not be found"
         )
         return None
+    else:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} filter node has row parents {[x.address for x in parent_row_concepts]} from node with output [{[x.address for x in row_parent.output_concepts]}] partial {row_parent.partial_concepts}"
+        )
     if global_filter_is_local_filter:
         logger.info(
             f"{padding(depth)}{LOGGER_PREFIX} filter node conditions match global conditions adding row parent {row_parent.output_concepts} with condition {where.conditional}"
         )
         row_parent.add_parents(core_parent_nodes)
+        # all local optional will be in the parent already, so we can set outputs
         row_parent.set_output_concepts([concept] + local_optional)
         return row_parent
     if optimized_pushdown:

trilogy/core/processing/node_generators/multiselect_node.py

@@ -157,19 +157,19 @@ def gen_multiselect_node(
     possible_joins = concept_to_relevant_joins(additional_relevant)
     if not local_optional:
         logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} no enrichment required for rowset node; exiting early"
+            f"{padding(depth)}{LOGGER_PREFIX} no enrichment required for multiselect node; exiting early"
         )
         return node
     if not possible_joins:
         logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} no possible joins for rowset node; exiting early"
+            f"{padding(depth)}{LOGGER_PREFIX} no possible joins for multiselect node; exiting early"
         )
         return node
     if all(
         [x.address in [y.address for y in node.output_concepts] for x in local_optional]
     ):
         logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} all enriched concepts returned from base rowset node; exiting early"
+            f"{padding(depth)}{LOGGER_PREFIX} all enriched concepts returned from base multiselect node; exiting early"
         )
         return node
     logger.info(

trilogy/core/processing/node_generators/unnest_node.py

@@ -9,6 +9,7 @@ from trilogy.core.models.build import (
 from trilogy.core.models.build_environment import BuildEnvironment
 from trilogy.core.processing.nodes import (
     History,
+    MergeNode,
     StrategyNode,
     UnnestNode,
     WhereSafetyNode,
@@ -18,6 +19,32 @@ from trilogy.core.processing.utility import padding
 LOGGER_PREFIX = "[GEN_UNNEST_NODE]"
 
 
+def get_pseudonym_parents(
+    concept: BuildConcept,
+    local_optional: List[BuildConcept],
+    source_concepts,
+    environment: BuildEnvironment,
+    g,
+    depth,
+    history,
+    conditions,
+) -> List[StrategyNode]:
+    for x in concept.pseudonyms:
+        attempt = source_concepts(
+            mandatory_list=[environment.alias_origin_lookup[x]] + local_optional,
+            environment=environment,
+            g=g,
+            depth=depth + 1,
+            history=history,
+            conditions=conditions,
+            accept_partial=True,
+        )
+        if not attempt:
+            continue
+        return [attempt]
+    return []
+
+
 def gen_unnest_node(
     concept: BuildConcept,
     local_optional: List[BuildConcept],
@@ -29,14 +56,34 @@ def gen_unnest_node(
     conditions: BuildWhereClause | None = None,
 ) -> StrategyNode | None:
     arguments = []
+    join_nodes: list[StrategyNode] = []
     depth_prefix = "\t" * depth
     if isinstance(concept.lineage, BuildFunction):
         arguments = concept.lineage.concept_arguments
+    search_optional = local_optional
+    if (not arguments) and (local_optional and concept.pseudonyms):
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} unnest node for {concept} has no parents; creating solo unnest node"
+        )
+        join_nodes += get_pseudonym_parents(
+            concept,
+            local_optional,
+            source_concepts,
+            environment,
+            g,
+            depth,
+            history,
+            conditions,
+        )
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} unnest node for {concept} got join nodes {join_nodes}"
+        )
+        search_optional = []
 
-    equivalent_optional = [x for x in local_optional if x.lineage == concept.lineage]
+    equivalent_optional = [x for x in search_optional if x.lineage == concept.lineage]
 
     non_equivalent_optional = [
-        x for x in local_optional if x not in equivalent_optional
+        x for x in search_optional if x not in equivalent_optional
     ]
     all_parents = arguments + non_equivalent_optional
     logger.info(
@@ -44,7 +91,8 @@ def gen_unnest_node(
     )
     local_conditions = False
     expected_outputs = [concept] + local_optional
-    if arguments or local_optional:
+    parent: StrategyNode | None = None
+    if arguments or search_optional:
         parent = source_concepts(
             mandatory_list=all_parents,
             environment=environment,
@@ -86,14 +134,37 @@ def gen_unnest_node(
     base = UnnestNode(
         unnest_concepts=[concept] + equivalent_optional,
         input_concepts=arguments + non_equivalent_optional,
-        output_concepts=[concept] + local_optional,
+        output_concepts=[concept] + search_optional,
         environment=environment,
         parents=([parent] if parent else []),
     )
+
+    conditional = conditions.conditional if conditions else None
+    if join_nodes:
+        logger.info(
+            f"{depth_prefix}{LOGGER_PREFIX} unnest node for {concept} needs to merge with join nodes {join_nodes}"
+        )
+        for x in join_nodes:
+            logger.info(
+                f"{depth_prefix}{LOGGER_PREFIX} join node {x} with partial {x.partial_concepts}"
+            )
+            pseudonyms = [
+                environment.alias_origin_lookup[p] for p in concept.pseudonyms
+            ]
+            x.add_partial_concepts(pseudonyms)
+        return MergeNode(
+            input_concepts=base.output_concepts
+            + [j for n in join_nodes for j in n.output_concepts],
+            output_concepts=[concept] + local_optional,
+            environment=environment,
+            parents=[base] + join_nodes,
+            conditions=conditional if local_conditions is True else None,
+            preexisting_conditions=(
+                conditional if conditional and local_conditions is False else None
+            ),
+        )
     # we need to sometimes nest an unnest node,
     # as unnest operations are not valid in all situations
-    # TODO: inline this node when we can detect it's safe
-    conditional = conditions.conditional if conditions else None
     new = WhereSafetyNode(
         input_concepts=base.output_concepts,
         output_concepts=base.output_concepts,

trilogy/core/statements/author.py

@@ -21,6 +21,7 @@ from trilogy.core.models.author import (
     Concept,
     ConceptRef,
     CustomType,
+    DeriveClause,
     Expr,
     FilterItem,
     Function,
@@ -197,7 +198,7 @@ class SelectStatement(HasUUID, SelectTypeMixin, BaseModel):
             for x in self.where_clause.concept_arguments:
                 if isinstance(x, UndefinedConcept):
                     validate = environment.concepts.get(x.address)
-                    if validate:
+                    if validate and self.where_clause:
                         self.where_clause = (
                             self.where_clause.with_reference_replacement(
                                 x.address, validate.reference
@@ -352,11 +353,13 @@ class MultiSelectStatement(HasUUID, SelectTypeMixin, BaseModel):
     local_concepts: Annotated[
         EnvironmentConceptDict, PlainValidator(validate_concepts)
     ] = Field(default_factory=EnvironmentConceptDict)
+    derive: DeriveClause | None = None
 
     def as_lineage(self, environment: Environment):
         return MultiSelectLineage(
             selects=[x.as_lineage(environment) for x in self.selects],
             align=self.align,
+            derive=self.derive,
             namespace=self.namespace,
             # derived_concepts = self.derived_concepts,
             limit=self.limit,