pytrilogy 0.0.2.58__py3-none-any.whl → 0.0.3.0__py3-none-any.whl

trilogy/core/processing/concept_strategies_v3.py

@@ -3,17 +3,19 @@ from enum import Enum
 from typing import List, Optional, Protocol, Union
 
 from trilogy.constants import logger
-from trilogy.core.enums import FunctionType, Granularity, PurposeLineage
+from trilogy.core.enums import Derivation, FunctionType, Granularity
 from trilogy.core.env_processor import generate_graph
 from trilogy.core.graph_models import ReferenceGraph
-from trilogy.core.models import (
-    Concept,
-    Environment,
-    Function,
-    RowsetItem,
+from trilogy.core.models.author import (
     UndefinedConcept,
-    WhereClause,
 )
+from trilogy.core.models.build import (
+    BuildConcept,
+    BuildFunction,
+    BuildRowsetItem,
+    BuildWhereClause,
+)
+from trilogy.core.models.build_environment import BuildEnvironment
 from trilogy.core.processing.node_generators import (
     gen_basic_node,
     gen_filter_node,
@@ -52,17 +54,17 @@ LOGGER_PREFIX = "[CONCEPT DETAIL]"
 class SearchConceptsType(Protocol):
     def __call__(
         self,
-        mandatory_list: List[Concept],
-        environment: Environment,
+        mandatory_list: List[BuildConcept],
+        history: History,
+        environment: BuildEnvironment,
         depth: int,
         g: ReferenceGraph,
         accept_partial: bool = False,
-        history: Optional[History] = None,
-        conditions: Optional[WhereClause] = None,
+        conditions: Optional[BuildWhereClause] = None,
     ) -> Union[StrategyNode, None]: ...
 
 
-def get_upstream_concepts(base: Concept, nested: bool = False) -> set[str]:
+def get_upstream_concepts(base: BuildConcept, nested: bool = False) -> set[str]:
     upstream = set()
     if nested:
         upstream.add(base.address)
@@ -70,20 +72,42 @@ def get_upstream_concepts(base: Concept, nested: bool = False) -> set[str]:
         return upstream
     for x in base.lineage.concept_arguments:
         # if it's derived from any value in a rowset, ALL rowset items are upstream
-        if x.derivation == PurposeLineage.ROWSET:
-            assert isinstance(x.lineage, RowsetItem)
-            for y in x.lineage.rowset.derived_concepts:
-                upstream = upstream.union(get_upstream_concepts(y, nested=True))
+        if x.derivation == Derivation.ROWSET:
+            assert isinstance(x.lineage, BuildRowsetItem), type(x.lineage)
+            for y in x.lineage.rowset.select.output_components:
+                upstream.add(f"{x.lineage.rowset.name}.{y.address}")
+                # upstream = upstream.union(get_upstream_concepts(y, nested=True))
         upstream = upstream.union(get_upstream_concepts(x, nested=True))
     return upstream
 
 
+def restrict_node_outputs_targets(
+    node: StrategyNode, targets: list[BuildConcept], depth: int
+) -> list[BuildConcept]:
+    ex_resolve = node.resolve()
+    extra = [
+        x
+        for x in ex_resolve.output_concepts
+        if x.address not in [y.address for y in targets]
+    ]
+
+    base = [x for x in ex_resolve.output_concepts if x.address not in extra]
+    logger.info(
+        f"{depth_to_prefix(depth)}{LOGGER_PREFIX} reducing final outputs, was {[c.address for c in ex_resolve.output_concepts]} with extra {[c.address for c in extra]}, remaining {base}"
+    )
+    for x in targets:
+        if x.address not in base:
+            base.append(x)
+    node.set_output_concepts(base)
+    return extra
+
+
 def get_priority_concept(
-    all_concepts: List[Concept],
+    all_concepts: List[BuildConcept],
     attempted_addresses: set[str],
     found_concepts: set[str],
     depth: int,
-) -> Concept:
+) -> BuildConcept:
     # optimized search for missing concepts
     pass_one = [
         c
@@ -99,34 +123,34 @@ def get_priority_concept(
             [
                 c
                 for c in remaining_concept
-                if c.derivation == PurposeLineage.CONSTANT
+                if c.derivation == Derivation.CONSTANT
                 and c.granularity == Granularity.SINGLE_ROW
             ]
             +
             # then multiselects to remove them from scope
-            [c for c in remaining_concept if c.derivation == PurposeLineage.MULTISELECT]
+            [c for c in remaining_concept if c.derivation == Derivation.MULTISELECT]
             +
             # then rowsets to remove them from scope, as they cannot get partials
-            [c for c in remaining_concept if c.derivation == PurposeLineage.ROWSET]
+            [c for c in remaining_concept if c.derivation == Derivation.ROWSET]
             +
             # then rowsets to remove them from scope, as they cannot get partials
-            [c for c in remaining_concept if c.derivation == PurposeLineage.UNION]
+            [c for c in remaining_concept if c.derivation == Derivation.UNION]
             # we should be home-free here
             +
             # then aggregates to remove them from scope, as they cannot get partials
-            [c for c in remaining_concept if c.derivation == PurposeLineage.AGGREGATE]
+            [c for c in remaining_concept if c.derivation == Derivation.AGGREGATE]
             # then windows to remove them from scope, as they cannot get partials
-            + [c for c in remaining_concept if c.derivation == PurposeLineage.WINDOW]
+            + [c for c in remaining_concept if c.derivation == Derivation.WINDOW]
             # then filters to remove them from scope, also cannot get partials
-            + [c for c in remaining_concept if c.derivation == PurposeLineage.FILTER]
+            + [c for c in remaining_concept if c.derivation == Derivation.FILTER]
             # unnests are weird?
-            + [c for c in remaining_concept if c.derivation == PurposeLineage.UNNEST]
-            + [c for c in remaining_concept if c.derivation == PurposeLineage.BASIC]
+            + [c for c in remaining_concept if c.derivation == Derivation.UNNEST]
+            + [c for c in remaining_concept if c.derivation == Derivation.BASIC]
             # finally our plain selects
             + [
-                c for c in remaining_concept if c.derivation == PurposeLineage.ROOT
+                c for c in remaining_concept if c.derivation == Derivation.ROOT
             ]  # and any non-single row constants
-            + [c for c in remaining_concept if c.derivation == PurposeLineage.CONSTANT]
+            + [c for c in remaining_concept if c.derivation == Derivation.CONSTANT]
         )
 
         priority += [
@@ -139,7 +163,15 @@ def get_priority_concept(
         # get the derived copy first
         # as this will usually resolve cleaner
         for x in priority:
-            if any([x.address in get_upstream_concepts(c) for c in priority]):
+            if any(
+                [
+                    x.address
+                    in get_upstream_concepts(
+                        c,
+                    )
+                    for c in priority
+                ]
+            ):
                 logger.info(
                     f"{depth_to_prefix(depth)}{LOGGER_PREFIX} delaying fetch of {x.address} as parent of another concept"
                 )
@@ -157,11 +189,12 @@ def get_priority_concept(
 
 
 def generate_candidates_restrictive(
-    priority_concept: Concept,
-    candidates: list[Concept],
+    priority_concept: BuildConcept,
+    candidates: list[BuildConcept],
     exhausted: set[str],
-    conditions: WhereClause | None = None,
-) -> List[List[Concept]]:
+    depth: int,
+    conditions: BuildWhereClause | None = None,
+) -> List[List[BuildConcept]]:
     # if it's single row, joins are irrelevant. Fetch without keys.
     if priority_concept.granularity == Granularity.SINGLE_ROW:
         return [[]]
@@ -175,26 +208,28 @@ def generate_candidates_restrictive(
         and priority_concept.address not in x.pseudonyms
     ]
     if conditions and priority_concept.derivation in (
-        PurposeLineage.ROOT,
-        PurposeLineage.CONSTANT,
+        Derivation.ROOT,
+        Derivation.CONSTANT,
     ):
-        return [unique(conditions.row_arguments + local_candidates, "address")]
+        logger.info(
+            f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Injecting additional conditional row arguments as all remaining concepts are roots or constant"
+        )
+        return [unique(list(conditions.row_arguments) + local_candidates, "address")]
     return [local_candidates]
 
 
 def generate_node(
-    concept: Concept,
-    local_optional: List[Concept],
-    environment: Environment,
+    concept: BuildConcept,
+    local_optional: List[BuildConcept],
+    environment: BuildEnvironment,
     g: ReferenceGraph,
     depth: int,
     source_concepts: SearchConceptsType,
+    history: History,
     accept_partial: bool = False,
-    history: History | None = None,
-    conditions: WhereClause | None = None,
+    conditions: BuildWhereClause | None = None,
 ) -> StrategyNode | None:
     # first check in case there is a materialized_concept
-    history = history or History()
     candidate = history.gen_select_node(
         concept,
         local_optional,
@@ -211,50 +246,50 @@ def generate_node(
     if candidate:
         return candidate
 
-    if concept.derivation == PurposeLineage.WINDOW:
+    if concept.derivation == Derivation.WINDOW:
         logger.info(
             f"{depth_to_prefix(depth)}{LOGGER_PREFIX} for {concept.address}, generating window node with optional {[x.address for x in local_optional]}"
         )
         return gen_window_node(
             concept,
             local_optional,
-            environment,
-            g,
-            depth + 1,
-            source_concepts,
-            history,
+            history=history,
+            environment=environment,
+            g=g,
+            depth=depth + 1,
+            source_concepts=source_concepts,
             conditions=conditions,
         )
 
-    elif concept.derivation == PurposeLineage.FILTER:
+    elif concept.derivation == Derivation.FILTER:
         logger.info(
             f"{depth_to_prefix(depth)}{LOGGER_PREFIX} for {concept.address}, generating filter node with optional {[x.address for x in local_optional]}"
         )
         return gen_filter_node(
             concept,
             local_optional,
-            environment,
-            g,
-            depth + 1,
-            source_concepts=source_concepts,
             history=history,
+            environment=environment,
+            g=g,
+            depth=depth + 1,
+            source_concepts=source_concepts,
             conditions=conditions,
         )
-    elif concept.derivation == PurposeLineage.UNNEST:
+    elif concept.derivation == Derivation.UNNEST:
         logger.info(
             f"{depth_to_prefix(depth)}{LOGGER_PREFIX} for {concept.address}, generating unnest node with optional {[x.address for x in local_optional]} and condition {conditions}"
         )
         return gen_unnest_node(
             concept,
             local_optional,
-            environment,
-            g,
-            depth + 1,
-            source_concepts,
-            history,
+            history=history,
+            environment=environment,
+            g=g,
+            depth=depth + 1,
+            source_concepts=source_concepts,
             conditions=conditions,
         )
-    elif concept.derivation == PurposeLineage.UNION:
+    elif concept.derivation == Derivation.UNION:
         logger.info(
             f"{depth_to_prefix(depth)}{LOGGER_PREFIX} for {concept.address}, generating union node with optional {[x.address for x in local_optional]} and condition {conditions}"
         )
@@ -268,7 +303,7 @@ def generate_node(
             history,
             conditions=conditions,
         )
-    elif concept.derivation == PurposeLineage.AGGREGATE:
+    elif concept.derivation == Derivation.AGGREGATE:
         # don't push constants up before aggregation
         # if not required
         # to avoid constants multiplication changing default aggregation results
@@ -284,14 +319,14 @@ def generate_node(
         return gen_group_node(
             concept,
             agg_optional,
-            environment,
-            g,
-            depth + 1,
-            source_concepts,
-            history,
+            history=history,
+            environment=environment,
+            g=g,
+            depth=depth + 1,
+            source_concepts=source_concepts,
             conditions=conditions,
         )
-    elif concept.derivation == PurposeLineage.ROWSET:
+    elif concept.derivation == Derivation.ROWSET:
         logger.info(
             f"{depth_to_prefix(depth)}{LOGGER_PREFIX} for {concept.address}, generating rowset node with optional {[x.address for x in local_optional]}"
         )
@@ -305,7 +340,7 @@ def generate_node(
             history,
             conditions=conditions,
         )
-    elif concept.derivation == PurposeLineage.MULTISELECT:
+    elif concept.derivation == Derivation.MULTISELECT:
         logger.info(
             f"{depth_to_prefix(depth)}{LOGGER_PREFIX} for {concept.address}, generating multiselect node with optional {[x.address for x in local_optional]}"
         )
@@ -319,16 +354,14 @@ def generate_node(
             history,
             conditions=conditions,
         )
-    elif concept.derivation == PurposeLineage.CONSTANT:
+    elif concept.derivation == Derivation.CONSTANT:
         constant_targets = [concept] + local_optional
         logger.info(
             f"{depth_to_prefix(depth)}{LOGGER_PREFIX} for {concept.address}, generating constant node"
         )
-        if any([x.derivation != PurposeLineage.CONSTANT for x in local_optional]):
+        if any([x.derivation != Derivation.CONSTANT for x in local_optional]):
             non_root = [
-                x.address
-                for x in local_optional
-                if x.derivation != PurposeLineage.CONSTANT
+                x.address for x in local_optional if x.derivation != Derivation.CONSTANT
             ]
             logger.info(
                 f"{depth_to_prefix(depth)}{LOGGER_PREFIX} including filter concepts, there is non root filter requirements {non_root}. Recursing with all of these as mandatory"
@@ -362,10 +395,10 @@ def generate_node(
             parents=[],
             depth=depth + 1,
         )
-    elif concept.derivation == PurposeLineage.BASIC:
+    elif concept.derivation == Derivation.BASIC:
         # this is special case handling for group bys
         if (
-            isinstance(concept.lineage, Function)
+            isinstance(concept.lineage, BuildFunction)
             and concept.lineage.operator == FunctionType.GROUP
         ):
             logger.info(
@@ -387,15 +420,15 @@ def generate_node(
         return gen_basic_node(
             concept,
             local_optional,
-            environment,
-            g,
-            depth + 1,
-            source_concepts,
-            history,
+            history=history,
+            environment=environment,
+            g=g,
+            depth=depth + 1,
+            source_concepts=source_concepts,
             conditions=conditions,
         )
 
-    elif concept.derivation == PurposeLineage.ROOT:
+    elif concept.derivation == Derivation.ROOT:
         logger.info(
             f"{depth_to_prefix(depth)}{LOGGER_PREFIX} for {concept.address}, generating select node with optional including condition inputs {[x.address for x in local_optional]}"
         )
@@ -405,14 +438,14 @@ def generate_node(
 
         if any(
             [
-                x.derivation not in (PurposeLineage.ROOT, PurposeLineage.CONSTANT)
+                x.derivation not in (Derivation.ROOT, Derivation.CONSTANT)
                 for x in local_optional
             ]
         ):
             non_root = [
                 x.address
                 for x in local_optional
-                if x.derivation not in (PurposeLineage.ROOT, PurposeLineage.CONSTANT)
+                if x.derivation not in (Derivation.ROOT, Derivation.CONSTANT)
             ]
             logger.info(
                 f"{depth_to_prefix(depth)}{LOGGER_PREFIX} including filter concepts, there is non root filter requirements {non_root}. Recursing with all of these as mandatory"
@@ -466,29 +499,12 @@ def generate_node(
                 )
 
                 if expanded:
-                    ex_resolve = expanded.resolve()
-                    extra = [
-                        x
-                        for x in ex_resolve.output_concepts
-                        if x.address not in [y.address for y in root_targets]
-                    ]
-
+                    extra = restrict_node_outputs_targets(expanded, root_targets, depth)
                     pseudonyms = [
                         x
                         for x in extra
                         if any(x.address in y.pseudonyms for y in root_targets)
                     ]
-                    logger.info(
-                        f"{depth_to_prefix(depth)}{LOGGER_PREFIX} reducing final outputs, was {[c.address for c in ex_resolve.output_concepts]} with extra {[c.address for c in extra]}"
-                    )
-                    base = [
-                        x for x in ex_resolve.output_concepts if x.address not in extra
-                    ]
-                    for x in root_targets:
-                        if x.address not in base:
-                            base.append(x)
-                    expanded.set_output_concepts(base)
-                    # but hide them
                     if pseudonyms:
                         expanded.add_output_concepts(pseudonyms)
                         logger.info(
@@ -505,21 +521,21 @@ def generate_node(
             )
             return None
     else:
-        raise ValueError(f"Unknown derivation {concept.derivation}")
+        raise ValueError(f"Unknown derivation {concept.derivation} on {concept}")
     return None
 
 
 def validate_concept(
-    concept: Concept,
+    concept: BuildConcept,
     node: StrategyNode,
     found_addresses: set[str],
     non_partial_addresses: set[str],
     partial_addresses: set[str],
     virtual_addresses: set[str],
-    found_map: dict[str, set[Concept]],
+    found_map: dict[str, set[BuildConcept]],
     accept_partial: bool,
     seen: set[str],
-    environment: Environment,
+    environment: BuildEnvironment,
 ):
     found_map[str(node)].add(concept)
     seen.add(concept.address)
@@ -546,7 +562,6 @@ def validate_concept(
             return
         if v.address == concept.address:
             return
-
         validate_concept(
             v,
             node,
@@ -562,14 +577,14 @@ def validate_concept(
 
 
 def validate_stack(
-    environment: Environment,
+    environment: BuildEnvironment,
     stack: List[StrategyNode],
-    concepts: List[Concept],
-    mandatory_with_filter: List[Concept],
-    conditions: WhereClause | None = None,
+    concepts: List[BuildConcept],
+    mandatory_with_filter: List[BuildConcept],
+    conditions: BuildWhereClause | None = None,
     accept_partial: bool = False,
 ) -> tuple[ValidationResult, set[str], set[str], set[str], set[str]]:
-    found_map: dict[str, set[Concept]] = defaultdict(set)
+    found_map: dict[str, set[BuildConcept]] = defaultdict(set)
     found_addresses: set[str] = set()
     non_partial_addresses: set[str] = set()
     partial_addresses: set[str] = set()
@@ -648,10 +663,10 @@ def depth_to_prefix(depth: int) -> str:
 
 def append_existence_check(
     node: StrategyNode,
-    environment: Environment,
+    environment: BuildEnvironment,
     graph: ReferenceGraph,
-    where: WhereClause,
-    history: History | None = None,
+    where: BuildWhereClause,
+    history: History,
 ):
     # we if we have a where clause doing an existence check
     # treat that as separate subquery
@@ -668,7 +683,10 @@ def append_existence_check(
                 f"{LOGGER_PREFIX} fetching existence clause inputs {[str(c) for c in subselect]}"
             )
             parent = source_query_concepts(
-                [*subselect], environment=environment, g=graph, history=history
+                [*subselect],
+                history=history,
+                environment=environment,
+                g=graph,
             )
             assert parent, "Could not resolve existence clause"
             node.add_parents([parent])
@@ -679,15 +697,14 @@ def append_existence_check(
 
 
 def search_concepts(
-    mandatory_list: List[Concept],
-    environment: Environment,
+    mandatory_list: List[BuildConcept],
+    history: History,
+    environment: BuildEnvironment,
     depth: int,
     g: ReferenceGraph,
     accept_partial: bool = False,
-    history: History | None = None,
-    conditions: WhereClause | None = None,
+    conditions: BuildWhereClause | None = None,
 ) -> StrategyNode | None:
-    history = history or History()
     hist = history.get_history(
         search=mandatory_list, accept_partial=accept_partial, conditions=conditions
     )
@@ -718,13 +735,13 @@ def search_concepts(
 
 
 def _search_concepts(
-    mandatory_list: List[Concept],
-    environment: Environment,
+    mandatory_list: List[BuildConcept],
+    environment: BuildEnvironment,
     depth: int,
     g: ReferenceGraph,
     history: History,
     accept_partial: bool = False,
-    conditions: WhereClause | None = None,
+    conditions: BuildWhereClause | None = None,
 ) -> StrategyNode | None:
     # these are the concepts we need in the output projection
     mandatory_list = unique(mandatory_list, "address")
@@ -738,22 +755,32 @@ def _search_concepts(
     # if we have a filter, we may need to get more values to support that.
     if conditions:
         completion_mandatory = unique(
-            mandatory_list + conditions.row_arguments, "address"
+            mandatory_list + list(conditions.row_arguments), "address"
         )
         # if anything we need to get is in the filter set and it's a computed value
         # we need to get _everything_ in this loop
+        logger.info(f"{[x.address for x in conditions.row_arguments]}")
         if any(
             [
-                x.derivation not in (PurposeLineage.ROOT, PurposeLineage.CONSTANT)
+                x.derivation not in (Derivation.ROOT, Derivation.CONSTANT)
                 and x.address in conditions.row_arguments
                 for x in mandatory_list
             ]
         ):
+            logger.info(
+                f"{depth_to_prefix(depth)}{LOGGER_PREFIX} derived condition row input present in mandatory list, forcing condition evaluation at this level. "
+            )
             mandatory_list = completion_mandatory
             must_evaluate_condition_on_this_level_not_push_down = True
+        else:
+            logger.info(
+                f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Do not need to evaluate conditions yet."
+            )
     else:
-        completion_mandatory = mandatory_list
 
+        completion_mandatory = mandatory_list
+    if "date.month_seq" in completion_mandatory:
+        raise SyntaxError(completion_mandatory)
     attempted: set[str] = set()
 
     found: set[str] = set()
@@ -777,7 +804,7 @@ def _search_concepts(
                 [x.address in mandatory_list for x in conditions.row_arguments]
             ) and not any(
                 [
-                    x.derivation not in (PurposeLineage.ROOT, PurposeLineage.CONSTANT)
+                    x.derivation not in (Derivation.ROOT, Derivation.CONSTANT)
                     for x in mandatory_list
                     if x.address not in conditions.row_arguments
                 ]
@@ -797,7 +824,7 @@ def _search_concepts(
         if (
             conditions
             and priority_concept.derivation
-            not in (PurposeLineage.ROOT, PurposeLineage.CONSTANT)
+            not in (Derivation.ROOT, Derivation.CONSTANT)
             and priority_concept.address not in conditions.row_arguments
         ):
             logger.info(
@@ -806,14 +833,14 @@ def _search_concepts(
             local_conditions = conditions
 
         logger.info(
-            f"{depth_to_prefix(depth)}{LOGGER_PREFIX} priority concept is {str(priority_concept)} derivation {priority_concept.derivation} with conditions {local_conditions}"
+            f"{depth_to_prefix(depth)}{LOGGER_PREFIX} priority concept is {str(priority_concept)} derivation {priority_concept.derivation} granularity {priority_concept.granularity} with conditions {local_conditions}"
         )
 
         candidates = [
             c for c in mandatory_list if c.address != priority_concept.address
         ]
         candidate_lists = generate_candidates_restrictive(
-            priority_concept, candidates, skip, conditions=conditions
+            priority_concept, candidates, skip, depth=depth, conditions=conditions
         )
         for clist in candidate_lists:
             logger.info(
@@ -842,14 +869,14 @@ def _search_concepts(
                 # these concepts should not be attempted to be sourced again
                 # as fetching them requires operating on a subset of concepts
                 if priority_concept.derivation in [
-                    PurposeLineage.AGGREGATE,
-                    PurposeLineage.FILTER,
-                    PurposeLineage.WINDOW,
-                    PurposeLineage.UNNEST,
-                    PurposeLineage.ROWSET,
-                    PurposeLineage.BASIC,
-                    PurposeLineage.MULTISELECT,
-                    PurposeLineage.UNION,
+                    Derivation.AGGREGATE,
+                    Derivation.FILTER,
+                    Derivation.WINDOW,
+                    Derivation.UNNEST,
+                    Derivation.ROWSET,
+                    Derivation.BASIC,
+                    Derivation.MULTISELECT,
+                    Derivation.UNION,
                 ]:
                     skip.add(priority_concept.address)
                 break
@@ -880,7 +907,7 @@ def _search_concepts(
             break
         # if we have attempted on root node, we've tried them all.
         # inject in another search with filter concepts
-        if priority_concept.derivation == PurposeLineage.ROOT:
+        if priority_concept.derivation == Derivation.ROOT:
             logger.info(
                 f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Breaking as attempted root with no results"
             )
@@ -910,10 +937,14 @@ def _search_concepts(
             )
         if len(stack) == 1:
             output: StrategyNode = stack[0]
+            # _ = restrict_node_outputs_targets(output, mandatory_list, depth)
             logger.info(
                 f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Source stack has single node, returning that {type(output)}"
             )
         else:
+            logger.info(
+                f"{depth_to_prefix(depth)}{LOGGER_PREFIX} wrapping multiple parent nodes {[type(x) for x in stack]} in merge node"
+            )
             output = MergeNode(
                 input_concepts=non_virtual,
                 output_concepts=non_virtual,
@@ -930,6 +961,8 @@ def _search_concepts(
                 append_existence_check(
                     output, environment, g, where=conditions, history=history
                 )
+        elif conditions:
+            output.preexisting_conditions = conditions.conditional
         logger.info(
             f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Graph is connected, returning {type(output)} node partial {[c.address for c in output.partial_concepts]}"
         )
@@ -963,18 +996,17 @@ def _search_concepts(
 
 
 def source_query_concepts(
-    output_concepts: List[Concept],
-    environment: Environment,
+    output_concepts: List[BuildConcept],
+    history: History,
+    environment: BuildEnvironment,
     g: Optional[ReferenceGraph] = None,
-    conditions: Optional[WhereClause] = None,
-    history: Optional[History] = None,
+    conditions: Optional[BuildWhereClause] = None,
 ):
     if not output_concepts:
         raise ValueError(f"No output concepts provided {output_concepts}")
     if not g:
         g = generate_graph(environment)
 
-    history = history or History()
     root = search_concepts(
         mandatory_list=output_concepts,
         environment=environment,
@@ -992,6 +1024,9 @@ def source_query_concepts(
             f"Could not resolve conections between {error_strings} from environment graph."
         )
     final = [x for x in root.output_concepts if x.address not in root.hidden_concepts]
+    logger.info(
+        f"{depth_to_prefix(0)}{LOGGER_PREFIX} final concepts are {[x.address for x in final]}"
+    )
     if GroupNode.check_if_required(
         downstream_concepts=final,
         parents=[root.resolve()],