pytrilogy 0.0.2.15__py3-none-any.whl → 0.0.2.18__py3-none-any.whl

trilogy/core/processing/concept_strategies_v3.py

@@ -1,11 +1,17 @@
 from collections import defaultdict
-from typing import List, Optional, Callable
+from typing import List, Optional, Protocol, Union
 
 from trilogy.constants import logger
 from trilogy.core.enums import PurposeLineage, Granularity, FunctionType
 from trilogy.core.env_processor import generate_graph
 from trilogy.core.graph_models import ReferenceGraph
-from trilogy.core.models import Concept, Environment, Function, Grain, WhereClause
+from trilogy.core.models import (
+    Concept,
+    Environment,
+    Function,
+    WhereClause,
+    RowsetItem,
+)
 from trilogy.core.processing.utility import (
     get_disconnected_components,
 )
@@ -36,11 +42,25 @@ class ValidationResult(Enum):
     COMPLETE = 1
     DISCONNECTED = 2
     INCOMPLETE = 3
+    INCOMPLETE_CONDITION = 4
 
 
 LOGGER_PREFIX = "[CONCEPT DETAIL]"
 
 
+class SearchConceptsType(Protocol):
+    def __call__(
+        self,
+        mandatory_list: List[Concept],
+        environment: Environment,
+        depth: int,
+        g: ReferenceGraph,
+        accept_partial: bool = False,
+        history: Optional[History] = None,
+        conditions: Optional[WhereClause] = None,
+    ) -> Union[StrategyNode, None]: ...
+
+
 def get_upstream_concepts(base: Concept, nested: bool = False) -> set[str]:
     upstream = set()
     if nested:
@@ -48,6 +68,11 @@ def get_upstream_concepts(base: Concept, nested: bool = False) -> set[str]:
     if not base.lineage:
         return upstream
     for x in base.lineage.concept_arguments:
+        # if it's derived from any value in a rowset, ALL rowset items are upstream
+        if x.derivation == PurposeLineage.ROWSET:
+            assert isinstance(x.lineage, RowsetItem)
+            for y in x.lineage.rowset.derived_concepts:
+                upstream = upstream.union(get_upstream_concepts(y, nested=True))
         upstream = upstream.union(get_upstream_concepts(x, nested=True))
     return upstream
 
@@ -64,10 +89,10 @@ def get_priority_concept(
         for c in all_concepts
         if c.address not in attempted_addresses and c.address not in found_concepts
     ]
-    # sometimes we need to scan intermediate concepts to get merge keys, so fall back
-    # to exhaustive search
-    pass_two = [c for c in all_concepts if c.address not in attempted_addresses]
-    for remaining_concept in (pass_one, pass_two):
+    # sometimes we need to scan intermediate concepts to get merge keys or filter keys,
+    # so do an exhaustive search
+    # pass_two = [c for c in all_concepts+filter_only if c.address not in attempted_addresses]
+    for remaining_concept in (pass_one,):
         priority = (
             # find anything that needs no joins first, so we can exit early
             [
@@ -85,60 +110,19 @@ def get_priority_concept(
             # we should be home-free here
             +
             # then aggregates to remove them from scope, as they cannot get partials
-            [
-                c
-                for c in remaining_concept
-                if c.derivation == PurposeLineage.AGGREGATE
-                and not c.granularity == Granularity.SINGLE_ROW
-            ]
+            [c for c in remaining_concept if c.derivation == PurposeLineage.AGGREGATE]
             # then windows to remove them from scope, as they cannot get partials
-            + [
-                c
-                for c in remaining_concept
-                if c.derivation == PurposeLineage.WINDOW
-                and not c.granularity == Granularity.SINGLE_ROW
-            ]
+            + [c for c in remaining_concept if c.derivation == PurposeLineage.WINDOW]
             # then filters to remove them from scope, also cannot get partials
-            + [
-                c
-                for c in remaining_concept
-                if c.derivation == PurposeLineage.FILTER
-                and not c.granularity == Granularity.SINGLE_ROW
-            ]
+            + [c for c in remaining_concept if c.derivation == PurposeLineage.FILTER]
             # unnests are weird?
-            + [
-                c
-                for c in remaining_concept
-                if c.derivation == PurposeLineage.UNNEST
-                and not c.granularity == Granularity.SINGLE_ROW
-            ]
-            + [
-                c
-                for c in remaining_concept
-                if c.derivation == PurposeLineage.BASIC
-                and not c.granularity == Granularity.SINGLE_ROW
-            ]
+            + [c for c in remaining_concept if c.derivation == PurposeLineage.UNNEST]
+            + [c for c in remaining_concept if c.derivation == PurposeLineage.BASIC]
             # finally our plain selects
             + [
-                c
-                for c in remaining_concept
-                if c.derivation == PurposeLineage.ROOT
-                and not c.granularity == Granularity.SINGLE_ROW
-            ]
-            # and any non-single row constants
-            + [
-                c
-                for c in remaining_concept
-                if c.derivation == PurposeLineage.CONSTANT
-                and not c.granularity == Granularity.SINGLE_ROW
-            ]
-            # catch all
-            + [
-                c
-                for c in remaining_concept
-                if c.derivation != PurposeLineage.CONSTANT
-                and c.granularity == Granularity.SINGLE_ROW
-            ]
+                c for c in remaining_concept if c.derivation == PurposeLineage.ROOT
+            ]  # and any non-single row constants
+            + [c for c in remaining_concept if c.derivation == PurposeLineage.CONSTANT]
         )
 
         priority += [
@@ -172,6 +156,7 @@ def generate_candidates_restrictive(
     priority_concept: Concept,
     candidates: list[Concept],
     exhausted: set[str],
+    conditions: WhereClause | None = None,
 ) -> List[List[Concept]]:
     # if it's single row, joins are irrelevant. Fetch without keys.
     if priority_concept.granularity == Granularity.SINGLE_ROW:
@@ -185,19 +170,12 @@ def generate_candidates_restrictive(
         and x.address not in priority_concept.pseudonyms
         and priority_concept.address not in x.pseudonyms
     ]
-    combos: list[list[Concept]] = []
-    grain_check = Grain(components=[*local_candidates]).components_copy
-    # for simple operations these, fetch as much as possible.
-    if priority_concept.derivation in (PurposeLineage.BASIC, PurposeLineage.ROOT):
-        if set([x.address for x in grain_check]) != set(
-            [x.address for x in local_candidates]
-        ):
-            combos.append(local_candidates)
-    combos.append(grain_check)
-    # combos.append(local_candidates)
-    # append the empty set for sourcing concept by itself last
-    combos.append([])
-    return combos
+    if conditions and priority_concept.derivation in (
+        PurposeLineage.ROOT,
+        PurposeLineage.CONSTANT,
+    ):
+        return [unique(conditions.row_arguments + local_candidates, "address")]
+    return [local_candidates]
 
 
 def generate_node(
@@ -206,7 +184,7 @@ def generate_node(
     environment: Environment,
     g: ReferenceGraph,
     depth: int,
-    source_concepts: Callable,
+    source_concepts: SearchConceptsType,
     accept_partial: bool = False,
     history: History | None = None,
     conditions: WhereClause | None = None,
@@ -260,7 +238,7 @@ def generate_node(
         )
     elif concept.derivation == PurposeLineage.UNNEST:
         logger.info(
-            f"{depth_to_prefix(depth)}{LOGGER_PREFIX} for {concept.address}, generating unnest node with optional {[x.address for x in local_optional]}"
+            f"{depth_to_prefix(depth)}{LOGGER_PREFIX} for {concept.address}, generating unnest node with optional {[x.address for x in local_optional]} and condition {conditions}"
         )
         return gen_unnest_node(
             concept,
@@ -300,22 +278,68 @@ def generate_node(
             f"{depth_to_prefix(depth)}{LOGGER_PREFIX} for {concept.address}, generating rowset node with optional {[x.address for x in local_optional]}"
         )
         return gen_rowset_node(
-            concept, local_optional, environment, g, depth + 1, source_concepts, history
+            concept,
+            local_optional,
+            environment,
+            g,
+            depth + 1,
+            source_concepts,
+            history,
+            conditions=conditions,
         )
     elif concept.derivation == PurposeLineage.MULTISELECT:
         logger.info(
             f"{depth_to_prefix(depth)}{LOGGER_PREFIX} for {concept.address}, generating multiselect node with optional {[x.address for x in local_optional]}"
         )
         return gen_multiselect_node(
-            concept, local_optional, environment, g, depth + 1, source_concepts, history
+            concept,
+            local_optional,
+            environment,
+            g,
+            depth + 1,
+            source_concepts,
+            history,
+            conditions=conditions,
         )
     elif concept.derivation == PurposeLineage.CONSTANT:
+        constant_targets = [concept] + local_optional
         logger.info(
             f"{depth_to_prefix(depth)}{LOGGER_PREFIX} for {concept.address}, generating constant node"
         )
+        if any([x.derivation != PurposeLineage.CONSTANT for x in local_optional]):
+            non_root = [
+                x.address
+                for x in local_optional
+                if x.derivation != PurposeLineage.CONSTANT
+            ]
+            logger.info(
+                f"{depth_to_prefix(depth)}{LOGGER_PREFIX} including filter concepts, there is non root filter requirements {non_root}. Recursing with all of these as mandatory"
+            )
+
+            if not history.check_started(
+                constant_targets, accept_partial=accept_partial, conditions=conditions
+            ):
+                history.log_start(
+                    constant_targets,
+                    accept_partial=accept_partial,
+                    conditions=conditions,
+                )
+                return source_concepts(
+                    mandatory_list=constant_targets,
+                    environment=environment,
+                    g=g,
+                    depth=depth + 1,
+                    accept_partial=accept_partial,
+                    history=history,
+                    # we DO NOT pass up conditions at this point, as we are now expanding to include conditions in search
+                    # which we do whenever we hit a root node
+                    # conditions=conditions,
+                )
+            else:
+                return None
         return ConstantNode(
             input_concepts=[],
-            output_concepts=[concept],
+            output_concepts=constant_targets,
             environment=environment,
             g=g,
             parents=[],
@@ -338,19 +362,64 @@ def generate_node(
                 depth + 1,
                 source_concepts,
                 history,
+                conditions=conditions,
             )
         logger.info(
             f"{depth_to_prefix(depth)}{LOGGER_PREFIX} for {concept.address}, generating basic node with optional {[x.address for x in local_optional]}"
         )
         return gen_basic_node(
-            concept, local_optional, environment, g, depth + 1, source_concepts, history
+            concept,
+            local_optional,
+            environment,
+            g,
+            depth + 1,
+            source_concepts,
+            history,
+            conditions=conditions,
         )
 
     elif concept.derivation == PurposeLineage.ROOT:
         logger.info(
-            f"{depth_to_prefix(depth)}{LOGGER_PREFIX} for {concept.address}, generating select node with optional {[x.address for x in local_optional]}"
+            f"{depth_to_prefix(depth)}{LOGGER_PREFIX} for {concept.address}, generating select node with optional including condition inputs {[x.address for x in local_optional]}"
         )
-        return history.gen_select_node(
+        # we've injected in any conditional concepts that may exist
+        # so if we don't still have just roots, we need to go up
+        root_targets = [concept] + local_optional
+
+        if any(
+            [
+                x.derivation not in (PurposeLineage.ROOT, PurposeLineage.CONSTANT)
+                for x in local_optional
+            ]
+        ):
+            non_root = [
+                x.address
+                for x in local_optional
+                if x.derivation not in (PurposeLineage.ROOT, PurposeLineage.CONSTANT)
+            ]
+            logger.info(
+                f"{depth_to_prefix(depth)}{LOGGER_PREFIX} including filter concepts, there is non root filter requirements {non_root}. Recursing with all of these as mandatory"
+            )
+
+            if not history.check_started(
+                root_targets, accept_partial=accept_partial, conditions=conditions
+            ):
+                history.log_start(
+                    root_targets, accept_partial=accept_partial, conditions=conditions
+                )
+                return source_concepts(
+                    mandatory_list=root_targets,
+                    environment=environment,
+                    g=g,
+                    depth=depth + 1,
+                    accept_partial=accept_partial,
+                    history=history,
+                    # we DO NOT pass up conditions at this point, as we are now expanding to include conditions in search
+                    # which we do whenever we hit a root node
+                    # conditions=conditions,
+                )
+
+        check = history.gen_select_node(
             concept,
             local_optional,
             environment,
@@ -358,12 +427,48 @@ def generate_node(
             depth + 1,
             fail_if_not_found=False,
             accept_partial=accept_partial,
-            accept_partial_optional=True,
+            accept_partial_optional=False,
             source_concepts=source_concepts,
             conditions=conditions,
         )
+        if not check:
+
+            logger.info(
+                f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Could not resolve root concepts, checking for expanded concepts"
+            )
+            for accept_partial in [False, True]:
+                expanded = gen_merge_node(
+                    all_concepts=root_targets,
+                    environment=environment,
+                    g=g,
+                    depth=depth,
+                    source_concepts=source_concepts,
+                    history=history,
+                    search_conditions=conditions,
+                    accept_partial=accept_partial,
+                )
+
+                if expanded:
+                    ex_resolve = expanded.resolve()
+                    extra = [
+                        x
+                        for x in ex_resolve.output_concepts
+                        if x.address not in [y.address for y in root_targets]
+                        and x not in ex_resolve.grain.components
+                    ]
+                    expanded.set_output_concepts(root_targets)
+
+                    logger.info(
+                        f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Found connections for {[c.address for c in root_targets]} via concept addition; removing extra {[c.address for c in extra]}"
+                    )
+                    return expanded
+            logger.info(
+                f"{depth_to_prefix(depth)}{LOGGER_PREFIX} could not find additional concept to inject"
+            )
+            return None
     else:
         raise ValueError(f"Unknown derivation {concept.derivation}")
+    return None
 
 
 def validate_concept(
@@ -417,6 +522,8 @@ def validate_concept(
 def validate_stack(
     stack: List[StrategyNode],
     concepts: List[Concept],
+    mandatory_with_filter: List[Concept],
+    conditions: WhereClause | None = None,
     accept_partial: bool = False,
 ) -> tuple[ValidationResult, set[str], set[str], set[str], set[str]]:
     found_map: dict[str, set[Concept]] = defaultdict(set)
@@ -425,6 +532,7 @@ def validate_stack(
     partial_addresses: set[str] = set()
     virtual_addresses: set[str] = set()
     seen: set[str] = set()
+
     for node in stack:
         resolved = node.resolve()
 
@@ -445,13 +553,26 @@ def validate_stack(
                 continue
             found_addresses.add(concept.address)
             virtual_addresses.add(concept.address)
-
+    if not conditions:
+        conditions_met = True
+    else:
+        conditions_met = all(
+            [node.preexisting_conditions == conditions.conditional for node in stack]
+        ) or all([c.address in found_addresses for c in mandatory_with_filter])
     # zip in those we know we found
-    if not all([c.address in found_addresses for c in concepts]):
+    if not all([c.address in found_addresses for c in concepts]) or not conditions_met:
+        if not all([c.address in found_addresses for c in concepts]):
+            return (
+                ValidationResult.INCOMPLETE,
+                found_addresses,
+                {c.address for c in concepts if c.address not in found_addresses},
+                partial_addresses,
+                virtual_addresses,
+            )
         return (
-            ValidationResult.INCOMPLETE,
+            ValidationResult.INCOMPLETE_CONDITION,
             found_addresses,
-            {c.address for c in concepts if c.address not in found_addresses},
+            {c.address for c in concepts if c.address not in mandatory_with_filter},
             partial_addresses,
             virtual_addresses,
         )
@@ -479,6 +600,38 @@ def depth_to_prefix(depth: int) -> str:
     return "\t" * depth
 
 
+def append_existence_check(
+    node: StrategyNode,
+    environment: Environment,
+    graph: ReferenceGraph,
+    where: WhereClause,
+    history: History | None = None,
+):
+    # we if we have a where clause doing an existence check
+    # treat that as separate subquery
+    if where.existence_arguments:
+        for subselect in where.existence_arguments:
+            if not subselect:
+                continue
+            if all([x.address in node.input_concepts for x in subselect]):
+                logger.info(
+                    f"{LOGGER_PREFIX} existance clause inputs already found {[str(c) for c in subselect]}"
+                )
+                continue
+            logger.info(
+                f"{LOGGER_PREFIX} fetching existence clause inputs {[str(c) for c in subselect]}"
+            )
+            parent = source_query_concepts(
+                [*subselect], environment=environment, g=graph, history=history
+            )
+            assert parent, "Could not resolve existence clause"
+            node.add_parents([parent])
+            logger.info(
+                f"{LOGGER_PREFIX} fetching existence clause inputs {[str(c) for c in subselect]}"
+            )
+            node.add_existence_concepts([*subselect])
+
+
 def search_concepts(
     mandatory_list: List[Concept],
     environment: Environment,
@@ -529,9 +682,32 @@ def _search_concepts(
     conditions: WhereClause | None = None,
 ) -> StrategyNode | None:
 
+    # these are the concepts we need in the output projection
     mandatory_list = unique(mandatory_list, "address")
 
     all_mandatory = set(c.address for c in mandatory_list)
+
+    must_evaluate_condition_on_this_level_not_push_down = False
+
+    # if we have a filter, we may need to get more values to support that.
+    if conditions:
+        completion_mandatory = unique(
+            mandatory_list + conditions.row_arguments, "address"
+        )
+        # if anything we need to get is in the filter set and it's a computed value
+        # we need to get _everything_ in this loop
+        if any(
+            [
+                x.derivation not in (PurposeLineage.ROOT, PurposeLineage.CONSTANT)
+                and x.address in conditions.row_arguments
+                for x in mandatory_list
+            ]
+        ):
+            mandatory_list = completion_mandatory
+            must_evaluate_condition_on_this_level_not_push_down = True
+    else:
+        completion_mandatory = mandatory_list
+
     attempted: set[str] = set()
 
     found: set[str] = set()
@@ -542,22 +718,60 @@ def _search_concepts(
 
     while attempted != all_mandatory:
         priority_concept = get_priority_concept(
-            mandatory_list, attempted, found_concepts=found, depth=depth
+            mandatory_list,
+            attempted,
+            found_concepts=found,
+            depth=depth,
         )
+        # filter evaluation
+        # always pass the filter up when we aren't looking at all filter inputs
+        # or there are any non-filter complex types
+        if conditions:
+            should_evaluate_filter_on_this_level_not_push_down = all(
+                [x.address in mandatory_list for x in conditions.row_arguments]
+            ) and not any(
+                [
+                    x.derivation not in (PurposeLineage.ROOT, PurposeLineage.CONSTANT)
+                    for x in mandatory_list
+                    if x.address not in conditions.row_arguments
+                ]
+            )
+        else:
+            should_evaluate_filter_on_this_level_not_push_down = True
+        local_conditions = (
+            conditions
+            if conditions
+            and not must_evaluate_condition_on_this_level_not_push_down
+            and not should_evaluate_filter_on_this_level_not_push_down
+            else None
+        )
+        # but if it's not basic, and it's not condition;
+        # we do need to push it down (and have another layer of filter evaluation)
+        # to ensure filtering happens before something like a SUM
+        if (
+            conditions
+            and priority_concept.derivation
+            not in (PurposeLineage.ROOT, PurposeLineage.CONSTANT)
+            and priority_concept.address not in conditions.row_arguments
+        ):
+            logger.info(
+                f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Force including conditions to push filtering above complex condition that is not condition member or parent"
+            )
+            local_conditions = conditions
 
         logger.info(
-            f"{depth_to_prefix(depth)}{LOGGER_PREFIX} priority concept is {str(priority_concept)}"
+            f"{depth_to_prefix(depth)}{LOGGER_PREFIX} priority concept is {str(priority_concept)} derivation {priority_concept.derivation} with conditions {local_conditions}"
         )
 
         candidates = [
             c for c in mandatory_list if c.address != priority_concept.address
         ]
         candidate_lists = generate_candidates_restrictive(
-            priority_concept, candidates, skip
+            priority_concept, candidates, skip, conditions=conditions
         )
         for clist in candidate_lists:
             logger.info(
-                f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Beginning sourcing loop for {str(priority_concept)}, accept_partial {accept_partial}, optional {[str(v) for v in clist]}, exhausted {[str(c) for c in skip]}"
+                f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Beginning sourcing loop for {priority_concept.address}, accept_partial {accept_partial}, optional {[v.address for v in clist]}, exhausted {[c for c in skip]}"
             )
             node = generate_node(
                 priority_concept,
@@ -568,11 +782,17 @@ def _search_concepts(
                 source_concepts=search_concepts,
                 accept_partial=accept_partial,
                 history=history,
-                conditions=conditions,
+                conditions=local_conditions,
             )
             if node:
                 stack.append(node)
-                node.resolve()
+                try:
+                    node.resolve()
+                except Exception as e:
+                    logger.error(
+                        f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Could not resolve node {node} {e}"
+                    )
+                    raise e
                 # these concepts should not be attempted to be sourced again
                 # as fetching them requires operating on a subset of concepts
                 if priority_concept.derivation in [
@@ -588,13 +808,21 @@ def _search_concepts(
                 break
         attempted.add(priority_concept.address)
         complete, found, missing, partial, virtual = validate_stack(
-            stack, mandatory_list, accept_partial
+            stack,
+            mandatory_list,
+            completion_mandatory,
+            conditions=conditions,
+            accept_partial=accept_partial,
         )
-
+        mandatory_completion = [c.address for c in completion_mandatory]
         logger.info(
-            f"{depth_to_prefix(depth)}{LOGGER_PREFIX} finished concept loop for {priority_concept} flag for accepting partial addresses is"
-            f" {accept_partial} (complete: {complete}), have {found} from {[n for n in stack]} (missing {missing} partial {partial} virtual {virtual}), attempted {attempted}"
+            f"{depth_to_prefix(depth)}{LOGGER_PREFIX} finished concept loop for {priority_concept} {priority_concept.derivation} condition {conditions} flag for accepting partial addresses is"
+            f" {accept_partial} (complete: {complete}), have {found} from {[n for n in stack]} (missing {missing} partial {partial} virtual {virtual}), attempted {attempted}, mandatory w/ filter {mandatory_completion}"
         )
+        if complete == ValidationResult.INCOMPLETE_CONDITION:
+            raise SyntaxError(
+                {str(node): node.preexisting_conditions for node in stack}
+            )
         # early exit if we have a complete stack with one node
         # we can only early exit if we have a complete stack
         # and we are not looking for more non-partial sources
@@ -602,73 +830,65 @@ def _search_concepts(
             not accept_partial or (accept_partial and not partial)
         ):
             break
+        # if we have attempted on root node, we've tried them all.
+        # inject in another search with filter concepts
+        if priority_concept.derivation == PurposeLineage.ROOT:
+            logger.info(
+                f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Breaking as attempted root with no results"
+            )
+            break
 
     logger.info(
         f"{depth_to_prefix(depth)}{LOGGER_PREFIX} finished sourcing loop (complete: {complete}), have {found} from {[n for n in stack]} (missing {all_mandatory - found}), attempted {attempted}, virtual {virtual}"
     )
     if complete == ValidationResult.COMPLETE:
-        non_virtual = [c for c in mandatory_list if c.address not in virtual]
+        condition_required = True
+        non_virtual = [c for c in completion_mandatory if c.address not in virtual]
+        if not conditions:
+            condition_required = False
+            non_virtual = [c for c in mandatory_list if c.address not in virtual]
+
+        elif all([x.preexisting_conditions == conditions.conditional for x in stack]):
+            condition_required = False
+            non_virtual = [c for c in mandatory_list if c.address not in virtual]
+
+        if conditions and not condition_required:
+            parent_map = {
+                str(x): x.preexisting_conditions == conditions.conditional
+                for x in stack
+            }
+            logger.info(
+                f"Condition {conditions} not required, parents included filtering! {parent_map }"
+            )
         if len(stack) == 1:
-            output = stack[0]
+            output: StrategyNode = stack[0]
             logger.info(
-                f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Source stack has single node, returning that {type(output)} with output {[x.address for x in output.output_concepts]} and {output.resolve().source_map}"
+                f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Source stack has single node, returning that {type(output)}"
+            )
+        else:
+            output = MergeNode(
+                input_concepts=non_virtual,
+                output_concepts=non_virtual,
+                environment=environment,
+                g=g,
+                parents=stack,
+                depth=depth,
             )
-            return output
-
-        output = MergeNode(
-            input_concepts=non_virtual,
-            output_concepts=non_virtual,
-            environment=environment,
-            g=g,
-            parents=stack,
-            depth=depth,
-        )
-
         # ensure we can resolve our final merge
         output.resolve()
+        if condition_required and conditions:
+            output.add_condition(conditions.conditional)
+            if conditions.existence_arguments:
+                append_existence_check(
+                    output, environment, g, where=conditions, history=history
+                )
         logger.info(
             f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Graph is connected, returning merge node, partial {[c.address for c in output.partial_concepts]}"
         )
         return output
 
-    # check that we're not already in a discovery loop
-    if not history.check_started(
-        mandatory_list, accept_partial=accept_partial, conditions=conditions
-    ):
-        logger.info(
-            f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Stack is not connected graph, flag for accepting partial addresses is {accept_partial}, checking for expanded concepts"
-        )
-        # gate against further recursion into this
-        history.log_start(
-            mandatory_list, accept_partial=accept_partial, conditions=conditions
-        )
-        expanded = gen_merge_node(
-            all_concepts=mandatory_list,
-            environment=environment,
-            g=g,
-            depth=depth,
-            source_concepts=search_concepts,
-            history=history,
-            search_conditions=conditions,
-        )
-
-        if expanded:
-            # we don't need to return the entire list; just the ones we needed pre-expansion
-            ex_resolve = expanded.resolve()
-            extra = [
-                x
-                for x in ex_resolve.output_concepts
-                if x.address not in [y.address for y in mandatory_list]
-                and x not in ex_resolve.grain.components
-            ]
-            expanded.set_output_concepts(mandatory_list)
-
-            logger.info(
-                f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Found connections for {[c.address for c in mandatory_list]} via concept addition; removing extra {[c.address for c in extra]}"
-            )
-            return expanded
     # if we can't find it after expanding to a merge, then
-    # attempt to accept partials in join paths
+    # accept partials in join paths
 
     if not accept_partial:
         logger.info(
@@ -723,11 +943,18 @@ def source_query_concepts(
         raise ValueError(
             f"Could not resolve conections between {error_strings} from environment graph."
         )
-    return GroupNode(
-        output_concepts=output_concepts,
-        input_concepts=output_concepts,
+    candidate = GroupNode(
+        output_concepts=[
+            x for x in root.output_concepts if x.address not in root.hidden_concepts
+        ],
+        input_concepts=[
+            x for x in root.output_concepts if x.address not in root.hidden_concepts
+        ],
         environment=environment,
         g=g,
         parents=[root],
         partial_concepts=root.partial_concepts,
     )
+    if not candidate.resolve().group_required:
+        return root
+    return candidate