pytrilogy 0.3.142__cp312-cp312-win_amd64.whl

trilogy/core/processing/node_generators/select_node.py

@@ -0,0 +1,95 @@
+from trilogy.constants import logger
+from trilogy.core.enums import Derivation
+from trilogy.core.exceptions import NoDatasourceException
+from trilogy.core.models.build import (
+    BuildConcept,
+    BuildWhereClause,
+    CanonicalBuildConceptList,
+)
+from trilogy.core.models.build_environment import BuildEnvironment
+from trilogy.core.processing.node_generators.select_merge_node import (
+    gen_select_merge_node,
+)
+from trilogy.core.processing.nodes import (
+    StrategyNode,
+)
+from trilogy.core.processing.utility import padding
+
+LOGGER_PREFIX = "[GEN_SELECT_NODE]"
+
+
+def validate_query_is_resolvable(
+    missing: list[str],
+    environment: BuildEnvironment,
+    materialized_lcl: CanonicalBuildConceptList,
+) -> None:
+    # if a query cannot ever be resolved, exit early with an error
+    for x in missing:
+        if x not in environment.concepts:
+            # if it's locally derived, we can assume it can be resolved
+            continue
+        validation_concept = environment.concepts[x]
+        # if the concept we look up isn't what we searched for,
+        # we're in a pseudonym anyway, don't worry about validating
+        if validation_concept.address != x:
+            continue
+        if validation_concept.derivation == Derivation.ROOT:
+            has_source = False
+            for x in validation_concept.pseudonyms:
+                if x in environment.alias_origin_lookup:
+                    pseudonym_concept = environment.alias_origin_lookup[x]
+                else:
+                    pseudonym_concept = environment.concepts[x]
+                # if it's not a root concept pseudonym,
+                # assume we can derive it
+                if pseudonym_concept.derivation != Derivation.ROOT:
+                    has_source = True
+                    break
+                if pseudonym_concept.address in materialized_lcl:
+                    has_source = True
+                    break
+            if not has_source:
+                raise NoDatasourceException(
+                    f"No datasource exists for root concept {validation_concept}, and no resolvable pseudonyms found from {validation_concept.pseudonyms}. This query is unresolvable from your environment. Check your datasources and imports to make sure this concept is bound."
+                )
+    return None
+
+
+def gen_select_node(
+    concepts: list[BuildConcept],
+    environment: BuildEnvironment,
+    g,
+    depth: int,
+    accept_partial: bool = False,
+    fail_if_not_found: bool = True,
+    conditions: BuildWhereClause | None = None,
+) -> StrategyNode | None:
+    all_lcl = CanonicalBuildConceptList(concepts=concepts)
+    # search all concepts here, including partial
+    materialized_lcl = CanonicalBuildConceptList(
+        concepts=[
+            x
+            for x in concepts
+            if x.canonical_address in environment.materialized_canonical_concepts
+            or x.derivation == Derivation.CONSTANT
+        ]
+    )
+    if materialized_lcl != all_lcl:
+        missing = all_lcl.difference(materialized_lcl)
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} Skipping select node generation for {concepts}"
+            f" as it + optional includes non-materialized concepts (looking for all {all_lcl}, missing {missing})."
+        )
+        validate_query_is_resolvable(missing, environment, materialized_lcl)
+        if fail_if_not_found:
+            raise NoDatasourceException(f"No datasource exists for {concepts}")
+        return None
+
+    return gen_select_merge_node(
+        concepts,
+        g=g,
+        environment=environment,
+        depth=depth,
+        accept_partial=accept_partial,
+        conditions=conditions,
+    )

trilogy/core/processing/node_generators/synonym_node.py

@@ -0,0 +1,98 @@
+import itertools
+from collections import defaultdict
+from typing import List
+
+from trilogy.constants import logger
+from trilogy.core.enums import Derivation
+from trilogy.core.models.build import BuildConcept, BuildWhereClause
+from trilogy.core.models.build_environment import BuildEnvironment
+from trilogy.core.processing.nodes import History, StrategyNode
+from trilogy.core.processing.utility import padding
+
+LOGGER_PREFIX = "[GEN_SYNONYM_NODE]"
+
+
+def gen_synonym_node(
+    all_concepts: List[BuildConcept],
+    environment: BuildEnvironment,
+    g,
+    depth: int,
+    source_concepts,
+    history: History | None = None,
+    conditions: BuildWhereClause | None = None,
+    accept_partial: bool = False,
+) -> StrategyNode | None:
+    local_prefix = f"{padding(depth)}[GEN_SYNONYM_NODE]"
+    base_fingerprint = tuple(sorted([x.address for x in all_concepts]))
+    synonyms = defaultdict(list)
+    has_synonyms = False
+    for x in all_concepts:
+        synonyms[x.address] = [x]
+        if x.address in environment.alias_origin_lookup:
+            parent = environment.concepts[x.address]
+            if parent.address != x.address:
+                synonyms[x.address].append(parent)
+                has_synonyms = True
+        for y in x.pseudonyms:
+            if y in environment.alias_origin_lookup:
+                synonyms[x.address].append(environment.alias_origin_lookup[y])
+                has_synonyms = True
+            elif y in environment.concepts:
+                synonyms[x.address].append(environment.concepts[y])
+                has_synonyms = True
+    for address in synonyms:
+        synonyms[address].sort(key=lambda obj: obj.address)
+    if not has_synonyms:
+        return None
+
+    logger.info(f"{local_prefix} Generating Synonym Node with {len(synonyms)} synonyms")
+    sorted_keys = sorted(synonyms.keys())
+    combinations_list: list[tuple[BuildConcept, ...]] = list(
+        itertools.product(*(synonyms[obj] for obj in sorted_keys))
+    )
+
+    def similarity_sort_key(combo: tuple[BuildConcept, ...]):
+        addresses = [x.address for x in combo]
+
+        # Calculate similarity score - count how many pairs share prefixes
+        similarity_score = 0
+        roots = sum(
+            [1 for x in combo if x.derivation in (Derivation.ROOT, Derivation.CONSTANT)]
+        )
+        for i in range(len(addresses)):
+            for j in range(i + 1, len(addresses)):
+                # Find common prefix length
+                addr1_parts = addresses[i].split(".")
+                addr2_parts = addresses[j].split(".")
+                common_prefix_len = 0
+                for k in range(min(len(addr1_parts), len(addr2_parts))):
+                    if addr1_parts[k] == addr2_parts[k]:
+                        common_prefix_len += 1
+                    else:
+                        break
+                similarity_score += common_prefix_len
+
+        # Sort by roots, similarity (descending), then by addresses (ascending) for ties
+        return (-roots, -similarity_score, addresses)
+
+    combinations_list.sort(key=similarity_sort_key)
+    for combo in combinations_list:
+        fingerprint = tuple(sorted([x.address for x in combo]))
+        if fingerprint == base_fingerprint:
+            continue
+        logger.info(
+            f"{local_prefix} checking combination {fingerprint} with {len(combo)} concepts"
+        )
+        attempt: StrategyNode | None = source_concepts(
+            list(combo),
+            history=history,
+            environment=environment,
+            depth=depth,
+            conditions=conditions,
+            g=g,
+            accept_partial=accept_partial,
+        )
+        if attempt:
+            logger.info(f"{local_prefix} found inputs with {combo}")
+            return attempt
+    return None

trilogy/core/processing/node_generators/union_node.py

@@ -0,0 +1,91 @@
+from typing import List
+
+from trilogy.constants import logger
+from trilogy.core.enums import FunctionType
+from trilogy.core.models.build import BuildConcept, BuildFunction, BuildWhereClause
+from trilogy.core.processing.nodes import History, StrategyNode, UnionNode
+from trilogy.core.processing.utility import padding
+
+LOGGER_PREFIX = "[GEN_UNION_NODE]"
+
+
+def is_union(c: BuildConcept):
+    return (
+        isinstance(c.lineage, BuildFunction)
+        and c.lineage.operator == FunctionType.UNION
+    )
+
+
+def build_layers(
+    concepts: list[BuildConcept],
+) -> tuple[list[list[BuildConcept]], list[BuildConcept]]:
+    sources = {
+        x.address: x.lineage.concept_arguments if x.lineage else [] for x in concepts
+    }
+    root = concepts[0]
+
+    built_layers = []
+    layers = root.lineage.concept_arguments if root.lineage else []
+    sourced = set()
+    while layers:
+        layer = []
+        current = layers.pop()
+        sourced.add(current.address)
+        layer.append(current)
+        for key, values in sources.items():
+            if key == current.address:
+                continue
+            for value in values:
+                if value.address in (current.keys or []) or current.address in (
+                    value.keys or []
+                ):
+                    layer.append(value)
+                    sourced.add(value.address)
+        built_layers.append(layer)
+    complete = [
+        x for x in concepts if all([x.address in sourced for x in sources[x.address]])
+    ]
+    return built_layers, complete
+
+
+def gen_union_node(
+    concept: BuildConcept,
+    local_optional: List[BuildConcept],
+    environment,
+    g,
+    depth: int,
+    source_concepts,
+    history: History | None = None,
+    conditions: BuildWhereClause | None = None,
+) -> StrategyNode | None:
+    all_unions = [x for x in local_optional if is_union(x)] + [concept]
+    logger.info(f"{padding(depth)}{LOGGER_PREFIX} found unions {all_unions}")
+    parent_nodes = []
+    layers, resolved = build_layers(all_unions)
+    for layer in layers:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} fetching layer {layer} with resolved {resolved}"
+        )
+        parent: StrategyNode = source_concepts(
+            mandatory_list=layer,
+            environment=environment,
+            g=g,
+            depth=depth + 1,
+            history=history,
+            conditions=conditions,
+        )
+
+        parent.add_output_concepts(resolved)
+        parent_nodes.append(parent)
+        if not parent:
+            logger.info(
+                f"{padding(depth)}{LOGGER_PREFIX} could not find union node parents"
+            )
+            return None
+
+    return UnionNode(
+        input_concepts=resolved,
+        output_concepts=resolved,
+        environment=environment,
+        parents=parent_nodes,
+    )

trilogy/core/processing/node_generators/unnest_node.py

@@ -0,0 +1,182 @@
+from typing import List
+
+from trilogy.constants import logger
+from trilogy.core.models.build import (
+    BuildConcept,
+    BuildFunction,
+    BuildWhereClause,
+)
+from trilogy.core.models.build_environment import BuildEnvironment
+from trilogy.core.processing.nodes import (
+    History,
+    MergeNode,
+    StrategyNode,
+    UnnestNode,
+    WhereSafetyNode,
+)
+from trilogy.core.processing.utility import padding
+
+LOGGER_PREFIX = "[GEN_UNNEST_NODE]"
+
+
+def get_pseudonym_parents(
+    concept: BuildConcept,
+    local_optional: List[BuildConcept],
+    source_concepts,
+    environment: BuildEnvironment,
+    g,
+    depth,
+    history,
+    conditions,
+) -> List[StrategyNode]:
+    for x in concept.pseudonyms:
+        attempt = source_concepts(
+            mandatory_list=[environment.alias_origin_lookup[x]] + local_optional,
+            environment=environment,
+            g=g,
+            depth=depth + 1,
+            history=history,
+            conditions=conditions,
+            accept_partial=True,
+        )
+        if not attempt:
+            continue
+        return [attempt]
+    return []
+
+
+def gen_unnest_node(
+    concept: BuildConcept,
+    local_optional: List[BuildConcept],
+    history: History,
+    environment: BuildEnvironment,
+    g,
+    depth: int,
+    source_concepts,
+    conditions: BuildWhereClause | None = None,
+) -> StrategyNode | None:
+    arguments = []
+    join_nodes: list[StrategyNode] = []
+    depth_prefix = "\t" * depth
+    if isinstance(concept.lineage, BuildFunction):
+        arguments = concept.lineage.concept_arguments
+    search_optional = local_optional
+    if (not arguments) and (local_optional and concept.pseudonyms):
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} unnest node for {concept} has no parents; creating solo unnest node"
+        )
+        join_nodes += get_pseudonym_parents(
+            concept,
+            local_optional,
+            source_concepts,
+            environment,
+            g,
+            depth,
+            history,
+            conditions,
+        )
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} unnest node for {concept} got join nodes {join_nodes}"
+        )
+        search_optional = []
+
+    equivalent_optional = [x for x in search_optional if x.lineage == concept.lineage]
+
+    non_equivalent_optional = [
+        x for x in search_optional if x not in equivalent_optional
+    ]
+    all_parents = arguments + non_equivalent_optional
+    logger.info(
+        f"{depth_prefix}{LOGGER_PREFIX} unnest node for {concept} with lineage {concept.lineage} has parents + optional {all_parents} and equivalent optional {equivalent_optional}"
+    )
+    local_conditions = False
+    expected_outputs = [concept] + local_optional
+    parent: StrategyNode | None = None
+    if arguments or search_optional:
+        parent = source_concepts(
+            mandatory_list=all_parents,
+            environment=environment,
+            g=g,
+            depth=depth + 1,
+            history=history,
+            conditions=conditions,
+        )
+        if not parent:
+            logger.info(
+                f"{padding(depth)}{LOGGER_PREFIX} could not find unnest node parents"
+            )
+            return None
+    elif conditions:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} unnest node has no parents but conditions inputs {conditions.row_arguments} vs expected output {expected_outputs}"
+        )
+        if all([x.address in expected_outputs for x in conditions.row_arguments]):
+            local_conditions = True
+        else:
+            parent = source_concepts(
+                mandatory_list=conditions.conditional.row_arguments,
+                environment=environment,
+                g=g,
+                depth=depth + 1,
+                history=history,
+                conditions=conditions,
+            )
+            if not parent:
+                logger.info(
+                    f"{padding(depth)}{LOGGER_PREFIX} could not find unnest node condition inputs with no parents"
+                )
+                return None
+    else:
+        parent = None
+    logger.info(
+        f"{depth_prefix}{LOGGER_PREFIX} unnest node for {concept} got parent {parent}"
+    )
+    base = UnnestNode(
+        unnest_concepts=[concept] + equivalent_optional,
+        input_concepts=arguments + non_equivalent_optional,
+        output_concepts=[concept] + search_optional,
+        environment=environment,
+        parents=([parent] if parent else []),
+    )
+
+    conditional = conditions.conditional if conditions else None
+    if join_nodes:
+        logger.info(
+            f"{depth_prefix}{LOGGER_PREFIX} unnest node for {concept} needs to merge with join nodes {join_nodes}"
+        )
+        for x in join_nodes:
+            logger.info(
+                f"{depth_prefix}{LOGGER_PREFIX} join node {x} with partial {x.partial_concepts}"
+            )
+            pseudonyms = [
+                environment.alias_origin_lookup[p] for p in concept.pseudonyms
+            ]
+            x.add_partial_concepts(pseudonyms)
+        return MergeNode(
+            input_concepts=base.output_concepts
+            + [j for n in join_nodes for j in n.output_concepts],
+            output_concepts=[concept] + local_optional,
+            environment=environment,
+            parents=[base] + join_nodes,
+            conditions=conditional if local_conditions is True else None,
+            preexisting_conditions=(
+                conditional if conditional and local_conditions is False else None
+            ),
+        )
+    # we need to sometimes nest an unnest node,
+    # as unnest operations are not valid in all situations
+    new = WhereSafetyNode(
+        input_concepts=base.output_concepts,
+        output_concepts=base.output_concepts,
+        environment=environment,
+        parents=[base],
+        conditions=conditional if local_conditions is True else None,
+        preexisting_conditions=(
+            conditional if conditional and local_conditions is False else None
+        ),
+    )
+    # qds = new.resolve()
+    # assert qds.source_map[concept.address] == {base.resolve()}
+    # for x in equivalent_optional:
+    #     assert qds.source_map[x.address] == {base.resolve()}
+    return new

trilogy/core/processing/node_generators/window_node.py

@@ -0,0 +1,201 @@
+from typing import List
+
+from trilogy.constants import logger
+from trilogy.core.models.build import (
+    BuildConcept,
+    BuildGrain,
+    BuildWhereClause,
+    BuildWindowItem,
+)
+from trilogy.core.models.build_environment import BuildEnvironment
+from trilogy.core.processing.node_generators.common import (
+    gen_enrichment_node,
+)
+from trilogy.core.processing.nodes import (
+    History,
+    StrategyNode,
+    WhereSafetyNode,
+    WindowNode,
+)
+from trilogy.core.processing.utility import create_log_lambda, padding
+from trilogy.utility import unique
+
+LOGGER_PREFIX = "[GEN_WINDOW_NODE]"
+
+
+WINDOW_TYPES = (BuildWindowItem,)
+
+
+def resolve_window_parent_concepts(
+    concept: BuildConcept, environment: BuildEnvironment, depth: int
+) -> tuple[BuildConcept, List[BuildConcept]]:
+    if not isinstance(concept.lineage, WINDOW_TYPES):
+        raise ValueError
+    base = []
+    if concept.lineage.over:
+        base += concept.lineage.over
+    if concept.lineage.order_by:
+        for item in concept.lineage.order_by:
+            base += item.concept_arguments
+    if concept.grain:
+        for gitem in concept.grain.components:
+            logger.info(
+                f"{padding(depth)}{LOGGER_PREFIX} appending grain item {gitem} to base"
+            )
+            base.append(environment.concepts[gitem])
+    return concept.lineage.content, unique(base, "address")
+
+
+def gen_window_node(
+    concept: BuildConcept,
+    local_optional: list[BuildConcept],
+    environment: BuildEnvironment,
+    g,
+    depth: int,
+    source_concepts,
+    history: History,
+    conditions: BuildWhereClause | None = None,
+) -> StrategyNode | None:
+    base, parent_concepts = resolve_window_parent_concepts(concept, environment, depth)
+    logger.info(
+        f"{padding(depth)}{LOGGER_PREFIX} generating window node for {concept} with parents {[x.address for x in parent_concepts]} and optional {local_optional}"
+    )
+    equivalent_optional = [
+        x
+        for x in local_optional
+        if isinstance(x.lineage, WINDOW_TYPES)
+        and resolve_window_parent_concepts(x, environment, depth)[1] == parent_concepts
+    ]
+
+    targets = [base]
+    # append in keys to get the right grain
+    if concept.keys:
+        for item in concept.keys:
+            if item in targets:
+                continue
+            logger.info(
+                f"{padding(depth)}{LOGGER_PREFIX} appending search for key {item}"
+            )
+            targets.append(environment.concepts[item])
+    additional_outputs = []
+    if equivalent_optional:
+        for x in equivalent_optional:
+            assert isinstance(x.lineage, WINDOW_TYPES)
+            base, parents = resolve_window_parent_concepts(x, environment, depth)
+            logger.info(
+                f"{padding(depth)}{LOGGER_PREFIX} found equivalent optional {x} with parents {parents}"
+            )
+            additional_outputs.append(x)
+            # also append the base concept it's being grouped over
+            targets.append(base)
+
+    grain_equivalents = [
+        x
+        for x in local_optional
+        if x.keys
+        and all([key in targets for key in x.keys])
+        and x.grain == concept.grain
+    ]
+
+    for x in grain_equivalents:
+        if x.address in additional_outputs:
+            continue
+        targets.append(x)
+
+    # finally, the ones we'll need to enrich
+    non_equivalent_optional = [x for x in local_optional if x.address not in targets]
+
+    logger.info(
+        f"{padding(depth)}{LOGGER_PREFIX} resolving final parents {parent_concepts + targets}"
+    )
+    parent_node: StrategyNode = source_concepts(
+        mandatory_list=parent_concepts + targets,
+        environment=environment,
+        g=g,
+        depth=depth + 1,
+        history=history,
+        conditions=conditions,
+    )
+    if not parent_node:
+        logger.info(f"{padding(depth)}{LOGGER_PREFIX} window node parents unresolvable")
+        return None
+    parent_node.resolve()
+    if not all(
+        [
+            x.address in [y.address for y in parent_node.output_concepts]
+            for x in parent_concepts
+        ]
+    ):
+        missing = [
+            x
+            for x in parent_concepts
+            if x.address not in [y.address for y in parent_node.output_concepts]
+        ]
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} window node parents unresolvable, missing {missing}"
+        )
+        raise SyntaxError
+    _window_node = WindowNode(
+        input_concepts=parent_concepts + targets,
+        output_concepts=[concept] + additional_outputs + parent_concepts + targets,
+        environment=environment,
+        parents=[
+            parent_node,
+        ],
+        depth=depth,
+        preexisting_conditions=conditions.conditional if conditions else None,
+    )
+    _window_node.rebuild_cache()
+    _window_node.resolve()
+
+    window_node = WhereSafetyNode(
+        input_concepts=[concept] + additional_outputs + parent_concepts + targets,
+        output_concepts=[concept] + additional_outputs + parent_concepts + targets,
+        environment=environment,
+        parents=[_window_node],
+        preexisting_conditions=conditions.conditional if conditions else None,
+        grain=BuildGrain.from_concepts(
+            concepts=[concept] + additional_outputs + parent_concepts + targets,
+            environment=environment,
+        ),
+    )
+    if not non_equivalent_optional:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} no optional concepts, returning window node"
+        )
+        # prune outputs if we don't need join keys
+        window_node.set_output_concepts([concept] + additional_outputs + targets)
+        return window_node
+
+    missing_optional = [
+        x.address
+        for x in local_optional
+        if x.address not in window_node.output_concepts
+    ]
+
+    if not missing_optional:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} no extra enrichment needed for window node, has all of {[x.address for x in local_optional]}"
+        )
+        return window_node
+    logger.info(
+        f"{padding(depth)}{LOGGER_PREFIX} window node for {concept.address} requires enrichment, missing {missing_optional}, has {[x.address for x in window_node.output_concepts]}"
+    )
+
+    return gen_enrichment_node(
+        window_node,
+        join_keys=[
+            environment.concepts[c]
+            for c in BuildGrain.from_concepts(
+                concepts=targets, environment=environment
+            ).components
+        ],
+        local_optional=local_optional,
+        environment=environment,
+        g=g,
+        depth=depth,
+        source_concepts=source_concepts,
+        log_lambda=create_log_lambda(LOGGER_PREFIX, depth, logger),
+        history=history,
+        conditions=conditions,
+    )

trilogy/core/processing/nodes/README.md

@@ -0,0 +1,28 @@
+# Nodes
+
+Nodes are the initial logical planning unit for a query path. 
+
+A query will initially resolve recursively to nodes, which are a lightweight operatoror representation.
+
+(Nodes will then later be instantiated as QueryDatasources/Datasources, a more complete intermediate representation,
+before finally becoming CTEs; a complete simplified object that is ready to be rendered as SQL).
+
+## Union Nodes
+
+Union nodes attempt to logically resolve union bindings.
+
+For a union concept:
+
+``` sql
+select
+    a,
+    b,
+    c,
+    union(a,b),
+    union(b,c),
+    a.prop,
+    b.prop,
+    c.prop
+```
+
+We