pytrilogy 0.3.148__cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl

trilogy/core/processing/discovery_validation.py

@@ -0,0 +1,167 @@
+from collections import defaultdict
+from enum import Enum
+from typing import List
+
+from trilogy.core.models.build import (
+    BuildConcept,
+    BuildWhereClause,
+)
+from trilogy.core.models.build_environment import BuildEnvironment
+from trilogy.core.processing.nodes import (
+    StrategyNode,
+)
+from trilogy.core.processing.utility import (
+    get_disconnected_components,
+)
+
+
+class ValidationResult(Enum):
+    COMPLETE = 1
+    DISCONNECTED = 2
+    INCOMPLETE = 3
+    INCOMPLETE_CONDITION = 4
+
+
+def validate_concept(
+    concept: BuildConcept,
+    node: StrategyNode,
+    found_addresses: set[str],
+    non_partial_addresses: set[str],
+    partial_addresses: set[str],
+    virtual_addresses: set[str],
+    found_map: dict[str, set[BuildConcept]],
+    accept_partial: bool,
+    seen: set[str],
+    environment: BuildEnvironment,
+):
+    # logger.debug(
+    #     f"Validating concept {concept.address} with accept_partial={accept_partial}"
+    # )
+    found_map[str(node)].add(concept)
+    seen.add(concept.address)
+    if concept not in node.partial_concepts:
+        found_addresses.add(concept.address)
+        non_partial_addresses.add(concept.address)
+        # remove it from our partial tracking
+        if concept.address in partial_addresses:
+            partial_addresses.remove(concept.address)
+        if concept.address in virtual_addresses:
+            virtual_addresses.remove(concept.address)
+    if concept in node.partial_concepts:
+        if concept.address in non_partial_addresses:
+            return None
+        partial_addresses.add(concept.address)
+        if accept_partial:
+            found_addresses.add(concept.address)
+            found_map[str(node)].add(concept)
+    for v_address in concept.pseudonyms:
+        if v_address in seen:
+            continue
+        if v_address in environment.alias_origin_lookup:
+            # logger.debug(
+            #     f"Found alias origin for {v_address}: {environment.alias_origin_lookup[v_address]} mapped to {environment.concepts[v_address]}")
+            v = environment.alias_origin_lookup[v_address]
+        else:
+            v = environment.concepts[v_address]
+
+        if v.address in seen:
+
+            continue
+
+        if v.address == concept.address:
+
+            continue
+        validate_concept(
+            v,
+            node,
+            found_addresses,
+            non_partial_addresses,
+            partial_addresses,
+            virtual_addresses,
+            found_map,
+            accept_partial,
+            seen=seen,
+            environment=environment,
+        )
+
+
+def validate_stack(
+    environment: BuildEnvironment,
+    stack: List[StrategyNode],
+    concepts: List[BuildConcept],
+    mandatory_with_filter: List[BuildConcept],
+    conditions: BuildWhereClause | None = None,
+    accept_partial: bool = False,
+) -> tuple[ValidationResult, set[str], set[str], set[str], set[str]]:
+    found_map: dict[str, set[BuildConcept]] = defaultdict(set)
+    found_addresses: set[str] = set()
+    non_partial_addresses: set[str] = set()
+    partial_addresses: set[str] = set()
+    virtual_addresses: set[str] = set()
+    seen: set[str] = set()
+
+    for node in stack:
+        resolved = node.resolve()
+
+        for concept in resolved.output_concepts:
+            if concept.address in resolved.hidden_concepts:
+                continue
+
+            validate_concept(
+                concept,
+                node,
+                found_addresses,
+                non_partial_addresses,
+                partial_addresses,
+                virtual_addresses,
+                found_map,
+                accept_partial,
+                seen,
+                environment,
+            )
+        for concept in node.virtual_output_concepts:
+            if concept.address in non_partial_addresses:
+                continue
+            found_addresses.add(concept.address)
+            virtual_addresses.add(concept.address)
+    if not conditions:
+        conditions_met = True
+    else:
+        conditions_met = all(
+            [node.preexisting_conditions == conditions.conditional for node in stack]
+        ) or all([c.address in found_addresses for c in mandatory_with_filter])
+    # zip in those we know we found
+    if not all([c.address in found_addresses for c in concepts]) or not conditions_met:
+        if not all([c.address in found_addresses for c in concepts]):
+            return (
+                ValidationResult.INCOMPLETE,
+                found_addresses,
+                {c.address for c in concepts if c.address not in found_addresses},
+                partial_addresses,
+                virtual_addresses,
+            )
+        return (
+            ValidationResult.INCOMPLETE_CONDITION,
+            found_addresses,
+            {c.address for c in concepts if c.address not in mandatory_with_filter},
+            partial_addresses,
+            virtual_addresses,
+        )
+
+    graph_count, _ = get_disconnected_components(found_map)
+    if graph_count in (0, 1):
+        return (
+            ValidationResult.COMPLETE,
+            found_addresses,
+            set(),
+            partial_addresses,
+            virtual_addresses,
+        )
+    # if we have too many subgraphs, we need to keep searching
+    return (
+        ValidationResult.DISCONNECTED,
+        found_addresses,
+        set(),
+        partial_addresses,
+        virtual_addresses,
+    )

trilogy/core/processing/graph_utils.py

@@ -0,0 +1,43 @@
+from collections import defaultdict
+from typing import Dict, List
+
+from trilogy.core.models.author import Concept
+from trilogy.utility import unique
+
+
+def extract_required_subgraphs(
+    assocs: defaultdict[str, list], path: List[str]
+) -> defaultdict[str, list]:
+    ds = path[0]
+    current: list[str] = []
+    for idx, val in enumerate(path):
+        if val.startswith("ds~"):
+            if current:
+                assocs[ds] += current
+                current = [path[idx - 1]] if idx > 0 else []
+            ds = val
+        else:
+            current.append(val)
+    else:
+        if current:
+            assocs[ds] += current
+
+    return assocs
+
+
+def extract_mandatory_subgraphs(paths: Dict[str, List[str]], g) -> List[List[Concept]]:
+    final: list[list[str]] = []
+    assocs: defaultdict[str, list] = defaultdict(list)
+    for path in paths.values():
+        extract_required_subgraphs(assocs, path)
+
+    for _, v in assocs.items():
+        final.append(v)
+    final_concepts = []
+    for value in final:
+        final_concepts.append(
+            unique(
+                [g.nodes[v]["concept"] for v in value if v.startswith("c~")], "address"
+            )
+        )
+    return final_concepts

trilogy/core/processing/node_generators/README.md

@@ -0,0 +1,9 @@
+
+
+
+For any specialized node, unpack the specialized concept X and fetch the rest.
+
+For unspecialized node, attempt to fetch concept X and the rest. If cannot, attempt to see 
+if all combinations of others + X can be found, and return a merge node with all of those.
+
+If not all combinations can be found, return what can be found.

trilogy/core/processing/node_generators/__init__.py

@@ -0,0 +1,31 @@
+from .basic_node import gen_basic_node
+from .constant_node import gen_constant_node
+from .filter_node import gen_filter_node
+from .group_node import gen_group_node
+from .group_to_node import gen_group_to_node
+from .multiselect_node import gen_multiselect_node
+from .node_merge_node import gen_merge_node
+from .recursive_node import gen_recursive_node
+from .rowset_node import gen_rowset_node
+from .select_node import gen_select_node
+from .synonym_node import gen_synonym_node
+from .union_node import gen_union_node
+from .unnest_node import gen_unnest_node
+from .window_node import gen_window_node
+
+__all__ = [
+    "gen_filter_node",
+    "gen_window_node",
+    "gen_group_node",
+    "gen_select_node",
+    "gen_basic_node",
+    "gen_unnest_node",
+    "gen_union_node",
+    "gen_merge_node",
+    "gen_group_to_node",
+    "gen_rowset_node",
+    "gen_multiselect_node",
+    "gen_synonym_node",
+    "gen_recursive_node",
+    "gen_constant_node",
+]

trilogy/core/processing/node_generators/basic_node.py

@@ -0,0 +1,160 @@
+from typing import List
+
+from trilogy.constants import logger
+from trilogy.core.enums import FunctionClass, FunctionType, SourceType
+from trilogy.core.models.build import BuildConcept, BuildFunction, BuildWhereClause
+from trilogy.core.models.build_environment import BuildEnvironment
+from trilogy.core.processing.node_generators.common import (
+    resolve_function_parent_concepts,
+)
+from trilogy.core.processing.nodes import ConstantNode, History, StrategyNode
+from trilogy.utility import unique
+
+LOGGER_PREFIX = "[GEN_BASIC_NODE]"
+
+
+def is_equivalent_basic_function_lineage(
+    x: BuildConcept,
+    y: BuildConcept,
+):
+    if not isinstance(x.lineage, BuildFunction) or not isinstance(
+        y.lineage, BuildFunction
+    ):
+        return False
+    if x.lineage.operator == y.lineage.operator == FunctionType.ATTR_ACCESS:
+        return x.lineage.concept_arguments == y.lineage.concept_arguments
+    if x.lineage.operator == y.lineage.operator:
+        return True
+    if (
+        y.lineage.operator in FunctionClass.AGGREGATE_FUNCTIONS.value
+        or y.lineage.operator in FunctionClass.ONE_TO_MANY.value
+    ):
+        return False
+    return True
+
+
+def gen_basic_node(
+    concept: BuildConcept,
+    local_optional: List[BuildConcept],
+    environment: BuildEnvironment,
+    g,
+    depth: int,
+    source_concepts,
+    history: History | None = None,
+    conditions: BuildWhereClause | None = None,
+):
+    depth_prefix = "\t" * depth
+    parent_concepts = resolve_function_parent_concepts(concept, environment=environment)
+
+    logger.info(
+        f"{depth_prefix}{LOGGER_PREFIX} basic node for {concept} with lineage {concept.lineage} has parents {[x for x in parent_concepts]}"
+    )
+    synonyms: list[BuildConcept] = []
+    ignored_optional: set[str] = set()
+
+    # when we are getting an attribute, if there is anything else
+    # that is an attribute of the same struct in local optional
+    # select that value for discovery as well
+    if (
+        isinstance(concept.lineage, BuildFunction)
+        and concept.lineage.operator == FunctionType.ATTR_ACCESS
+    ):
+        logger.info(
+            f"{depth_prefix}{LOGGER_PREFIX} checking for synonyms for attribute access"
+        )
+        for x in local_optional:
+            found = False
+            for z in x.pseudonyms:
+                # gate to ensure we don't match to multiple synonyms
+                if found:
+                    continue
+                if z in environment.concepts:
+                    s_concept = environment.concepts[z]
+                else:
+                    s_concept = environment.alias_origin_lookup[z]
+                if is_equivalent_basic_function_lineage(concept, s_concept):
+                    found = True
+                    synonyms.append(s_concept)
+                    ignored_optional.add(x.address)
+    equivalent_optional = [
+        x
+        for x in local_optional
+        if is_equivalent_basic_function_lineage(concept, x)
+        and x.address != concept.address
+    ] + synonyms
+
+    if equivalent_optional:
+        logger.info(
+            f"{depth_prefix}{LOGGER_PREFIX} basic node for {concept} has equivalent optional {[x.address for x in equivalent_optional]}"
+        )
+    for eo in equivalent_optional:
+        new_parents = resolve_function_parent_concepts(eo, environment=environment)
+        logger.info(
+            f"{depth_prefix}{LOGGER_PREFIX} equivalent optional {eo.address} has parents {[x.address for x in new_parents]}"
+        )
+        parent_concepts += new_parents
+    non_equivalent_optional = [
+        x
+        for x in local_optional
+        if x not in equivalent_optional
+        and not any(x.address in y.pseudonyms for y in equivalent_optional)
+        and x.address not in ignored_optional
+    ]
+    logger.info(
+        f"{depth_prefix}{LOGGER_PREFIX} basic node for {concept} has non-equivalent optional {[x.address for x in non_equivalent_optional]}"
+    )
+    all_parents: list[BuildConcept] = unique(
+        parent_concepts + non_equivalent_optional, "address"
+    )
+    logger.info(
+        f"{depth_prefix}{LOGGER_PREFIX} Fetching parents {[x.address for x in all_parents]} with conditions {conditions}"
+    )
+    if all_parents:
+        parent_node: StrategyNode | None = source_concepts(
+            mandatory_list=all_parents,
+            environment=environment,
+            g=g,
+            depth=depth + 1,
+            history=history,
+            conditions=conditions,
+        )
+
+        if not parent_node:
+            logger.info(
+                f"{depth_prefix}{LOGGER_PREFIX} No basic node could be generated for {concept}"
+            )
+            return None
+    else:
+        return ConstantNode(
+            input_concepts=[],
+            output_concepts=[concept],
+            environment=environment,
+            depth=depth,
+        )
+    if parent_node.source_type != SourceType.CONSTANT:
+        parent_node.source_type = SourceType.BASIC
+    parent_node.add_output_concept(concept)
+    for x in equivalent_optional:
+        parent_node.add_output_concept(x)
+
+    logger.info(
+        f"{depth_prefix}{LOGGER_PREFIX} Returning basic select for {concept}: output {[x.address for x in parent_node.output_concepts]}"
+    )
+    # if it's a constant, don't prune outputs
+    if parent_node.source_type == SourceType.CONSTANT:
+        return parent_node
+    targets = [concept] + local_optional + equivalent_optional
+    targets = [
+        s
+        for s in parent_node.output_concepts
+        if any(s.address in y.pseudonyms for y in targets)
+    ] + targets
+    hidden = [x for x in parent_node.output_concepts if x.address not in targets]
+    parent_node.hide_output_concepts(hidden)
+    parent_node.source_type = SourceType.BASIC
+
+    logger.info(
+        f"{depth_prefix}{LOGGER_PREFIX} Returning basic select for {concept}: input: {[x.address for x in parent_node.input_concepts]} output {[x.address for x in parent_node.output_concepts]} hidden {[x for x in parent_node.hidden_concepts]}"
+    )
+
+    return parent_node

trilogy/core/processing/node_generators/common.py

@@ -0,0 +1,270 @@
+from collections import defaultdict
+from typing import Callable, List, Tuple
+
+from trilogy.core.enums import Derivation, Purpose
+from trilogy.core.models.build import (
+    BuildAggregateWrapper,
+    BuildComparison,
+    BuildConcept,
+    BuildFilterItem,
+    BuildFunction,
+    BuildWhereClause,
+    LooseBuildConceptList,
+)
+from trilogy.core.models.build_environment import BuildEnvironment
+from trilogy.core.processing.nodes import (
+    History,
+    NodeJoin,
+)
+from trilogy.core.processing.nodes.base_node import StrategyNode
+from trilogy.core.processing.nodes.merge_node import MergeNode
+from trilogy.utility import unique
+
+AGGREGATE_TYPES = (BuildAggregateWrapper,)
+FUNCTION_TYPES = (BuildFunction,)
+
+
+def resolve_function_parent_concepts(
+    concept: BuildConcept, environment: BuildEnvironment
+) -> List[BuildConcept]:
+    if not isinstance(
+        concept.lineage, (*FUNCTION_TYPES, *AGGREGATE_TYPES, BuildComparison)
+    ):
+        raise ValueError(
+            f"Concept {concept} lineage is not function or aggregate, is {type(concept.lineage)}"
+        )
+    if concept.derivation == Derivation.AGGREGATE:
+        base: list[BuildConcept] = []
+        if not concept.grain.abstract:
+            base = concept.lineage.concept_arguments + [
+                environment.concepts[c] for c in concept.grain.components
+            ]
+            # if the base concept being aggregated is a property with a key
+            # keep the key as a parent
+        else:
+            base = concept.lineage.concept_arguments
+        if isinstance(concept.lineage, AGGREGATE_TYPES):
+            # for aggregate wrapper, don't include the by
+            extra_property_grain = concept.lineage.function.concept_arguments
+        else:
+            extra_property_grain = concept.lineage.concept_arguments
+        for x in extra_property_grain:
+            if isinstance(x, BuildConcept) and x.purpose == Purpose.PROPERTY and x.keys:
+                base += [environment.concepts[c] for c in x.keys]
+        return unique(base, "address")
+    # TODO: handle basic lineage chains?
+    return unique(concept.lineage.concept_arguments, "address")
+
+
+def resolve_condition_parent_concepts(
+    condition: BuildWhereClause,
+) -> Tuple[List[BuildConcept], List[Tuple[BuildConcept, ...]]]:
+    base_existence = []
+    base_rows: list[BuildConcept] = []
+    base_rows += condition.row_arguments
+    for ctuple in condition.existence_arguments:
+        base_existence.append(ctuple)
+    return unique(base_rows, "address"), base_existence
+
+
+def resolve_filter_parent_concepts(
+    concept: BuildConcept,
+    environment: BuildEnvironment,
+) -> Tuple[List[BuildConcept], List[Tuple[BuildConcept, ...]]]:
+    if not isinstance(concept.lineage, (BuildFilterItem,)):
+        raise ValueError(
+            f"Concept {concept} lineage is not filter item, is {type(concept.lineage)}"
+        )
+    direct_parent = concept.lineage.content
+    base_existence = []
+    base_rows = [direct_parent] if isinstance(direct_parent, BuildConcept) else []
+    condition_rows, condition_existence = resolve_condition_parent_concepts(
+        concept.lineage.where
+    )
+    base_rows += condition_rows
+    base_existence += condition_existence
+    # this is required so that
+    if (
+        isinstance(direct_parent, BuildConcept)
+        and direct_parent.purpose in (Purpose.PROPERTY, Purpose.METRIC)
+        and direct_parent.keys
+    ):
+        base_rows += [environment.concepts[c] for c in direct_parent.keys]
+
+    if concept.lineage.where.existence_arguments:
+        return (
+            unique(base_rows, "address"),
+            base_existence,
+        )
+    return unique(base_rows, "address"), []
+
+
+def gen_property_enrichment_node(
+    base_node: StrategyNode,
+    extra_properties: list[BuildConcept],
+    history: History,
+    environment: BuildEnvironment,
+    g,
+    depth: int,
+    source_concepts,
+    log_lambda: Callable,
+    conditions: BuildWhereClause | None = None,
+):
+    required_keys: dict[str, set[str]] = defaultdict(set)
+    for x in extra_properties:
+        if not x.keys:
+            raise SyntaxError(f"Property {x.address} missing keys in lookup")
+        keys = "-".join([y for y in x.keys])
+        required_keys[keys].add(x.address)
+    final_nodes = []
+    for _k, vs in required_keys.items():
+        log_lambda(f"Generating enrichment node for {_k} with {vs}")
+        ks = _k.split("-")
+        enrich_node: StrategyNode = source_concepts(
+            mandatory_list=[environment.concepts[k] for k in ks]
+            + [environment.concepts[v] for v in vs],
+            environment=environment,
+            g=g,
+            depth=depth + 1,
+            history=history,
+            conditions=conditions,
+        )
+        if not enrich_node:
+            return None
+        final_nodes.append(enrich_node)
+    return MergeNode(
+        input_concepts=unique(
+            base_node.output_concepts
+            + extra_properties
+            + [
+                environment.concepts[v]
+                for k, values in required_keys.items()
+                for v in values
+            ],
+            "address",
+        ),
+        output_concepts=base_node.output_concepts + extra_properties,
+        environment=environment,
+        parents=[
+            base_node,
+        ]
+        + final_nodes,
+        preexisting_conditions=conditions.conditional if conditions else None,
+    )
+
+
+def gen_enrichment_node(
+    base_node: StrategyNode,
+    join_keys: List[BuildConcept],
+    local_optional: list[BuildConcept],
+    environment: BuildEnvironment,
+    g,
+    depth: int,
+    source_concepts,
+    log_lambda,
+    history: History,
+    conditions: BuildWhereClause | None = None,
+):
+    local_opts = LooseBuildConceptList(concepts=local_optional)
+
+    extra_required = [
+        x
+        for x in local_opts
+        if x not in base_node.output_lcl or x in base_node.partial_lcl
+    ]
+
+    # property lookup optimization
+    # this helps create ergonomic merge nodes when evaluating a normalized star schema
+    # as we only want to lookup the missing properties based on the relevant keys
+    if all([x.purpose == Purpose.PROPERTY for x in extra_required]):
+        if all(
+            x.keys and all([key in base_node.output_lcl for key in x.keys])
+            for x in extra_required
+        ):
+            log_lambda(
+                f"{str(type(base_node).__name__)} returning property optimized enrichment node for {extra_required[0].keys}"
+            )
+            return gen_property_enrichment_node(
+                base_node,
+                extra_required,
+                environment=environment,
+                g=g,
+                depth=depth,
+                source_concepts=source_concepts,
+                history=history,
+                conditions=conditions,
+                log_lambda=log_lambda,
+            )
+    log_lambda(
+        f"{str(type(base_node).__name__)} searching for join keys {LooseBuildConceptList(concepts=join_keys)} and extra required {local_opts}"
+    )
+    enrich_node: StrategyNode = source_concepts(  # this fetches the parent + join keys
+        # to then connect to the rest of the query
+        mandatory_list=join_keys + extra_required,
+        environment=environment,
+        g=g,
+        depth=depth + 1,
+        history=history,
+        conditions=conditions,
+    )
+    if not enrich_node:
+        log_lambda(
+            f"{str(type(base_node).__name__)} enrichment node unresolvable, returning just group node"
+        )
+        return base_node
+    log_lambda(
+        f"{str(type(base_node).__name__)} returning merge node with group node + enrichment node"
+    )
+    non_hidden = [
+        x
+        for x in base_node.output_concepts
+        if x.address not in base_node.hidden_concepts
+    ]
+    return MergeNode(
+        input_concepts=unique(join_keys + extra_required + non_hidden, "address"),
+        output_concepts=unique(join_keys + extra_required + non_hidden, "address"),
+        environment=environment,
+        parents=[enrich_node, base_node],
+        force_group=False,
+        preexisting_conditions=conditions.conditional if conditions else None,
+        depth=depth,
+    )
+
+
+def resolve_join_order(joins: List[NodeJoin]) -> List[NodeJoin]:
+    if not joins:
+        return []
+    available_aliases: set[StrategyNode] = set()
+    final_joins_pre = [*joins]
+    final_joins = []
+    left = set()
+    right = set()
+    for join in joins:
+        left.add(join.left_node)
+        right.add(join.right_node)
+
+    potential_basis = left.difference(right)
+    base_candidates = [x for x in final_joins_pre if x.left_node in potential_basis]
+    if not base_candidates:
+        raise SyntaxError(
+            f"Unresolvable join dependencies, left requires {left} and right requires {right}"
+        )
+    base = base_candidates[0]
+    final_joins.append(base)
+    available_aliases.add(base.left_node)
+    available_aliases.add(base.right_node)
+    while final_joins_pre:
+        new_final_joins_pre: List[NodeJoin] = []
+        for join in final_joins_pre:
+            if join.left_node in available_aliases:
+                # we don't need to join twice
+                # so whatever join we found first, works
+                if join.right_node in available_aliases:
+                    continue
+                final_joins.append(join)
+                available_aliases.add(join.left_node)
+                available_aliases.add(join.right_node)
+            else:
+                new_final_joins_pre.append(join)
+        final_joins_pre = new_final_joins_pre
+    return final_joins