pytrilogy 0.0.1.102__py3-none-any.whl

trilogy/core/processing/node_generators/basic_node.py

@@ -0,0 +1,71 @@
+# directly select out a basic derivation
+from typing import List
+
+from trilogy.core.models import (
+    Concept,
+)
+from trilogy.core.processing.nodes import StrategyNode, History, MergeNode
+from trilogy.core.processing.node_generators.common import (
+    resolve_function_parent_concepts,
+)
+from trilogy.constants import logger
+
+LOGGER_PREFIX = "[GEN_BASIC_NODE]"
+
+
+def gen_basic_node(
+    concept: Concept,
+    local_optional: List[Concept],
+    environment,
+    g,
+    depth: int,
+    source_concepts,
+    history: History | None = None,
+):
+    depth_prefix = "\t" * depth
+    parent_concepts = resolve_function_parent_concepts(concept)
+
+    logger.info(
+        f"{depth_prefix}{LOGGER_PREFIX} basic node for {concept} has parents {[x.address for x in parent_concepts]}"
+    )
+
+    output_concepts = [concept] + local_optional
+    partials = []
+
+    attempts = [(parent_concepts, [concept])]
+    if local_optional:
+        attempts.append((parent_concepts + local_optional, local_optional + [concept]))
+
+    for attempt, output in reversed(attempts):
+        parent_node = source_concepts(
+            mandatory_list=attempt,
+            environment=environment,
+            g=g,
+            depth=depth + 1,
+            history=history,
+        )
+        if not parent_node:
+            continue
+        parents: List[StrategyNode] = [parent_node]
+        for x in output_concepts:
+            sources = [p for p in parents if x in p.output_concepts]
+            if not sources:
+                continue
+            if all(x in source.partial_concepts for source in sources):
+                partials.append(x)
+        logger.info(
+            f"{depth_prefix}{LOGGER_PREFIX} Returning basic select for {concept} with attempted extra {[x.address for x in attempt]}"
+        )
+        return MergeNode(
+            input_concepts=attempt,
+            output_concepts=output,
+            environment=environment,
+            g=g,
+            parents=parents,
+            depth=depth,
+            partial_concepts=partials,
+        )
+    logger.info(
+        f"{depth_prefix}{LOGGER_PREFIX} No basic node could be generated for {concept}"
+    )
+    return None

trilogy/core/processing/node_generators/common.py

@@ -0,0 +1,239 @@
+from typing import List, Tuple
+
+
+from trilogy.core.enums import PurposeLineage, Purpose
+from trilogy.core.models import (
+    Concept,
+    Function,
+    AggregateWrapper,
+    FilterItem,
+    Environment,
+    LooseConceptList,
+)
+from trilogy.utility import unique
+from trilogy.core.processing.nodes.base_node import StrategyNode
+from trilogy.core.processing.nodes.merge_node import MergeNode
+from trilogy.core.processing.nodes import History
+from trilogy.core.enums import JoinType
+from trilogy.core.processing.nodes import (
+    NodeJoin,
+)
+from collections import defaultdict
+from trilogy.core.processing.utility import concept_to_relevant_joins
+
+
+def resolve_function_parent_concepts(concept: Concept) -> List[Concept]:
+    if not isinstance(concept.lineage, (Function, AggregateWrapper)):
+        raise ValueError(f"Concept {concept} lineage is not function or aggregate")
+    if concept.derivation == PurposeLineage.AGGREGATE:
+        if not concept.grain.abstract:
+            base = concept.lineage.concept_arguments + concept.grain.components_copy
+            # if the base concept being aggregated is a property with a key
+            # keep the key as a parent
+        else:
+            base = concept.lineage.concept_arguments
+        if isinstance(concept.lineage, AggregateWrapper):
+            # for aggregate wrapper, don't include the by
+            extra_property_grain = concept.lineage.function.concept_arguments
+        else:
+            extra_property_grain = concept.lineage.concept_arguments
+        for x in extra_property_grain:
+            if isinstance(x, Concept) and x.purpose == Purpose.PROPERTY and x.keys:
+                base += x.keys
+        return unique(base, "address")
+    # TODO: handle basic lineage chains?
+    return unique(concept.lineage.concept_arguments, "address")
+
+
+def resolve_filter_parent_concepts(concept: Concept) -> Tuple[Concept, List[Concept]]:
+    if not isinstance(concept.lineage, FilterItem):
+        raise ValueError
+    direct_parent = concept.lineage.content
+    base = [direct_parent]
+    base += concept.lineage.where.concept_arguments
+    if direct_parent.grain:
+        base += direct_parent.grain.components_copy
+    if (
+        isinstance(direct_parent, Concept)
+        and direct_parent.purpose == Purpose.PROPERTY
+        and direct_parent.keys
+    ):
+        base += direct_parent.keys
+    return concept.lineage.content, unique(base, "address")
+
+
+def gen_property_enrichment_node(
+    base_node: StrategyNode,
+    extra_properties: list[Concept],
+    environment: Environment,
+    g,
+    depth: int,
+    source_concepts,
+    history: History | None = None,
+):
+    required_keys: dict[str, set[str]] = defaultdict(set)
+    for x in extra_properties:
+        if not x.keys:
+            raise SyntaxError(f"Property {x.address} missing keys in lookup")
+        keys = "-".join([y.address for y in x.keys])
+        required_keys[keys].add(x.address)
+    final_nodes = []
+    node_joins = []
+    for _k, vs in required_keys.items():
+        ks = _k.split("-")
+        enrich_node: StrategyNode = source_concepts(
+            mandatory_list=[environment.concepts[k] for k in ks]
+            + [environment.concepts[v] for v in vs],
+            environment=environment,
+            g=g,
+            depth=depth + 1,
+            history=history,
+        )
+        final_nodes.append(enrich_node)
+        node_joins.append(
+            NodeJoin(
+                left_node=enrich_node,
+                right_node=base_node,
+                concepts=concept_to_relevant_joins(
+                    [environment.concepts[k] for k in ks]
+                ),
+                filter_to_mutual=False,
+                join_type=JoinType.LEFT_OUTER,
+            )
+        )
+    return MergeNode(
+        input_concepts=unique(
+            base_node.output_concepts
+            + extra_properties
+            + [
+                environment.concepts[v]
+                for k, values in required_keys.items()
+                for v in values
+            ],
+            "address",
+        ),
+        output_concepts=base_node.output_concepts + extra_properties,
+        environment=environment,
+        g=g,
+        parents=[
+            base_node,
+            enrich_node,
+        ],
+        node_joins=node_joins,
+    )
+
+
+def gen_enrichment_node(
+    base_node: StrategyNode,
+    join_keys: List[Concept],
+    local_optional: list[Concept],
+    environment: Environment,
+    g,
+    depth: int,
+    source_concepts,
+    log_lambda,
+    history: History | None = None,
+):
+
+    local_opts = LooseConceptList(concepts=local_optional)
+
+    if local_opts.issubset(LooseConceptList(concepts=base_node.output_concepts)):
+        log_lambda(
+            f"{str(type(base_node).__name__)} has all optional { base_node.output_lcl}, skipping enrichmennt"
+        )
+        return base_node
+    extra_required = [
+        x
+        for x in local_opts
+        if x not in base_node.output_lcl or x in base_node.partial_lcl
+    ]
+
+    # property lookup optimization
+    # this helps when evaluating a normalized star schema as you only want to lookup the missing properties based on the relevant keys
+    if all([x.purpose == Purpose.PROPERTY for x in extra_required]):
+        if all(
+            x.keys and all([key in base_node.output_lcl for key in x.keys])
+            for x in extra_required
+        ):
+            log_lambda(
+                f"{str(type(base_node).__name__)} returning property optimized enrichment node"
+            )
+            return gen_property_enrichment_node(
+                base_node,
+                extra_required,
+                environment,
+                g,
+                depth,
+                source_concepts,
+                history=history,
+            )
+
+    enrich_node: StrategyNode = source_concepts(  # this fetches the parent + join keys
+        # to then connect to the rest of the query
+        mandatory_list=join_keys + extra_required,
+        environment=environment,
+        g=g,
+        depth=depth,
+        history=history,
+    )
+    if not enrich_node:
+        log_lambda(
+            f"{str(type(base_node).__name__)} enrichment node unresolvable, returning just group node"
+        )
+        return base_node
+    log_lambda(
+        f"{str(type(base_node).__name__)} returning merge node with group node + enrichment node"
+    )
+    return MergeNode(
+        input_concepts=unique(
+            join_keys + extra_required + base_node.output_concepts, "address"
+        ),
+        output_concepts=unique(
+            join_keys + extra_required + base_node.output_concepts, "address"
+        ),
+        environment=environment,
+        g=g,
+        parents=[enrich_node, base_node],
+        node_joins=[
+            NodeJoin(
+                left_node=enrich_node,
+                right_node=base_node,
+                concepts=concept_to_relevant_joins(
+                    [x for x in join_keys if x in enrich_node.output_lcl]
+                ),
+                filter_to_mutual=False,
+                join_type=JoinType.LEFT_OUTER,
+            )
+        ],
+    )
+
+
+def resolve_join_order(joins: List[NodeJoin]) -> List[NodeJoin]:
+    available_aliases: set[StrategyNode] = set()
+    final_joins_pre = [*joins]
+    final_joins = []
+    while final_joins_pre:
+        new_final_joins_pre: List[NodeJoin] = []
+        for join in final_joins_pre:
+            if not available_aliases:
+                final_joins.append(join)
+                available_aliases.add(join.left_node)
+                available_aliases.add(join.right_node)
+            elif join.left_node in available_aliases:
+                # we don't need to join twice
+                # so whatever join we found first, works
+                if join.right_node in available_aliases:
+                    continue
+                final_joins.append(join)
+                available_aliases.add(join.left_node)
+                available_aliases.add(join.right_node)
+            else:
+                new_final_joins_pre.append(join)
+        if len(new_final_joins_pre) == len(final_joins_pre):
+            remaining = [join.left_node for join in new_final_joins_pre]
+            remaining_right = [join.right_node for join in new_final_joins_pre]
+            raise SyntaxError(
+                f"did not find any new joins, available {available_aliases} remaining is {remaining + remaining_right} "
+            )
+        final_joins_pre = new_final_joins_pre
+    return final_joins

trilogy/core/processing/node_generators/concept_merge.py

@@ -0,0 +1,152 @@
+from trilogy.core.models import (
+    Concept,
+    Environment,
+    MergeStatement,
+)
+from trilogy.core.processing.nodes import MergeNode, NodeJoin, History
+from trilogy.core.processing.nodes.base_node import concept_list_to_grain, StrategyNode
+from typing import List
+
+from trilogy.core.enums import JoinType
+from trilogy.constants import logger
+from trilogy.core.processing.utility import padding
+from trilogy.core.processing.node_generators.common import concept_to_relevant_joins
+from itertools import combinations
+from trilogy.core.processing.node_generators.common import resolve_join_order
+
+LOGGER_PREFIX = "[GEN_CONCEPT_MERGE_NODE]"
+
+
+def merge_joins(base: MergeStatement, parents: List[StrategyNode]) -> List[NodeJoin]:
+    output = []
+    for left, right in combinations(parents, 2):
+        output.append(
+            NodeJoin(
+                left_node=left,
+                right_node=right,
+                concepts=[
+                    base.merge_concept,
+                ],
+                join_type=JoinType.FULL,
+            )
+        )
+    return resolve_join_order(output)
+
+
+def gen_concept_merge_node(
+    concept: Concept,
+    local_optional: List[Concept],
+    environment: Environment,
+    g,
+    depth: int,
+    source_concepts,
+    history: History | None = None,
+) -> MergeNode | None:
+    if not isinstance(concept.lineage, MergeStatement):
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} Cannot generate merge node for {concept}"
+        )
+        return None
+    lineage: MergeStatement = concept.lineage
+
+    base_parents: List[StrategyNode] = []
+    for select in lineage.concepts:
+        # if it's a merge concept, filter it out of the optional
+        sub_optional = [
+            x
+            for x in local_optional
+            if x.address not in lineage.concepts_lcl and x.namespace == select.namespace
+        ]
+        snode: StrategyNode = source_concepts(
+            mandatory_list=[select] + sub_optional,
+            environment=environment,
+            g=g,
+            depth=depth + 1,
+            history=history,
+        )
+        if not snode:
+            logger.info(
+                f"{padding(depth)}{LOGGER_PREFIX} Cannot generate merge node for {concept}"
+            )
+            return None
+        snode.add_output_concept(lineage.merge_concept)
+        base_parents.append(snode)
+
+    node_joins = merge_joins(lineage, base_parents)
+
+    enrichment = set([x.address for x in local_optional])
+    outputs = [x for y in base_parents for x in y.output_concepts]
+
+    additional_relevant = [x for x in outputs if x.address in enrichment]
+    node = MergeNode(
+        input_concepts=[x for y in base_parents for x in y.output_concepts],
+        output_concepts=outputs + additional_relevant + [concept],
+        environment=environment,
+        g=g,
+        depth=depth,
+        parents=base_parents,
+        node_joins=node_joins,
+    )
+
+    qds = node.rebuild_cache()
+
+    # assume grain to be outoput of select
+    # but don't include anything aggregate at this point
+    qds.grain = concept_list_to_grain(
+        node.output_concepts, parent_sources=qds.datasources
+    )
+    possible_joins = concept_to_relevant_joins(additional_relevant)
+    if not local_optional:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} no enriched required for merge concept node; exiting early"
+        )
+        return node
+    if not possible_joins:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} no possible joins for merge concept node; exiting early"
+        )
+        return node
+    if all(
+        [x.address in [y.address for y in node.output_concepts] for x in local_optional]
+    ):
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} all enriched concepts returned from base merge concept node; exiting early"
+        )
+        return node
+    enrich_node: MergeNode = source_concepts(  # this fetches the parent + join keys
+        # to then connect to the rest of the query
+        mandatory_list=additional_relevant + local_optional,
+        environment=environment,
+        g=g,
+        depth=depth + 1,
+        history=history,
+    )
+    if not enrich_node:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} Cannot generate merge concept enrichment node for {concept} with optional {local_optional}, returning just merge concept"
+        )
+        return node
+
+    return MergeNode(
+        input_concepts=enrich_node.output_concepts + node.output_concepts,
+        output_concepts=node.output_concepts + local_optional,
+        environment=environment,
+        g=g,
+        depth=depth,
+        parents=[
+            # this node gets the window
+            node,
+            # this node gets enrichment
+            enrich_node,
+        ],
+        node_joins=[
+            NodeJoin(
+                left_node=enrich_node,
+                right_node=node,
+                concepts=possible_joins,
+                filter_to_mutual=False,
+                join_type=JoinType.LEFT_OUTER,
+            )
+        ],
+        partial_concepts=node.partial_concepts,
+    )

trilogy/core/processing/node_generators/filter_node.py

@@ -0,0 +1,83 @@
+from typing import List
+
+
+from trilogy.core.enums import JoinType
+from trilogy.core.models import (
+    Concept,
+    Environment,
+)
+from trilogy.core.processing.nodes import FilterNode, MergeNode, NodeJoin, History
+from trilogy.core.processing.node_generators.common import (
+    resolve_filter_parent_concepts,
+)
+from trilogy.constants import logger
+from trilogy.core.processing.utility import padding
+from trilogy.core.processing.node_generators.common import concept_to_relevant_joins
+
+LOGGER_PREFIX = "[GEN_FILTER_NODE]"
+
+
+def gen_filter_node(
+    concept: Concept,
+    local_optional: List[Concept],
+    environment: Environment,
+    g,
+    depth: int,
+    source_concepts,
+    history: History | None = None,
+) -> MergeNode | FilterNode | None:
+    immediate_parent, parent_concepts = resolve_filter_parent_concepts(concept)
+
+    logger.info(f"{padding(depth)}{LOGGER_PREFIX} fetching filter node parents")
+    parent = source_concepts(
+        mandatory_list=parent_concepts,
+        environment=environment,
+        g=g,
+        depth=depth + 1,
+        history=history,
+    )
+    if not parent:
+        return None
+    filter_node = FilterNode(
+        input_concepts=[immediate_parent] + parent_concepts,
+        output_concepts=[concept, immediate_parent] + parent_concepts,
+        environment=environment,
+        g=g,
+        parents=[parent],
+    )
+    if not local_optional:
+        return filter_node
+    enrich_node = source_concepts(  # this fetches the parent + join keys
+        # to then connect to the rest of the query
+        mandatory_list=[immediate_parent] + parent_concepts + local_optional,
+        environment=environment,
+        g=g,
+        depth=depth + 1,
+        history=history,
+    )
+    x = MergeNode(
+        input_concepts=[concept, immediate_parent] + local_optional,
+        output_concepts=[
+            concept,
+        ]
+        + local_optional,
+        environment=environment,
+        g=g,
+        parents=[
+            # this node fetches only what we need to filter
+            filter_node,
+            enrich_node,
+        ],
+        node_joins=[
+            NodeJoin(
+                left_node=enrich_node,
+                right_node=filter_node,
+                concepts=concept_to_relevant_joins(
+                    [immediate_parent] + parent_concepts
+                ),
+                join_type=JoinType.LEFT_OUTER,
+                filter_to_mutual=False,
+            )
+        ],
+    )
+    return x

trilogy/core/processing/node_generators/group_node.py

@@ -0,0 +1,92 @@
+from trilogy.core.models import Concept, Environment, LooseConceptList
+from trilogy.utility import unique
+from trilogy.core.processing.nodes import GroupNode, StrategyNode, History
+from typing import List
+from trilogy.core.processing.node_generators.common import (
+    resolve_function_parent_concepts,
+)
+from trilogy.constants import logger
+from trilogy.core.processing.utility import padding, create_log_lambda
+from trilogy.core.processing.node_generators.common import (
+    gen_enrichment_node,
+)
+
+LOGGER_PREFIX = "[GEN_GROUP_NODE]"
+
+
+def gen_group_node(
+    concept: Concept,
+    local_optional: List[Concept],
+    environment: Environment,
+    g,
+    depth: int,
+    source_concepts,
+    history: History | None = None,
+):
+    # aggregates MUST always group to the proper grain
+    # except when the
+    parent_concepts: List[Concept] = unique(
+        resolve_function_parent_concepts(concept), "address"
+    )
+    logger.info(
+        f"{padding(depth)}{LOGGER_PREFIX} parent_concepts are {[x.address for x in parent_concepts]} from group grain {concept.grain}"
+    )
+
+    # if the aggregation has a grain, we need to ensure these are the ONLY optional in the output of the select
+    output_concepts = [concept]
+
+    if concept.grain and len(concept.grain.components_copy) > 0:
+        grain_components = (
+            concept.grain.components_copy if not concept.grain.abstract else []
+        )
+        parent_concepts += grain_components
+        output_concepts += grain_components
+
+    if parent_concepts:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} fetching group node parents {LooseConceptList(concepts=parent_concepts)}"
+        )
+        parent_concepts = unique(parent_concepts, "address")
+        parent = source_concepts(
+            mandatory_list=parent_concepts,
+            environment=environment,
+            g=g,
+            depth=depth,
+            history=history,
+        )
+        if not parent:
+            logger.info(
+                f"{padding(depth)}{LOGGER_PREFIX} group by node parents unresolvable"
+            )
+            return None
+        parents: List[StrategyNode] = [parent]
+    else:
+        parents = []
+
+    # the keys we group by
+    # are what we can use for enrichment
+    group_key_parents = concept.grain.components_copy
+
+    group_node = GroupNode(
+        output_concepts=output_concepts,
+        input_concepts=parent_concepts,
+        environment=environment,
+        g=g,
+        parents=parents,
+        depth=depth,
+    )
+
+    # early exit if no optional
+    if not local_optional:
+        return group_node
+    logger.info(f"{padding(depth)}{LOGGER_PREFIX} group node requires enrichment")
+    return gen_enrichment_node(
+        group_node,
+        join_keys=group_key_parents,
+        local_optional=local_optional,
+        environment=environment,
+        g=g,
+        depth=depth,
+        source_concepts=source_concepts,
+        log_lambda=create_log_lambda(LOGGER_PREFIX, depth, logger),
+    )