pytrilogy 0.3.138__cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl

trilogy/core/processing/node_generators/filter_node.py

@@ -0,0 +1,315 @@
+from typing import List
+
+from trilogy.constants import logger
+from trilogy.core.models.build import (
+    BuildConcept,
+    BuildFilterItem,
+    BuildWhereClause,
+)
+from trilogy.core.models.build_environment import BuildEnvironment
+from trilogy.core.processing.node_generators.common import (
+    resolve_filter_parent_concepts,
+)
+from trilogy.core.processing.nodes import (
+    FilterNode,
+    History,
+    MergeNode,
+    SelectNode,
+    StrategyNode,
+)
+from trilogy.core.processing.utility import is_scalar_condition, padding, unique
+
+LOGGER_PREFIX = "[GEN_FILTER_NODE]"
+
+FILTER_TYPES = (BuildFilterItem,)
+
+
+def pushdown_filter_to_parent(
+    local_optional: List[BuildConcept],
+    conditions: BuildWhereClause | None,
+    filter_where: BuildWhereClause,
+    same_filter_optional: list[BuildConcept],
+    depth: int,
+) -> bool:
+    optimized_pushdown = False
+    if not is_scalar_condition(filter_where.conditional):
+        optimized_pushdown = False
+    elif not local_optional:
+        optimized_pushdown = True
+    elif conditions and conditions == filter_where:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} query conditions are the same as filter conditions, can optimize across all concepts"
+        )
+        optimized_pushdown = True
+    elif same_filter_optional == local_optional:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} all optional concepts are included in the filter, can optimize across all concepts"
+        )
+        optimized_pushdown = True
+
+    return optimized_pushdown
+
+
+def build_parent_concepts(
+    concept: BuildConcept,
+    environment: BuildEnvironment,
+    local_optional: List[BuildConcept],
+    conditions: BuildWhereClause | None = None,
+    depth: int = 0,
+) -> tuple[
+    list[BuildConcept],
+    list[tuple[BuildConcept, ...]],
+    list[BuildConcept],
+    bool,
+    bool,
+]:
+    parent_row_concepts, parent_existence_concepts = resolve_filter_parent_concepts(
+        concept, environment
+    )
+    if not isinstance(concept.lineage, FILTER_TYPES):
+        raise SyntaxError('Filter node must have a filter type lineage"')
+    filter_where = concept.lineage.where
+
+    same_filter_optional: list[BuildConcept] = []
+    # mypy struggled here? we shouldn't need explicit bools
+    global_filter_is_local_filter: bool = (
+        True if (conditions and conditions == filter_where) else False
+    )
+
+    exact_partial_matches = True
+    for x in local_optional:
+        if isinstance(x.lineage, FILTER_TYPES):
+            if set([x.address for x in x.lineage.where.concept_arguments]) == set(
+                [x.address for x in filter_where.concept_arguments]
+            ):
+                exact_partial_matches = (
+                    exact_partial_matches and x.lineage.where == filter_where
+                )
+                logger.info(
+                    f"{padding(depth)}{LOGGER_PREFIX} fetching parents for peer {x.address} (of {concept.address})"
+                )
+
+                for arg in x.lineage.content_concept_arguments:
+                    if arg.address not in parent_row_concepts:
+                        parent_row_concepts.append(arg)
+                same_filter_optional.append(x)
+                continue
+        elif global_filter_is_local_filter:
+            same_filter_optional.append(x)
+            # also append it to the parent row concepts
+            parent_row_concepts.append(x)
+
+    # sometimes, it's okay to include other local optional above the filter
+    # in case it is, prep our list
+    extra_row_level_optional: list[BuildConcept] = []
+
+    for x in local_optional:
+        if x.address in same_filter_optional:
+            continue
+        extra_row_level_optional.append(x)
+    is_optimized_pushdown = exact_partial_matches and pushdown_filter_to_parent(
+        local_optional, conditions, filter_where, same_filter_optional, depth
+    )
+    if not is_optimized_pushdown:
+        parent_row_concepts += extra_row_level_optional
+    return (
+        parent_row_concepts,
+        parent_existence_concepts,
+        same_filter_optional,
+        is_optimized_pushdown,
+        global_filter_is_local_filter,
+    )
+
+
+def add_existence_sources(
+    core_parent_nodes: list[StrategyNode],
+    parent_existence_concepts: list[tuple[BuildConcept, ...]],
+    source_concepts,
+    environment,
+    g,
+    depth,
+    history,
+):
+    for existence_tuple in parent_existence_concepts:
+        if not existence_tuple:
+            continue
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} fetching filter node existence parents {[x.address for x in existence_tuple]}"
+        )
+        parent_existence = source_concepts(
+            mandatory_list=list(existence_tuple),
+            environment=environment,
+            g=g,
+            depth=depth + 1,
+            history=history,
+        )
+        if not parent_existence:
+            logger.info(
+                f"{padding(depth)}{LOGGER_PREFIX} filter existence node parents could not be found"
+            )
+            return None
+        core_parent_nodes.append(parent_existence)
+
+
+def gen_filter_node(
+    concept: BuildConcept,
+    local_optional: List[BuildConcept],
+    environment: BuildEnvironment,
+    g,
+    depth: int,
+    source_concepts,
+    history: History | None = None,
+    conditions: BuildWhereClause | None = None,
+) -> StrategyNode | None:
+    if not isinstance(concept.lineage, FILTER_TYPES):
+        raise SyntaxError('Filter node must have a filter type lineage"')
+    where = concept.lineage.where
+
+    (
+        parent_row_concepts,
+        parent_existence_concepts,
+        same_filter_optional,
+        optimized_pushdown,
+        global_filter_is_local_filter,
+    ) = build_parent_concepts(
+        concept,
+        environment=environment,
+        local_optional=local_optional,
+        conditions=conditions,
+        depth=depth,
+    )
+
+    row_parent: StrategyNode = source_concepts(
+        mandatory_list=parent_row_concepts,
+        environment=environment,
+        g=g,
+        depth=depth + 1,
+        history=history,
+        conditions=conditions,
+    )
+
+    core_parent_nodes: list[StrategyNode] = []
+    flattened_existence = [x for y in parent_existence_concepts for x in y]
+    if parent_existence_concepts:
+        add_existence_sources(
+            core_parent_nodes,
+            parent_existence_concepts,
+            source_concepts,
+            environment,
+            g,
+            depth,
+            history,
+        )
+
+    if not row_parent:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} filter node row parents {[x.address for x in parent_row_concepts]} could not be found"
+        )
+        return None
+    else:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} filter node has row parents {[x.address for x in parent_row_concepts]} from node with output [{[x.address for x in row_parent.output_concepts]}] partial {row_parent.partial_concepts}"
+        )
+    if global_filter_is_local_filter:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} filter node conditions match global conditions adding row parent {row_parent.output_concepts} with condition {where.conditional}"
+        )
+        row_parent.add_parents(core_parent_nodes)
+        # all local optional will be in the parent already, so we can set outputs
+        row_parent.set_output_concepts([concept] + local_optional)
+        return row_parent
+    if optimized_pushdown:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} returning optimized filter node with pushdown to parent with condition {where.conditional} across {[concept] + same_filter_optional + row_parent.output_concepts} "
+        )
+        if isinstance(row_parent, SelectNode):
+            logger.info(
+                f"{padding(depth)}{LOGGER_PREFIX} nesting select node in strategy node"
+            )
+            parent = StrategyNode(
+                input_concepts=row_parent.output_concepts,
+                output_concepts=[concept]
+                + same_filter_optional
+                + row_parent.output_concepts,
+                environment=row_parent.environment,
+                parents=[row_parent],
+                depth=row_parent.depth,
+                partial_concepts=row_parent.partial_concepts,
+                force_group=False,
+            )
+        else:
+            parent = row_parent
+            parent.add_output_concepts([concept] + same_filter_optional)
+        parent.add_parents(core_parent_nodes)
+        if not parent.preexisting_conditions == where.conditional:
+            parent.add_condition(where.conditional)
+        parent.add_existence_concepts(flattened_existence, False)
+        # parent.grain = BuildGrain.from_concepts(
+        #     parent.output_concepts,
+        #     environment=environment,
+        # )
+        parent.rebuild_cache()
+        filter_node = parent
+    else:
+        core_parent_nodes.append(row_parent)
+        filter_node = FilterNode(
+            input_concepts=unique(
+                parent_row_concepts + flattened_existence,
+                "address",
+            ),
+            output_concepts=[concept] + same_filter_optional + parent_row_concepts,
+            environment=environment,
+            parents=core_parent_nodes,
+            preexisting_conditions=conditions.conditional if conditions else None,
+        )
+
+    if not local_optional or all(
+        [x.address in filter_node.output_concepts for x in local_optional]
+    ):
+        optional_outputs = [
+            x for x in filter_node.output_concepts if x.address in local_optional
+        ]
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} no extra enrichment needed for filter node, has all of {[x.address for x in local_optional]}"
+        )
+        filter_node.set_output_concepts(
+            [
+                concept,
+            ]
+            + optional_outputs
+        )
+        return filter_node
+    logger.info(
+        f"{padding(depth)}{LOGGER_PREFIX} need to enrich filter node with additional concepts {[x.address for x in local_optional if x.address not in filter_node.output_concepts]}"
+    )
+    enrich_node: StrategyNode = source_concepts(  # this fetches the parent + join keys
+        # to then connect to the rest of the query
+        mandatory_list=parent_row_concepts
+        + [x for x in local_optional if x.address not in filter_node.output_concepts],
+        environment=environment,
+        g=g,
+        depth=depth + 1,
+        history=history,
+        conditions=conditions,
+    )
+    if not enrich_node:
+        logger.error(
+            f"{padding(depth)}{LOGGER_PREFIX} filter node enrichment node could not be found"
+        )
+        return filter_node
+    logger.info(
+        f"{padding(depth)}{LOGGER_PREFIX} returning filter node and enrich node with {enrich_node.output_concepts} and {enrich_node.input_concepts}"
+    )
+    return MergeNode(
+        input_concepts=filter_node.output_concepts + enrich_node.output_concepts,
+        output_concepts=[
+            concept,
+        ]
+        + local_optional,
+        environment=environment,
+        parents=[
+            filter_node,
+            enrich_node,
+        ],
+        preexisting_conditions=conditions.conditional if conditions else None,
+    )

trilogy/core/processing/node_generators/group_node.py

@@ -0,0 +1,213 @@
+from typing import List
+
+from trilogy.constants import logger
+from trilogy.core.internal import ALL_ROWS_CONCEPT
+from trilogy.core.models.build import (
+    BuildAggregateWrapper,
+    BuildConcept,
+    BuildFunction,
+    BuildGrain,
+    BuildWhereClause,
+    LooseBuildConceptList,
+)
+from trilogy.core.models.build_environment import BuildEnvironment
+from trilogy.core.processing.node_generators.common import (
+    gen_enrichment_node,
+    resolve_function_parent_concepts,
+)
+from trilogy.core.processing.nodes import GroupNode, History, StrategyNode
+from trilogy.core.processing.utility import create_log_lambda, padding
+from trilogy.utility import unique
+
+LOGGER_PREFIX = "[GEN_GROUP_NODE]"
+
+
+def get_aggregate_grain(
+    concept: BuildConcept, environment: BuildEnvironment
+) -> BuildGrain:
+    parent_concepts: List[BuildConcept] = unique(
+        resolve_function_parent_concepts(concept, environment=environment), "address"
+    )
+
+    if (
+        concept.grain
+        and len(concept.grain.components) > 0
+        and not concept.grain.abstract
+    ):
+        grain_components = [environment.concepts[c] for c in concept.grain.components]
+        parent_concepts += grain_components
+        return BuildGrain.from_concepts(parent_concepts)
+    else:
+        return BuildGrain.from_concepts(parent_concepts)
+
+
+def gen_group_node(
+    concept: BuildConcept,
+    local_optional: List[BuildConcept],
+    environment: BuildEnvironment,
+    g,
+    depth: int,
+    source_concepts,
+    history: History,
+    conditions: BuildWhereClause | None = None,
+) -> StrategyNode | None:
+    # aggregates MUST always group to the proper grain
+    # except when the
+    parent_concepts: List[BuildConcept] = unique(
+        resolve_function_parent_concepts(concept, environment=environment), "address"
+    )
+    logger.info(
+        f"{padding(depth)}{LOGGER_PREFIX} parent concepts for {concept} {concept.lineage} are {[x.address for x in parent_concepts]} from group grain {concept.grain}"
+    )
+
+    # if the aggregation has a grain, we need to ensure these are the ONLY optional in the output of the select
+    output_concepts = [concept]
+    grain_components = [environment.concepts[c] for c in concept.grain.components]
+    if (
+        concept.grain
+        and len(concept.grain.components) > 0
+        and not concept.grain.abstract
+    ):
+
+        parent_concepts += grain_components
+        build_grain_parents = get_aggregate_grain(concept, environment)
+        output_concepts += grain_components
+        for possible_agg in local_optional:
+
+            if not isinstance(
+                possible_agg.lineage,
+                (BuildAggregateWrapper, BuildFunction),
+            ):
+                continue
+            if possible_agg.grain and possible_agg.grain != concept.grain:
+                logger.info(
+                    f"{padding(depth)}{LOGGER_PREFIX} mismatched equivalent group by with grain {possible_agg.grain} for {concept.address}"
+                )
+
+            if possible_agg.grain and possible_agg.grain == concept.grain:
+                agg_parents: List[BuildConcept] = resolve_function_parent_concepts(
+                    possible_agg,
+                    environment=environment,
+                )
+                comp_grain = get_aggregate_grain(possible_agg, environment)
+                if set([x.address for x in agg_parents]).issubset(
+                    set([x.address for x in parent_concepts])
+                ):
+                    output_concepts.append(possible_agg)
+                    logger.info(
+                        f"{padding(depth)}{LOGGER_PREFIX} found equivalent group by optional concept {possible_agg.address} for {concept.address}"
+                    )
+                elif comp_grain == build_grain_parents:
+                    extra = [x for x in agg_parents if x.address not in parent_concepts]
+                    parent_concepts += extra
+                    output_concepts.append(possible_agg)
+                    logger.info(
+                        f"{padding(depth)}{LOGGER_PREFIX} found equivalent group by optional concept {possible_agg.address} for {concept.address}"
+                    )
+                else:
+                    logger.info(
+                        f"{padding(depth)}{LOGGER_PREFIX} cannot include optional agg {possible_agg.address}; it has mismatched parent grain {comp_grain } vs local parent {build_grain_parents}"
+                    )
+    elif concept.grain.abstract:
+        for possible_agg in local_optional:
+            if not isinstance(
+                possible_agg.lineage,
+                (BuildAggregateWrapper, BuildFunction),
+            ):
+
+                continue
+            logger.info(
+                f"{padding(depth)}{LOGGER_PREFIX} considering optional agg {possible_agg.address} for {concept.address}"
+            )
+            agg_parents = resolve_function_parent_concepts(
+                possible_agg,
+                environment=environment,
+            )
+            comp_grain = get_aggregate_grain(possible_agg, environment)
+            if not possible_agg.grain.abstract:
+                continue
+            if set([x.address for x in agg_parents]) == set(
+                [x.address for x in parent_concepts]
+            ):
+                output_concepts.append(possible_agg)
+                logger.info(
+                    f"{padding(depth)}{LOGGER_PREFIX} found equivalent group by optional concept {possible_agg.address} for {concept.address}"
+                )
+            elif comp_grain == get_aggregate_grain(concept, environment):
+                extra = [x for x in agg_parents if x.address not in parent_concepts]
+                parent_concepts += extra
+                output_concepts.append(possible_agg)
+                logger.info(
+                    f"{padding(depth)}{LOGGER_PREFIX} found equivalent group by optional concept {possible_agg.address} for {concept.address}"
+                )
+            else:
+                logger.info(
+                    f"{padding(depth)}{LOGGER_PREFIX} cannot include optional agg {possible_agg.address}; it has mismatched parent grain {comp_grain } vs local parent {get_aggregate_grain(concept, environment)}"
+                )
+    if parent_concepts:
+        target_grain = BuildGrain.from_concepts(parent_concepts)
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} fetching group node parents {LooseBuildConceptList(concepts=parent_concepts)} with expected grain {target_grain}"
+        )
+        parent_concepts = unique(
+            [x for x in parent_concepts if not x.name == ALL_ROWS_CONCEPT], "address"
+        )
+        parent: StrategyNode | None = source_concepts(
+            mandatory_list=parent_concepts,
+            environment=environment,
+            g=g,
+            depth=depth,
+            history=history,
+            conditions=conditions,
+        )
+        if not parent:
+            logger.info(
+                f"{padding(depth)}{LOGGER_PREFIX} group by node parents unresolvable"
+            )
+            return None
+        parents: List[StrategyNode] = [parent]
+    else:
+        parents = []
+
+    group_node = GroupNode(
+        output_concepts=output_concepts,
+        input_concepts=parent_concepts,
+        environment=environment,
+        parents=parents,
+        depth=depth,
+        preexisting_conditions=conditions.conditional if conditions else None,
+        required_outputs=parent_concepts,
+    )
+
+    # early exit if no optional
+
+    if not local_optional:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} no optional concepts, returning group node"
+        )
+        return group_node
+    missing_optional = [
+        x.address for x in local_optional if x.address not in group_node.usable_outputs
+    ]
+    if not missing_optional:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} no extra enrichment needed for group node, has all of {[x.address for x in local_optional]}"
+        )
+        return group_node
+    logger.info(
+        f"{padding(depth)}{LOGGER_PREFIX} group node for {concept.address} requires enrichment, missing {missing_optional}"
+    )
+    return gen_enrichment_node(
+        group_node,
+        join_keys=grain_components,
+        local_optional=local_optional,
+        environment=environment,
+        g=g,
+        depth=depth,
+        source_concepts=source_concepts,
+        log_lambda=create_log_lambda(
+            LOGGER_PREFIX + f" for {concept.address}", depth, logger
+        ),
+        history=history,
+        conditions=conditions,
+    )

trilogy/core/processing/node_generators/group_to_node.py

@@ -0,0 +1,117 @@
+from typing import List
+
+from trilogy.constants import logger
+from trilogy.core.models.build import BuildConcept, BuildFunction, BuildWhereClause
+from trilogy.core.models.build_environment import BuildEnvironment
+
+# C:\Users\ethan\coding_projects\pytrilogy\trilogy\core\processing\node_generators\group_to_node.py
+from trilogy.core.processing.nodes import (
+    GroupNode,
+    History,
+    MergeNode,
+    StrategyNode,
+)
+from trilogy.core.processing.utility import padding
+
+LOGGER_PREFIX = "[GEN_GROUP_TO_NODE]"
+
+
+def gen_group_to_node(
+    concept: BuildConcept,
+    local_optional,
+    environment: BuildEnvironment,
+    g,
+    depth: int,
+    source_concepts,
+    history: History | None = None,
+    conditions: BuildWhereClause | None = None,
+) -> GroupNode | MergeNode:
+    # aggregates MUST always group to the proper grain
+    if not isinstance(concept.lineage, BuildFunction):
+        raise SyntaxError(
+            f"Group to should have function lineage, is {type(concept.lineage)}"
+        )
+
+    parent_concepts: List[BuildConcept] = concept.lineage.concept_arguments
+    root = parent_concepts[0]
+    logger.info(
+        f"{padding(depth)}{LOGGER_PREFIX} group by node has required parents {[x.address for x in parent_concepts]}"
+    )
+    parents: List[StrategyNode] = [
+        source_concepts(
+            mandatory_list=parent_concepts,
+            environment=environment,
+            g=g,
+            depth=depth + 1,
+            history=history,
+            conditions=conditions,
+        )
+    ]
+    outputs = parent_concepts + [concept]
+    group_node = GroupNode(
+        output_concepts=parent_concepts + [concept],
+        input_concepts=parent_concepts,
+        environment=environment,
+        parents=parents,
+        depth=depth,
+        preexisting_conditions=conditions.conditional if conditions else None,
+        hidden_concepts=set(
+            [
+                x.address
+                for x in outputs
+                if x.address not in local_optional
+                and x.address != concept.address
+                and x.address != root.address
+            ]
+        ),
+    )
+
+    # early exit if no optional
+    missing_local_option: list[BuildConcept] = [
+        x for x in local_optional if x not in group_node.output_concepts
+    ]
+    if not missing_local_option:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} no missing local optional required, returning group node only."
+        )
+        return group_node
+
+    logger.info(
+        f"{padding(depth)}{LOGGER_PREFIX} group by node is missing required optional {[x.address for x in missing_local_option]}"
+    )
+
+    # the keys we group by
+    # are what we can use for enrichment
+    enrich_node = source_concepts(  # this fetches the parent + join keys
+        # to then connect to the rest of the query
+        mandatory_list=parent_concepts + local_optional,
+        environment=environment,
+        g=g,
+        depth=depth + 1,
+        history=history,
+        conditions=conditions,
+    )
+    if not enrich_node:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} group by node enrich node, returning group node only."
+        )
+        return group_node
+    logger.info(
+        f"{padding(depth)}{LOGGER_PREFIX} returning group to node with enrichment."
+    )
+    return MergeNode(
+        input_concepts=[concept]
+        + local_optional
+        + [x for x in parent_concepts if x.address != concept.address],
+        output_concepts=[concept] + local_optional,
+        environment=environment,
+        parents=[
+            # this node gets the group
+            group_node,
+            # this node gets enrichment
+            enrich_node,
+        ],
+        whole_grain=True,
+        depth=depth,
+        preexisting_conditions=conditions.conditional if conditions else None,
+    )