pytrilogy 0.0.1.102__py3-none-any.whl

trilogy/core/processing/node_generators/group_to_node.py

@@ -0,0 +1,99 @@
+from trilogy.core.models import Concept, Environment, Function
+from trilogy.core.processing.nodes import (
+    GroupNode,
+    StrategyNode,
+    MergeNode,
+    NodeJoin,
+    History,
+)
+from typing import List
+from trilogy.core.enums import JoinType
+
+from trilogy.constants import logger
+from trilogy.core.processing.utility import padding
+from trilogy.core.processing.node_generators.common import concept_to_relevant_joins
+
+LOGGER_PREFIX = "[GEN_GROUP_TO_NODE]"
+
+
+def gen_group_to_node(
+    concept: Concept,
+    local_optional,
+    environment: Environment,
+    g,
+    depth: int,
+    source_concepts,
+    history: History | None = None,
+) -> GroupNode | MergeNode:
+    # aggregates MUST always group to the proper grain
+    if not isinstance(concept.lineage, Function):
+        raise SyntaxError("Group to should have function lineage")
+    parent_concepts: List[Concept] = concept.lineage.concept_arguments
+    logger.info(
+        f"{padding(depth)}{LOGGER_PREFIX} group by node has required parents {[x.address for x in parent_concepts]}"
+    )
+    parents: List[StrategyNode] = [
+        source_concepts(
+            mandatory_list=parent_concepts,
+            environment=environment,
+            g=g,
+            depth=depth + 1,
+            history=history,
+        )
+    ]
+
+    group_node = GroupNode(
+        output_concepts=parent_concepts + [concept],
+        input_concepts=parent_concepts,
+        environment=environment,
+        g=g,
+        parents=parents,
+        depth=depth,
+    )
+
+    # early exit if no optional
+    if not local_optional:
+        return group_node
+
+    # the keys we group by
+    # are what we can use for enrichment
+    enrich_node = source_concepts(  # this fetches the parent + join keys
+        # to then connect to the rest of the query
+        mandatory_list=parent_concepts + local_optional,
+        environment=environment,
+        g=g,
+        depth=depth + 1,
+        history=history,
+    )
+    if not enrich_node:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} group by node enrich node, returning group node only."
+        )
+        return group_node
+
+    return MergeNode(
+        input_concepts=[concept]
+        + local_optional
+        + [x for x in parent_concepts if x.address != concept.address],
+        output_concepts=[concept] + local_optional,
+        environment=environment,
+        g=g,
+        parents=[
+            # this node gets the group
+            group_node,
+            # this node gets enrichment
+            enrich_node,
+        ],
+        node_joins=[
+            NodeJoin(
+                left_node=group_node,
+                right_node=enrich_node,
+                concepts=concept_to_relevant_joins(parent_concepts),
+                filter_to_mutual=False,
+                join_type=JoinType.LEFT_OUTER,
+            )
+        ],
+        whole_grain=True,
+        depth=depth,
+        partial_concepts=group_node.partial_concepts,
+    )

trilogy/core/processing/node_generators/merge_node.py

@@ -0,0 +1,148 @@
+from typing import List, Optional
+
+from trilogy.core.models import Concept, Environment, Datasource, Conditional
+from trilogy.core.processing.nodes import MergeNode, History
+import networkx as nx
+from trilogy.core.graph_models import concept_to_node, datasource_to_node
+from trilogy.core.processing.utility import PathInfo
+from trilogy.constants import logger
+from trilogy.utility import unique
+from trilogy.core.exceptions import AmbiguousRelationshipResolutionException
+from trilogy.core.processing.utility import padding
+from trilogy.core.processing.graph_utils import extract_mandatory_subgraphs
+
+LOGGER_PREFIX = "[GEN_MERGE_NODE]"
+
+
+def reduce_path_concepts(paths, g) -> set[str]:
+    concept_nodes: List[Concept] = []
+    # along our path, find all the concepts required
+    for _, value in paths.items():
+        concept_nodes += [g.nodes[v]["concept"] for v in value if v.startswith("c~")]
+    final: List[Concept] = unique(concept_nodes, "address")
+    return set([x.address for x in final])
+
+
+def identify_ds_join_paths(
+    all_concepts: List[Concept],
+    g,
+    datasource: Datasource,
+    accept_partial: bool = False,
+    fail: bool = False,
+) -> PathInfo | None:
+    all_found = True
+    any_direct_found = False
+    paths = {}
+    for bitem in all_concepts:
+        item = bitem.with_default_grain()
+        target_node = concept_to_node(item)
+        try:
+            path = nx.shortest_path(
+                g,
+                source=datasource_to_node(datasource),
+                target=target_node,
+            )
+            paths[target_node] = path
+            if sum([1 for x in path if x.startswith("ds~")]) == 1:
+                any_direct_found = True
+        except nx.exception.NodeNotFound:
+            # TODO: support Verbose logging mode configuration and reenable these
+            all_found = False
+            if fail:
+                raise
+            return None
+        except nx.exception.NetworkXNoPath:
+            all_found = False
+            if fail:
+                raise
+            return None
+    if all_found and any_direct_found:
+        partial = [
+            c.concept
+            for c in datasource.columns
+            if not c.is_complete
+            and c.concept.address in [x.address for x in all_concepts]
+        ]
+        if partial and not accept_partial:
+            return None
+        # join_candidates.append({"paths": paths, "datasource": datasource})
+        return PathInfo(
+            paths=paths,
+            datasource=datasource,
+            reduced_concepts=reduce_path_concepts(paths, g),
+            concept_subgraphs=extract_mandatory_subgraphs(paths, g),
+        )  # {"paths": paths, "datasource": datasource}
+    return None
+
+
+def gen_merge_node(
+    all_concepts: List[Concept],
+    g: nx.DiGraph,
+    environment: Environment,
+    depth: int,
+    source_concepts,
+    accept_partial: bool = False,
+    history: History | None = None,
+    conditions: Conditional | None = None,
+) -> Optional[MergeNode]:
+    join_candidates: List[PathInfo] = []
+    # anchor on datasources
+    for datasource in environment.datasources.values():
+        path = identify_ds_join_paths(all_concepts, g, datasource, accept_partial)
+        if path and path.reduced_concepts:
+            join_candidates.append(path)
+    join_candidates.sort(key=lambda x: sum([len(v) for v in x.paths.values()]))
+    if not join_candidates:
+        return None
+    for join_candidate in join_candidates:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} Join candidate: {join_candidate.paths}"
+        )
+    join_additions: List[set[str]] = []
+    for candidate in join_candidates:
+        join_additions.append(candidate.reduced_concepts)
+    if not all(
+        [x.issubset(y) or y.issubset(x) for x in join_additions for y in join_additions]
+    ):
+        raise AmbiguousRelationshipResolutionException(
+            f"Ambiguous concept join resolution - possible paths =  {join_additions}. Include an additional concept to disambiguate",
+            join_additions,
+        )
+    if not join_candidates:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} No additional join candidates could be found"
+        )
+        return None
+    shortest: PathInfo = sorted(join_candidates, key=lambda x: len(x.reduced_concepts))[
+        0
+    ]
+    logger.info(f"{padding(depth)}{LOGGER_PREFIX} final path is {shortest.paths}")
+    # logger.info(f'{padding(depth)}{LOGGER_PREFIX} final reduced concepts are {shortest.concs}')
+    parents = []
+    for graph in shortest.concept_subgraphs:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} fetching subgraph {[c.address for c in graph]}"
+        )
+        parent = source_concepts(
+            mandatory_list=graph,
+            environment=environment,
+            g=g,
+            depth=depth + 1,
+            history=history,
+        )
+        if not parent:
+            logger.info(
+                f"{padding(depth)}{LOGGER_PREFIX} Unable to instantiate target subgraph"
+            )
+            return None
+        parents.append(parent)
+
+    return MergeNode(
+        input_concepts=[environment.concepts[x] for x in shortest.reduced_concepts],
+        output_concepts=all_concepts,
+        environment=environment,
+        g=g,
+        parents=parents,
+        depth=depth,
+        conditions=conditions,
+    )

trilogy/core/processing/node_generators/multiselect_node.py

@@ -0,0 +1,189 @@
+from trilogy.core.models import (
+    Concept,
+    Environment,
+    MultiSelectStatement,
+)
+from trilogy.core.processing.nodes import MergeNode, NodeJoin, History
+from trilogy.core.processing.nodes.base_node import concept_list_to_grain, StrategyNode
+from typing import List
+
+from trilogy.core.enums import JoinType
+from trilogy.constants import logger
+from trilogy.core.processing.utility import padding
+from trilogy.core.processing.node_generators.common import concept_to_relevant_joins
+from collections import defaultdict
+from itertools import combinations
+from trilogy.core.enums import Purpose
+from trilogy.core.processing.node_generators.common import resolve_join_order
+
+LOGGER_PREFIX = "[GEN_MULTISELECT_NODE]"
+
+
+def extra_align_joins(
+    base: MultiSelectStatement, parents: List[StrategyNode]
+) -> List[NodeJoin]:
+    node_merge_concept_map = defaultdict(list)
+    output = []
+    for align in base.align.items:
+        jc = align.gen_concept(base)
+        if jc.purpose == Purpose.CONSTANT:
+            continue
+        for node in parents:
+            for item in align.concepts:
+                if item in node.output_lcl:
+                    node_merge_concept_map[node].append(jc)
+
+    for left, right in combinations(node_merge_concept_map.keys(), 2):
+        matched_concepts = [
+            x
+            for x in node_merge_concept_map[left]
+            if x in node_merge_concept_map[right]
+        ]
+        output.append(
+            NodeJoin(
+                left_node=left,
+                right_node=right,
+                concepts=matched_concepts,
+                join_type=JoinType.FULL,
+            )
+        )
+    return resolve_join_order(output)
+
+
+def gen_multiselect_node(
+    concept: Concept,
+    local_optional: List[Concept],
+    environment: Environment,
+    g,
+    depth: int,
+    source_concepts,
+    history: History | None = None,
+) -> MergeNode | None:
+    if not isinstance(concept.lineage, MultiSelectStatement):
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} Cannot generate multiselect node for {concept}"
+        )
+        return None
+    lineage: MultiSelectStatement = concept.lineage
+
+    base_parents: List[StrategyNode] = []
+    for select in lineage.selects:
+        snode: StrategyNode = source_concepts(
+            mandatory_list=select.output_components,
+            environment=environment,
+            g=g,
+            depth=depth + 1,
+            history=history,
+        )
+        if not snode:
+            logger.info(
+                f"{padding(depth)}{LOGGER_PREFIX} Cannot generate multiselect node for {concept}"
+            )
+            return None
+        if select.where_clause:
+            snode.conditions = select.where_clause.conditional
+        merge_concepts = []
+        for x in [*snode.output_concepts]:
+            merge = lineage.get_merge_concept(x)
+            if merge:
+                snode.output_concepts.append(merge)
+                merge_concepts.append(merge)
+        # clear cache so QPS
+        snode.rebuild_cache()
+        for mc in merge_concepts:
+            assert mc in snode.resolve().output_concepts
+        base_parents.append(snode)
+
+    node_joins = extra_align_joins(lineage, base_parents)
+    node = MergeNode(
+        input_concepts=[x for y in base_parents for x in y.output_concepts],
+        output_concepts=[x for y in base_parents for x in y.output_concepts],
+        environment=environment,
+        g=g,
+        depth=depth,
+        parents=base_parents,
+        node_joins=node_joins,
+    )
+
+    enrichment = set([x.address for x in local_optional])
+
+    rowset_relevant = [
+        x
+        for x in lineage.derived_concepts
+        if x.address == concept.address or x.address in enrichment
+    ]
+    additional_relevant = [
+        x for x in select.output_components if x.address in enrichment
+    ]
+    # add in other other concepts
+    for item in rowset_relevant:
+        node.output_concepts.append(item)
+    for item in additional_relevant:
+        node.output_concepts.append(item)
+    if select.where_clause:
+        for item in additional_relevant:
+            node.partial_concepts.append(item)
+
+    # we need a better API for refreshing a nodes QDS
+    node.resolution_cache = node._resolve()
+
+    # assume grain to be output of select
+    # but don't include anything aggregate at this point
+    node.resolution_cache.grain = concept_list_to_grain(
+        node.output_concepts, parent_sources=node.resolution_cache.datasources
+    )
+    possible_joins = concept_to_relevant_joins(additional_relevant)
+    if not local_optional:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} no enriched required for rowset node; exiting early"
+        )
+        return node
+    if not possible_joins:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} no possible joins for rowset node; exiting early"
+        )
+        return node
+    if all(
+        [x.address in [y.address for y in node.output_concepts] for x in local_optional]
+    ):
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} all enriched concepts returned from base rowset node; exiting early"
+        )
+        return node
+    enrich_node: MergeNode = source_concepts(  # this fetches the parent + join keys
+        # to then connect to the rest of the query
+        mandatory_list=additional_relevant + local_optional,
+        environment=environment,
+        g=g,
+        depth=depth + 1,
+        history=history,
+    )
+    if not enrich_node:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} Cannot generate rowset enrichment node for {concept} with optional {local_optional}, returning just rowset node"
+        )
+        return node
+
+    return MergeNode(
+        input_concepts=enrich_node.output_concepts + node.output_concepts,
+        output_concepts=node.output_concepts + local_optional,
+        environment=environment,
+        g=g,
+        depth=depth,
+        parents=[
+            # this node gets the multiselect
+            node,
+            # this node gets enrichment
+            enrich_node,
+        ],
+        node_joins=[
+            NodeJoin(
+                left_node=enrich_node,
+                right_node=node,
+                concepts=possible_joins,
+                filter_to_mutual=False,
+                join_type=JoinType.LEFT_OUTER,
+            )
+        ],
+        partial_concepts=node.partial_concepts,
+    )

trilogy/core/processing/node_generators/rowset_node.py

@@ -0,0 +1,130 @@
+from trilogy.core.models import (
+    Concept,
+    Environment,
+    SelectStatement,
+    RowsetDerivationStatement,
+    RowsetItem,
+    MultiSelectStatement,
+)
+from trilogy.core.processing.nodes import MergeNode, NodeJoin, History, StrategyNode
+from trilogy.core.processing.nodes.base_node import concept_list_to_grain
+from typing import List
+
+from trilogy.core.enums import JoinType
+from trilogy.constants import logger
+from trilogy.core.processing.utility import padding
+from trilogy.core.processing.node_generators.common import concept_to_relevant_joins
+
+
+LOGGER_PREFIX = "[GEN_ROWSET_NODE]"
+
+
+def gen_rowset_node(
+    concept: Concept,
+    local_optional: List[Concept],
+    environment: Environment,
+    g,
+    depth: int,
+    source_concepts,
+    history: History | None = None,
+) -> StrategyNode | None:
+    if not isinstance(concept.lineage, RowsetItem):
+        raise SyntaxError(
+            f"Invalid lineage passed into rowset fetch, got {type(concept.lineage)}, expected {RowsetItem}"
+        )
+    lineage: RowsetItem = concept.lineage
+    rowset: RowsetDerivationStatement = lineage.rowset
+    select: SelectStatement | MultiSelectStatement = lineage.rowset.select
+    node: StrategyNode = source_concepts(
+        mandatory_list=select.output_components,
+        environment=environment,
+        g=g,
+        depth=depth + 1,
+        history=history,
+    )
+    node.conditions = select.where_clause.conditional if select.where_clause else None
+    # rebuild any cached info with the new condition clause
+    node.rebuild_cache()
+    if not node:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} Cannot generate rowset node for {concept}"
+        )
+        return None
+    enrichment = set([x.address for x in local_optional])
+    rowset_relevant = [
+        x
+        for x in rowset.derived_concepts
+        if x.address == concept.address or x.address in enrichment
+    ]
+    additional_relevant = [
+        x for x in select.output_components if x.address in enrichment
+    ]
+    # add in other other concepts
+    for item in rowset_relevant:
+        node.output_concepts.append(item)
+    for item in additional_relevant:
+        node.output_concepts.append(item)
+    if select.where_clause:
+        for item in additional_relevant:
+            node.partial_concepts.append(item)
+
+    # assume grain to be outoput of select
+    # but don't include anything aggregate at this point
+    assert node.resolution_cache
+    node.resolution_cache.grain = concept_list_to_grain(
+        node.output_concepts, parent_sources=node.resolution_cache.datasources
+    )
+    possible_joins = concept_to_relevant_joins(additional_relevant)
+    if not local_optional:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} no enriched required for rowset node; exiting early"
+        )
+        return node
+    if not possible_joins:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} no possible joins for rowset node; exiting early"
+        )
+        return node
+    if all(
+        [x.address in [y.address for y in node.output_concepts] for x in local_optional]
+    ):
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} all enriched concepts returned from base rowset node; exiting early"
+        )
+        return node
+    enrich_node: MergeNode = source_concepts(  # this fetches the parent + join keys
+        # to then connect to the rest of the query
+        mandatory_list=additional_relevant + local_optional,
+        environment=environment,
+        g=g,
+        depth=depth + 1,
+    )
+    if not enrich_node:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} Cannot generate rowset enrichment node for {concept} with optional {local_optional}, returning just rowset node"
+        )
+        return node
+
+    return MergeNode(
+        input_concepts=enrich_node.output_concepts + node.output_concepts,
+        output_concepts=node.output_concepts + local_optional,
+        environment=environment,
+        g=g,
+        depth=depth,
+        parents=[
+            # this node gets the window
+            node,
+            # this node gets enrichment
+            enrich_node,
+        ],
+        node_joins=[
+            NodeJoin(
+                left_node=enrich_node,
+                right_node=node,
+                concepts=concept_to_relevant_joins(additional_relevant),
+                filter_to_mutual=False,
+                join_type=JoinType.LEFT_OUTER,
+            )
+        ],
+        partial_concepts=node.partial_concepts,
+    )