pytrilogy 0.0.1.117__py3-none-any.whl → 0.0.2.1__py3-none-any.whl

trilogy/core/processing/node_generators/basic_node.py

@@ -4,7 +4,7 @@ from typing import List
 from trilogy.core.models import (
     Concept,
 )
-from trilogy.core.processing.nodes import StrategyNode, History, MergeNode
+from trilogy.core.processing.nodes import StrategyNode, History
 from trilogy.core.processing.node_generators.common import (
     resolve_function_parent_concepts,
 )
@@ -30,16 +30,25 @@ def gen_basic_node(
         f"{depth_prefix}{LOGGER_PREFIX} basic node for {concept} has parents {[x.address for x in parent_concepts]}"
     )
 
-    output_concepts = [concept] + local_optional
-    partials = []
+    local_optional_redundant = [x for x in local_optional if x in parent_concepts]
+    attempts = [(parent_concepts, [concept] + local_optional_redundant)]
+    from itertools import combinations
 
-    attempts = [(parent_concepts, [concept])]
     if local_optional:
-        attempts.append((parent_concepts + local_optional, local_optional + [concept]))
+        for combo in range(1, len(local_optional) + 1):
+            combos = combinations(local_optional, combo)
+            for optional_set in combos:
+                attempts.append(
+                    (
+                        unique(parent_concepts + list(optional_set), "address"),
+                        list(optional_set) + [concept],
+                    )
+                )
 
-    for attempt, output in reversed(attempts):
+    for attempt, basic_output in reversed(attempts):
+        partials = []
         attempt = unique(attempt, "address")
-        parent_node = source_concepts(
+        parent_node: StrategyNode = source_concepts(
             mandatory_list=attempt,
             environment=environment,
             g=g,
@@ -49,24 +58,28 @@ def gen_basic_node(
         if not parent_node:
             continue
         parents: List[StrategyNode] = [parent_node]
-        for x in output_concepts:
+        for x in basic_output:
             sources = [p for p in parents if x in p.output_concepts]
             if not sources:
                 continue
             if all(x in source.partial_concepts for source in sources):
                 partials.append(x)
+        outputs = parent_node.output_concepts + [concept]
         logger.info(
-            f"{depth_prefix}{LOGGER_PREFIX} Returning basic select for {concept} with attempted extra {[x.address for x in attempt]}"
+            f"{depth_prefix}{LOGGER_PREFIX} Returning basic select for {concept} with attempted extra {[x.address for x in attempt]}, output {[x.address for x in outputs]}"
         )
-        return MergeNode(
-            input_concepts=attempt,
-            output_concepts=output,
-            environment=environment,
-            g=g,
-            parents=parents,
-            depth=depth,
-            partial_concepts=partials,
+        # parents.resolve()
+
+        parent_node.add_output_concept(concept)
+
+        parent_node.remove_output_concepts(
+            [
+                x
+                for x in parent_node.output_concepts
+                if x.address not in [y.address for y in basic_output]
+            ]
         )
+        return parent_node
     logger.info(
         f"{depth_prefix}{LOGGER_PREFIX} No basic node could be generated for {concept}"
     )

trilogy/core/processing/node_generators/filter_node.py

@@ -2,7 +2,7 @@ from typing import List
 
 
 from trilogy.core.enums import JoinType
-from trilogy.core.models import Concept, Environment, FilterItem
+from trilogy.core.models import Concept, Environment, FilterItem, Grain
 from trilogy.core.processing.nodes import (
     FilterNode,
     MergeNode,
@@ -65,7 +65,9 @@ def gen_filter_node(
         else:
             parent.conditions = where.conditional
         parent.output_concepts = [concept]
+        parent.grain = Grain(components=[concept])
         parent.rebuild_cache()
+
         logger.info(
             f"{padding(depth)}{LOGGER_PREFIX} returning optimized filter node with pushdown to parent with condition {where.conditional}"
         )

trilogy/core/processing/node_generators/node_merge_node.py

@@ -1,79 +1,301 @@
 from typing import List, Optional
 
-from trilogy.core.models import Concept, Environment, Datasource, Conditional
-from trilogy.core.processing.nodes import MergeNode, History
+from trilogy.core.models import Concept, Environment, Conditional
+from trilogy.core.processing.nodes import MergeNode, History, StrategyNode
 import networkx as nx
-from trilogy.core.graph_models import concept_to_node, datasource_to_node
+from trilogy.core.graph_models import concept_to_node
 from trilogy.core.processing.utility import PathInfo
 from trilogy.constants import logger
 from trilogy.utility import unique
 from trilogy.core.exceptions import AmbiguousRelationshipResolutionException
 from trilogy.core.processing.utility import padding
-from trilogy.core.processing.graph_utils import extract_mandatory_subgraphs
+from networkx.algorithms import approximation as ax
 from trilogy.core.enums import PurposeLineage
 
+
 LOGGER_PREFIX = "[GEN_MERGE_NODE]"
+AMBIGUITY_CHECK_LIMIT = 20
+
+
+def filter_pseudonyms_for_source(ds_graph: nx.DiGraph, node: str):
+    to_remove = set()
+
+    for edge in ds_graph.edges:
+        if ds_graph.edges[edge].get("pseudonym", False):
+            lengths = {}
+            for n in edge:
+                lengths[n] = nx.shortest_path_length(ds_graph, node, n)
+            to_remove.add(max(lengths, key=lambda x: lengths.get(x, 0)))
+    for node in to_remove:
+        ds_graph.remove_node(node)
+
+
+def extract_address(node: str):
+    return node.split("~")[1].split("@")[0]
+
+
+def extract_concept(node: str, env: Environment):
+    if node in env.alias_origin_lookup:
+        return env.alias_origin_lookup[node]
+    return env.concepts[node]
 
 
-def reduce_path_concepts(paths, g) -> set[str]:
-    concept_nodes: List[Concept] = []
-    # along our path, find all the concepts required
-    for _, value in paths.items():
-        concept_nodes += [g.nodes[v]["concept"] for v in value if v.startswith("c~")]
-    final: List[Concept] = unique(concept_nodes, "address")
-    return set([x.address for x in final])
+def filter_unique_graphs(graphs: list[list[str]]) -> list[list[str]]:
+    unique_graphs: list[set[str]] = []
+    for graph in graphs:
+        if not any(set(graph).issubset(x) for x in unique_graphs):
+            unique_graphs.append(set(graph))
 
+    return [list(x) for x in unique_graphs]
 
-def identify_ds_join_paths(
+
+def extract_ds_components(g: nx.DiGraph, nodelist: list[str]) -> list[list[str]]:
+    graphs = []
+    for node in g.nodes:
+        if node.startswith("ds~"):
+            local = g.copy()
+            filter_pseudonyms_for_source(local, node)
+            ds_graph: nx.DiGraph = nx.ego_graph(local, node, radius=10).copy()
+            graphs.append(
+                [
+                    extract_address(x)
+                    for x in ds_graph.nodes
+                    if not str(x).startswith("ds~")
+                ]
+            )
+
+    graphs = filter_unique_graphs(graphs)
+    for node in nodelist:
+        parsed = extract_address(node)
+        if not any(parsed in x for x in graphs):
+            graphs.append([parsed])
+    return graphs
+
+
+def determine_induced_minimal_nodes(
+    G: nx.DiGraph,
+    nodelist: list[str],
+    environment: Environment,
+    filter_downstream: bool,
+    accept_partial: bool = False,
+) -> nx.DiGraph | None:
+    H: nx.Graph = nx.to_undirected(G).copy()
+    nodes_to_remove = []
+    concepts = nx.get_node_attributes(G, "concept")
+    for node in G.nodes:
+        if concepts.get(node):
+            lookup = concepts[node]
+            if lookup.derivation not in (PurposeLineage.BASIC, PurposeLineage.ROOT):
+                nodes_to_remove.append(node)
+            elif lookup.derivation == PurposeLineage.BASIC and G.out_degree(node) == 0:
+                nodes_to_remove.append(node)
+            # purge a node if we're already looking for all it's parents
+            elif filter_downstream and lookup.derivation == PurposeLineage.BASIC:
+                nodes_to_remove.append(node)
+
+    H.remove_nodes_from(nodes_to_remove)
+
+    H.remove_nodes_from(list(nx.isolates(H)))
+
+    zero_out = list(x for x in H.nodes if G.out_degree(x) == 0 and x not in nodelist)
+    while zero_out:
+        H.remove_nodes_from(zero_out)
+        zero_out = list(
+            x for x in H.nodes if G.out_degree(x) == 0 and x not in nodelist
+        )
+    try:
+        paths = nx.multi_source_dijkstra_path(H, nodelist)
+    except nx.exception.NodeNotFound:
+        return None
+    H.remove_nodes_from(list(x for x in H.nodes if x not in paths))
+    sG: nx.Graph = ax.steinertree.steiner_tree(H, nodelist).copy()
+    final: nx.DiGraph = nx.subgraph(G, sG.nodes).copy()
+    for edge in G.edges:
+        if edge[1] in final.nodes and edge[0].startswith("ds~"):
+            ds_name = extract_address(edge[0])
+            ds = environment.datasources[ds_name]
+            concept = environment.concepts[extract_address(edge[1])]
+            if concept.address in [x.address for x in ds.partial_concepts]:
+                if not accept_partial:
+                    continue
+            final.add_edge(*edge)
+    # all concept nodes must have a parent
+
+    if not all(
+        [final.in_degree(node) > 0 for node in final.nodes if node.startswith("c~")]
+    ):
+        return None
+    if not all([node in final.nodes for node in nodelist]):
+        return None
+    return final
+
+
+def detect_ambiguity_and_raise(all_concepts, reduced_concept_sets) -> None:
+    final_candidates: list[set[str]] = []
+    common: set[str] = set()
+    # find all values that show up in every join_additions
+    for ja in reduced_concept_sets:
+        if not common:
+            common = ja
+        else:
+            common = common.intersection(ja)
+        if all(set(ja).issubset(y) for y in reduced_concept_sets):
+            final_candidates.append(ja)
+
+    if not final_candidates:
+        filtered_paths = [x.difference(common) for x in reduced_concept_sets]
+        raise AmbiguousRelationshipResolutionException(
+            message=f"Multiple possible concept injections found for {[x.address for x in all_concepts]}, got {' or '.join([str(x) for x in reduced_concept_sets])}",
+            parents=filtered_paths,
+        )
+
+
+def resolve_weak_components(
     all_concepts: List[Concept],
-    g,
-    datasource: Datasource,
+    environment: Environment,
+    environment_graph: nx.DiGraph,
+    filter_downstream: bool = True,
     accept_partial: bool = False,
-    fail: bool = False,
-) -> PathInfo | None:
-    all_found = True
-    any_direct_found = False
-    paths = {}
-    for bitem in all_concepts:
-        item = bitem.with_default_grain()
-        target_node = concept_to_node(item)
+) -> list[list[Concept]] | None:
+
+    break_flag = False
+    found = []
+    search_graph = environment_graph.copy()
+    reduced_concept_sets: list[set[str]] = []
+
+    # loop through, removing new nodes we find
+    # to ensure there are not ambiguous loops
+    # (if we did not care about raising ambiguity errors, we could just use the first one)
+    count = 0
+    node_list = [
+        concept_to_node(c.with_default_grain())
+        for c in all_concepts
+        if "__preql_internal" not in c.address
+    ]
+    while break_flag is not True:
+        count += 1
+        if count > AMBIGUITY_CHECK_LIMIT:
+            break_flag = True
         try:
-            path = nx.shortest_path(
-                g,
-                source=datasource_to_node(datasource),
-                target=target_node,
+            g = determine_induced_minimal_nodes(
+                search_graph,
+                node_list,
+                filter_downstream=filter_downstream,
+                accept_partial=accept_partial,
+                environment=environment,
             )
-            paths[target_node] = path
-            if sum([1 for x in path if x.startswith("ds~")]) == 1:
-                any_direct_found = True
-        except nx.exception.NodeNotFound:
-            # TODO: support Verbose logging mode configuration and reenable these
-            all_found = False
-            if fail:
-                raise
-            return None
+
+            if not g or not g.nodes:
+                break_flag = True
+                continue
+            if not nx.is_weakly_connected(g):
+                break_flag = True
+                continue
+
+            all_graph_concepts = [
+                extract_concept(extract_address(node), environment)
+                for node in g.nodes
+                if node.startswith("c~")
+            ]
+            new = [
+                x
+                for x in all_graph_concepts
+                if x.address not in [y.address for y in all_concepts]
+            ]
+
+            new_addresses = set([x.address for x in new])
+            if not new:
+                break_flag = True
+            # remove our new nodes for the next search path
+            for n in new:
+                node = concept_to_node(n)
+                if node in search_graph:
+                    search_graph.remove_node(node)
+            # TODO: figure out better place for debugging
+            # from trilogy.hooks.graph_hook import GraphHook
+            # GraphHook().query_graph_built(g, highlight_nodes=[concept_to_node(c.with_default_grain()) for c in all_concepts if "__preql_internal" not in c.address])
+            found.append(g)
+            reduced_concept_sets.append(new_addresses)
+
         except nx.exception.NetworkXNoPath:
-            all_found = False
-            if fail:
-                raise
-            return None
-    if all_found and any_direct_found:
-        partial = [
-            c.concept
-            for c in datasource.columns
-            if not c.is_complete
-            and c.concept.address in [x.address for x in all_concepts]
-        ]
-        if partial and not accept_partial:
-            return None
+            break_flag = True
+        if g and not g.nodes:
+            break_flag = True
+    if not found:
+        return None
+
+    detect_ambiguity_and_raise(all_concepts, reduced_concept_sets)
+
+    # take our first one as the actual graph
+    g = found[0]
 
-        return PathInfo(
-            paths=paths,
-            datasource=datasource,
-            reduced_concepts=reduce_path_concepts(paths, g),
-            concept_subgraphs=extract_mandatory_subgraphs(paths, g),
+    subgraphs: list[list[Concept]] = []
+    # components = nx.strongly_connected_components(g)
+    components = extract_ds_components(g, node_list)
+    for component in components:
+        # we need to take unique again as different addresses may map to the same concept
+        sub_component = unique(
+            # sorting here is required for reproducibility
+            # todo: we should sort in an optimized order
+            [extract_concept(x, environment) for x in sorted(component)],
+            "address",
         )
-    return None
+        if not sub_component:
+            continue
+        subgraphs.append(sub_component)
+    return subgraphs
+
+
+def subgraphs_to_merge_node(
+    concept_subgraphs: list[list[Concept]],
+    depth: int,
+    all_concepts: List[Concept],
+    environment,
+    g,
+    source_concepts,
+    history,
+    conditions,
+):
+    parents: List[StrategyNode] = []
+    logger.info(
+        f"{padding(depth)}{LOGGER_PREFIX} fetching subgraphs {[[c.address for c in subgraph] for subgraph in concept_subgraphs]}"
+    )
+    for graph in concept_subgraphs:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} fetching subgraph {[c.address for c in graph]}"
+        )
+
+        parent: StrategyNode | None = source_concepts(
+            mandatory_list=graph,
+            environment=environment,
+            g=g,
+            depth=depth + 1,
+            history=history,
+        )
+        if not parent:
+            logger.info(
+                f"{padding(depth)}{LOGGER_PREFIX} Unable to instantiate target subgraph"
+            )
+            return None
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} finished subgraph fetch for {[c.address for c in graph]}, have parent {type(parent)} w/ {[c.address for c in parent.output_concepts]}"
+        )
+        parents.append(parent)
+    input_c = []
+    for x in parents:
+        for y in x.output_concepts:
+            input_c.append(y)
+
+    return MergeNode(
+        input_concepts=unique(input_c, "address"),
+        output_concepts=[x for x in all_concepts],
+        environment=environment,
+        g=g,
+        parents=parents,
+        depth=depth,
+        conditions=conditions,
+        # node_joins=[]
+    )
 
 
 def gen_merge_node(
@@ -87,15 +309,32 @@ def gen_merge_node(
     conditions: Conditional | None = None,
 ) -> Optional[MergeNode]:
     join_candidates: List[PathInfo] = []
-    # anchor on datasources
-    final_all_concepts = []
-    for x in all_concepts:
-        final_all_concepts.append(x)
-    for datasource in environment.datasources.values():
-        path = identify_ds_join_paths(final_all_concepts, g, datasource, accept_partial)
-        if path and path.reduced_concepts:
-            join_candidates.append(path)
-    join_candidates.sort(key=lambda x: sum([len(v) for v in x.paths.values()]))
+
+    # inject new concepts into search, and identify if two dses can reach there
+    if not join_candidates:
+        for filter_downstream in [True, False]:
+            weak_resolve = resolve_weak_components(
+                all_concepts,
+                environment,
+                g,
+                filter_downstream=filter_downstream,
+                accept_partial=accept_partial,
+            )
+            if weak_resolve:
+                log_graph = [[y.address for y in x] for x in weak_resolve]
+                logger.info(
+                    f"{padding(depth)}{LOGGER_PREFIX} Was able to resolve graph through weak component resolution - final graph {log_graph}"
+                )
+                return subgraphs_to_merge_node(
+                    weak_resolve,
+                    depth=depth,
+                    all_concepts=all_concepts,
+                    environment=environment,
+                    g=g,
+                    source_concepts=source_concepts,
+                    history=history,
+                    conditions=conditions,
+                )
     if not join_candidates:
         return None
     join_additions: list[set[str]] = []
@@ -129,41 +368,51 @@ def gen_merge_node(
         key=lambda x: len(x.reduced_concepts),
     )[0]
     logger.info(f"{padding(depth)}{LOGGER_PREFIX} final path is {shortest.paths}")
-    # logger.info(f'{padding(depth)}{LOGGER_PREFIX} final reduced concepts are {shortest.concs}')
-    parents = []
-    for graph in shortest.concept_subgraphs:
-        logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} fetching subgraph {[c.address for c in graph]}"
-        )
-        parent = source_concepts(
-            mandatory_list=graph,
-            environment=environment,
-            g=g,
-            depth=depth + 1,
-            history=history,
-        )
-        if not parent:
-            logger.info(
-                f"{padding(depth)}{LOGGER_PREFIX} Unable to instantiate target subgraph"
-            )
-            return None
-        logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} finished subgraph fetch for {[c.address for c in graph]}, have parent {type(parent)}"
-        )
-        parents.append(parent)
 
-    return MergeNode(
-        input_concepts=[
-            environment.concepts[x]
-            for x in shortest.reduced_concepts
-            if environment.concepts[x].derivation != PurposeLineage.MERGE
-        ],
-        output_concepts=[
-            x for x in all_concepts if x.derivation != PurposeLineage.MERGE
-        ],
+    return subgraphs_to_merge_node(
+        shortest.concept_subgraphs,
+        depth=depth,
+        all_concepts=all_concepts,
         environment=environment,
         g=g,
-        parents=parents,
-        depth=depth,
+        source_concepts=source_concepts,
+        history=history,
         conditions=conditions,
     )
+    # parents = []
+    # for graph in shortest.concept_subgraphs:
+    #     logger.info(
+    #         f"{padding(depth)}{LOGGER_PREFIX} fetching subgraph {[c.address for c in graph]}"
+    #     )
+    #     parent = source_concepts(
+    #         mandatory_list=graph,
+    #         environment=environment,
+    #         g=g,
+    #         depth=depth + 1,
+    #         history=history,
+    #     )
+    #     if not parent:
+    #         logger.info(
+    #             f"{padding(depth)}{LOGGER_PREFIX} Unable to instantiate target subgraph"
+    #         )
+    #         return None
+    #     logger.info(
+    #         f"{padding(depth)}{LOGGER_PREFIX} finished subgraph fetch for {[c.address for c in graph]}, have parent {type(parent)}"
+    #     )
+    #     parents.append(parent)
+
+    # return MergeNode(
+    #     input_concepts=[
+    #         environment.concepts[x]
+    #         for x in shortest.reduced_concepts
+    #         if environment.concepts[x].derivation != PurposeLineage.MERGE
+    #     ],
+    #     output_concepts=[
+    #         x for x in all_concepts if x.derivation != PurposeLineage.MERGE
+    #     ],
+    #     environment=environment,
+    #     g=g,
+    #     parents=parents,
+    #     depth=depth,
+    #     conditions=conditions,
+    # )

trilogy/core/processing/node_generators/rowset_node.py

@@ -35,7 +35,7 @@ def gen_rowset_node(
     lineage: RowsetItem = concept.lineage
     rowset: RowsetDerivationStatement = lineage.rowset
     select: SelectStatement | MultiSelectStatement = lineage.rowset.select
-    parents: List[StrategyNode] = []
+    existence_parents: List[StrategyNode] = []
     if where := select.where_clause:
         targets = select.output_components + where.conditional.row_arguments
         for sub_select in where.conditional.existence_arguments:
@@ -54,7 +54,7 @@ def gen_rowset_node(
                     f"{padding(depth)}{LOGGER_PREFIX} Cannot generate parent existence node for rowset node for {concept}"
                 )
                 return None
-            parents.append(parent_check)
+            existence_parents.append(parent_check)
     else:
         targets = select.output_components
     node: StrategyNode = source_concepts(
@@ -65,18 +65,22 @@ def gen_rowset_node(
         history=history,
     )
 
-    # add our existence concepts in
-    if parents:
-        node.parents += parents
-        for parent in parents:
-            for x in parent.output_concepts:
-                if x.address not in node.output_lcl:
-                    node.existence_concepts.append(x)
     if not node:
         logger.info(
             f"{padding(depth)}{LOGGER_PREFIX} Cannot generate rowset node for {concept}"
         )
         return None
+    # add our existence concepts in
+    if existence_parents:
+        node.parents += existence_parents
+        # we don't need to join to any existence parents
+        if isinstance(node, MergeNode):
+            node.node_joins = []
+        for parent in existence_parents:
+            for x in parent.output_concepts:
+                if x.address not in node.output_lcl:
+                    node.existence_concepts.append(x)
+
     node.conditions = select.where_clause.conditional if select.where_clause else None
     enrichment = set([x.address for x in local_optional])
     rowset_relevant = [x for x in rowset.derived_concepts]
@@ -91,22 +95,21 @@ def gen_rowset_node(
         x for x in select.output_components if x.address in enrichment
     ]
     # add in other other concepts
-    for item in rowset_relevant:
-        node.output_concepts.append(item)
-    for item in additional_relevant:
-        node.output_concepts.append(item)
+
+    node.add_output_concepts(rowset_relevant + additional_relevant)
     if select.where_clause:
         for item in additional_relevant:
             node.partial_concepts.append(item)
-    node.hidden_concepts = rowset_hidden + [
+
+    final_hidden = rowset_hidden + [
         x
         for x in node.output_concepts
         if x.address not in [y.address for y in local_optional + [concept]]
         and x.derivation != PurposeLineage.ROWSET
     ]
+    node.hide_output_concepts(final_hidden)
     # assume grain to be output of select
     # but don't include anything aggregate at this point
-    node.rebuild_cache()
     assert node.resolution_cache
 
     node.resolution_cache.grain = concept_list_to_grain(
@@ -143,7 +146,6 @@ def gen_rowset_node(
             f"{padding(depth)}{LOGGER_PREFIX} Cannot generate rowset enrichment node for {concept} with optional {local_optional}, returning just rowset node"
         )
         return node
-
     return MergeNode(
         input_concepts=enrich_node.output_concepts + node.output_concepts,
         output_concepts=node.output_concepts + local_optional,