pytrilogy 0.0.2.58__py3-none-any.whl → 0.0.3.0__py3-none-any.whl

trilogy/core/processing/node_generators/multiselect_node.py

@@ -3,15 +3,14 @@ from itertools import combinations
 from typing import List
 
 from trilogy.constants import logger
-from trilogy.core.enums import BooleanOperator, JoinType, Purpose
-from trilogy.core.models import (
-    Concept,
-    Conditional,
-    Environment,
-    Grain,
-    MultiSelectStatement,
-    WhereClause,
+from trilogy.core.enums import JoinType, Purpose
+from trilogy.core.models.build import (
+    BuildConcept,
+    BuildGrain,
+    BuildMultiSelectLineage,
+    BuildWhereClause,
 )
+from trilogy.core.models.build_environment import BuildEnvironment
 from trilogy.core.processing.node_generators.common import resolve_join_order
 from trilogy.core.processing.nodes import History, MergeNode, NodeJoin
 from trilogy.core.processing.nodes.base_node import StrategyNode
@@ -21,12 +20,14 @@ LOGGER_PREFIX = "[GEN_MULTISELECT_NODE]"
 
 
 def extra_align_joins(
-    base: MultiSelectStatement, parents: List[StrategyNode]
+    base: BuildMultiSelectLineage,
+    environment: BuildEnvironment,
+    parents: List[StrategyNode],
 ) -> List[NodeJoin]:
     node_merge_concept_map = defaultdict(list)
     output = []
     for align in base.align.items:
-        jc = align.gen_concept(base)
+        jc = environment.concepts[align.aligned_concept]
         if jc.purpose == Purpose.CONSTANT:
             continue
         for node in parents:
@@ -52,51 +53,43 @@ def extra_align_joins(
 
 
 def gen_multiselect_node(
-    concept: Concept,
-    local_optional: List[Concept],
-    environment: Environment,
+    concept: BuildConcept,
+    local_optional: List[BuildConcept],
+    environment: BuildEnvironment,
     g,
     depth: int,
     source_concepts,
-    history: History | None = None,
-    conditions: WhereClause | None = None,
+    history: History,
+    conditions: BuildWhereClause | None = None,
 ) -> MergeNode | None:
-    if not isinstance(concept.lineage, MultiSelectStatement):
+    from trilogy.core.query_processor import get_query_node
+
+    if not isinstance(concept.lineage, BuildMultiSelectLineage):
         logger.info(
             f"{padding(depth)}{LOGGER_PREFIX} Cannot generate multiselect node for {concept}"
         )
         return None
-    lineage: MultiSelectStatement = concept.lineage
+    lineage: BuildMultiSelectLineage = concept.lineage
 
     base_parents: List[StrategyNode] = []
     partial = []
     for select in lineage.selects:
-        snode: StrategyNode = source_concepts(
-            mandatory_list=select.output_components,
-            environment=environment,
-            g=g,
-            depth=depth + 1,
-            history=history,
-            conditions=select.where_clause,
+
+        snode: StrategyNode = get_query_node(history.base_environment, select)
+        # raise SyntaxError(select.output_components)
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} Fetched parent node with outputs {select.output_components}"
         )
         if not snode:
             logger.info(
                 f"{padding(depth)}{LOGGER_PREFIX} Cannot generate multiselect node for {concept}"
             )
             return None
-        if select.having_clause:
-            if snode.conditions:
-                snode.conditions = Conditional(
-                    left=snode.conditions,
-                    right=select.having_clause.conditional,
-                    operator=BooleanOperator.AND,
-                )
-            else:
-                snode.conditions = select.having_clause.conditional
         merge_concepts = []
         for x in [*snode.output_concepts]:
-            merge = lineage.get_merge_concept(x)
-            if merge:
+            merge_name = lineage.get_merge_concept(x)
+            if merge_name:
+                merge = environment.concepts[merge_name]
                 snode.output_concepts.append(merge)
                 merge_concepts.append(merge)
         # clear cache so QPS
@@ -107,24 +100,46 @@ def gen_multiselect_node(
         if select.where_clause:
             for item in select.output_components:
                 partial.append(item)
+        logger.info(snode.hidden_concepts)
 
-    node_joins = extra_align_joins(lineage, base_parents)
+    node_joins = extra_align_joins(lineage, environment, base_parents)
+    logger.info(
+        f"Non-hidden {[x for y in base_parents for x in y.output_concepts if x.address not in y.hidden_concepts]}"
+    )
     node = MergeNode(
-        input_concepts=[x for y in base_parents for x in y.output_concepts],
-        output_concepts=[x for y in base_parents for x in y.output_concepts],
+        input_concepts=[
+            x
+            for y in base_parents
+            for x in y.output_concepts
+            if x.address not in y.hidden_concepts
+        ],
+        output_concepts=[
+            x
+            for y in base_parents
+            for x in y.output_concepts
+            if x.address not in y.hidden_concepts
+        ],
         environment=environment,
         depth=depth,
         parents=base_parents,
         node_joins=node_joins,
-        hidden_concepts=set([x for y in base_parents for x in y.hidden_concepts]),
+        grain=BuildGrain.from_concepts(
+            [
+                x
+                for y in base_parents
+                for x in y.output_concepts
+                if x.address not in y.hidden_concepts
+            ],
+            environment=environment,
+        ),
     )
 
     enrichment = set([x.address for x in local_optional])
 
     multiselect_relevant = [
-        x
+        environment.concepts[x]
         for x in lineage.derived_concepts
-        if x.address == concept.address or x.address in enrichment
+        if x == concept.address or x in enrichment
     ]
     additional_relevant = [x for x in node.output_concepts if x.address in enrichment]
     # add in other other concepts
@@ -135,7 +150,7 @@ def gen_multiselect_node(
     # if select.where_clause:
     #     for item in additional_relevant:
     #         node.partial_concepts.append(item)
-    node.grain = Grain.from_concepts(node.output_concepts, environment=environment)
+    node.grain = BuildGrain.from_concepts(node.output_concepts, environment=environment)
     node.rebuild_cache()
     # we need a better API for refreshing a nodes QDS
     possible_joins = concept_to_relevant_joins(additional_relevant)
@@ -156,7 +171,9 @@ def gen_multiselect_node(
             f"{padding(depth)}{LOGGER_PREFIX} all enriched concepts returned from base rowset node; exiting early"
         )
         return node
-
+    logger.info(
+        f"{padding(depth)}{LOGGER_PREFIX} Missing required concepts {[x for x in local_optional if x.address not in [y.address for y in node.output_concepts]]}"
+    )
     enrich_node: MergeNode = source_concepts(  # this fetches the parent + join keys
         # to then connect to the rest of the query
         mandatory_list=additional_relevant + local_optional,

trilogy/core/processing/node_generators/node_merge_node.py

@@ -4,10 +4,11 @@ import networkx as nx
 from networkx.algorithms import approximation as ax
 
 from trilogy.constants import logger
-from trilogy.core.enums import PurposeLineage
+from trilogy.core.enums import Derivation
 from trilogy.core.exceptions import AmbiguousRelationshipResolutionException
 from trilogy.core.graph_models import concept_to_node
-from trilogy.core.models import Concept, Conditional, Environment, WhereClause
+from trilogy.core.models.build import BuildConcept, BuildConditional, BuildWhereClause
+from trilogy.core.models.build_environment import BuildEnvironment
 from trilogy.core.processing.nodes import History, MergeNode, StrategyNode
 from trilogy.core.processing.utility import padding
 from trilogy.utility import unique
@@ -33,7 +34,7 @@ def extract_address(node: str):
     return node.split("~")[1].split("@")[0]
 
 
-def extract_concept(node: str, env: Environment):
+def extract_concept(node: str, env: BuildEnvironment):
     if node in env.alias_origin_lookup:
         return env.alias_origin_lookup[node]
     return env.concepts[node]
@@ -76,7 +77,7 @@ def extract_ds_components(g: nx.DiGraph, nodelist: list[str]) -> list[list[str]]
 def determine_induced_minimal_nodes(
     G: nx.DiGraph,
     nodelist: list[str],
-    environment: Environment,
+    environment: BuildEnvironment,
     filter_downstream: bool,
     accept_partial: bool = False,
 ) -> nx.DiGraph | None:
@@ -86,11 +87,11 @@ def determine_induced_minimal_nodes(
 
     for node in G.nodes:
         if concepts.get(node):
-            lookup: Concept = concepts[node]
-            if lookup.derivation in (PurposeLineage.CONSTANT,):
+            lookup: BuildConcept = concepts[node]
+            if lookup.derivation in (Derivation.CONSTANT,):
                 nodes_to_remove.append(node)
             # purge a node if we're already looking for all it's parents
-            if filter_downstream and lookup.derivation not in (PurposeLineage.ROOT,):
+            if filter_downstream and lookup.derivation not in (Derivation.ROOT,):
                 nodes_to_remove.append(node)
 
     H.remove_nodes_from(nodes_to_remove)
@@ -106,8 +107,8 @@ def determine_induced_minimal_nodes(
 
     try:
         paths = nx.multi_source_dijkstra_path(H, nodelist)
-    except nx.exception.NodeNotFound:
-        logger.debug(f"Unable to find paths for {nodelist}")
+    except nx.exception.NodeNotFound as e:
+        logger.debug(f"Unable to find paths for {nodelist}- {str(e)}")
         return None
     H.remove_nodes_from(list(x for x in H.nodes if x not in paths))
     sG: nx.Graph = ax.steinertree.steiner_tree(H, nodelist).copy()
@@ -140,14 +141,17 @@ def determine_induced_minimal_nodes(
         return None
 
     if not all([node in final.nodes for node in nodelist]):
-        logger.debug(f"Skipping graph for {nodelist} as missing nodes")
+        missing = [node for node in nodelist if node not in final.nodes]
+        logger.debug(
+            f"Skipping graph for {nodelist} as missing nodes {missing} from {final.nodes}"
+        )
         return None
     logger.debug(f"Found final graph {final.nodes}")
     return final
 
 
 def detect_ambiguity_and_raise(
-    all_concepts: list[Concept], reduced_concept_sets: list[set[str]]
+    all_concepts: list[BuildConcept], reduced_concept_sets: list[set[str]]
 ) -> None:
     final_candidates: list[set[str]] = []
     common: set[str] = set()
@@ -167,7 +171,7 @@ def detect_ambiguity_and_raise(
         )
 
 
-def has_synonym(concept: Concept, others: list[list[Concept]]) -> bool:
+def has_synonym(concept: BuildConcept, others: list[list[BuildConcept]]) -> bool:
     return any(
         c.address in concept.pseudonyms or concept.address in c.pseudonyms
         for sublist in others
@@ -175,7 +179,9 @@ def has_synonym(concept: Concept, others: list[list[Concept]]) -> bool:
     )
 
 
-def filter_relevant_subgraphs(subgraphs: list[list[Concept]]) -> list[list[Concept]]:
+def filter_relevant_subgraphs(
+    subgraphs: list[list[BuildConcept]],
+) -> list[list[BuildConcept]]:
     return [
         subgraph
         for subgraph in subgraphs
@@ -187,20 +193,39 @@ def filter_relevant_subgraphs(subgraphs: list[list[Concept]]) -> list[list[Conce
     ]
 
 
+def filter_duplicate_subgraphs(
+    subgraphs: list[list[BuildConcept]],
+) -> list[list[BuildConcept]]:
+    seen: list[set[str]] = []
+
+    for graph in subgraphs:
+        seen.append(set([x.address for x in graph]))
+    final = []
+    # sometimes w can get two subcomponents that are the same
+    # due to alias resolution
+    # if so, drop any that are strict subsets.
+    for graph in subgraphs:
+        set_x = set([x.address for x in graph])
+        if any([set_x.issubset(y) and set_x != y for y in seen]):
+            continue
+        final.append(graph)
+    return final
+
+
 def resolve_weak_components(
-    all_concepts: List[Concept],
-    environment: Environment,
+    all_concepts: List[BuildConcept],
+    environment: BuildEnvironment,
     environment_graph: nx.DiGraph,
     filter_downstream: bool = True,
     accept_partial: bool = False,
-) -> list[list[Concept]] | None:
+) -> list[list[BuildConcept]] | None:
     break_flag = False
     found = []
     search_graph = environment_graph.copy()
     reduced_concept_sets: list[set[str]] = []
 
     # loop through, removing new nodes we find
-    # to ensure there are not ambiguous loops
+    # to ensure there are not ambiguous discovery paths
     # (if we did not care about raising ambiguity errors, we could just use the first one)
     count = 0
     node_list = [
@@ -211,6 +236,8 @@ def resolve_weak_components(
     synonyms: set[str] = set()
     for x in all_concepts:
         synonyms = synonyms.union(x.pseudonyms)
+    # from trilogy.hooks.graph_hook import GraphHook
+    # GraphHook().query_graph_built(search_graph, highlight_nodes=[concept_to_node(c.with_default_grain()) for c in all_concepts if "__preql_internal" not in c.address])
     while break_flag is not True:
         count += 1
         if count > AMBIGUITY_CHECK_LIMIT:
@@ -239,6 +266,7 @@ def resolve_weak_components(
             new = [x for x in all_graph_concepts if x.address not in all_concepts]
 
             new_addresses = set([x.address for x in new if x.address not in synonyms])
+
             if not new:
                 break_flag = True
             # remove our new nodes for the next search path
@@ -264,7 +292,7 @@ def resolve_weak_components(
     # take our first one as the actual graph
     g = found[0]
 
-    subgraphs: list[list[Concept]] = []
+    subgraphs: list[list[BuildConcept]] = []
     # components = nx.strongly_connected_components(g)
     node_list = [x for x in g.nodes if x.startswith("c~")]
     components = extract_ds_components(g, node_list)
@@ -280,20 +308,22 @@ def resolve_weak_components(
         if not sub_component:
             continue
         subgraphs.append(sub_component)
-    return subgraphs
+    final = filter_duplicate_subgraphs(subgraphs)
+    return final
     # return filter_relevant_subgraphs(subgraphs)
 
 
 def subgraphs_to_merge_node(
-    concept_subgraphs: list[list[Concept]],
+    concept_subgraphs: list[list[BuildConcept]],
     depth: int,
-    all_concepts: List[Concept],
+    all_concepts: List[BuildConcept],
     environment,
     g,
     source_concepts,
     history,
     conditions,
-    search_conditions: WhereClause | None = None,
+    output_concepts: List[BuildConcept],
+    search_conditions: BuildWhereClause | None = None,
     enable_early_exit: bool = True,
 ):
     parents: List[StrategyNode] = []
@@ -331,16 +361,13 @@ def subgraphs_to_merge_node(
             f"{padding(depth)}{LOGGER_PREFIX} only one parent node, exiting early w/ {[c.address for c in parents[0].output_concepts]}"
         )
         return parents[0]
-    base_output = [x for x in all_concepts]
-    # for x in base_output:
-    #     if x not in input_c:
-    #         input_c.append(x)
     return MergeNode(
         input_concepts=unique(input_c, "address"),
-        output_concepts=base_output,
+        output_concepts=output_concepts,
         environment=environment,
         parents=parents,
         depth=depth,
+        # hidden_concepts=[]
         # conditions=conditions,
         # conditions=search_conditions.conditional,
         # preexisting_conditions=search_conditions.conditional,
@@ -349,33 +376,41 @@ def subgraphs_to_merge_node(
 
 
 def gen_merge_node(
-    all_concepts: List[Concept],
+    all_concepts: List[BuildConcept],
     g: nx.DiGraph,
-    environment: Environment,
+    environment: BuildEnvironment,
     depth: int,
     source_concepts,
     accept_partial: bool = False,
     history: History | None = None,
-    conditions: Conditional | None = None,
-    search_conditions: WhereClause | None = None,
+    conditions: BuildConditional | None = None,
+    search_conditions: BuildWhereClause | None = None,
 ) -> Optional[MergeNode]:
     if search_conditions:
-        all_concepts = unique(all_concepts + search_conditions.row_arguments, "address")
+        all_search_concepts = unique(
+            all_concepts + list(search_conditions.row_arguments), "address"
+        )
+    else:
+        all_search_concepts = all_concepts
     for filter_downstream in [True, False]:
         weak_resolve = resolve_weak_components(
-            all_concepts,
+            all_search_concepts,
             environment,
             g,
             filter_downstream=filter_downstream,
             accept_partial=accept_partial,
         )
-        if weak_resolve:
+        if not weak_resolve:
+            logger.info(
+                f"{padding(depth)}{LOGGER_PREFIX} wasn't able to resolve graph through intermediate concept injection with accept_partial {accept_partial}, filter_downstream {filter_downstream}"
+            )
+        else:
             log_graph = [[y.address for y in x] for x in weak_resolve]
             logger.info(
                 f"{padding(depth)}{LOGGER_PREFIX} Was able to resolve graph through weak component resolution - final graph {log_graph}"
             )
             for flat in log_graph:
-                if set(flat) == set([x.address for x in all_concepts]):
+                if set(flat) == set([x.address for x in all_search_concepts]):
                     logger.info(
                         f"{padding(depth)}{LOGGER_PREFIX} expanded concept resolution was identical to search resolution; breaking to avoid recursion error."
                     )
@@ -383,17 +418,19 @@ def gen_merge_node(
             return subgraphs_to_merge_node(
                 weak_resolve,
                 depth=depth,
-                all_concepts=all_concepts,
+                all_concepts=all_search_concepts,
                 environment=environment,
                 g=g,
                 source_concepts=source_concepts,
                 history=history,
                 conditions=conditions,
                 search_conditions=search_conditions,
+                output_concepts=all_concepts,
             )
+
     # one concept handling may need to be kicked to alias
-    if len(all_concepts) == 1:
-        concept = all_concepts[0]
+    if len(all_search_concepts) == 1:
+        concept = all_search_concepts[0]
         for v in concept.pseudonyms:
             test = subgraphs_to_merge_node(
                 [[concept, environment.alias_origin_lookup[v]]],
@@ -406,6 +443,7 @@ def gen_merge_node(
                 conditions=conditions,
                 enable_early_exit=False,
                 search_conditions=search_conditions,
+                output_concepts=[concept],
             )
             if test:
                 return test

trilogy/core/processing/node_generators/rowset_node.py

@@ -1,17 +1,18 @@
 from typing import List
 
 from trilogy.constants import logger
-from trilogy.core.enums import PurposeLineage
-from trilogy.core.models import (
-    Concept,
-    Environment,
-    Grain,
-    MultiSelectStatement,
-    RowsetDerivationStatement,
-    RowsetItem,
-    SelectStatement,
-    WhereClause,
+from trilogy.core.enums import Derivation
+from trilogy.core.exceptions import UnresolvableQueryException
+from trilogy.core.models.author import MultiSelectLineage, SelectLineage
+from trilogy.core.models.build import (
+    BuildConcept,
+    BuildGrain,
+    BuildRowsetItem,
+    BuildRowsetLineage,
+    BuildWhereClause,
+    Factory,
 )
+from trilogy.core.models.build_environment import BuildEnvironment
 from trilogy.core.processing.nodes import History, MergeNode, StrategyNode
 from trilogy.core.processing.utility import concept_to_relevant_joins, padding
 
@@ -19,62 +20,76 @@ LOGGER_PREFIX = "[GEN_ROWSET_NODE]"
 
 
 def gen_rowset_node(
-    concept: Concept,
-    local_optional: List[Concept],
-    environment: Environment,
+    concept: BuildConcept,
+    local_optional: List[BuildConcept],
+    environment: BuildEnvironment,
     g,
     depth: int,
     source_concepts,
-    history: History | None = None,
-    conditions: WhereClause | None = None,
+    history: History,
+    conditions: BuildWhereClause | None = None,
 ) -> StrategyNode | None:
     from trilogy.core.query_processor import get_query_node
 
-    if not isinstance(concept.lineage, RowsetItem):
+    if not isinstance(concept.lineage, BuildRowsetItem):
         raise SyntaxError(
-            f"Invalid lineage passed into rowset fetch, got {type(concept.lineage)}, expected {RowsetItem}"
+            f"Invalid lineage passed into rowset fetch, got {type(concept.lineage)}, expected {BuildRowsetItem}"
         )
-    lineage: RowsetItem = concept.lineage
-    rowset: RowsetDerivationStatement = lineage.rowset
-    select: SelectStatement | MultiSelectStatement = lineage.rowset.select
+    lineage: BuildRowsetItem = concept.lineage
+    rowset: BuildRowsetLineage = lineage.rowset
+    select: SelectLineage | MultiSelectLineage = lineage.rowset.select
 
-    node = get_query_node(environment, select)
+    node = get_query_node(history.base_environment, select)
 
     if not node:
         logger.info(
             f"{padding(depth)}{LOGGER_PREFIX} Cannot generate parent rowset node for {concept}"
         )
-        return None
+        raise UnresolvableQueryException(
+            f"Cannot generate parent select for concept {concept} in rowset {rowset.name}; ensure the rowset is a valid statement."
+        )
     enrichment = set([x.address for x in local_optional])
-    rowset_relevant = [x for x in rowset.derived_concepts]
-    select_hidden = select.hidden_components
+
+    factory = Factory(environment=history.base_environment, grain=select.grain)
+    rowset_relevant: list[BuildConcept] = [
+        v
+        for v in environment.concepts.values()
+        if isinstance(v.lineage, BuildRowsetItem)
+        and v.lineage.rowset.name == rowset.name
+    ]
+    # logger.info(
+    #     f"{padding(depth)}{LOGGER_PREFIX} rowset relevant nodes are {rowset_relevant}"
+    # )
+    select_hidden = node.hidden_concepts
     rowset_hidden = [
         x
-        for x in rowset.derived_concepts
-        if isinstance(x.lineage, RowsetItem)
+        for x in rowset_relevant
+        if isinstance(x.lineage, BuildRowsetItem)
         and x.lineage.content.address in select_hidden
     ]
     additional_relevant = [
-        x for x in select.output_components if x.address in enrichment
+        factory.build(x) for x in select.output_components if x.address in enrichment
     ]
     # add in other other concepts
-
     node.add_output_concepts(rowset_relevant + additional_relevant)
     if select.where_clause:
         for item in additional_relevant:
+            logger.info(
+                f"{padding(depth)}{LOGGER_PREFIX} adding {item} to partial concepts"
+            )
             node.partial_concepts.append(item)
 
     final_hidden = rowset_hidden + [
         x
         for x in node.output_concepts
         if x.address not in local_optional + [concept]
-        and x.derivation != PurposeLineage.ROWSET
+        and x.derivation != Derivation.ROWSET
     ]
     node.hide_output_concepts(final_hidden)
     assert node.resolution_cache
     # assume grain to be output of select
     # but don't include anything hidden(the non-rowset concepts)
-    node.grain = Grain.from_concepts(
+    node.grain = BuildGrain.from_concepts(
         [
             x
             for x in node.output_concepts
@@ -82,29 +97,35 @@ def gen_rowset_node(
             not in [
                 y
                 for y in node.hidden_concepts
-                if environment.concepts[y].derivation != PurposeLineage.ROWSET
+                if y in environment.concepts
+                and environment.concepts[y].derivation != Derivation.ROWSET
             ]
         ],
     )
 
     node.rebuild_cache()
-
     if not local_optional or all(
-        x.address in node.output_concepts for x in local_optional
+        x.address in node.output_concepts and x.address not in node.partial_concepts
+        for x in local_optional
     ):
         logger.info(
             f"{padding(depth)}{LOGGER_PREFIX} no enrichment required for rowset node as all optional {[x.address for x in local_optional]} found or no optional; exiting early."
         )
         return node
+    remaining = [
+        x
+        for x in local_optional
+        if x not in node.output_concepts or x in node.partial_concepts
+    ]
     possible_joins = concept_to_relevant_joins(
-        [x for x in node.output_concepts if x.derivation != PurposeLineage.ROWSET]
+        [x for x in node.output_concepts if x.derivation != Derivation.ROWSET]
     )
     if not possible_joins:
         logger.info(
             f"{padding(depth)}{LOGGER_PREFIX} no possible joins for rowset node to get {[x.address for x in local_optional]}; have {[x.address for x in node.output_concepts]}"
         )
         return node
-    if any(x.derivation == PurposeLineage.ROWSET for x in possible_joins):
+    if any(x.derivation == Derivation.ROWSET for x in possible_joins):
 
         logger.info(
             f"{padding(depth)}{LOGGER_PREFIX} cannot enrich rowset node with rowset concepts; exiting early"
@@ -113,7 +134,7 @@ def gen_rowset_node(
     logger.info([x.address for x in possible_joins + local_optional])
     enrich_node: MergeNode = source_concepts(  # this fetches the parent + join keys
         # to then connect to the rest of the query
-        mandatory_list=possible_joins + local_optional,
+        mandatory_list=possible_joins + remaining,
         environment=environment,
         g=g,
         depth=depth + 1,