pytrilogy 0.0.2.11__py3-none-any.whl → 0.0.2.13__py3-none-any.whl

trilogy/core/processing/node_generators/basic_node.py

@@ -10,6 +10,8 @@ from trilogy.core.processing.node_generators.common import (
 )
 from trilogy.utility import unique
 from trilogy.constants import logger
+from trilogy.core.enums import SourceType
+from itertools import combinations
 
 LOGGER_PREFIX = "[GEN_BASIC_NODE]"
 
@@ -31,12 +33,21 @@ def gen_basic_node(
     )
 
     local_optional_redundant = [x for x in local_optional if x in parent_concepts]
-    attempts = [(parent_concepts, [concept] + local_optional_redundant)]
-    from itertools import combinations
+    attempts: List[tuple[list[Concept], list[Concept]]] = [
+        (parent_concepts, [concept] + local_optional_redundant)
+    ]
+    equivalent_optional = [
+        x
+        for x in local_optional
+        if x.lineage == concept.lineage and x.address != concept.address
+    ]
+    non_equivalent_optional = [
+        x for x in local_optional if x not in equivalent_optional
+    ]
 
     if local_optional:
-        for combo in range(1, len(local_optional) + 1):
-            combos = combinations(local_optional, combo)
+        for combo in range(1, len(non_equivalent_optional) + 1):
+            combos = combinations(non_equivalent_optional, combo)
             for optional_set in combos:
                 attempts.append(
                     (
@@ -55,8 +66,10 @@ def gen_basic_node(
             depth=depth + 1,
             history=history,
         )
+
         if not parent_node:
             continue
+        parent_node.source_type = SourceType.BASIC
         parents: List[StrategyNode] = [parent_node]
         for x in basic_output:
             sources = [p for p in parents if x in p.output_concepts]
@@ -64,13 +77,10 @@ def gen_basic_node(
                 continue
             if all(x in source.partial_concepts for source in sources):
                 partials.append(x)
-        outputs = parent_node.output_concepts + [concept]
-        logger.info(
-            f"{depth_prefix}{LOGGER_PREFIX} Returning basic select for {concept} with attempted extra {[x.address for x in attempt]}, output {[x.address for x in outputs]}"
-        )
-        # parents.resolve()
 
         parent_node.add_output_concept(concept)
+        for x in equivalent_optional:
+            parent_node.add_output_concept(x)
 
         parent_node.remove_output_concepts(
             [
@@ -79,6 +89,9 @@ def gen_basic_node(
                 if x.address not in [y.address for y in basic_output]
             ]
         )
+        logger.info(
+            f"{depth_prefix}{LOGGER_PREFIX} Returning basic select for {concept} with attempted extra {[x.address for x in attempt]}, output {[x.address for x in parent_node.output_concepts]}"
+        )
         return parent_node
     logger.info(
         f"{depth_prefix}{LOGGER_PREFIX} No basic node could be generated for {concept}"

trilogy/core/processing/node_generators/common.py

@@ -1,4 +1,4 @@
-from typing import List, Tuple
+from typing import List, Tuple, Callable
 
 
 from trilogy.core.enums import PurposeLineage, Purpose
@@ -96,6 +96,7 @@ def gen_property_enrichment_node(
     g,
     depth: int,
     source_concepts,
+    log_lambda: Callable,
     history: History | None = None,
     conditions: WhereClause | None = None,
 ):
@@ -106,8 +107,8 @@ def gen_property_enrichment_node(
         keys = "-".join([y.address for y in x.keys])
         required_keys[keys].add(x.address)
     final_nodes = []
-    node_joins = []
     for _k, vs in required_keys.items():
+        log_lambda(f"Generating enrichment node for {_k} with {vs}")
         ks = _k.split("-")
         enrich_node: StrategyNode = source_concepts(
             mandatory_list=[environment.concepts[k] for k in ks]
@@ -119,17 +120,6 @@ def gen_property_enrichment_node(
             conditions=conditions,
         )
         final_nodes.append(enrich_node)
-        node_joins.append(
-            NodeJoin(
-                left_node=enrich_node,
-                right_node=base_node,
-                concepts=concept_to_relevant_joins(
-                    [environment.concepts[k] for k in ks]
-                ),
-                filter_to_mutual=False,
-                join_type=JoinType.LEFT_OUTER,
-            )
-        )
     return MergeNode(
         input_concepts=unique(
             base_node.output_concepts
@@ -146,9 +136,8 @@ def gen_property_enrichment_node(
         g=g,
         parents=[
             base_node,
-            enrich_node,
-        ],
-        node_joins=node_joins,
+        ]
+        + final_nodes,
     )
 
 
@@ -197,6 +186,7 @@ def gen_enrichment_node(
                 source_concepts,
                 history=history,
                 conditions=conditions,
+                log_lambda=log_lambda,
             )
 
     enrich_node: StrategyNode = source_concepts(  # this fetches the parent + join keys
@@ -216,14 +206,14 @@ def gen_enrichment_node(
     log_lambda(
         f"{str(type(base_node).__name__)} returning merge node with group node + enrichment node"
     )
-
+    non_hidden = [
+        x
+        for x in base_node.output_concepts
+        if x.address not in [y.address for y in base_node.hidden_concepts]
+    ]
     return MergeNode(
-        input_concepts=unique(
-            join_keys + extra_required + base_node.output_concepts, "address"
-        ),
-        output_concepts=unique(
-            join_keys + extra_required + base_node.output_concepts, "address"
-        ),
+        input_concepts=unique(join_keys + extra_required + non_hidden, "address"),
+        output_concepts=unique(join_keys + extra_required + non_hidden, "address"),
         environment=environment,
         g=g,
         parents=[enrich_node, base_node],

trilogy/core/processing/node_generators/node_merge_node.py

@@ -86,7 +86,7 @@ def determine_induced_minimal_nodes(
 
     for node in G.nodes:
         if concepts.get(node):
-            lookup = concepts[node]
+            lookup: Concept = concepts[node]
             if lookup.derivation not in (PurposeLineage.BASIC, PurposeLineage.ROOT):
                 nodes_to_remove.append(node)
             elif lookup.derivation == PurposeLineage.BASIC and G.out_degree(node) == 0:
@@ -155,6 +155,26 @@ def detect_ambiguity_and_raise(all_concepts, reduced_concept_sets) -> None:
         )
 
 
+def has_synonym(concept: Concept, others: list[list[Concept]]) -> bool:
+    return any(
+        c.address in concept.pseudonyms or concept.address in c.pseudonyms
+        for sublist in others
+        for c in sublist
+    )
+
+
+def filter_relevant_subgraphs(subgraphs: list[list[Concept]]) -> list[list[Concept]]:
+    return [
+        subgraph
+        for subgraph in subgraphs
+        if len(subgraph) > 1
+        or (
+            len(subgraph) == 1
+            and not has_synonym(subgraph[0], [x for x in subgraphs if x != subgraph])
+        )
+    ]
+
+
 def resolve_weak_components(
     all_concepts: List[Concept],
     environment: Environment,
@@ -249,6 +269,7 @@ def resolve_weak_components(
             continue
         subgraphs.append(sub_component)
     return subgraphs
+    # return filter_relevant_subgraphs(subgraphs)
 
 
 def subgraphs_to_merge_node(

trilogy/core/processing/node_generators/unnest_node.py

@@ -22,9 +22,14 @@ def gen_unnest_node(
     arguments = []
     if isinstance(concept.lineage, Function):
         arguments = concept.lineage.concept_arguments
+
+    equivalent_optional = [x for x in local_optional if x.lineage == concept.lineage]
+    non_equivalent_optional = [
+        x for x in local_optional if x not in equivalent_optional
+    ]
     if arguments or local_optional:
         parent = source_concepts(
-            mandatory_list=arguments + local_optional,
+            mandatory_list=arguments + non_equivalent_optional,
             environment=environment,
             g=g,
             depth=depth + 1,
@@ -38,8 +43,8 @@ def gen_unnest_node(
             return None
 
     base = UnnestNode(
-        unnest_concept=concept,
-        input_concepts=arguments + local_optional,
+        unnest_concepts=[concept] + equivalent_optional,
+        input_concepts=arguments + non_equivalent_optional,
         output_concepts=[concept] + local_optional,
         environment=environment,
         g=g,
@@ -57,4 +62,6 @@ def gen_unnest_node(
     )
     qds = new.resolve()
     assert qds.source_map[concept.address] == {base.resolve()}
+    for x in equivalent_optional:
+        assert qds.source_map[x.address] == {base.resolve()}
     return new

trilogy/core/processing/nodes/base_node.py

@@ -61,26 +61,28 @@ def resolve_concept_map(
         for concept in input.output_concepts:
             if concept.address not in input.non_partial_concept_addresses:
                 continue
-            if concept.address not in inherited:
-                continue
-            if (
-                isinstance(input, QueryDatasource)
-                and concept.address in input.hidden_concepts
-            ):
+            if isinstance(input, QueryDatasource) and concept.address in [
+                x.address for x in input.hidden_concepts
+            ]:
                 continue
             if concept.address in full_addresses:
+
                 concept_map[concept.address].add(input)
             elif concept.address not in concept_map:
+                # equi_targets = [x for x in targets if concept.address in x.pseudonyms or x.address in concept.pseudonyms]
+                # if equi_targets:
+                #     for equi in equi_targets:
+                #         concept_map[equi.address] = set()
                 concept_map[concept.address].add(input)
+
     # second loop, include partials
     for input in inputs:
         for concept in input.output_concepts:
             if concept.address not in [t.address for t in inherited_inputs]:
                 continue
-            if (
-                isinstance(input, QueryDatasource)
-                and concept.address in input.hidden_concepts
-            ):
+            if isinstance(input, QueryDatasource) and concept.address in [
+                x.address for x in input.hidden_concepts
+            ]:
                 continue
             if len(concept_map.get(concept.address, [])) == 0:
                 concept_map[concept.address].add(input)
@@ -89,6 +91,7 @@ def resolve_concept_map(
         if target.address not in inherited:
             # an empty source means it is defined in this CTE
             concept_map[target.address] = set()
+
     return concept_map
 
 
@@ -221,7 +224,10 @@ class StrategyNode:
     def remove_output_concepts(self, concepts: List[Concept]):
         for x in concepts:
             self.hidden_concepts.append(x)
-        self.output_concepts = [x for x in self.output_concepts if x not in concepts]
+        addresses = [x.address for x in concepts]
+        self.output_concepts = [
+            x for x in self.output_concepts if x.address not in addresses
+        ]
         self.rebuild_cache()
 
     @property
@@ -252,6 +258,7 @@ class StrategyNode:
             targets=self.output_concepts,
             inherited_inputs=self.input_concepts + self.existence_concepts,
         )
+
         return QueryDatasource(
             input_concepts=self.input_concepts,
             output_concepts=self.output_concepts,

trilogy/core/processing/nodes/group_node.py

@@ -121,7 +121,6 @@ class GroupNode(StrategyNode):
             source_type=source_type,
             source_map=resolve_concept_map(
                 parent_sources,
-                # targets = self.output_concepts,
                 targets=(
                     unique(
                         self.output_concepts + self.conditions.concept_arguments,

trilogy/core/processing/nodes/merge_node.py

@@ -28,14 +28,18 @@ LOGGER_PREFIX = "[CONCEPT DETAIL - MERGE NODE]"
 
 
 def deduplicate_nodes(
-    merged: dict[str, QueryDatasource | Datasource], logging_prefix: str
+    merged: dict[str, QueryDatasource | Datasource],
+    logging_prefix: str,
+    environment: Environment,
 ) -> tuple[bool, dict[str, QueryDatasource | Datasource], set[str]]:
     duplicates = False
     removed: set[str] = set()
     set_map: dict[str, set[str]] = {}
     for k, v in merged.items():
         unique_outputs = [
-            x.address for x in v.output_concepts if x not in v.partial_concepts
+            environment.concepts[x.address].address
+            for x in v.output_concepts
+            if x not in v.partial_concepts
         ]
         set_map[k] = set(unique_outputs)
     for k1, v1 in set_map.items():
@@ -71,12 +75,15 @@ def deduplicate_nodes_and_joins(
     joins: List[NodeJoin] | None,
     merged: dict[str, QueryDatasource | Datasource],
     logging_prefix: str,
+    environment: Environment,
 ) -> Tuple[List[NodeJoin] | None, dict[str, QueryDatasource | Datasource]]:
     # it's possible that we have more sources than we need
     duplicates = True
     while duplicates:
         duplicates = False
-        duplicates, merged, removed = deduplicate_nodes(merged, logging_prefix)
+        duplicates, merged, removed = deduplicate_nodes(
+            merged, logging_prefix, environment=environment
+        )
         # filter out any removed joins
         if joins is not None:
             joins = [
@@ -245,7 +252,7 @@ class MergeNode(StrategyNode):
 
         # it's possible that we have more sources than we need
         final_joins, merged = deduplicate_nodes_and_joins(
-            final_joins, merged, self.logging_prefix
+            final_joins, merged, self.logging_prefix, self.environment
         )
         # early exit if we can just return the parent
         final_datasets: List[QueryDatasource | Datasource] = list(merged.values())
@@ -364,7 +371,7 @@ class MergeNode(StrategyNode):
             conditions=self.conditions,
             hidden_concepts=list(self.hidden_concepts),
             virtual_output_concepts=list(self.virtual_output_concepts),
-            node_joins=self.node_joins,
+            node_joins=list(self.node_joins) if self.node_joins else None,
             join_concepts=list(self.join_concepts) if self.join_concepts else None,
             force_join_type=self.force_join_type,
             existence_concepts=list(self.existence_concepts),

trilogy/core/processing/nodes/unnest_node.py

@@ -6,6 +6,7 @@ from trilogy.core.models import (
     SourceType,
     Concept,
     UnnestJoin,
+    Function,
 )
 from trilogy.core.processing.nodes.base_node import StrategyNode
 
@@ -19,7 +20,7 @@ class UnnestNode(StrategyNode):
 
     def __init__(
         self,
-        unnest_concept: Concept,
+        unnest_concepts: List[Concept],
         input_concepts: List[Concept],
         output_concepts: List[Concept],
         environment,
@@ -37,25 +38,28 @@ class UnnestNode(StrategyNode):
             parents=parents,
             depth=depth,
         )
-        self.unnest_concept = unnest_concept
+        self.unnest_concepts = unnest_concepts
 
     def _resolve(self) -> QueryDatasource:
         """We need to ensure that any filtered values are removed from the output to avoid inappropriate references"""
         base = super()._resolve()
-
+        lineage = self.unnest_concepts[0].lineage
+        assert isinstance(lineage, Function)
+        final = "_".join(set([c.address for c in self.unnest_concepts]))
         unnest = UnnestJoin(
-            concept=self.unnest_concept,
-            alias=f'unnest_{self.unnest_concept.address.replace(".", "_")}',
+            concepts=self.unnest_concepts,
+            parent=lineage,
+            alias=f'unnest_{final.replace(".", "_")}',
         )
         base.joins.append(unnest)
-
-        base.source_map[self.unnest_concept.address] = {unnest}
-        base.join_derived_concepts = [self.unnest_concept]
+        for unnest_concept in self.unnest_concepts:
+            base.source_map[unnest_concept.address] = {unnest}
+            base.join_derived_concepts = [unnest_concept]
         return base
 
     def copy(self) -> "UnnestNode":
         return UnnestNode(
-            unnest_concept=self.unnest_concept,
+            unnest_concepts=self.unnest_concepts,
             input_concepts=list(self.input_concepts),
             output_concepts=list(self.output_concepts),
             environment=self.environment,

trilogy/core/processing/utility.py

@@ -285,7 +285,9 @@ def get_node_joins(
                     raise SyntaxError(
                         f"Could not find {joinc.address} in {right_datasource.identifier} output {[c.address for c in right_datasource.output_concepts]}"
                     )
-            join_tuples.append((left_arg, right_arg))
+            narg = (left_arg, right_arg)
+            if narg not in join_tuples:
+                join_tuples.append((left_arg, right_arg))
         final_joins_pre.append(
             BaseJoin(
                 left_datasource=identifier_map[left],

trilogy/core/query_processor.py

@@ -35,7 +35,6 @@ from trilogy.core.ergonomics import CTE_NAMES
 from trilogy.core.optimization import optimize_ctes
 from math import ceil
 from collections import defaultdict
-from random import shuffle
 
 LOGGER_PREFIX = "[QUERY BUILD]"
 
@@ -46,7 +45,10 @@ def base_join_to_join(
     """This function converts joins at the datasource level
     to joins at the CTE level"""
     if isinstance(base_join, UnnestJoin):
-        return InstantiatedUnnestJoin(concept=base_join.concept, alias=base_join.alias)
+        return InstantiatedUnnestJoin(
+            concept_to_unnest=base_join.parent.concept_arguments[0],
+            alias=base_join.alias,
+        )
     if base_join.left_datasource.identifier == base_join.right_datasource.identifier:
         raise ValueError(f"Joining on same datasource {base_join}")
     left_ctes = [
@@ -125,8 +127,6 @@ def generate_source_map(
                 if qdk in output_address:
                     source_map[qdk].append(cte.name)
             # now do a pass that accepts partials
-            # TODO: move this into a second loop by first creationg all sub sources
-            # then loop through this
             for cte in matches:
                 if qdk not in source_map:
                     source_map[qdk] = [cte.name]
@@ -177,7 +177,6 @@ def generate_cte_name(full_name: str, name_map: dict[str, str]) -> str:
             int = ceil(idx / len(CTE_NAMES))
             suffix = f"_{int}"
         valid = [x for x in CTE_NAMES if x + suffix not in name_map.values()]
-        shuffle(valid)
         lookup = valid[0]
         new_name = f"{lookup}{suffix}"
         name_map[full_name] = new_name
@@ -306,7 +305,11 @@ def datasource_to_ctes(
     if cte.grain != query_datasource.grain:
         raise ValueError("Grain was corrupted in CTE generation")
     for x in cte.output_columns:
-        if x.address not in cte.source_map and CONFIG.validate_missing:
+        if (
+            x.address not in cte.source_map
+            and not any(y in cte.source_map for y in x.pseudonyms)
+            and CONFIG.validate_missing
+        ):
             raise ValueError(
                 f"Missing {x.address} in {cte.source_map}, source map {cte.source.source_map.keys()} "
             )
@@ -327,12 +330,13 @@ def append_existence_check(
         for subselect in where.existence_arguments:
             if not subselect:
                 continue
-            logger.info(
-                f"{LOGGER_PREFIX} fetching existance clause inputs {[str(c) for c in subselect]}"
-            )
+
             eds = source_query_concepts(
                 [*subselect], environment=environment, g=graph, history=history
             )
+            logger.info(
+                f"{LOGGER_PREFIX} fetching existence clause inputs {[str(c) for c in subselect]}"
+            )
             node.add_parents([eds])
             node.add_existence_concepts([*subselect])
 
@@ -377,9 +381,7 @@ def get_query_node(
     if nest_where and statement.where_clause:
         if not all_aggregate:
             ods.conditions = statement.where_clause.conditional
-        ods.output_concepts = statement.output_components
-        # ods.hidden_concepts = where_delta
-        ods.rebuild_cache()
+        ods.set_output_concepts(statement.output_components)
         append_existence_check(ods, environment, graph, history)
         ds = GroupNode(
             output_concepts=statement.output_components,