pytrilogy 0.0.2.10__py3-none-any.whl → 0.0.2.12__py3-none-any.whl

trilogy/core/processing/node_generators/basic_node.py

@@ -10,6 +10,7 @@ from trilogy.core.processing.node_generators.common import (
 )
 from trilogy.utility import unique
 from trilogy.constants import logger
+from itertools import combinations
 
 LOGGER_PREFIX = "[GEN_BASIC_NODE]"
 
@@ -31,12 +32,17 @@ def gen_basic_node(
     )
 
     local_optional_redundant = [x for x in local_optional if x in parent_concepts]
-    attempts = [(parent_concepts, [concept] + local_optional_redundant)]
-    from itertools import combinations
+    attempts: List[tuple[list[Concept], list[Concept]]] = [
+        (parent_concepts, [concept] + local_optional_redundant)
+    ]
+    equivalent_optional = [x for x in local_optional if x.lineage == concept.lineage]
+    non_equivalent_optional = [
+        x for x in local_optional if x not in equivalent_optional
+    ]
 
     if local_optional:
-        for combo in range(1, len(local_optional) + 1):
-            combos = combinations(local_optional, combo)
+        for combo in range(1, len(non_equivalent_optional) + 1):
+            combos = combinations(non_equivalent_optional, combo)
             for optional_set in combos:
                 attempts.append(
                     (
@@ -64,13 +70,10 @@ def gen_basic_node(
                 continue
             if all(x in source.partial_concepts for source in sources):
                 partials.append(x)
-        outputs = parent_node.output_concepts + [concept]
-        logger.info(
-            f"{depth_prefix}{LOGGER_PREFIX} Returning basic select for {concept} with attempted extra {[x.address for x in attempt]}, output {[x.address for x in outputs]}"
-        )
-        # parents.resolve()
 
         parent_node.add_output_concept(concept)
+        for x in equivalent_optional:
+            parent_node.add_output_concept(x)
 
         parent_node.remove_output_concepts(
             [
@@ -79,6 +82,9 @@ def gen_basic_node(
                 if x.address not in [y.address for y in basic_output]
             ]
         )
+        logger.info(
+            f"{depth_prefix}{LOGGER_PREFIX} Returning basic select for {concept} with attempted extra {[x.address for x in attempt]}, output {[x.address for x in parent_node.output_concepts]}"
+        )
         return parent_node
     logger.info(
         f"{depth_prefix}{LOGGER_PREFIX} No basic node could be generated for {concept}"

trilogy/core/processing/node_generators/filter_node.py

@@ -39,6 +39,15 @@ def gen_filter_node(
         raise SyntaxError('Filter node must have a lineage of type "FilterItem"')
     where = concept.lineage.where
 
+    optional_included: list[Concept] = []
+    for x in local_optional:
+        if isinstance(x.lineage, FilterItem):
+            if concept.lineage.where == where:
+                logger.info(
+                    f"{padding(depth)}{LOGGER_PREFIX} fetching {x.lineage.content.address} as optional parent with same filter conditions "
+                )
+                parent_row_concepts.append(x.lineage.content)
+                optional_included.append(x)
     logger.info(
         f"{padding(depth)}{LOGGER_PREFIX} filter {concept.address} derived from {immediate_parent.address} row parents {[x.address for x in parent_row_concepts]} and {[[y.address] for x  in parent_existence_concepts for y  in x]} existence parents"
     )
@@ -49,6 +58,7 @@ def gen_filter_node(
         g=g,
         depth=depth + 1,
         history=history,
+        conditions=conditions,
     )
 
     flattened_existence = [x for y in parent_existence_concepts for x in y]
@@ -88,6 +98,11 @@ def gen_filter_node(
             f"{padding(depth)}{LOGGER_PREFIX} query conditions are the same as filter conditions, can optimize across all concepts"
         )
         optimized_pushdown = True
+    elif optional_included == local_optional:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} all optional concepts are included in the filter, can optimize across all concepts"
+        )
+        optimized_pushdown = True
     if optimized_pushdown:
         if isinstance(row_parent, SelectNode):
             logger.info(
@@ -116,6 +131,7 @@ def gen_filter_node(
             x
             for x in local_optional
             if x.address in [y.address for y in parent.output_concepts]
+            or x.address in [y.address for y in optional_included]
         ]
         parent.add_parents(core_parents)
         parent.add_condition(where.conditional)
@@ -175,6 +191,7 @@ def gen_filter_node(
         ] + outputs
         filter_node.rebuild_cache()
         return filter_node
+
     enrich_node = source_concepts(  # this fetches the parent + join keys
         # to then connect to the rest of the query
         mandatory_list=[immediate_parent] + parent_row_concepts + local_optional,
@@ -182,10 +199,11 @@ def gen_filter_node(
         g=g,
         depth=depth + 1,
         history=history,
+        conditions=conditions,
     )
     if not enrich_node:
         return filter_node
-    x = MergeNode(
+    return MergeNode(
         input_concepts=[concept, immediate_parent] + local_optional,
         output_concepts=[
             concept,
@@ -206,8 +224,7 @@ def gen_filter_node(
                     [immediate_parent] + parent_row_concepts
                 ),
                 join_type=JoinType.LEFT_OUTER,
-                filter_to_mutual=False,
+                filter_to_mutual=True,
             )
         ],
     )
-    return x

trilogy/core/processing/node_generators/group_node.py

@@ -91,4 +91,6 @@ def gen_group_node(
         depth=depth,
         source_concepts=source_concepts,
         log_lambda=create_log_lambda(LOGGER_PREFIX, depth, logger),
+        history=history,
+        conditions=conditions,
     )

trilogy/core/processing/node_generators/node_merge_node.py

@@ -1,6 +1,6 @@
 from typing import List, Optional
 
-from trilogy.core.models import Concept, Environment, Conditional
+from trilogy.core.models import Concept, Environment, Conditional, WhereClause
 from trilogy.core.processing.nodes import MergeNode, History, StrategyNode
 import networkx as nx
 from trilogy.core.graph_models import concept_to_node
@@ -86,7 +86,7 @@ def determine_induced_minimal_nodes(
 
     for node in G.nodes:
         if concepts.get(node):
-            lookup = concepts[node]
+            lookup: Concept = concepts[node]
             if lookup.derivation not in (PurposeLineage.BASIC, PurposeLineage.ROOT):
                 nodes_to_remove.append(node)
             elif lookup.derivation == PurposeLineage.BASIC and G.out_degree(node) == 0:
@@ -155,6 +155,26 @@ def detect_ambiguity_and_raise(all_concepts, reduced_concept_sets) -> None:
         )
 
 
+def has_synonym(concept: Concept, others: list[list[Concept]]) -> bool:
+    return any(
+        c.address in concept.pseudonyms or concept.address in c.pseudonyms
+        for sublist in others
+        for c in sublist
+    )
+
+
+def filter_relevant_subgraphs(subgraphs: list[list[Concept]]) -> list[list[Concept]]:
+    return [
+        subgraph
+        for subgraph in subgraphs
+        if len(subgraph) > 1
+        or (
+            len(subgraph) == 1
+            and not has_synonym(subgraph[0], [x for x in subgraphs if x != subgraph])
+        )
+    ]
+
+
 def resolve_weak_components(
     all_concepts: List[Concept],
     environment: Environment,
@@ -249,6 +269,7 @@ def resolve_weak_components(
             continue
         subgraphs.append(sub_component)
     return subgraphs
+    # return filter_relevant_subgraphs(subgraphs)
 
 
 def subgraphs_to_merge_node(
@@ -260,6 +281,7 @@ def subgraphs_to_merge_node(
     source_concepts,
     history,
     conditions,
+    search_conditions: WhereClause | None = None,
     enable_early_exit: bool = True,
 ):
     parents: List[StrategyNode] = []
@@ -277,6 +299,7 @@ def subgraphs_to_merge_node(
             g=g,
             depth=depth + 1,
             history=history,
+            conditions=search_conditions,
         )
         if not parent:
             logger.info(
@@ -315,6 +338,7 @@ def gen_merge_node(
     accept_partial: bool = False,
     history: History | None = None,
     conditions: Conditional | None = None,
+    search_conditions: WhereClause | None = None,
 ) -> Optional[MergeNode]:
 
     for filter_downstream in [True, False]:
@@ -339,6 +363,7 @@ def gen_merge_node(
                 source_concepts=source_concepts,
                 history=history,
                 conditions=conditions,
+                search_conditions=search_conditions,
             )
     # one concept handling may need to be kicked to alias
     if len(all_concepts) == 1:
@@ -354,6 +379,7 @@ def gen_merge_node(
                 history=history,
                 conditions=conditions,
                 enable_early_exit=False,
+                search_conditions=search_conditions,
             )
             if test:
                 return test

trilogy/core/processing/node_generators/unnest_node.py

@@ -22,9 +22,14 @@ def gen_unnest_node(
     arguments = []
     if isinstance(concept.lineage, Function):
         arguments = concept.lineage.concept_arguments
+
+    equivalent_optional = [x for x in local_optional if x.lineage == concept.lineage]
+    non_equivalent_optional = [
+        x for x in local_optional if x not in equivalent_optional
+    ]
     if arguments or local_optional:
         parent = source_concepts(
-            mandatory_list=arguments + local_optional,
+            mandatory_list=arguments + non_equivalent_optional,
             environment=environment,
             g=g,
             depth=depth + 1,
@@ -38,8 +43,8 @@ def gen_unnest_node(
             return None
 
     base = UnnestNode(
-        unnest_concept=concept,
-        input_concepts=arguments + local_optional,
+        unnest_concepts=[concept] + equivalent_optional,
+        input_concepts=arguments + non_equivalent_optional,
         output_concepts=[concept] + local_optional,
         environment=environment,
         g=g,
@@ -57,4 +62,6 @@ def gen_unnest_node(
     )
     qds = new.resolve()
     assert qds.source_map[concept.address] == {base.resolve()}
+    for x in equivalent_optional:
+        assert qds.source_map[x.address] == {base.resolve()}
     return new

trilogy/core/processing/nodes/base_node.py

@@ -61,17 +61,22 @@ def resolve_concept_map(
         for concept in input.output_concepts:
             if concept.address not in input.non_partial_concept_addresses:
                 continue
-            if concept.address not in inherited:
-                continue
+
             if (
                 isinstance(input, QueryDatasource)
                 and concept.address in input.hidden_concepts
             ):
                 continue
             if concept.address in full_addresses:
+
                 concept_map[concept.address].add(input)
             elif concept.address not in concept_map:
+                # equi_targets = [x for x in targets if concept.address in x.pseudonyms or x.address in concept.pseudonyms]
+                # if equi_targets:
+                #     for equi in equi_targets:
+                #         concept_map[equi.address] = set()
                 concept_map[concept.address].add(input)
+
     # second loop, include partials
     for input in inputs:
         for concept in input.output_concepts:

trilogy/core/processing/nodes/group_node.py

@@ -121,7 +121,6 @@ class GroupNode(StrategyNode):
             source_type=source_type,
             source_map=resolve_concept_map(
                 parent_sources,
-                # targets = self.output_concepts,
                 targets=(
                     unique(
                         self.output_concepts + self.conditions.concept_arguments,

trilogy/core/processing/nodes/merge_node.py

@@ -28,14 +28,18 @@ LOGGER_PREFIX = "[CONCEPT DETAIL - MERGE NODE]"
 
 
 def deduplicate_nodes(
-    merged: dict[str, QueryDatasource | Datasource], logging_prefix: str
+    merged: dict[str, QueryDatasource | Datasource],
+    logging_prefix: str,
+    environment: Environment,
 ) -> tuple[bool, dict[str, QueryDatasource | Datasource], set[str]]:
     duplicates = False
     removed: set[str] = set()
     set_map: dict[str, set[str]] = {}
     for k, v in merged.items():
         unique_outputs = [
-            x.address for x in v.output_concepts if x not in v.partial_concepts
+            environment.concepts[x.address].address
+            for x in v.output_concepts
+            if x not in v.partial_concepts
         ]
         set_map[k] = set(unique_outputs)
     for k1, v1 in set_map.items():
@@ -71,12 +75,15 @@ def deduplicate_nodes_and_joins(
     joins: List[NodeJoin] | None,
     merged: dict[str, QueryDatasource | Datasource],
     logging_prefix: str,
+    environment: Environment,
 ) -> Tuple[List[NodeJoin] | None, dict[str, QueryDatasource | Datasource]]:
     # it's possible that we have more sources than we need
     duplicates = True
     while duplicates:
         duplicates = False
-        duplicates, merged, removed = deduplicate_nodes(merged, logging_prefix)
+        duplicates, merged, removed = deduplicate_nodes(
+            merged, logging_prefix, environment=environment
+        )
         # filter out any removed joins
         if joins is not None:
             joins = [
@@ -245,7 +252,7 @@ class MergeNode(StrategyNode):
 
         # it's possible that we have more sources than we need
         final_joins, merged = deduplicate_nodes_and_joins(
-            final_joins, merged, self.logging_prefix
+            final_joins, merged, self.logging_prefix, self.environment
         )
         # early exit if we can just return the parent
         final_datasets: List[QueryDatasource | Datasource] = list(merged.values())

trilogy/core/processing/nodes/unnest_node.py

@@ -6,6 +6,7 @@ from trilogy.core.models import (
     SourceType,
     Concept,
     UnnestJoin,
+    Function,
 )
 from trilogy.core.processing.nodes.base_node import StrategyNode
 
@@ -19,7 +20,7 @@ class UnnestNode(StrategyNode):
 
     def __init__(
         self,
-        unnest_concept: Concept,
+        unnest_concepts: List[Concept],
         input_concepts: List[Concept],
         output_concepts: List[Concept],
         environment,
@@ -37,25 +38,28 @@ class UnnestNode(StrategyNode):
             parents=parents,
             depth=depth,
         )
-        self.unnest_concept = unnest_concept
+        self.unnest_concepts = unnest_concepts
 
     def _resolve(self) -> QueryDatasource:
         """We need to ensure that any filtered values are removed from the output to avoid inappropriate references"""
         base = super()._resolve()
-
+        lineage = self.unnest_concepts[0].lineage
+        assert isinstance(lineage, Function)
+        final = "_".join(set([c.address for c in self.unnest_concepts]))
         unnest = UnnestJoin(
-            concept=self.unnest_concept,
-            alias=f'unnest_{self.unnest_concept.address.replace(".", "_")}',
+            concepts=self.unnest_concepts,
+            parent=lineage,
+            alias=f'unnest_{final.replace(".", "_")}',
         )
         base.joins.append(unnest)
-
-        base.source_map[self.unnest_concept.address] = {unnest}
-        base.join_derived_concepts = [self.unnest_concept]
+        for unnest_concept in self.unnest_concepts:
+            base.source_map[unnest_concept.address] = {unnest}
+            base.join_derived_concepts = [unnest_concept]
         return base
 
     def copy(self) -> "UnnestNode":
         return UnnestNode(
-            unnest_concept=self.unnest_concept,
+            unnest_concepts=self.unnest_concepts,
             input_concepts=list(self.input_concepts),
             output_concepts=list(self.output_concepts),
             environment=self.environment,

trilogy/core/processing/utility.py

@@ -285,7 +285,9 @@ def get_node_joins(
                     raise SyntaxError(
                         f"Could not find {joinc.address} in {right_datasource.identifier} output {[c.address for c in right_datasource.output_concepts]}"
                     )
-            join_tuples.append((left_arg, right_arg))
+            narg = (left_arg, right_arg)
+            if narg not in join_tuples:
+                join_tuples.append((left_arg, right_arg))
         final_joins_pre.append(
             BaseJoin(
                 left_datasource=identifier_map[left],

trilogy/core/query_processor.py

@@ -46,7 +46,10 @@ def base_join_to_join(
     """This function converts joins at the datasource level
     to joins at the CTE level"""
     if isinstance(base_join, UnnestJoin):
-        return InstantiatedUnnestJoin(concept=base_join.concept, alias=base_join.alias)
+        return InstantiatedUnnestJoin(
+            concept_to_unnest=base_join.parent.concept_arguments[0],
+            alias=base_join.alias,
+        )
     if base_join.left_datasource.identifier == base_join.right_datasource.identifier:
         raise ValueError(f"Joining on same datasource {base_join}")
     left_ctes = [
@@ -145,7 +148,9 @@ def generate_source_map(
         names = set([x.name for x in ev])
         ematches = [cte.name for cte in all_new_ctes if cte.source.name in names]
         existence_source_map[ek] = ematches
-    return {k: [] if not v else v for k, v in source_map.items()}, existence_source_map
+    return {
+        k: [] if not v else list(set(v)) for k, v in source_map.items()
+    }, existence_source_map
 
 
 def datasource_to_query_datasource(datasource: Datasource) -> QueryDatasource:
@@ -191,6 +196,8 @@ def resolve_cte_base_name_and_alias_v2(
     raw_joins: List[Join | InstantiatedUnnestJoin],
 ) -> Tuple[str | None, str | None]:
     joins: List[Join] = [join for join in raw_joins if isinstance(join, Join)]
+    # INFO     trilogy:query_processor.py:263 Finished building source map for civet with 3 parents, have {'local.relevant_customers': ['fowl', 'fowl'],
+    # 'customer.demographics.gender': ['mandrill'], 'customer.id': ['mandrill'], 'customer.demographics.id': ['mandrill'], 'customer.id_9268029262289908': [], 'customer.demographics.gender_1513806568509111': []}, query_datasource had non-empty keys ['local.relevant_customers', 'customer.demographics.gender', 'customer.id', 'customer.demographics.id'] and existence had non-empty keys []
     if (
         len(source.datasources) == 1
         and isinstance(source.datasources[0], Datasource)
@@ -212,12 +219,16 @@ def resolve_cte_base_name_and_alias_v2(
 
     counts: dict[str, int] = defaultdict(lambda: 0)
     output_addresses = [x.address for x in source.output_concepts]
+    input_address = [x.address for x in source.input_concepts]
     for k, v in source_map.items():
         for vx in v:
             if k in output_addresses:
                 counts[vx] = counts[vx] + 1
-            else:
-                counts[vx] = counts[vx]
+
+            if k in input_address:
+                counts[vx] = counts[vx] + 1
+
+            counts[vx] = counts[vx]
     if counts:
         return max(counts, key=counts.get), max(counts, key=counts.get)  # type: ignore
     return None, None
@@ -298,7 +309,11 @@ def datasource_to_ctes(
     if cte.grain != query_datasource.grain:
         raise ValueError("Grain was corrupted in CTE generation")
     for x in cte.output_columns:
-        if x.address not in cte.source_map and CONFIG.validate_missing:
+        if (
+            x.address not in cte.source_map
+            and not any(y in cte.source_map for y in x.pseudonyms)
+            and CONFIG.validate_missing
+        ):
             raise ValueError(
                 f"Missing {x.address} in {cte.source_map}, source map {cte.source.source_map.keys()} "
             )