pytrilogy 0.0.2.17__py3-none-any.whl → 0.0.2.19__py3-none-any.whl

trilogy/core/processing/node_generators/basic_node.py

@@ -1,17 +1,13 @@
 # directly select out a basic derivation
 from typing import List
 
-from trilogy.core.models import (
-    Concept,
-)
+from trilogy.core.models import Concept, WhereClause
 from trilogy.core.processing.nodes import StrategyNode, History
 from trilogy.core.processing.node_generators.common import (
     resolve_function_parent_concepts,
 )
-from trilogy.utility import unique
 from trilogy.constants import logger
 from trilogy.core.enums import SourceType
-from itertools import combinations
 
 LOGGER_PREFIX = "[GEN_BASIC_NODE]"
 
@@ -24,6 +20,7 @@ def gen_basic_node(
     depth: int,
     source_concepts,
     history: History | None = None,
+    conditions: WhereClause | None = None,
 ):
     depth_prefix = "\t" * depth
     parent_concepts = resolve_function_parent_concepts(concept)
@@ -32,10 +29,6 @@ def gen_basic_node(
         f"{depth_prefix}{LOGGER_PREFIX} basic node for {concept} has parents {[x.address for x in parent_concepts]}"
     )
 
-    local_optional_redundant = [x for x in local_optional if x in parent_concepts]
-    attempts: List[tuple[list[Concept], list[Concept]]] = [
-        (parent_concepts, [concept] + local_optional_redundant)
-    ]
     equivalent_optional = [
         x
         for x in local_optional
@@ -44,56 +37,35 @@ def gen_basic_node(
     non_equivalent_optional = [
         x for x in local_optional if x not in equivalent_optional
     ]
+    parent_node: StrategyNode = source_concepts(
+        mandatory_list=parent_concepts + non_equivalent_optional,
+        environment=environment,
+        g=g,
+        depth=depth + 1,
+        history=history,
+        conditions=conditions,
+    )
 
-    if local_optional:
-        for combo in range(1, len(non_equivalent_optional) + 1):
-            combos = combinations(non_equivalent_optional, combo)
-            for optional_set in combos:
-                attempts.append(
-                    (
-                        unique(parent_concepts + list(optional_set), "address"),
-                        list(optional_set) + [concept],
-                    )
-                )
-    # check for the concept by itself
-    for attempt, basic_output in reversed(attempts):
-        partials = []
-        attempt = unique(attempt, "address")
-        parent_node: StrategyNode = source_concepts(
-            mandatory_list=attempt,
-            environment=environment,
-            g=g,
-            depth=depth + 1,
-            history=history,
+    if not parent_node:
+        logger.info(
+            f"{depth_prefix}{LOGGER_PREFIX} No basic node could be generated for {concept}"
         )
+        return None
 
-        if not parent_node:
-            continue
-        parent_node.source_type = SourceType.BASIC
-        parents: List[StrategyNode] = [parent_node]
-        for x in basic_output:
-            sources = [p for p in parents if x in p.output_concepts]
-            if not sources:
-                continue
-            if all(x in source.partial_concepts for source in sources):
-                partials.append(x)
+    parent_node.source_type = SourceType.BASIC
+    parent_node.add_output_concept(concept)
+    for x in equivalent_optional:
+        parent_node.add_output_concept(x)
 
-        parent_node.add_output_concept(concept)
-        for x in equivalent_optional:
-            parent_node.add_output_concept(x)
+    parent_node.remove_output_concepts(
+        [
+            x
+            for x in parent_node.output_concepts
+            if x.address not in [concept] + local_optional
+        ]
+    )
 
-        parent_node.remove_output_concepts(
-            [
-                x
-                for x in parent_node.output_concepts
-                if x.address not in [y.address for y in basic_output]
-            ]
-        )
-        logger.info(
-            f"{depth_prefix}{LOGGER_PREFIX} Returning basic select for {concept} with attempted extra {[x.address for x in attempt]}, output {[x.address for x in parent_node.output_concepts]}"
-        )
-        return parent_node
     logger.info(
-        f"{depth_prefix}{LOGGER_PREFIX} No basic node could be generated for {concept}"
+        f"{depth_prefix}{LOGGER_PREFIX} Returning basic select for {concept}: output {[x.address for x in parent_node.output_concepts]}"
     )
-    return None
+    return parent_node

trilogy/core/processing/node_generators/common.py

@@ -136,6 +136,7 @@ def gen_property_enrichment_node(
             base_node,
         ]
         + final_nodes,
+        preexisting_conditions=conditions.conditional if conditions else None,
     )
 
 
@@ -154,11 +155,6 @@ def gen_enrichment_node(
 
     local_opts = LooseConceptList(concepts=local_optional)
 
-    if local_opts.issubset(base_node.output_lcl):
-        log_lambda(
-            f"{str(type(base_node).__name__)} has all optional { base_node.output_lcl}, skipping enrichmennt"
-        )
-        return base_node
     extra_required = [
         x
         for x in local_opts
@@ -186,13 +182,15 @@ def gen_enrichment_node(
                 conditions=conditions,
                 log_lambda=log_lambda,
             )
-
+    log_lambda(
+        f"{str(type(base_node).__name__)} searching for join keys {LooseConceptList(concepts=join_keys)} and extra required {local_opts}"
+    )
     enrich_node: StrategyNode = source_concepts(  # this fetches the parent + join keys
         # to then connect to the rest of the query
         mandatory_list=join_keys + extra_required,
         environment=environment,
         g=g,
-        depth=depth,
+        depth=depth + 1,
         history=history,
         conditions=conditions,
     )
@@ -216,6 +214,7 @@ def gen_enrichment_node(
         g=g,
         parents=[enrich_node, base_node],
         force_group=False,
+        preexisting_conditions=conditions.conditional if conditions else None,
     )
 
 

trilogy/core/processing/node_generators/filter_node.py

@@ -37,6 +37,7 @@ def gen_filter_node(
     where = concept.lineage.where
 
     optional_included: list[Concept] = []
+
     for x in local_optional:
         if isinstance(x.lineage, FilterItem):
             if concept.lineage.where == where:
@@ -45,17 +46,23 @@ def gen_filter_node(
                 )
                 parent_row_concepts.append(x.lineage.content)
                 optional_included.append(x)
+                continue
+        if conditions and conditions == where:
+            optional_included.append(x)
     logger.info(
         f"{padding(depth)}{LOGGER_PREFIX} filter {concept.address} derived from {immediate_parent.address} row parents {[x.address for x in parent_row_concepts]} and {[[y.address] for x  in parent_existence_concepts for y  in x]} existence parents"
     )
+    # we'll populate this with the row parent
+    # and the existence parent(s)
     core_parents = []
+
     row_parent: StrategyNode = source_concepts(
         mandatory_list=parent_row_concepts,
         environment=environment,
         g=g,
         depth=depth + 1,
         history=history,
-        conditions=conditions,
+        # conditions=conditions,
     )
 
     flattened_existence = [x for y in parent_existence_concepts for x in y]
@@ -101,6 +108,9 @@ def gen_filter_node(
         )
         optimized_pushdown = True
     if optimized_pushdown:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} returning optimized filter node with pushdown to parent with condition {where.conditional}"
+        )
         if isinstance(row_parent, SelectNode):
             logger.info(
                 f"{padding(depth)}{LOGGER_PREFIX} nesting select node in strategy node"
@@ -110,16 +120,10 @@ def gen_filter_node(
                 output_concepts=[concept] + row_parent.output_concepts,
                 environment=row_parent.environment,
                 g=row_parent.g,
-                parents=[row_parent] + core_parents,
+                parents=[row_parent],
                 depth=row_parent.depth,
                 partial_concepts=row_parent.partial_concepts,
                 force_group=False,
-                conditions=(
-                    row_parent.conditions + where.conditional
-                    if row_parent.conditions
-                    else where.conditional
-                ),
-                existence_concepts=row_parent.existence_concepts,
             )
         else:
             parent = row_parent
@@ -127,13 +131,14 @@ def gen_filter_node(
         expected_output = [concept] + [
             x
             for x in local_optional
-            if x.address in [y.address for y in parent.output_concepts]
-            or x.address in [y.address for y in optional_included]
+            if x.address in [y for y in parent.output_concepts]
+            or x.address in [y for y in optional_included]
         ]
         parent.add_parents(core_parents)
         parent.add_condition(where.conditional)
-        parent.add_existence_concepts(flattened_existence)
-        parent.set_output_concepts(expected_output)
+        parent.add_existence_concepts(flattened_existence, False).set_output_concepts(
+            expected_output, False
+        )
         parent.grain = Grain(
             components=(
                 list(immediate_parent.keys)
@@ -147,10 +152,6 @@ def gen_filter_node(
             ]
         )
         parent.rebuild_cache()
-
-        logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} returning optimized filter node with pushdown to parent with condition {where.conditional}"
-        )
         filter_node = parent
     else:
         core_parents.append(row_parent)
@@ -167,26 +168,24 @@ def gen_filter_node(
             grain=Grain(
                 components=[immediate_parent] + parent_row_concepts,
             ),
+            preexisting_conditions=conditions.conditional if conditions else None,
         )
 
     if not local_optional or all(
-        [
-            x.address in [y.address for y in filter_node.output_concepts]
-            for x in local_optional
-        ]
+        [x.address in filter_node.output_concepts for x in local_optional]
     ):
-        outputs = [
-            x
-            for x in filter_node.output_concepts
-            if x.address in [y.address for y in local_optional]
+        optional_outputs = [
+            x for x in filter_node.output_concepts if x.address in local_optional
         ]
         logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} no extra enrichment needed for filter node"
+            f"{padding(depth)}{LOGGER_PREFIX} no extra enrichment needed for filter node, has all of {[x.address for x in local_optional]}"
+        )
+        filter_node.set_output_concepts(
+            [
+                concept,
+            ]
+            + optional_outputs
         )
-        filter_node.output_concepts = [
-            concept,
-        ] + outputs
-        filter_node.rebuild_cache()
         return filter_node
 
     enrich_node = source_concepts(  # this fetches the parent + join keys
@@ -198,8 +197,6 @@ def gen_filter_node(
         history=history,
         conditions=conditions,
     )
-    if not enrich_node:
-        return filter_node
     return MergeNode(
         input_concepts=[concept, immediate_parent] + local_optional,
         output_concepts=[

trilogy/core/processing/node_generators/group_node.py

@@ -5,6 +5,7 @@ from trilogy.core.models import (
     WhereClause,
     Function,
     AggregateWrapper,
+    Grain,
 )
 from trilogy.utility import unique
 from trilogy.core.processing.nodes import GroupNode, StrategyNode, History
@@ -30,7 +31,7 @@ def gen_group_node(
     source_concepts,
     history: History | None = None,
     conditions: WhereClause | None = None,
-):
+) -> StrategyNode | None:
     # aggregates MUST always group to the proper grain
     # except when the
     parent_concepts: List[Concept] = unique(
@@ -50,9 +51,9 @@ def gen_group_node(
         parent_concepts += grain_components
         output_concepts += grain_components
         for possible_agg in local_optional:
+            if not isinstance(possible_agg.lineage, (AggregateWrapper, Function)):
+                continue
             if possible_agg.grain and possible_agg.grain == concept.grain:
-                if not isinstance(possible_agg.lineage, (AggregateWrapper, Function)):
-                    continue
                 agg_parents: List[Concept] = resolve_function_parent_concepts(
                     possible_agg
                 )
@@ -60,7 +61,16 @@ def gen_group_node(
                     set([x.address for x in parent_concepts])
                 ):
                     output_concepts.append(possible_agg)
-
+                    logger.info(
+                        f"{padding(depth)}{LOGGER_PREFIX} found equivalent group by optional concept {possible_agg.address} for {concept.address}"
+                    )
+                elif Grain(components=agg_parents) == Grain(components=parent_concepts):
+                    extra = [x for x in agg_parents if x.address not in parent_concepts]
+                    parent_concepts += extra
+                    output_concepts.append(possible_agg)
+                    logger.info(
+                        f"{padding(depth)}{LOGGER_PREFIX} found equivalent group by optional concept {possible_agg.address} for {concept.address}"
+                    )
     if parent_concepts:
         logger.info(
             f"{padding(depth)}{LOGGER_PREFIX} fetching group node parents {LooseConceptList(concepts=parent_concepts)}"
@@ -94,12 +104,24 @@ def gen_group_node(
         g=g,
         parents=parents,
         depth=depth,
+        preexisting_conditions=conditions.conditional if conditions else None,
     )
 
     # early exit if no optional
+
     if not local_optional:
         return group_node
-    logger.info(f"{padding(depth)}{LOGGER_PREFIX} group node requires enrichment")
+    missing_optional = [
+        x.address for x in local_optional if x.address not in group_node.output_concepts
+    ]
+    if not missing_optional:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} no extra enrichment needed for group node, has all of {[x.address for x in local_optional]}"
+        )
+        return group_node
+    logger.info(
+        f"{padding(depth)}{LOGGER_PREFIX} group node requires enrichment, missing {missing_optional}"
+    )
     return gen_enrichment_node(
         group_node,
         join_keys=group_key_parents,

trilogy/core/processing/node_generators/group_to_node.py

@@ -1,4 +1,4 @@
-from trilogy.core.models import Concept, Environment, Function
+from trilogy.core.models import Concept, Environment, Function, WhereClause
 from trilogy.core.processing.nodes import (
     GroupNode,
     StrategyNode,
@@ -21,6 +21,7 @@ def gen_group_to_node(
     depth: int,
     source_concepts,
     history: History | None = None,
+    conditions: WhereClause | None = None,
 ) -> GroupNode | MergeNode:
     # aggregates MUST always group to the proper grain
     if not isinstance(concept.lineage, Function):
@@ -36,6 +37,7 @@ def gen_group_to_node(
             g=g,
             depth=depth + 1,
             history=history,
+            conditions=conditions,
         )
     ]
 

trilogy/core/processing/node_generators/multiselect_node.py

@@ -2,6 +2,7 @@ from trilogy.core.models import (
     Concept,
     Environment,
     MultiSelectStatement,
+    WhereClause,
 )
 from trilogy.core.processing.nodes import MergeNode, NodeJoin, History
 from trilogy.core.processing.nodes.base_node import concept_list_to_grain, StrategyNode
@@ -58,6 +59,7 @@ def gen_multiselect_node(
     depth: int,
     source_concepts,
     history: History | None = None,
+    conditions: WhereClause | None = None,
 ) -> MergeNode | None:
     if not isinstance(concept.lineage, MultiSelectStatement):
         logger.info(
@@ -157,6 +159,7 @@ def gen_multiselect_node(
         g=g,
         depth=depth + 1,
         history=history,
+        conditions=conditions,
     )
     if not enrich_node:
         logger.info(

trilogy/core/processing/node_generators/node_merge_node.py

@@ -135,7 +135,9 @@ def determine_induced_minimal_nodes(
     return final
 
 
-def detect_ambiguity_and_raise(all_concepts, reduced_concept_sets) -> None:
+def detect_ambiguity_and_raise(
+    all_concepts: list[Concept], reduced_concept_sets: list[set[str]]
+) -> None:
     final_candidates: list[set[str]] = []
     common: set[str] = set()
     # find all values that show up in every join_additions
@@ -146,7 +148,6 @@ def detect_ambiguity_and_raise(all_concepts, reduced_concept_sets) -> None:
             common = common.intersection(ja)
         if all(set(ja).issubset(y) for y in reduced_concept_sets):
             final_candidates.append(ja)
-
     if not final_candidates:
         filtered_paths = [x.difference(common) for x in reduced_concept_sets]
         raise AmbiguousRelationshipResolutionException(
@@ -197,6 +198,9 @@ def resolve_weak_components(
         for c in all_concepts
         if "__preql_internal" not in c.address
     ]
+    synonyms: list[Concept] = []
+    for x in all_concepts:
+        synonyms += x.pseudonyms.values()
     while break_flag is not True:
         count += 1
         if count > AMBIGUITY_CHECK_LIMIT:
@@ -222,13 +226,9 @@ def resolve_weak_components(
                 for node in g.nodes
                 if node.startswith("c~")
             ]
-            new = [
-                x
-                for x in all_graph_concepts
-                if x.address not in [y.address for y in all_concepts]
-            ]
+            new = [x for x in all_graph_concepts if x.address not in all_concepts]
 
-            new_addresses = set([x.address for x in new])
+            new_addresses = set([x.address for x in new if x.address not in synonyms])
             if not new:
                 break_flag = True
             # remove our new nodes for the next search path
@@ -299,7 +299,7 @@ def subgraphs_to_merge_node(
             g=g,
             depth=depth + 1,
             history=history,
-            conditions=search_conditions,
+            # conditions=search_conditions,
         )
         if not parent:
             logger.info(
@@ -324,7 +324,9 @@ def subgraphs_to_merge_node(
         g=g,
         parents=parents,
         depth=depth,
-        conditions=conditions,
+        # conditions=conditions,
+        # conditions=search_conditions.conditional,
+        # preexisting_conditions=search_conditions.conditional,
         # node_joins=[]
     )
 
@@ -341,6 +343,8 @@ def gen_merge_node(
     search_conditions: WhereClause | None = None,
 ) -> Optional[MergeNode]:
 
+    if search_conditions:
+        all_concepts = unique(all_concepts + search_conditions.row_arguments, "address")
     for filter_downstream in [True, False]:
         weak_resolve = resolve_weak_components(
             all_concepts,

trilogy/core/processing/node_generators/rowset_node.py

@@ -5,16 +5,15 @@ from trilogy.core.models import (
     RowsetDerivationStatement,
     RowsetItem,
     MultiSelectStatement,
+    WhereClause,
 )
 from trilogy.core.processing.nodes import MergeNode, History, StrategyNode
-from trilogy.core.processing.nodes.base_node import concept_list_to_grain
 from typing import List
 
 from trilogy.core.enums import PurposeLineage
 from trilogy.constants import logger
-from trilogy.core.processing.utility import padding
-from trilogy.core.processing.utility import concept_to_relevant_joins
-
+from trilogy.core.processing.utility import padding, concept_to_relevant_joins
+from trilogy.core.processing.nodes.base_node import concept_list_to_grain
 
 LOGGER_PREFIX = "[GEN_ROWSET_NODE]"
 
@@ -27,6 +26,7 @@ def gen_rowset_node(
     depth: int,
     source_concepts,
     history: History | None = None,
+    conditions: WhereClause | None = None,
 ) -> StrategyNode | None:
     from trilogy.core.query_processor import get_query_node
 
@@ -44,7 +44,6 @@ def gen_rowset_node(
             f"{padding(depth)}{LOGGER_PREFIX} Cannot generate parent rowset node for {concept}"
         )
         return None
-
     enrichment = set([x.address for x in local_optional])
     rowset_relevant = [x for x in rowset.derived_concepts]
     select_hidden = set([x.address for x in select.hidden_components])
@@ -67,11 +66,10 @@ def gen_rowset_node(
     final_hidden = rowset_hidden + [
         x
         for x in node.output_concepts
-        if x.address not in [y.address for y in local_optional + [concept]]
+        if x.address not in local_optional + [concept]
         and x.derivation != PurposeLineage.ROWSET
     ]
     node.hide_output_concepts(final_hidden)
-
     assert node.resolution_cache
     # assume grain to be output of select
     # but don't include anything hidden(the non-rowset concepts)
@@ -81,9 +79,7 @@ def gen_rowset_node(
             for x in node.output_concepts
             if x.address
             not in [
-                y.address
-                for y in node.hidden_concepts
-                if y.derivation != PurposeLineage.ROWSET
+                y for y in node.hidden_concepts if y.derivation != PurposeLineage.ROWSET
             ]
         ],
         parent_sources=node.resolution_cache.datasources,
@@ -91,14 +87,16 @@ def gen_rowset_node(
 
     node.rebuild_cache()
 
-    possible_joins = concept_to_relevant_joins(additional_relevant)
     if not local_optional or all(
-        x.address in [y.address for y in node.output_concepts] for x in local_optional
+        x.address in node.output_concepts for x in local_optional
     ):
         logger.info(
             f"{padding(depth)}{LOGGER_PREFIX} no enrichment required for rowset node as all optional found or no optional; exiting early."
         )
+        # node.set_preexisting_conditions(conditions.conditional if conditions else None)
         return node
+
+    possible_joins = concept_to_relevant_joins(additional_relevant)
     if not possible_joins:
         logger.info(
             f"{padding(depth)}{LOGGER_PREFIX} no possible joins for rowset node to get {[x.address for x in local_optional]}; have {[x.address for x in node.output_concepts]}"
@@ -110,6 +108,7 @@ def gen_rowset_node(
         environment=environment,
         g=g,
         depth=depth + 1,
+        conditions=conditions,
     )
     if not enrich_node:
         logger.info(
@@ -127,4 +126,5 @@ def gen_rowset_node(
             enrich_node,
         ],
         partial_concepts=node.partial_concepts,
+        preexisting_conditions=conditions.conditional if conditions else None,
     )