pytrilogy 0.0.2.2__py3-none-any.whl → 0.0.2.4__py3-none-any.whl

trilogy/core/processing/node_generators/common.py

@@ -47,7 +47,7 @@ def resolve_function_parent_concepts(concept: Concept) -> List[Concept]:
 
 def resolve_filter_parent_concepts(
     concept: Concept,
-) -> Tuple[Concept, List[Concept], List[Concept]]:
+) -> Tuple[Concept, List[Concept], List[Tuple[Concept, ...]]]:
     if not isinstance(concept.lineage, FilterItem):
         raise ValueError(
             f"Concept {concept} lineage is not filter item, is {type(concept.lineage)}"
@@ -58,7 +58,7 @@ def resolve_filter_parent_concepts(
     base_rows += concept.lineage.where.row_arguments
     # TODO: pass tuple groups through
     for ctuple in concept.lineage.where.existence_arguments:
-        base_existence += list(ctuple)
+        base_existence.append(ctuple)
     if direct_parent.grain:
         base_rows += direct_parent.grain.components_copy
     if (
@@ -71,7 +71,7 @@ def resolve_filter_parent_concepts(
         return (
             concept.lineage.content,
             unique(base_rows, "address"),
-            unique(base_existence, "address"),
+            base_existence,
         )
     return concept.lineage.content, unique(base_rows, "address"), []
 

trilogy/core/processing/node_generators/filter_node.py

@@ -74,27 +74,31 @@ def gen_filter_node(
         return parent
 
     core_parents.append(parent)
+
     if parent_existence_concepts:
-        logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} fetching filter node existence parents {[x.address for x in parent_existence_concepts]}"
-        )
-        parent_existence = source_concepts(
-            mandatory_list=parent_existence_concepts,
-            environment=environment,
-            g=g,
-            depth=depth + 1,
-            history=history,
-        )
-        if not parent_existence:
+        for existence_tuple in parent_existence_concepts:
+            if not existence_tuple:
+                continue
             logger.info(
-                f"{padding(depth)}{LOGGER_PREFIX} filter existence node parents could not be found"
+                f"{padding(depth)}{LOGGER_PREFIX} fetching filter node existence parents {[x.address for x in existence_tuple]}"
             )
-            return None
-        core_parents.append(parent_existence)
-
+            parent_existence = source_concepts(
+                mandatory_list=list(existence_tuple),
+                environment=environment,
+                g=g,
+                depth=depth + 1,
+                history=history,
+            )
+            if not parent_existence:
+                logger.info(
+                    f"{padding(depth)}{LOGGER_PREFIX} filter existence node parents could not be found"
+                )
+                return None
+            core_parents.append(parent_existence)
+    flattened_existence = [x for y in parent_existence_concepts for x in y]
     filter_node = FilterNode(
         input_concepts=unique(
-            [immediate_parent] + parent_row_concepts + parent_existence_concepts,
+            [immediate_parent] + parent_row_concepts + flattened_existence,
             "address",
         ),
         output_concepts=[concept, immediate_parent] + parent_row_concepts,

trilogy/core/processing/node_generators/node_merge_node.py

@@ -4,7 +4,6 @@ from trilogy.core.models import Concept, Environment, Conditional
 from trilogy.core.processing.nodes import MergeNode, History, StrategyNode
 import networkx as nx
 from trilogy.core.graph_models import concept_to_node
-from trilogy.core.processing.utility import PathInfo
 from trilogy.constants import logger
 from trilogy.utility import unique
 from trilogy.core.exceptions import AmbiguousRelationshipResolutionException
@@ -63,7 +62,9 @@ def extract_ds_components(g: nx.DiGraph, nodelist: list[str]) -> list[list[str]]
                     if not str(x).startswith("ds~")
                 ]
             )
-
+    # if we had no ego graphs, return all concepts
+    if not graphs:
+        return [[extract_address(node) for node in nodelist]]
     graphs = filter_unique_graphs(graphs)
     for node in nodelist:
         parsed = extract_address(node)
@@ -82,6 +83,7 @@ def determine_induced_minimal_nodes(
     H: nx.Graph = nx.to_undirected(G).copy()
     nodes_to_remove = []
     concepts = nx.get_node_attributes(G, "concept")
+
     for node in G.nodes:
         if concepts.get(node):
             lookup = concepts[node]
@@ -107,9 +109,11 @@ def determine_induced_minimal_nodes(
         paths = nx.multi_source_dijkstra_path(H, nodelist)
     except nx.exception.NodeNotFound:
         return None
+
     H.remove_nodes_from(list(x for x in H.nodes if x not in paths))
     sG: nx.Graph = ax.steinertree.steiner_tree(H, nodelist).copy()
     final: nx.DiGraph = nx.subgraph(G, sG.nodes).copy()
+
     for edge in G.edges:
         if edge[1] in final.nodes and edge[0].startswith("ds~"):
             ds_name = extract_address(edge[0])
@@ -125,6 +129,7 @@ def determine_induced_minimal_nodes(
         [final.in_degree(node) > 0 for node in final.nodes if node.startswith("c~")]
     ):
         return None
+
     if not all([node in final.nodes for node in nodelist]):
         return None
     return final
@@ -308,111 +313,44 @@ def gen_merge_node(
     history: History | None = None,
     conditions: Conditional | None = None,
 ) -> Optional[MergeNode]:
-    join_candidates: List[PathInfo] = []
-
-    # inject new concepts into search, and identify if two dses can reach there
-    if not join_candidates:
-        for filter_downstream in [True, False]:
-            weak_resolve = resolve_weak_components(
-                all_concepts,
-                environment,
-                g,
-                filter_downstream=filter_downstream,
-                accept_partial=accept_partial,
-            )
-            if weak_resolve:
-                log_graph = [[y.address for y in x] for x in weak_resolve]
-                logger.info(
-                    f"{padding(depth)}{LOGGER_PREFIX} Was able to resolve graph through weak component resolution - final graph {log_graph}"
-                )
-                return subgraphs_to_merge_node(
-                    weak_resolve,
-                    depth=depth,
-                    all_concepts=all_concepts,
-                    environment=environment,
-                    g=g,
-                    source_concepts=source_concepts,
-                    history=history,
-                    conditions=conditions,
-                )
-    if not join_candidates:
-        return None
-    join_additions: list[set[str]] = []
-    for candidate in join_candidates:
-        join_additions.append(candidate.reduced_concepts)
-
-    common: set[str] = set()
-    final_candidates: list[set[str]] = []
-    # find all values that show up in every join_additions
-    for ja in join_additions:
-        if not common:
-            common = ja
-        else:
-            common = common.intersection(ja)
-        if all(ja.issubset(y) for y in join_additions):
-            final_candidates.append(ja)
 
-    if not final_candidates:
-        filtered_paths = [x.difference(common) for x in join_additions]
-        raise AmbiguousRelationshipResolutionException(
-            f"Ambiguous concept join resolution fetching {[x.address for x in all_concepts]} - unique values in possible paths = {filtered_paths}. Include an additional concept to disambiguate",
-            join_additions,
-        )
-    if not join_candidates:
-        logger.info(
-            f"{padding(depth)}{LOGGER_PREFIX} No additional join candidates could be found"
+    for filter_downstream in [True, False]:
+        weak_resolve = resolve_weak_components(
+            all_concepts,
+            environment,
+            g,
+            filter_downstream=filter_downstream,
+            accept_partial=accept_partial,
         )
-        return None
-    shortest: PathInfo = sorted(
-        [x for x in join_candidates if x.reduced_concepts in final_candidates],
-        key=lambda x: len(x.reduced_concepts),
-    )[0]
-    logger.info(f"{padding(depth)}{LOGGER_PREFIX} final path is {shortest.paths}")
-
-    return subgraphs_to_merge_node(
-        shortest.concept_subgraphs,
-        depth=depth,
-        all_concepts=all_concepts,
-        environment=environment,
-        g=g,
-        source_concepts=source_concepts,
-        history=history,
-        conditions=conditions,
-    )
-    # parents = []
-    # for graph in shortest.concept_subgraphs:
-    #     logger.info(
-    #         f"{padding(depth)}{LOGGER_PREFIX} fetching subgraph {[c.address for c in graph]}"
-    #     )
-    #     parent = source_concepts(
-    #         mandatory_list=graph,
-    #         environment=environment,
-    #         g=g,
-    #         depth=depth + 1,
-    #         history=history,
-    #     )
-    #     if not parent:
-    #         logger.info(
-    #             f"{padding(depth)}{LOGGER_PREFIX} Unable to instantiate target subgraph"
-    #         )
-    #         return None
-    #     logger.info(
-    #         f"{padding(depth)}{LOGGER_PREFIX} finished subgraph fetch for {[c.address for c in graph]}, have parent {type(parent)}"
-    #     )
-    #     parents.append(parent)
-
-    # return MergeNode(
-    #     input_concepts=[
-    #         environment.concepts[x]
-    #         for x in shortest.reduced_concepts
-    #         if environment.concepts[x].derivation != PurposeLineage.MERGE
-    #     ],
-    #     output_concepts=[
-    #         x for x in all_concepts if x.derivation != PurposeLineage.MERGE
-    #     ],
-    #     environment=environment,
-    #     g=g,
-    #     parents=parents,
-    #     depth=depth,
-    #     conditions=conditions,
-    # )
+        if weak_resolve:
+            log_graph = [[y.address for y in x] for x in weak_resolve]
+            logger.info(
+                f"{padding(depth)}{LOGGER_PREFIX} Was able to resolve graph through weak component resolution - final graph {log_graph}"
+            )
+            return subgraphs_to_merge_node(
+                weak_resolve,
+                depth=depth,
+                all_concepts=all_concepts,
+                environment=environment,
+                g=g,
+                source_concepts=source_concepts,
+                history=history,
+                conditions=conditions,
+            )
+    # one concept handling may need to be kicked to alias
+    if len(all_concepts) == 1:
+        concept = all_concepts[0]
+        for k, v in concept.pseudonyms.items():
+            test = subgraphs_to_merge_node(
+                [[concept, v]],
+                g=g,
+                all_concepts=[concept],
+                environment=environment,
+                depth=depth,
+                source_concepts=source_concepts,
+                history=history,
+                conditions=conditions,
+            )
+            if test:
+                return test
+    return None

trilogy/core/processing/nodes/group_node.py

@@ -19,7 +19,7 @@ from trilogy.core.processing.nodes.base_node import (
     concept_list_to_grain,
 )
 from trilogy.utility import unique
-
+from trilogy.core.processing.utility import is_scalar_condition
 
 LOGGER_PREFIX = "[CONCEPT DETAIL - GROUP NODE]"
 
@@ -111,7 +111,8 @@ class GroupNode(StrategyNode):
                     f" {parent.grain}"
                 )
             source_type = SourceType.GROUP
-        return QueryDatasource(
+
+        base = QueryDatasource(
             input_concepts=self.input_concepts,
             output_concepts=self.output_concepts,
             datasources=parent_sources,
@@ -134,6 +135,31 @@ class GroupNode(StrategyNode):
             partial_concepts=self.partial_concepts,
             condition=self.conditions,
         )
+        # if there is a condition on a group node and it's not scalar
+        # inject an additional CTE
+        if self.conditions:
+            logger.info("CONDITIONS")
+            logger.info(str(self.conditions))
+            logger.info(is_scalar_condition(self.conditions))
+        if self.conditions and not is_scalar_condition(self.conditions):
+            base.condition = None
+            base.output_concepts = self.output_concepts + self.conditions.row_arguments
+            return QueryDatasource(
+                input_concepts=base.output_concepts,
+                output_concepts=self.output_concepts,
+                datasources=[base],
+                source_type=SourceType.SELECT,
+                source_map=resolve_concept_map(
+                    [base],
+                    targets=self.output_concepts,
+                    inherited_inputs=base.output_concepts,
+                ),
+                joins=[],
+                grain=grain,
+                partial_concepts=self.partial_concepts,
+                condition=self.conditions,
+            )
+        return base
 
     def copy(self) -> "GroupNode":
         return GroupNode(

trilogy/core/processing/utility.py

@@ -8,6 +8,16 @@ from trilogy.core.models import (
     QueryDatasource,
     LooseConceptList,
     Environment,
+    Conditional,
+    SubselectComparison,
+    Comparison,
+    Parenthetical,
+    Function,
+    FilterItem,
+    MagicConstants,
+    WindowItem,
+    AggregateWrapper,
+    DataType,
 )
 
 from trilogy.core.enums import Purpose, Granularity
@@ -18,6 +28,8 @@ from collections import defaultdict
 from logging import Logger
 from pydantic import BaseModel
 
+from trilogy.core.enums import FunctionClass
+
 
 class NodeType(Enum):
     CONCEPT = 1
@@ -124,15 +136,20 @@ def resolve_join_order(joins: List[BaseJoin]) -> List[BaseJoin]:
     return final_joins
 
 
-def add_node_join_concept(graph, concept, datasource, concepts):
-    # we don't need to join on a concept if all of the keys exist in the grain
-    # if concept.keys and all([x in grain for x in concept.keys]):
-    #     continue
+def add_node_join_concept(
+    graph: nx.DiGraph,
+    concept: Concept,
+    datasource: Datasource | QueryDatasource,
+    concepts: List[Concept],
+):
+
     concepts.append(concept)
 
     graph.add_node(concept.address, type=NodeType.CONCEPT)
     graph.add_edge(datasource.identifier, concept.address)
-    for k, v in concept.pseudonyms.items():
+    for _, v in concept.pseudonyms.items():
+        if v in concepts:
+            continue
         if v.address != concept.address:
             add_node_join_concept(graph, v, datasource, concepts)
 
@@ -149,13 +166,6 @@ def get_node_joins(
         graph.add_node(datasource.identifier, type=NodeType.NODE)
         for concept in datasource.output_concepts:
             add_node_join_concept(graph, concept, datasource, concepts)
-            # we don't need to join on a concept if all of the keys exist in the grain
-            # if concept.keys and all([x in grain for x in concept.keys]):
-            #     continue
-            # concepts.append(concept)
-
-            # graph.add_node(concept.address, type=NodeType.CONCEPT)
-            # graph.add_edge(datasource.identifier, concept.address)
 
     # add edges for every constant to every datasource
     for datasource in datasources:
@@ -195,26 +205,6 @@ def get_node_joins(
         ),
     )
 
-    node_map = {
-        x[0:20]: len(
-            [
-                partial
-                for partial in identifier_map[x].partial_concepts
-                if partial in grain
-            ]
-            + [
-                output
-                for output in identifier_map[x].output_concepts
-                if output.address in grain_pseudonyms
-            ]
-        )
-        for x in node_list
-    }
-    print("NODE MAP")
-    print(node_map)
-    print([x.address for x in grain])
-    print(grain_pseudonyms)
-
     for left in node_list:
         # the constant dataset is a special case
         # and can never be on the left of a join
@@ -354,3 +344,37 @@ def get_disconnected_components(
         x for x in sub_graphs if calculate_graph_relevance(graph, x, all_concepts) > 0
     ]
     return len(sub_graphs), sub_graphs
+
+
+def is_scalar_condition(
+    element: (
+        int
+        | str
+        | float
+        | list
+        | WindowItem
+        | FilterItem
+        | Concept
+        | Comparison
+        | Conditional
+        | Parenthetical
+        | Function
+        | AggregateWrapper
+        | MagicConstants
+        | DataType
+    ),
+) -> bool:
+    if isinstance(element, Parenthetical):
+        return is_scalar_condition(element.content)
+    elif isinstance(element, SubselectComparison):
+        return True
+    elif isinstance(element, Comparison):
+        return is_scalar_condition(element.left) and is_scalar_condition(element.right)
+    elif isinstance(element, Function):
+        if element.operator in FunctionClass.AGGREGATE_FUNCTIONS.value:
+            return False
+    elif isinstance(element, AggregateWrapper):
+        return is_scalar_condition(element.function)
+    elif isinstance(element, Conditional):
+        return is_scalar_condition(element.left) and is_scalar_condition(element.right)
+    return True

trilogy/core/query_processor.py

@@ -350,7 +350,11 @@ def get_query_datasources(
 
     search_concepts: list[Concept] = statement.output_components
     nest_where = statement.where_clause_category == SelectFiltering.IMPLICIT
-    if nest_where and statement.where_clause:
+
+    # if all are aggregates, we've pushed the filtering inside the aggregates anyway
+    all_aggregate = all([x.is_aggregate for x in search_concepts])
+
+    if nest_where and statement.where_clause and not all_aggregate:
         search_concepts = unique(
             statement.where_clause.row_arguments + search_concepts, "address"
         )
@@ -363,7 +367,8 @@ def get_query_datasources(
     )
     ds: GroupNode | SelectNode
     if nest_where and statement.where_clause:
-        ods.conditions = statement.where_clause.conditional
+        if not all_aggregate:
+            ods.conditions = statement.where_clause.conditional
         ods.output_concepts = search_concepts
         # ods.hidden_concepts = where_delta
         ods.rebuild_cache()
@@ -467,9 +472,11 @@ def process_query(
     for cte in raw_ctes:
         cte.parent_ctes = [seen[x.name] for x in cte.parent_ctes]
     deduped_ctes: List[CTE] = list(seen.values())
+    root_cte.order_by = statement.order_by
+    root_cte.limit = statement.limit
+    root_cte.hidden_concepts = [x for x in statement.hidden_components]
 
     final_ctes = optimize_ctes(deduped_ctes, root_cte, statement)
-
     return ProcessedQuery(
         order_by=statement.order_by,
         grain=statement.grain,