pytrilogy 0.0.2.12__py3-none-any.whl → 0.0.2.14__py3-none-any.whl

trilogy/core/processing/node_generators/common.py

@@ -1,4 +1,4 @@
-from typing import List, Tuple
+from typing import List, Tuple, Callable
 
 
 from trilogy.core.enums import PurposeLineage, Purpose
@@ -15,12 +15,10 @@ from trilogy.utility import unique
 from trilogy.core.processing.nodes.base_node import StrategyNode
 from trilogy.core.processing.nodes.merge_node import MergeNode
 from trilogy.core.processing.nodes import History
-from trilogy.core.enums import JoinType
 from trilogy.core.processing.nodes import (
     NodeJoin,
 )
 from collections import defaultdict
-from trilogy.core.processing.utility import concept_to_relevant_joins
 
 
 def resolve_function_parent_concepts(concept: Concept) -> List[Concept]:
@@ -96,6 +94,7 @@ def gen_property_enrichment_node(
     g,
     depth: int,
     source_concepts,
+    log_lambda: Callable,
     history: History | None = None,
     conditions: WhereClause | None = None,
 ):
@@ -106,8 +105,8 @@ def gen_property_enrichment_node(
         keys = "-".join([y.address for y in x.keys])
         required_keys[keys].add(x.address)
     final_nodes = []
-    node_joins = []
     for _k, vs in required_keys.items():
+        log_lambda(f"Generating enrichment node for {_k} with {vs}")
         ks = _k.split("-")
         enrich_node: StrategyNode = source_concepts(
             mandatory_list=[environment.concepts[k] for k in ks]
@@ -119,17 +118,6 @@ def gen_property_enrichment_node(
             conditions=conditions,
         )
         final_nodes.append(enrich_node)
-        node_joins.append(
-            NodeJoin(
-                left_node=enrich_node,
-                right_node=base_node,
-                concepts=concept_to_relevant_joins(
-                    [environment.concepts[k] for k in ks]
-                ),
-                filter_to_mutual=False,
-                join_type=JoinType.LEFT_OUTER,
-            )
-        )
     return MergeNode(
         input_concepts=unique(
             base_node.output_concepts
@@ -146,9 +134,8 @@ def gen_property_enrichment_node(
         g=g,
         parents=[
             base_node,
-            enrich_node,
-        ],
-        node_joins=node_joins,
+        ]
+        + final_nodes,
     )
 
 
@@ -197,6 +184,7 @@ def gen_enrichment_node(
                 source_concepts,
                 history=history,
                 conditions=conditions,
+                log_lambda=log_lambda,
             )
 
     enrich_node: StrategyNode = source_concepts(  # this fetches the parent + join keys
@@ -216,29 +204,18 @@ def gen_enrichment_node(
     log_lambda(
         f"{str(type(base_node).__name__)} returning merge node with group node + enrichment node"
     )
-
+    non_hidden = [
+        x
+        for x in base_node.output_concepts
+        if x.address not in [y.address for y in base_node.hidden_concepts]
+    ]
     return MergeNode(
-        input_concepts=unique(
-            join_keys + extra_required + base_node.output_concepts, "address"
-        ),
-        output_concepts=unique(
-            join_keys + extra_required + base_node.output_concepts, "address"
-        ),
+        input_concepts=unique(join_keys + extra_required + non_hidden, "address"),
+        output_concepts=unique(join_keys + extra_required + non_hidden, "address"),
         environment=environment,
         g=g,
         parents=[enrich_node, base_node],
         force_group=False,
-        node_joins=[
-            NodeJoin(
-                left_node=enrich_node,
-                right_node=base_node,
-                concepts=concept_to_relevant_joins(
-                    [x for x in join_keys if x in enrich_node.output_lcl]
-                ),
-                filter_to_mutual=False,
-                join_type=JoinType.LEFT_OUTER,
-            )
-        ],
     )
 
 

trilogy/core/processing/node_generators/filter_node.py

@@ -1,12 +1,10 @@
 from typing import List
 
 
-from trilogy.core.enums import JoinType
 from trilogy.core.models import Concept, Environment, FilterItem, Grain, WhereClause
 from trilogy.core.processing.nodes import (
     FilterNode,
     MergeNode,
-    NodeJoin,
     History,
     StrategyNode,
     SelectNode,
@@ -16,7 +14,6 @@ from trilogy.core.processing.node_generators.common import (
 )
 from trilogy.constants import logger
 from trilogy.core.processing.utility import padding, unique
-from trilogy.core.processing.node_generators.common import concept_to_relevant_joins
 from trilogy.core.processing.utility import is_scalar_condition
 
 LOGGER_PREFIX = "[GEN_FILTER_NODE]"
@@ -215,16 +212,5 @@ def gen_filter_node(
             # this node fetches only what we need to filter
             filter_node,
             enrich_node,
-        ],
-        node_joins=[
-            NodeJoin(
-                left_node=enrich_node,
-                right_node=filter_node,
-                concepts=concept_to_relevant_joins(
-                    [immediate_parent] + parent_row_concepts
-                ),
-                join_type=JoinType.LEFT_OUTER,
-                filter_to_mutual=True,
-            )
-        ],
+        ]
     )

trilogy/core/processing/node_generators/group_node.py

@@ -1,4 +1,11 @@
-from trilogy.core.models import Concept, Environment, LooseConceptList, WhereClause
+from trilogy.core.models import (
+    Concept,
+    Environment,
+    LooseConceptList,
+    WhereClause,
+    Function,
+    AggregateWrapper,
+)
 from trilogy.utility import unique
 from trilogy.core.processing.nodes import GroupNode, StrategyNode, History
 from typing import List
@@ -42,6 +49,17 @@ def gen_group_node(
         )
         parent_concepts += grain_components
         output_concepts += grain_components
+        for possible_agg in local_optional:
+            if possible_agg.grain and possible_agg.grain == concept.grain:
+                if not isinstance(possible_agg.lineage, (AggregateWrapper, Function)):
+                    continue
+                agg_parents: List[Concept] = resolve_function_parent_concepts(
+                    possible_agg
+                )
+                if set([x.address for x in agg_parents]).issubset(
+                    set([x.address for x in parent_concepts])
+                ):
+                    output_concepts.append(possible_agg)
 
     if parent_concepts:
         logger.info(

trilogy/core/processing/node_generators/group_to_node.py

@@ -3,15 +3,12 @@ from trilogy.core.processing.nodes import (
     GroupNode,
     StrategyNode,
     MergeNode,
-    NodeJoin,
     History,
 )
 from typing import List
-from trilogy.core.enums import JoinType
 
 from trilogy.constants import logger
 from trilogy.core.processing.utility import padding
-from trilogy.core.processing.node_generators.common import concept_to_relevant_joins
 
 LOGGER_PREFIX = "[GEN_GROUP_TO_NODE]"
 
@@ -84,15 +81,6 @@ def gen_group_to_node(
             # this node gets enrichment
             enrich_node,
         ],
-        node_joins=[
-            NodeJoin(
-                left_node=group_node,
-                right_node=enrich_node,
-                concepts=concept_to_relevant_joins(parent_concepts),
-                filter_to_mutual=False,
-                join_type=JoinType.LEFT_OUTER,
-            )
-        ],
         whole_grain=True,
         depth=depth,
     )

trilogy/core/processing/node_generators/multiselect_node.py

@@ -10,7 +10,7 @@ from typing import List
 from trilogy.core.enums import JoinType
 from trilogy.constants import logger
 from trilogy.core.processing.utility import padding
-from trilogy.core.processing.node_generators.common import concept_to_relevant_joins
+from trilogy.core.processing.utility import concept_to_relevant_joins
 from collections import defaultdict
 from itertools import combinations
 from trilogy.core.enums import Purpose
@@ -176,14 +176,5 @@ def gen_multiselect_node(
             # this node gets enrichment
             enrich_node,
         ],
-        node_joins=[
-            NodeJoin(
-                left_node=enrich_node,
-                right_node=node,
-                concepts=possible_joins,
-                filter_to_mutual=False,
-                join_type=JoinType.LEFT_OUTER,
-            )
-        ],
         partial_concepts=node.partial_concepts,
     )

trilogy/core/processing/node_generators/rowset_node.py

@@ -6,14 +6,14 @@ from trilogy.core.models import (
     RowsetItem,
     MultiSelectStatement,
 )
-from trilogy.core.processing.nodes import MergeNode, NodeJoin, History, StrategyNode
+from trilogy.core.processing.nodes import MergeNode, History, StrategyNode
 from trilogy.core.processing.nodes.base_node import concept_list_to_grain
 from typing import List
 
-from trilogy.core.enums import JoinType, PurposeLineage
+from trilogy.core.enums import PurposeLineage
 from trilogy.constants import logger
 from trilogy.core.processing.utility import padding
-from trilogy.core.processing.node_generators.common import concept_to_relevant_joins
+from trilogy.core.processing.utility import concept_to_relevant_joins
 
 
 LOGGER_PREFIX = "[GEN_ROWSET_NODE]"
@@ -113,19 +113,8 @@ def gen_rowset_node(
         g=g,
         depth=depth,
         parents=[
-            # this node gets the window
             node,
-            # this node gets enrichment
             enrich_node,
         ],
-        node_joins=[
-            NodeJoin(
-                left_node=enrich_node,
-                right_node=node,
-                concepts=concept_to_relevant_joins(additional_relevant),
-                filter_to_mutual=False,
-                join_type=JoinType.LEFT_OUTER,
-            )
-        ],
         partial_concepts=node.partial_concepts,
     )

trilogy/core/processing/node_generators/select_node.py

@@ -32,6 +32,7 @@ class DatasourceMatch:
     datasource: Datasource
     matched: LooseConceptList
     partial: LooseConceptList
+    nullable: LooseConceptList
 
     def __repr__(self):
         return f"DatasourceMatch({self.key}, {self.datasource.identifier}, {str(self.matched)}, {str(self.partial)})"
@@ -76,6 +77,7 @@ def dm_to_strategy_node(
         parents=[],
         depth=depth,
         partial_concepts=dm.partial.concepts,
+        nullable_concepts=dm.nullable.concepts,
         accept_partial=accept_partial,
         datasource=datasource,
         grain=datasource.grain,
@@ -189,6 +191,13 @@ def gen_select_nodes_from_tables_v2(
                     if not c.is_complete and c.concept.address in all_lcl
                 ]
             ),
+            nullable=LooseConceptList(
+                concepts=[
+                    c.concept
+                    for c in datasource.columns
+                    if c.is_nullable and c.concept in all_lcl
+                ]
+            ),
         )
         if not matched:
             continue
@@ -318,6 +327,12 @@ def gen_select_node_from_table(
             if not c.is_complete and c.concept in all_lcl
         ]
         partial_lcl = LooseConceptList(concepts=partial_concepts)
+        nullable_concepts = [
+            c.concept
+            for c in datasource.columns
+            if c.is_nullable and c.concept in all_lcl
+        ]
+        nullable_lcl = LooseConceptList(concepts=nullable_concepts)
         if not accept_partial and target_concept in partial_lcl:
             continue
         logger.info(
@@ -359,6 +374,7 @@ def gen_select_node_from_table(
             parents=[],
             depth=depth,
             partial_concepts=[c for c in all_concepts if c in partial_lcl],
+            nullable_concepts=[c for c in all_concepts if c in nullable_lcl],
             accept_partial=accept_partial,
             datasource=datasource,
             grain=Grain(components=all_concepts),
@@ -377,6 +393,7 @@ def gen_select_node_from_table(
                 parents=[bcandidate],
                 depth=depth,
                 partial_concepts=bcandidate.partial_concepts,
+                nullable_concepts=bcandidate.nullable_concepts,
             )
         else:
             candidate = bcandidate
@@ -469,6 +486,14 @@ def gen_select_node(
             )
         ]
 
+        all_nullable = [
+            c
+            for c in all_concepts
+            if any(
+                [c.address in [x.address for x in p.nullable_concepts] for p in parents]
+            )
+        ]
+
         if all_found:
             logger.info(
                 f"{padding(depth)}{LOGGER_PREFIX} found all optional {[c.address for c in local_optional]} via joins"
@@ -494,6 +519,7 @@ def gen_select_node(
                 parents=parents,
                 depth=depth,
                 partial_concepts=all_partial,
+                nullable_concepts=all_nullable,
                 grain=inferred_grain,
             )
 

trilogy/core/processing/node_generators/window_node.py

@@ -12,8 +12,8 @@ from trilogy.constants import logger
 from trilogy.core.processing.utility import padding, create_log_lambda
 from trilogy.core.processing.node_generators.common import (
     gen_enrichment_node,
-    concept_to_relevant_joins,
 )
+from trilogy.core.processing.utility import concept_to_relevant_joins
 
 LOGGER_PREFIX = "[GEN_WINDOW_NODE]"
 

trilogy/core/processing/nodes/base_node.py

@@ -13,6 +13,7 @@ from trilogy.core.models import (
     Comparison,
     Parenthetical,
     LooseConceptList,
+    ConceptPair,
 )
 from trilogy.core.enums import Purpose, JoinType, PurposeLineage, Granularity
 from trilogy.utility import unique
@@ -61,11 +62,9 @@ def resolve_concept_map(
         for concept in input.output_concepts:
             if concept.address not in input.non_partial_concept_addresses:
                 continue
-
-            if (
-                isinstance(input, QueryDatasource)
-                and concept.address in input.hidden_concepts
-            ):
+            if isinstance(input, QueryDatasource) and concept.address in [
+                x.address for x in input.hidden_concepts
+            ]:
                 continue
             if concept.address in full_addresses:
 
@@ -82,10 +81,9 @@ def resolve_concept_map(
         for concept in input.output_concepts:
             if concept.address not in [t.address for t in inherited_inputs]:
                 continue
-            if (
-                isinstance(input, QueryDatasource)
-                and concept.address in input.hidden_concepts
-            ):
+            if isinstance(input, QueryDatasource) and concept.address in [
+                x.address for x in input.hidden_concepts
+            ]:
                 continue
             if len(concept_map.get(concept.address, [])) == 0:
                 concept_map[concept.address].add(input)
@@ -94,6 +92,7 @@ def resolve_concept_map(
         if target.address not in inherited:
             # an empty source means it is defined in this CTE
             concept_map[target.address] = set()
+
     return concept_map
 
 
@@ -124,6 +123,26 @@ def get_all_parent_partial(
     )
 
 
+def get_all_parent_nullable(
+    all_concepts: List[Concept], parents: List["StrategyNode"]
+) -> List[Concept]:
+    return unique(
+        [
+            c
+            for c in all_concepts
+            if len(
+                [
+                    p
+                    for p in parents
+                    if c.address in [x.address for x in p.nullable_concepts]
+                ]
+            )
+            >= 1
+        ],
+        "address",
+    )
+
+
 class StrategyNode:
     source_type = SourceType.ABSTRACT
 
@@ -136,6 +155,7 @@ class StrategyNode:
         whole_grain: bool = False,
         parents: List["StrategyNode"] | None = None,
         partial_concepts: List[Concept] | None = None,
+        nullable_concepts: List[Concept] | None = None,
         depth: int = 0,
         conditions: Conditional | Comparison | Parenthetical | None = None,
         force_group: bool | None = None,
@@ -159,6 +179,9 @@ class StrategyNode:
         self.partial_concepts = partial_concepts or get_all_parent_partial(
             self.output_concepts, self.parents
         )
+        self.nullable_concepts = nullable_concepts or get_all_parent_nullable(
+            self.output_concepts, self.parents
+        )
 
         self.depth = depth
         self.conditions = conditions
@@ -226,7 +249,10 @@ class StrategyNode:
     def remove_output_concepts(self, concepts: List[Concept]):
         for x in concepts:
             self.hidden_concepts.append(x)
-        self.output_concepts = [x for x in self.output_concepts if x not in concepts]
+        addresses = [x.address for x in concepts]
+        self.output_concepts = [
+            x for x in self.output_concepts if x.address not in addresses
+        ]
         self.rebuild_cache()
 
     @property
@@ -257,6 +283,7 @@ class StrategyNode:
             targets=self.output_concepts,
             inherited_inputs=self.input_concepts + self.existence_concepts,
         )
+
         return QueryDatasource(
             input_concepts=self.input_concepts,
             output_concepts=self.output_concepts,
@@ -267,6 +294,7 @@ class StrategyNode:
             grain=grain,
             condition=self.conditions,
             partial_concepts=self.partial_concepts,
+            nullable_concepts=self.nullable_concepts,
             force_group=self.force_group,
             hidden_concepts=self.hidden_concepts,
         )
@@ -295,6 +323,7 @@ class StrategyNode:
             whole_grain=self.whole_grain,
             parents=list(self.parents),
             partial_concepts=list(self.partial_concepts),
+            nullable_concepts=list(self.nullable_concepts),
             depth=self.depth,
             conditions=self.conditions,
             force_group=self.force_group,
@@ -312,7 +341,7 @@ class NodeJoin:
     concepts: List[Concept]
     join_type: JoinType
     filter_to_mutual: bool = False
-    concept_pairs: list[tuple[Concept, Concept]] | None = None
+    concept_pairs: list[ConceptPair] | None = None
 
     def __post_init__(self):
         if self.concept_pairs:

trilogy/core/processing/nodes/group_node.py

@@ -20,6 +20,7 @@ from trilogy.core.processing.nodes.base_node import (
 )
 from trilogy.utility import unique
 from trilogy.core.processing.utility import is_scalar_condition
+from trilogy.core.processing.utility import find_nullable_concepts
 
 LOGGER_PREFIX = "[CONCEPT DETAIL - GROUP NODE]"
 
@@ -37,6 +38,7 @@ class GroupNode(StrategyNode):
         parents: List["StrategyNode"] | None = None,
         depth: int = 0,
         partial_concepts: Optional[List[Concept]] = None,
+        nullable_concepts: Optional[List[Concept]] = None,
         force_group: bool | None = None,
         conditions: Conditional | Comparison | Parenthetical | None = None,
         existence_concepts: List[Concept] | None = None,
@@ -50,6 +52,7 @@ class GroupNode(StrategyNode):
             parents=parents,
             depth=depth,
             partial_concepts=partial_concepts,
+            nullable_concepts=nullable_concepts,
             force_group=force_group,
             conditions=conditions,
             existence_concepts=existence_concepts,
@@ -113,27 +116,34 @@ class GroupNode(StrategyNode):
                     f" {parent.grain}"
                 )
             source_type = SourceType.GROUP
-
+        source_map = resolve_concept_map(
+            parent_sources,
+            targets=(
+                unique(
+                    self.output_concepts + self.conditions.concept_arguments,
+                    "address",
+                )
+                if self.conditions
+                else self.output_concepts
+            ),
+            inherited_inputs=self.input_concepts + self.existence_concepts,
+        )
+        nullable_addresses = find_nullable_concepts(
+            source_map=source_map, joins=[], datasources=parent_sources
+        )
+        nullable_concepts = [
+            x for x in self.output_concepts if x.address in nullable_addresses
+        ]
         base = QueryDatasource(
             input_concepts=self.input_concepts,
             output_concepts=self.output_concepts,
             datasources=parent_sources,
             source_type=source_type,
-            source_map=resolve_concept_map(
-                parent_sources,
-                targets=(
-                    unique(
-                        self.output_concepts + self.conditions.concept_arguments,
-                        "address",
-                    )
-                    if self.conditions
-                    else self.output_concepts
-                ),
-                inherited_inputs=self.input_concepts + self.existence_concepts,
-            ),
+            source_map=source_map,
             joins=[],
             grain=grain,
             partial_concepts=self.partial_concepts,
+            nullable_concepts=nullable_concepts,
             condition=self.conditions,
         )
         # if there is a condition on a group node and it's not scalar
@@ -141,18 +151,20 @@ class GroupNode(StrategyNode):
         if self.conditions and not is_scalar_condition(self.conditions):
             base.condition = None
             base.output_concepts = self.output_concepts + self.conditions.row_arguments
+            source_map = resolve_concept_map(
+                [base],
+                targets=self.output_concepts,
+                inherited_inputs=base.output_concepts,
+            )
             return QueryDatasource(
                 input_concepts=base.output_concepts,
                 output_concepts=self.output_concepts,
                 datasources=[base],
                 source_type=SourceType.SELECT,
-                source_map=resolve_concept_map(
-                    [base],
-                    targets=self.output_concepts,
-                    inherited_inputs=base.output_concepts,
-                ),
+                source_map=source_map,
                 joins=[],
                 grain=grain,
+                nullable_concepts=base.nullable_concepts,
                 partial_concepts=self.partial_concepts,
                 condition=self.conditions,
             )
@@ -168,6 +180,7 @@ class GroupNode(StrategyNode):
             parents=self.parents,
             depth=self.depth,
             partial_concepts=list(self.partial_concepts),
+            nullable_concepts=list(self.nullable_concepts),
             force_group=self.force_group,
             conditions=self.conditions,
             existence_concepts=list(self.existence_concepts),

trilogy/core/processing/nodes/merge_node.py

@@ -22,7 +22,7 @@ from trilogy.core.processing.nodes.base_node import (
     resolve_concept_map,
     NodeJoin,
 )
-from trilogy.core.processing.utility import get_node_joins
+from trilogy.core.processing.utility import get_node_joins, find_nullable_concepts
 
 LOGGER_PREFIX = "[CONCEPT DETAIL - MERGE NODE]"
 
@@ -110,6 +110,7 @@ class MergeNode(StrategyNode):
         join_concepts: Optional[List] = None,
         force_join_type: Optional[JoinType] = None,
         partial_concepts: Optional[List[Concept]] = None,
+        nullable_concepts: Optional[List[Concept]] = None,
         force_group: bool | None = None,
         depth: int = 0,
         grain: Grain | None = None,
@@ -127,6 +128,7 @@ class MergeNode(StrategyNode):
             parents=parents,
             depth=depth,
             partial_concepts=partial_concepts,
+            nullable_concepts=nullable_concepts,
             force_group=force_group,
             grain=grain,
             conditions=conditions,
@@ -192,7 +194,7 @@ class MergeNode(StrategyNode):
         pregrain: Grain,
         grain: Grain,
         environment: Environment,
-    ) -> List[BaseJoin]:
+    ) -> List[BaseJoin | UnnestJoin]:
         # only finally, join between them for unique values
         dataset_list: List[QueryDatasource] = sorted(
             final_datasets, key=lambda x: -len(x.grain.components_copy)
@@ -308,7 +310,7 @@ class MergeNode(StrategyNode):
         )
         join_candidates = [x for x in final_datasets if x not in existence_final]
         if len(join_candidates) > 1:
-            joins = self.generate_joins(
+            joins: List[BaseJoin | UnnestJoin] = self.generate_joins(
                 join_candidates, final_joins, pregrain, grain, self.environment
             )
         else:
@@ -318,7 +320,7 @@ class MergeNode(StrategyNode):
         )
         full_join_concepts = []
         for join in joins:
-            if join.join_type == JoinType.FULL:
+            if isinstance(join, BaseJoin) and join.join_type == JoinType.FULL:
                 full_join_concepts += join.concepts
         if self.whole_grain:
             force_group = False
@@ -341,6 +343,9 @@ class MergeNode(StrategyNode):
             inherited_inputs=self.input_concepts + self.existence_concepts,
             full_joins=full_join_concepts,
         )
+        nullable_concepts = find_nullable_concepts(
+            source_map=source_map, joins=joins, datasources=final_datasets
+        )
         qds = QueryDatasource(
             input_concepts=unique(self.input_concepts, "address"),
             output_concepts=unique(self.output_concepts, "address"),
@@ -349,6 +354,9 @@ class MergeNode(StrategyNode):
             source_map=source_map,
             joins=qd_joins,
             grain=grain,
+            nullable_concepts=[
+                x for x in self.output_concepts if x.address in nullable_concepts
+            ],
             partial_concepts=self.partial_concepts,
             force_group=force_group,
             condition=self.conditions,
@@ -369,9 +377,10 @@ class MergeNode(StrategyNode):
             force_group=self.force_group,
             grain=self.grain,
             conditions=self.conditions,
+            nullable_concepts=list(self.nullable_concepts),
             hidden_concepts=list(self.hidden_concepts),
             virtual_output_concepts=list(self.virtual_output_concepts),
-            node_joins=self.node_joins,
+            node_joins=list(self.node_joins) if self.node_joins else None,
             join_concepts=list(self.join_concepts) if self.join_concepts else None,
             force_join_type=self.force_join_type,
             existence_concepts=list(self.existence_concepts),