pytrilogy 0.0.2.17__py3-none-any.whl → 0.0.2.18__py3-none-any.whl

trilogy/core/processing/nodes/base_node.py

@@ -79,11 +79,12 @@ def resolve_concept_map(
     # second loop, include partials
     for input in inputs:
         for concept in input.output_concepts:
-            if concept.address not in [t.address for t in inherited_inputs]:
+            if concept.address not in [t for t in inherited_inputs]:
                 continue
-            if isinstance(input, QueryDatasource) and concept.address in [
-                x.address for x in input.hidden_concepts
-            ]:
+            if (
+                isinstance(input, QueryDatasource)
+                and concept.address in input.hidden_concepts
+            ):
                 continue
             if len(concept_map.get(concept.address, [])) == 0:
                 concept_map[concept.address].add(input)
@@ -158,6 +159,7 @@ class StrategyNode:
         nullable_concepts: List[Concept] | None = None,
         depth: int = 0,
         conditions: Conditional | Comparison | Parenthetical | None = None,
+        preexisting_conditions: Conditional | Comparison | Parenthetical | None = None,
         force_group: bool | None = None,
         grain: Optional[Grain] = None,
         hidden_concepts: List[Concept] | None = None,
@@ -191,11 +193,32 @@ class StrategyNode:
         self.hidden_concepts = hidden_concepts or []
         self.existence_concepts = existence_concepts or []
         self.virtual_output_concepts = virtual_output_concepts or []
+        self.preexisting_conditions = preexisting_conditions
+        if self.conditions and not self.preexisting_conditions:
+            self.preexisting_conditions = self.conditions
+        elif (
+            self.conditions
+            and self.preexisting_conditions
+            and self.conditions != self.preexisting_conditions
+        ):
+            self.preexisting_conditions = Conditional(
+                left=self.conditions,
+                right=self.preexisting_conditions,
+                operator=BooleanOperator.AND,
+            )
         self.validate_parents()
+        self.log = True
 
     def add_parents(self, parents: list["StrategyNode"]):
         self.parents += parents
         self.validate_parents()
+        return self
+
+    def set_preexisting_conditions(
+        self, conditions: Conditional | Comparison | Parenthetical
+    ):
+        self.preexisting_conditions = conditions
+        return self
 
     def add_condition(self, condition: Conditional | Comparison | Parenthetical):
         if self.conditions:
@@ -204,6 +227,9 @@ class StrategyNode:
             )
         else:
             self.conditions = condition
+        self.set_preexisting_conditions(condition)
+        self.rebuild_cache()
+        return self
 
     def validate_parents(self):
         # validate parents exist
@@ -220,40 +246,54 @@ class StrategyNode:
 
         self.partial_lcl = LooseConceptList(concepts=self.partial_concepts)
 
-    def add_output_concepts(self, concepts: List[Concept]):
+    def add_output_concepts(self, concepts: List[Concept], rebuild: bool = True):
         for concept in concepts:
             if concept.address not in self.output_lcl.addresses:
                 self.output_concepts.append(concept)
         self.output_lcl = LooseConceptList(concepts=self.output_concepts)
-        self.rebuild_cache()
+        if rebuild:
+            self.rebuild_cache()
+        return self
 
-    def add_existence_concepts(self, concepts: List[Concept]):
+    def add_existence_concepts(self, concepts: List[Concept], rebuild: bool = True):
         for concept in concepts:
-            if concept.address not in [x.address for x in self.output_concepts]:
+            if concept.address not in self.output_concepts:
                 self.existence_concepts.append(concept)
-        self.rebuild_cache()
-
-    def set_output_concepts(self, concepts: List[Concept]):
+        if rebuild:
+            self.rebuild_cache()
+        return self
+
+    def set_output_concepts(self, concepts: List[Concept], rebuild: bool = True):
+        # exit if no changes
+        if self.output_concepts == concepts:
+            return self
         self.output_concepts = concepts
         self.output_lcl = LooseConceptList(concepts=self.output_concepts)
-        self.rebuild_cache()
 
-    def add_output_concept(self, concept: Concept):
-        self.add_output_concepts([concept])
+        if rebuild:
+            self.rebuild_cache()
+        return self
 
-    def hide_output_concepts(self, concepts: List[Concept]):
+    def add_output_concept(self, concept: Concept, rebuild: bool = True):
+        return self.add_output_concepts([concept], rebuild)
+
+    def hide_output_concepts(self, concepts: List[Concept], rebuild: bool = True):
         for x in concepts:
             self.hidden_concepts.append(x)
-        self.rebuild_cache()
+        if rebuild:
+            self.rebuild_cache()
+        return self
 
-    def remove_output_concepts(self, concepts: List[Concept]):
+    def remove_output_concepts(self, concepts: List[Concept], rebuild: bool = True):
         for x in concepts:
             self.hidden_concepts.append(x)
         addresses = [x.address for x in concepts]
         self.output_concepts = [
             x for x in self.output_concepts if x.address not in addresses
         ]
-        self.rebuild_cache()
+        if rebuild:
+            self.rebuild_cache()
+        return self
 
     @property
     def logging_prefix(self) -> str:
@@ -269,7 +309,11 @@ class StrategyNode:
 
     def __repr__(self):
         concepts = self.all_concepts
-        contents = ",".join(sorted([c.address for c in concepts]))
+        addresses = [c.address for c in concepts]
+        contents = ",".join(sorted(addresses[:3]))
+        if len(addresses) > 3:
+            extra = len(addresses) - 3
+            contents += f"...{extra} more"
         return f"{self.__class__.__name__}<{contents}>"
 
     def _resolve(self) -> QueryDatasource:
@@ -277,7 +321,11 @@ class StrategyNode:
             p.resolve() for p in self.parents
         ]
 
-        grain = self.grain if self.grain else Grain(components=self.output_concepts)
+        grain = (
+            self.grain
+            if self.grain
+            else concept_list_to_grain(self.output_concepts, [])
+        )
         source_map = resolve_concept_map(
             parent_sources,
             targets=self.output_concepts,
@@ -326,6 +374,7 @@ class StrategyNode:
             nullable_concepts=list(self.nullable_concepts),
             depth=self.depth,
             conditions=self.conditions,
+            preexisting_conditions=self.preexisting_conditions,
             force_group=self.force_group,
             grain=self.grain,
             hidden_concepts=list(self.hidden_concepts),

trilogy/core/processing/nodes/filter_node.py

@@ -33,6 +33,7 @@ class FilterNode(StrategyNode):
         parents: List["StrategyNode"] | None = None,
         depth: int = 0,
         conditions: Conditional | Comparison | Parenthetical | None = None,
+        preexisting_conditions: Conditional | Comparison | Parenthetical | None = None,
         partial_concepts: List[Concept] | None = None,
         force_group: bool | None = False,
         grain: Grain | None = None,
@@ -47,6 +48,7 @@ class FilterNode(StrategyNode):
             depth=depth,
             input_concepts=input_concepts,
             conditions=conditions,
+            preexisting_conditions=preexisting_conditions,
             partial_concepts=partial_concepts,
             force_group=force_group,
             grain=grain,
@@ -63,6 +65,7 @@ class FilterNode(StrategyNode):
             parents=self.parents,
             depth=self.depth,
             conditions=self.conditions,
+            preexisting_conditions=self.preexisting_conditions,
             partial_concepts=list(self.partial_concepts),
             force_group=self.force_group,
             grain=self.grain,

trilogy/core/processing/nodes/group_node.py

@@ -41,7 +41,9 @@ class GroupNode(StrategyNode):
         nullable_concepts: Optional[List[Concept]] = None,
         force_group: bool | None = None,
         conditions: Conditional | Comparison | Parenthetical | None = None,
+        preexisting_conditions: Conditional | Comparison | Parenthetical | None = None,
         existence_concepts: List[Concept] | None = None,
+        hidden_concepts: List[Concept] | None = None,
     ):
         super().__init__(
             input_concepts=input_concepts,
@@ -56,6 +58,8 @@ class GroupNode(StrategyNode):
             force_group=force_group,
             conditions=conditions,
             existence_concepts=existence_concepts,
+            preexisting_conditions=preexisting_conditions,
+            hidden_concepts=hidden_concepts,
         )
 
     def _resolve(self) -> QueryDatasource:
@@ -63,23 +67,18 @@ class GroupNode(StrategyNode):
             p.resolve() for p in self.parents
         ]
 
-        grain = concept_list_to_grain(self.output_concepts, [])
+        grain = self.grain or concept_list_to_grain(self.output_concepts, [])
         comp_grain = Grain()
         for source in parent_sources:
             comp_grain += source.grain
 
         # dynamically select if we need to group
         # because sometimes, we are already at required grain
-        if (
-            comp_grain == grain
-            and self.output_lcl == self.input_lcl
-            and self.force_group is not True
-        ):
+        if comp_grain == grain and self.force_group is not True:
             # if there is no group by, and inputs equal outputs
             # return the parent
             logger.info(
-                f"{self.logging_prefix}{LOGGER_PREFIX} Output of group by node equals input of group by node"
-                f" {self.output_lcl}"
+                f"{self.logging_prefix}{LOGGER_PREFIX} Grain of group by equals output"
                 f" grains {comp_grain} and {grain}"
             )
             if (
@@ -88,7 +87,7 @@ class GroupNode(StrategyNode):
                 == self.output_lcl
             ) and isinstance(parent_sources[0], QueryDatasource):
                 logger.info(
-                    f"{self.logging_prefix}{LOGGER_PREFIX} No group by required, returning parent node"
+                    f"{self.logging_prefix}{LOGGER_PREFIX} No group by required as inputs match outputs of parent; returning parent node"
                 )
                 will_return: QueryDatasource = parent_sources[0]
                 if self.conditions:
@@ -99,21 +98,19 @@ class GroupNode(StrategyNode):
         else:
 
             logger.info(
-                f"{self.logging_prefix}{LOGGER_PREFIX} Group node has different output than input, forcing group"
-                f" {self.input_lcl}"
-                " vs"
-                f" {self.output_lcl}"
-                " and"
+                f"{self.logging_prefix}{LOGGER_PREFIX} Group node has different grain than parents; forcing group"
                 f" upstream grains {[str(source.grain) for source in parent_sources]}"
-                " vs"
+                f" with final grain {comp_grain} vs"
                 f" target grain {grain}"
             )
-            for parent in parent_sources:
+            for parent in self.parents:
                 logger.info(
                     f"{self.logging_prefix}{LOGGER_PREFIX} Parent node"
-                    f" {[c.address for c in parent.output_concepts]}"
+                    f" {[c.address for c in parent.output_concepts[:2]]}... has"
                     " grain"
                     f" {parent.grain}"
+                    f" resolved grain {parent.resolve().grain}"
+                    f" {type(parent)}"
                 )
             source_type = SourceType.GROUP
         source_map = resolve_concept_map(
@@ -144,6 +141,7 @@ class GroupNode(StrategyNode):
             grain=grain,
             partial_concepts=self.partial_concepts,
             nullable_concepts=nullable_concepts,
+            hidden_concepts=self.hidden_concepts,
             condition=self.conditions,
         )
         # if there is a condition on a group node and it's not scalar
@@ -167,6 +165,7 @@ class GroupNode(StrategyNode):
                 nullable_concepts=base.nullable_concepts,
                 partial_concepts=self.partial_concepts,
                 condition=self.conditions,
+                hidden_concepts=self.hidden_concepts,
             )
         return base
 
@@ -183,5 +182,7 @@ class GroupNode(StrategyNode):
             nullable_concepts=list(self.nullable_concepts),
             force_group=self.force_group,
             conditions=self.conditions,
+            preexisting_conditions=self.preexisting_conditions,
             existence_concepts=list(self.existence_concepts),
+            hidden_concepts=list(self.hidden_concepts),
         )

trilogy/core/processing/nodes/merge_node.py

@@ -115,6 +115,7 @@ class MergeNode(StrategyNode):
         depth: int = 0,
         grain: Grain | None = None,
         conditions: Conditional | Comparison | Parenthetical | None = None,
+        preexisting_conditions: Conditional | Comparison | Parenthetical | None = None,
         hidden_concepts: List[Concept] | None = None,
         virtual_output_concepts: List[Concept] | None = None,
         existence_concepts: List[Concept] | None = None,
@@ -132,6 +133,7 @@ class MergeNode(StrategyNode):
             force_group=force_group,
             grain=grain,
             conditions=conditions,
+            preexisting_conditions=preexisting_conditions,
             hidden_concepts=hidden_concepts,
             virtual_output_concepts=virtual_output_concepts,
             existence_concepts=existence_concepts,
@@ -203,14 +205,6 @@ class MergeNode(StrategyNode):
         logger.info(
             f"{self.logging_prefix}{LOGGER_PREFIX} Merge node has {len(dataset_list)} parents, starting merge"
         )
-        for item in dataset_list:
-            logger.info(
-                f"{self.logging_prefix}{LOGGER_PREFIX} for {item.full_name} partial concepts {[x.address for x in item.partial_concepts]}"
-            )
-            logger.info(
-                f"{self.logging_prefix}{LOGGER_PREFIX} potential merge keys {[x.address+str(x.purpose) for x in item.output_concepts]} partial {[x.address for x in item.partial_concepts]}"
-            )
-
         if final_joins is None:
             if not pregrain.components:
                 logger.info(
@@ -246,7 +240,7 @@ class MergeNode(StrategyNode):
         for source in parent_sources:
             if source.full_name in merged:
                 logger.info(
-                    f"{self.logging_prefix}{LOGGER_PREFIX} parent node with {source.full_name} into existing"
+                    f"{self.logging_prefix}{LOGGER_PREFIX} merging parent node with {source.full_name} into existing"
                 )
                 merged[source.full_name] = merged[source.full_name] + source
             else:
@@ -304,7 +298,6 @@ class MergeNode(StrategyNode):
             pregrain += source.grain
 
         grain = self.grain if self.grain else pregrain
-
         logger.info(
             f"{self.logging_prefix}{LOGGER_PREFIX} has pre grain {pregrain} and final merge node grain {grain}"
         )
@@ -377,6 +370,7 @@ class MergeNode(StrategyNode):
             force_group=self.force_group,
             grain=self.grain,
             conditions=self.conditions,
+            preexisting_conditions=self.preexisting_conditions,
             nullable_concepts=list(self.nullable_concepts),
             hidden_concepts=list(self.hidden_concepts),
             virtual_output_concepts=list(self.virtual_output_concepts),

trilogy/core/processing/nodes/select_node_v2.py

@@ -18,8 +18,7 @@ from trilogy.core.models import (
     Parenthetical,
 )
 from trilogy.utility import unique
-from trilogy.core.processing.nodes.base_node import StrategyNode
-from trilogy.core.exceptions import NoDatasourceException
+from trilogy.core.processing.nodes.base_node import StrategyNode, resolve_concept_map
 
 
 LOGGER_PREFIX = "[CONCEPT DETAIL - SELECT NODE]"
@@ -48,6 +47,7 @@ class SelectNode(StrategyNode):
         grain: Optional[Grain] = None,
         force_group: bool | None = False,
         conditions: Conditional | Comparison | Parenthetical | None = None,
+        preexisting_conditions: Conditional | Comparison | Parenthetical | None = None,
         hidden_concepts: List[Concept] | None = None,
     ):
         super().__init__(
@@ -63,6 +63,7 @@ class SelectNode(StrategyNode):
             force_group=force_group,
             grain=grain,
             conditions=conditions,
+            preexisting_conditions=preexisting_conditions,
             hidden_concepts=hidden_concepts,
         )
         self.accept_partial = accept_partial
@@ -144,9 +145,7 @@ class SelectNode(StrategyNode):
 
     def _resolve(self) -> QueryDatasource:
         # if we have parent nodes, we do not need to go to a datasource
-        if self.parents:
-            return super()._resolve()
-        resolution: QueryDatasource | None
+        resolution: QueryDatasource | None = None
         if all(
             [
                 (
@@ -163,17 +162,30 @@ class SelectNode(StrategyNode):
                 f"{self.logging_prefix}{LOGGER_PREFIX} have a constant datasource"
             )
             resolution = self.resolve_from_constant_datasources()
-            if resolution:
-                return resolution
-        if self.datasource:
+        if self.datasource and not resolution:
             resolution = self.resolve_from_provided_datasource()
-            if resolution:
-                return resolution
 
-        required = [c.address for c in self.all_concepts]
-        raise NoDatasourceException(
-            f"Could not find any way to resolve datasources for required concepts {required} with derivation {[x.derivation for x in self.all_concepts]}"
-        )
+        if self.parents:
+            if not resolution:
+                return super()._resolve()
+            # zip in our parent source map
+            parent_sources: List[QueryDatasource | Datasource] = [
+                p.resolve() for p in self.parents
+            ]
+
+            resolution.datasources += parent_sources
+
+            source_map = resolve_concept_map(
+                parent_sources,
+                targets=self.output_concepts,
+                inherited_inputs=self.input_concepts + self.existence_concepts,
+            )
+            for k, v in source_map.items():
+                if v and k not in resolution.source_map:
+                    resolution.source_map[k] = v
+        if not resolution:
+            raise ValueError("No select node could be generated")
+        return resolution
 
     def copy(self) -> "SelectNode":
         return SelectNode(
@@ -191,6 +203,7 @@ class SelectNode(StrategyNode):
             grain=self.grain,
             force_group=self.force_group,
             conditions=self.conditions,
+            preexisting_conditions=self.preexisting_conditions,
             hidden_concepts=self.hidden_concepts,
         )
 
@@ -208,5 +221,6 @@ class ConstantNode(SelectNode):
             depth=self.depth,
             partial_concepts=list(self.partial_concepts),
             conditions=self.conditions,
+            preexisting_conditions=self.preexisting_conditions,
             hidden_concepts=self.hidden_concepts,
         )

trilogy/core/processing/nodes/window_node.py

@@ -1,7 +1,7 @@
 from typing import List
 
 
-from trilogy.core.models import SourceType, Concept, Grain
+from trilogy.core.models import SourceType, Concept
 from trilogy.core.processing.nodes.base_node import StrategyNode, QueryDatasource
 
 
@@ -30,7 +30,6 @@ class WindowNode(StrategyNode):
 
     def _resolve(self) -> QueryDatasource:
         base = super()._resolve()
-        base.grain = Grain(components=self.input_concepts)
         return base
 
     def copy(self) -> "WindowNode":

trilogy/core/processing/utility.py

@@ -1,4 +1,4 @@
-from typing import List, Tuple, Dict, Set
+from typing import List, Tuple, Dict, Set, Any
 import networkx as nx
 from trilogy.core.models import (
     Datasource,
@@ -20,6 +20,14 @@ from trilogy.core.models import (
     DataType,
     ConceptPair,
     UnnestJoin,
+    CaseWhen,
+    CaseElse,
+    MapWrapper,
+    ListWrapper,
+    MapType,
+    DatePart,
+    NumericType,
+    ListType,
 )
 
 from trilogy.core.enums import Purpose, Granularity, BooleanOperator, Modifier
@@ -107,9 +115,19 @@ def resolve_join_order(joins: List[BaseJoin]) -> List[BaseJoin]:
     available_aliases: set[str] = set()
     final_joins_pre = [*joins]
     final_joins = []
+    partial = set()
     while final_joins_pre:
         new_final_joins_pre: List[BaseJoin] = []
         for join in final_joins_pre:
+            if join.join_type != JoinType.INNER:
+                partial.add(join.right_datasource.identifier)
+            # an inner join after a left outer implicitly makes that outer an inner
+            # so fix that
+            if (
+                join.left_datasource.identifier in partial
+                and join.join_type == JoinType.INNER
+            ):
+                join.join_type = JoinType.LEFT_OUTER
             if not available_aliases:
                 final_joins.append(join)
                 available_aliases.add(join.left_datasource.identifier)
@@ -270,7 +288,7 @@ def get_node_joins(
         relevant = concept_to_relevant_joins(local_concepts)
         left_datasource = identifier_map[left]
         right_datasource = identifier_map[right]
-        join_tuples = []
+        join_tuples: list[ConceptPair] = []
         for joinc in relevant:
             left_arg = joinc
             right_arg = joinc
@@ -316,6 +334,19 @@ def get_node_joins(
                         left=left_arg, right=right_arg, modifiers=list(modifiers)
                     )
                 )
+
+        # deduplication
+        all_right = []
+        for tuple in join_tuples:
+            all_right.append(tuple.right.address)
+        right_grain = identifier_map[right].grain
+        # if the join includes all the right grain components
+        # we only need to join on those, not everything
+        if all([x.address in all_right for x in right_grain.components]):
+            join_tuples = [
+                x for x in join_tuples if x.right.address in right_grain.components
+            ]
+
         final_joins_pre.append(
             BaseJoin(
                 left_datasource=identifier_map[left],
@@ -373,7 +404,7 @@ def is_scalar_condition(
         int
         | str
         | float
-        | list
+        | list[Any]
         | WindowItem
         | FilterItem
         | Concept
@@ -384,6 +415,14 @@ def is_scalar_condition(
         | AggregateWrapper
         | MagicConstants
         | DataType
+        | CaseWhen
+        | CaseElse
+        | MapWrapper[Any, Any]
+        | ListType
+        | MapType
+        | NumericType
+        | DatePart
+        | ListWrapper[Any]
     ),
     materialized: set[str] | None = None,
 ) -> bool:
@@ -398,6 +437,7 @@ def is_scalar_condition(
     elif isinstance(element, Function):
         if element.operator in FunctionClass.AGGREGATE_FUNCTIONS.value:
             return False
+        return all([is_scalar_condition(x, materialized) for x in element.arguments])
     elif isinstance(element, Concept):
         if materialized and element.address in materialized:
             return True
@@ -410,6 +450,14 @@ def is_scalar_condition(
         return is_scalar_condition(element.left, materialized) and is_scalar_condition(
             element.right, materialized
         )
+    elif isinstance(element, CaseWhen):
+        return is_scalar_condition(
+            element.comparison, materialized
+        ) and is_scalar_condition(element.expr, materialized)
+    elif isinstance(element, CaseElse):
+        return is_scalar_condition(element.expr, materialized)
+    elif isinstance(element, MagicConstants):
+        return True
     return True