pytrilogy 0.0.2.58__py3-none-any.whl → 0.0.3.1__py3-none-any.whl

trilogy/core/processing/nodes/group_node.py

@@ -2,23 +2,23 @@ from dataclasses import dataclass
 from typing import List, Optional
 
 from trilogy.constants import logger
-from trilogy.core.models import (
-    Comparison,
-    Concept,
-    Conditional,
-    Datasource,
-    Environment,
-    Grain,
-    Parenthetical,
-    QueryDatasource,
-    SourceType,
+from trilogy.core.enums import SourceType
+from trilogy.core.models.build import (
+    BuildComparison,
+    BuildConcept,
+    BuildConditional,
+    BuildDatasource,
+    BuildGrain,
+    BuildOrderBy,
+    BuildParenthetical,
 )
+from trilogy.core.models.build_environment import BuildEnvironment
+from trilogy.core.models.execute import QueryDatasource
 from trilogy.core.processing.nodes.base_node import (
     StrategyNode,
     resolve_concept_map,
 )
 from trilogy.core.processing.utility import find_nullable_concepts, is_scalar_condition
-from trilogy.parsing.common import concepts_to_grain_concepts
 from trilogy.utility import unique
 
 LOGGER_PREFIX = "[CONCEPT DETAIL - GROUP NODE]"
@@ -26,8 +26,8 @@ LOGGER_PREFIX = "[CONCEPT DETAIL - GROUP NODE]"
 
 @dataclass
 class GroupRequiredResponse:
-    target: Grain
-    upstream: Grain
+    target: BuildGrain
+    upstream: BuildGrain
     required: bool
 
 
@@ -36,19 +36,24 @@ class GroupNode(StrategyNode):
 
     def __init__(
         self,
-        output_concepts: List[Concept],
-        input_concepts: List[Concept],
-        environment: Environment,
+        output_concepts: List[BuildConcept],
+        input_concepts: List[BuildConcept],
+        environment: BuildEnvironment,
         whole_grain: bool = False,
         parents: List["StrategyNode"] | None = None,
         depth: int = 0,
-        partial_concepts: Optional[List[Concept]] = None,
-        nullable_concepts: Optional[List[Concept]] = None,
+        partial_concepts: Optional[List[BuildConcept]] = None,
+        nullable_concepts: Optional[List[BuildConcept]] = None,
         force_group: bool | None = None,
-        conditions: Conditional | Comparison | Parenthetical | None = None,
-        preexisting_conditions: Conditional | Comparison | Parenthetical | None = None,
-        existence_concepts: List[Concept] | None = None,
+        conditions: (
+            BuildConditional | BuildComparison | BuildParenthetical | None
+        ) = None,
+        preexisting_conditions: (
+            BuildConditional | BuildComparison | BuildParenthetical | None
+        ) = None,
+        existence_concepts: List[BuildConcept] | None = None,
         hidden_concepts: set[str] | None = None,
+        ordering: BuildOrderBy | None = None,
     ):
         super().__init__(
             input_concepts=input_concepts,
@@ -64,27 +69,33 @@ class GroupNode(StrategyNode):
             existence_concepts=existence_concepts,
             preexisting_conditions=preexisting_conditions,
             hidden_concepts=hidden_concepts,
+            ordering=ordering,
         )
 
     @classmethod
     def check_if_required(
         cls,
-        downstream_concepts: List[Concept],
-        parents: list[QueryDatasource | Datasource],
-        environment: Environment,
+        downstream_concepts: List[BuildConcept],
+        parents: list[QueryDatasource | BuildDatasource],
+        environment: BuildEnvironment,
     ) -> GroupRequiredResponse:
-        target_grain = Grain.from_concepts(
-            concepts_to_grain_concepts(
-                downstream_concepts,
-                environment=environment,
-            )
+        target_grain = BuildGrain.from_concepts(
+            downstream_concepts,
+            environment=environment,
         )
-        comp_grain = Grain()
+
+        # the concepts of the souce grain might not exist in the output environment
+        # so we need to construct a new
+        concept_map: dict[str, BuildConcept] = {}
+        comp_grain = BuildGrain()
         for source in parents:
             comp_grain += source.grain
-        comp_grain = Grain.from_concepts(
-            concepts_to_grain_concepts(comp_grain.components, environment=environment)
-        )
+            for x in source.output_concepts:
+                concept_map[x.address] = x
+        lookups = [
+            concept_map[x] if x in concept_map else x for x in comp_grain.components
+        ]
+        comp_grain = BuildGrain.from_concepts(lookups, environment=environment)
         # dynamically select if we need to group
         # because sometimes, we are already at required grain
         if comp_grain.issubset(target_grain):
@@ -93,7 +104,7 @@ class GroupNode(StrategyNode):
         return GroupRequiredResponse(target_grain, comp_grain, True)
 
     def _resolve(self) -> QueryDatasource:
-        parent_sources: List[QueryDatasource | Datasource] = [
+        parent_sources: List[QueryDatasource | BuildDatasource] = [
             p.resolve() for p in self.parents
         ]
         grains = self.check_if_required(
@@ -145,13 +156,15 @@ class GroupNode(StrategyNode):
             nullable_concepts=nullable_concepts,
             hidden_concepts=self.hidden_concepts,
             condition=self.conditions,
+            ordering=self.ordering,
         )
         # if there is a condition on a group node and it's not scalar
         # inject an additional CTE
         if self.conditions and not is_scalar_condition(self.conditions):
             base.condition = None
             base.output_concepts = unique(
-                base.output_concepts + self.conditions.row_arguments, "address"
+                list(base.output_concepts) + list(self.conditions.row_arguments),
+                "address",
             )
             # re-visible any hidden concepts
             base.hidden_concepts = set(
@@ -174,6 +187,7 @@ class GroupNode(StrategyNode):
                 partial_concepts=self.partial_concepts,
                 condition=self.conditions,
                 hidden_concepts=self.hidden_concepts,
+                ordering=self.ordering,
             )
         return base
 
@@ -192,4 +206,5 @@ class GroupNode(StrategyNode):
             preexisting_conditions=self.preexisting_conditions,
             existence_concepts=list(self.existence_concepts),
             hidden_concepts=set(self.hidden_concepts),
+            ordering=self.ordering,
         )

trilogy/core/processing/nodes/merge_node.py

@@ -1,20 +1,21 @@
 from typing import List, Optional, Tuple
 
 from trilogy.constants import logger
-from trilogy.core.models import (
-    BaseJoin,
-    Comparison,
-    Concept,
-    Conditional,
-    Datasource,
-    Environment,
-    Grain,
+from trilogy.core.enums import (
     JoinType,
-    Parenthetical,
-    QueryDatasource,
     SourceType,
-    UnnestJoin,
 )
+from trilogy.core.models.build import (
+    BuildComparison,
+    BuildConcept,
+    BuildConditional,
+    BuildDatasource,
+    BuildGrain,
+    BuildOrderBy,
+    BuildParenthetical,
+)
+from trilogy.core.models.build_environment import BuildEnvironment
+from trilogy.core.models.execute import BaseJoin, QueryDatasource, UnnestJoin
 from trilogy.core.processing.nodes.base_node import (
     NodeJoin,
     StrategyNode,
@@ -27,10 +28,10 @@ LOGGER_PREFIX = "[CONCEPT DETAIL - MERGE NODE]"
 
 
 def deduplicate_nodes(
-    merged: dict[str, QueryDatasource | Datasource],
+    merged: dict[str, QueryDatasource | BuildDatasource],
     logging_prefix: str,
-    environment: Environment,
-) -> tuple[bool, dict[str, QueryDatasource | Datasource], set[str]]:
+    environment: BuildEnvironment,
+) -> tuple[bool, dict[str, QueryDatasource | BuildDatasource], set[str]]:
     duplicates = False
     removed: set[str] = set()
     set_map: dict[str, set[str]] = {}
@@ -73,10 +74,10 @@ def deduplicate_nodes(
 
 def deduplicate_nodes_and_joins(
     joins: List[NodeJoin] | None,
-    merged: dict[str, QueryDatasource | Datasource],
+    merged: dict[str, QueryDatasource | BuildDatasource],
     logging_prefix: str,
-    environment: Environment,
-) -> Tuple[List[NodeJoin] | None, dict[str, QueryDatasource | Datasource]]:
+    environment: BuildEnvironment,
+) -> Tuple[List[NodeJoin] | None, dict[str, QueryDatasource | BuildDatasource]]:
     # it's possible that we have more sources than we need
     duplicates = True
     while duplicates:
@@ -100,24 +101,29 @@ class MergeNode(StrategyNode):
 
     def __init__(
         self,
-        input_concepts: List[Concept],
-        output_concepts: List[Concept],
+        input_concepts: List[BuildConcept],
+        output_concepts: List[BuildConcept],
         environment,
         whole_grain: bool = False,
         parents: List["StrategyNode"] | None = None,
         node_joins: List[NodeJoin] | None = None,
         join_concepts: Optional[List] = None,
         force_join_type: Optional[JoinType] = None,
-        partial_concepts: Optional[List[Concept]] = None,
-        nullable_concepts: Optional[List[Concept]] = None,
+        partial_concepts: Optional[List[BuildConcept]] = None,
+        nullable_concepts: Optional[List[BuildConcept]] = None,
         force_group: bool | None = None,
         depth: int = 0,
-        grain: Grain | None = None,
-        conditions: Conditional | Comparison | Parenthetical | None = None,
-        preexisting_conditions: Conditional | Comparison | Parenthetical | None = None,
+        grain: BuildGrain | None = None,
+        conditions: (
+            BuildConditional | BuildComparison | BuildParenthetical | None
+        ) = None,
+        preexisting_conditions: (
+            BuildConditional | BuildComparison | BuildParenthetical | None
+        ) = None,
         hidden_concepts: set[str] | None = None,
-        virtual_output_concepts: List[Concept] | None = None,
-        existence_concepts: List[Concept] | None = None,
+        virtual_output_concepts: List[BuildConcept] | None = None,
+        existence_concepts: List[BuildConcept] | None = None,
+        ordering: BuildOrderBy | None = None,
     ):
         super().__init__(
             input_concepts=input_concepts,
@@ -135,6 +141,7 @@ class MergeNode(StrategyNode):
             hidden_concepts=hidden_concepts,
             virtual_output_concepts=virtual_output_concepts,
             existence_concepts=existence_concepts,
+            ordering=ordering,
         )
         self.join_concepts = join_concepts
         self.force_join_type = force_join_type
@@ -166,7 +173,7 @@ class MergeNode(StrategyNode):
             )
         return joins
 
-    def create_full_joins(self, dataset_list: List[QueryDatasource | Datasource]):
+    def create_full_joins(self, dataset_list: List[QueryDatasource | BuildDatasource]):
         joins = []
         seen = set()
         for left_value in dataset_list:
@@ -191,12 +198,12 @@ class MergeNode(StrategyNode):
         self,
         final_datasets,
         final_joins: List[NodeJoin] | None,
-        pregrain: Grain,
-        grain: Grain,
-        environment: Environment,
+        pregrain: BuildGrain,
+        grain: BuildGrain,
+        environment: BuildEnvironment,
     ) -> List[BaseJoin | UnnestJoin]:
         # only finally, join between them for unique values
-        dataset_list: List[QueryDatasource | Datasource] = sorted(
+        dataset_list: List[QueryDatasource | BuildDatasource] = sorted(
             final_datasets, key=lambda x: -len(x.grain.components)
         )
 
@@ -227,10 +234,10 @@ class MergeNode(StrategyNode):
         return joins
 
     def _resolve(self) -> QueryDatasource:
-        parent_sources: List[QueryDatasource | Datasource] = [
+        parent_sources: List[QueryDatasource | BuildDatasource] = [
             p.resolve() for p in self.parents
         ]
-        merged: dict[str, QueryDatasource | Datasource] = {}
+        merged: dict[str, QueryDatasource | BuildDatasource] = {}
         final_joins: List[NodeJoin] | None = self.node_joins
         for source in parent_sources:
             if source.identifier in merged:
@@ -246,7 +253,7 @@ class MergeNode(StrategyNode):
             final_joins, merged, self.logging_prefix, self.environment
         )
         # early exit if we can just return the parent
-        final_datasets: List[QueryDatasource | Datasource] = list(merged.values())
+        final_datasets: List[QueryDatasource | BuildDatasource] = list(merged.values())
 
         existence_final = [
             x
@@ -254,7 +261,7 @@ class MergeNode(StrategyNode):
             if all([y in self.existence_concepts for y in x.output_concepts])
         ]
         if len(merged.keys()) == 1:
-            final: QueryDatasource | Datasource = list(merged.values())[0]
+            final: QueryDatasource | BuildDatasource = list(merged.values())[0]
             if (
                 set([c.address for c in final.output_concepts])
                 == set([c.address for c in self.output_concepts])
@@ -288,7 +295,7 @@ class MergeNode(StrategyNode):
                 )
                 return dataset
 
-        pregrain = Grain()
+        pregrain = BuildGrain()
         for source in final_datasets:
             pregrain += source.grain
 
@@ -352,6 +359,7 @@ class MergeNode(StrategyNode):
             force_group=force_group,
             condition=self.conditions,
             hidden_concepts=self.hidden_concepts,
+            ordering=self.ordering,
         )
         return qds
 
@@ -375,4 +383,5 @@ class MergeNode(StrategyNode):
             join_concepts=list(self.join_concepts) if self.join_concepts else None,
             force_join_type=self.force_join_type,
             existence_concepts=list(self.existence_concepts),
+            ordering=self.ordering,
         )

trilogy/core/processing/nodes/select_node_v2.py

@@ -2,20 +2,19 @@ from typing import List, Optional
 
 from trilogy.constants import logger
 from trilogy.core.constants import CONSTANT_DATASET
-from trilogy.core.enums import Purpose, PurposeLineage
-from trilogy.core.models import (
-    Comparison,
-    Concept,
-    Conditional,
-    Datasource,
-    Environment,
-    Function,
-    Grain,
-    Parenthetical,
-    QueryDatasource,
-    SourceType,
-    UnnestJoin,
+from trilogy.core.enums import Derivation, Purpose, SourceType
+from trilogy.core.models.build import (
+    BuildComparison,
+    BuildConcept,
+    BuildConditional,
+    BuildDatasource,
+    BuildFunction,
+    BuildGrain,
+    BuildOrderBy,
+    BuildParenthetical,
 )
+from trilogy.core.models.build_environment import BuildEnvironment
+from trilogy.core.models.execute import QueryDatasource, UnnestJoin
 from trilogy.core.processing.nodes.base_node import StrategyNode, resolve_concept_map
 from trilogy.utility import unique
 
@@ -31,21 +30,26 @@ class SelectNode(StrategyNode):
 
     def __init__(
         self,
-        input_concepts: List[Concept],
-        output_concepts: List[Concept],
-        environment: Environment,
-        datasource: Datasource | None = None,
+        input_concepts: List[BuildConcept],
+        output_concepts: List[BuildConcept],
+        environment: BuildEnvironment,
+        datasource: BuildDatasource | None = None,
         whole_grain: bool = False,
         parents: List["StrategyNode"] | None = None,
         depth: int = 0,
-        partial_concepts: List[Concept] | None = None,
-        nullable_concepts: List[Concept] | None = None,
+        partial_concepts: List[BuildConcept] | None = None,
+        nullable_concepts: List[BuildConcept] | None = None,
         accept_partial: bool = False,
-        grain: Optional[Grain] = None,
+        grain: Optional[BuildGrain] = None,
         force_group: bool | None = False,
-        conditions: Conditional | Comparison | Parenthetical | None = None,
-        preexisting_conditions: Conditional | Comparison | Parenthetical | None = None,
+        conditions: (
+            BuildConditional | BuildComparison | BuildParenthetical | None
+        ) = None,
+        preexisting_conditions: (
+            BuildConditional | BuildComparison | BuildParenthetical | None
+        ) = None,
         hidden_concepts: set[str] | None = None,
+        ordering: BuildOrderBy | None = None,
     ):
         super().__init__(
             input_concepts=input_concepts,
@@ -61,6 +65,7 @@ class SelectNode(StrategyNode):
             conditions=conditions,
             preexisting_conditions=preexisting_conditions,
             hidden_concepts=hidden_concepts,
+            ordering=ordering,
         )
         self.accept_partial = accept_partial
         self.datasource = datasource
@@ -75,31 +80,31 @@ class SelectNode(StrategyNode):
     ) -> QueryDatasource:
         if not self.datasource:
             raise ValueError("Datasource not provided")
-        datasource: Datasource = self.datasource
+        datasource: BuildDatasource = self.datasource
 
-        all_concepts_final: List[Concept] = unique(self.all_concepts, "address")
-        source_map: dict[str, set[Datasource | QueryDatasource | UnnestJoin]] = {
+        all_concepts_final: List[BuildConcept] = unique(self.all_concepts, "address")
+        source_map: dict[str, set[BuildDatasource | QueryDatasource | UnnestJoin]] = {
             concept.address: {datasource} for concept in self.input_concepts
         }
 
         derived_concepts = [
             c
             for c in datasource.columns
-            if isinstance(c.alias, Function) and c.concept.address in source_map
+            if isinstance(c.alias, BuildFunction) and c.concept.address in source_map
         ]
         for c in derived_concepts:
-            if not isinstance(c.alias, Function):
+            if not isinstance(c.alias, BuildFunction):
                 continue
             for x in c.alias.concept_arguments:
                 source_map[x.address] = {datasource}
         for x in all_concepts_final:
             if x.address not in source_map and x.derivation in (
-                PurposeLineage.MULTISELECT,
-                PurposeLineage.FILTER,
-                PurposeLineage.BASIC,
-                PurposeLineage.ROWSET,
-                PurposeLineage.BASIC,
-                PurposeLineage.UNION,
+                Derivation.MULTISELECT,
+                Derivation.FILTER,
+                Derivation.BASIC,
+                Derivation.ROWSET,
+                Derivation.BASIC,
+                Derivation.UNION,
             ):
                 source_map[x.address] = set()
 
@@ -109,7 +114,7 @@ class SelectNode(StrategyNode):
         if self.force_group is False:
             grain = datasource.grain
         else:
-            grain = self.grain or Grain()
+            grain = self.grain or BuildGrain()
         return QueryDatasource(
             input_concepts=self.input_concepts,
             output_concepts=all_concepts_final,
@@ -127,10 +132,11 @@ class SelectNode(StrategyNode):
             # select nodes should never group
             force_group=self.force_group,
             hidden_concepts=self.hidden_concepts,
+            ordering=self.ordering,
         )
 
     def resolve_from_constant_datasources(self) -> QueryDatasource:
-        datasource = Datasource(
+        datasource = BuildDatasource(
             name=CONSTANT_DATASET, address=CONSTANT_DATASET, columns=[]
         )
         return QueryDatasource(
@@ -144,6 +150,7 @@ class SelectNode(StrategyNode):
             partial_concepts=[],
             source_type=SourceType.CONSTANT,
             hidden_concepts=self.hidden_concepts,
+            ordering=self.ordering,
         )
 
     def _resolve(self) -> QueryDatasource:
@@ -152,10 +159,10 @@ class SelectNode(StrategyNode):
         if all(
             [
                 (
-                    c.derivation == PurposeLineage.CONSTANT
+                    c.derivation == Derivation.CONSTANT
                     or (
                         c.purpose == Purpose.CONSTANT
-                        and c.derivation == PurposeLineage.MULTISELECT
+                        and c.derivation == Derivation.MULTISELECT
                     )
                 )
                 for c in self.all_concepts
@@ -165,6 +172,8 @@ class SelectNode(StrategyNode):
                 f"{self.logging_prefix}{LOGGER_PREFIX} have a constant datasource"
             )
             resolution = self.resolve_from_constant_datasources()
+            return resolution
+
         if self.datasource and not resolution:
             resolution = self.resolve_from_provided_datasource()
 
@@ -172,7 +181,7 @@ class SelectNode(StrategyNode):
             if not resolution:
                 return super()._resolve()
             # zip in our parent source map
-            parent_sources: List[QueryDatasource | Datasource] = [
+            parent_sources: List[QueryDatasource | BuildDatasource] = [
                 p.resolve() for p in self.parents
             ]
 
@@ -187,7 +196,7 @@ class SelectNode(StrategyNode):
                 if v and k not in resolution.source_map:
                     resolution.source_map[k] = v
         if not resolution:
-            raise ValueError("No select node could be generated")
+            raise ValueError(f"No select node could be generated for {self}")
         return resolution
 
     def copy(self) -> "SelectNode":
@@ -207,6 +216,7 @@ class SelectNode(StrategyNode):
             conditions=self.conditions,
             preexisting_conditions=self.preexisting_conditions,
             hidden_concepts=self.hidden_concepts,
+            ordering=self.ordering,
         )
 
 
@@ -224,4 +234,8 @@ class ConstantNode(SelectNode):
             conditions=self.conditions,
             preexisting_conditions=self.preexisting_conditions,
             hidden_concepts=self.hidden_concepts,
+            ordering=self.ordering,
         )
+
+    def _resolve(self) -> QueryDatasource:
+        return self.resolve_from_constant_datasources()

trilogy/core/processing/nodes/union_node.py

@@ -1,10 +1,8 @@
 from typing import List
 
-from trilogy.core.models import (
-    Concept,
-    QueryDatasource,
-    SourceType,
-)
+from trilogy.core.enums import SourceType
+from trilogy.core.models.build import BuildConcept
+from trilogy.core.models.execute import QueryDatasource
 from trilogy.core.processing.nodes.base_node import StrategyNode
 
 
@@ -15,8 +13,8 @@ class UnionNode(StrategyNode):
 
     def __init__(
         self,
-        input_concepts: List[Concept],
-        output_concepts: List[Concept],
+        input_concepts: List[BuildConcept],
+        output_concepts: List[BuildConcept],
         environment,
         whole_grain: bool = False,
         parents: List["StrategyNode"] | None = None,

trilogy/core/processing/nodes/unnest_node.py

@@ -1,12 +1,8 @@
 from typing import List
 
-from trilogy.core.models import (
-    Concept,
-    Function,
-    QueryDatasource,
-    SourceType,
-    UnnestJoin,
-)
+from trilogy.core.enums import SourceType
+from trilogy.core.models.build import BuildConcept, BuildFunction
+from trilogy.core.models.execute import QueryDatasource, UnnestJoin
 from trilogy.core.processing.nodes.base_node import StrategyNode
 
 
@@ -19,9 +15,9 @@ class UnnestNode(StrategyNode):
 
     def __init__(
         self,
-        unnest_concepts: List[Concept],
-        input_concepts: List[Concept],
-        output_concepts: List[Concept],
+        unnest_concepts: List[BuildConcept],
+        input_concepts: List[BuildConcept],
+        output_concepts: List[BuildConcept],
         environment,
         whole_grain: bool = False,
         parents: List["StrategyNode"] | None = None,
@@ -41,7 +37,7 @@ class UnnestNode(StrategyNode):
         """We need to ensure that any filtered values are removed from the output to avoid inappropriate references"""
         base = super()._resolve()
         lineage = self.unnest_concepts[0].lineage
-        assert isinstance(lineage, Function)
+        assert isinstance(lineage, BuildFunction)
         final = "_".join(set([c.address for c in self.unnest_concepts]))
         unnest = UnnestJoin(
             concepts=self.unnest_concepts,

trilogy/core/processing/nodes/window_node.py

@@ -1,7 +1,9 @@
 from typing import List
 
-from trilogy.core.models import Concept, SourceType
-from trilogy.core.processing.nodes.base_node import QueryDatasource, StrategyNode
+from trilogy.core.enums import SourceType
+from trilogy.core.models.build import BuildConcept, BuildOrderBy
+from trilogy.core.models.execute import QueryDatasource
+from trilogy.core.processing.nodes.base_node import StrategyNode
 
 
 class WindowNode(StrategyNode):
@@ -9,12 +11,13 @@ class WindowNode(StrategyNode):
 
     def __init__(
         self,
-        input_concepts: List[Concept],
-        output_concepts: List[Concept],
+        input_concepts: List[BuildConcept],
+        output_concepts: List[BuildConcept],
         environment,
         whole_grain: bool = False,
         parents: List["StrategyNode"] | None = None,
         depth: int = 0,
+        ordering: BuildOrderBy | None = None,
     ):
         super().__init__(
             input_concepts=input_concepts,
@@ -23,6 +26,7 @@ class WindowNode(StrategyNode):
             whole_grain=whole_grain,
             parents=parents,
             depth=depth,
+            ordering=ordering,
         )
 
     def _resolve(self) -> QueryDatasource:
@@ -37,4 +41,5 @@ class WindowNode(StrategyNode):
             whole_grain=self.whole_grain,
             parents=self.parents,
             depth=self.depth,
+            ordering=self.ordering,
         )