pytrilogy 0.0.2.57__py3-none-any.whl → 0.0.3.0__py3-none-any.whl

trilogy/core/processing/node_generators/unnest_node.py

@@ -1,7 +1,8 @@
 from typing import List
 
 from trilogy.constants import logger
-from trilogy.core.models import Concept, Function, WhereClause
+from trilogy.core.models.build import BuildConcept, BuildFunction, BuildWhereClause
+from trilogy.core.models.build_environment import BuildEnvironment
 from trilogy.core.processing.nodes import History, StrategyNode, UnnestNode
 from trilogy.core.processing.utility import padding
 
@@ -9,17 +10,17 @@ LOGGER_PREFIX = "[GEN_UNNEST_NODE]"
 
 
 def gen_unnest_node(
-    concept: Concept,
-    local_optional: List[Concept],
-    environment,
+    concept: BuildConcept,
+    local_optional: List[BuildConcept],
+    history: History,
+    environment: BuildEnvironment,
     g,
     depth: int,
     source_concepts,
-    history: History | None = None,
-    conditions: WhereClause | None = None,
+    conditions: BuildWhereClause | None = None,
 ) -> StrategyNode | None:
     arguments = []
-    if isinstance(concept.lineage, Function):
+    if isinstance(concept.lineage, BuildFunction):
         arguments = concept.lineage.concept_arguments
 
     equivalent_optional = [x for x in local_optional if x.lineage == concept.lineage]
@@ -56,6 +57,7 @@ def gen_unnest_node(
         output_concepts=base.output_concepts,
         environment=environment,
         parents=[base],
+        # conditions=conditions.conditional if conditions else None,
         preexisting_conditions=conditions.conditional if conditions else None,
     )
     qds = new.resolve()

trilogy/core/processing/node_generators/window_node.py

@@ -1,7 +1,8 @@
 from typing import List
 
 from trilogy.constants import logger
-from trilogy.core.models import Concept, Environment, WhereClause, WindowItem
+from trilogy.core.models.build import BuildConcept, BuildWhereClause, BuildWindowItem
+from trilogy.core.models.build_environment import BuildEnvironment
 from trilogy.core.processing.nodes import History, StrategyNode, WindowNode
 from trilogy.core.processing.utility import padding
 from trilogy.utility import unique
@@ -9,40 +10,42 @@ from trilogy.utility import unique
 LOGGER_PREFIX = "[GEN_WINDOW_NODE]"
 
 
+WINDOW_TYPES = (BuildWindowItem,)
+
+
 def resolve_window_parent_concepts(
-    concept: Concept, environment: Environment
-) -> tuple[Concept, List[Concept]]:
-    if not isinstance(concept.lineage, WindowItem):
+    concept: BuildConcept, environment: BuildEnvironment
+) -> tuple[BuildConcept, List[BuildConcept]]:
+    if not isinstance(concept.lineage, WINDOW_TYPES):
         raise ValueError
     base = []
+    logger.info(concept.lineage)
     if concept.lineage.over:
         base += concept.lineage.over
     if concept.lineage.order_by:
         for item in concept.lineage.order_by:
-            # TODO: we do want to use the rehydrated value, but
-            # that introduces a circular dependency on an aggregate
-            # that is grouped by a window
-            # need to figure out how to resolve this
-            # base += [environment.concepts[item.expr.output.address]]
-            base += [item.expr.output]
+            base += item.concept_arguments
     return concept.lineage.content, unique(base, "address")
 
 
 def gen_window_node(
-    concept: Concept,
-    local_optional: list[Concept],
-    environment: Environment,
+    concept: BuildConcept,
+    local_optional: list[BuildConcept],
+    environment: BuildEnvironment,
     g,
     depth: int,
     source_concepts,
     history: History | None = None,
-    conditions: WhereClause | None = None,
+    conditions: BuildWhereClause | None = None,
 ) -> StrategyNode | None:
     base, parent_concepts = resolve_window_parent_concepts(concept, environment)
+    logger.info(
+        f"{padding(depth)}{LOGGER_PREFIX} generating window node for {concept} with parents {parent_concepts}"
+    )
     equivalent_optional = [
         x
         for x in local_optional
-        if isinstance(x.lineage, WindowItem)
+        if isinstance(x.lineage, WINDOW_TYPES)
         and resolve_window_parent_concepts(x, environment)[1] == parent_concepts
     ]
 
@@ -52,7 +55,7 @@ def gen_window_node(
     targets = [base]
     if equivalent_optional:
         for x in equivalent_optional:
-            assert isinstance(x.lineage, WindowItem)
+            assert isinstance(x.lineage, WINDOW_TYPES)
             targets.append(x.lineage.content)
 
     parent_node: StrategyNode = source_concepts(

trilogy/core/processing/nodes/__init__.py

@@ -1,6 +1,8 @@
 from pydantic import BaseModel, ConfigDict, Field
 
-from trilogy.core.models import Concept, Environment, WhereClause
+from trilogy.core.models.build import BuildConcept, BuildWhereClause
+from trilogy.core.models.build_environment import BuildEnvironment
+from trilogy.core.models.environment import Environment
 
 from .base_node import NodeJoin, StrategyNode
 from .filter_node import FilterNode
@@ -13,6 +15,7 @@ from .window_node import WindowNode
 
 
 class History(BaseModel):
+    base_environment: Environment
     history: dict[str, StrategyNode | None] = Field(default_factory=dict)
     select_history: dict[str, StrategyNode | None] = Field(default_factory=dict)
     started: set[str] = Field(default_factory=set)
@@ -20,9 +23,9 @@ class History(BaseModel):
 
     def _concepts_to_lookup(
         self,
-        search: list[Concept],
+        search: list[BuildConcept],
         accept_partial: bool,
-        conditions: WhereClause | None = None,
+        conditions: BuildWhereClause | None = None,
     ) -> str:
         if conditions:
             return (
@@ -34,10 +37,10 @@ class History(BaseModel):
 
     def search_to_history(
         self,
-        search: list[Concept],
+        search: list[BuildConcept],
         accept_partial: bool,
         output: StrategyNode | None,
-        conditions: WhereClause | None = None,
+        conditions: BuildWhereClause | None = None,
     ):
         self.history[
             self._concepts_to_lookup(search, accept_partial, conditions=conditions)
@@ -45,8 +48,8 @@ class History(BaseModel):
 
     def get_history(
         self,
-        search: list[Concept],
-        conditions: WhereClause | None = None,
+        search: list[BuildConcept],
+        conditions: BuildWhereClause | None = None,
         accept_partial: bool = False,
         parent_key: str = "",
     ) -> StrategyNode | None | bool:
@@ -68,9 +71,9 @@ class History(BaseModel):
 
     def log_start(
         self,
-        search: list[Concept],
+        search: list[BuildConcept],
         accept_partial: bool = False,
-        conditions: WhereClause | None = None,
+        conditions: BuildWhereClause | None = None,
     ):
         self.started.add(
             self._concepts_to_lookup(
@@ -82,9 +85,9 @@ class History(BaseModel):
 
     def check_started(
         self,
-        search: list[Concept],
+        search: list[BuildConcept],
         accept_partial: bool = False,
-        conditions: WhereClause | None = None,
+        conditions: BuildWhereClause | None = None,
     ):
         return (
             self._concepts_to_lookup(
@@ -97,12 +100,12 @@ class History(BaseModel):
 
     def _select_concepts_to_lookup(
         self,
-        main: Concept,
-        search: list[Concept],
+        main: BuildConcept,
+        search: list[BuildConcept],
         accept_partial: bool,
         fail_if_not_found: bool,
         accept_partial_optional: bool,
-        conditions: WhereClause | None = None,
+        conditions: BuildWhereClause | None = None,
     ) -> str:
         return (
             str(main.address)
@@ -116,16 +119,16 @@ class History(BaseModel):
 
     def gen_select_node(
         self,
-        concept: Concept,
-        local_optional: list[Concept],
-        environment: Environment,
+        concept: BuildConcept,
+        local_optional: list[BuildConcept],
+        environment: BuildEnvironment,
         g,
         depth: int,
         source_concepts,
         fail_if_not_found: bool = False,
         accept_partial: bool = False,
         accept_partial_optional: bool = False,
-        conditions: WhereClause | None = None,
+        conditions: BuildWhereClause | None = None,
     ) -> StrategyNode | None:
         from trilogy.core.processing.node_generators.select_node import gen_select_node
 

trilogy/core/processing/nodes/base_node.py

@@ -4,45 +4,45 @@ from typing import List, Optional
 
 from trilogy.core.enums import (
     BooleanOperator,
+    Derivation,
     JoinType,
-    PurposeLineage,
-)
-from trilogy.core.models import (
-    Comparison,
-    Concept,
-    ConceptPair,
-    Conditional,
-    Datasource,
-    Environment,
-    Grain,
-    LooseConceptList,
-    Parenthetical,
-    QueryDatasource,
     SourceType,
-    UnnestJoin,
 )
+from trilogy.core.models.build import (
+    BuildComparison,
+    BuildConcept,
+    BuildConditional,
+    BuildDatasource,
+    BuildGrain,
+    BuildOrderBy,
+    BuildParenthetical,
+    LooseBuildConceptList,
+)
+from trilogy.core.models.build_environment import BuildEnvironment
+from trilogy.core.models.execute import ConceptPair, QueryDatasource, UnnestJoin
 from trilogy.utility import unique
 
 
 def resolve_concept_map(
-    inputs: List[QueryDatasource | Datasource],
-    targets: List[Concept],
-    inherited_inputs: List[Concept],
-    full_joins: List[Concept] | None = None,
-) -> dict[str, set[Datasource | QueryDatasource | UnnestJoin]]:
+    inputs: List[QueryDatasource | BuildDatasource],
+    targets: List[BuildConcept],
+    inherited_inputs: List[BuildConcept],
+    full_joins: List[BuildConcept] | None = None,
+) -> dict[str, set[BuildDatasource | QueryDatasource | UnnestJoin]]:
     targets = targets or []
-    concept_map: dict[str, set[Datasource | QueryDatasource | UnnestJoin]] = (
+    concept_map: dict[str, set[BuildDatasource | QueryDatasource | UnnestJoin]] = (
         defaultdict(set)
     )
     full_addresses = {c.address for c in full_joins} if full_joins else set()
     inherited = set([t.address for t in inherited_inputs])
     for input in inputs:
         for concept in input.output_concepts:
-            if concept.address not in input.non_partial_concept_addresses:
+            if concept.address not in input.full_concepts:
                 continue
-            if isinstance(input, QueryDatasource) and concept.address in [
-                x.address for x in input.hidden_concepts
-            ]:
+            if (
+                isinstance(input, QueryDatasource)
+                and concept.address in input.hidden_concepts
+            ):
                 continue
             if concept.address in full_addresses:
                 concept_map[concept.address].add(input)
@@ -75,8 +75,8 @@ def resolve_concept_map(
 
 
 def get_all_parent_partial(
-    all_concepts: List[Concept], parents: List["StrategyNode"]
-) -> List[Concept]:
+    all_concepts: List[BuildConcept], parents: List["StrategyNode"]
+) -> List[BuildConcept]:
     return unique(
         [
             c
@@ -102,8 +102,8 @@ def get_all_parent_partial(
 
 
 def get_all_parent_nullable(
-    all_concepts: List[Concept], parents: List["StrategyNode"]
-) -> List[Concept]:
+    all_concepts: List[BuildConcept], parents: List["StrategyNode"]
+) -> List[BuildConcept]:
     return unique(
         [
             c
@@ -126,28 +126,33 @@ class StrategyNode:
 
     def __init__(
         self,
-        input_concepts: List[Concept],
-        output_concepts: List[Concept],
-        environment: Environment,
+        input_concepts: List[BuildConcept],
+        output_concepts: List[BuildConcept],
+        environment: BuildEnvironment,
         whole_grain: bool = False,
         parents: List["StrategyNode"] | None = None,
-        partial_concepts: List[Concept] | None = None,
-        nullable_concepts: List[Concept] | None = None,
+        partial_concepts: List[BuildConcept] | None = None,
+        nullable_concepts: List[BuildConcept] | None = None,
         depth: int = 0,
-        conditions: Conditional | Comparison | Parenthetical | None = None,
-        preexisting_conditions: Conditional | Comparison | Parenthetical | None = None,
+        conditions: (
+            BuildConditional | BuildComparison | BuildParenthetical | None
+        ) = None,
+        preexisting_conditions: (
+            BuildConditional | BuildComparison | BuildParenthetical | None
+        ) = None,
         force_group: bool | None = None,
-        grain: Optional[Grain] = None,
-        hidden_concepts: List[Concept] | None = None,
-        existence_concepts: List[Concept] | None = None,
-        virtual_output_concepts: List[Concept] | None = None,
+        grain: Optional[BuildGrain] = None,
+        hidden_concepts: set[str] | None = None,
+        existence_concepts: List[BuildConcept] | None = None,
+        virtual_output_concepts: List[BuildConcept] | None = None,
+        ordering: BuildOrderBy | None = None,
     ):
-        self.input_concepts: List[Concept] = (
+        self.input_concepts: List[BuildConcept] = (
             unique(input_concepts, "address") if input_concepts else []
         )
-        self.input_lcl = LooseConceptList(concepts=self.input_concepts)
-        self.output_concepts: List[Concept] = unique(output_concepts, "address")
-        self.output_lcl = LooseConceptList(concepts=self.output_concepts)
+        self.input_lcl = LooseBuildConceptList(concepts=self.input_concepts)
+        self.output_concepts: List[BuildConcept] = unique(output_concepts, "address")
+        self.output_lcl = LooseBuildConceptList(concepts=self.output_concepts)
 
         self.environment = environment
         self.whole_grain = whole_grain
@@ -159,13 +164,13 @@ class StrategyNode:
         self.nullable_concepts = nullable_concepts or get_all_parent_nullable(
             self.output_concepts, self.parents
         )
-
+        self.ordering = ordering
         self.depth = depth
         self.conditions = conditions
         self.grain = grain
         self.force_group = force_group
         self.tainted = False
-        self.hidden_concepts = hidden_concepts or []
+        self.hidden_concepts = hidden_concepts or set()
         self.existence_concepts = existence_concepts or []
         self.virtual_output_concepts = virtual_output_concepts or []
         self.preexisting_conditions = preexisting_conditions
@@ -176,7 +181,7 @@ class StrategyNode:
             and self.preexisting_conditions
             and self.conditions != self.preexisting_conditions
         ):
-            self.preexisting_conditions = Conditional(
+            self.preexisting_conditions = BuildConditional(
                 left=self.conditions,
                 right=self.preexisting_conditions,
                 operator=BooleanOperator.AND,
@@ -206,16 +211,18 @@ class StrategyNode:
         return self
 
     def set_preexisting_conditions(
-        self, conditions: Conditional | Comparison | Parenthetical
+        self, conditions: BuildConditional | BuildComparison | BuildParenthetical
     ):
         self.preexisting_conditions = conditions
         return self
 
-    def add_condition(self, condition: Conditional | Comparison | Parenthetical):
+    def add_condition(
+        self, condition: BuildConditional | BuildComparison | BuildParenthetical
+    ):
         if self.conditions and condition == self.conditions:
             return self
         if self.conditions:
-            self.conditions = Conditional(
+            self.conditions = BuildConditional(
                 left=self.conditions, right=condition, operator=BooleanOperator.AND
             )
         else:
@@ -237,27 +244,29 @@ class StrategyNode:
                 self.output_concepts, self.parents
             )
 
-        self.partial_lcl = LooseConceptList(concepts=self.partial_concepts)
+        self.partial_lcl = LooseBuildConceptList(concepts=self.partial_concepts)
 
-    def add_output_concepts(self, concepts: List[Concept], rebuild: bool = True):
+    def add_output_concepts(self, concepts: List[BuildConcept], rebuild: bool = True):
         for concept in concepts:
             if concept.address not in self.output_lcl.addresses:
                 self.output_concepts.append(concept)
-        self.output_lcl = LooseConceptList(concepts=self.output_concepts)
+        self.output_lcl = LooseBuildConceptList(concepts=self.output_concepts)
         if rebuild:
             self.rebuild_cache()
         return self
 
-    def add_partial_concepts(self, concepts: List[Concept], rebuild: bool = True):
+    def add_partial_concepts(self, concepts: List[BuildConcept], rebuild: bool = True):
         for concept in concepts:
             if concept.address not in self.partial_lcl.addresses:
                 self.partial_concepts.append(concept)
-        self.partial_lcl = LooseConceptList(concepts=self.partial_concepts)
+        self.partial_lcl = LooseBuildConceptList(concepts=self.partial_concepts)
         if rebuild:
             self.rebuild_cache()
         return self
 
-    def add_existence_concepts(self, concepts: List[Concept], rebuild: bool = True):
+    def add_existence_concepts(
+        self, concepts: List[BuildConcept], rebuild: bool = True
+    ):
         for concept in concepts:
             if concept.address not in self.output_concepts:
                 self.existence_concepts.append(concept)
@@ -265,38 +274,40 @@ class StrategyNode:
             self.rebuild_cache()
         return self
 
-    def set_output_concepts(self, concepts: List[Concept], rebuild: bool = True):
+    def set_output_concepts(self, concepts: List[BuildConcept], rebuild: bool = True):
         # exit if no changes
         if self.output_concepts == concepts:
             return self
         self.output_concepts = concepts
-        self.output_lcl = LooseConceptList(concepts=self.output_concepts)
+        self.output_lcl = LooseBuildConceptList(concepts=self.output_concepts)
 
         if rebuild:
             self.rebuild_cache()
         return self
 
-    def add_output_concept(self, concept: Concept, rebuild: bool = True):
+    def add_output_concept(self, concept: BuildConcept, rebuild: bool = True):
         return self.add_output_concepts([concept], rebuild)
 
-    def hide_output_concepts(self, concepts: List[Concept], rebuild: bool = True):
+    def hide_output_concepts(self, concepts: List[BuildConcept], rebuild: bool = True):
         for x in concepts:
-            self.hidden_concepts.append(x)
+            self.hidden_concepts.add(x.address)
         if rebuild:
             self.rebuild_cache()
         return self
 
-    def unhide_output_concepts(self, concepts: List[Concept], rebuild: bool = True):
-        self.hidden_concepts = [
-            x for x in self.hidden_concepts if x.address not in concepts
-        ]
+    def unhide_output_concepts(
+        self, concepts: List[BuildConcept], rebuild: bool = True
+    ):
+        self.hidden_concepts = set(x for x in self.hidden_concepts if x not in concepts)
         if rebuild:
             self.rebuild_cache()
         return self
 
-    def remove_output_concepts(self, concepts: List[Concept], rebuild: bool = True):
+    def remove_output_concepts(
+        self, concepts: List[BuildConcept], rebuild: bool = True
+    ):
         for x in concepts:
-            self.hidden_concepts.append(x)
+            self.hidden_concepts.add(x.address)
         addresses = [x.address for x in concepts]
         self.output_concepts = [
             x for x in self.output_concepts if x.address not in addresses
@@ -306,7 +317,7 @@ class StrategyNode:
         return self
 
     @property
-    def usable_outputs(self) -> list[Concept]:
+    def usable_outputs(self) -> list[BuildConcept]:
         return [
             x for x in self.output_concepts if x.address not in self.hidden_concepts
         ]
@@ -316,11 +327,11 @@ class StrategyNode:
         return "\t" * self.depth
 
     @property
-    def all_concepts(self) -> list[Concept]:
+    def all_concepts(self) -> list[BuildConcept]:
         return [*self.output_concepts]
 
     @property
-    def all_used_concepts(self) -> list[Concept]:
+    def all_used_concepts(self) -> list[BuildConcept]:
         return [*self.input_concepts, *self.existence_concepts]
 
     def __repr__(self):
@@ -333,11 +344,13 @@ class StrategyNode:
         return f"{self.__class__.__name__}<{contents}>"
 
     def _resolve(self) -> QueryDatasource:
-        parent_sources: List[QueryDatasource | Datasource] = [
+        parent_sources: List[QueryDatasource | BuildDatasource] = [
             p.resolve() for p in self.parents
         ]
 
-        grain = self.grain if self.grain else Grain.from_concepts(self.output_concepts)
+        grain = (
+            self.grain if self.grain else BuildGrain.from_concepts(self.output_concepts)
+        )
         source_map = resolve_concept_map(
             parent_sources,
             targets=self.output_concepts,
@@ -357,11 +370,12 @@ class StrategyNode:
             nullable_concepts=self.nullable_concepts,
             force_group=self.force_group,
             hidden_concepts=self.hidden_concepts,
+            ordering=self.ordering,
         )
 
     def rebuild_cache(self) -> QueryDatasource:
         self.tainted = True
-        self.output_lcl = LooseConceptList(concepts=self.output_concepts)
+        self.output_lcl = LooseBuildConceptList(concepts=self.output_concepts)
         if not self.resolution_cache:
             return self.resolve()
         self.resolution_cache = None
@@ -388,9 +402,10 @@ class StrategyNode:
             preexisting_conditions=self.preexisting_conditions,
             force_group=self.force_group,
             grain=self.grain,
-            hidden_concepts=list(self.hidden_concepts),
+            hidden_concepts=set(self.hidden_concepts),
             existence_concepts=list(self.existence_concepts),
             virtual_output_concepts=list(self.virtual_output_concepts),
+            ordering=self.ordering,
         )
 
 
@@ -398,12 +413,14 @@ class StrategyNode:
 class NodeJoin:
     left_node: StrategyNode
     right_node: StrategyNode
-    concepts: List[Concept]
+    concepts: List[BuildConcept]
     join_type: JoinType
     filter_to_mutual: bool = False
     concept_pairs: list[ConceptPair] | None = None
 
     def __post_init__(self):
+        if self.left_node == self.right_node:
+            raise SyntaxError("Invalid join, left and right nodes are the same")
         if self.concept_pairs:
             return
         final_concepts = []
@@ -424,9 +441,7 @@ class NodeJoin:
             # if one datasource only has constants
             # we can join on 1=1
             for ds in [self.left_node, self.right_node]:
-                if all(
-                    [c.derivation == PurposeLineage.CONSTANT for c in ds.all_concepts]
-                ):
+                if all([c.derivation == Derivation.CONSTANT for c in ds.all_concepts]):
                     self.concepts = []
                     return
 

trilogy/core/processing/nodes/filter_node.py

@@ -1,13 +1,15 @@
 from typing import List
 
-from trilogy.core.models import (
-    Comparison,
-    Concept,
-    Conditional,
-    Grain,
-    Parenthetical,
+from trilogy.core.enums import (
     SourceType,
 )
+from trilogy.core.models.build import (
+    BuildComparison,
+    BuildConcept,
+    BuildConditional,
+    BuildGrain,
+    BuildParenthetical,
+)
 from trilogy.core.processing.nodes.base_node import StrategyNode
 
 
@@ -24,18 +26,22 @@ class FilterNode(StrategyNode):
 
     def __init__(
         self,
-        input_concepts: List[Concept],
-        output_concepts: List[Concept],
+        input_concepts: List[BuildConcept],
+        output_concepts: List[BuildConcept],
         environment,
         whole_grain: bool = False,
         parents: List["StrategyNode"] | None = None,
         depth: int = 0,
-        conditions: Conditional | Comparison | Parenthetical | None = None,
-        preexisting_conditions: Conditional | Comparison | Parenthetical | None = None,
-        partial_concepts: List[Concept] | None = None,
+        conditions: (
+            BuildConditional | BuildComparison | BuildParenthetical | None
+        ) = None,
+        preexisting_conditions: (
+            BuildConditional | BuildComparison | BuildParenthetical | None
+        ) = None,
+        partial_concepts: List[BuildConcept] | None = None,
         force_group: bool | None = False,
-        grain: Grain | None = None,
-        existence_concepts: List[Concept] | None = None,
+        grain: BuildGrain | None = None,
+        existence_concepts: List[BuildConcept] | None = None,
     ):
         super().__init__(
             output_concepts=output_concepts,