pytrilogy 0.0.3.65__tar.gz → 0.0.3.67__tar.gz

{pytrilogy-0.0.3.65/pytrilogy.egg-info → pytrilogy-0.0.3.67}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.65
+Version: 0.0.3.67
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.3.65 → pytrilogy-0.0.3.67/pytrilogy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.0.3.65
+Version: 0.0.3.67
 Summary: Declarative, typed query language that compiles to SQL.
 Home-page: 
 Author: 

{pytrilogy-0.0.3.65 → pytrilogy-0.0.3.67}/tests/test_parsing.py

@@ -2,7 +2,7 @@ from trilogy import Dialects
 from trilogy.constants import MagicConstants
 from trilogy.core.enums import BooleanOperator, ComparisonOperator, Purpose
 from trilogy.core.functions import argument_to_purpose, function_args_to_output_purpose
-from trilogy.core.models.author import Comparison
+from trilogy.core.models.author import Comparison, Conditional, SubselectComparison
 from trilogy.core.models.core import (
     DataType,
     TupleWrapper,
@@ -16,6 +16,7 @@ from trilogy.core.models.environment import (
 from trilogy.core.statements.author import SelectStatement, ShowStatement
 from trilogy.core.statements.execute import ProcessedQuery
 from trilogy.dialect.base import BaseDialect
+from trilogy.parsing.common import atom_is_relevant
 from trilogy.parsing.parse_engine import (
     arg_to_datatype,
     parse_text,
@@ -685,3 +686,45 @@ select x % 10 -> x_mod_10;
             
 """
     )
+
+
+def test_is_atom():
+    env = Environment()
+
+    env.parse(
+        """
+key x int;
+auto x_sum <- sum(x);
+"""
+    )
+
+    assert not atom_is_relevant(
+        Comparison(left=env.concepts["x_sum"], right=0, operator=ComparisonOperator.GT),
+        [env.concepts["x"]],
+        env,
+    )
+    assert atom_is_relevant(
+        Comparison(left=env.concepts["x"], right=0, operator=ComparisonOperator.GT),
+        [env.concepts["x_sum"]],
+        env,
+    )
+
+    assert not atom_is_relevant(
+        Conditional(left=env.concepts["x_sum"], right=0, operator=BooleanOperator.AND),
+        [env.concepts["x"]],
+        env,
+    )
+    assert atom_is_relevant(
+        Conditional(left=env.concepts["x"], right=0, operator=BooleanOperator.AND),
+        [env.concepts["x_sum"]],
+        env,
+    )
+    assert not atom_is_relevant(
+        SubselectComparison(
+            left=env.concepts["x_sum"],
+            right=env.concepts["x"],
+            operator=ComparisonOperator.IN,
+        ),
+        [env.concepts["x_sum"]],
+        env,
+    )

{pytrilogy-0.0.3.65 → pytrilogy-0.0.3.67}/trilogy/__init__.py

@@ -4,6 +4,6 @@ from trilogy.dialect.enums import Dialects
 from trilogy.executor import Executor
 from trilogy.parser import parse
 
-__version__ = "0.0.3.65"
+__version__ = "0.0.3.67"
 
 __all__ = ["parse", "Executor", "Dialects", "Environment", "CONFIG"]

{pytrilogy-0.0.3.65 → pytrilogy-0.0.3.67}/trilogy/core/exceptions.py

@@ -8,7 +8,7 @@ class UndefinedConceptException(Exception):
         self.suggestions = suggestions
 
 
-class UnresolvableQueryException(Exception):
+class FrozenEnvironmentException(Exception):
     pass
 
 
@@ -16,15 +16,15 @@ class InvalidSyntaxException(Exception):
     pass
 
 
-class NoDatasourceException(Exception):
+class UnresolvableQueryException(Exception):
     pass
 
 
-class FrozenEnvironmentException(Exception):
+class NoDatasourceException(UnresolvableQueryException):
     pass
 
 
-class AmbiguousRelationshipResolutionException(Exception):
+class AmbiguousRelationshipResolutionException(UnresolvableQueryException):
     def __init__(self, message, parents: List[set[str]]):
         super().__init__(self, message)
         self.message = message

{pytrilogy-0.0.3.65 → pytrilogy-0.0.3.67}/trilogy/core/models/author.py

@@ -271,6 +271,20 @@ class Conditional(Mergeable, ConceptArgs, Namespaced, DataTyped, BaseModel):
     right: Expr
     operator: BooleanOperator
 
+    @field_validator("left", mode="before")
+    @classmethod
+    def left_validator(cls, v, info: ValidationInfo):
+        if isinstance(v, Concept):
+            return v.reference
+        return v
+
+    @field_validator("right", mode="before")
+    @classmethod
+    def right_validator(cls, v, info: ValidationInfo):
+        if isinstance(v, Concept):
+            return v.reference
+        return v
+
     def __add__(self, other) -> "Conditional":
         if other is None:
             return self
@@ -346,7 +360,6 @@ class Conditional(Mergeable, ConceptArgs, Namespaced, DataTyped, BaseModel):
 
     @property
     def concept_arguments(self) -> Sequence[ConceptRef]:
-        """Return concepts directly referenced in where clause"""
         output = []
         output += get_concept_arguments(self.left)
         output += get_concept_arguments(self.right)
@@ -575,11 +588,11 @@ class Comparison(ConceptArgs, Mergeable, DataTyped, Namespaced, BaseModel):
         date,
         Function,
         ConceptRef,
-        "Conditional",
+        Conditional,
         DataType,
-        "Comparison",
+        Comparison,
         FunctionCallWrapper,
-        "Parenthetical",
+        Parenthetical,
         MagicConstants,
         WindowItem,
         AggregateWrapper,

{pytrilogy-0.0.3.65 → pytrilogy-0.0.3.67}/trilogy/core/models/build.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from abc import ABC
+from collections import defaultdict
 from datetime import date, datetime
 from functools import cached_property, singledispatchmethod
 from typing import (
@@ -1466,6 +1467,20 @@ BuildExpr = (
 BuildConcept.model_rebuild()
 
 
+def get_canonical_pseudonyms(environment: Environment) -> dict[str, set[str]]:
+    roots: dict[str, set[str]] = defaultdict(set)
+    for k, v in environment.concepts.items():
+        roots[v.address].add(k)
+        for x in v.pseudonyms:
+            roots[v.address].add(x)
+    for k, v in environment.alias_origin_lookup.items():
+        lookup = environment.concepts[k].address
+        roots[lookup].add(v.address)
+        for x2 in v.pseudonyms:
+            roots[lookup].add(x2)
+    return roots
+
+
 class Factory:
 
     def __init__(
@@ -1479,6 +1494,7 @@ class Factory:
         self.local_concepts: dict[str, BuildConcept] = (
             {} if local_concepts is None else local_concepts
         )
+        self.pseudonym_map = get_canonical_pseudonyms(environment)
 
     def instantiate_concept(
         self,
@@ -1631,7 +1647,10 @@ class Factory:
 
     @build.register
     def _(self, base: CaseElse) -> BuildCaseElse:
-        return BuildCaseElse.model_construct(expr=self.build(base.expr))
+        expr: Concept | FuncArgs = base.expr
+        if isinstance(expr, (AggregateWrapper, FilterItem, WindowItem)):
+            expr, _ = self.instantiate_concept(expr)
+        return BuildCaseElse.model_construct(expr=self.build(expr))
 
     @build.register
     def _(self, base: Concept) -> BuildConcept:
@@ -1653,6 +1672,17 @@ class Factory:
         )
         is_aggregate = Concept.calculate_is_aggregate(build_lineage)
 
+        # if this is a pseudonym, we need to look up the base address
+        if base.address in self.environment.alias_origin_lookup:
+            lookup_address = self.environment.concepts[base.address].address
+            # map only to the canonical concept, not to other merged concepts
+            base_pseudonyms = {lookup_address}
+        else:
+            base_pseudonyms = {
+                x
+                for x in self.pseudonym_map.get(base.address, set())
+                if x != base.address
+            }
         rval = BuildConcept.model_construct(
             name=base.name,
             datatype=base.datatype,
@@ -1663,7 +1693,7 @@ class Factory:
             namespace=base.namespace,
             keys=base.keys,
             modifiers=base.modifiers,
-            pseudonyms=base.pseudonyms,
+            pseudonyms=base_pseudonyms,
             ## instantiated values
             derivation=derivation,
             granularity=granularity,
@@ -1685,14 +1715,22 @@ class Factory:
 
     @build.register
     def _(self, base: ColumnAssignment) -> BuildColumnAssignment:
-        fetched = self.environment.concepts[base.concept.address]
+        address = base.concept.address
+        fetched = (
+            self.build(
+                self.environment.alias_origin_lookup[address].with_grain(self.grain)
+            )
+            if address in self.environment.alias_origin_lookup
+            else self.build(self.environment.concepts[address].with_grain(self.grain))
+        )
+
         return BuildColumnAssignment.model_construct(
             alias=(
                 self.build(base.alias)
                 if isinstance(base.alias, Function)
                 else base.alias
             ),
-            concept=self.build(fetched.with_grain(self.grain)),
+            concept=fetched,
             modifiers=base.modifiers,
         )
 

{pytrilogy-0.0.3.65 → pytrilogy-0.0.3.67}/trilogy/core/models/environment.py

@@ -362,9 +362,6 @@ class Environment(BaseModel):
                         and x.concept.address != deriv_lookup
                     ]
                     assert len(datasource.columns) < clen
-                    for x in datasource.columns:
-                        logger.info(x)
-
             return None
 
         if existing and self.config.allow_duplicate_declaration:
@@ -607,15 +604,15 @@ class Environment(BaseModel):
                 )
                 persisted = f"{PERSISTED_CONCEPT_PREFIX}_" + new_persisted_concept.name
                 # override the current concept source to reflect that it's now coming from a datasource
+                base_pseudonyms = new_persisted_concept.pseudonyms or set()
+                original_pseudonyms = {*base_pseudonyms, new_persisted_concept.address}
                 if (
                     new_persisted_concept.metadata.concept_source
                     != ConceptSource.PERSIST_STATEMENT
                 ):
                     original_concept = new_persisted_concept.model_copy(
                         deep=True,
-                        update={
-                            "name": persisted,
-                        },
+                        update={"name": persisted, "pseudonyms": original_pseudonyms},
                     )
                     self.add_concept(
                         original_concept,
@@ -629,6 +626,7 @@ class Environment(BaseModel):
                         ),
                         "derivation": Derivation.ROOT,
                         "purpose": new_persisted_concept.purpose,
+                        "pseudonyms": {*original_pseudonyms, original_concept.address},
                     }
                     # purpose is used in derivation calculation
                     # which should be fixed, but we'll do in a followup
@@ -650,6 +648,7 @@ class Environment(BaseModel):
                         new_persisted_concept,
                         meta=meta,
                     )
+
         return datasource
 
     def delete_datasource(

{pytrilogy-0.0.3.65 → pytrilogy-0.0.3.67}/trilogy/core/processing/concept_strategies_v3.py

@@ -224,6 +224,9 @@ def initialize_loop_context(
     else:
 
         completion_mandatory = mandatory_list
+    logger.info(
+        f"{depth_to_prefix(depth)}{LOGGER_PREFIX} Initialized loop context with mandatory list {[c.address for c in mandatory_list]} and completion mandatory {[c.address for c in completion_mandatory]}"
+    )
     return LoopContext(
         mandatory_list=mandatory_list,
         environment=environment,
@@ -330,7 +333,7 @@ def check_for_early_exit(
     return False
 
 
-def generate_loop_completion(context: LoopContext, virtual) -> StrategyNode:
+def generate_loop_completion(context: LoopContext, virtual: set[str]) -> StrategyNode:
     condition_required = True
     non_virtual = [c for c in context.completion_mandatory if c.address not in virtual]
     non_virtual_output = [
@@ -367,10 +370,15 @@ def generate_loop_completion(context: LoopContext, virtual) -> StrategyNode:
         output: StrategyNode = context.stack[0]
         if non_virtual_different:
             logger.info(
-                f"{depth_to_prefix(context.depth)}{LOGGER_PREFIX} Found different non-virtual output concepts ({non_virtual_difference_values}), removing condition injected values"
+                f"{depth_to_prefix(context.depth)}{LOGGER_PREFIX} Found different non-virtual output concepts ({non_virtual_difference_values}), removing condition injected values by setting outputs to {[x.address for x in output.output_concepts if x.address in non_virtual_output]}"
             )
             output.set_output_concepts(
-                [x for x in output.output_concepts if x.address in non_virtual_output],
+                [
+                    x
+                    for x in output.output_concepts
+                    if x.address in non_virtual_output
+                    or any(c in non_virtual_output for c in x.pseudonyms)
+                ],
                 rebuild=False,
             )
 
@@ -404,7 +412,7 @@ def generate_loop_completion(context: LoopContext, virtual) -> StrategyNode:
     elif context.conditions:
         output.preexisting_conditions = context.conditions.conditional
     logger.info(
-        f"{depth_to_prefix(context.depth)}{LOGGER_PREFIX} Graph is connected, returning {type(output)} node partial {[c.address for c in output.partial_concepts]} with {context.conditions}"
+        f"{depth_to_prefix(context.depth)}{LOGGER_PREFIX} Graph is connected, returning {type(output)} node output {[x.address for x in output.usable_outputs]} partial {[c.address for c in output.partial_concepts]} with {context.conditions}"
     )
     if condition_required and context.conditions and non_virtual_different:
         logger.info(
@@ -419,7 +427,7 @@ def generate_loop_completion(context: LoopContext, virtual) -> StrategyNode:
         )
         if result.required:
             logger.info(
-                f"{depth_to_prefix(context.depth)}{LOGGER_PREFIX} Adding group node"
+                f"{depth_to_prefix(context.depth)}{LOGGER_PREFIX} Adding group node with outputs {[x.address for x in context.original_mandatory]}"
             )
             return GroupNode(
                 output_concepts=context.original_mandatory,
@@ -466,6 +474,7 @@ def _search_concepts(
     )
 
     while context.incomplete:
+
         priority_concept = get_priority_concept(
             context.mandatory_list,
             context.attempted,
@@ -478,7 +487,7 @@ def _search_concepts(
         candidates = [
             c for c in context.mandatory_list if c.address != priority_concept.address
         ]
-        # the local conditions list may be override if we end up injecting conditions
+        # the local conditions list may be overriden if we end up injecting conditions
         candidate_list, local_conditions = generate_candidates_restrictive(
             priority_concept,
             candidates,

{pytrilogy-0.0.3.65 → pytrilogy-0.0.3.67}/trilogy/core/processing/discovery_node_factory.py

@@ -351,7 +351,10 @@ class RootNodeHandler:
         extra = restrict_node_outputs_targets(expanded, root_targets, self.ctx.depth)
 
         pseudonyms = [
-            x for x in extra if any(x.address in y.pseudonyms for y in root_targets)
+            x
+            for x in extra
+            if any(x.address in y.pseudonyms for y in root_targets)
+            and x.address not in root_targets
         ]
 
         if pseudonyms:

{pytrilogy-0.0.3.65 → pytrilogy-0.0.3.67}/trilogy/core/processing/discovery_utility.py

@@ -1,7 +1,7 @@
 from typing import List
 
 from trilogy.constants import logger
-from trilogy.core.enums import Derivation, Granularity
+from trilogy.core.enums import Derivation
 from trilogy.core.models.build import (
     BuildConcept,
     BuildRowsetItem,
@@ -52,14 +52,6 @@ def get_priority_concept(
     # pass_two = [c for c in all_concepts+filter_only if c.address not in attempted_addresses]
     for remaining_concept in (pass_one,):
         priority = (
-            # find anything that needs no joins first, so we can exit early
-            [
-                c
-                for c in remaining_concept
-                if c.derivation == Derivation.CONSTANT
-                and c.granularity == Granularity.SINGLE_ROW
-            ]
-            +
             # then multiselects to remove them from scope
             [c for c in remaining_concept if c.derivation == Derivation.MULTISELECT]
             +

{pytrilogy-0.0.3.65 → pytrilogy-0.0.3.67}/trilogy/core/processing/discovery_validation.py

@@ -34,6 +34,9 @@ def validate_concept(
     seen: set[str],
     environment: BuildEnvironment,
 ):
+    # logger.debug(
+    #     f"Validating concept {concept.address} with accept_partial={accept_partial}"
+    # )
     found_map[str(node)].add(concept)
     seen.add(concept.address)
     if concept not in node.partial_concepts:
@@ -53,12 +56,21 @@ def validate_concept(
             found_map[str(node)].add(concept)
     for v_address in concept.pseudonyms:
         if v_address in seen:
-            return
-        v = environment.concepts[v_address]
+            continue
+        if v_address in environment.alias_origin_lookup:
+            # logger.debug(
+            #     f"Found alias origin for {v_address}: {environment.alias_origin_lookup[v_address]} mapped to {environment.concepts[v_address]}")
+            v = environment.alias_origin_lookup[v_address]
+        else:
+            v = environment.concepts[v_address]
+
         if v.address in seen:
-            return
+
+            continue
+
         if v.address == concept.address:
-            return
+
+            continue
         validate_concept(
             v,
             node,
@@ -93,6 +105,7 @@ def validate_stack(
 
         for concept in resolved.output_concepts:
             if concept.address in resolved.hidden_concepts:
+
                 continue
 
             validate_concept(

{pytrilogy-0.0.3.65 → pytrilogy-0.0.3.67}/trilogy/core/processing/node_generators/filter_node.py

@@ -57,7 +57,13 @@ def build_parent_concepts(
     local_optional: List[BuildConcept],
     conditions: BuildWhereClause | None = None,
     depth: int = 0,
-):
+) -> tuple[
+    list[BuildConcept],
+    list[tuple[BuildConcept, ...]],
+    list[BuildConcept],
+    bool,
+    bool,
+]:
     parent_row_concepts, parent_existence_concepts = resolve_filter_parent_concepts(
         concept, environment
     )
@@ -66,6 +72,10 @@ def build_parent_concepts(
     filter_where = concept.lineage.where
 
     same_filter_optional: list[BuildConcept] = []
+    # mypy struggled here? we shouldn't need explicit bools
+    global_filter_is_local_filter: bool = (
+        True if (conditions and conditions == filter_where) else False
+    )
 
     for x in local_optional:
         if isinstance(x.lineage, FILTER_TYPES):
@@ -79,7 +89,7 @@ def build_parent_concepts(
                         parent_row_concepts.append(arg)
                 same_filter_optional.append(x)
                 continue
-        elif conditions and conditions == filter_where:
+        elif global_filter_is_local_filter:
             same_filter_optional.append(x)
 
     # sometimes, it's okay to include other local optional above the filter
@@ -100,6 +110,7 @@ def build_parent_concepts(
         parent_existence_concepts,
         same_filter_optional,
         is_optimized_pushdown,
+        global_filter_is_local_filter,
     )
 
 
@@ -152,6 +163,7 @@ def gen_filter_node(
         parent_existence_concepts,
         same_filter_optional,
         optimized_pushdown,
+        global_filter_is_local_filter,
     ) = build_parent_concepts(
         concept,
         environment=environment,
@@ -187,7 +199,13 @@ def gen_filter_node(
             f"{padding(depth)}{LOGGER_PREFIX} filter node row parents {[x.address for x in parent_row_concepts]} could not be found"
         )
         return None
-
+    if global_filter_is_local_filter:
+        logger.info(
+            f"{padding(depth)}{LOGGER_PREFIX} filter node conditions match global conditions adding row parent {row_parent.output_concepts} with condition {where.conditional}"
+        )
+        row_parent.add_parents(core_parent_nodes)
+        row_parent.set_output_concepts([concept] + local_optional)
+        return row_parent
     if optimized_pushdown:
         logger.info(
             f"{padding(depth)}{LOGGER_PREFIX} returning optimized filter node with pushdown to parent with condition {where.conditional} across {[concept] + same_filter_optional + row_parent.output_concepts} "
@@ -211,7 +229,8 @@ def gen_filter_node(
             parent = row_parent
             parent.add_output_concepts([concept] + same_filter_optional)
         parent.add_parents(core_parent_nodes)
-        parent.add_condition(where.conditional)
+        if not parent.preexisting_conditions == where.conditional:
+            parent.add_condition(where.conditional)
         parent.add_existence_concepts(flattened_existence, False)
         parent.grain = BuildGrain.from_concepts(
             parent.output_concepts,
@@ -225,7 +244,8 @@ def gen_filter_node(
         parents_for_grain = [
             x.lineage.content
             for x in filters
-            if isinstance(x.lineage.content, BuildConcept)
+            if isinstance(x.lineage, BuildFilterItem)
+            and isinstance(x.lineage.content, BuildConcept)
         ]
         filter_node = FilterNode(
             input_concepts=unique(

{pytrilogy-0.0.3.65 → pytrilogy-0.0.3.67}/trilogy/core/processing/node_generators/group_node.py

@@ -133,7 +133,7 @@ def gen_group_node(
         )
         return group_node
     missing_optional = [
-        x.address for x in local_optional if x.address not in group_node.output_concepts
+        x.address for x in local_optional if x.address not in group_node.usable_outputs
     ]
     if not missing_optional:
         logger.info(