pytrilogy 0.3.330__tar.gz → 0.3.332__tar.gz

{pytrilogy-0.3.330 → pytrilogy-0.3.332}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.3.330
+Version: 0.3.332
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.9

{pytrilogy-0.3.330 → pytrilogy-0.3.332}/trilogy/__init__.py

@@ -7,7 +7,7 @@ if TYPE_CHECKING:
     from trilogy.executor import Executor
     from trilogy.parser import parse
 
-__version__ = "0.3.330"
+__version__ = "0.3.332"
 
 __all__ = [
     "CONFIG",

{pytrilogy-0.3.330 → pytrilogy-0.3.332}/trilogy/authoring/__init__.py

@@ -55,6 +55,7 @@ from trilogy.core.models.datasource import Address, Datasource, DatasourceMetada
 from trilogy.core.models.environment import DictImportResolver, Environment
 from trilogy.core.statements.author import (
     STATEMENT_TYPES,
+    CallStatement,
     ConceptDeclarationStatement,
     ConceptTransform,
     CopyStatement,
@@ -89,6 +90,7 @@ __all__ = [
     "ArrayType",
     # trilogy.core.enums
     "BooleanOperator",
+    "CallStatement",
     "CaseElse",
     "CaseWhen",
     "Comment",

{pytrilogy-0.3.330 → pytrilogy-0.3.332}/trilogy/core/models/execute.py

@@ -1830,11 +1830,27 @@ class UnionCTE:
 
     @property
     def group_to_grain(self) -> bool:
+        """A union never dedupes itself — only its consumer can decide.
+
+        A UNION ALL whose projection drops its arms' grain keys does hold
+        duplicate rows against its own declared grain, so this is NOT the
+        "already a unique set" it looks like. But the duplicates are only noise
+        to a consumer joining the union as a key set; to a consumer aggregating
+        it they are the data. `select species, sum(dbh)` over two tree-grain
+        partitions projects (species, dbh) and must keep all three of three
+        identically-valued trees — deduping here silently under-counts the sum.
+
+        The union cannot tell those apart, so the obligation sits with whoever
+        reads it at a coarser grain: that consumer must group. See
+        `_grain_claim_needs_group` in v4_node_generators/basic.py, which is the
+        projection-shaped case of exactly that.
+        """
         return False
 
     @property
     def group_concepts(self) -> list[BuildConcept]:
-        # unions should always be on unique sets
+        # Nothing to group by: see group_to_grain — deduping is the consumer's
+        # obligation, never the union's.
         return []
 
     def __add__(self, other):

{pytrilogy-0.3.330 → pytrilogy-0.3.332}/trilogy/core/optimizations/union_dim_pushdown.py

@@ -192,6 +192,23 @@ def _datasource_matches_raw_id(
     return isinstance(base, BuildDatasource) and base.identifier == source_id
 
 
+def _derives_from(node: CTE | UnionCTE, container_name: str) -> bool:
+    """``node`` IS ``container_name`` or reads from it, at any depth."""
+    seen: set[str] = set()
+    stack: list[CTE | UnionCTE] = [node]
+    while stack:
+        current = stack.pop()
+        if current.name == container_name:
+            return True
+        if current.name in seen:
+            continue
+        seen.add(current.name)
+        stack.extend(current.dependency_nodes())
+        if isinstance(current, UnionCTE):
+            stack.extend(current.internal_ctes)
+    return False
+
+
 def _dim_local_atoms(
     cte: CTE,
     dim_qds: BuildDatasource | QueryDatasource,
@@ -604,12 +621,25 @@ class UnionDimPushdown(OptimizationRule):
         return needed.issubset(out)
 
     def _resolve_push_context(
-        self, consumers: list[CTE], d: _DimDescriptor
+        self, consumers: list[CTE], d: _DimDescriptor, container: CTE | UnionCTE
     ) -> _PushContext | None:
         for c in consumers:
             found = _find_dim_cte_for_qds(c, d.join_qds_id)
-            if found is not None and self._dim_cte_exposes(found, d):
-                return _PushContext(dim_cte=found, source_consumer=c)
+            if found is None or not self._dim_cte_exposes(found, d):
+                continue
+            # A "dim" carved OUT of the container (a single-key projection of the
+            # union, which the planner peels when the union is the only source of
+            # that key) cannot be computed before it. Pushing it in would make the
+            # container's branches depend on a CTE that depends on the container —
+            # `reorder_ctes` then fails the whole query with "CTE dependency graph
+            # contains a cycle".
+            if _derives_from(found, container.name):
+                self.log(
+                    f"Skipping dim {d.dim_qds.identifier}: its CTE {found.name} "
+                    f"is derived from {container.name}"
+                )
+                continue
+            return _PushContext(dim_cte=found, source_consumer=c)
         return None
 
     def _can_push_into_branch(self, branch: CTE, d: _DimDescriptor) -> bool:
@@ -634,7 +664,7 @@ class UnionDimPushdown(OptimizationRule):
         if not d.strip_safe and not d.where_atoms:
             return False
 
-        context = self._resolve_push_context(consumers, d)
+        context = self._resolve_push_context(consumers, d, union)
         if context is None:
             return False
 
@@ -707,7 +737,7 @@ class UnionDimPushdown(OptimizationRule):
         """
         if not d.strip_safe or not d.where_atoms:
             return False
-        context = self._resolve_push_context(consumers, d)
+        context = self._resolve_push_context(consumers, d, target)
         if context is None:
             return False
         if not self._can_push_into_branch(target, d):

{pytrilogy-0.3.330 → pytrilogy-0.3.332}/trilogy/core/processing/node_generators/select_helpers/datasource_injection.py

@@ -1,5 +1,4 @@
 from collections import defaultdict
-from itertools import product
 
 from trilogy.core.enums import (
     AddressType,
@@ -76,9 +75,12 @@ def _best_enum_union(
 ) -> list[list[BuildDatasource]] | None:
     """Find the best minimal covering combinations for an enum-partitioned key.
 
-    Groups by covered enum value, enumerates one-source-per-value combos,
-    computes the combo's actual concept overlap (minus the merge key), and
-    keeps the highest-scoring combo per distinct overlap signature. Returning
+    Groups by covered enum value, then searches one-source-per-value combos via
+    a dynamic program over values whose state is the combo's concept overlap so
+    far (minus the merge key), keeping the highest-scoring combo per distinct
+    overlap signature. The score is separable (a per-source sum) and the
+    overlap is the only coupling between values, so this is exhaustive over
+    achievable signatures without enumerating the k^V combo product. Returning
     one combo per overlap lets parallel partitionings (e.g., sales vs.
     returns vs. dim, all keyed by the same channel enum) each contribute
     their own union datasource instead of collapsing into the single best.
@@ -100,36 +102,92 @@ def _best_enum_union(
         return None
 
     values = list(by_value.keys())
-    merge_key_addr = {merge_key.address}
-
-    best_per_overlap: dict[frozenset[str], tuple[list[BuildDatasource], int]] = {}
+    # A union requires at least 2 distinct sources; a single source is not a union
+    if len(values) < 2:
+        return None
 
-    for combo in product(*[by_value[v] for v in values]):
-        combo_list = list(combo)
+    cols: dict[int, frozenset[str]] = {}
+    scores: dict[int, int] = {}
+    for candidates in by_value.values():
+        for ds in candidates:
+            cols[id(ds)] = frozenset(col.concept.address for col in ds.columns)
+            scores[id(ds)] = _datasource_score(ds)
 
-        # A union requires at least 2 distinct sources; a single source is not a union
-        if len(combo_list) < 2:
+    # Members MAY disagree on intrinsic (~) partiality of a shared column
+    # (a "mixed-family" combo, e.g. web_sales + catalog/store_returns).
+    # Such a union is a legitimate per-channel provider of columns it binds
+    # complete (q05: a web return's return-site lives on web_sales), and
+    # union partial propagation keeps its ~-partial keys from ever outranking
+    # a pure family that binds them complete (q14) — so don't reject it here.
+    # An empty overlap beyond the merge key IS rejected, at the first step it
+    # appears: intersection only shrinks, so no continuation can revive it.
+    # Ties reproduce the old product enumeration exactly: per signature the
+    # winner is the first max-scoring combo in product order (`combo_key` =
+    # candidate index tuple, lexicographic = product order), and signatures
+    # order by their first-achieving combo (`min_key`, tracked over ALL combos
+    # reaching a signature, not just the best-scoring one).
+    merge_key_addr = merge_key.address
+    # signature -> (score, combo, combo_key, min_key)
+    states: dict[
+        frozenset[str],
+        tuple[int, list[BuildDatasource], tuple[int, ...], tuple[int, ...]],
+    ] = {}
+    for idx, ds in enumerate(by_value[values[0]]):
+        sig = cols[id(ds)] - {merge_key_addr}
+        if not sig:
             continue
+        existing = states.get(sig)
+        if existing is None:
+            states[sig] = (scores[id(ds)], [ds], (idx,), (idx,))
+        elif scores[id(ds)] > existing[0]:
+            states[sig] = (scores[id(ds)], [ds], (idx,), existing[3])
 
-        # Require at least one shared concept beyond the merge key
-        overlap = {col.concept.address for col in combo_list[0].columns}
-        for ds in combo_list[1:]:
-            overlap &= {col.concept.address for col in ds.columns}
-        signature = frozenset(overlap - merge_key_addr)
-        if not signature:
-            continue
-        # Members MAY disagree on intrinsic (~) partiality of a shared column
-        # (a "mixed-family" combo, e.g. web_sales + catalog/store_returns).
-        # Such a union is a legitimate per-channel provider of columns it binds
-        # complete (q05: a web return's return-site lives on web_sales), and
-        # union partial propagation keeps its ~-partial keys from ever outranking
-        # a pure family that binds them complete (q14) — so don't reject it here.
-
-        score = sum(_datasource_score(ds) for ds in combo_list)
-        existing = best_per_overlap.get(signature)
-        if existing is None or score > existing[1]:
-            best_per_overlap[signature] = (combo_list, score)
+    for v in values[1:]:
+        next_states: dict[
+            frozenset[str],
+            tuple[int, list[BuildDatasource], tuple[int, ...], tuple[int, ...]],
+        ] = {}
+        for sig, (score, combo, combo_key, min_key) in states.items():
+            for idx, ds in enumerate(by_value[v]):
+                new_sig = sig & cols[id(ds)]
+                if not new_sig:
+                    continue
+                new_score = score + scores[id(ds)]
+                new_key = combo_key + (idx,)
+                existing = next_states.get(new_sig)
+                if existing is None:
+                    next_states[new_sig] = (
+                        new_score,
+                        combo + [ds],
+                        new_key,
+                        min_key + (idx,),
+                    )
+                    continue
+                new_min_key = min(existing[3], min_key + (idx,))
+                if new_score > existing[0] or (
+                    new_score == existing[0] and new_key < existing[2]
+                ):
+                    next_states[new_sig] = (
+                        new_score,
+                        combo + [ds],
+                        new_key,
+                        new_min_key,
+                    )
+                else:
+                    next_states[new_sig] = (
+                        existing[0],
+                        existing[1],
+                        existing[2],
+                        new_min_key,
+                    )
+        states = next_states
+        if not states:
+            return None
 
+    best_per_overlap: dict[frozenset[str], tuple[list[BuildDatasource], int]] = {
+        sig: (state[1], state[0])
+        for sig, state in sorted(states.items(), key=lambda kv: kv[1][3])
+    }
     if not best_per_overlap:
         return None
     # Keep only maximal overlap signatures: drop a signature whose concept set

{pytrilogy-0.3.330 → pytrilogy-0.3.332}/trilogy/core/processing/utility.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from collections.abc import Sequence
 from dataclasses import dataclass
 from enum import Enum
 from typing import TYPE_CHECKING
@@ -8,6 +9,7 @@ if TYPE_CHECKING:
     from trilogy.core import graph as nx
 
 from trilogy.core.enums import Derivation, Granularity, JoinType, Purpose
+from trilogy.core.models.author import ConceptRef
 from trilogy.core.models.build import (
     BuildConcept,
     BuildDatasource,
@@ -210,6 +212,44 @@ def find_nullable_concepts(
     return sorted(final_nullable)
 
 
+def _resolve_output_target(
+    cte: CTE | UnionCTE,
+    target: BuildConcept | ConceptRef,
+    scoped_merge_map: dict[str, str],
+) -> tuple[BuildConcept, bool] | None:
+    """The CTE column that renders `target`, and whether it must be re-aliased
+    to the written name. None when the CTE cannot render it at all."""
+    mapping = {x.address: x for x in cte.output_columns}
+    if target.address in mapping:
+        return mapping[target.address], False
+    for oc in cte.output_columns:
+        if target.address in oc.pseudonyms:
+            return oc, True
+    # an in-query JOIN collapses a source onto its canonical target; the
+    # target's column is present, render it under the written source name
+    canonical = scoped_merge_map.get(target.address)
+    if canonical is not None and canonical in mapping:
+        return mapping[canonical], True
+    return None
+
+
+def unrenderable_outputs(
+    cte: CTE | UnionCTE,
+    targets: Sequence[BuildConcept | ConceptRef],
+    scoped_merge_map: dict[str, str] | None = None,
+) -> list[str]:
+    """Target addresses `sort_select_output` would silently omit from the
+    projection. A caller that cannot tolerate a short SELECT — a persist writes
+    positionally, so a missing column shifts every later one — asks here and
+    fails naming the column instead."""
+    merge_map = scoped_merge_map or {}
+    return [
+        target.address
+        for target in targets
+        if _resolve_output_target(cte, target, merge_map) is None
+    ]
+
+
 def sort_select_output_processed(
     cte: CTE | UnionCTE, query: SelectStatement | MultiSelectStatement | ProcessedQuery
 ) -> CTE | UnionCTE:
@@ -221,7 +261,6 @@ def sort_select_output_processed(
         hidden = query.hidden_components
 
     output_addresses = {c.address for c in targets}
-    mapping = {x.address: x for x in cte.output_columns}
     scoped_merge_map: dict[str, str] = (
         query.scoped_merge_map if isinstance(query, ProcessedQuery) else {}
     )
@@ -243,19 +282,14 @@ def sort_select_output_processed(
 
     new_output: list[BuildConcept] = []
     for x in targets:
-        if x.address in mapping:
-            new_output.append(mapping[x.address])
+        # A target the CTE cannot render is dropped here; `unrenderable_outputs`
+        # is the same resolution, asked ahead of time by callers that must fail
+        # rather than emit a short projection.
+        resolved = _resolve_output_target(cte, x, scoped_merge_map)
+        if resolved is None:
             continue
-        for oc in cte.output_columns:
-            if x.address in oc.pseudonyms:
-                new_output.append(render_as(x, oc))
-                break
-        else:
-            # an in-query JOIN collapses a source onto its canonical target; the
-            # target's column is present, render it under the written source name
-            canonical = scoped_merge_map.get(x.address)
-            if canonical is not None and canonical in mapping:
-                new_output.append(render_as(x, mapping[canonical]))
+        source, rename = resolved
+        new_output.append(render_as(x, source) if rename else source)
 
     for oc in cte.output_columns:
         # add hidden back

{pytrilogy-0.3.330 → pytrilogy-0.3.332}/trilogy/core/processing/v4_helper/strategy_builder.py

@@ -2019,7 +2019,19 @@ def _satisfy_parent_projection_contract(
     if not any(_contains_shape_barrier(parent) for parent in parents):
         return parents
 
+    # Two different questions, two different sets. "What is available TO this
+    # parent" is its own parents' outputs (`parent_output_addresses`) — that is
+    # what `parent_needed` asks, and a leaf scan's empty answer is what keeps
+    # leaf scans out of this projection entirely (q66). "What does a SIBLING
+    # supply to the merge" is that sibling's own projection: a WINDOW sibling
+    # READS `event_time` and emits only `lag(event_time)`, so crediting it with
+    # its inputs made this projection strip `event_time` off the dimension
+    # parent — and the consumer computing `event_time - prior_event_time` then
+    # had no source for it and was dropped from the plan outright.
     outputs_by_parent = [parent_output_addresses(parent) for parent in parents]
+    own_outputs_by_parent = [
+        {output.address for output in parent.usable_outputs} for parent in parents
+    ]
     projected: list[StrategyNode] = []
     for idx, parent in enumerate(parents):
         if _contains_shape_barrier(parent):
@@ -2027,7 +2039,7 @@ def _satisfy_parent_projection_contract(
             continue
         parent_needed = outputs_by_parent[idx] & needed
         other_outputs = set().union(
-            *(outputs for j, outputs in enumerate(outputs_by_parent) if j != idx)
+            *(outputs for j, outputs in enumerate(own_outputs_by_parent) if j != idx)
         )
         fd_candidates = {
             addr
@@ -2049,6 +2061,28 @@ def _satisfy_parent_projection_contract(
         if not concepts or non_fd_needed:
             projected.append(parent)
             continue
+        # `parent_needed` is measured off the parent's INPUTS, so a value the
+        # parent computes itself — a BASIC sibling's `revenue`, a date
+        # projection — is invisible to it and the projection below would strip
+        # it, leaving the consuming aggregate with no source for that output
+        # (sibling aggregates where one reads a derived row value: the plan
+        # computed `revenue` in a CTE and then never projected `total_revenue`).
+        # Carry those through. Restricted to FD-at-grain so the projection's
+        # row count is unchanged, and to what no sibling parent already
+        # supplies, so this never re-shapes a plain dimension re-join.
+        carry = {
+            output.address
+            for output in parent.usable_outputs
+            if output.address in needed
+            and output.address not in parent_needed
+            and output.address not in other_outputs
+            and _fd_at_grain(output, projection_grain_components)
+        }
+        concepts.extend(
+            c
+            for addr in sorted(carry)
+            if (c := _concept_at(environment, addr)) is not None
+        )
         # When the dimension's projected grain (fd_needed) shares NO key with the
         # barrier sibling, the post-projection merge has nothing to join on and
         # cross-joins ON 1=1 — the bridge between the two is a projection-grain key
@@ -2079,7 +2113,7 @@ def _satisfy_parent_projection_contract(
             parent_outputs = {o.address for o in parent.usable_outputs}
             grain_concepts = [
                 c
-                for addr in sorted((fd_needed | join_keys) & parent_outputs)
+                for addr in sorted((fd_needed | join_keys | carry) & parent_outputs)
                 if (c := _concept_at(environment, addr)) is not None
             ]
             projected.append(

pytrilogy-0.3.332/trilogy/core/processing/v4_node_generators/basic.py

@@ -0,0 +1,104 @@
+from trilogy.core.models.build import BuildConcept, BuildGrain, BuildWhereClause
+from trilogy.core.models.build_environment import BuildEnvironment
+from trilogy.core.processing.nodes import (
+    ConstantNode,
+    MergeNode,
+    SelectNode,
+    StrategyNode,
+)
+from trilogy.core.processing.v4_helper.functional_dependency import (
+    build_fd_determines,
+)
+
+from .common import outputs_with_parent_grain_keys, parent_outputs_needed
+
+
+def _grain_claim_needs_group(
+    outputs: list[BuildConcept],
+    parent: StrategyNode,
+    environment: BuildEnvironment,
+) -> bool:
+    """True when this projection can only make its grain claim true by grouping.
+
+    A projection's grain is inferred from what it projects, but a plain SELECT
+    emits one row per PARENT row. Project `species` and `upper(species)` off a
+    `tree_id`-grain union and the node advertises grain `species` while emitting
+    one row per tree; a consumer then joins it on `species` and fans the other
+    side out — silently, since the SQL is valid.
+
+    Two things can make the claim true, and only the second is ours to do here:
+
+    - The claim is already true. Either the parent's grain keys survive in the
+      projection, or the projected grain functionally determines them (the
+      parent holds at most one row per it, so dropping them loses nothing).
+    - The projection can be widened back to the parent's grain. That is
+      ``outputs_with_parent_grain_keys``'s job, and it needs the parent to
+      actually expose those keys.
+
+    When the parent does NOT expose its own grain there is nothing to widen with,
+    and grouping is the only repair left. The outputs are functionally determined
+    by the grouped set, so the dedupe cannot drop a row — it removes exactly the
+    duplicates the parent's finer grain introduced.
+
+    Returns SelectNode's own default (False) when it does not fire -- passing
+    None instead would flip every other basic projection onto ``_resolve``'s
+    grain-less branch, which is a plan change for nodes this has no business
+    touching.
+    """
+    parent_grain = set(parent.resolve().grain.components)
+    if not parent_grain or parent_grain.issubset({c.address for c in outputs}):
+        return False
+    if parent_grain.issubset({c.address for c in parent.output_concepts}):
+        return False
+    claimed = BuildGrain.from_concepts(outputs).components
+    return not all(
+        component in claimed
+        or build_fd_determines(
+            environment, claimed, component, include_empty_grain=False
+        )
+        for component in parent_grain
+    )
+
+
+def gen_basic(
+    outputs: list[BuildConcept],
+    parents: list[StrategyNode],
+    environment: BuildEnvironment,
+    conditions: BuildWhereClause | None = None,
+    preexisting_conditions: BuildWhereClause | None = None,
+) -> StrategyNode | None:
+    """Projection of derived basic expressions over already-built parents.
+
+    Zero parents → ConstantNode. One parent → SelectNode (just projection).
+    Multiple parents → MergeNode, which auto-joins on shared output concepts
+    (typically the common grain). A SelectNode here would render with no
+    join and emit `INVALID_REFERENCE_BUG_<...>` for the unjoined parent."""
+    pre = preexisting_conditions.conditional if preexisting_conditions else None
+    if not parents:
+        return ConstantNode(
+            input_concepts=[],
+            output_concepts=outputs,
+            environment=environment,
+            conditions=conditions.conditional if conditions else None,
+            preexisting_conditions=pre,
+        )
+    full_outputs = outputs_with_parent_grain_keys(outputs, parents)
+    inputs = parent_outputs_needed(full_outputs, parents, conditions)
+    if len(parents) == 1:
+        return SelectNode(
+            input_concepts=inputs,
+            output_concepts=full_outputs,
+            environment=environment,
+            parents=parents,
+            conditions=conditions.conditional if conditions else None,
+            preexisting_conditions=pre,
+            force_group=_grain_claim_needs_group(full_outputs, parents[0], environment),
+        )
+    return MergeNode(
+        input_concepts=inputs,
+        output_concepts=full_outputs,
+        environment=environment,
+        parents=parents,
+        conditions=conditions.conditional if conditions else None,
+        preexisting_conditions=pre,
+    )