pytrilogy 0.3.332__tar.gz → 0.3.334__tar.gz

{pytrilogy-0.3.332 → pytrilogy-0.3.334}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrilogy
-Version: 0.3.332
+Version: 0.3.334
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.9

{pytrilogy-0.3.332 → pytrilogy-0.3.334}/trilogy/__init__.py

@@ -7,7 +7,7 @@ if TYPE_CHECKING:
     from trilogy.executor import Executor
     from trilogy.parser import parse
 
-__version__ = "0.3.332"
+__version__ = "0.3.334"
 
 __all__ = [
     "CONFIG",

{pytrilogy-0.3.332 → pytrilogy-0.3.334}/trilogy/ai/constants.py

@@ -26,24 +26,20 @@ parameter NAME TYPE [default <literal>]; — a runtime value supplied via `--par
 
 ### Query-scoped join
 
-A typical fact already has its dimensions merged in — no join needed. When blending fact models or rowset outputs, a join DECLARES how the key domains relate (allowing a path, not defining a join):
+A typical fact already has its dimensions merged in, so no join is needed. Blending fact models or rowset outputs takes a scoped join, placed right after the select list, which DECLARES how the key domains relate: `subset join a = b` (a's values are contained in b's, a ⊆ b; b authoritative for the key) or `union join a = b` (neither contains the other; the key is the coalesce of both sides and unmatched rows from BOTH sides are kept).
 
-- `subset join a = b` — a's values are contained in b's (a ⊆ b); b is authoritative for the key.
-- `union join a = b` — neither domain contains the other; the key is the coalesce of both sides and unmatched rows from BOTH sides are kept.
+Two rules decide whether the numbers are right, so apply them before reaching for the example:
 
-A union key-group must be entirely union (`union join a = b = c` chains one group); subset joins mix freely. Keys may be expressions (`union join a.id + 53 = b.id`), aggregates, or windows; only `=` is supported. Joins never drop nulls (NULL keys match null-safely) and are ALWAYS effectively full — no join EVER implicitly drops a row; there is NO inner join. For an intersection, add an explicit `where <optional-side attr> is not null`.
+- Joins NEVER drop a row and NULL keys match null-safely; there is NO inner join. An intersection is an explicit `where <optional-side attr> is not null`.
+- JOIN ON THE FULL GRAIN: `explore` prints each fact's grain as @<k1, k2>; write one clause PER key. Matching only one key of a multi-key grain causes duplication.
 
-JOIN ON THE FULL GRAIN. `explore` prints each fact's grain as @<k1, k2>; a composite grain needs one join clause PER key (BOTH `union join a.order_number = b.order_number` AND `union join a.item.id = b.item.id`). Matching only one key of a multi-key grain causes duplication.
-
-Joins go right after the select list. Full example: trilogy agent-info syntax example scoped-join.
+Chaining, expression keys, union/subset mixing, multi-key and self-pair shapes: trilogy agent-info syntax example scoped-join.
 
 merge <a> into ~<b>; is the persistent (whole-file) equivalent of `subset join a = b`; plain `merge a into b;` declares EXACT domain equivalence. Standalone statements; prefer a scoped join unless the connection is universal.
 
 ### union / except / intersect (row set operations)
 
-union((armA), (armB), ...) -> (out1, out2, ...) row-stacks self-contained select arms positionally (SQL UNION ALL) into one named result, usable in a rowset (`with combined as union(...) -> (...)`). Each arm is an ordinary select and DEDUPLICATES to its own output grain before stacking — so aggregate INSIDE each arm (preferred), or carry each arm's grain key through as an extra output column; stacking raw measure rows and aggregating outside silently undersums. Example: trilogy agent-info syntax example union-stack-channels.
-
-except((armA), (armB), ...) -> (...) and intersect(...) share union's arm shape but are SQL SET operators: output rows are DISTINCT, whole rows compare null-safely, and except subtracts later arms from the FIRST (arm order matters). Prefer except over multi-column `not in` for "in A but never in B". Example: trilogy agent-info syntax example except-intersect-setops.
+`union(...)` row-stacks self-contained select arms positionally (SQL UNION ALL) into one named rowset (`with combined as union((armA), (armB), ...) -> (out1, out2, ...)`); `except(...)` / `intersect(...)` share that arm shape but are SQL SET operators (DISTINCT rows, null-safe whole-row compare, and except subtracts later arms from the FIRST, so arm order matters). Prefer except over a multi-column `not in` for "in A but never in B". Each arm dedups to its own output grain BEFORE stacking, so aggregate INSIDE each arm or the total silently undersums. Examples: trilogy agent-info syntax example union-stack-channels / except-intersect-setops.
 
 ## Query structure
 

{pytrilogy-0.3.332 → pytrilogy-0.3.334}/trilogy/ai/syntax_examples.py

@@ -94,12 +94,7 @@ order by index;
     SyntaxExample(
         name="query-structure",
         title="Full query structure - every clause and where it goes (+ rowsets)",
-        summary=(
-            "the clause order of a query (`where` -> `select` <cols> -> join(s) -> "
-            "`having` -> `order by` -> `limit`) and how to define a rowset (a NAME "
-            "then a select); joins sit right after the select list - filter a "
-            "joined or aggregated RESULT in `having`, input rows in `where`"
-        ),
+        summary="the exact clause order of a query, and how to define a rowset",
         body="""\
 # Every Trilogy query has the SAME fixed skeleton. Top-level statements come
 # first (each ends with `;`), then ONE query whose clauses appear in this exact
@@ -169,9 +164,8 @@ limit 100;
         name="filtered-aggregate",
         title="Filtered aggregate - aggregate only the rows matching a condition",
         summary=(
-            "`sum(x ? cond)` / `count(x ? cond)` aggregate just the matching rows; "
-            "to COUNT ROWS count the unique grain/row key, or `count(grain(k1, k2))` "
-            "when several keys identify it; `by <grain>` pins the aggregate's grain"
+            "aggregating only the rows matching a condition, and counting rows "
+            "correctly"
         ),
         body="""\
 # `?` is a local filter - a filter that applies only to the prior expression-
@@ -207,11 +201,7 @@ order by iris.species asc;
             "Row-level filter (WHERE) vs aggregate filter (HAVING) - don't wrap a "
             "per-row threshold in max()/min()"
         ),
-        summary=(
-            "a plain per-ROW condition (`x.col > N`) goes in WHERE (filters the rows "
-            "feeding every aggregate); HAVING filters an aggregate RESULT. "
-            "`max(col) by g > N` in HAVING filters GROUPS, not rows - different meaning"
-        ),
+        summary="choosing between WHERE and HAVING for a threshold condition",
         body="""\
 # A non-aggregate, per-ROW condition belongs in WHERE - it filters the rows that
 # feed every aggregate, BEFORE grouping. HAVING is ONLY for filtering on an
@@ -396,12 +386,7 @@ order by enroll.year asc nulls first;
     SyntaxExample(
         name="union-stack-channels",
         title="Stack rows from several sources/channels with the union(...) TVF",
-        summary=(
-            "`with combined as union((armA), (armB), (armC), ...) -> (o1, o2, ...)` "
-            "row-STACKS self-contained inline selects positionally (SQL UNION ALL) - "
-            "the PREFERRED way to combine channels/sources; pass TWO OR MORE arms, one "
-            "per source; reference outputs as `combined.o1`"
-        ),
+        summary="stacking several sources/channels together as rows",
         body="""\
 # `union(...)` is a relational table-valued function: it STACKS the rows of TWO OR
 # MORE self-contained `select` arms, positionally (like SQL `UNION ALL`), into one
@@ -476,14 +461,7 @@ limit 100;
     SyntaxExample(
         name="except-intersect-setops",
         title="Set difference / co-occurrence across sources with except(...) / intersect(...)",
-        summary=(
-            "`with only_a as except((armA), (armB)) -> (o1, o2)` keeps the distinct "
-            "rows of the FIRST arm that appear in NO later arm (SQL EXCEPT); "
-            "`intersect(...)` keeps rows present in EVERY arm. PREFERRED over "
-            "multi-column `not in` for 'in A but not B' questions when you also "
-            "want the distinct combinations themselves: whole ROWS are compared "
-            "(any column mix) and the output is deduplicated"
-        ),
+        summary="set difference or co-occurrence across sources",
         body="""\
 # `except(...)` and `intersect(...)` are relational table-valued functions with the
 # SAME shape as `union(...)`: TWO OR MORE inline self-contained select arms, matched
@@ -529,12 +507,7 @@ limit 100;
     SyntaxExample(
         name="scoped-join",
         title="Blend two models in one query with a scoped subset/union join",
-        summary=(
-            "`select <cols> subset|union join a.key = b.key (= other.key ...)?` (right after "
-            "the select list) blends a second model into ONE query - the PREFERRED "
-            "alternative to `merge`. JOIN ON THE FULL GRAIN: one clause per key in "
-            "the shared `@<k1, k2>` grain"
-        ),
+        summary="blending a second fact model or rowset into one query",
         body="""\
 # A query-scoped `join` blends a second model into ONE select without editing the
 # model files (the query-local equivalent of `merge`). A join DECLARES how the
@@ -643,11 +616,7 @@ limit 100;
     SyntaxExample(
         name="existence-anti-join",
         title="Existence / anti-join across models",
-        summary=(
-            "keep or exclude rows by whether a key matches another model: "
-            "`key in other.key` (semi-join) / `not in` (anti-join); a MULTI-part "
-            "key uses a row-wise tuple `(a, b) in (set.a, set.b)`"
-        ),
+        summary=("keeping or excluding rows by whether a key matches another model"),
         body="""\
 # `in` / `not in` against another model's key is the semi-join / anti-join
 # idiom - no manual JOIN. Use it instead of `count(...) is null` hacks.
@@ -757,12 +726,7 @@ order by unique_sc.course asc;
     SyntaxExample(
         name="rollup",
         title="Subtotals + grand total in one pass with `by rollup (…)`",
-        summary=(
-            "the select-level `by rollup (d1, d2)` clause computes EVERY aggregate "
-            "in the select at every level (leaf, per-d1 subtotal, grand total) in one "
-            "pass; `grouping(d)` = 1 on a rolled-up row - use it to LABEL "
-            "subtotal/total rows and to sort by level"
-        ),
+        summary="subtotals and a grand total in one pass, and labeling the levels",
         body="""\
 # A `by rollup (d1, d2)` clause AFTER the select list (before having/order by)
 # computes the whole select at MULTIPLE grain levels in one pass - the grouping
@@ -858,10 +822,7 @@ limit 100;
     SyntaxExample(
         name="staged-membership",
         title="Stage a coarse-grain membership set, then filter a fine-grain query",
-        summary=(
-            "compute a membership set in a `rowset` (keys meeting a count/HAVING), "
-            "then filter the main query with `<key> in <rowset>.<col>`"
-        ),
+        summary=("filtering a fine-grain query by a set computed at a coarser grain"),
         body="""\
 # When a query filters on a condition computed at a COARSER grain (e.g. "items
 # sold in all 3 channels", "courses offered in more than one year"), compute that

{pytrilogy-0.3.332 → pytrilogy-0.3.334}/trilogy/core/models/build.py

@@ -3607,6 +3607,12 @@ class Factory:
             self._build_over_items(list(grouping_set))
             for grouping_set in grouping_sets_source
         ]
+        if grouping != AggregateGroupingMode.STANDARD and not by and not grouping_sets:
+            # A rollup/cube over zero keys has exactly one grouping set - the
+            # grand total - which is what a standard grouping with no keys
+            # already renders. `by rollup ()` on a grainless select (only
+            # constants and aggregates projected) lands here.
+            grouping = AggregateGroupingMode.STANDARD
         if grouping == AggregateGroupingMode.STANDARD:
             by = sorted(by, key=lambda x: x.address)
 

{pytrilogy-0.3.332 → pytrilogy-0.3.334}/trilogy/core/models/core.py

@@ -652,11 +652,10 @@ def constant_domain_violation(
     comparison provably never match; None when satisfiable or undecidable.
 
     Only provably-FALSE predicates are flagged. In-domain predicates — even
-    ones matching every declared value — must parse untouched: they can carry
-    load-bearing join null-rejection through nullable FK paths
-    (evals/tpcds_agent/bug_q16_enum_tautology_drops_joined_null_rejection.md),
-    and a narrower authored intent than the current domain snapshot is
-    legitimate. Error messages must never advise removing a predicate."""
+    ones matching every declared value — must parse untouched: through a
+    nullable FK they carry load-bearing join null-rejection, and a narrower
+    authored intent than the current domain snapshot is legitimate. Error
+    messages must never advise removing a predicate."""
     while isinstance(expected, TraitDataType):
         expected = expected.type
     if isinstance(value, Enum):

{pytrilogy-0.3.332 → pytrilogy-0.3.334}/trilogy/core/models/execute.py

@@ -1196,11 +1196,17 @@ class QueryDatasource:
 
     @property
     def effective_grain(self) -> BuildGrain:
+        # A NULL-padded key is absence, not a row identity: an anti-join
+        # (`orders.id is null`) declares the preserved side's grain but still
+        # projects the rejected key, and counting that key here re-inflates the
+        # grain to the pre-join side and forces a regroup that dedups nothing.
+        nullable = {concept.address for concept in self.nullable_concepts}
         key_outputs = {
             concept.address
             for concept in self.output_concepts
             if concept.purpose == Purpose.KEY
             and concept.name != RECURSIVE_GATING_CONCEPT
+            and concept.address not in nullable
         }
         return self.grain + BuildGrain(components=key_outputs)
 
@@ -1422,12 +1428,22 @@ class QueryDatasource:
             limited = f"_limited_{self.limit}"
             if self.ordering:
                 limited += f"_{string_to_hash(str(self.ordering))}"
+        # An UNNEST in the FROM is identity for the mirror-image reason a limit
+        # is: the unnested source is a row MULTIPLE of the same shape without
+        # it, so the two must never collide and merge (``__add__`` rejects the
+        # mismatch rather than silently folding an unnest into a plain scan).
+        unnested = ""
+        if self.join_derived_concepts:
+            unnested = "_unnest_" + "_".join(
+                sorted(c.address.replace(".", "_") for c in self.join_derived_concepts)
+            )
         return (
             "_join_".join(sorted(d.identifier for d in self.datasources))
             + group
             + (f"_at_{grain}" if grain else "_at_abstract")
             + (f"_filtered_by_{filters}" if filters else "")
             + limited
+            + unnested
         )
 
     def get_alias(

{pytrilogy-0.3.332 → pytrilogy-0.3.334}/trilogy/core/processing/join_resolution.py

@@ -525,16 +525,77 @@ def side_nullable(concept: BuildConcept, side: DataSource | None) -> bool:
     return bool(args & nullable_addrs)
 
 
+def _side_outputs(concept: BuildConcept, side: DataSource) -> bool:
+    equivalent = concept.equivalent_addresses
+    return any(equivalent & c.equivalent_addresses for c in side.output_concepts)
+
+
+def nulls_are_values(
+    concept: BuildConcept,
+    side: DataSource,
+    _seen: frozenset[tuple[str, int]] = frozenset(),
+) -> bool:
+    """Whether the NULLs this side carries for ``concept`` are VALUES (a `?`
+    column, a nullable derivation, a nullable input to a null-propagating
+    expression, a ROLLUP grouping key) rather than pure outer-join extension.
+
+    Outer-join extension means *absent*: there is no row on that side, so no
+    key. Pairing that against a real NULL group cross-joins the two (q30: 311
+    padded address rows x 20 unknown-customer groups = 6,220)."""
+    if concept.is_nullable:
+        return True
+    # Argument chains can be mutually recursive (q25); a repeat visit of the
+    # same concept on the same source contributes nothing new.
+    visit = (concept.address, id(side))
+    if visit in _seen:
+        return False
+    seen = _seen | {visit}
+    equivalent = concept.equivalent_addresses
+    if isinstance(side, BuildDatasource):
+        # Column-level `?` is the only value-NULL source on a physical table.
+        return any(
+            equivalent & nc.equivalent_addresses for nc in side.nullable_concepts
+        )
+    # A grouping-set NULL is padding too, but a twin-rollup partner pads the
+    # same key, so it stays pairable here; get_join_type handles the mismatch.
+    if equivalent & rollup_padded_addresses(side):
+        return True
+    carriers = [p for p in side.datasources if _side_outputs(concept, p)]
+    if not carriers:
+        # Nothing upstream to attribute the NULL to; stay conservative rather
+        # than call an unexplained nullability extension.
+        return True
+    if any(nulls_are_values(concept, p, seen) for p in carriers):
+        return True
+    if not propagates_argument_nulls(concept):
+        return False
+    args = {a.address for a in concept.concept_arguments}
+    if not args:
+        return False
+    return any(
+        (args & nc.equivalent_addresses) and nulls_are_values(nc, side, seen)
+        for nc in side.nullable_concepts
+    )
+
+
 def get_modifiers(
     left_concept: BuildConcept,
     right_concept: BuildConcept,
     left: DataSource | None,
     right: DataSource | None,
 ) -> list[Modifier]:
-    """Use null-safe equality only when both exposed join keys can be NULL."""
-    if side_nullable(left_concept, left) and side_nullable(right_concept, right):
-        return [Modifier.NULLABLE]
-    return []
+    """Use null-safe equality only when both exposed join keys can be NULL.
+
+    Asymmetric padding is the exception: when one side's NULLs are outer-join
+    extension (absence) and the other's are values, they name nothing in
+    common and null-safe equality cross-joins them. Both sides extended is the
+    ordinary case again: the padding shares provenance, so those rows pair."""
+    if not (side_nullable(left_concept, left) and side_nullable(right_concept, right)):
+        return []
+    assert left is not None and right is not None
+    if nulls_are_values(left_concept, left) != nulls_are_values(right_concept, right):
+        return []
+    return [Modifier.NULLABLE]
 
 
 def _collect_deep_partial_addresses(

{pytrilogy-0.3.332 → pytrilogy-0.3.334}/trilogy/core/processing/nodes/base_node.py

@@ -177,6 +177,10 @@ def get_all_parent_nullable(
 
 class StrategyNode:
     source_type = SourceType.ABSTRACT
+    # A node that only projects or filters emits its parents' rows. Subclasses
+    # that set this take their parents' grain when none was passed, rather than
+    # deriving one from their own outputs; see `_default_grain`.
+    inherits_parent_grain: bool = False
 
     def __init__(
         self,
@@ -539,14 +543,37 @@ class StrategyNode:
                     source_map[concept.address] = {ps}
                     break
 
+    def _default_grain(
+        self, parent_sources: list[QueryDatasource | BuildDatasource]
+    ) -> BuildGrain:
+        """The grain to declare when the planner passed none.
+
+        Deriving it from ``output_concepts`` states the grain the projection
+        *selects*, which is only the row grain if something deduped to it. For a
+        row-preserving node nothing did, so it reports its parents' rows: a
+        narrowing projection that claims the narrow grain reads as already
+        deduped, and consumers skip the GROUP BY that would make it true.
+        """
+        if self.inherits_parent_grain and parent_sources and not self.force_group:
+            # An existence feeder is read through a subselect: it rejects rows,
+            # it never supplies them, so its grain is not part of ours.
+            existence = {concept.address for concept in self.existence_concepts}
+            inherited = BuildGrain()
+            for source in parent_sources:
+                supplied = {concept.address for concept in source.output_concepts}
+                if existence and supplied <= existence:
+                    continue
+                inherited += source.grain
+            if inherited.components:
+                return inherited
+        return BuildGrain.from_concepts(self.output_concepts)
+
     def _resolve(self) -> QueryDatasource:
         parent_sources: list[QueryDatasource | BuildDatasource] = [
             p.resolve() for p in self.parents
         ]
 
-        grain = (
-            self.grain if self.grain else BuildGrain.from_concepts(self.output_concepts)
-        )
+        grain = self.grain if self.grain else self._default_grain(parent_sources)
         source_map = resolve_concept_map(
             parent_sources,
             targets=self.output_concepts,

{pytrilogy-0.3.332 → pytrilogy-0.3.334}/trilogy/core/processing/nodes/filter_node.py

@@ -20,6 +20,8 @@ class FilterNode(StrategyNode):
 
     source_type = SourceType.FILTER
 
+    inherits_parent_grain = True
+
     def __init__(
         self,
         input_concepts: list[BuildConcept],

{pytrilogy-0.3.332 → pytrilogy-0.3.334}/trilogy/core/processing/nodes/select_node_v2.py

@@ -28,6 +28,8 @@ class SelectNode(StrategyNode):
 
     source_type = SourceType.SELECT
 
+    inherits_parent_grain = True
+
     def __init__(
         self,
         input_concepts: list[BuildConcept],

{pytrilogy-0.3.332 → pytrilogy-0.3.334}/trilogy/core/processing/v4_helper/group_graph.py

@@ -1783,16 +1783,37 @@ def _consumer_required_input_grain(
     # a raw scan with it is unrenderable. The parent's grain (added below) is
     # the joinable identity.
     parent_computed: set[str] = set()
-    for pred in group_graph.predecessors(gid):
-        if pred == FINAL_NODE_ID or pred not in attrs:
-            continue
-        if edge_kind(group_edges, pred, gid) == EdgeKind.EXISTENCE:
-            continue
+    row_preds = [
+        pred
+        for pred in group_graph.predecessors(gid)
+        if pred != FINAL_NODE_ID
+        and pred in attrs
+        and edge_kind(group_edges, pred, gid) != EdgeKind.EXISTENCE
+    ]
+    for pred in row_preds:
         if (
             attrs[pred].derivation in GROUPING_DERIVATIONS
             or attrs[pred].derivation == Derivation.ROWSET
         ):
-            grain |= set(attrs[pred].grain_components)
+            # A pred whose row stream reaches this consumer through a GROUPING
+            # descendant pred that grouped the axis away has no such axis at
+            # this merge: requiring it resurrects the raw pred as a parent CTE
+            # beside the descendant, and the two share no joinable key (rollup
+            # over union-joined rowsets: the raw `r` rowset beside the rollup
+            # aggregate, whose ROLLUP grain is `channel` only). The
+            # descendant's own grain, declared below on its own iteration, is
+            # the merge axis.
+            pred_grain = set(attrs[pred].grain_components)
+            axis_grouped_away = any(
+                other != pred
+                and pred in nx.ancestors(group_graph, other)
+                and attrs[other].derivation in GROUPING_DERIVATIONS
+                and not (pred_grain <= set(attrs[other].output_concepts))
+                for other in row_preds
+            )
+            if axis_grouped_away:
+                continue
+            grain |= pred_grain
     return frozenset(grain - parent_computed)
 
 
@@ -2271,6 +2292,21 @@ def _compute_concept_sets(
     pseudonym_mates = {k: frozenset(v) for k, v in mate_accumulator.items()}
 
     io = GroupIOPlan.for_groups(group_graph)
+    # Non-ROWSET members of authored statement relations that NO group hosts:
+    # the axis vocabulary a fresh scan may advertise below. A member some group
+    # already carries as a primary needs no re-sourcing, and advertising it
+    # there only widens the scan and the merge keys with equalities the fact
+    # grain already determines.
+    scan_advertisable_members: list[str] = []
+    if environment is not None:
+        for member in sorted(_statement_scoped_relation_members(environment)):
+            member_concept = environment.concepts.get(member)
+            if (
+                member_concept is not None
+                and member_concept.derivation != Derivation.ROWSET
+                and member not in all_primary_members
+            ):
+                scan_advertisable_members.append(member)
     for gid in topo:
         if gid == FINAL_NODE_ID:
             continue
@@ -2279,6 +2315,20 @@ def _compute_concept_sets(
         if fact.behavior is None or fact.derivation == Derivation.ROOT:
             for addr in list(cap):
                 cap.update(source_grain_of.get(addr, frozenset()))
+            # A fresh scan can re-source an authored scoped-join axis member
+            # its columns FD-determine (`billing_customer.sk` determines the
+            # unique `billing_customer.id` a `subset join cust.cid = ...id`
+            # names). Advertise it so the demand rules can elect the axis for
+            # the merge; without it the address intersection fails and the
+            # FINAL merge loses the authored join key (all_sales pivot).
+            # Rowset handles are excluded: a scan that absorbs one would drop
+            # the rowset's internal filter.
+            if environment is not None:
+                for member in scan_advertisable_members:
+                    if member not in cap and build_fd_determines(
+                        environment, cap, member, include_empty_grain=False
+                    ):
+                        cap.add(member)
             io.capability[gid] = cap
             continue
         # A grouping group's grain component that is a STATEMENT-scoped join axis