mostlyright-data 0.18.2__tar.gz → 0.18.3__tar.gz

{mostlyright_data-0.18.2 → mostlyright_data-0.18.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mostlyright-data
-Version: 0.18.2
+Version: 0.18.3
 Summary: Mostly Right CLI for reproducible, reviewed Parquet datasets
 Project-URL: Homepage, https://mostlyright.md/
 Project-URL: Documentation, https://mostlyright.md/docs/guides/cli/

{mostlyright_data-0.18.2 → mostlyright_data-0.18.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "mostlyright-data"
-version = "0.18.2"
+version = "0.18.3"
 description = "Mostly Right CLI for reproducible, reviewed Parquet datasets"
 readme = "README.md"
 requires-python = ">=3.13"

{mostlyright_data-0.18.2 → mostlyright_data-0.18.3}/skills/mr-data-build/SKILL.md

@@ -1665,6 +1665,12 @@ What it costs the protocol, and what to do instead:
   revision behavior, entity and geographic coverage, cost, and refresh fit. Reject prohibited use.
 - Preserve event time, available time, acquisition time, source revision, and timezone. Do not use
   post-cutoff information in targets, labels, features, joins, or validation decisions.
+- When the question is **what changed over time**, or the table needs point-in-time features, and
+  the source answers only with the present, declare a `snapshot` window on that source: it stamps
+  every row with the UTC day it was acquired on and keeps one partition per day. Compute a
+  transition with `lag(column) over (partition by <identity> order by <snapshot column>)` and keep
+  the rows where the two are `is distinct from` each other. Never diff two sealed versions by hand;
+  a version is not a date, and nothing outside the seal can be replayed.
 
 ### Authoring a collection source
 

{mostlyright_data-0.18.2 → mostlyright_data-0.18.3}/src/mostlyright/data_harness/hosted_run_worker.py

@@ -307,10 +307,15 @@ from mostlyright.data_harness.v4_window import (
     align_down,
     applied_window,
     assert_mergeable_format,
+    assert_snapshot_has_begun,
+    assert_snapshot_slice_is_whole,
+    assert_snapshot_within_bounds,
     instant,
     merge_slice,
     partition_keys,
     resolve_physical_scope,
+    snapshot_day,
+    stamp_snapshot,
 )
 
 # ------------------------------------------------------------------------------------------------
@@ -5511,6 +5516,13 @@ def _source_history_refresh_enabled(job: RunExecution) -> bool:
     Keeping this deliberately narrow is important: a source-history predecessor is authority for
     one direct CSV projection, not a hint that an arbitrary multi-source SQL graph may reuse raw
     bytes.  The old path remains responsible for every other document.
+
+    A SNAPSHOT SOURCE IS NOT THIS SHAPE, THOUGH IT IS SINGLE AND IT IS WINDOWED.  This route
+    acquires with ``merge_windowed=False``, and :func:`_windowed_relation` -- the only place the
+    observed day is written onto a snapshot's rows and the only place its merge runs -- is
+    reachable only when that is true.  A snapshot admitted here would hand the build a relation
+    with no day on it at all, so the run either dies in the transform on a missing column or seals
+    a table that records no days, which is the whole member absent under a green run.
     """
 
     return bool(
@@ -5521,11 +5533,17 @@ def _source_history_refresh_enabled(job: RunExecution) -> bool:
         and job.predecessor.source_history is not None
         and len(job.recipe.sources) == 1
         and job.recipe.sources[0].window is not None
+        and job.recipe.sources[0].window.snapshot is None
     )
 
 
 def _source_history_initial_enabled(job: RunExecution) -> bool:
-    """Return whether a full run emits the first bounded source-history version."""
+    """Return whether a full run emits the first bounded source-history version.
+
+    A snapshot source is excluded for the reason :func:`_source_history_refresh_enabled` records,
+    and excluded HERE rather than only there: an initial run that took this route seals a
+    predecessor carrying ``source_history``, and every refresh after it takes the route again.
+    """
 
     return bool(
         (job.mode == "full" or is_initial_bootstrap_refresh(job))
@@ -5534,6 +5552,7 @@ def _source_history_initial_enabled(job: RunExecution) -> bool:
         and job.predecessor is None
         and len(job.recipe.sources) == 1
         and job.recipe.sources[0].window is not None
+        and job.recipe.sources[0].window.snapshot is None
     )
 
 
@@ -10214,6 +10233,13 @@ def _applied_windows(job: RunExecution, *, requested_at: str) -> Mapping[str, Ap
     Empty for a recipe that declares no window, which is what makes every registered document
     written before this member behave exactly as it did: no address is rendered, no merge runs and
     no coverage entry claims a range.
+
+    THIS IS WHAT BINDS THE ACQUISITION CACHE KEY, and for a snapshot that is what keeps a day out
+    of the day before's bytes. The key is ``(source_id, query_digest, window)`` and this mapping
+    supplies the third member, so two runs whose ``requested_at`` fall on different UTC days can
+    never be served each other's fetch -- a snapshot's derived range IS its day. A snapshot does
+    not take its partition from here, though: the day it stamps comes from the receipt the fetch
+    wrote, and this range is what routes the source into the merge at all.
     """
 
     moment = instant(requested_at, "the run's requested_at")
@@ -10236,6 +10262,82 @@ def _applied_windows(job: RunExecution, *, requested_at: str) -> Mapping[str, Ap
     )
 
 
+def _stamped_snapshot(
+    job: RunExecution,
+    source: V4Source,
+    *,
+    relation: RelationInput,
+    coverage: V4AcquisitionCoverage,
+    destination: Path,
+) -> tuple[AppliedWindow, RelationInput, V4AcquisitionCoverage]:
+    """Write this source's own acquisition day onto every decoded row, and say so.
+
+    ⚠ THE DAY COMES FROM THE RECEIPT AND NEVER FROM A CLOCK. ``coverage.fetched_at`` is the instant
+    this acquisition already recorded and the instant the sealed receipt states for this source, so
+    a rerun of the same sealed run stamps the day it sealed rather than the day it is re-run on.
+    Reading a clock here would make a replay a different relation from the one it is replaying.
+
+    ONE INSTANT ANSWERS EVERY QUESTION ABOUT THE DAY. The stamp on each row, the range this run
+    reports as ``coverage.window``, the receipt's ``observed_date`` and the partition the merge
+    replaces are all derived from ``coverage.fetched_at`` and from nothing else, so the four can
+    never be four renderings. The ``applied`` window this function is handed is the run's own,
+    derived before the first fetch over the clock the dispatch carried; it is what routed this
+    source into the merge and what bound its acquisition cache key, and a snapshot derives its
+    partition from the receipt rather than from it so that a replay of a sealed run answers with
+    the day it sealed.
+
+    THE CACHE CANNOT PUT A DAY IN ANOTHER DAY'S BYTES. That derived range is the third member of
+    the acquisition cache key (``inprocess_acquisition.AcquisitionCacheKey``), and for a snapshot
+    the range IS the day, so a cached read is never served across a UTC midnight and
+    ``coverage.fetched_at`` always falls on the day the run asked for.
+
+    The stamped column is appended to ``relation.columns`` because that tuple is what binds the
+    transform's source columns; a relation whose bytes carry a column its header does not name is a
+    column no statement can select.
+    """
+
+    window = source.window
+    assert window is not None and window.snapshot is not None
+    observed = instant(coverage.fetched_at, f"the acquisition receipt for {source.name}")
+    applied = applied_window(job.mode, window, clamps_window=None, now=observed)
+    # THE DECLARED FIRST DAY IS HELD OVER THE RECEIPT'S OWN WINDOW, which is the window every
+    # other fact about this day is derived from. Holding it over the run's pre-fetch derivation
+    # instead would be a second authority on one question, and two authorities on one question are
+    # two answers waiting to disagree.
+    assert_snapshot_has_begun(window, applied)
+    assert_mergeable_format(source.name, relation.data_format)
+    stamped = stamp_snapshot(
+        window, source_id=source.name, acquired=relation.content, applied=applied
+    )
+    # ONLY THE BYTE CEILING IS MEANINGFUL BEFORE THE MERGE. This slice is one day by construction
+    # and `max_partitions` is at least one, so the partition arm cannot answer here; what it can
+    # answer on is the merged relation below.
+    assert_snapshot_within_bounds(
+        window.snapshot,
+        source_id=source.name,
+        partitions=1,
+        byte_count=len(stamped.content),
+    )
+    digest = hashlib.sha256(stamped.content).hexdigest()
+    return (
+        applied,
+        replace(
+            relation,
+            path=destination,
+            content=stamped.content,
+            columns=stamped.columns,
+            row_count=stamped.rows,
+            content_sha256=digest,
+        ),
+        replace(
+            coverage,
+            snapshot_sha256=digest,
+            snapshot_bytes=len(stamped.content),
+            rows_kept=stamped.rows,
+        ),
+    )
+
+
 def _windowed_relation(
     job: RunExecution,
     source: V4Source,
@@ -10260,19 +10362,50 @@ def _windowed_relation(
     A first, full or sample run of a windowed source MERGES NOTHING. Its slice IS the relation: it
     fetched the whole range it claims to cover, and there is no earlier version to continue.
 
-    ``truncated`` says whether the sealed relation is less than the whole table. A refresh's is
-    never truncated, a clipped one included -- after the merge the relation covers everything from
-    the corpus's start, and nothing was cut. A sample that narrowed its window IS truncated, and it
-    has to be: a latest build whose coverage is untruncated on every source is read as the full
-    build whatever mode produced it, so a windowed sample claiming otherwise would present one
-    day's rows as the finished table.
+    ``truncated`` says whether the sealed relation is less than the whole table. A REQUEST window's
+    refresh is never truncated, a clipped one included -- after the merge the relation covers
+    everything from the corpus's start, and nothing was cut. A sample that narrowed its window IS
+    truncated, and it has to be: a latest build whose coverage is untruncated on every source is
+    read as the full build whatever mode produced it, so a windowed sample claiming otherwise
+    would present one day's rows as the finished table.
+
+    A SNAPSHOT'S REFRESH REFUSES A CLAMPED SLICE RATHER THAN MERGING IT, AND THAT IS THE
+    DIFFERENCE BETWEEN THE TWO GRAMMARS. A clipped request window will be re-requested: the
+    partitions it did not reach are the predecessor's and the next tick fetches them. A snapshot
+    cannot re-request anything -- the publisher answers with now and cannot be asked for a day that
+    has passed -- so a day whose rows a row clamp cut short is a hole nothing later repairs, and
+    the run refuses with ``SNAPSHOT_RELATION_TRUNCATED`` before the merge rather than sealing a day
+    it cannot complete. A full or a sample carries the clamp through instead: neither merges, so
+    neither has a predecessor's days to lose, and an incomplete first build is what a later full
+    replaces.
+
+    A SNAPSHOT IS NEVER NARROWED, AND THAT IS NOT THE SAME SENTENCE. A request window's applied
+    range is a cut of a corpus the publisher already holds, so a range that starts after
+    ``start_at`` really is less of it. A snapshot's range is an OBSERVATION: the publisher serves
+    one document saying what is true now, this run read the whole of it, and the one partition it
+    seals is everything a run of any mode could have produced today. Deriving `truncated` from the
+    start here would mark every snapshot run from the second day of the table's life onward as an
+    incomplete build -- and `truncated` is what Studio's one definition of a finished build reads
+    (`build_profile.build_is_complete`), so the button that starts a full would never say the table
+    already has one, and a complete sample would never release the full it is holding. A row clamp
+    that really did cut the fetched relation short arrives on ``relation.truncated``, and what
+    becomes of it is the mode's: carried through by a run that merges nothing, refused above by
+    one that merges.
+
+    A SNAPSHOT SOURCE IS STAMPED BEFORE ANY OF THAT, and the stamp is what makes the rest of this
+    function true of it: its rows arrive carrying no time at all, and the column the merge reads
+    its partition off is written here.
     """
 
     window = source.window
     assert window is not None
+    if window.snapshot is not None:
+        applied, relation, coverage = _stamped_snapshot(
+            job, source, relation=relation, coverage=coverage, destination=destination
+        )
     fetched_window = applied.wire
     if job.mode not in WINDOW_MERGE_MODES:
-        narrowed = applied.start_inclusive > align_down(window.start_at)
+        narrowed = window.snapshot is None and applied.start_inclusive > align_down(window.start_at)
         return (
             replace(relation, truncated=relation.truncated or narrowed),
             replace(
@@ -10281,6 +10414,8 @@ def _windowed_relation(
                 truncated=coverage.truncated or narrowed,
             ),
         )
+    if window.snapshot is not None:
+        assert_snapshot_slice_is_whole(source_id=source.name, truncated=relation.truncated)
     if predecessor is None or predecessor_sources is None:
         raise _predecessor_refusal(
             f"source {source.name} is windowed and this run holds none of the bytes its "
@@ -10306,7 +10441,21 @@ def _windowed_relation(
         acquired=relation.content,
         keys=partition_keys(window, applied),
     )
+    # BEFORE THE RELATION IS WRITTEN INTO THE BUILD'S INPUT TREE, which is before anything this
+    # run produces is uploaded. A snapshot grows by one partition a day forever, so the two
+    # ceilings its registrant declared are met on an ordinary day by an ordinary run; what that run
+    # must not do is seal a version whose successor cannot be built.
+    if window.snapshot is not None:
+        assert_snapshot_within_bounds(
+            window.snapshot,
+            source_id=source.name,
+            partitions=merged.partitions,
+            byte_count=len(merged.content),
+        )
     digest = hashlib.sha256(merged.content).hexdigest()
+    # NOTHING THAT REACHES HERE IS SHORT. A request window's merge leaves the relation covering
+    # the corpus from its own start, so a clip it recorded on the way in is no longer a shortfall
+    # of the sealed bytes; a snapshot's clipped slice never reaches the merge at all.
     return (
         replace(
             relation,
@@ -10343,6 +10492,20 @@ def _windowed_relation(
                     "predecessor_digest": committed,
                     "predecessor_rows": merged.predecessor_rows,
                     "slice_rows": merged.slice_rows,
+                    # WHICH DAY THIS RUN WROTE, AND HOW MANY THE RELATION NOW HOLDS. For a
+                    # request window both are already legible from the range; for a snapshot the
+                    # range IS the observed day, and a reader of the receipt should not have to
+                    # recover the table's depth by counting partitions in the sealed bytes. The
+                    # day is spelled by the one function that spells it, so the receipt and the
+                    # cell the merge reads its partition off can never be two renderings.
+                    **(
+                        {}
+                        if window.snapshot is None
+                        else {
+                            "observed_date": snapshot_day(applied),
+                            "partitions": merged.partitions,
+                        }
+                    ),
                 }
             ),
         ),

{mostlyright_data-0.18.2 → mostlyright_data-0.18.3}/src/mostlyright/data_harness/run_execution_v4.py

@@ -279,6 +279,17 @@ V4_FAILURE_CODES: tuple[str, ...] = (
     "RUN_RECORDING_INVALID",
     "RUN_WATERMARK_INVALID",
     "RUN_WINDOW_REQUIRED",
+    "SNAPSHOT_BEFORE_START_AT",
+    "SNAPSHOT_COLUMN_RESERVED",
+    "SNAPSHOT_IDENTITY_INCOMPLETE",
+    "SNAPSHOT_PARTITIONS_EXHAUSTED",
+    "SNAPSHOT_PARTITION_MISMATCH",
+    "SNAPSHOT_RELATION_EMPTY",
+    "SNAPSHOT_RELATION_TOO_LARGE",
+    "SNAPSHOT_RELATION_TRUNCATED",
+    "SNAPSHOT_REQUEST_EXCLUSIVE",
+    "SNAPSHOT_SCOPES_UNSUPPORTED",
+    "SNAPSHOT_SPAN_FIXED",
     "SOURCE_HTTP_5XX",
     "SOURCE_PHYSICAL_SCOPE_UNKNOWN",
     "SOURCE_PROXY_BUSY",
@@ -2585,6 +2596,10 @@ def _applied_execution_window(job: RunExecution, *, now: str) -> RunWindow | Non
     A refresh of a recipe with a windowed source and no ``clamps.window`` is refused here. An
     absent range on a continuing run is not a neutral default: it would re-request the whole corpus
     from the publisher, which is the cost this member exists to remove.
+
+    A SNAPSHOT SOURCE IS NOT ONE OF THOSE. It re-requests nothing: its address is fetched literally
+    and its range is the day the fetch happened in, so a dispatch that states no range costs a
+    snapshot-only recipe nothing and there is no resumption point for one to continue from.
     """
 
     from mostlyright.data_harness.v4_window import applied_window, instant, intersect
@@ -2592,7 +2607,8 @@ def _applied_execution_window(job: RunExecution, *, now: str) -> RunWindow | Non
     windows = [source.window for source in job.recipe.sources if source.window is not None]
     if not windows:
         return None
-    if job.mode == "refresh" and job.clamps.window is None:
+    requested = [window for window in windows if window.snapshot is None]
+    if job.mode == "refresh" and requested and job.clamps.window is None:
         raise _refuse(
             "RUN_WINDOW_REQUIRED",
             "a refresh of a recipe with a windowed source must state the clamps.window it "

{mostlyright_data-0.18.2 → mostlyright_data-0.18.3}/src/mostlyright/data_harness/studio_boundary.py

@@ -85,7 +85,7 @@ PINNED_OPENAPI_SOURCE_SHA256 = "2dfdef306229bf211333211354a948742e9759dd5399f66b
 #: One line, and it stays one line: ``_replace_exactly`` in the repin matches the whole assignment
 #: as a literal and refuses on any count but one, so a wrapped constant is a constant the repin can
 #: no longer rewrite.
-PINNED_V4_OPENAPI_SOURCE_SHA256 = "450d913fcf4af526fb9e7d9d415771364ee4c1916ae78fc1c534a46ad0138fab"
+PINNED_V4_OPENAPI_SOURCE_SHA256 = "244d4953c4d650da1ef448cb6846582c0d139b3e16b2798d743ec02f129e55ea"
 
 _DIGEST = re.compile(r"^sha256:[0-9a-f]{64}$")
 _IMAGE_DIGEST = re.compile(r"^[a-z0-9][a-z0-9._/-]{0,254}@sha256:[0-9a-f]{64}$")

{mostlyright_data-0.18.2 → mostlyright_data-0.18.3}/src/mostlyright/data_harness/thin/v4_tables.py

@@ -942,6 +942,8 @@ def cadence_state(record: Mapping[str, Any]) -> dict[str, Any] | None:
     applied = applied if isinstance(applied, Mapping) else {}
     basis = applied.get("basis")
     basis = basis if isinstance(basis, Mapping) else {}
+    floor = cadence.get("floor")
+    floor = floor if isinstance(floor, Mapping) else {}
     return {
         # ⚠ THE WIRE OBJECT TRAVELS WHOLE, BESIDE THE PROSE RATHER THAN INSTEAD OF IT. The members
         # below are a rendering for somebody reading a terminal, and a rendering necessarily drops
@@ -959,6 +961,10 @@ def cadence_state(record: Mapping[str, Any]) -> dict[str, Any] | None:
         "updates_seen": basis.get("evidence_count"),
         "measured_gap": _interval_phrase(basis.get("interval_seconds")),
         "quickest_source": basis.get("fastest_source"),
+        # A source with a snapshot window records the day it was read, so the loop never lets the
+        # table refresh slower than that day; the bound is shown as the gap it never exceeds.
+        "no_slower_than": _interval_phrase(floor.get("max_interval_seconds")),
+        "bounded_by_source": floor.get("source"),
     }
 
 

{mostlyright_data-0.18.2 → mostlyright_data-0.18.3}/src/mostlyright/data_harness/ux/remediation.py

@@ -5,7 +5,7 @@ for the commonest failure of all -- pointing a command at a folder that is not t
 engineering wording reads like a security incident. This module turns a code into the two or three
 plain sentences that say what happened and what to do next.
 
-There are two layers, because 1859 typed codes across 217 prefixes can reach a person at a
+There are two layers, because 1861 typed codes across 217 prefixes can reach a person at a
 workbench, and a hand-written sentence for each one would be neither affordable nor honest:
 
 * **Families.** Every code prefix is assigned to a family. A family is one paragraph: what this