mostlyright-data 0.30.1__tar.gz → 0.30.3__tar.gz

{mostlyright_data-0.30.1 → mostlyright_data-0.30.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mostlyright-data
-Version: 0.30.1
+Version: 0.30.3
 Summary: Mostly Right hosted CLI for reviewed datasets
 Project-URL: Homepage, https://mostlyright.md/
 Project-URL: Documentation, https://mostlyright.md/docs/guides/cli/

{mostlyright_data-0.30.1 → mostlyright_data-0.30.3}/skills/mr-data-build/SKILL.md

@@ -152,6 +152,9 @@ step 6. The stage 3 reference holds its rule. Gate 4 is step 6.
    quality checks.
 7. Follow the returned run with `mr-data watch RUN_ID --json`. Its `source_rates` and
    `mr-data status RUN_ID --json` on a live run give each collection source's rate and time left.
+   On a queued or running run, `status` also gives `progress`: the current attempt, how many of
+   its sources it fetched of how many, the source it is on with its pages, and how the attempt
+   before it ended. A handed-back run's new attempt counts from zero.
    Quote those numbers rather than doing the arithmetic. Read `run.status`: returning an existing
    run does not mean new work was queued. Resume the known run after an interruption, using the
    saved cursor where available. Read [run lifecycle](references/live-run.md) for holds and

{mostlyright_data-0.30.1 → mostlyright_data-0.30.3}/src/mostlyright/data_harness/page_coverage.py

@@ -20,6 +20,7 @@ __all__ = [
     "PAGE_BUDGET_KINDS",
     "PAGE_COVERAGE_COUNT_FIELDS",
     "PAGE_COVERAGE_FIELDS",
+    "SOURCE_ROWS_INCOMPLETE",
     "page_coverage_sentence",
 ]
 
@@ -42,6 +43,9 @@ __all__ = [
 #: predecessor rows, but current collection refresh is refused before acquisition, except a
 #: per-row request collection's, which refreshes with the direct request window it inherits and
 #: asks again for a failed row whenever a refresh covers that row's partition.
+#: A ``source_rows`` collection also states it on a full run, as ``SOURCE_ROWS_INCOMPLETE``, when
+#: the upstream relation it lists from is incomplete: that is not a listing anybody failed to read
+#: but a list that may be short by what the upstream source is missing, and the sentence says so.
 PAGE_COVERAGE_FIELDS: tuple[str, ...] = (
     "discovered",
     "discovery_requests",
@@ -84,6 +88,46 @@ PAGE_COVERAGE_COUNT_FIELDS: tuple[str, ...] = (
 )
 
 
+#: The discovery failure a ``source_rows`` collection states when the upstream relation it lists
+#: from is itself incomplete (failed, pending or unlisted members upstream). Nothing failed to be
+#: LISTED: the list is built from rows another source did not finish delivering, so it may be
+#: short by exactly what that source is missing. Said as what it is, never as an unreadable
+#: listing, because a reader told "the listing could not be read" goes looking for an outage.
+SOURCE_ROWS_INCOMPLETE = "SOURCE_ROWS_INCOMPLETE"
+
+
+def _count(block: Mapping[str, Any], member: str) -> int | None:
+    value = block.get(member)
+    if isinstance(value, bool) or not isinstance(value, int) or value < 0:
+        return None
+    return value
+
+
+def _pages_held(block: Mapping[str, Any]) -> int | None:
+    """How many of the collection's members hold an answer, or ``None`` when the block cannot say.
+
+    MEMBERS, NOT CALLS. ``fetched_this_run`` counts the requests this run made: a per-row request
+    collection asks once for every member that renders the same request, a batch answers up to a
+    hundred members in one call, and a page kept from an earlier capture is not fetched again. So
+    ``fetched_this_run`` over ``discovered`` divides calls by members, and a run that answered
+    every member it listed reads as half done (food run 23161446: "459,156 of 1,005,223" for a
+    run with 1,004,768 members answered and 455 failed). The members that hold an answer are the
+    ones the ledger knows that are neither still owed, nor failed, nor skipped -- the same count
+    the dataset page's sources card prints as fetched.
+
+    Read with ``get``: a record written before these members existed carries only the four the
+    sentence always needed, and says what it always said.
+    """
+
+    known = _count(block, "known")
+    pending = _count(block, "pending")
+    if known is None or pending is None:
+        return None
+    failed = _count(block, "failed") or 0
+    skipped = _count(block, "skipped") or 0
+    return max(known - pending - failed - skipped, 0)
+
+
 def page_coverage_sentence(block: Mapping[str, Any]) -> str:
     """The one sentence a person reads off a page collection's coverage.
 
@@ -96,13 +140,29 @@ def page_coverage_sentence(block: Mapping[str, Any]) -> str:
     failed = int(block["failed"])
     if block["complete"]:
         return f"all {discovered:,} pages fetched"
-    parts = [f"{fetched:,} of {discovered:,} pages fetched"]
+    held = _pages_held(block)
+    if held is None:
+        parts = [f"{fetched:,} of {discovered:,} pages fetched"]
+    else:
+        # A ledger can know pages its latest listing no longer names (a refresh keeps them), so
+        # the whole is the larger of the two and the sentence never says "more of fewer".
+        whole = max(discovered, _count(block, "known") or 0)
+        lead = f"{held:,} of {whole:,} pages fetched"
+        if fetched != held:
+            lead += f" ({fetched:,} requests this run)"
+        parts = [lead]
     # READ WITH ``get``, AND ONLY THIS MEMBER. The block a run produces always carries it, and the
     # block Studio hands back may not: it is optional on the run record, so a record written before
     # the member existed is still one sentence rather than a KeyError this lane swallows whole.
     listing = block.get("discovery_failure")
     if isinstance(listing, Mapping):
-        parts.append(f"the page listing could not be read ({listing['code']})")
+        if listing.get("code") == SOURCE_ROWS_INCOMPLETE:
+            parts.append(
+                "a page list built from another source's rows may be short, because that source "
+                f"is incomplete ({SOURCE_ROWS_INCOMPLETE})"
+            )
+        else:
+            parts.append(f"the page listing could not be read ({listing['code']})")
     if failed:
         parts.append(f"{failed:,} failed")
     # The block does not say which kind of collection it counts, so the remedy names both. A

{mostlyright_data-0.30.1 → mostlyright_data-0.30.3}/src/mostlyright/data_harness/thin/progress_rate.py

@@ -6,10 +6,10 @@ Each tick also carries the time it happened. Two ticks of one source give a rate
 count gives what is left. Nothing here is a new Studio fact: it is the arithmetic an agent used to
 do by hand while polling.
 
-``pages_processed`` counts saved answers replayed as well as requests sent, so a run that starts
-by replaying runs fast at first. The live rate is taken over a trailing window of ticks, so it
-settles on the sending rate once the replays are done. The time remaining is an estimate at that
-rate and says so.
+``pages_processed`` combines saved answers replayed and requests sent. Its rate is therefore
+source-member throughput, never publisher throughput or whole-run completion. Live status rates
+require reaching the exact snapshot event, matching its attempt, and a fresh source observation.
+Reaching that cursor does not establish complete source inventory or complete event history.
 """
 
 from __future__ import annotations
@@ -19,7 +19,7 @@ import time
 from collections import deque
 from collections.abc import Callable, Mapping
 from dataclasses import dataclass, field
-from datetime import datetime
+from datetime import UTC, datetime
 from typing import Any
 
 from mostlyright.data_harness.thin.stream import RunEventStream
@@ -32,6 +32,13 @@ TRAILING_WINDOW_SECONDS = 600.0
 #: How often ``watch`` prints a rate line for one source, in seconds of event time.
 RATE_LINE_EVERY_SECONDS = 60.0
 
+#: A rate is a live estimate only while its own latest observation is this recent.
+LIVE_FRESHNESS_SECONDS = 30.0
+
+
+def utc_now() -> datetime:
+    return datetime.now(UTC)
+
 
 def parse_instant(value: Any) -> datetime | None:
     """An RFC 3339 instant as Studio stamps it, or ``None`` when it is absent or unreadable."""
@@ -74,16 +81,16 @@ class SourceRate:
     planned: int | None = None
     discovery_complete: bool = False
     printed_at: datetime | None = None
+    unavailable_reason: str | None = None
 
-    def observe(self, at: datetime, done: int) -> None:
-        if self.ticks and done < self.ticks[-1][1]:
-            # The count went back, so this is a different acquisition. Start again.
-            self.ticks.clear()
-            self.first = None
+    def observe(self, at: datetime, done: int) -> bool:
         if self.ticks and at <= self.ticks[-1][0]:
-            if done > self.ticks[-1][1]:
-                self.ticks[-1] = (self.ticks[-1][0], done)
-            return
+            return False
+        if self.ticks and done < self.ticks[-1][1]:
+            # There is no batch identity on these counters. A decrease cannot establish one.
+            self.unavailable_reason = "counter_regressed_without_scope_change"
+            self.planned = None
+            self.discovery_complete = False
         self.ticks.append((at, done))
         if self.first is None:
             self.first = (at, done)
@@ -92,6 +99,7 @@ class SourceRate:
             and (at - self.ticks[1][0]).total_seconds() >= TRAILING_WINDOW_SECONDS
         ):
             self.ticks.popleft()
+        return True
 
     @property
     def done(self) -> int | None:
@@ -100,7 +108,7 @@ class SourceRate:
     def per_second(self) -> float | None:
         """The trailing rate, or ``None`` before two ticks with time between them."""
 
-        if len(self.ticks) < 2:
+        if self.unavailable_reason is not None or len(self.ticks) < 2:
             return None
         (start, start_done), (end, end_done) = self.ticks[0], self.ticks[-1]
         elapsed = (end - start).total_seconds()
@@ -109,7 +117,7 @@ class SourceRate:
     def average_per_second(self) -> float | None:
         """The rate from the first tick to the last, or ``None`` before two ticks."""
 
-        if self.first is None or not self.ticks:
+        if self.unavailable_reason is not None or self.first is None or not self.ticks:
             return None
         elapsed = (self.ticks[-1][0] - self.first[0]).total_seconds()
         return (self.ticks[-1][1] - self.first[1]) / elapsed if elapsed > 0 else None
@@ -127,6 +135,7 @@ class SourceRate:
             or done is None
             or rate is None
             or rate <= 0
+            or done > self.planned
         ):
             return None
         return max(0, self.planned - done) / rate
@@ -171,21 +180,46 @@ class RateTracker:
 
     def __init__(self) -> None:
         self.sources: dict[str, SourceRate] = {}
+        self.attempt_id: str | None = None
+        self.retired_attempts: set[str] = set()
+        self.last_sequence = 0
+        self.live_observation: dict[str, Any] | None = None
 
     def observe(self, body: Mapping[str, Any]) -> SourceRate | None:
         """Fold one run event record, and answer the source it moved, or ``None``."""
 
+        sequence = _count(body.get("sequence"))
+        if sequence is None or sequence <= self.last_sequence:
+            return None
+        self.last_sequence = sequence
         event_type = body.get("event_type")
-        if isinstance(event_type, str) and not event_type.startswith("run_progress_"):
+        if not isinstance(event_type, str) or not event_type.startswith("run_progress_"):
             return None
+        attempt = body.get("attempt_id")
+        if isinstance(attempt, str) and attempt:
+            if attempt in self.retired_attempts:
+                return None
+            if attempt != self.attempt_id:
+                if self.attempt_id is not None:
+                    self.retired_attempts.add(self.attempt_id)
+                self.sources.clear()
+                self.attempt_id = attempt
         facts = tick_facts(body)
         at = parse_instant(body.get("occurred_at"))
         done = _count(facts.get("pages_processed"))
         name = facts.get("source_id") or facts.get("source_name")
-        if at is None or done is None or not isinstance(name, str) or not name:
+        if (
+            at is None
+            or done is None
+            or not isinstance(name, str)
+            or not name
+            or not isinstance(attempt, str)
+            or not attempt
+        ):
             return None
         source = self.sources.setdefault(name, SourceRate(name))
-        source.observe(at, done)
+        if not source.observe(at, done) or source.unavailable_reason is not None:
+            return None
         planned = _count(facts.get("pages_planned"))
         if planned is not None:
             source.planned = planned
@@ -193,12 +227,17 @@ class RateTracker:
             source.discovery_complete = facts["discovery_complete"]
         return source
 
-    def due_line(self, source: SourceRate) -> str | None:
+    def due_line(
+        self, source: SourceRate, *, wall_clock: Callable[[], datetime] = utc_now
+    ) -> str | None:
         """The source's line when one is due, at most once a minute of event time per source."""
 
         if not source.ticks:
             return None
         now = source.ticks[-1][0]
+        age = (wall_clock() - now).total_seconds()
+        if age < 0 or age > LIVE_FRESHNESS_SECONDS:
+            return None
         if (
             source.printed_at is not None
             and (now - source.printed_at).total_seconds() < RATE_LINE_EVERY_SECONDS
@@ -265,7 +304,8 @@ def live_rates(
     """The rates of a live run, from the last ``tail`` records of its event log.
 
     Reads the progress snapshot for the newest sequence, then replays the log from ``tail``
-    records before it and stops at that sequence. Answers ``None`` when the run has reported no
+    records before it and stops at or beyond that sequence. Only observing the exact target
+    certifies completion of this bounded replay. Answers ``None`` when the run has reported no
     progress yet. A refusal on the way is raised, and the caller decides what it costs.
     """
 
@@ -274,14 +314,51 @@ def live_rates(
     if type(sequence) is not int or sequence < 1:
         return None
     tracker = RateTracker()
-    RunEventStream(
+    target_seen = False
+    target_attempt: str | None = None
+    observed_at: str | None = None
+    from_sequence = max(0, sequence - tail)
+    expected_sequence = from_sequence + 1
+    sequence_gap = False
+
+    def observe(body: Mapping[str, Any], _frame_name: str) -> bool:
+        nonlocal target_seen, target_attempt, observed_at, expected_sequence, sequence_gap
+        observed = _count(body.get("sequence"))
+        if observed is None:
+            return False
+        if observed != expected_sequence:
+            sequence_gap = True
+        expected_sequence = observed + 1
+        instant = parse_instant(body.get("occurred_at"))
+        observed_at = instant.isoformat() if instant is not None else None
+        if observed <= sequence:
+            tracker.observe(body)
+        if observed == sequence:
+            target_seen = True
+            attempt = body.get("attempt_id")
+            target_attempt = attempt if isinstance(attempt, str) and attempt else None
+        return observed >= sequence
+
+    followed = RunEventStream(
         client.session, run_id, transport=client.transport, sleep=sleep, path=RUN_EVENTS_PATH
     ).follow(
-        from_seq=max(0, sequence - tail),
-        observe=follow_into(tracker, until_sequence=sequence),
+        from_seq=from_sequence,
+        observe=observe,
         deadline=clock() + deadline_seconds,
         clock=clock,
     )
+    tracker.live_observation = {
+        "scope": "bounded_event_tail",
+        "from_sequence_exclusive": from_sequence,
+        "target_sequence": sequence,
+        "observed_sequence": followed.last_sequence,
+        "replay_complete": target_seen and not sequence_gap,
+        "sequence_gap": sequence_gap,
+        "observed_at": observed_at,
+        "attempt_id": target_attempt,
+        "deadline_expired": followed.expired,
+        "end_reason": followed.end_reason,
+    }
     return tracker
 
 
@@ -314,19 +391,58 @@ def history_rates(
     return tracker
 
 
-def live_rate_members(client: Any, run_id: str, **kwargs: Any) -> dict[str, Any]:
+def live_rate_members(
+    client: Any, run_id: str, *, wall_clock: Callable[[], datetime] = utc_now, **kwargs: Any
+) -> dict[str, Any]:
     """The ``status`` receipt members for a live run's rates, or why there are none."""
 
     try:
         tracker = live_rates(client, run_id, **kwargs)
     except ThinLaneError as refusal:
         return {"source_rates_unavailable": f"{refusal.code}: {refusal.detail}"}
-    if tracker is None or not tracker.receipts():
+    if tracker is None:
         return {"source_rates_unavailable": "no collection source has reported two ticks yet"}
-    return {
-        "source_rates": tracker.receipts(),
-        "rates": {f"source {index}": line for index, line in enumerate(tracker.lines(), 1)},
-    }
+    observation = dict(tracker.live_observation or {})
+    now = wall_clock()
+    observation["checked_at"] = now.isoformat()
+    result: dict[str, Any] = {"source_rate_observation": observation}
+    if not observation.get("replay_complete"):
+        result["source_rates_unavailable"] = (
+            "incomplete replay: event gap or snapshot event missing"
+        )
+        return result
+    if not observation.get("attempt_id") or observation["attempt_id"] != tracker.attempt_id:
+        result["source_rates_unavailable"] = "snapshot attempt could not be established"
+        return result
+    receipts, lines, unavailable = [], {}, {}
+    for source in tracker.sources.values():
+        at = source.ticks[-1][0] if source.ticks else None
+        age = (now - at).total_seconds() if at is not None else None
+        reason = source.unavailable_reason
+        if reason is None and (age is None or age < 0):
+            reason = "observation_time_unavailable_or_in_future"
+        if reason is None and age is not None and age > LIVE_FRESHNESS_SECONDS:
+            reason = "stale_observation"
+        if reason is None and source.per_second() is None:
+            reason = "insufficient_observations"
+        if reason is not None:
+            unavailable[source.name] = {
+                "reason": reason,
+                "observed_at": at.isoformat() if at else None,
+                "age_seconds": age,
+            }
+            continue
+        receipt = source.to_receipt()
+        receipt.update(observed_at=at.isoformat(), age_seconds=age, attempt_id=tracker.attempt_id)
+        receipts.append(receipt)
+        lines[f"source {len(lines) + 1}"] = source.line()
+    if unavailable:
+        result["source_rates_unavailable_by_source"] = unavailable
+    if receipts:
+        result.update(source_rates=receipts, rates=lines)
+    else:
+        result["source_rates_unavailable"] = "no fresh, same-attempt source rate is available"
+    return result
 
 
 __all__ = [

{mostlyright_data-0.30.1 → mostlyright_data-0.30.3}/src/mostlyright/data_harness/thin/v4_runs.py

@@ -2355,6 +2355,8 @@ def status(args: argparse.Namespace, *, client: StudioV4RunClient | None = None)
         held = {member: run[member] for member in _PROJECTION_MEMBERS[1:] if member in run}
         if held:
             payload["projection"] = held
+    if state in ATTEMPTING_RUN_STATUSES:
+        payload.update(attempt_progress_members(selected, run_id))
     if getattr(args, "rate", False) and state not in TERMINAL_RUN_STATUSES:
         # Imported here: progress_rate reads the event stream, whose module imports this one.
         from mostlyright.data_harness.thin import progress_rate
@@ -2378,6 +2380,112 @@ def status(args: argparse.Namespace, *, client: StudioV4RunClient | None = None)
     return payload
 
 
+#: The states a run holds while it has, or is between, producer attempts. A run held at a spend
+#: or sample gate has never been claimed, so there is no attempt to read for it.
+ATTEMPTING_RUN_STATUSES: frozenset[str] = frozenset({"queued", "running"})
+
+#: How each way an attempt can end reads in the progress sentence. The keys are
+#: ``run-attempts.schema.json``'s closed ``end`` vocabulary.
+_ATTEMPT_ENDS: dict[str, str] = {
+    "running": "is still running",
+    "completed": "completed",
+    "requeued_by_worker": "was handed back by its own worker",
+    "handed_back": "was handed back after its worker went silent",
+    "failed": "failed",
+    "lost": "went silent",
+}
+
+
+def attempt_progress_members(client: StudioV4RunClient, run_id: str) -> dict[str, Any]:
+    """The ``status`` members for a live run's current attempt and its sources, or why not.
+
+    Read off the attempt ledger (``GET /v4/runs/{run_id}/attempts``), which Studio folds from
+    each progress tick as it arrives and never from the run log. So, unlike the rate, this does
+    not depend on replaying the log in time, and an attempt that has run for hours still says
+    how many of its sources it fetched. Counts are the current attempt's own: a run handed back
+    walks its sources again from the first.
+
+    A Studio before 4.37.0 serves no per-source member, and then the attempt and how the one
+    before it ended are still stated. A refused read is reported, never fatal: the run record
+    has already been read.
+    """
+
+    try:
+        answer = client.run_attempts(run_id)
+    except ThinLaneError as refusal:
+        return {"progress_unavailable": f"{refusal.code}: {refusal.detail}"}
+    rows = answer.get("attempts") if isinstance(answer, Mapping) else None
+    rows = [
+        row for row in rows or () if isinstance(row, Mapping) and type(row.get("generation")) is int
+    ]
+    if not rows:
+        return {"progress_unavailable": "Studio lists no attempt of this run yet"}
+    rows.sort(key=lambda row: row["generation"])
+    current = rows[-1]
+    sources = current.get("sources") if isinstance(current.get("sources"), Mapping) else None
+    named = sources.get("current") if sources is not None else None
+    progress: dict[str, Any] = {
+        "attempt": current["generation"],
+        "attempts": len(rows),
+        "source_count": sources.get("source_count") if sources is not None else None,
+        "sources_fetched": sources.get("fetched") if sources is not None else None,
+        "sources_refused": sources.get("refused") if sources is not None else None,
+        "counted_from_first_tick": (
+            sources.get("counted_from_first_tick") if sources is not None else None
+        ),
+        "current_source": dict(named) if isinstance(named, Mapping) else None,
+        "previous_attempt": None,
+    }
+    if len(rows) > 1:
+        previous = rows[-2]
+        loss = previous.get("loss") if isinstance(previous.get("loss"), Mapping) else {}
+        progress["previous_attempt"] = {
+            "attempt": previous["generation"],
+            "end": previous.get("end"),
+            "ended_at": previous.get("ended_at"),
+            "cause": loss.get("cause"),
+            "cause_inferred": loss.get("inferred"),
+        }
+    return {"progress": progress, "progress_sentence": attempt_progress_sentence(progress)}
+
+
+def attempt_progress_sentence(progress: Mapping[str, Any]) -> str:
+    """``Attempt 4 of 4 has fetched 27 of 86 sources and is on ...``, from the members above."""
+
+    said = f"Attempt {progress['attempt']} of {progress['attempts']}"
+    fetched, count = progress.get("sources_fetched"), progress.get("source_count")
+    if fetched is None:
+        said += "; this Studio reports no per-source progress for it."
+    else:
+        said += f" has fetched {fetched:,}"
+        said += f" of {count:,} sources" if count is not None else " sources"
+        if progress.get("counted_from_first_tick") is False:
+            said += " (at least: counting began after the attempt started)"
+        named = progress.get("current_source")
+        if isinstance(named, Mapping):
+            said += f" and is on {named.get('source_name')}"
+            if named.get("source_index") is not None and count is not None:
+                said += f", source {named['source_index']:,} of {count:,}"
+            done, planned = named.get("pages_processed"), named.get("pages_planned")
+            unit = "members" if named.get("pages_unit") == "member" else "pages"
+            if done is not None and planned is not None and named.get("discovery_complete"):
+                said += f": {done:,} of {planned:,} {unit}"
+            elif done is not None:
+                said += f": {done:,} {unit} so far"
+        said += "."
+    previous = progress.get("previous_attempt")
+    if isinstance(previous, Mapping):
+        end = _ATTEMPT_ENDS.get(str(previous.get("end")), f"ended {previous.get('end')}")
+        said += f" Attempt {previous['attempt']} {end}"
+        if previous.get("cause") is not None:
+            said += (
+                f"; {'inferred ' if previous.get('cause_inferred') else ''}cause: "
+                f"{previous['cause']}"
+            )
+        said += "."
+    return said
+
+
 def status_exit_code(payload: Mapping[str, Any]) -> int:
     """A failed run exits non-zero, so a script gating on a build cannot read it as a success."""
 

{mostlyright_data-0.30.1 → mostlyright_data-0.30.3}/src/mostlyright/data_harness/thin/v4_stream.py

@@ -40,10 +40,11 @@ import argparse
 import json
 import time
 from collections.abc import Callable, Mapping
+from datetime import datetime
 from typing import Any
 
 from mostlyright.data_harness.thin import THIN_SCHEMA_PREFIX
-from mostlyright.data_harness.thin.progress_rate import RateTracker
+from mostlyright.data_harness.thin.progress_rate import RateTracker, utc_now
 from mostlyright.data_harness.thin.session import (
     StudioSession,
     StudioSessionProvider,
@@ -216,6 +217,7 @@ def watch(
     *,
     client: StudioV4RunClient | None = None,
     clock: Callable[[], float] = time.monotonic,
+    wall_clock: Callable[[], datetime] = utc_now,
     sessions: StudioSessionProvider | None = None,
 ) -> dict[str, Any]:
     """``mr-data watch RUN``: the live timeline, and the record the run stopped on.
@@ -259,7 +261,7 @@ def watch(
             print(render_v4_frame(body, frame_name), flush=True)
         moved = rates.observe(body)
         if moved is not None and announce:
-            line = rates.due_line(moved)
+            line = rates.due_line(moved, wall_clock=wall_clock)
             if line is not None:
                 print(f"       rate  {line}", flush=True)
         if body.get("event_type") in V4_TERMINAL_EVENT_TYPES:
@@ -334,6 +336,12 @@ def terminal_payload(
     if measured:
         # The last rate each collection source reached, from its own progress ticks.
         payload["source_rates"] = measured
+        payload["source_rates_basis"] = "historical_last_observation_per_source"
+        payload["source_rate_observed_at"] = {
+            name: source.ticks[-1][0].isoformat()
+            for name, source in rates.sources.items()
+            if source.per_second() is not None
+        }
     if event_type == "run_failed":
         triple = failure_from(event_facts(terminal), subject="a run_failed event")
         payload["failure"] = triple