mostlyright-data 0.30.1__tar.gz → 0.30.2__tar.gz

{mostlyright_data-0.30.1 → mostlyright_data-0.30.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mostlyright-data
-Version: 0.30.1
+Version: 0.30.2
 Summary: Mostly Right hosted CLI for reviewed datasets
 Project-URL: Homepage, https://mostlyright.md/
 Project-URL: Documentation, https://mostlyright.md/docs/guides/cli/

{mostlyright_data-0.30.1 → mostlyright_data-0.30.2}/src/mostlyright/data_harness/thin/progress_rate.py

@@ -6,10 +6,10 @@ Each tick also carries the time it happened. Two ticks of one source give a rate
 count gives what is left. Nothing here is a new Studio fact: it is the arithmetic an agent used to
 do by hand while polling.
 
-``pages_processed`` counts saved answers replayed as well as requests sent, so a run that starts
-by replaying runs fast at first. The live rate is taken over a trailing window of ticks, so it
-settles on the sending rate once the replays are done. The time remaining is an estimate at that
-rate and says so.
+``pages_processed`` combines saved answers replayed and requests sent. Its rate is therefore
+source-member throughput, never publisher throughput or whole-run completion. Live status rates
+require reaching the exact snapshot event, matching its attempt, and a fresh source observation.
+Reaching that cursor does not establish complete source inventory or complete event history.
 """
 
 from __future__ import annotations
@@ -19,7 +19,7 @@ import time
 from collections import deque
 from collections.abc import Callable, Mapping
 from dataclasses import dataclass, field
-from datetime import datetime
+from datetime import UTC, datetime
 from typing import Any
 
 from mostlyright.data_harness.thin.stream import RunEventStream
@@ -32,6 +32,13 @@ TRAILING_WINDOW_SECONDS = 600.0
 #: How often ``watch`` prints a rate line for one source, in seconds of event time.
 RATE_LINE_EVERY_SECONDS = 60.0
 
+#: A rate is a live estimate only while its own latest observation is this recent.
+LIVE_FRESHNESS_SECONDS = 30.0
+
+
+def utc_now() -> datetime:
+    return datetime.now(UTC)
+
 
 def parse_instant(value: Any) -> datetime | None:
     """An RFC 3339 instant as Studio stamps it, or ``None`` when it is absent or unreadable."""
@@ -74,16 +81,16 @@ class SourceRate:
     planned: int | None = None
     discovery_complete: bool = False
     printed_at: datetime | None = None
+    unavailable_reason: str | None = None
 
-    def observe(self, at: datetime, done: int) -> None:
-        if self.ticks and done < self.ticks[-1][1]:
-            # The count went back, so this is a different acquisition. Start again.
-            self.ticks.clear()
-            self.first = None
+    def observe(self, at: datetime, done: int) -> bool:
         if self.ticks and at <= self.ticks[-1][0]:
-            if done > self.ticks[-1][1]:
-                self.ticks[-1] = (self.ticks[-1][0], done)
-            return
+            return False
+        if self.ticks and done < self.ticks[-1][1]:
+            # There is no batch identity on these counters. A decrease cannot establish one.
+            self.unavailable_reason = "counter_regressed_without_scope_change"
+            self.planned = None
+            self.discovery_complete = False
         self.ticks.append((at, done))
         if self.first is None:
             self.first = (at, done)
@@ -92,6 +99,7 @@ class SourceRate:
             and (at - self.ticks[1][0]).total_seconds() >= TRAILING_WINDOW_SECONDS
         ):
             self.ticks.popleft()
+        return True
 
     @property
     def done(self) -> int | None:
@@ -100,7 +108,7 @@ class SourceRate:
     def per_second(self) -> float | None:
         """The trailing rate, or ``None`` before two ticks with time between them."""
 
-        if len(self.ticks) < 2:
+        if self.unavailable_reason is not None or len(self.ticks) < 2:
             return None
         (start, start_done), (end, end_done) = self.ticks[0], self.ticks[-1]
         elapsed = (end - start).total_seconds()
@@ -109,7 +117,7 @@ class SourceRate:
     def average_per_second(self) -> float | None:
         """The rate from the first tick to the last, or ``None`` before two ticks."""
 
-        if self.first is None or not self.ticks:
+        if self.unavailable_reason is not None or self.first is None or not self.ticks:
             return None
         elapsed = (self.ticks[-1][0] - self.first[0]).total_seconds()
         return (self.ticks[-1][1] - self.first[1]) / elapsed if elapsed > 0 else None
@@ -127,6 +135,7 @@ class SourceRate:
             or done is None
             or rate is None
             or rate <= 0
+            or done > self.planned
         ):
             return None
         return max(0, self.planned - done) / rate
@@ -171,21 +180,46 @@ class RateTracker:
 
     def __init__(self) -> None:
         self.sources: dict[str, SourceRate] = {}
+        self.attempt_id: str | None = None
+        self.retired_attempts: set[str] = set()
+        self.last_sequence = 0
+        self.live_observation: dict[str, Any] | None = None
 
     def observe(self, body: Mapping[str, Any]) -> SourceRate | None:
         """Fold one run event record, and answer the source it moved, or ``None``."""
 
+        sequence = _count(body.get("sequence"))
+        if sequence is None or sequence <= self.last_sequence:
+            return None
+        self.last_sequence = sequence
         event_type = body.get("event_type")
-        if isinstance(event_type, str) and not event_type.startswith("run_progress_"):
+        if not isinstance(event_type, str) or not event_type.startswith("run_progress_"):
             return None
+        attempt = body.get("attempt_id")
+        if isinstance(attempt, str) and attempt:
+            if attempt in self.retired_attempts:
+                return None
+            if attempt != self.attempt_id:
+                if self.attempt_id is not None:
+                    self.retired_attempts.add(self.attempt_id)
+                self.sources.clear()
+                self.attempt_id = attempt
         facts = tick_facts(body)
         at = parse_instant(body.get("occurred_at"))
         done = _count(facts.get("pages_processed"))
         name = facts.get("source_id") or facts.get("source_name")
-        if at is None or done is None or not isinstance(name, str) or not name:
+        if (
+            at is None
+            or done is None
+            or not isinstance(name, str)
+            or not name
+            or not isinstance(attempt, str)
+            or not attempt
+        ):
             return None
         source = self.sources.setdefault(name, SourceRate(name))
-        source.observe(at, done)
+        if not source.observe(at, done) or source.unavailable_reason is not None:
+            return None
         planned = _count(facts.get("pages_planned"))
         if planned is not None:
             source.planned = planned
@@ -193,12 +227,17 @@ class RateTracker:
             source.discovery_complete = facts["discovery_complete"]
         return source
 
-    def due_line(self, source: SourceRate) -> str | None:
+    def due_line(
+        self, source: SourceRate, *, wall_clock: Callable[[], datetime] = utc_now
+    ) -> str | None:
         """The source's line when one is due, at most once a minute of event time per source."""
 
         if not source.ticks:
             return None
         now = source.ticks[-1][0]
+        age = (wall_clock() - now).total_seconds()
+        if age < 0 or age > LIVE_FRESHNESS_SECONDS:
+            return None
         if (
             source.printed_at is not None
             and (now - source.printed_at).total_seconds() < RATE_LINE_EVERY_SECONDS
@@ -265,7 +304,8 @@ def live_rates(
     """The rates of a live run, from the last ``tail`` records of its event log.
 
     Reads the progress snapshot for the newest sequence, then replays the log from ``tail``
-    records before it and stops at that sequence. Answers ``None`` when the run has reported no
+    records before it and stops at or beyond that sequence. Only observing the exact target
+    certifies completion of this bounded replay. Answers ``None`` when the run has reported no
     progress yet. A refusal on the way is raised, and the caller decides what it costs.
     """
 
@@ -274,14 +314,51 @@ def live_rates(
     if type(sequence) is not int or sequence < 1:
         return None
     tracker = RateTracker()
-    RunEventStream(
+    target_seen = False
+    target_attempt: str | None = None
+    observed_at: str | None = None
+    from_sequence = max(0, sequence - tail)
+    expected_sequence = from_sequence + 1
+    sequence_gap = False
+
+    def observe(body: Mapping[str, Any], _frame_name: str) -> bool:
+        nonlocal target_seen, target_attempt, observed_at, expected_sequence, sequence_gap
+        observed = _count(body.get("sequence"))
+        if observed is None:
+            return False
+        if observed != expected_sequence:
+            sequence_gap = True
+        expected_sequence = observed + 1
+        instant = parse_instant(body.get("occurred_at"))
+        observed_at = instant.isoformat() if instant is not None else None
+        if observed <= sequence:
+            tracker.observe(body)
+        if observed == sequence:
+            target_seen = True
+            attempt = body.get("attempt_id")
+            target_attempt = attempt if isinstance(attempt, str) and attempt else None
+        return observed >= sequence
+
+    followed = RunEventStream(
         client.session, run_id, transport=client.transport, sleep=sleep, path=RUN_EVENTS_PATH
     ).follow(
-        from_seq=max(0, sequence - tail),
-        observe=follow_into(tracker, until_sequence=sequence),
+        from_seq=from_sequence,
+        observe=observe,
         deadline=clock() + deadline_seconds,
         clock=clock,
     )
+    tracker.live_observation = {
+        "scope": "bounded_event_tail",
+        "from_sequence_exclusive": from_sequence,
+        "target_sequence": sequence,
+        "observed_sequence": followed.last_sequence,
+        "replay_complete": target_seen and not sequence_gap,
+        "sequence_gap": sequence_gap,
+        "observed_at": observed_at,
+        "attempt_id": target_attempt,
+        "deadline_expired": followed.expired,
+        "end_reason": followed.end_reason,
+    }
     return tracker
 
 
@@ -314,19 +391,58 @@ def history_rates(
     return tracker
 
 
-def live_rate_members(client: Any, run_id: str, **kwargs: Any) -> dict[str, Any]:
+def live_rate_members(
+    client: Any, run_id: str, *, wall_clock: Callable[[], datetime] = utc_now, **kwargs: Any
+) -> dict[str, Any]:
     """The ``status`` receipt members for a live run's rates, or why there are none."""
 
     try:
         tracker = live_rates(client, run_id, **kwargs)
     except ThinLaneError as refusal:
         return {"source_rates_unavailable": f"{refusal.code}: {refusal.detail}"}
-    if tracker is None or not tracker.receipts():
+    if tracker is None:
         return {"source_rates_unavailable": "no collection source has reported two ticks yet"}
-    return {
-        "source_rates": tracker.receipts(),
-        "rates": {f"source {index}": line for index, line in enumerate(tracker.lines(), 1)},
-    }
+    observation = dict(tracker.live_observation or {})
+    now = wall_clock()
+    observation["checked_at"] = now.isoformat()
+    result: dict[str, Any] = {"source_rate_observation": observation}
+    if not observation.get("replay_complete"):
+        result["source_rates_unavailable"] = (
+            "incomplete replay: event gap or snapshot event missing"
+        )
+        return result
+    if not observation.get("attempt_id") or observation["attempt_id"] != tracker.attempt_id:
+        result["source_rates_unavailable"] = "snapshot attempt could not be established"
+        return result
+    receipts, lines, unavailable = [], {}, {}
+    for source in tracker.sources.values():
+        at = source.ticks[-1][0] if source.ticks else None
+        age = (now - at).total_seconds() if at is not None else None
+        reason = source.unavailable_reason
+        if reason is None and (age is None or age < 0):
+            reason = "observation_time_unavailable_or_in_future"
+        if reason is None and age is not None and age > LIVE_FRESHNESS_SECONDS:
+            reason = "stale_observation"
+        if reason is None and source.per_second() is None:
+            reason = "insufficient_observations"
+        if reason is not None:
+            unavailable[source.name] = {
+                "reason": reason,
+                "observed_at": at.isoformat() if at else None,
+                "age_seconds": age,
+            }
+            continue
+        receipt = source.to_receipt()
+        receipt.update(observed_at=at.isoformat(), age_seconds=age, attempt_id=tracker.attempt_id)
+        receipts.append(receipt)
+        lines[f"source {len(lines) + 1}"] = source.line()
+    if unavailable:
+        result["source_rates_unavailable_by_source"] = unavailable
+    if receipts:
+        result.update(source_rates=receipts, rates=lines)
+    else:
+        result["source_rates_unavailable"] = "no fresh, same-attempt source rate is available"
+    return result
 
 
 __all__ = [

{mostlyright_data-0.30.1 → mostlyright_data-0.30.2}/src/mostlyright/data_harness/thin/v4_stream.py

@@ -40,10 +40,11 @@ import argparse
 import json
 import time
 from collections.abc import Callable, Mapping
+from datetime import datetime
 from typing import Any
 
 from mostlyright.data_harness.thin import THIN_SCHEMA_PREFIX
-from mostlyright.data_harness.thin.progress_rate import RateTracker
+from mostlyright.data_harness.thin.progress_rate import RateTracker, utc_now
 from mostlyright.data_harness.thin.session import (
     StudioSession,
     StudioSessionProvider,
@@ -216,6 +217,7 @@ def watch(
     *,
     client: StudioV4RunClient | None = None,
     clock: Callable[[], float] = time.monotonic,
+    wall_clock: Callable[[], datetime] = utc_now,
     sessions: StudioSessionProvider | None = None,
 ) -> dict[str, Any]:
     """``mr-data watch RUN``: the live timeline, and the record the run stopped on.
@@ -259,7 +261,7 @@ def watch(
             print(render_v4_frame(body, frame_name), flush=True)
         moved = rates.observe(body)
         if moved is not None and announce:
-            line = rates.due_line(moved)
+            line = rates.due_line(moved, wall_clock=wall_clock)
             if line is not None:
                 print(f"       rate  {line}", flush=True)
         if body.get("event_type") in V4_TERMINAL_EVENT_TYPES:
@@ -334,6 +336,12 @@ def terminal_payload(
     if measured:
         # The last rate each collection source reached, from its own progress ticks.
         payload["source_rates"] = measured
+        payload["source_rates_basis"] = "historical_last_observation_per_source"
+        payload["source_rate_observed_at"] = {
+            name: source.ticks[-1][0].isoformat()
+            for name, source in rates.sources.items()
+            if source.per_second() is not None
+        }
     if event_type == "run_failed":
         triple = failure_from(event_facts(terminal), subject="a run_failed event")
         payload["failure"] = triple