mostlyright-data 0.30.2__tar.gz → 0.30.3__tar.gz

{mostlyright_data-0.30.2 → mostlyright_data-0.30.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mostlyright-data
-Version: 0.30.2
+Version: 0.30.3
 Summary: Mostly Right hosted CLI for reviewed datasets
 Project-URL: Homepage, https://mostlyright.md/
 Project-URL: Documentation, https://mostlyright.md/docs/guides/cli/

{mostlyright_data-0.30.2 → mostlyright_data-0.30.3}/skills/mr-data-build/SKILL.md

@@ -152,6 +152,9 @@ step 6. The stage 3 reference holds its rule. Gate 4 is step 6.
    quality checks.
 7. Follow the returned run with `mr-data watch RUN_ID --json`. Its `source_rates` and
    `mr-data status RUN_ID --json` on a live run give each collection source's rate and time left.
+   On a queued or running run, `status` also gives `progress`: the current attempt, how many of
+   its sources it fetched of how many, the source it is on with its pages, and how the attempt
+   before it ended. A handed-back run's new attempt counts from zero.
    Quote those numbers rather than doing the arithmetic. Read `run.status`: returning an existing
    run does not mean new work was queued. Resume the known run after an interruption, using the
    saved cursor where available. Read [run lifecycle](references/live-run.md) for holds and

{mostlyright_data-0.30.2 → mostlyright_data-0.30.3}/src/mostlyright/data_harness/page_coverage.py

@@ -20,6 +20,7 @@ __all__ = [
     "PAGE_BUDGET_KINDS",
     "PAGE_COVERAGE_COUNT_FIELDS",
     "PAGE_COVERAGE_FIELDS",
+    "SOURCE_ROWS_INCOMPLETE",
     "page_coverage_sentence",
 ]
 
@@ -42,6 +43,9 @@ __all__ = [
 #: predecessor rows, but current collection refresh is refused before acquisition, except a
 #: per-row request collection's, which refreshes with the direct request window it inherits and
 #: asks again for a failed row whenever a refresh covers that row's partition.
+#: A ``source_rows`` collection also states it on a full run, as ``SOURCE_ROWS_INCOMPLETE``, when
+#: the upstream relation it lists from is incomplete: that is not a listing anybody failed to read
+#: but a list that may be short by what the upstream source is missing, and the sentence says so.
 PAGE_COVERAGE_FIELDS: tuple[str, ...] = (
     "discovered",
     "discovery_requests",
@@ -84,6 +88,46 @@ PAGE_COVERAGE_COUNT_FIELDS: tuple[str, ...] = (
 )
 
 
+#: The discovery failure a ``source_rows`` collection states when the upstream relation it lists
+#: from is itself incomplete (failed, pending or unlisted members upstream). Nothing failed to be
+#: LISTED: the list is built from rows another source did not finish delivering, so it may be
+#: short by exactly what that source is missing. Said as what it is, never as an unreadable
+#: listing, because a reader told "the listing could not be read" goes looking for an outage.
+SOURCE_ROWS_INCOMPLETE = "SOURCE_ROWS_INCOMPLETE"
+
+
+def _count(block: Mapping[str, Any], member: str) -> int | None:
+    value = block.get(member)
+    if isinstance(value, bool) or not isinstance(value, int) or value < 0:
+        return None
+    return value
+
+
+def _pages_held(block: Mapping[str, Any]) -> int | None:
+    """How many of the collection's members hold an answer, or ``None`` when the block cannot say.
+
+    MEMBERS, NOT CALLS. ``fetched_this_run`` counts the requests this run made: a per-row request
+    collection asks once for every member that renders the same request, a batch answers up to a
+    hundred members in one call, and a page kept from an earlier capture is not fetched again. So
+    ``fetched_this_run`` over ``discovered`` divides calls by members, and a run that answered
+    every member it listed reads as half done (food run 23161446: "459,156 of 1,005,223" for a
+    run with 1,004,768 members answered and 455 failed). The members that hold an answer are the
+    ones the ledger knows that are neither still owed, nor failed, nor skipped -- the same count
+    the dataset page's sources card prints as fetched.
+
+    Read with ``get``: a record written before these members existed carries only the four the
+    sentence always needed, and says what it always said.
+    """
+
+    known = _count(block, "known")
+    pending = _count(block, "pending")
+    if known is None or pending is None:
+        return None
+    failed = _count(block, "failed") or 0
+    skipped = _count(block, "skipped") or 0
+    return max(known - pending - failed - skipped, 0)
+
+
 def page_coverage_sentence(block: Mapping[str, Any]) -> str:
     """The one sentence a person reads off a page collection's coverage.
 
@@ -96,13 +140,29 @@ def page_coverage_sentence(block: Mapping[str, Any]) -> str:
     failed = int(block["failed"])
     if block["complete"]:
         return f"all {discovered:,} pages fetched"
-    parts = [f"{fetched:,} of {discovered:,} pages fetched"]
+    held = _pages_held(block)
+    if held is None:
+        parts = [f"{fetched:,} of {discovered:,} pages fetched"]
+    else:
+        # A ledger can know pages its latest listing no longer names (a refresh keeps them), so
+        # the whole is the larger of the two and the sentence never says "more of fewer".
+        whole = max(discovered, _count(block, "known") or 0)
+        lead = f"{held:,} of {whole:,} pages fetched"
+        if fetched != held:
+            lead += f" ({fetched:,} requests this run)"
+        parts = [lead]
     # READ WITH ``get``, AND ONLY THIS MEMBER. The block a run produces always carries it, and the
     # block Studio hands back may not: it is optional on the run record, so a record written before
     # the member existed is still one sentence rather than a KeyError this lane swallows whole.
     listing = block.get("discovery_failure")
     if isinstance(listing, Mapping):
-        parts.append(f"the page listing could not be read ({listing['code']})")
+        if listing.get("code") == SOURCE_ROWS_INCOMPLETE:
+            parts.append(
+                "a page list built from another source's rows may be short, because that source "
+                f"is incomplete ({SOURCE_ROWS_INCOMPLETE})"
+            )
+        else:
+            parts.append(f"the page listing could not be read ({listing['code']})")
     if failed:
         parts.append(f"{failed:,} failed")
     # The block does not say which kind of collection it counts, so the remedy names both. A

{mostlyright_data-0.30.2 → mostlyright_data-0.30.3}/src/mostlyright/data_harness/thin/v4_runs.py

@@ -2355,6 +2355,8 @@ def status(args: argparse.Namespace, *, client: StudioV4RunClient | None = None)
         held = {member: run[member] for member in _PROJECTION_MEMBERS[1:] if member in run}
         if held:
             payload["projection"] = held
+    if state in ATTEMPTING_RUN_STATUSES:
+        payload.update(attempt_progress_members(selected, run_id))
     if getattr(args, "rate", False) and state not in TERMINAL_RUN_STATUSES:
         # Imported here: progress_rate reads the event stream, whose module imports this one.
         from mostlyright.data_harness.thin import progress_rate
@@ -2378,6 +2380,112 @@ def status(args: argparse.Namespace, *, client: StudioV4RunClient | None = None)
     return payload
 
 
+#: The states a run holds while it has, or is between, producer attempts. A run held at a spend
+#: or sample gate has never been claimed, so there is no attempt to read for it.
+ATTEMPTING_RUN_STATUSES: frozenset[str] = frozenset({"queued", "running"})
+
+#: How each way an attempt can end reads in the progress sentence. The keys are
+#: ``run-attempts.schema.json``'s closed ``end`` vocabulary.
+_ATTEMPT_ENDS: dict[str, str] = {
+    "running": "is still running",
+    "completed": "completed",
+    "requeued_by_worker": "was handed back by its own worker",
+    "handed_back": "was handed back after its worker went silent",
+    "failed": "failed",
+    "lost": "went silent",
+}
+
+
+def attempt_progress_members(client: StudioV4RunClient, run_id: str) -> dict[str, Any]:
+    """The ``status`` members for a live run's current attempt and its sources, or why not.
+
+    Read off the attempt ledger (``GET /v4/runs/{run_id}/attempts``), which Studio folds from
+    each progress tick as it arrives and never from the run log. So, unlike the rate, this does
+    not depend on replaying the log in time, and an attempt that has run for hours still says
+    how many of its sources it fetched. Counts are the current attempt's own: a run handed back
+    walks its sources again from the first.
+
+    A Studio before 4.37.0 serves no per-source member, and then the attempt and how the one
+    before it ended are still stated. A refused read is reported, never fatal: the run record
+    has already been read.
+    """
+
+    try:
+        answer = client.run_attempts(run_id)
+    except ThinLaneError as refusal:
+        return {"progress_unavailable": f"{refusal.code}: {refusal.detail}"}
+    rows = answer.get("attempts") if isinstance(answer, Mapping) else None
+    rows = [
+        row for row in rows or () if isinstance(row, Mapping) and type(row.get("generation")) is int
+    ]
+    if not rows:
+        return {"progress_unavailable": "Studio lists no attempt of this run yet"}
+    rows.sort(key=lambda row: row["generation"])
+    current = rows[-1]
+    sources = current.get("sources") if isinstance(current.get("sources"), Mapping) else None
+    named = sources.get("current") if sources is not None else None
+    progress: dict[str, Any] = {
+        "attempt": current["generation"],
+        "attempts": len(rows),
+        "source_count": sources.get("source_count") if sources is not None else None,
+        "sources_fetched": sources.get("fetched") if sources is not None else None,
+        "sources_refused": sources.get("refused") if sources is not None else None,
+        "counted_from_first_tick": (
+            sources.get("counted_from_first_tick") if sources is not None else None
+        ),
+        "current_source": dict(named) if isinstance(named, Mapping) else None,
+        "previous_attempt": None,
+    }
+    if len(rows) > 1:
+        previous = rows[-2]
+        loss = previous.get("loss") if isinstance(previous.get("loss"), Mapping) else {}
+        progress["previous_attempt"] = {
+            "attempt": previous["generation"],
+            "end": previous.get("end"),
+            "ended_at": previous.get("ended_at"),
+            "cause": loss.get("cause"),
+            "cause_inferred": loss.get("inferred"),
+        }
+    return {"progress": progress, "progress_sentence": attempt_progress_sentence(progress)}
+
+
+def attempt_progress_sentence(progress: Mapping[str, Any]) -> str:
+    """``Attempt 4 of 4 has fetched 27 of 86 sources and is on ...``, from the members above."""
+
+    said = f"Attempt {progress['attempt']} of {progress['attempts']}"
+    fetched, count = progress.get("sources_fetched"), progress.get("source_count")
+    if fetched is None:
+        said += "; this Studio reports no per-source progress for it."
+    else:
+        said += f" has fetched {fetched:,}"
+        said += f" of {count:,} sources" if count is not None else " sources"
+        if progress.get("counted_from_first_tick") is False:
+            said += " (at least: counting began after the attempt started)"
+        named = progress.get("current_source")
+        if isinstance(named, Mapping):
+            said += f" and is on {named.get('source_name')}"
+            if named.get("source_index") is not None and count is not None:
+                said += f", source {named['source_index']:,} of {count:,}"
+            done, planned = named.get("pages_processed"), named.get("pages_planned")
+            unit = "members" if named.get("pages_unit") == "member" else "pages"
+            if done is not None and planned is not None and named.get("discovery_complete"):
+                said += f": {done:,} of {planned:,} {unit}"
+            elif done is not None:
+                said += f": {done:,} {unit} so far"
+        said += "."
+    previous = progress.get("previous_attempt")
+    if isinstance(previous, Mapping):
+        end = _ATTEMPT_ENDS.get(str(previous.get("end")), f"ended {previous.get('end')}")
+        said += f" Attempt {previous['attempt']} {end}"
+        if previous.get("cause") is not None:
+            said += (
+                f"; {'inferred ' if previous.get('cause_inferred') else ''}cause: "
+                f"{previous['cause']}"
+            )
+        said += "."
+    return said
+
+
 def status_exit_code(payload: Mapping[str, Any]) -> int:
     """A failed run exits non-zero, so a script gating on a build cannot read it as a success."""