mostlyright-data 0.19.6__tar.gz → 0.19.7__tar.gz

{mostlyright_data-0.19.6 → mostlyright_data-0.19.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mostlyright-data
-Version: 0.19.6
+Version: 0.19.7
 Summary: Mostly Right hosted CLI for reviewed datasets
 Project-URL: Homepage, https://mostlyright.md/
 Project-URL: Documentation, https://mostlyright.md/docs/guides/cli/

{mostlyright_data-0.19.6 → mostlyright_data-0.19.7}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "mostlyright-data"
-version = "0.19.6"
+version = "0.19.7"
 description = "Mostly Right hosted CLI for reviewed datasets"
 readme = "README.md"
 requires-python = ">=3.13"

{mostlyright_data-0.19.6 → mostlyright_data-0.19.7}/skills/mr-data-build/SKILL.md

@@ -933,20 +933,23 @@ finished work rather than a shape test. Where a source was never opened, and its
 a guess, two thousand rows a source is the fallback: it exercises every column, every cast and
 every declared check, and it comes back while the person who asked is still watching the page.
 
-**One Reader holds about 3 MiB of one source at a time, and that bounds what a source DELIVERS.**
-It is the largest single source rather than the total — one Reader opens one source at a time — and
-a source that really delivers more than that dies mid-run with `SANDBOX_MEMORY_LIMIT`. This is
-yours to apply to the sizes stage 2 measured, not something the command decides for you:
+**What bounds a source is how much it DECODES, not how much it weighs.** One confined decode may
+hand back 6,845,533 bytes, and a source over that is refused with `READER_BUDGET`, which names both
+sizes. The two quantities come apart constantly: a 10.4 MB document projected to a few columns
+decodes small and runs, while an 8.6 MB table read whole does not, and two members of one 3.6 MB
+archive land on opposite sides. So the fetched size predicts nothing on its own.
+
 `limits.max_source_bytes` states where fetching STOPS rather than how big the source is, so
 `mr-data recipe` does not refuse a large one. It registers the document and puts one sentence on
-the receipt under `warnings`, naming the source and both numbers. Read that as "go and check this
-one", not as a failure: a document declaring 8 MiB over a feed that answers 40 KB is fine.
-
-**If a source really does deliver more than the Reader holds, do not answer it by lowering the
-ceiling on that source:** that does not fetch less data, it truncates the sample or refuses the
-full run. Split the window across several sources — one per month, per region, or per whatever the
-publisher paginates on — and union them in a first transform step. A recipe names up to 256
-sources, which is a great many windows. [The recipe document](https://mostlyright.md/docs/reference/recipe/)
+the receipt under `warnings` when the declared number is larger than any decode could return. Read
+that as "go and check this one", not as a failure.
+
+**Which remedy applies depends on which way the source is too big, and lowering the ceiling is
+never it** — that does not fetch less data, it truncates the sample or refuses the full run. A WIDE
+source, many columns or long text, is narrowed by projecting fewer columns in the reader's own
+settings; splitting it would not help at all. A LONG source is split across several sources — one
+per month, per region, or per whatever the publisher paginates on — and unioned in a first
+transform step. A recipe names up to 256 sources. [The recipe document](https://mostlyright.md/docs/reference/recipe/)
 carries the measured numbers behind the ceiling.
 
 **A corpus of many pages behind one index is a COLLECTION, not many sources.** See

{mostlyright_data-0.19.6 → mostlyright_data-0.19.7}/src/mostlyright/data_harness/thin/recipe.py

@@ -69,31 +69,24 @@ REQUIRED_DOCUMENT_MEMBERS: tuple[str, ...] = (
 
 #: The declared ``limits.max_source_bytes`` above which one source is worth a word, in bytes.
 #:
-#: WHY THE NUMBER IS WORTH SAYING, AND WHY THIS ONE. The Reader that opens a source's bytes runs
-#: inside one Clean room with a fixed memory allowance, and it opens ONE source at a time -- so
-#: what decides whether a recipe can run is the largest single source in it, not the total. The
-#: number is measured rather than reasoned: a source that DELIVERED 5.76 MB died mid-run with
-#: ``SANDBOX_MEMORY_LIMIT``, and one that delivered 1.15 MB was accepted and completed. 3 MiB sits
-#: between the two with room on both sides.
+#: ⚠ THIS NUMBER BOUNDS THE FETCH, AND WHAT REFUSES IS THE DECODE.  The two are different
+#: quantities and the old value conflated them.  3 MiB came from two observations -- a source that
+#: delivered 5.76 MB and died, one that delivered 1.15 MB and lived -- and neither was a
+#: measurement of anything the run enforces.  Measured properly: a 10,449,878-byte JSON source
+#: projected to five columns SUCCEEDS, while an 8,578,215-byte CSV read whole FAILS, and two
+#: members of ONE 3.6 MB archive land on opposite sides -- so the fetched size, which is what this
+#: member states, cannot order the outcomes at all.
 #:
-#: ⚠ AND IT IS A WARNING RATHER THAN A REFUSAL, BECAUSE ``max_source_bytes`` IS NOT A SIZE. It is
-#: the point at which the Reader STOPS fetching -- a refusal ceiling, stated by whoever wrote the
-#: document -- so a document that writes 8 MiB over a feed answering 40 KB is a document that runs
-#: and delivers. Refusing on the declared number would therefore refuse working recipes for
-#: something they have not done, and it did: three of this repository's own acceptance recipes
-#: declare between 4 and 16 MiB, and every one of them was built, run and delivered. What the
-#: ceiling is genuinely about is the DELIVERED size, which this side of the wire cannot know before
-#: the run. So the command says so and submits, and the sentence is addressed to the agent that
-#: wrote the number, because it is the one party that can go and measure the source.
+#: WHAT ORDERS THEM is how much the Reader DECODES, because the confined decode hands its output
+#: back over a bounded channel and carries it twice there: base64 and as parsed rows.  That
+#: ceiling is ``DELIVERABLE_READER_OUTPUT_BYTES`` on the backend, 6,845,533 bytes, and a source
+#: over it is refused by name with ``READER_BUDGET`` naming both sizes.
 #:
-#: ⚠ THE REMEDY IS NOT A SMALLER CEILING ON THE SAME SOURCE. Lowering ``max_source_bytes`` under a
-#: window that genuinely holds more bytes does not make the source smaller; on `full` it turns the
-#: run into a ``CLAMP_EXCEEDED`` refusal, and on `sample` it truncates, which is worse because a
-#: truncated source looks like a complete one until somebody reads ``coverage``. The remedy is to
-#: SPLIT the window across several sources -- one per month, per region, or per whatever the
-#: publisher paginates on -- and union them in a first transform step, which is what the warning
-#: says.
-READER_SOURCE_BYTE_CEILING = 3 * 1024 * 1024
+#: So this stays a warning about a DECLARED ceiling being implausibly large, which is worth one
+#: sentence, and it no longer promises to predict a failure it cannot see.  It is set at the
+#: backend's own output ceiling: a document declaring more than one decode can return is a
+#: document whose author has not measured the source, whatever the source turns out to weigh.
+READER_SOURCE_BYTE_CEILING = 6_845_533
 
 #: How many undescribed columns the warning names before it counts the rest. A table plan admits
 #: 512 columns, and a sentence that spelt every one of them out would be the wall the `numbered`
@@ -215,11 +208,10 @@ def source_byte_warnings(sources: Any) -> list[str]:
     will not.
 
     ⚠ AND THE SENTENCE NAMES THE FIX, which is the whole reason it is written here rather than left
-    to a mid-run ``SANDBOX_MEMORY_LIMIT``. That failure costs a registration, a run and the minutes
-    before it died, and it leaves somebody to guess whether the answer is a smaller ceiling (it is
-    not -- that truncates or refuses) or a different source (usually not either). Splitting the
-    window into several sources and unioning them in a first transform step is the answer nearly
-    every time, so it is said.
+    to a mid-run refusal. It also says what the fix is NOT: a smaller declared ceiling, which
+    truncates the fetch rather than decoding less. Narrowing what the reader projects is the answer
+    when the source is wide; splitting the source is the answer when it is long. Splitting a wide
+    source does nothing, which is why the two cases are named separately.
 
     A declared ceiling that is not a whole number of bytes is left alone: the document admits no
     fractional number at all, ``read_document`` has already refused one, and re-deciding here what
@@ -241,16 +233,16 @@ def source_byte_warnings(sources: Any) -> list[str]:
         named = source.get("name")
         where = f"{named!r}" if isinstance(named, str) and named else f"sources[{index}]"
         said.append(
-            f"source {where} declares limits.max_source_bytes of {declared:,} bytes, and one "
-            f"Reader has been measured to hold about {READER_SOURCE_BYTE_CEILING:,}. That is a "
-            "ceiling on fetching rather than a measurement of the source, so this registers and "
-            "runs -- but if the source really delivers that much, the run dies with "
-            "SANDBOX_MEMORY_LIMIT. One Reader opens one source at a time in one Clean room, so it "
-            "is the largest single source that has to fit, not the total. Do not simply lower the "
-            "ceiling -- that truncates the sample or refuses the full run rather than fetching "
-            "less data. Measure what the source answers, and if it is over the ceiling: split the "
-            "window into several sources, one per month or per region or per whatever the "
-            "publisher paginates on, and union them in a first transform step"
+            f"source {where} declares limits.max_source_bytes of {declared:,} bytes, which is "
+            f"more than one confined decode can hand back ({READER_SOURCE_BYTE_CEILING:,}). That "
+            "is a ceiling on fetching rather than a measurement of the source, so this registers "
+            "and runs. What decides the run is how much the Reader DECODES, not how much is "
+            "fetched: a wide document projected to a few columns decodes small however large it "
+            "arrives, and a compact archive member can decode past the ceiling. A source that "
+            "does is refused by name with READER_BUDGET, which states both sizes. Lowering this "
+            "ceiling does not help, because it bounds the fetch and truncates rather than "
+            "decoding less. Narrow the projection, or split the source and union the parts in a "
+            "first transform step"
         )
     return said
 

{mostlyright_data-0.19.6 → mostlyright_data-0.19.7}/src/mostlyright/data_harness/thin/router.py

@@ -508,6 +508,39 @@ def _emit(payload: dict[str, Any], *, as_json: bool) -> None:
     print(render_human(payload), flush=True)
 
 
+#: How many of Studio's own pointer-and-reason details one refusal renders before it counts the
+#: rest. A document refused for one reason is the ordinary case; a document refused for forty is a
+#: document to fix a few at a time, and forty paragraphs is the wall a person stops reading at.
+MAX_RENDERED_DETAILS = 8
+
+
+def _refusal_details(refusal: BaseException) -> tuple[str, ...]:
+    """The sentences Studio wrote about WHICH part of a document it refused.
+
+    Studio answers a refused write with one entry per fault, each carrying a JSON pointer into the
+    document and a reason -- "chart 'share' describes boolean columns; 'event_type' is declared
+    string" names the column and the fix in one line. ``_studio_refusal`` already carries them onto
+    the refusal; nothing rendered them, so every one of those refusals reached a person as
+    "The recipe document is internally inconsistent." and nothing else, and the only way to find
+    out which of five hundred members was wrong was to bisect the document by hand.
+
+    Rendered rather than interpreted: the pointer is Studio's and the reason is Studio's, and a
+    client that paraphrased either would be a second opinion about a document it did not validate.
+    """
+
+    details = getattr(refusal, "details", ()) or ()
+    said: list[str] = []
+    for entry in details:
+        if not isinstance(entry, Mapping):
+            continue
+        reason = entry.get("reason")
+        if not isinstance(reason, str) or not reason:
+            continue
+        pointer = entry.get("pointer")
+        said.append(f"{pointer}: {reason}" if isinstance(pointer, str) and pointer else reason)
+    return tuple(said[:MAX_RENDERED_DETAILS])
+
+
 def _refuse(
     code: str,
     detail: str,
@@ -515,6 +548,7 @@ def _refuse(
     as_json: bool,
     retry_after_seconds: int | None = None,
     extra: Mapping[str, Any] | None = None,
+    details: Sequence[str] = (),
 ) -> int:
     """Report one refusal the way every other ``mr-data`` refusal is reported.
 
@@ -543,6 +577,8 @@ def _refuse(
         }
         if retry_after_seconds is not None:
             error["retry_after_seconds"] = retry_after_seconds
+        if details:
+            error["details"] = list(details)
         if extra:
             error.update(extra)
         print(
@@ -551,7 +587,14 @@ def _refuse(
             flush=True,
         )
     else:
-        print(render_error(detail, code=code, remediation=remediation), file=sys.stderr, flush=True)
+        # Studio's own sentences come FIRST, before the generic remediation: they name the member
+        # and the fix, and burying them under three paragraphs of lane advice is how a refusal
+        # that said exactly what was wrong reached a person saying nothing.
+        print(
+            render_error(detail, code=code, remediation=tuple(details) + tuple(remediation)),
+            file=sys.stderr,
+            flush=True,
+        )
     return 2
 
 
@@ -789,6 +832,7 @@ def _hosted(command: str, arguments: Sequence[str], *, as_json: bool) -> int:
             refusal.detail,
             as_json=as_json or parsed.json,
             retry_after_seconds=getattr(refusal, "retry_after_seconds", None),
+            details=_refusal_details(refusal),
         )
     except Exception as error:
         code = getattr(error, "code", None)