mostlyright-data 0.21.3__tar.gz → 0.22.1__tar.gz

{mostlyright_data-0.21.3 → mostlyright_data-0.22.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mostlyright-data
-Version: 0.21.3
+Version: 0.22.1
 Summary: Mostly Right hosted CLI for reviewed datasets
 Project-URL: Homepage, https://mostlyright.md/
 Project-URL: Documentation, https://mostlyright.md/docs/guides/cli/

{mostlyright_data-0.21.3 → mostlyright_data-0.22.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "mostlyright-data"
-version = "0.21.3"
+version = "0.22.1"
 description = "Mostly Right hosted CLI for reviewed datasets"
 readme = "README.md"
 requires-python = ">=3.11"

{mostlyright_data-0.21.3 → mostlyright_data-0.22.1}/skills/mr-data-build/SKILL.md

@@ -1867,8 +1867,10 @@ Write it as a source whose connector is that coordinate plus a `collection` memb
 
 - `collection.discovery` says how the corpus is LISTED — `json_api` with RFC 6901 pointers,
   `html_index` with the reader's closed selectors, `sitemap`, or an `explicit` member list — and
-  which pagination grammar the listing uses. The run follows only that grammar and never a link
-  found inside a detail page.
+  which pagination grammar the listing uses. `source_rows` instead names an upstream source and
+  its `url_column`, optionally `base_url_column`: acquisition runs upstream first, resolves and
+  deduplicates links, and preserves incomplete upstream coverage. Use only declared dependencies;
+  this is not arbitrary recursive crawling. It requires the matching Studio API/worker release.
 - `collection.pages` says what may be fetched and how much: `allowed_origins` (which must include
   the discovery address's own origin), `max_pages`, `max_fetches_per_run`, `concurrency` (1 or 2),
   `min_interval_seconds` as a whole number of seconds (one or more), `request_timeout_seconds`,
@@ -1877,7 +1879,8 @@ Write it as a source whose connector is that coordinate plus a `collection` memb
 - `limits.max_requests` is REQUIRED and bounds every request the run makes, discovery and detail
   pages together. `limits.max_source_bytes` bounds the fetched bytes; `limits.max_rows` clamps the
   merged relation.
-- The Reader pin is required, must be `html.web_extract`, and is the SAME settings for every page.
+- The Reader pin is required, must be `html.web_extract`, `html.tabular`, `json.tabular`, or
+  `xml.tabular`, and uses the same settings for every page.
 
 **`limits.max_requests` is charged per HTTP request, redirect hops and retries included** — a page
 that redirects once costs two. Registration only checks that
@@ -1888,6 +1891,10 @@ redirects are likely — trailing slashes, `http`→`https`, `www`, which is mos
 `limits.max_requests >= discovery.max_requests + 2 * max_fetches_per_run`. A budget larger than the
 corpus can absorb costs nothing.
 
+For a dependent graph, declare recipe-level `acquisition_limits` with `max_requests` and
+`max_source_bytes`. Every source must explicitly state both ceilings; their sums must fit the
+totals. Unused capacity does not transfer between sources.
+
 The relation the transform reads is the reader's columns plus `page_id`, `page_url`,
 `page_fetched_at`, `page_content_sha256`, `page_revision`, `page_discovered_at` and `page_ordinal`.
 Those names are reserved: a reader that declares one is a registration refusal. Select them in the
@@ -2024,6 +2031,7 @@ decoder. Zero matches and multiple matches both refuse.
 | `html.tabular@1.0.0` | Absolute record element path and relative text/attribute field selectors; fields are required unless `required: false`. |
 | `html.web_extract@1.0.0` | `mode: dom` with closed CSS-like record/field selectors, or `mode: json` with an inert JSON script selector and RFC 6901 field pointers. |
 | `html.web_extract@1.1.0` | Same settings as 1.0.0; extracted text follows DOM order, with one space at block-element boundaries and `<br>` and none between inline runs. |
+| `html.web_extract@1.2.0` | Same text as 1.1.0; bounded integer `:nth-child(N)` / `:nth-of-type(N)` and field `scope: ancestor` with `ancestor_selector`. Requires the corresponding deployed certification. |
 | `protobuf.tabular@1.0.0` | Digest-pinned base64 FileDescriptorSet, fully-qualified message type, `single` or `length_delimited` framing, and fields with an explicit repeated `explode` or `refuse` choice. |
 | `parquet.tabular@1.0.0` | Optional `columns` selects a bounded ordered Parquet projection. |
 | `xbrl.facts@1.0.0` | `document_kind` (`xbrl` or constrained XML `ixbrl`) and exact namespace/name fact projection; contexts, units, and metadata remain evidence fields, not joins. |

{mostlyright_data-0.21.3 → mostlyright_data-0.22.1}/src/mostlyright/data_harness/thin/v4.py

@@ -105,7 +105,7 @@ from mostlyright.data_harness.thin.transport import ThinLaneError
 #: ``JOB_INVALID`` outright, which in production was every refresh a collection epoch was offered
 #: for rather than only the collection runs. This package must not reach a Studio older than the
 #: commit it pins; ``docs/V4-WORKER-PROTOCOL.md`` states it beside the layout's own ordering rule.
-PINNED_V4_OPENAPI_SOURCE_SHA256 = "2a83dda60e84c166f95ea18af747985bfeeb5887cb48d2c72d4335e83720dc37"
+PINNED_V4_OPENAPI_SOURCE_SHA256 = "2b854c3c4fae7c30a6219ee99ea814f80c57fbf0251ed22a0b66b002b598e6f7"
 PINNED_V4_CONTRACT_VERSION = "4.7.0"
 
 # --------------------------------------------------------------------------------------------

{mostlyright_data-0.21.3 → mostlyright_data-0.22.1}/src/mostlyright/data_harness/thin/v4_artifacts.py

@@ -277,9 +277,10 @@ class StudioV4ArtifactClient(StudioV4RunClient):
         ⚠ THE ONE ROUTE THAT REACHES A PART ANOTHER RUN SEALED. A version is a manifest over
         parts and the parts may come from many runs, so ``GET /v4/runs/{run_id}/artifacts``
         answers about one run's contribution and never about the whole table. This route is
-        addressed by the version the run's OWN manifest names -- the coordinate is sealed inside
-        the artifact this command already fetched -- so nothing here resolves a live pointer or
-        asks whether the version is one anybody published.
+        addressed by the physical table and version identifiers on that public run. A sealed
+        manifest selects and verifies the parts, but legacy manifests can contain logical
+        identifiers that cannot address this route; nothing here resolves a live pointer or asks
+        whether the version is one anybody published.
         """
 
         body: dict[str, Any] = {"limit": PART_SESSION_PAGE_LIMIT}
@@ -548,19 +549,17 @@ def download(
     """``mr-data download RUN``: this run's artifacts, on this computer, each digest checked.
 
     ⚠ NOTHING ON THIS COMMAND'S OWN PATH ASKS WHETHER THE RUN MAY BE READ. The listing is fetched,
-    a session is minted per artifact and the bytes are verified; the run record is never read on
-    the way to the answer, so there is no place a status, a version or an approval could be
-    consulted even by accident. That is REQ-H407's sentence made structural rather than promised.
-
-    ⚠ AND THE SCOPE OF THAT SENTENCE IS THIS PATH, WHICH SAYING SO NARROWS RATHER THAN WEAKENS.
-    After the answer is printed, ``thin.router`` makes one best-effort progress report, and for
-    this command -- which holds a run identifier and nothing else -- that report reads
-    ``GET /v4/runs/{run_id}`` to learn which dataset page to write on. It happens AFTER the
-    download has been decided, executed and rendered, on its own bounded budget, and it cannot
-    reach anything this command did or printed: nothing on the way to the artifacts consults the
-    run record, which is the property REQ-H407 is about. A docstring claiming the record is never
-    read at all would be false the first time somebody grepped for the route, and a false absolute
-    is worse than a scoped true one.
+    a session is minted per artifact and the bytes are verified. A multipart version with retained
+    parts additionally reads the run's public table and version identifiers solely to address the
+    version-part session route; it never reads the status, approval, or promotion to decide whether
+    bytes may be read. That is REQ-H407's sentence made structural rather than promised.
+
+    ⚠ THE RUN RECORD'S NARROW ROLE IS TO ADDRESS, NEVER TO ADMIT. A manifest is a sealed logical
+    document and historical producers can put logical table identifiers in it. Studio's public
+    version-part route is addressed by the physical identifiers on the run record instead. Those
+    values are used only after the manifest identified retained parts that need signed sessions;
+    every fetched byte is still checked against the manifest's part digest. A status, approval, or
+    promotion value is neither read nor used as a predicate.
 
     ⚠ A TABLE VERSION ARRIVES WHOLE OR NOT AT ALL. The run listing answers about the parts THIS
     run sealed, and a version composed over several refreshes names parts earlier runs sealed; a
@@ -629,7 +628,7 @@ def _version_parts(
     sealed = {str(row.get("digest")): row for row in listing if row.get("kind") == "table_part"}
     repeated = Counter(part.part_key for part in manifest.parts)
     outstanding = tuple(part for part in manifest.parts if part.part_id not in sealed)
-    minted = _part_sessions(client, manifest, outstanding)
+    minted = _part_sessions(client, run_id, manifest, outstanding)
     written: list[dict[str, Any]] = []
     for part in manifest.parts:
         destination = output / _part_destination(part, repeated=repeated[part.part_key] > 1)
@@ -660,19 +659,20 @@ def _part_destination(part: Any, *, repeated: bool) -> Path:
 
 
 def _part_sessions(
-    client: StudioV4ArtifactClient, manifest: Any, outstanding: Sequence[Any]
+    client: StudioV4ArtifactClient, run_id: str, manifest: Any, outstanding: Sequence[Any]
 ) -> dict[str, dict[str, Any]]:
     """One signed read per part this run did not seal, or a refusal naming how many are missing."""
 
     if not outstanding:
         return {}
+    public_run = StudioV4RunClient(client.session, transport=client.transport).run(run_id)
+    table_id = _run_identifier(public_run, "table_id")
+    version_id = _run_identifier(public_run, "table_version_id")
     wanted = {part.part_id for part in outstanding}
     minted: dict[str, dict[str, Any]] = {}
     cursor: str | None = None
     for _page in range(MAX_PART_SESSION_PAGES):
-        page = client.version_part_sessions(
-            manifest.table_id, manifest.table_version_id, cursor=cursor
-        )
+        page = client.version_part_sessions(table_id, version_id, cursor=cursor)
         entries = page.get("parts")
         if not isinstance(entries, list):
             raise ThinLaneError(
@@ -701,6 +701,18 @@ def _part_sessions(
     return minted
 
 
+def _run_identifier(run: Mapping[str, Any], field: str) -> str:
+    """One public physical version coordinate, never a lifecycle predicate."""
+
+    value = run.get(field)
+    if not isinstance(value, str) or not value:
+        raise ThinLaneError(
+            "THIN_RESPONSE_INVALID",
+            f"the run's public {field} is absent while retained table parts need signed reads",
+        )
+    return identifier(value, f"run {field}")
+
+
 def _fetch_version_part(
     client: StudioV4ArtifactClient,
     part: Any,