mostlyright-data 0.30.0__tar.gz → 0.30.1__tar.gz

{mostlyright_data-0.30.0 → mostlyright_data-0.30.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mostlyright-data
-Version: 0.30.0
+Version: 0.30.1
 Summary: Mostly Right hosted CLI for reviewed datasets
 Project-URL: Homepage, https://mostlyright.md/
 Project-URL: Documentation, https://mostlyright.md/docs/guides/cli/

{mostlyright_data-0.30.0 → mostlyright_data-0.30.1}/skills/mr-data-build/references/preparing-table-search.md

@@ -199,8 +199,8 @@ The `local` block lists each refusal with a JSON pointer. Repair every diagnosti
 
 Register and build the table as usual. A finished build of a table with `table.search` prepares
 search in the background. Data publication never waits for it. The live search package that
-`search try` and `search locate` use exists only for a table published publicly. Read the
-version ID from the run receipt, then check readiness:
+`search try` uses exists only for a table published publicly. Read the version ID from the run
+receipt, then check readiness:
 
 ```sh
 mr-data search status TABLE_ID --version VERSION_ID --json
@@ -280,12 +280,14 @@ mr-data search locate TABLE_ID --public-table PUBLIC_TABLE_ID --title "Dune" --y
 mr-data search locate TABLE_ID --public-table PUBLIC_TABLE_ID --id 27205 --json
 ```
 
-`locate` searches the live package and maps each matching row to its locator. It reads the
-title, date and identifier columns from the field roles. Override them with `--title-column`,
-`--date-column` and `--id-column`. Each column it reads must be in `display_columns`. A target
-with a year reads the date column, so declare `display_columns` with the date in it. A title
-without a year can match several rows. `locate` then lists them all and exits 4. Add the year or
-use the id. It writes `--out` only when every target resolved and the cases meet the contract.
+`locate` reads the public table's rows and maps each matching row to its locator. It needs no
+search package, so write the cases before the version has one. The table must be published
+publicly with its preview on. It reads the title, date and identifier columns from the field
+roles. Override them with `--title-column`, `--date-column` and `--id-column`. A title matches
+exactly, with the same case and spacing as the cell. A year matches the start of the date
+column's value. A title without a year can match several rows. `locate` then lists them all and
+exits 4. Add the year or use the id. It writes `--out` only when every target resolved and the
+cases meet the contract.
 
 Locators belong to one sealed version. `locate` resolves rows of the live version only. Configure
 the cases on that same version. A later version has a different row layout, so locate again after
@@ -313,6 +315,8 @@ Leave out `--spec` to declare the `table.search` spec of the recipe that sealed
 Use that to add evaluation cases without retyping the spec. Without `--spec` the client cannot
 check locally that semantic cases include a paraphrase and a hard negative. Studio checks them
 against the recipe's spec. A recipe with no `table.search` answers 409 `SEARCH_NOT_CONFIGURED`.
+`search status --job` reads any search job: the build, its package and embed steps, the
+evaluator and a title job. A job still `queued`, `dispatched` or `running` exits 3.
 `--expected-revision` is the current draft from `search status` (`selected_draft_revision_id`),
 or `none`. Reuse `--idempotency-key` only to retry the same configuration. When the job succeeds,
 `search status TABLE_ID --version VERSION_ID --revision REVISION_ID` reports `not_activated` and

{mostlyright_data-0.30.0 → mostlyright_data-0.30.1}/src/mostlyright/data_harness/thin/table_search_contract.json

@@ -1,6 +1,6 @@
 {
  "schema_version": "mostlyright-thin-table-search-contract.v1",
- "source_commit": "a0555f0152fe14beefaa5e082c7e3352d052f87a",
+ "source_commit": "b308bc2e9ebb424b4cfe1130438ab9f7d0b6736d",
  "recipe_search_definition": "spec",
  "$defs": {
   "evaluation_case": {

{mostlyright_data-0.30.0 → mostlyright_data-0.30.1}/src/mostlyright/data_harness/thin/v4.py

@@ -148,8 +148,8 @@ from mostlyright.data_harness.thin.transport import ThinLaneError
 #: ``JOB_INVALID`` outright, which in production was every refresh a collection epoch was offered
 #: for rather than only the collection runs. This package must not reach a Studio older than the
 #: commit it pins; ``docs/V4-WORKER-PROTOCOL.md`` states it beside the layout's own ordering rule.
-PINNED_V4_OPENAPI_SOURCE_SHA256 = "fc5229095aacb4fb7fe57b34c206802b381c81088c40af9dc19c05889b486a41"
-PINNED_V4_CONTRACT_VERSION = "4.34.0"
+PINNED_V4_OPENAPI_SOURCE_SHA256 = "001a5e132e8af9872ac61168e41df4465627f34ac062066b212f1b48a88385f4"
+PINNED_V4_CONTRACT_VERSION = "4.35.0"
 
 #: The ``schema_version`` every command body that predates 4.14.0 stamps: ``dataset activity``,
 #: ``table rename``, ``table archive`` and ``dataset archive``. It is the number the 0.25 line

{mostlyright_data-0.30.0 → mostlyright_data-0.30.1}/src/mostlyright/data_harness/thin/v4_search.py

@@ -197,6 +197,7 @@ def search_exit_code(payload: dict[str, Any]) -> int:
         "queued",
         "dispatched",
         "running",
+        "deferred",
         "pending",
         "cancellation_requested",
     }:
@@ -217,6 +218,12 @@ def search_exit_code(payload: dict[str, Any]) -> int:
 
 
 SEARCH_JOB_SCHEMAS = {"table-search-job.v1", "table-search-job.v2"}
+#: Every job Studio's job read answers. The whole-version (v1) and sharded (v2) parents carry
+#: their schema; the title job carries v3; a package, embed or evaluation child carries none.
+JOB_STATUS_SCHEMAS = {*SEARCH_JOB_SCHEMAS, "table-search-job.v3", None}
+JOB_STATUSES = frozenset(
+    {"queued", "dispatched", "running", "deferred", "succeeded", "failed", "cancelled"}
+)
 
 RECIPE_INSPECTION_PATH = "/v4/search/recipe-inspection"
 TABLE_SEARCH_PATH = "/v4/tables/{table_id}/search"
@@ -715,26 +722,60 @@ def _try(
     return _payload("try", result)
 
 
-def _locator(row_id: int, counts: list[int]) -> list[int]:
-    start = 0
-    for part, count in enumerate(counts):
-        if row_id < start + count:
-            return [part, row_id - start]
-        start += count
-    raise ThinLaneError("THIN_RESPONSE_INVALID", "a search row lies outside the sealed version")
+#: Rows per page of a locate read. Cloud returns each row's physical locator only on a compact
+#: page, and a compact page holds at most 50 rows.
+LOCATE_PAGE_ROWS = 50
+#: Pages one locate target may read before it reports the rows it found as ambiguous.
+MAX_LOCATE_PAGES = 20
+
+
+def _public_table(public_table_id: str, public: Any) -> dict[str, Any]:
+    """The public schema's table: which Studio version it serves, and whether it is queryable."""
+
+    from mostlyright.data_harness.thin.v4_public import PublicCatalogClient, resolve_public_origin
+
+    selected = public or PublicCatalogClient(resolve_public_origin())
+    body = selected.get_table_schema(public_table_id)
+    table = body.get("table") if isinstance(body, dict) else None
+    if not isinstance(table, dict):
+        raise ThinLaneError("THIN_RESPONSE_INVALID", "Cloud returned a schema without a table")
+    return table
 
 
 class _Locator:
-    """Resolves rows of the published live version to evaluation-case locators."""
+    """Resolves rows of the published live version to evaluation-case locators.
+
+    It reads through Cloud's public table query, filtered on the title or identifier column, so
+    it needs no search package: cases are written before the version has one. Each row comes
+    back with its physical ``[part, row]`` locator. Every read is pinned to the version the first
+    one served, and that version must be the one Studio reports live."""
 
     def __init__(self, args, client, public, query_client) -> None:
         self.table_id = identifier(_required(args.target, "a table identifier"), "table identifier")
         self.public_table_id = identifier(args.public_table_id, "public table ID")
         self.studio = _studio(client)
-        self.search, metadata = _bind_public(
-            self.public_table_id, public, self.studio, self.table_id
-        )
-        self.version = metadata["table_version_id"]
+        table = _public_table(self.public_table_id, public)
+        metadata = _metadata(self.studio, self.table_id, None)
+        self.version = metadata.get("table_version_id")
+        capabilities = table.get("capabilities") or {}
+        search = capabilities.get("search") if isinstance(capabilities, dict) else None
+        served = {table.get("version_id")}
+        if isinstance(search, dict):
+            served.add(search.get("table_version_id"))
+        if not self.version or served != {self.version}:
+            raise ThinLaneError(
+                "THIN_SEARCH_VERSION_MISMATCH",
+                f"public table {self.public_table_id} serves version "
+                f"{table.get('version_id')}, but table {self.table_id} has live version "
+                f"{self.version}. Check --public-table with mr-data public tables DATASET_SLUG. "
+                "The public schema can lag a new live version by a few minutes, so retry then.",
+            )
+        if isinstance(capabilities, dict) and capabilities.get("query") is False:
+            raise ThinLaneError(
+                "THIN_SEARCH_CAPABILITY_UNAVAILABLE",
+                f"public table {self.public_table_id} does not serve queries, so locate cannot "
+                "read its rows. Turn on its preview in the dataset's publication settings.",
+            )
         if (
             args.table_version_id is not None
             and identifier(args.table_version_id, "table version") != self.version
@@ -751,22 +792,84 @@ class _Locator:
         self.id_column = args.id_column or roles.get("identifier")
         self.counts = _manifest_record_counts(self.studio, self.table_id, self.version)
         self.query_client = query_client
-
-    def _rows(self, text: str, plan: dict[str, Any], columns: list[str]) -> list[dict[str, Any]]:
-        answer, _ = _cloud_query(
-            self.public_table_id,
-            _resident_body(_query_text(text), self.search, columns, MAX_TRY_ROWS, plan),
-            self.query_client,
-        )
-        if answer.get("path") != "resident" or answer.get("outcome") not in (None, "ok"):
+        self.pin: tuple[str, str] | None = None
+
+    def _bound(self, answer: dict[str, Any]) -> None:
+        served = answer.get("table")
+        if not isinstance(served, dict):
+            raise ThinLaneError("THIN_RESPONSE_INVALID", "Cloud returned rows without a table")
+        coordinates = (served.get("version_id"), served.get("content_digest"))
+        if not all(isinstance(value, str) and value for value in coordinates):
+            raise ThinLaneError("THIN_RESPONSE_INVALID", "Cloud returned rows without a version")
+        live = served.get("live_version_id")
+        rows = served.get("row_count")
+        if (
+            coordinates[0] != self.version
+            or live not in (None, self.version)
+            or (rows is not None and rows != sum(self.counts))
+            or (self.pin is not None and coordinates != self.pin)
+        ):
             raise ThinLaneError(
-                "THIN_SEARCH_CAPABILITY_UNAVAILABLE",
-                "the live search package did not answer ("
-                + str(answer.get("note") or answer.get("outcome") or "no package")
-                + f"). locate reads {', '.join(columns)}. Each must be in the spec's "
-                "display_columns, and the package must be ready in mr-data search status.",
+                "THIN_SEARCH_VERSION_MISMATCH",
+                f"public table {self.public_table_id} stopped serving live version "
+                f"{self.version} while locate read it. Run locate again.",
             )
-        return [row for row in answer.get("rows", []) if isinstance(row, dict)]
+        self.pin = coordinates
+
+    def _rows(self, filters: list[dict[str, Any]], columns: list[str]) -> tuple[list, bool]:
+        """Every row matching ``filters``, each with its locator, and whether pages remain."""
+
+        found: list[dict[str, Any]] = []
+        cursor: str | None = None
+        for _ in range(MAX_LOCATE_PAGES):
+            # A cursor is honoured only beside the query it was issued for, and it carries its
+            # own version pin.
+            body: dict[str, Any] = {
+                "columns": columns,
+                "filters": filters,
+                "limit": LOCATE_PAGE_ROWS,
+                "compact_cells": True,
+            }
+            if cursor is not None:
+                body["cursor"] = cursor
+            elif self.pin is not None:
+                body.update(version_id=self.pin[0], content_digest=self.pin[1])
+            answer, _ = _cloud_query(self.public_table_id, body, self.query_client)
+            self._bound(answer)
+            rows, refs = answer.get("rows"), answer.get("row_refs")
+            if (
+                not isinstance(rows, list)
+                or not isinstance(refs, list)
+                or len(rows) != len(refs)
+                or len(rows) > LOCATE_PAGE_ROWS
+            ):
+                raise ThinLaneError(
+                    "THIN_RESPONSE_INVALID", "Cloud returned rows without their locators"
+                )
+            for row, ref in zip(rows, refs, strict=True):
+                part = ref.get("part_index") if isinstance(ref, dict) else None
+                index = ref.get("row_index") if isinstance(ref, dict) else None
+                if (
+                    not isinstance(row, dict)
+                    or type(part) is not int
+                    or type(index) is not int
+                    or not 0 <= part < len(self.counts)
+                    or not 0 <= index < self.counts[part]
+                ):
+                    raise ThinLaneError(
+                        "THIN_RESPONSE_INVALID", "a located row lies outside the sealed version"
+                    )
+                found.append(
+                    {
+                        "row_id": sum(self.counts[:part]) + index,
+                        "locator": [part, index],
+                        **row,
+                    }
+                )
+            cursor = answer.get("next_cursor")
+            if not cursor:
+                return found, False
+        return found, True
 
     def resolve(self, target: Any) -> dict[str, Any]:
         """One draft target to its matches: ``{"title", "year"}``, ``{"id"}`` or a locator."""
@@ -796,23 +899,9 @@ class _Locator:
             value = target["id"]
             if self.types.get(column) == "integer" and isinstance(value, str) and value.isdigit():
                 value = int(value)
-            plan = {
-                "schema_version": PLAN_SCHEMA,
-                "kind": "browse",
-                "text": "",
-                "predicates": [
-                    {"column": column, "operator": "eq", "value": value, "origin": "explicit"}
-                ],
-                "prefer": [],
-                "order": [],
-                "scope": "matching_rows",
-                "unresolved": [],
-            }
-            rows = [
-                row
-                for row in self._rows(str(target["id"]), plan, [column])
-                if str((row.get("fields") or {}).get(column)) == str(value)
-            ]
+            matches, more = self._rows(
+                [{"column": column, "operator": "eq", "value": value}], [column]
+            )
         else:
             column = _required(self.title_column, "--title-column (no title field role)")
             title = str(target["title"])
@@ -821,37 +910,18 @@ class _Locator:
             if year is not None:
                 date = _required(self.date_column, "--date-column (no date field role)")
                 columns.append(date)
-            plan = {
-                "schema_version": PLAN_SCHEMA,
-                "kind": "title",
-                "text": title,
-                "predicates": [],
-                "prefer": [],
-                "order": [],
-                "scope": "relevance_window",
-                "unresolved": [],
-            }
-            rows = [
-                row
-                for row in self._rows(title, plan, list(dict.fromkeys(columns)))
-                if _norm((row.get("fields") or {}).get(column)) == _norm(title)
-            ]
+            matches, more = self._rows(
+                [{"column": column, "operator": "eq", "value": title}],
+                list(dict.fromkeys(columns)),
+            )
             if year is not None:
-                rows = [
-                    row
-                    for row in rows
-                    if str((row.get("fields") or {}).get(date) or "").startswith(str(year))
+                matches = [
+                    row for row in matches if str(row.get(date) or "").startswith(str(year))
                 ]
-        matches = [
-            {
-                "row_id": row["row_id"],
-                "locator": _locator(row["row_id"], self.counts),
-                **row["fields"],
-            }
-            for row in rows
-            if type(row.get("row_id")) is int and isinstance(row.get("fields"), dict)
-        ]
-        status = "located" if len(matches) == 1 else "not_found" if not matches else "ambiguous"
+        if len(matches) == 1 and not more:
+            status = "located"
+        else:
+            status = "not_found" if not matches and not more else "ambiguous"
         return {
             "target": target,
             "status": status,
@@ -1057,9 +1127,8 @@ def search(
     if job_operation:
         if (
             answer.get("job_id") != job_id
-            or answer.get("status")
-            not in {"queued", "dispatched", "running", "succeeded", "failed", "cancelled"}
-            or (verb == "status" and answer.get("schema_version") not in SEARCH_JOB_SCHEMAS)
+            or answer.get("status") not in JOB_STATUSES
+            or (verb == "status" and answer.get("schema_version") not in JOB_STATUS_SCHEMAS)
         ):
             raise ThinLaneError(
                 "THIN_RESPONSE_INVALID", "Studio returned invalid search job coordinates"