mostlyright-data 0.29.8__tar.gz → 0.29.10__tar.gz

{mostlyright_data-0.29.8 → mostlyright_data-0.29.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mostlyright-data
-Version: 0.29.8
+Version: 0.29.10
 Summary: Mostly Right hosted CLI for reviewed datasets
 Project-URL: Homepage, https://mostlyright.md/
 Project-URL: Documentation, https://mostlyright.md/docs/guides/cli/

{mostlyright_data-0.29.8 → mostlyright_data-0.29.10}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "mostlyright-data"
-version = "0.29.8"
+version = "0.29.10"
 description = "Mostly Right hosted CLI for reviewed datasets"
 readme = "README.md"
 requires-python = ">=3.11"

{mostlyright_data-0.29.8 → mostlyright_data-0.29.10}/skills/mr-data-build/references/preparing-table-search.md

@@ -15,8 +15,9 @@ mr-data search inspect --recipe recipe.json --json
 
 Local CLI help describes installed commands; it does not prove the deployed server serves them.
 For the recipe-target probe above, read JSON refusals from stderr: `schema_version` is
-`mr-data-error.v1`, `code` is the typed discriminator, and `error` is the detail. There is no
-public `http_status` field in that envelope. Treat `code: THIN_NOT_FOUND` as absent search
+`mr-data-error.v1`, `code` is the typed discriminator, and `error` is the detail. Cloud refusals
+also expose `http_status`, `request_id`, `detail` and `retry_after_seconds` when supplied;
+Studio inspection refusals retain their typed code. Treat `code: THIN_NOT_FOUND` as absent search
 registration support only when `error` begins exactly with
 `Studio has no such resource for /v4/search/recipe-inspection`; the CLI maps this route's HTTP 404
 that way. An explicit `code: THIN_SEARCH_CAPABILITY_UNAVAILABLE` also states a capability limit.
@@ -141,3 +142,28 @@ an exact filter.
 
 This workflow opts in newly authored recipes only. It does not migrate old datasets, reauthor users'
 existing recipes, launch a full data rebuild or imply successful indexing of every table.
+
+## Diagnose preparation and query a published table
+
+`preparation_state: unavailable` can mean a never-built table, a failed or cancelled job,
+or an incompatible specification. Keep `job_id` from the search build receipt and run
+`mr-data search status --job JOB_ID --json` to inspect the job's status and failure.
+Studio is adding `preparation_detail`; print it when present rather than inferring a reason
+from `unavailable`. Table status passes this additive field through unchanged.
+
+For a published table, first run `mr-data public schema TABLE_ID --json`.
+The table schema's `capabilities.search` contains `table_version_id`, `spec_digest`,
+`keyword_columns`, `semantic_columns`, and optional stored `lexical`/`semantic` preparation values.
+Use `keyword_columns` for `search.columns`; top-level `columns` is the non-empty output projection.
+The following lexical example uses a syntactically valid illustrative digest:
+
+```sh
+mr-data public query TABLE_ID --body '{"columns":["title"],"search":{"text":"example","columns":["title"],"spec_digest":"sha256:0000000000000000000000000000000000000000000000000000000000000000","mode":"lexical"},"limit":10}' --json
+```
+
+Replace the illustrative columns and digest with the actual schema values. Hybrid mode additionally
+needs `generation_digest` (`sha256:` plus 64 lowercase hex digits) from readiness/explain;
+it is not in `capabilities.search`. Omit it for this lexical example. Public query prints
+Cloud stage durations in milliseconds under `result.timings` when Server-Timing is present,
+and reports HTTP status, request id and retry timing. Keep refusal `detail` with the request id.
+Missing timing metadata does not establish that a stage did no work.

{mostlyright_data-0.29.8 → mostlyright_data-0.29.10}/src/mostlyright/data_harness/thin/recipe.py

@@ -80,11 +80,9 @@ SHOW_SCHEMA = f"{THIN_SCHEMA_PREFIX}-recipe-read.v1"
 READINESS_SCHEMA = f"{THIN_SCHEMA_PREFIX}-recipe-readiness.v1"
 PROPOSE_SCHEMA = f"{THIN_SCHEMA_PREFIX}-recipe-propose.v1"
 
-#: The bound on the document. The same number the authoring lane bounded one of its six documents
-#: by -- four times the contract's own 262144-byte ceiling on a canonical recipe -- rather than a
-#: second number chosen here, because this document is those six documents in one and the ceiling
-#: it has to clear is the canonical form's.
-MAX_DOCUMENT_BYTES = 4 * 262_144
+#: A plain input file may occupy up to Studio's 2 MiB body ceiling. Before submission,
+#: the complete canonical UTF-8 request envelope must also fit that ceiling.
+MAX_DOCUMENT_BYTES = v4.MAX_RECIPE_REQUEST_BYTES
 
 #: ``recipe.schema.json#/$defs/recipe_document``'s required members, in the order the schema lists
 #: them. Checked here before the network, for the reason every early refusal in this client
@@ -1187,6 +1185,9 @@ def register(
     """
 
     document = read_document(Path(args.target))
+    # UUIDs have fixed wire length. Refuse an oversized envelope before credentials,
+    # capability reads or delegation writes; the actual client checks it again.
+    v4.recipe_registration_body(document, "00000000-0000-0000-0000-000000000000")
     # New grammar must be refused as unserved before local lint can diagnose its shape.
     keyless = any(
         capability == "keyless_window" for _, capability in recipe_explain.gated_grammar(document)

{mostlyright_data-0.29.8 → mostlyright_data-0.29.10}/src/mostlyright/data_harness/thin/recipe_explain.py

@@ -39,6 +39,7 @@ from mostlyright.data_harness.thin.transport import ThinLaneError
 from mostlyright.data_harness.thin.v4 import (
     EXPLAIN_AND_FORECAST_REQUEST_VERSION,
     StudioV4Client,
+    check_recipe_request_size,
     identifier,
 )
 
@@ -1231,17 +1232,18 @@ def explain_document_at_studio(
     nothing, which is what makes it safe to ask before a registration.
     """
 
+    body = {
+        "schema_version": ROUTES_REQUEST_VERSION,
+        "workspace_id": str(client.session.workspace_id),
+        "document": dict(document),
+    }
+    check_recipe_request_size(body)
     try:
-        # The one inherited `_call` (thin/v4.py): the route rides the same transport, byte
-        # ceiling and refusal translation as every other V4 call.
+        # The one inherited `_call` carries the ordinary response ceiling.
         answer = client._call(
             "POST",
             EXPLAIN_REFRESH_PATH,
-            body={
-                "schema_version": ROUTES_REQUEST_VERSION,
-                "workspace_id": str(client.session.workspace_id),
-                "document": dict(document),
-            },
+            body=body,
             expected=(200,),
         )
     except ThinLaneError as refusal:

{mostlyright_data-0.29.8 → mostlyright_data-0.29.10}/src/mostlyright/data_harness/thin/router.py

@@ -888,7 +888,10 @@ def _hosted(command: str, arguments: Sequence[str], *, as_json: bool) -> int:
             as_json=as_json or parsed.json,
             retry_after_seconds=getattr(refusal, "retry_after_seconds", None),
             details=_refusal_details(refusal),
-            extra={"sessions": sessions} if sessions else None,
+            extra={
+                **getattr(refusal, "cloud_metadata", {}),
+                **({"sessions": sessions} if sessions else {}),
+            },
         )
     except Exception as error:
         code = getattr(error, "code", None)

{mostlyright_data-0.29.8 → mostlyright_data-0.29.10}/src/mostlyright/data_harness/thin/session.py

@@ -17,6 +17,7 @@ recorded production failures on 2026-08-20.
 from __future__ import annotations
 
 import json
+import math
 import re
 import time
 from collections.abc import Callable, Mapping
@@ -523,8 +524,59 @@ def stated_sentence(value: Any) -> str | None:
     return stated
 
 
+def _cloud_sentence(value: Any) -> str | None:
+    stated = stated_sentence(value)
+    if stated is None:
+        return None
+    # Refusal prose must never echo bearer/device tokens or URLs (including signed URLs).
+    stated = re.sub(r"https?://\S+", "[redacted URL]", stated, flags=re.IGNORECASE)
+    stated = re.sub(r"mr_(?:cli|use)_[A-Za-z0-9_-]+", "[redacted token]", stated)
+    stated = re.sub(r"(?i)bearer\s+\S+", "Bearer [redacted token]", stated)
+    return stated
+
+
+def cloud_response_metadata(status: int, parsed: Any, headers: Mapping[str, str]) -> dict[str, Any]:
+    """Only bounded diagnostic fields, never arbitrary response headers or authority."""
+    normalized = {key.lower(): value for key, value in headers.items()}
+    metadata: dict[str, Any] = {"http_status": status}
+    for request_id in (
+        normalized.get("x-request-id"),
+        parsed.get("request_id") if isinstance(parsed, dict) else None,
+    ):
+        if (
+            isinstance(request_id, str)
+            and re.fullmatch(r"[A-Za-z0-9_.:-]{1,128}", request_id)
+            and not request_id.startswith("mr_")
+        ):
+            metadata["request_id"] = request_id
+            break
+    raw_retry = normalized.get("retry-after", "")
+    if isinstance(raw_retry, str) and re.fullmatch(r"[0-9]{1,4}", raw_retry.strip()):
+        seconds = int(raw_retry.strip())
+        if 0 <= seconds <= 3600:
+            metadata["retry_after_seconds"] = seconds
+    timings: dict[str, float] = {}
+    for metric in normalized.get("server-timing", "")[:8192].split(",")[:64]:
+        match = re.fullmatch(
+            r"\s*([A-Za-z][A-Za-z0-9_-]{0,63})\s*;\s*dur=([0-9]+(?:\.[0-9]+)?)(?:\s*;.*)?\s*",
+            metric,
+        )
+        if match:
+            duration = float(match[2])
+            if math.isfinite(duration):
+                timings[match[1]] = duration
+    if timings:
+        metadata["timings"] = timings
+    return metadata
+
+
 def cloud_refusal(
-    status: int, parsed: Any, *, act: str, absent: str | None = None
+    status: int,
+    parsed: Any,
+    *,
+    act: str,
+    absent: str | None = None,
+    headers: Mapping[str, str] | None = None,
 ) -> ThinLaneError:
     """One Cloud refusal, folded the way ``runs._studio_refusal`` folds Studio's.
 
@@ -564,7 +616,7 @@ def cloud_refusal(
     stated = parsed.get("code") if isinstance(parsed, dict) else None
     named = stated if isinstance(stated, str) and CLOUD_REFUSAL_CODE.fullmatch(stated) else None
     said = (
-        stated_sentence(parsed.get("error")) if named is None and isinstance(parsed, dict) else None
+        _cloud_sentence(parsed.get("error")) if named is None and isinstance(parsed, dict) else None
     )
     if status in {401, 403}:
         if named:
@@ -588,7 +640,16 @@ def cloud_refusal(
         refusal = ThinLaneError(
             "THIN_CLOUD_REFUSED", f"{detail}. Cloud said: {said}" if said else detail
         )
-    refusal.http_status = status  # type: ignore[attr-defined]
+    detail = _cloud_sentence(parsed.get("detail")) if isinstance(parsed, dict) else None
+    if detail:
+        refusal.detail += f". Cloud detail: {detail}"
+        refusal.args = (f"{refusal.code}: {refusal.detail}",)
+    refusal.cloud_metadata = cloud_response_metadata(status, parsed, headers or {})
+    if detail:
+        refusal.cloud_metadata["detail"] = detail
+    for field, value in refusal.cloud_metadata.items():
+        if field != "detail":
+            setattr(refusal, field, value)
     return refusal
 
 
@@ -613,6 +674,7 @@ class CloudClient:
         self._credentials = credentials
         self._origin = service_origin(credentials.cloud_url, "Cloud URL", allow_loopback_http=False)
         self._transport = transport or UrlLibTransport()
+        self.response_metadata: dict[str, Any] = {}
 
     @property
     def cloud_url(self) -> str:
@@ -661,7 +723,7 @@ class CloudClient:
         if body is not None:
             headers["Content-Type"] = "application/json"
             encoded = json.dumps(dict(body), separators=(",", ":")).encode("utf-8")
-        status, raw, _headers = self._transport.request(
+        status, raw, response_headers = self._transport.request(
             method,
             f"{self._origin}{path}",
             headers,
@@ -676,8 +738,9 @@ class CloudClient:
             )
         except CanonicalJSONError:
             parsed = None
+        self.response_metadata = cloud_response_metadata(status, parsed, response_headers)
         if status not in expected:
-            raise cloud_refusal(status, parsed, act=act, absent=absent)
+            raise cloud_refusal(status, parsed, act=act, absent=absent, headers=response_headers)
         if not isinstance(parsed, dict):
             raise ThinLaneError(
                 "THIN_RESPONSE_INVALID",

{mostlyright_data-0.29.8 → mostlyright_data-0.29.10}/src/mostlyright/data_harness/thin/v4.py

@@ -39,6 +39,7 @@ from collections.abc import Mapping
 from typing import Any
 from uuid import UUID
 
+from mostlyright.data_harness.canonical import canonical_json_bytes
 from mostlyright.data_harness.thin.runs import StudioApiClient
 from mostlyright.data_harness.thin.transport import ThinLaneError
 
@@ -915,6 +916,35 @@ def identifier(value: str, field: str) -> str:
         raise ThinLaneError("THIN_REQUEST_INVALID", f"{value!r} is not a {field}") from error
 
 
+# Studio's API request-body middleware accepts at most 2 MiB. Bound the complete
+# UTF-8 wire envelope, not the recipe's character count or its formatted file size.
+MAX_RECIPE_REQUEST_BYTES = 2 * 1024 * 1024
+# A GET recipe includes the document plus identifiers and registration metadata.
+# Give that envelope 1 MiB headroom; ordinary JSON routes retain their 1 MiB cap.
+MAX_RECIPE_RESPONSE_BYTES = MAX_RECIPE_REQUEST_BYTES + 1024 * 1024
+
+
+def check_recipe_request_size(body: Mapping[str, Any]) -> None:
+    size = len(canonical_json_bytes(body))
+    if size > MAX_RECIPE_REQUEST_BYTES:
+        raise ThinLaneError(
+            "THIN_RECIPE_REQUEST_TOO_LARGE",
+            f"the serialized recipe request is {size} bytes; Studio accepts at most "
+            f"{MAX_RECIPE_REQUEST_BYTES} bytes including its request envelope. "
+            "Reduce the recipe before submitting it",
+        )
+
+
+def recipe_registration_body(document: Mapping[str, Any], workspace_id: str) -> dict[str, Any]:
+    body = {
+        "workspace_id": workspace_id,
+        "document": dict(document),
+        "require_category": True,
+    }
+    check_recipe_request_size(body)
+    return body
+
+
 class StudioV4Client(StudioApiClient):
     """The V4 routes, over the transport and the hardening :class:`StudioApiClient` already has.
 
@@ -993,11 +1023,7 @@ class StudioV4Client(StudioApiClient):
         return self._call(
             "POST",
             CREATE_RECIPE_PATH,
-            body={
-                "workspace_id": str(self.session.workspace_id),
-                "document": dict(document),
-                "require_category": True,
-            },
+            body=recipe_registration_body(document, str(self.session.workspace_id)),
             expected=(201,),
         )
 
@@ -1008,6 +1034,7 @@ class StudioV4Client(StudioApiClient):
             "GET",
             GET_RECIPE_PATH.format(recipe_id=identifier(recipe_id, "recipe identifier")),
             expected=(200,),
+            maximum=MAX_RECIPE_RESPONSE_BYTES,
         )
 
     def recipe_readiness(
@@ -1049,14 +1076,16 @@ class StudioV4Client(StudioApiClient):
     def explain_recipe_refresh(self, document: Mapping[str, Any]) -> dict[str, Any]:
         """Judge one unregistered recipe document for refresh, and read what this Studio serves."""
 
+        body = {
+            "schema_version": EXPLAIN_AND_FORECAST_REQUEST_VERSION,
+            "workspace_id": str(self.session.workspace_id),
+            "document": dict(document),
+        }
+        check_recipe_request_size(body)
         return self._call(
             "POST",
             EXPLAIN_RECIPE_REFRESH_PATH,
-            body={
-                "schema_version": EXPLAIN_AND_FORECAST_REQUEST_VERSION,
-                "workspace_id": str(self.session.workspace_id),
-                "document": dict(document),
-            },
+            body=body,
             expected=(200,),
         )
 

{mostlyright_data-0.29.8 → mostlyright_data-0.29.10}/src/mostlyright/data_harness/thin/v4_public.py

@@ -39,6 +39,7 @@ from mostlyright.data_harness.thin.session import (
     MAX_TOKEN_RESPONSE_BYTES,
     CloudClient,
     cloud_refusal,
+    cloud_response_metadata,
     require_device_key,
     stated_sentence,
 )
@@ -193,7 +194,6 @@ def declare_arguments(parser: argparse.ArgumentParser) -> None:
         ),
     )
 
-
     parser.add_argument(
         "--body",
         default=None,
@@ -287,8 +287,10 @@ class PublicCatalogClient:
                 )
             error = parsed.get("error")
             detail = parsed.get("detail") if isinstance(parsed.get("detail"), str) else None
-            code = f"THIN_CLOUD_{error}".upper() if isinstance(error, str) and error else (
-                "THIN_CLOUD_REFUSED"
+            code = (
+                f"THIN_CLOUD_{error}".upper()
+                if isinstance(error, str) and error
+                else ("THIN_CLOUD_REFUSED")
             )
             message = detail or f"Cloud refused to {act} (HTTP {status})"
             refusal = ThinLaneError(code, message)
@@ -471,9 +473,7 @@ def _search_hit(dataset: Mapping[str, Any]) -> dict[str, Any]:
         "id": dataset.get("id"),
         "slug": dataset.get("slug"),
         "title": dataset.get("title"),
-        "summary": (
-            f"{summary[: SEARCH_SUMMARY_LIMIT - 1].rstrip()}…" if cut else summary
-        ),
+        "summary": (f"{summary[: SEARCH_SUMMARY_LIMIT - 1].rstrip()}…" if cut else summary),
         "topics": list(dataset.get("topics") or []),
         "publisher": _publisher(publisher),
         "published_at": dataset.get("published_at"),
@@ -861,7 +861,6 @@ def _fetch(args: argparse.Namespace, client: PublicCatalogClient) -> dict[str, A
     }
 
 
-
 CLOUD_PUBLIC_TABLE_QUERY_PATH = "/api/cli/public/tables/{table_id}/query"
 CLOUD_PUBLIC_TABLE_DOWNLOAD_URL_PATH = "/api/cli/public/tables/{table_id}/download-url"
 QUERY_SCHEMA = f"{THIN_SCHEMA_PREFIX}-v4-public-query.v1"
@@ -885,8 +884,15 @@ MAX_PUBLIC_QUERY_RESPONSE_MEMBERS = (
 class PublicQueryClient(CloudClient):
     """Authenticated public table query + download-url over the device-key client."""
 
+    def __init__(self, credentials, *, transport=None) -> None:
+        # Let Cloud's gateway deadline return its own refusal and request id first.
+        super().__init__(credentials, transport=transport)
+        self._query_client = CloudClient(
+            credentials, transport=transport or UrlLibTransport(timeout=150.0)
+        )
+
     def query_table(self, table_id: str, body: Mapping[str, Any]) -> dict[str, Any]:
-        return self.call(
+        answer = self._query_client.call(
             "POST",
             CLOUD_PUBLIC_TABLE_QUERY_PATH.format(table_id=table_id),
             act="query a public table",
@@ -897,6 +903,9 @@ class PublicQueryClient(CloudClient):
             max_members=MAX_PUBLIC_QUERY_RESPONSE_MEMBERS,
         )
 
+        self.response_metadata = self._query_client.response_metadata
+        return answer
+
     def download_url(self, table_id: str) -> dict[str, Any]:
         """Read download instructions; keep download_not_enabled distinct from missing table.
 
@@ -911,7 +920,7 @@ class PublicQueryClient(CloudClient):
             "Accept": "application/json",
             "Authorization": f"Bearer {self._credentials.raw_key}",
         }
-        status, raw, _headers = self._transport.request(
+        status, raw, response_headers = self._transport.request(
             "GET",
             f"{self.cloud_url}{path}",
             headers,
@@ -941,6 +950,7 @@ class PublicQueryClient(CloudClient):
                 "THIN_CLOUD_DOWNLOAD_NOT_ENABLED",
                 f"Cloud refused to {act} (download_not_enabled). Cloud said: {said}",
             )
+            refusal.cloud_metadata = cloud_response_metadata(404, parsed, response_headers)
             refusal.http_status = 404  # type: ignore[attr-defined]
             raise refusal
         raise cloud_refusal(
@@ -948,6 +958,7 @@ class PublicQueryClient(CloudClient):
             parsed,
             act=act,
             absent="no public table has that id",
+            headers=response_headers,
         )
 
 
@@ -993,12 +1004,16 @@ def _query(args: argparse.Namespace, client: Any = None) -> dict[str, Any]:
     body = _read_query_body(args)
     selected = _auth_query_client(client if isinstance(client, PublicQueryClient) else None)
     answer = selected.query_table(table_id, body)
+    metadata = getattr(selected, "response_metadata", {})
+    if "timings" in metadata:
+        answer = {**answer, "timings": metadata["timings"]}
     return {
         "schema_version": QUERY_SCHEMA,
         "status": "public_table_queried",
         "lane": "hosted",
         "table_id": table_id,
         "result": answer,
+        **{key: value for key, value in metadata.items() if key != "timings"},
         "next_step": _NEXT["public_table_queried"],
     }
 
@@ -1061,16 +1076,20 @@ def public(
                 "THIN_REQUEST_INVALID",
                 f"--cursor applies to public search, not public {action}",
             )
-    if action in {
-        "show",
-        "tables",
-        "schema",
-        "sample",
-        "fetch",
-        "lookup",
-        "query",
-        "download-url",
-    } and not args.target:
+    if (
+        action
+        in {
+            "show",
+            "tables",
+            "schema",
+            "sample",
+            "fetch",
+            "lookup",
+            "query",
+            "download-url",
+        }
+        and not args.target
+    ):
         needed = {
             "show": "a dataset slug",
             "tables": "a dataset slug",

{mostlyright_data-0.29.8 → mostlyright_data-0.29.10}/src/mostlyright/data_harness/thin/v4_query.py

@@ -54,7 +54,6 @@ from mostlyright.data_harness.thin.session import (
 )
 from mostlyright.data_harness.thin.transport import ThinLaneError, UrlLibTransport
 from mostlyright.data_harness.thin.v4 import (
-    GET_RECIPE_PATH,
     GET_RUN_PATH,
     GET_RUN_QUERY_PATH,
     RUN_QUERY_PATH,
@@ -278,7 +277,7 @@ class StudioV4QueryClient(StudioV4Client):
     def recipe_source_ids(self, recipe_id: str) -> list[str]:
         """The sources a recipe registered, which a session opened for its dataset may probe."""
 
-        record = self._call("GET", GET_RECIPE_PATH.format(recipe_id=recipe_id), expected=(200,))
+        record = self.recipe(recipe_id)
         listed = record.get("source_ids")
         sources = (
             [item for item in listed if isinstance(item, str) and item]