mostlyright-data 0.31.0__tar.gz → 0.31.2__tar.gz

{mostlyright_data-0.31.0 → mostlyright_data-0.31.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mostlyright-data
-Version: 0.31.0
+Version: 0.31.2
 Summary: Mostly Right hosted CLI for reviewed datasets
 Project-URL: Homepage, https://mostlyright.md/
 Project-URL: Documentation, https://mostlyright.md/docs/guides/cli/

{mostlyright_data-0.31.0 → mostlyright_data-0.31.2}/skills/mr-data-build/references/5-draft-one-recipe-document-one-call.md

@@ -141,6 +141,7 @@ The allowed IDs and labels are:
 | `people-society` | People & Society |
 | `health-medicine` | Health & Medicine |
 | `science-research` | Science & Research |
+| `ai-models` | AI & Models |
 | `technology` | Technology |
 | `energy` | Energy |
 | `agriculture-food` | Agriculture & Food |

{mostlyright_data-0.31.0 → mostlyright_data-0.31.2}/skills/mr-data-build/references/enriching-with-small-models.md

@@ -26,6 +26,16 @@ The person pays nothing for these calls. They count against the workspace's dail
 `mr-data recipe forecast` reports. Prefer `classify` over `openrouter-decisions` for new work:
 Mostly Right pays OpenRouter per Jev call, and Clef answers the same request body.
 
+Each binding also comes with a `pages_template`: `request_timeout_seconds` 120 and `retries` 3.
+Write both into the collection's `pages`. Clef and the extraction model scale to zero when idle,
+and their first instance takes three to five minutes to load. Studio starts loading it when the
+run is admitted, and until the service has answered once it spreads each member's attempts over
+six minutes rather than seconds. The longer timeout and the fourth attempt let the last one land
+after a slow start. Size `limits.max_requests` at `pages.max_fetches_per_run` times four. A run
+that still fails says `it never answered, likely a 3 to 5 minute cold start` in its
+`ROW_REQUEST_UNAVAILABLE` detail: tell the person the model was starting up and run it again, which keeps every saved
+answer.
+
 A task listed `not_served` or `planned` is offered as coming, never written into a recipe. For
 `match`, join on a normalized deterministic key now and say that model matching for the rows the
 key cannot settle is coming.

{mostlyright_data-0.31.0 → mostlyright_data-0.31.2}/skills/mr-data-build/references/sources.md

@@ -291,7 +291,11 @@ The worker retries 408, 429 and every 5xx but 501 and 505. It also retries when
 arrived: a transport, resolver or TLS failure, or a response refused before its status was
 checked, such as a compressed body or a malformed header, whatever its status. It sends a request
 at most `pages.retries` more times, then fails the run with `ROW_REQUEST_UNAVAILABLE`. A
-`Retry-After` longer than 60 seconds fails the run at once, and more retries cannot help it. When
+`Retry-After` longer than 60 seconds fails the run at once, and more retries cannot help it. A
+platform credential's source that calls this deployment's own model service (Clef, the
+extraction model) waits longer before that service has answered once: a failure
+with no answer, or a 502, 503 or 504 without `Retry-After`, is that service's cold start, and the
+attempts left are spread so the last goes out six minutes after the first. When
 the request presents a key, a checked 401 or 403 stops the source with
 `ROW_REQUEST_CREDENTIAL_REFUSED`, and a checked 402 with `ROW_REQUEST_PAYMENT_REQUIRED`. Any
 other answer the upstream gave fails that one member, and the run goes on. That covers another

{mostlyright_data-0.31.0 → mostlyright_data-0.31.2}/src/mostlyright/data_harness/thin/dataset-categories-v1.json

@@ -13,7 +13,7 @@
       "id": "crypto",
       "label": "Crypto",
       "definition": "Cryptocurrencies, blockchain networks, DeFi and digital-asset markets.",
-      "exclusions": "Use Finance for conventional securities, banking and non-crypto prediction markets.",
+      "exclusions": "Use Finance for conventional securities, banking and other non-crypto financial markets.",
       "examples": "Bitcoin prices; Ethereum transactions; DeFi lending"
     },
     {
@@ -21,7 +21,7 @@
       "label": "Finance",
       "definition": "Banking, securities, investment, insurance and financial-market activity.",
       "exclusions": "Crypto assets belong to Crypto; economy-wide indicators belong to Economics & Business.",
-      "examples": "Stock prices; interest-rate products; prediction-market order books"
+      "examples": "Stock prices; interest-rate products; exchange trade records"
     },
     {
       "id": "economics-business",
@@ -58,12 +58,19 @@
       "exclusions": "Use the subject-specific category for weather, health, technology or other covered research.",
       "examples": "Astronomical observations; particle experiments; research publications"
     },
+    {
+      "id": "ai-models",
+      "label": "AI & Models",
+      "definition": "AI systems, specialized and frontier models, evaluations, training-run metadata, agent traces and model-related corpora.",
+      "exclusions": "General software, internet and telecommunications measurements belong to Technology. A dataset about a non-AI subject stays in that subject even when a model was used to build it.",
+      "examples": "Frontier-model benchmark scores; model cards; agent-workflow traces"
+    },
     {
       "id": "technology",
       "label": "Technology",
-      "definition": "Software, computing, AI, internet and telecommunications.",
-      "exclusions": "Blockchain belongs to Crypto; media consumption and gaming belong to Arts, Media & Entertainment.",
-      "examples": "Software packages; internet performance; AI benchmarks"
+      "definition": "Software, computing, internet and telecommunications.",
+      "exclusions": "AI systems, models, evaluations and agent traces belong to AI & Models. Blockchain belongs to Crypto; media consumption and gaming belong to Arts, Media & Entertainment.",
+      "examples": "Software packages; internet performance; package registries"
     },
     {
       "id": "energy",

{mostlyright_data-0.31.0 → mostlyright_data-0.31.2}/src/mostlyright/data_harness/thin/enrichment.py

@@ -135,6 +135,13 @@ def binding_task(entry: Mapping[str, Any]) -> str:
     return "other"
 
 
+#: The ``pages`` members a model source states. A self-hosted model scales to zero and its first
+#: instance loads for three to five minutes; Studio waits out that cold start between a member's
+#: attempts, and a longer request timeout and a fourth attempt let the last one land after it.
+#: ``limits.max_requests`` must then cover ``pages.max_fetches_per_run`` times four.
+MODEL_PAGES_TEMPLATE: Mapping[str, int] = {"request_timeout_seconds": 120, "retries": 3}
+
+
 def request_template(task: str, entry: Mapping[str, Any]) -> dict[str, Any] | None:
     """A per-row ``request`` to start from, with this deployment's route and pin filled in."""
 
@@ -199,6 +206,8 @@ def models_listing(listing: Mapping[str, Any] | None) -> dict[str, Any]:
             else:
                 row["for_recipes"] = True
                 row["request_template"] = request_template(name, entry)
+                if row["request_template"] is not None:
+                    row["pages_template"] = dict(MODEL_PAGES_TEMPLATE)
             served.append(row)
         order = TASK_BINDINGS[name]
         served.sort(
@@ -656,7 +665,8 @@ def models(args: argparse.Namespace, *, client: Any | None = None) -> dict[str,
             "reference": REFERENCE,
             "next_step": (
                 "Write an available binding's request_template into a per-row source, keeping "
-                "origin, method, path and model exactly as listed. Run mr-data models suggest "
+                "origin, method, path and model exactly as listed, and its pages_template into "
+                "the collection's pages. Run mr-data models suggest "
                 "RUN_ID after a run to see where the table could use one."
             ),
         }

{mostlyright_data-0.31.0 → mostlyright_data-0.31.2}/src/mostlyright/data_harness/thin/recipe.py

@@ -169,7 +169,7 @@ FILE_DECODER_COORDINATES: frozenset[tuple[str, str]] = frozenset(
     (family, version)
     for family, versions in {
         "archive.gzip": ("1.0.0", "1.1.0", "2.0.0"),
-        "archive.tar": ("1.0.0", "1.1.0", "3.0.0"),
+        "archive.tar": ("1.0.0", "1.1.0", "3.0.0", "4.0.0"),
         "archive.zip": ("1.0.0", "1.1.0", "1.2.0", "1.3.0", "1.4.0", "1.5.0", "3.0.0"),
         "delimited_text": ("1.0.0", "1.1.0", "2.0.0"),
         "geoparquet.vector": ("1.0.0",),

{mostlyright_data-0.31.0 → mostlyright_data-0.31.2}/src/mostlyright/data_harness/thin/table_search_contract.json

@@ -1,7 +1,7 @@
 {
  "schema_version": "mostlyright-thin-table-search-contract.v1",
- "source_commit": "115eb05cd9356ebadc7090e47481f69df257570d",
- "recipe_search_definition": "spec",
+ "source_commit": "27b7e13f5d15471c6386f8797586c1cf332dd6a1",
+ "recipe_search_definition": "recipe_search",
  "$defs": {
   "evaluation_case": {
    "type": "object",
@@ -107,6 +107,21 @@
     "cases"
    ]
   },
+  "recipe_search": {
+   "allOf": [
+    {
+     "$ref": "#/$defs/spec"
+    },
+    {
+     "not": {
+      "required": [
+       "long_text_columns"
+      ]
+     },
+     "description": "Keel's recipe reader does not admit long_text_columns yet (4.42.0): a search revision declares it."
+    }
+   ]
+  },
   "spec": {
    "type": "object",
    "additionalProperties": false,
@@ -418,6 +433,18 @@
       "minLength": 1,
       "maxLength": 128
      }
+    },
+    "long_text_columns": {
+     "description": "Long string columns (4.42.0) whose text is searched by meaning, Mojirama's subtitles: plot synopses, descriptions, dialogue. Each row's text is cut into chunks, embedded and averaged into one vector per row in the package's `media.long_text`, and a plan's `long_text` ranks with it. Declared columns of type string; a public table only (Studio declares no media for another). Admitted on a search revision; a recipe's `table.search` may not name it yet.",
+     "type": "array",
+     "minItems": 1,
+     "maxItems": 4,
+     "uniqueItems": true,
+     "items": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 128
+     }
     }
    },
    "required": [

{mostlyright_data-0.31.0 → mostlyright_data-0.31.2}/src/mostlyright/data_harness/thin/transport.py

@@ -10,8 +10,12 @@ to relax any of it.
 
 from __future__ import annotations
 
+import http.client
 import ipaddress
+import math
 import ssl
+import threading
+import time
 from collections.abc import Callable, Iterator, Mapping
 from contextlib import contextmanager
 from typing import Any, Protocol
@@ -119,8 +123,9 @@ class UrlLibTransport:
 
     def __init__(self, *, timeout: float | Callable[[], float] = REQUEST_TIMEOUT_SECONDS) -> None:
         self._timeout = timeout if callable(timeout) else float(timeout)
+        self._context = ssl.create_default_context()
         self._opener = build_opener(
-            ProxyHandler({}), HTTPSHandler(context=ssl.create_default_context()), _NoRedirect()
+            ProxyHandler({}), HTTPSHandler(context=self._context), _NoRedirect()
         )
 
     def _clock(self) -> float:
@@ -171,6 +176,128 @@ class UrlLibTransport:
             response.close()
 
 
+class ReusableHttpsTransport(UrlLibTransport):
+    """One serialized, verified HTTPS connection for repeated query requests.
+
+    urllib deliberately sends Connection: close. A query client reuses its
+    own connection instead, never a global credential-bearing pool. Changing
+    origin closes the old socket. Responses are bounded and fully framed;
+    any failure discards the socket and is raised without replaying a POST.
+    Streams retain the original urllib/proxy-free/redirect-free behavior.
+    """
+
+    def __init__(self, *, timeout: float | Callable[[], float] = REQUEST_TIMEOUT_SECONDS) -> None:
+        super().__init__(timeout=timeout)
+        self._lock = threading.Lock()
+        self._connection: http.client.HTTPSConnection | None = None
+        self._connection_origin: tuple[str, str, int] | None = None
+
+    def _discard(self) -> None:
+        connection, self._connection = self._connection, None
+        self._connection_origin = None
+        if connection is not None:
+            connection.close()
+
+    def close(self) -> None:
+        with self._lock:
+            self._discard()
+
+    def __enter__(self) -> ReusableHttpsTransport:
+        return self
+
+    def __exit__(self, *_args: Any) -> None:
+        self.close()
+
+    def __del__(self) -> None:
+        connection = getattr(self, "_connection", None)
+        if connection is not None:
+            connection.close()
+
+    def request(
+        self, method: str, url: str, headers: Mapping[str, str], body: bytes | None, maximum: int
+    ) -> tuple[int, bytes, Mapping[str, str]]:
+        if type(maximum) is not int or maximum < 0:
+            raise ThinLaneError("THIN_REQUEST_INVALID", "the response ceiling must be nonnegative")
+        parts = urlsplit(url)
+        service_origin(
+            f"{parts.scheme}://{parts.netloc}", "request origin", allow_loopback_http=False
+        )
+        if parts.scheme != "https" or parts.fragment or parts.hostname is None:
+            raise ThinLaneError(
+                "THIN_REQUEST_INVALID", "a reusable request requires a valid HTTPS URL"
+            )
+        origin = (parts.scheme, parts.hostname.lower(), parts.port or 443)
+        if any(
+            name.lower() == "host" and value.lower() != parts.netloc.lower()
+            for name, value in headers.items()
+        ):
+            raise ThinLaneError(
+                "THIN_REQUEST_INVALID", "the Host header differs from the request origin"
+            )
+        budget = self._clock()
+        if not math.isfinite(budget) or budget <= 0:
+            raise TimeoutError("the request deadline has elapsed")
+        deadline = time.monotonic() + budget
+        if not self._lock.acquire(timeout=budget):
+            raise TimeoutError("the request deadline elapsed waiting for its connection")
+
+        def remaining() -> float:
+            value = deadline - time.monotonic()
+            if value <= 0:
+                raise TimeoutError("the request deadline has elapsed")
+            return value
+
+        try:
+            if self._connection_origin != origin:
+                self._discard()
+            if self._connection is None:
+                self._connection = http.client.HTTPSConnection(
+                    origin[1], origin[2], timeout=remaining(), context=self._context
+                )
+                self._connection_origin = origin
+            connection = self._connection
+            connection.timeout = remaining()
+            if connection.sock is not None:
+                connection.sock.settimeout(remaining())
+            path = parts.path or "/"
+            if parts.query:
+                path += "?" + parts.query
+            connection.request(method, path, body=body, headers=dict(with_user_agent(headers)))
+            if connection.sock is not None:
+                connection.sock.settimeout(remaining())
+            response = connection.getresponse()
+            expected = response.length
+            chunks: list[bytes] = []
+            length = 0
+            while length <= maximum:
+                if connection.sock is not None:
+                    connection.sock.settimeout(remaining())
+                chunk = response.read1(min(64 * 1024, maximum + 1 - length))
+                if not chunk:
+                    break
+                chunks.append(chunk)
+                length += len(chunk)
+            raw = b"".join(chunks)
+            if length > maximum:
+                raise ThinLaneError(
+                    "THIN_RESPONSE_TOO_LARGE",
+                    "the response exceeded the byte ceiling for its route",
+                )
+            if expected is not None and length != expected:
+                raise http.client.IncompleteRead(raw, expected - length)
+            status = int(response.status)
+            response_headers = {key.lower(): value for key, value in response.headers.items()}
+            response.close()
+            if response.will_close or connection.sock is None or status < 200:
+                self._discard()
+            return status, raw, response_headers
+        except BaseException:
+            self._discard()
+            raise
+        finally:
+            self._lock.release()
+
+
 def service_origin(value: str, label: str, *, allow_loopback_http: bool) -> str:
     """Return ``value`` when it is a bare HTTPS origin, and refuse it otherwise.
 

{mostlyright_data-0.31.0 → mostlyright_data-0.31.2}/src/mostlyright/data_harness/thin/v4.py

@@ -149,8 +149,8 @@ from mostlyright.data_harness.thin.v4_matching import CONTRACT_PATHS as MATCHING
 #: ``JOB_INVALID`` outright, which in production was every refresh a collection epoch was offered
 #: for rather than only the collection runs. This package must not reach a Studio older than the
 #: commit it pins; ``docs/V4-WORKER-PROTOCOL.md`` states it beside the layout's own ordering rule.
-PINNED_V4_OPENAPI_SOURCE_SHA256 = "94fec61ae25722c5a96670bb7b2786103ec7808e1dbae1866a0db5a886beb13b"
-PINNED_V4_CONTRACT_VERSION = "4.41.0"
+PINNED_V4_OPENAPI_SOURCE_SHA256 = "7fc74682ecae99966e007fd97631786ec10e32c3916946b692d4bd90c904d097"
+PINNED_V4_CONTRACT_VERSION = "4.42.0"
 
 #: The ``schema_version`` every command body that predates 4.14.0 stamps: ``dataset activity``,
 #: ``table rename``, ``table archive`` and ``dataset archive``. It is the number the 0.25 line

{mostlyright_data-0.31.0 → mostlyright_data-0.31.2}/src/mostlyright/data_harness/thin/v4_public.py

@@ -46,6 +46,7 @@ from mostlyright.data_harness.thin.session import (
 )
 from mostlyright.data_harness.thin.transport import (
     MAX_JSON_RESPONSE_BYTES,
+    ReusableHttpsTransport,
     ThinLaneError,
     Transport,
     UrlLibTransport,
@@ -896,10 +897,25 @@ class PublicQueryClient(CloudClient):
     def __init__(self, credentials, *, transport=None) -> None:
         # Let Cloud's gateway deadline return its own refusal and request id first.
         super().__init__(credentials, transport=transport)
+        self._owned_query_transport = (
+            ReusableHttpsTransport(timeout=150.0) if transport is None else None
+        )
         self._query_client = CloudClient(
-            credentials, transport=transport or UrlLibTransport(timeout=150.0)
+            credentials,
+            transport=transport if transport is not None else self._owned_query_transport,
         )
 
+    def close(self) -> None:
+        """Close this client's owned query connection; injected transports stay caller-owned."""
+        if self._owned_query_transport is not None:
+            self._owned_query_transport.close()
+
+    def __enter__(self) -> PublicQueryClient:
+        return self
+
+    def __exit__(self, *_args: Any) -> None:
+        self.close()
+
     def query_table(self, table_id: str, body: Mapping[str, Any]) -> dict[str, Any]:
         answer = self._query_client.call(
             "POST",

{mostlyright_data-0.31.0 → mostlyright_data-0.31.2}/src/mostlyright/data_harness/thin/v4_runs.py

@@ -164,6 +164,11 @@ ROOM_FAULT_DETAILS: frozenset[str] = frozenset({"SANDBOX_MEMORY_BOUNDARY"})
 #: ``limits.max_source_bytes`` and this run clamp. A clamp at or above 4 GiB therefore cannot
 #: narrow any source, whatever the recipe declares and however many sources it names.
 #:
+#: Under the ``bounded-streaming.v1`` execution policy the worker starts that minimum at the run's
+#: capacity grant instead, which may exceed this floor. The clamp stays inert there because every
+#: source must declare ``limits.max_source_bytes`` and :func:`retry_clamps` raises the clamp to at
+#: least the declared total, which is at least each source's own declaration.
+#:
 #: ⚠ IT WAS 256 TIMES THIS, AND THAT WAS A REFUSAL RATHER THAN A SAFETY. Multiplying by the 256
 #: sources a recipe may declare read the composition as a sum when it is a per-source minimum, so
 #: a run bounded at, say, 8 GiB -- a ceiling that narrows nothing, since 4 GiB binds first -- was

{mostlyright_data-0.31.0 → mostlyright_data-0.31.2}/src/mostlyright/data_harness/thin/v4_search_contract.py

@@ -31,24 +31,36 @@ def _ref_name(ref: str) -> str:
     return ref[len(prefix) :]
 
 
-def _recipe_search_definition(recipe: dict[str, Any]) -> str:
-    """The table-search definition the recipe's ``table.search`` member names."""
+def _recipe_search_definition(recipe: dict[str, Any]) -> dict[str, Any]:
+    """Preserve recipe-only restrictions while localizing its search-schema references."""
 
-    search = recipe["$defs"]["table_plan"]["properties"]["search"]
-    ref = search.get("$ref", "")
     marker = "table-search.schema.json#/$defs/"
-    if not ref.startswith(marker):
-        raise ValueError("the recipe search member does not reference the table search schema")
-    return ref[len(marker) :]
+
+    def localize(value: Any) -> Any:
+        if isinstance(value, list):
+            return [localize(item) for item in value]
+        if not isinstance(value, dict):
+            return value
+        result = {}
+        for key, item in value.items():
+            if key == "$ref":
+                if not isinstance(item, str) or not item.startswith(marker):
+                    raise ValueError("unsupported recipe search reference")
+                result[key] = "#/$defs/" + item[len(marker) :]
+            else:
+                result[key] = localize(item)
+        return result
+
+    return localize(recipe["$defs"]["table_plan"]["properties"]["search"])
 
 
 def build_bundle(table_search: dict[str, Any], recipe: dict[str, Any], pin: dict[str, Any]) -> dict:
     """The closure of the definitions the client checks, from the vendored documents."""
 
     definitions = table_search["$defs"]
-    recipe_member = _recipe_search_definition(recipe)
+    recipe_member = "recipe_search"
     wanted = [SPEC, recipe_member, CASES]
-    selected: dict[str, Any] = {}
+    selected: dict[str, Any] = {recipe_member: _recipe_search_definition(recipe)}
 
     def collect(value: Any) -> None:
         if isinstance(value, dict):
@@ -64,6 +76,7 @@ def build_bundle(table_search: dict[str, Any], recipe: dict[str, Any], pin: dict
             for item in value:
                 collect(item)
 
+    collect(selected[recipe_member])
     for name in wanted:
         if name not in selected:
             selected[name] = definitions[name]
@@ -123,6 +136,7 @@ KEYWORDS = frozenset(
         "else",
         "anyOf",
         "oneOf",
+        "not",
         "description",
     }
 )
@@ -230,6 +244,11 @@ def _check(schema: Any, value: Any, at: str, definitions: dict, out: list) -> No
             _check(branch, value, at, definitions, out)
     for part in schema.get("allOf", []):
         _check(part, value, at, definitions, out)
+    if "not" in schema:
+        probe: list = []
+        _check(schema["not"], value, at, definitions, probe)
+        if not probe:
+            out.append({"pointer": at or "/", "reason": "matches a forbidden form (not)"})
     for member, exactly_one in (("anyOf", False), ("oneOf", True)):
         options = schema.get(member)
         if options: