mostlyright-data 0.31.0__tar.gz → 0.31.1__tar.gz

{mostlyright_data-0.31.0 → mostlyright_data-0.31.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mostlyright-data
-Version: 0.31.0
+Version: 0.31.1
 Summary: Mostly Right hosted CLI for reviewed datasets
 Project-URL: Homepage, https://mostlyright.md/
 Project-URL: Documentation, https://mostlyright.md/docs/guides/cli/

{mostlyright_data-0.31.0 → mostlyright_data-0.31.1}/skills/mr-data-build/references/5-draft-one-recipe-document-one-call.md

@@ -141,6 +141,7 @@ The allowed IDs and labels are:
 | `people-society` | People & Society |
 | `health-medicine` | Health & Medicine |
 | `science-research` | Science & Research |
+| `ai-models` | AI & Models |
 | `technology` | Technology |
 | `energy` | Energy |
 | `agriculture-food` | Agriculture & Food |

{mostlyright_data-0.31.0 → mostlyright_data-0.31.1}/skills/mr-data-build/references/enriching-with-small-models.md

@@ -26,6 +26,16 @@ The person pays nothing for these calls. They count against the workspace's dail
 `mr-data recipe forecast` reports. Prefer `classify` over `openrouter-decisions` for new work:
 Mostly Right pays OpenRouter per Jev call, and Clef answers the same request body.
 
+Each binding also comes with a `pages_template`: `request_timeout_seconds` 120 and `retries` 3.
+Write both into the collection's `pages`. Clef and the extraction model scale to zero when idle,
+and their first instance takes three to five minutes to load. Studio starts loading it when the
+run is admitted, and until the service has answered once it spreads each member's attempts over
+six minutes rather than seconds. The longer timeout and the fourth attempt let the last one land
+after a slow start. Size `limits.max_requests` at `pages.max_fetches_per_run` times four. A run
+that still fails says `it never answered, likely a 3 to 5 minute cold start` in its
+`ROW_REQUEST_UNAVAILABLE` detail: tell the person the model was starting up and run it again, which keeps every saved
+answer.
+
 A task listed `not_served` or `planned` is offered as coming, never written into a recipe. For
 `match`, join on a normalized deterministic key now and say that model matching for the rows the
 key cannot settle is coming.

{mostlyright_data-0.31.0 → mostlyright_data-0.31.1}/skills/mr-data-build/references/sources.md

@@ -291,7 +291,11 @@ The worker retries 408, 429 and every 5xx but 501 and 505. It also retries when
 arrived: a transport, resolver or TLS failure, or a response refused before its status was
 checked, such as a compressed body or a malformed header, whatever its status. It sends a request
 at most `pages.retries` more times, then fails the run with `ROW_REQUEST_UNAVAILABLE`. A
-`Retry-After` longer than 60 seconds fails the run at once, and more retries cannot help it. When
+`Retry-After` longer than 60 seconds fails the run at once, and more retries cannot help it. A
+platform credential's source that calls this deployment's own model service (Clef, the
+extraction model) waits longer before that service has answered once: a failure
+with no answer, or a 502, 503 or 504 without `Retry-After`, is that service's cold start, and the
+attempts left are spread so the last goes out six minutes after the first. When
 the request presents a key, a checked 401 or 403 stops the source with
 `ROW_REQUEST_CREDENTIAL_REFUSED`, and a checked 402 with `ROW_REQUEST_PAYMENT_REQUIRED`. Any
 other answer the upstream gave fails that one member, and the run goes on. That covers another

{mostlyright_data-0.31.0 → mostlyright_data-0.31.1}/src/mostlyright/data_harness/thin/dataset-categories-v1.json

@@ -13,7 +13,7 @@
       "id": "crypto",
       "label": "Crypto",
       "definition": "Cryptocurrencies, blockchain networks, DeFi and digital-asset markets.",
-      "exclusions": "Use Finance for conventional securities, banking and non-crypto prediction markets.",
+      "exclusions": "Use Finance for conventional securities, banking and other non-crypto financial markets.",
       "examples": "Bitcoin prices; Ethereum transactions; DeFi lending"
     },
     {
@@ -21,7 +21,7 @@
       "label": "Finance",
       "definition": "Banking, securities, investment, insurance and financial-market activity.",
       "exclusions": "Crypto assets belong to Crypto; economy-wide indicators belong to Economics & Business.",
-      "examples": "Stock prices; interest-rate products; prediction-market order books"
+      "examples": "Stock prices; interest-rate products; exchange trade records"
     },
     {
       "id": "economics-business",
@@ -58,12 +58,19 @@
       "exclusions": "Use the subject-specific category for weather, health, technology or other covered research.",
       "examples": "Astronomical observations; particle experiments; research publications"
     },
+    {
+      "id": "ai-models",
+      "label": "AI & Models",
+      "definition": "AI systems, specialized and frontier models, evaluations, training-run metadata, agent traces and model-related corpora.",
+      "exclusions": "General software, internet and telecommunications measurements belong to Technology. A dataset about a non-AI subject stays in that subject even when a model was used to build it.",
+      "examples": "Frontier-model benchmark scores; model cards; agent-workflow traces"
+    },
     {
       "id": "technology",
       "label": "Technology",
-      "definition": "Software, computing, AI, internet and telecommunications.",
-      "exclusions": "Blockchain belongs to Crypto; media consumption and gaming belong to Arts, Media & Entertainment.",
-      "examples": "Software packages; internet performance; AI benchmarks"
+      "definition": "Software, computing, internet and telecommunications.",
+      "exclusions": "AI systems, models, evaluations and agent traces belong to AI & Models. Blockchain belongs to Crypto; media consumption and gaming belong to Arts, Media & Entertainment.",
+      "examples": "Software packages; internet performance; package registries"
     },
     {
       "id": "energy",

{mostlyright_data-0.31.0 → mostlyright_data-0.31.1}/src/mostlyright/data_harness/thin/enrichment.py

@@ -135,6 +135,13 @@ def binding_task(entry: Mapping[str, Any]) -> str:
     return "other"
 
 
+#: The ``pages`` members a model source states. A self-hosted model scales to zero and its first
+#: instance loads for three to five minutes; Studio waits out that cold start between a member's
+#: attempts, and a longer request timeout and a fourth attempt let the last one land after it.
+#: ``limits.max_requests`` must then cover ``pages.max_fetches_per_run`` times four.
+MODEL_PAGES_TEMPLATE: Mapping[str, int] = {"request_timeout_seconds": 120, "retries": 3}
+
+
 def request_template(task: str, entry: Mapping[str, Any]) -> dict[str, Any] | None:
     """A per-row ``request`` to start from, with this deployment's route and pin filled in."""
 
@@ -199,6 +206,8 @@ def models_listing(listing: Mapping[str, Any] | None) -> dict[str, Any]:
             else:
                 row["for_recipes"] = True
                 row["request_template"] = request_template(name, entry)
+                if row["request_template"] is not None:
+                    row["pages_template"] = dict(MODEL_PAGES_TEMPLATE)
             served.append(row)
         order = TASK_BINDINGS[name]
         served.sort(
@@ -656,7 +665,8 @@ def models(args: argparse.Namespace, *, client: Any | None = None) -> dict[str,
             "reference": REFERENCE,
             "next_step": (
                 "Write an available binding's request_template into a per-row source, keeping "
-                "origin, method, path and model exactly as listed. Run mr-data models suggest "
+                "origin, method, path and model exactly as listed, and its pages_template into "
+                "the collection's pages. Run mr-data models suggest "
                 "RUN_ID after a run to see where the table could use one."
             ),
         }

{mostlyright_data-0.31.0 → mostlyright_data-0.31.1}/src/mostlyright/data_harness/thin/recipe.py

@@ -169,7 +169,7 @@ FILE_DECODER_COORDINATES: frozenset[tuple[str, str]] = frozenset(
     (family, version)
     for family, versions in {
         "archive.gzip": ("1.0.0", "1.1.0", "2.0.0"),
-        "archive.tar": ("1.0.0", "1.1.0", "3.0.0"),
+        "archive.tar": ("1.0.0", "1.1.0", "3.0.0", "4.0.0"),
         "archive.zip": ("1.0.0", "1.1.0", "1.2.0", "1.3.0", "1.4.0", "1.5.0", "3.0.0"),
         "delimited_text": ("1.0.0", "1.1.0", "2.0.0"),
         "geoparquet.vector": ("1.0.0",),

{mostlyright_data-0.31.0 → mostlyright_data-0.31.1}/src/mostlyright/data_harness/thin/table_search_contract.json

@@ -1,7 +1,7 @@
 {
  "schema_version": "mostlyright-thin-table-search-contract.v1",
- "source_commit": "115eb05cd9356ebadc7090e47481f69df257570d",
- "recipe_search_definition": "spec",
+ "source_commit": "27b7e13f5d15471c6386f8797586c1cf332dd6a1",
+ "recipe_search_definition": "recipe_search",
  "$defs": {
   "evaluation_case": {
    "type": "object",
@@ -107,6 +107,21 @@
     "cases"
    ]
   },
+  "recipe_search": {
+   "allOf": [
+    {
+     "$ref": "#/$defs/spec"
+    },
+    {
+     "not": {
+      "required": [
+       "long_text_columns"
+      ]
+     },
+     "description": "Keel's recipe reader does not admit long_text_columns yet (4.42.0): a search revision declares it."
+    }
+   ]
+  },
   "spec": {
    "type": "object",
    "additionalProperties": false,
@@ -418,6 +433,18 @@
       "minLength": 1,
       "maxLength": 128
      }
+    },
+    "long_text_columns": {
+     "description": "Long string columns (4.42.0) whose text is searched by meaning, Mojirama's subtitles: plot synopses, descriptions, dialogue. Each row's text is cut into chunks, embedded and averaged into one vector per row in the package's `media.long_text`, and a plan's `long_text` ranks with it. Declared columns of type string; a public table only (Studio declares no media for another). Admitted on a search revision; a recipe's `table.search` may not name it yet.",
+     "type": "array",
+     "minItems": 1,
+     "maxItems": 4,
+     "uniqueItems": true,
+     "items": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 128
+     }
     }
    },
    "required": [

{mostlyright_data-0.31.0 → mostlyright_data-0.31.1}/src/mostlyright/data_harness/thin/v4.py

@@ -149,8 +149,8 @@ from mostlyright.data_harness.thin.v4_matching import CONTRACT_PATHS as MATCHING
 #: ``JOB_INVALID`` outright, which in production was every refresh a collection epoch was offered
 #: for rather than only the collection runs. This package must not reach a Studio older than the
 #: commit it pins; ``docs/V4-WORKER-PROTOCOL.md`` states it beside the layout's own ordering rule.
-PINNED_V4_OPENAPI_SOURCE_SHA256 = "94fec61ae25722c5a96670bb7b2786103ec7808e1dbae1866a0db5a886beb13b"
-PINNED_V4_CONTRACT_VERSION = "4.41.0"
+PINNED_V4_OPENAPI_SOURCE_SHA256 = "7fc74682ecae99966e007fd97631786ec10e32c3916946b692d4bd90c904d097"
+PINNED_V4_CONTRACT_VERSION = "4.42.0"
 
 #: The ``schema_version`` every command body that predates 4.14.0 stamps: ``dataset activity``,
 #: ``table rename``, ``table archive`` and ``dataset archive``. It is the number the 0.25 line

{mostlyright_data-0.31.0 → mostlyright_data-0.31.1}/src/mostlyright/data_harness/thin/v4_runs.py

@@ -164,6 +164,11 @@ ROOM_FAULT_DETAILS: frozenset[str] = frozenset({"SANDBOX_MEMORY_BOUNDARY"})
 #: ``limits.max_source_bytes`` and this run clamp. A clamp at or above 4 GiB therefore cannot
 #: narrow any source, whatever the recipe declares and however many sources it names.
 #:
+#: Under the ``bounded-streaming.v1`` execution policy the worker starts that minimum at the run's
+#: capacity grant instead, which may exceed this floor. The clamp stays inert there because every
+#: source must declare ``limits.max_source_bytes`` and :func:`retry_clamps` raises the clamp to at
+#: least the declared total, which is at least each source's own declaration.
+#:
 #: ⚠ IT WAS 256 TIMES THIS, AND THAT WAS A REFUSAL RATHER THAN A SAFETY. Multiplying by the 256
 #: sources a recipe may declare read the composition as a sum when it is a per-source minimum, so
 #: a run bounded at, say, 8 GiB -- a ceiling that narrows nothing, since 4 GiB binds first -- was

{mostlyright_data-0.31.0 → mostlyright_data-0.31.1}/src/mostlyright/data_harness/thin/v4_search_contract.py

@@ -31,24 +31,36 @@ def _ref_name(ref: str) -> str:
     return ref[len(prefix) :]
 
 
-def _recipe_search_definition(recipe: dict[str, Any]) -> str:
-    """The table-search definition the recipe's ``table.search`` member names."""
+def _recipe_search_definition(recipe: dict[str, Any]) -> dict[str, Any]:
+    """Preserve recipe-only restrictions while localizing its search-schema references."""
 
-    search = recipe["$defs"]["table_plan"]["properties"]["search"]
-    ref = search.get("$ref", "")
     marker = "table-search.schema.json#/$defs/"
-    if not ref.startswith(marker):
-        raise ValueError("the recipe search member does not reference the table search schema")
-    return ref[len(marker) :]
+
+    def localize(value: Any) -> Any:
+        if isinstance(value, list):
+            return [localize(item) for item in value]
+        if not isinstance(value, dict):
+            return value
+        result = {}
+        for key, item in value.items():
+            if key == "$ref":
+                if not isinstance(item, str) or not item.startswith(marker):
+                    raise ValueError("unsupported recipe search reference")
+                result[key] = "#/$defs/" + item[len(marker) :]
+            else:
+                result[key] = localize(item)
+        return result
+
+    return localize(recipe["$defs"]["table_plan"]["properties"]["search"])
 
 
 def build_bundle(table_search: dict[str, Any], recipe: dict[str, Any], pin: dict[str, Any]) -> dict:
     """The closure of the definitions the client checks, from the vendored documents."""
 
     definitions = table_search["$defs"]
-    recipe_member = _recipe_search_definition(recipe)
+    recipe_member = "recipe_search"
     wanted = [SPEC, recipe_member, CASES]
-    selected: dict[str, Any] = {}
+    selected: dict[str, Any] = {recipe_member: _recipe_search_definition(recipe)}
 
     def collect(value: Any) -> None:
         if isinstance(value, dict):
@@ -64,6 +76,7 @@ def build_bundle(table_search: dict[str, Any], recipe: dict[str, Any], pin: dict
             for item in value:
                 collect(item)
 
+    collect(selected[recipe_member])
     for name in wanted:
         if name not in selected:
             selected[name] = definitions[name]
@@ -123,6 +136,7 @@ KEYWORDS = frozenset(
         "else",
         "anyOf",
         "oneOf",
+        "not",
         "description",
     }
 )
@@ -230,6 +244,11 @@ def _check(schema: Any, value: Any, at: str, definitions: dict, out: list) -> No
             _check(branch, value, at, definitions, out)
     for part in schema.get("allOf", []):
         _check(part, value, at, definitions, out)
+    if "not" in schema:
+        probe: list = []
+        _check(schema["not"], value, at, definitions, probe)
+        if not probe:
+            out.append({"pointer": at or "/", "reason": "matches a forbidden form (not)"})
     for member, exactly_one in (("anyOf", False), ("oneOf", True)):
         options = schema.get(member)
         if options: