whileai 0.52__tar.gz → 0.53__tar.gz

{whileai-0.52/whileai.egg-info → whileai-0.53}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: whileai
-Version: 0.52
+Version: 0.53
 Summary: While Python SDK: trace ingestion and verification client, plus agent simulations grounded in your tools and system prompt.
 Author: While
 License: Apache-2.0

{whileai-0.52 → whileai-0.53}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "whileai"
-version = "0.52"
+version = "0.53"
 description = "While Python SDK: trace ingestion and verification client, plus agent simulations grounded in your tools and system prompt."
 readme = "README.md"
 license = {text = "Apache-2.0"}

{whileai-0.52 → whileai-0.53}/whileai/simulations/score/passat.py

@@ -195,6 +195,25 @@ class PassAt:
         return f"{head} {tail})"
 
 
+def _nothing_to_score(rows: Sequence[dict]) -> str:
+    """Why no row carried a binary reward.
+
+    "grade first" is right when nothing has been judged, and wrong -- it
+    sends the user back to the step that already ran -- when grading did
+    happen and every row failed. A cold hosted judge does exactly that: all
+    the concurrent calls time out together and the whole set reads as
+    ungraded.
+    """
+    judged = [r for r in rows if isinstance(r, dict) and r.get("judge_status")]
+    failed = [r for r in judged if str(r.get("judge_status")) != "ok"]
+    if not judged or len(failed) != len(judged):
+        return "no binary rewards; grade first"
+    statuses = "/".join(sorted({str(r.get("judge_status")) for r in failed}))
+    reason = next((str(r.get("reason") or "").strip() for r in failed if r.get("reason")), "")
+    tail = f": {reason[:120]}" if reason else ""
+    return f"the judge failed on all {len(failed)} rows ({statuses}){tail}; re-run the judge"
+
+
 def pass_at(
     rows: Sequence[dict] | Any,
     *,
@@ -249,7 +268,7 @@ def pass_at(
             pass_at_k=None,
             n_groups=0,
             n_rows=0,
-            note="no binary rewards; grade first",
+            note=_nothing_to_score(row_list),
             config=run_config(row_list, n_tasks=0, k=int(k or 1)),
         )
 

{whileai-0.52 → whileai-0.53}/whileai/simulations/score/privileged.py

@@ -63,9 +63,15 @@ def leak_report(rows: Sequence[dict], *, min_len: int = 12) -> dict[str, Any]:
     the block, so the result is vacuous), ``leaked`` (up to 20 rows:
     ``scenario_id``, ``rollout_index``, ``field``, ``needle``) and
     ``summary``. Does not mutate ``rows``.
+
+    Pass ``data.trajectories``, not ``data.rows()``: the export scrubs
+    ``privileged`` at any depth, so exported rows carry nothing to check
+    and the report is vacuous. When it can tell the rows came through the
+    export, ``summary`` says so and names the accessor to use instead.
     """
     n_rows = 0
     n_checked = 0
+    exported = False
     leaked: list[dict[str, Any]] = []
     for row in rows:
         if not isinstance(row, dict):
@@ -73,6 +79,10 @@ def leak_report(rows: Sequence[dict], *, min_len: int = 12) -> dict[str, Any]:
         n_rows += 1
         needles = _needles(row.get("privileged"), min_len=min_len)
         if not needles:
+            # ``export_row`` always writes ``scenario_id`` (``""`` when the
+            # row has none), so a row that has the key but no privileged
+            # block was scrubbed on the way out rather than never filled.
+            exported = exported or "scenario_id" in row
             continue
         n_checked += 1
         hay = _norm(assistant_text(row))
@@ -94,6 +104,11 @@ def leak_report(rows: Sequence[dict], *, min_len: int = 12) -> dict[str, Any]:
             f"checked 0 of {n_rows} rows: none carried privileged context, "
             "so this says nothing about leaks"
         )
+        if exported:
+            summary += (
+                ". These came through the export (rows(), save(), push()), which scrubs "
+                "privileged at any depth -- pass data.trajectories, which keeps it"
+            )
     elif not n_leaked:
         summary = f"checked {n_checked} of {n_rows} rows: no reply quoted its privileged context"
     else:

{whileai-0.52 → whileai-0.53}/whileai/simulations/score/rubric.py

@@ -26,12 +26,19 @@ import concurrent.futures
 import hashlib
 import json
 import re
+import threading
 from collections.abc import Callable, Mapping, Sequence
 from dataclasses import dataclass, field
 from typing import Any, Literal
 
 from ..generate.agents import complete, parse_backend_spec
-from .grade_llm import JUDGE_TEMPERATURE, _render_payload, judge_spec, judge_version
+from .grade_llm import (
+    JUDGE_TEMPERATURE,
+    _render_payload,
+    judge_spec,
+    judge_version,
+    warm_judge,
+)
 
 Kind = Literal["hard", "principle", "pitfall"]
 KINDS: tuple[str, ...] = ("hard", "principle", "pitfall")
@@ -375,10 +382,28 @@ def rubric_judge(
     ``markers`` (``rubric:<slug>`` = 1.0 met / 0.0 not, and for a pitfall
     1.0 clean / 0.0 exhibited), ``criteria`` (the raw verdicts),
     ``rubric_version`` and the score breakdown. The judge's name folds the
-    rubric version in when one is fixed."""
+    rubric version in when one is fixed.
+
+    The hosted judge scales to zero, so the first row through warms it once
+    (``warm_judge``, a 600s budget) while the rest of the fan-out waits.
+    Without that, ``run_judge``'s eight concurrent calls all raced a
+    container that was still loading its weights and every row came back
+    ``invalid_result`` with a ``TimeoutError``. Warm-up failure is not
+    fatal: the rows are judged anyway and report the real error."""
     resolved = judge_spec(spec=spec)
     url, model = parse_backend_spec(resolved)
     system = str(prompt or "").strip() or RUBRIC_JUDGE_SYSTEM
+    warm_lock = threading.Lock()
+    warmed: list[dict] = []
+
+    def ensure_warm() -> None:
+        # once per judge, and the other workers block here rather than
+        # opening their own request against a cold server
+        if warmed:
+            return
+        with warm_lock:
+            if not warmed:
+                warmed.append(warm_judge(resolved, api_key=api_key))
 
     def judge(row: dict) -> dict[str, Any]:
         use = rubric or rubric_of(row)
@@ -391,6 +416,7 @@ def rubric_judge(
             },
             default=str,
         )
+        ensure_warm()
         try:
             reply = complete(
                 url,

{whileai-0.52 → whileai-0.53/whileai.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: whileai
-Version: 0.52
+Version: 0.53
 Summary: While Python SDK: trace ingestion and verification client, plus agent simulations grounded in your tools and system prompt.
 Author: While
 License: Apache-2.0