whileai 0.57__tar.gz → 0.59__tar.gz

{whileai-0.57/whileai.egg-info → whileai-0.59}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: whileai
-Version: 0.57
+Version: 0.59
 Summary: While Python SDK: trace ingestion and verification client, plus agent simulations grounded in your tools and system prompt.
 Author: While
 License: Apache-2.0
@@ -219,7 +219,11 @@ rollout touched, a marker that fired on no row. A 1.00 on a run like that is not
 
 Bring your own model. Any OpenAI-compatible chat endpoint that returns tool
 calls works; it writes the situations and plays the agent, so both run on
-your key:
+your key. To put a number on a model you serve (`wai.serve`, or your own
+vLLM), make it the agent: `wai.simulate(tasks=pinned,
+agent=wai.local_model(endpoint, name, tools=TOOLS, system=POLICY,
+thinking=False))`, and run both arms of a before/after through that same
+call so the only difference is the weights.
 
 ```bash
 export OPENAI_API_KEY=...

{whileai-0.57 → whileai-0.59}/README.md

@@ -183,7 +183,11 @@ rollout touched, a marker that fired on no row. A 1.00 on a run like that is not
 
 Bring your own model. Any OpenAI-compatible chat endpoint that returns tool
 calls works; it writes the situations and plays the agent, so both run on
-your key:
+your key. To put a number on a model you serve (`wai.serve`, or your own
+vLLM), make it the agent: `wai.simulate(tasks=pinned,
+agent=wai.local_model(endpoint, name, tools=TOOLS, system=POLICY,
+thinking=False))`, and run both arms of a before/after through that same
+call so the only difference is the weights.
 
 ```bash
 export OPENAI_API_KEY=...

{whileai-0.57 → whileai-0.59}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "whileai"
-version = "0.57"
+version = "0.59"
 description = "While Python SDK: trace ingestion and verification client, plus agent simulations grounded in your tools and system prompt."
 readme = "README.md"
 license = {text = "Apache-2.0"}

{whileai-0.57 → whileai-0.59}/whileai/simulations/ingest/traces.py

@@ -278,15 +278,20 @@ def dimensions_from_traces(
 def _task_key(row: dict, index: int) -> tuple[str, object]:
     """What makes two rows the same task for splitting purposes.
 
-    ``prompt`` first, because that is what a train/eval split has to keep
-    disjoint and what ``leakage_report`` measures. ``scenario_id`` covers
-    rows that carry no prompt. A row with neither is its own task, so rows
-    that merely lack both are not swept onto one side together.
+    The unit every report counts in: ``task_key`` (``scenario_id`` when
+    the row has one, else ``task_id``, else the prompt), so repeats and
+    rephrasings of one situation land on the same side and the held-out
+    slice is disjoint from train in the unit ``pass_at``, ``compare_runs``
+    and ``delta_report`` group by (#268). Splitting on the prompt alone
+    left 16 of 28 held-out situations in train, and ``decontaminate``
+    cannot see that because it compares prompts. A row with no key is its
+    own task, so rows that merely lack one are not swept onto one side.
     """
-    for field in ("prompt", "scenario_id"):
-        value = row.get(field)
-        if value is not None and str(value).strip():
-            return (field, str(value))
+    from ..score.stats import task_key
+
+    key = task_key(row) if isinstance(row, dict) else ""
+    if key and str(key).strip():
+        return ("task", str(key))
     return ("index", index)
 
 
@@ -295,9 +300,10 @@ def split_pseudo_production(
 ) -> tuple[list[dict], list[dict]]:
     """Set aside a pseudo-production slice; the rest stays for training.
 
-    The split is by task, not by row: every row sharing a ``prompt`` (or a
-    ``scenario_id``, for rows without a prompt) lands on the same side, so
-    the held-out slice is prompt-disjoint from the training side. Splitting
+    The split is by task, not by row: every row sharing a ``task_key``
+    (the ``scenario_id``, else the prompt) lands on the same side, so the
+    held-out slice is disjoint from the training side in the unit every
+    report groups by, not just prompt-disjoint. Splitting
     by row is not enough — under ``mode="rl"`` with ``repeats=k`` each
     prompt has k rows, and scattering siblings across the two sides trains
     the student on every prompt it is then evaluated on.

{whileai-0.57 → whileai-0.59}/whileai/simulations/score/delta.py

@@ -33,6 +33,7 @@ from .stats import (
     eval_variance,
     holdout_size,
     marker_names,
+    metric_summary,
     task_means,
 )
 
@@ -214,6 +215,13 @@ def delta_report(
 
     guarded = {_key(m) for m in must_not_regress}
     target_key = _key(target) if target else None
+    degenerate_guards: list[str] = []
+    for m in sorted(guarded):
+        if m not in results:
+            continue
+        sides = [metric_summary(rows, m, n_boot=10) for rows in (before, after)]
+        if all(s.get("degenerate") for s in sides):
+            degenerate_guards.append(m)
     eval_runs = {"before": len(_eval_runs(before)), "after": len(_eval_runs(after))}
     run_std_source = "given" if run_std is not None else None
     if run_std is None and min(eval_runs.values()) >= 2:
@@ -255,6 +263,12 @@ def delta_report(
             f"One eval run on {where}, so this could be noise. Run each side three times with "
             "simulate(tasks=..., runs=3) and the report will say."
         )
+    for m in degenerate_guards:
+        warnings.append(
+            f"must_not_regress {m} is degenerate on both sides (every applicable row scored the "
+            "same value): this guard cannot fail, so it catches nothing. Check that the marker "
+            "fires at all."
+        )
     if run_std_source == "eval_run" and min(eval_runs.values()) < 3:
         warnings.append(
             "Two eval runs on a side is a difference, not a distribution, so run_std is rough; "
@@ -434,6 +448,7 @@ def delta_report(
         "ceiling": ceiling,
         "detectable_effect": can_prove,
         "tasks_needed": tasks_needed,
+        "degenerate_guards": degenerate_guards,
         "proxy": proxy_key,
         "proxy_verdict": proxy_verdict,
         "proxy_delta": proxy_result["delta"] if proxy_result else None,

{whileai-0.57 → whileai-0.59}/whileai/simulations/score/judge_trust.py

@@ -526,11 +526,32 @@ def judge_trust(
     if trusted and agree["n"]:
         low = agree["ci95"][0]
         if low < min_agreement:
-            warnings.append(
-                f"Judge agreement with human labels is {low:.2f} (lower bound), under the "
-                f"{min_agreement:.2f} floor. Change the judge prompt or the judge model, then "
-                "run judge_trust again."
-            )
+            point = float(agree.get("agreement") or 0.0)
+            need = int(agree["n"])
+            if point > min_agreement:
+                while need < 1000:
+                    ci = wilson_interval(round(point * need), need)
+                    if ci is not None and ci[0] >= min_agreement:
+                        break
+                    need += 1
+            if point > min_agreement and need <= 200:
+                # The judge agrees often enough; the sample is what is short.
+                # Say how many labels the bound needs at this agreement rate,
+                # or a perfect judge on 14 labels reads as "change the judge". At
+                # the floor exactly, or when the count is out of reach, the old
+                # advice stands: the judge is what to change.
+                warnings.append(
+                    f"Judge agreement with human labels is {point:.2f} on {agree['n']} labels, "
+                    f"but the lower bound is {low:.2f}, under the {min_agreement:.2f} floor. "
+                    f"The judge is not the problem; the sample is. Label about "
+                    f'{need} rows (attach_labels(kind="human")) and run judge_trust again.'
+                )
+            else:
+                warnings.append(
+                    f"Judge agreement with human labels is {low:.2f} (lower bound), under the "
+                    f"{min_agreement:.2f} floor. Change the judge prompt or the judge model, then "
+                    "run judge_trust again."
+                )
         kappa = agree["kappa"]
         if not degenerate_gold and kappa is not None and kappa < min_kappa:
             warnings.append(

{whileai-0.57 → whileai-0.59}/whileai/simulations/score/stats.py

@@ -250,21 +250,46 @@ def task_means(rows: Sequence[dict], metric: str = "pass_at_1") -> dict[str, flo
 def metric_summary(
     rows: Sequence[dict], metric: str = "pass_at_1", *, n_boot: int = DEFAULT_BOOT, seed: int = 0
 ) -> dict[str, Any]:
-    """Mean over tasks with a task-bootstrap 95% interval."""
+    """Mean over tasks with a task-bootstrap 95% interval.
+
+    ``degenerate`` is set when every applicable row scored the same
+    value: the metric has not been shown to be able to come out any
+    other way, so ``ci95`` is ``None`` (the way ``pass_at`` returns
+    ``None`` below three groups) and ``warning`` says so. A marker that
+    is silently unfireable (a key-name mismatch) and one that is
+    genuinely always true look identical otherwise, and either one passed
+    to ``must_not_regress`` is a guard that cannot fail (#270).
+    ``n_rows_at_1`` and ``n_rows_at_0`` put the row-level split next to
+    the mean.
+    """
     means = task_means(rows, metric)
     values = list(means.values())
-    return {
+    per_task = _by_task(
+        rows, _binary if metric == "pass_at_1" else _marker(metric.split(":", 1)[1])
+    )
+    row_values = [v for vs in per_task.values() for v in vs]
+    distinct = {round(float(v), 9) for v in row_values}
+    degenerate = len(row_values) > 0 and len(distinct) == 1
+    out: dict[str, Any] = {
         "metric": metric,
         "n_tasks": len(values),
-        "n_rows": sum(
-            len(v)
-            for v in _by_task(
-                rows, _binary if metric == "pass_at_1" else _marker(metric.split(":", 1)[1])
-            ).values()
-        ),
+        "n_rows": len(row_values),
+        "n_rows_at_1": sum(1 for v in row_values if float(v) == 1.0),
+        "n_rows_at_0": sum(1 for v in row_values if float(v) == 0.0),
         "mean": _mean(values) if values else None,
-        "ci95": bootstrap_ci(values, n_boot=n_boot, seed=seed),
+        "ci95": None if degenerate else bootstrap_ci(values, n_boot=n_boot, seed=seed),
+        "degenerate": degenerate,
     }
+    if degenerate:
+        only = next(iter(distinct))
+        name = metric.split(":", 1)[1] if metric.startswith("marker:") else metric
+        out["warning"] = (
+            f"all {len(row_values)} applicable rows scored {only:g}; {name} has not been shown "
+            "to be able to come out any other way. Check the marker fires at all (a key-name "
+            "mismatch looks exactly like this) before reading the mean, and do not put it in "
+            "must_not_regress: a guard that cannot fail catches nothing."
+        )
+    return out
 
 
 def marker_names(rows: Sequence[dict]) -> list[str]:

{whileai-0.57 → whileai-0.59/whileai.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: whileai
-Version: 0.57
+Version: 0.59
 Summary: While Python SDK: trace ingestion and verification client, plus agent simulations grounded in your tools and system prompt.
 Author: While
 License: Apache-2.0
@@ -219,7 +219,11 @@ rollout touched, a marker that fired on no row. A 1.00 on a run like that is not
 
 Bring your own model. Any OpenAI-compatible chat endpoint that returns tool
 calls works; it writes the situations and plays the agent, so both run on
-your key:
+your key. To put a number on a model you serve (`wai.serve`, or your own
+vLLM), make it the agent: `wai.simulate(tasks=pinned,
+agent=wai.local_model(endpoint, name, tools=TOOLS, system=POLICY,
+thinking=False))`, and run both arms of a before/after through that same
+call so the only difference is the weights.
 
 ```bash
 export OPENAI_API_KEY=...