whileai 0.54__tar.gz → 0.56__tar.gz

{whileai-0.54/whileai.egg-info → whileai-0.56}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: whileai
-Version: 0.54
+Version: 0.56
 Summary: While Python SDK: trace ingestion and verification client, plus agent simulations grounded in your tools and system prompt.
 Author: While
 License: Apache-2.0

{whileai-0.54 → whileai-0.56}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "whileai"
-version = "0.54"
+version = "0.56"
 description = "While Python SDK: trace ingestion and verification client, plus agent simulations grounded in your tools and system prompt."
 readme = "README.md"
 license = {text = "Apache-2.0"}

{whileai-0.54 → whileai-0.56}/whileai/simulations/__init__.py

@@ -135,6 +135,7 @@ from .score.optimize import (
     DEFAULT_BAND,
     filter_rl_rows,
     group_signal,
+    next_round,
     optimize,
     recommend,
     select_for_rl,
@@ -162,7 +163,9 @@ from .score.stage import STAGES, format_stages, stage_of, stage_report, stamp_st
 from .score.stats import (
     compare_runs,
     decontaminate,
+    detectable_effect,
     eval_variance,
+    holdout_size,
     marker_summary,
     metric_summary,
     task_key,
@@ -175,6 +178,7 @@ from .training import (
     TrainingRun,
     attach_delta,
     attach_holdout,
+    delete_model,
     delete_run,
     get_run,
     list_runs,
@@ -183,6 +187,7 @@ from .training import (
     serve,
     train,
     training_run,
+    unserve,
 )
 from .verify import Verifier, verifier
 from .world.sandbox import MockEnvironment
@@ -253,8 +258,10 @@ __all__ = [
     "dedupe_groups",
     "delete_dataset",
     "delete_empty_datasets",
+    "delete_model",
     "delete_run",
     "delta_report",
+    "detectable_effect",
     "dimensions_from_traces",
     "drop_leaky_rows",
     "eval_variance",
@@ -287,6 +294,7 @@ __all__ = [
     "hf_publish",
     "hf_publish_run",
     "hf_status",
+    "holdout_size",
     "hosted_model",
     "import_hf",
     "inspect",
@@ -314,6 +322,7 @@ __all__ = [
     "mine_traces",
     "models",
     "near_duplicate_prompts",
+    "next_round",
     "normalize_judge_result",
     "novelty",
     "open_ended_probes",
@@ -382,6 +391,7 @@ __all__ = [
     "trim_unanimous_groups",
     "ungrounded_arguments",
     "unpublish",
+    "unserve",
     "update_dataset",
     "validate",
     "verifier",

{whileai-0.54 → whileai-0.56}/whileai/simulations/export.py

@@ -734,6 +734,7 @@ export_dataset = export_training
 _PAIR_KEYS = (
     "tie",
     "pairwise",
+    "first_turn_differs",
     "chosen_score",
     "rejected_score",
     "margin",
@@ -853,6 +854,12 @@ def export_preference(
         length_note = length_confound_warning(chosen_longer, len(deltas))
         if length_note:
             report["warnings"] = [length_note]
+    identical = sum(1 for r in out_rows if r.get("first_turn_differs") is False)
+    if identical:
+        from .score.judging import first_turn_note
+
+        report["first_turn_identical"] = identical
+        report.setdefault("warnings", []).append(first_turn_note(identical, len(out_rows)))
     margins = [r["margin"] for r in out_rows if isinstance(r.get("margin"), (int, float))]
     if margins:
         report["mean_margin"] = round(sum(margins) / len(margins), 4)

{whileai-0.54 → whileai-0.56}/whileai/simulations/generate/agents.py

@@ -10,7 +10,7 @@ import os
 import re
 import threading
 import time
-from collections.abc import Callable
+from collections.abc import Callable, Mapping
 from typing import Any
 from urllib.parse import urlparse
 
@@ -531,6 +531,7 @@ def complete(
     timeout: float = 60,
     n: int = 1,
     logprobs: bool | str = False,
+    extra: Mapping[str, Any] | None = None,
 ) -> dict:
     """POST /chat/completions. Reuses a thread-local keep-alive connection.
 
@@ -577,6 +578,8 @@ def complete(
     if tools:
         payload["tools"] = _wire_tools(tools)
     payload.update(_request_extras(base_url, model))
+    if extra:
+        payload.update(dict(extra))
     headers = {"Content-Type": "application/json", "Connection": "keep-alive"}
     if key:
         headers["Authorization"] = f"Bearer {key}"
@@ -730,8 +733,21 @@ def _strip_tool_markup(text: str) -> str:
     return cleaned.strip()
 
 
+_THINK_BLOCK = re.compile(r"<think>.*?</think>\s*", re.S | re.I)
+_THINK_OPEN = re.compile(r"<think>.*\Z", re.S | re.I)
+
+
+def _strip_think(text: str) -> str:
+    """Drop a thinking model's reasoning markup. A closed block goes whole;
+    an unclosed ``<think>`` (the token cap landed inside it) goes to the
+    end. What is left is the reply, which is what a grader, a marker and
+    the next turn's history should see (#264)."""
+    text = _THINK_BLOCK.sub("", text)
+    return _THINK_OPEN.sub("", text)
+
+
 def _spoken_text(reply: dict) -> str:
-    return _strip_tool_markup(str(reply.get("content") or ""))
+    return _strip_tool_markup(_strip_think(str(reply.get("content") or "")))
 
 
 def _calls_from_reply(reply: dict) -> tuple[list[dict], dict]:
@@ -1391,9 +1407,23 @@ def local_model(
     timeout: float = 60,
     max_tokens: int | None = None,
     user_model: str | None = None,
+    thinking: bool | None = None,
 ) -> Callable:
+    """An agent that talks to an OpenAI-compatible endpoint (a served
+    adapter, a local vLLM, any chat server) for ``simulate(agent=...)``.
+
+    ``thinking`` is for reasoning bases such as Qwen3: ``False`` sends
+    ``chat_template_kwargs={"enable_thinking": False}`` so the reply is
+    the answer, not the reasoning, the way the hosted Qwen path already
+    does; ``True`` asks for it; ``None`` (the default) sends nothing and
+    leaves the server's default. Either way ``<think>`` markup never
+    reaches ``step["text"]`` or ``final_text``.
+    """
     local = threading.local()
     plans = fault_plans if fault_plans is not None else {}
+    extras: dict[str, Any] | None = (
+        None if thinking is None else {"chat_template_kwargs": {"enable_thinking": bool(thinking)}}
+    )
     # The simulated user's model. None means the agent's own model plays
     # the user (the default); a backend spec moves that role to another
     # model, with the key resolved for that endpoint.
@@ -1438,6 +1468,7 @@ def local_model(
                     temperature=temperature,
                     timeout=timeout,
                     max_tokens=120,
+                    extra=extras,
                 )
                 opener_text = (_spoken_text(greet) or "").strip()
                 if opener_text:
@@ -1477,6 +1508,7 @@ def local_model(
                 timeout=timeout,
                 max_tokens=reply_budget(max_tokens),
                 logprobs=logprobs,
+                extra=extras,
             )
             calls, assistant = _calls_from_reply(reply)
             # One agent turn, one set of sampling facts, on its first step.

{whileai-0.54 → whileai-0.56}/whileai/simulations/ingest/platform.py

@@ -22,6 +22,7 @@ import time
 import urllib.error
 import urllib.parse
 import urllib.request
+import warnings as _warnings
 from collections.abc import Sequence
 from typing import Any
 
@@ -242,6 +243,36 @@ def _meta_body(
     return body
 
 
+HOLDOUT_PROVE_EFFECT = 0.05
+
+
+def _warn_small_holdout(rows: Sequence[dict]) -> None:
+    """A holdout too small to prove a 5-point gain reads every round as
+    ``no_change_detected``; say so at push time, not after training (#257)."""
+    from ..score.stats import holdout_size, task_key
+
+    groups: dict[str, int] = {}
+    for row in rows:
+        if isinstance(row, dict):
+            key = task_key(row)
+            groups[key] = groups.get(key, 0) + 1
+    n_tasks = len(groups)
+    if not n_tasks:
+        return
+    try:
+        need = holdout_size(HOLDOUT_PROVE_EFFECT, rows=rows)
+    except ValueError:
+        need = holdout_size(HOLDOUT_PROVE_EFFECT, k=min(groups.values()))
+    if n_tasks < need["n_tasks"]:
+        _warnings.warn(
+            f"holdout has {n_tasks} tasks at k={need['k']}; proving a "
+            f"{HOLDOUT_PROVE_EFFECT:.0%} gain at 80% power needs about {need['n_tasks']} "
+            "(holdout_size). A smaller holdout reads a real gain that size as "
+            "no_change_detected.",
+            stacklevel=3,
+        )
+
+
 def push_rows(
     rows: list[dict],
     name: str,
@@ -279,6 +310,8 @@ def push_rows(
 
         gate_report = publish_gate(rows, mode=mode, endorsed=endorsed, strict_hacks=strict_hacks)
     check(rows, where="push_rows")
+    if purpose == "holdout":
+        _warn_small_holdout(rows)
     body: dict = {
         "name": name,
         **_meta_body(purpose, mode if mode in MODES else None, agent, description),

{whileai-0.54 → whileai-0.56}/whileai/simulations/score/delta.py

@@ -26,7 +26,15 @@ from collections.abc import Callable, Sequence
 from typing import Any
 
 from .passat import pass_at
-from .stats import DEFAULT_BOOT, compare_runs, eval_variance, marker_names, task_means
+from .stats import (
+    DEFAULT_BOOT,
+    compare_runs,
+    detectable_effect,
+    eval_variance,
+    holdout_size,
+    marker_names,
+    task_means,
+)
 
 GROUP_KEYS = ("delta", "ci95", "verdict", "mean_a", "mean_b", "n_used", "n_paired", "paired")
 
@@ -327,6 +335,32 @@ def delta_report(
     headline_key = target_key if target_result else "pass_at_1"
     if headline.get("note"):
         warnings.append(f"{headline_key}: {headline['note']}")
+    # Eval size: a no-change verdict is only as strong as the band the
+    # task count allows. Say what this holdout can prove and what the
+    # delta seen here would have needed (#257).
+    n_paired = int(headline.get("n_paired") or 0)
+    k_eval = int(pass_at(before).config.get("k") or 1)
+    base_rate = float(mean_a) if mean_a is not None else 0.6
+    can_prove = detectable_effect(n_paired, base=base_rate, k=k_eval) if n_paired >= 2 else None
+    tasks_needed: int | None = None
+    delta_seen: float | None = None
+    raw_delta = headline.get("delta")
+    if isinstance(raw_delta, (int, float)) and 0 < raw_delta < 1:
+        delta_seen = float(raw_delta)
+        tasks_needed = holdout_size(delta_seen, base=base_rate, k=k_eval)["n_tasks"]
+    verdict_word = (
+        target_verdict if target_result else _verdict_word(results["pass_at_1"], replicated)
+    )
+    if verdict_word == "no_change_detected" and can_prove is not None:
+        line = (
+            f"{n_paired} paired tasks at k={k_eval} can prove a gain of about "
+            f"+{can_prove:.2f} at 80% power"
+        )
+        if tasks_needed is not None and delta_seen is not None:
+            line += (
+                f"; to prove the {delta_seen:+.3f} seen here you need about {tasks_needed} tasks"
+            )
+        warnings.append(line + " (holdout_size).")
     if target_verdict == "target_not_measured":
         warnings.append(f"target {target!r} is not on both row sets")
     groups: dict[str, dict[str, Any]] | None = None
@@ -398,6 +432,8 @@ def delta_report(
         "eval_runs": eval_runs,
         "replicated": replicated,
         "ceiling": ceiling,
+        "detectable_effect": can_prove,
+        "tasks_needed": tasks_needed,
         "proxy": proxy_key,
         "proxy_verdict": proxy_verdict,
         "proxy_delta": proxy_result["delta"] if proxy_result else None,

{whileai-0.54 → whileai-0.56}/whileai/simulations/score/grading.py

@@ -40,6 +40,15 @@ _FAULT_ALIASES = {
     "entity already acted on": "already_done",
 }
 _CLEAN_FAULT = {"", "ok", "created", "deleted", "success", "exit_0", "no fault", "clean"}
+#: Statuses a tool result can carry that mean the call failed. A ``status``
+#: key with any other value (``paid``, ``open``, ``completed``) is the
+#: tool's own vocabulary, not a fault (#261).
+_KNOWN_FAULTS = (
+    _BAD_STATUS
+    | set(_FAULT_ALIASES)
+    | set(_FAULT_ALIASES.values())
+    | {"stale", "malformed", "already_done", "not_found", "deny", "garbled"}
+)
 _HTTP_FAIL = re.compile(r"^[45]\d\d$")
 _INFRA_STUB = re.compile(
     r"^<agent error:|returned\s+[45]\d\d\b|^(https?://\S+\s+)?[45]\d\d(\s|$)", re.I
@@ -426,10 +435,10 @@ def _fault_from_result(result) -> str:
     if status in {"already_done", "already_acted_on"} or reason == "already_acted_on":
         return "already_done"
     name = normalize_fault_name(status)
-    if name:
+    if name and (name in _KNOWN_FAULTS or status in _KNOWN_FAULTS):
         return name
     if _step_faulted(result):
-        return normalize_fault_name(status) or "error"
+        return name or "error"
     return ""
 
 

{whileai-0.54 → whileai-0.56}/whileai/simulations/score/judging.py

@@ -563,6 +563,42 @@ def length_confound_warning(chosen_longer: int, n: int) -> str | None:
     return None
 
 
+def _first_turn(row: dict) -> str:
+    """What the policy emitted first, read the way the hosted DPO trainer
+    reads it: the first tool step as its call, else the first assistant
+    text, else ``final_text``."""
+    steps = [s for s in (row.get("steps") or []) if isinstance(s, dict)]
+    for step in steps:
+        if step.get("tool"):
+            args = step.get("arguments")
+            if args is None:
+                args = step.get("args")
+            return json.dumps({"name": step["tool"], "arguments": args or {}}, sort_keys=True)
+    for step in steps:
+        if str(step.get("text") or "").strip():
+            return str(step["text"]).strip()
+    for message in row.get("messages") or []:
+        if isinstance(message, dict) and message.get("role") == "assistant":
+            text = str(message.get("content") or "").strip()
+            if text:
+                return text
+    return str(row.get("final_text") or "").strip()
+
+
+def first_turn_note(identical: int, n: int) -> str:
+    """The warning for pairs whose first assistant turns read the same."""
+    if not identical:
+        return ""
+    left = n - identical
+    return (
+        f"{identical}/{n} pairs have identical first assistant turns (same opening tool call "
+        "or line); their contrast is later in the rollout. The hosted DPO trainer compares "
+        f"first turns only and will drop them, leaving {left} (it needs at least 8). Keep "
+        "[p for p in pairs if p['first_turn_differs']] to see what it will train on, or "
+        "export_preference(pairs) for a trainer that reads whole conversations."
+    )
+
+
 def build_preference_pairs(
     rows: Sequence[dict],
     *,
@@ -651,6 +687,7 @@ def build_preference_pairs(
                     "rejected_model": r_model,
                     "same_policy": (c_model == r_model) if c_model and r_model else None,
                     "length_delta": reply_length(chosen) - reply_length(rejected),
+                    "first_turn_differs": _first_turn(chosen) != _first_turn(rejected),
                     "chosen_reason": str(chosen.get("reason") or ""),
                     "rejected_reason": str(rejected.get("reason") or ""),
                     "rejected_failure_class": rejected.get("failure_class"),
@@ -673,10 +710,13 @@ def build_preference_pairs(
         for p in pairs
         if p["chosen_score"] not in (0.0, 1.0) or p["rejected_score"] not in (0.0, 1.0)
     )
+    identical = sum(1 for p in pairs if not p["first_turn_differs"])
     warnings: list[str] = []
     length_note = length_confound_warning(chosen_longer, n)
     if length_note:
         warnings.append(length_note)
+    if identical:
+        warnings.append(first_turn_note(identical, n))
     if mixed_policy:
         warnings.append(
             f"{mixed_policy}/{n} pairs mix policies (chosen and rejected from different "
@@ -694,6 +734,8 @@ def build_preference_pairs(
         "min_margin": min_margin,
         "mean_margin": round(sum(p["margin"] for p in pairs) / n, 4) if n else None,
         "partial_score_pairs": partial,
+        "first_turn_identical": identical,
+        "trainer_pairs": n - identical,
         "same_policy_pairs": same_policy,
         "mixed_policy_pairs": mixed_policy,
         "eval_sourced": eval_pairs,

{whileai-0.54 → whileai-0.56}/whileai/simulations/score/optimize.py

@@ -614,6 +614,105 @@ def _spread_by(prompts: list[str], key) -> list[str]:
     return out
 
 
+def _task_keys_of(tasks: Sequence[Any]) -> list[str]:
+    out: list[str] = []
+    for t in tasks:
+        if isinstance(t, dict):
+            out.append(task_key(t))
+        else:
+            out.append(task_key({"prompt": str(t)}))
+    return out
+
+
+def next_round(
+    prior: Sequence[dict],
+    *,
+    tasks: Sequence[Any] | None = None,
+    lo: float = DEFAULT_BAND[0],
+    hi: float = DEFAULT_BAND[1],
+) -> dict[str, Any]:
+    """The prompt set for the next round, from the last round's graded
+    rollouts.
+
+    A round trained on the file it started from keeps paying for groups
+    that give no gradient: at a 0.65 training reward about half the
+    groups are all-pass or all-fail. The band is the fix the book already
+    names (rlhf-book ch. 7: filter to the 20-80% band; ch. 6, DAPO's
+    dynamic sampling drops groups with no contrast), applied to what the
+    *current* policy does rather than what the base did. ``prior`` is
+    round N's graded rollouts (``simulate(tasks=..., repeats=k)`` on the
+    round-N policy, or the trainer's own sampled rows); each task's pass
+    rate over them decides: inside ``[lo, hi]`` it is kept, above ``hi``
+    it is solved and dropped, below ``lo`` it is unsolved and dropped.
+    ``tasks`` restricts the candidates (rows, task dicts with a
+    ``prompt``, or prompt strings); a task with no prior rollouts is
+    ``unknown`` and kept, since nothing says it is flat.
+
+    Returns ``tasks`` (one representative row per kept task: the prior
+    row, with ``calibration.pass_rate`` and the band), the counts
+    ``kept``, ``dropped_solved``, ``dropped_unsolved``, ``unknown``,
+    ``pass_rates`` per task, ``band``, ``from_policy`` (the policy
+    versions the prior rows came from) and ``prompt_set_sha``: the
+    identity of the kept set, for lineage on the run. Push the kept rows
+    as the next train set with ``parent=`` the last one.
+    """
+    if not 0 <= lo < hi <= 1:
+        raise ValueError("band is 0 <= lo < hi <= 1")
+    labels = _group_label_lists(prior)
+    rates = {key: sum(v) / len(v) for key, v in labels.items() if v}
+    first: dict[str, dict] = {}
+    policies: set[str] = set()
+    for row in prior:
+        if not isinstance(row, dict):
+            continue
+        first.setdefault(task_key(row), row)
+        if row.get("policy_version"):
+            policies.add(str(row["policy_version"]))
+    if tasks is None:
+        candidates = list(rates)
+        given: dict[str, Any] = {}
+    else:
+        given = {}
+        for t in tasks:
+            key = task_key(t) if isinstance(t, dict) else task_key({"prompt": str(t)})
+            given.setdefault(key, t)
+        candidates = list(given)
+    kept: list[dict] = []
+    solved = unsolved = unknown = 0
+    for key in candidates:
+        rate = rates.get(key)
+        if rate is None:
+            unknown += 1
+            rep = given.get(key)
+            rep = dict(rep) if isinstance(rep, dict) else {"prompt": str(rep)}
+            kept.append(rep)
+            continue
+        if rate > hi:
+            solved += 1
+            continue
+        if rate < lo:
+            unsolved += 1
+            continue
+        rep = dict(first.get(key) or given.get(key) or {"prompt": key})
+        cal = dict(rep.get("calibration") or {})
+        cal.update({"pass_rate": round(rate, 4), "n": len(labels[key]), "band": [lo, hi]})
+        rep["calibration"] = cal
+        kept.append(rep)
+    sha = hashlib.sha256("\n".join(sorted(task_key(r) for r in kept)).encode()).hexdigest()[:16]
+    return {
+        "tasks": kept,
+        "kept": len(kept) - unknown,
+        "dropped_solved": solved,
+        "dropped_unsolved": unsolved,
+        "unknown": unknown,
+        "n_prior_tasks": len(rates),
+        "pass_rates": {k: round(v, 4) for k, v in rates.items()},
+        "band": [lo, hi],
+        "from_policy": sorted(policies),
+        "prompt_set_sha": sha,
+    }
+
+
 def select_for_rl(
     rows: Sequence[dict],
     *,
@@ -627,9 +726,18 @@ def select_for_rl(
     endorsed: Sequence[str] = (),
     truncated: str = "drop",
     order: str = "spread",
+    prior: Sequence[dict] | None = None,
 ) -> tuple[list[dict], dict[str, Any]]:
     """Whole mixed groups up to roughly ``target`` rows. Groups never split.
 
+    ``prior`` is the previous round's graded rollouts: tasks the round-N
+    policy already solves (pass rate above ``hi`` on ``prior``) or never
+    solves (below ``lo``) are dropped before anything else, so round N+1
+    trains on what that policy gets right 20-80% of the time rather than
+    on the file round 1 started from (``next_round``; rlhf-book ch. 7).
+    The report's ``prior`` block counts kept, dropped_solved,
+    dropped_unsolved and unknown.
+
     ``truncated`` says what happens to a rollout cut at the token cap
     (rlhf-book ch. 6, DAPO's overlong handling; ch. 7 overlong filtering):
     ``"drop"`` removes it (the default; ``drop_truncated=False`` is the old
@@ -691,6 +799,24 @@ def select_for_rl(
         raise ValueError(f"order must be one of {', '.join(RL_ORDERS)}; got {order!r}")
     if not drop_truncated and truncated == "drop":
         truncated = "keep"
+    prior_report: dict[str, Any] | None = None
+    if prior is not None:
+        plan = next_round(prior, lo=lo, hi=hi)
+        rates = plan["pass_rates"]
+        before_n = len(rows)
+        rows = [
+            r
+            for r in rows
+            if not isinstance(r, dict)
+            or rates.get(task_key(r)) is None
+            or lo <= rates[task_key(r)] <= hi
+        ]
+        prior_report = {
+            k: plan[k]
+            for k in ("kept", "dropped_solved", "dropped_unsolved", "unknown", "from_policy")
+        }
+        prior_report["rows_dropped"] = before_n - len(rows)
+        prior_report["prompt_set_sha"] = plan["prompt_set_sha"]
     penalized = kept_overlong = 0
     if truncated != "drop":
         marked: list[dict] = []
@@ -838,6 +964,7 @@ def select_for_rl(
         "enforce_band": bool(enforce_band),
         "band_groups_dropped": band_report["n_groups_dropped"],
         "band_dropped": {"too_easy": band_report["too_easy"], "too_hard": band_report["too_hard"]},
+        "prior": prior_report,
         "duplicates": dup_report,
         "truncated_dropped": trunc_report["n_dropped"],
         "truncated_policy": truncated,

{whileai-0.54 → whileai-0.56}/whileai/simulations/score/stats.py

@@ -57,6 +57,116 @@ def wilson_interval(successes: int, n: int, *, z: float = 1.96) -> tuple[float,
     return (max(0.0, center - half), min(1.0, center + half))
 
 
+def _z(p: float) -> float:
+    from statistics import NormalDist
+
+    return NormalDist().inv_cdf(p)
+
+
+def _paired_task_sd(base: float, effect: float, k: int) -> float:
+    """Standard deviation of one task's paired difference (after minus
+    before pass rate over ``k`` rollouts each side) when the gain lands
+    uniformly: before at ``base``, after at ``base + effect``."""
+    p = min(1.0, max(0.0, float(base)))
+    q = min(1.0, max(0.0, p + float(effect)))
+    kk = max(1, int(k))
+    return math.sqrt((p * (1 - p) + q * (1 - q)) / kk)
+
+
+def _rows_base_and_k(rows: Sequence[dict]) -> tuple[float, int]:
+    """Mean per-task pass rate and the smallest rollouts-per-task on graded
+    rows: what ``delta_report`` would pair on."""
+    groups: dict[str, list[float]] = {}
+    for row in rows:
+        if not isinstance(row, dict):
+            continue
+        value = _binary(row)
+        if value is None:
+            continue
+        groups.setdefault(task_key(row), []).append(value)
+    if not groups:
+        raise ValueError("rows carry no 0/1 rewards; grade them first, or pass base= and k=")
+    base = _mean([_mean(v) for v in groups.values()])
+    k = min(len(v) for v in groups.values())
+    return base, k
+
+
+def holdout_size(
+    effect: float,
+    *,
+    base: float = 0.6,
+    k: int = 4,
+    power: float = 0.8,
+    alpha: float = 0.05,
+    rows: Sequence[dict] | None = None,
+) -> dict[str, Any]:
+    """How many paired tasks a holdout needs to prove a gain of ``effect``.
+
+    Models the test ``delta_report`` runs: each task's pass rate over
+    ``k`` rollouts on each side, the delta as the mean of the paired
+    differences, the interval from a bootstrap over tasks. A task's
+    difference then has standard deviation
+    ``sqrt((p(1-p) + q(1-q)) / k)`` with ``p = base`` and ``q = base +
+    effect``, and the usual two-sided power calculation gives
+    ``n = ((z_{1-alpha/2} + z_power) * sd / effect) ** 2`` (rlhf-book ch. 16,
+    appendix C: the eval's own variance decides what a delta can mean).
+    It assumes the gain lands uniformly across tasks; a gain concentrated
+    on a few tasks needs more.
+
+    ``rows`` (graded before-side rows) reads ``base`` and ``k`` off the
+    data instead. Returns ``n_tasks`` plus the inputs, ``sd_task``, and
+    ``half_width``: the 95% band on the delta at that ``n``.
+
+    The recipe that asked for this had 140 tasks at k=4 around 0.6: a
+    band of about +-0.06, so a real 3-point gain reads
+    ``no_change_detected`` every round. This says so before training.
+    """
+    if not 0 < float(effect) < 1:
+        raise ValueError(
+            "effect is the gain in pass rate to prove, between 0 and 1 (0.05 = 5 points)"
+        )
+    if not 0 < power < 1 or not 0 < alpha < 1:
+        raise ValueError("power and alpha are probabilities strictly between 0 and 1")
+    if rows is not None:
+        base, k = _rows_base_and_k(rows)
+    sd = _paired_task_sd(base, effect, k)
+    z = _z(1 - alpha / 2) + _z(power)
+    n = math.ceil((z * sd / float(effect)) ** 2) if sd > 0 else 1
+    n = max(n, 2)
+    return {
+        "n_tasks": n,
+        "effect": float(effect),
+        "base": float(base),
+        "k": int(k),
+        "power": float(power),
+        "alpha": float(alpha),
+        "sd_task": round(sd, 4),
+        "half_width": round(_z(1 - alpha / 2) * sd / math.sqrt(n), 4),
+    }
+
+
+def detectable_effect(
+    n_tasks: int,
+    *,
+    base: float = 0.6,
+    k: int = 4,
+    power: float = 0.8,
+    alpha: float = 0.05,
+) -> float | None:
+    """The smallest gain ``n_tasks`` paired tasks can prove at ``power``:
+    ``holdout_size`` solved for the effect (a few fixed-point steps, since
+    the after-side variance depends on it). ``None`` below two tasks."""
+    n = int(n_tasks)
+    if n < 2:
+        return None
+    z = _z(1 - alpha / 2) + _z(power)
+    effect = 0.0
+    for _ in range(12):
+        sd = _paired_task_sd(base, effect, k)
+        effect = z * sd / math.sqrt(n)
+    return round(min(1.0, effect), 4)
+
+
 def bootstrap_ci(
     values: Sequence[float],
     *,

{whileai-0.54 → whileai-0.56}/whileai/simulations/training.py

@@ -474,6 +474,11 @@ class TrainingRun:
         else:
             self.finish("done")
 
+    @property
+    def id(self) -> str:
+        """The run id, the handle ``get_run``, ``serve`` and ``delete_run`` take."""
+        return self.run_id
+
     def __repr__(self) -> str:
         return (
             f"TrainingRun({self.run_id!r}, {self.name!r}, status={self.status!r}, step={self.step})"
@@ -825,11 +830,40 @@ def models(*, api_key: str | None = None) -> list[dict[str, Any]]:
     """The account's hosted models: ``name``, ``baseModel``, ``adapter``,
     ``adapterRunId``, ``version``, ``endpoint`` (an OpenAI-compatible
     base URL; send the account key as the bearer and ``name`` as the
-    model)."""
+    model).
+
+    A row here is a registry entry, not a running GPU: the endpoint
+    behind it idles to zero on its own and an unused model costs nothing.
+    The row stays until ``unserve(name)`` removes it; serving the same
+    name again bumps its ``version`` rather than adding a row."""
     out = _call("GET", "/models", api_key)
     return list(out.get("models") or []) if isinstance(out, dict) else []
 
 
+def unserve(
+    name: str,
+    *,
+    api_key: str | None = None,
+    transport: Callable[..., Any] | None = None,
+) -> dict[str, Any]:
+    """Stop hosting ``name``: removes the model row from the account, so
+    ``models()`` no longer lists it and its endpoint stops answering for
+    that name. The inverse of ``serve``, the way ``delete_dataset`` is the
+    inverse of ``push``. The adapter weights and the training run stay;
+    ``serve`` the run again to bring it back (at version 1).
+    Returns ``{"name": ..., "deleted": True}``."""
+    call = transport or _call
+    key = str(name).strip().lower()
+    if not key:
+        raise ValueError("unserve: name is the hosted model's name, as models() lists it")
+    out = call("DELETE", f"/models/{key}", api_key)
+    return dict(out) if isinstance(out, dict) else {"name": key, "deleted": True}
+
+
+#: Same call, the other spelling: symmetric with ``delete_dataset``.
+delete_model = unserve
+
+
 def serve(
     name: str,
     run: TrainingRun | str | None = None,
@@ -842,18 +876,36 @@ def serve(
     row; ``endpoint`` is the OpenAI-compatible base URL and ``name`` the
     model id to send. Posting an existing name bumps ``version``.
 
-    ``run`` is a ``TrainingRun`` or its id; the adapter and base model
-    come from the run record unless ``base_model`` is given. No ``run``
-    serves the bare base (``base_model`` required).
+    ``run`` is a ``TrainingRun``, the record ``get_run`` returns, or the
+    run id; the adapter and base model come from the run record unless
+    ``base_model`` is given. No ``run`` serves the bare base
+    (``base_model`` required). ``unserve`` is the inverse.
     """
     call = transport or _call
     adapter: str | None = None
     base = base_model
+    run_id: str | None
     if isinstance(run, TrainingRun):
         adapter = run.adapter
-        run_id: str | None = run.run_id
+        run_id = run.run_id
+    elif isinstance(run, dict):
+        # The record ``get_run`` returns. Reading the id here is what lets
+        # train in one process and serve in the next compose (#262).
+        run_id = str(run.get("runId") or run.get("run_id") or run.get("id") or "").strip() or None
+        if not run_id:
+            raise TypeError(
+                "run is a dict with no runId; pass the record wai.get_run(run_id) returns, "
+                "a TrainingRun, or the run id string"
+            )
+        adapter = str(run.get("adapter") or "").strip() or None
+        base = base or run.get("baseModel") or run.get("base_model")
+    elif run is None or isinstance(run, str):
+        run_id = (str(run).strip() or None) if run else None
     else:
-        run_id = str(run) if run else None
+        raise TypeError(
+            "run must be a TrainingRun, the record wai.get_run(run_id) returns, or the run "
+            f"id string; got {type(run).__name__}"
+        )
     if run_id and (adapter is None or base is None):
         meta = call("GET", f"/runs/{run_id}", api_key)
         meta = meta if isinstance(meta, dict) else {}

{whileai-0.54 → whileai-0.56/whileai.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: whileai
-Version: 0.54
+Version: 0.56
 Summary: While Python SDK: trace ingestion and verification client, plus agent simulations grounded in your tools and system prompt.
 Author: While
 License: Apache-2.0