whileai 0.55__tar.gz → 0.56__tar.gz

{whileai-0.55/whileai.egg-info → whileai-0.56}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: whileai
-Version: 0.55
+Version: 0.56
 Summary: While Python SDK: trace ingestion and verification client, plus agent simulations grounded in your tools and system prompt.
 Author: While
 License: Apache-2.0

{whileai-0.55 → whileai-0.56}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "whileai"
-version = "0.55"
+version = "0.56"
 description = "While Python SDK: trace ingestion and verification client, plus agent simulations grounded in your tools and system prompt."
 readme = "README.md"
 license = {text = "Apache-2.0"}

{whileai-0.55 → whileai-0.56}/whileai/simulations/__init__.py

@@ -135,6 +135,7 @@ from .score.optimize import (
     DEFAULT_BAND,
     filter_rl_rows,
     group_signal,
+    next_round,
     optimize,
     recommend,
     select_for_rl,
@@ -162,7 +163,9 @@ from .score.stage import STAGES, format_stages, stage_of, stage_report, stamp_st
 from .score.stats import (
     compare_runs,
     decontaminate,
+    detectable_effect,
     eval_variance,
+    holdout_size,
     marker_summary,
     metric_summary,
     task_key,
@@ -258,6 +261,7 @@ __all__ = [
     "delete_model",
     "delete_run",
     "delta_report",
+    "detectable_effect",
     "dimensions_from_traces",
     "drop_leaky_rows",
     "eval_variance",
@@ -290,6 +294,7 @@ __all__ = [
     "hf_publish",
     "hf_publish_run",
     "hf_status",
+    "holdout_size",
     "hosted_model",
     "import_hf",
     "inspect",
@@ -317,6 +322,7 @@ __all__ = [
     "mine_traces",
     "models",
     "near_duplicate_prompts",
+    "next_round",
     "normalize_judge_result",
     "novelty",
     "open_ended_probes",

{whileai-0.55 → whileai-0.56}/whileai/simulations/ingest/platform.py

@@ -22,6 +22,7 @@ import time
 import urllib.error
 import urllib.parse
 import urllib.request
+import warnings as _warnings
 from collections.abc import Sequence
 from typing import Any
 
@@ -242,6 +243,36 @@ def _meta_body(
     return body
 
 
+HOLDOUT_PROVE_EFFECT = 0.05
+
+
+def _warn_small_holdout(rows: Sequence[dict]) -> None:
+    """A holdout too small to prove a 5-point gain reads every round as
+    ``no_change_detected``; say so at push time, not after training (#257)."""
+    from ..score.stats import holdout_size, task_key
+
+    groups: dict[str, int] = {}
+    for row in rows:
+        if isinstance(row, dict):
+            key = task_key(row)
+            groups[key] = groups.get(key, 0) + 1
+    n_tasks = len(groups)
+    if not n_tasks:
+        return
+    try:
+        need = holdout_size(HOLDOUT_PROVE_EFFECT, rows=rows)
+    except ValueError:
+        need = holdout_size(HOLDOUT_PROVE_EFFECT, k=min(groups.values()))
+    if n_tasks < need["n_tasks"]:
+        _warnings.warn(
+            f"holdout has {n_tasks} tasks at k={need['k']}; proving a "
+            f"{HOLDOUT_PROVE_EFFECT:.0%} gain at 80% power needs about {need['n_tasks']} "
+            "(holdout_size). A smaller holdout reads a real gain that size as "
+            "no_change_detected.",
+            stacklevel=3,
+        )
+
+
 def push_rows(
     rows: list[dict],
     name: str,
@@ -279,6 +310,8 @@ def push_rows(
 
         gate_report = publish_gate(rows, mode=mode, endorsed=endorsed, strict_hacks=strict_hacks)
     check(rows, where="push_rows")
+    if purpose == "holdout":
+        _warn_small_holdout(rows)
     body: dict = {
         "name": name,
         **_meta_body(purpose, mode if mode in MODES else None, agent, description),

{whileai-0.55 → whileai-0.56}/whileai/simulations/score/delta.py

@@ -26,7 +26,15 @@ from collections.abc import Callable, Sequence
 from typing import Any
 
 from .passat import pass_at
-from .stats import DEFAULT_BOOT, compare_runs, eval_variance, marker_names, task_means
+from .stats import (
+    DEFAULT_BOOT,
+    compare_runs,
+    detectable_effect,
+    eval_variance,
+    holdout_size,
+    marker_names,
+    task_means,
+)
 
 GROUP_KEYS = ("delta", "ci95", "verdict", "mean_a", "mean_b", "n_used", "n_paired", "paired")
 
@@ -327,6 +335,32 @@ def delta_report(
     headline_key = target_key if target_result else "pass_at_1"
     if headline.get("note"):
         warnings.append(f"{headline_key}: {headline['note']}")
+    # Eval size: a no-change verdict is only as strong as the band the
+    # task count allows. Say what this holdout can prove and what the
+    # delta seen here would have needed (#257).
+    n_paired = int(headline.get("n_paired") or 0)
+    k_eval = int(pass_at(before).config.get("k") or 1)
+    base_rate = float(mean_a) if mean_a is not None else 0.6
+    can_prove = detectable_effect(n_paired, base=base_rate, k=k_eval) if n_paired >= 2 else None
+    tasks_needed: int | None = None
+    delta_seen: float | None = None
+    raw_delta = headline.get("delta")
+    if isinstance(raw_delta, (int, float)) and 0 < raw_delta < 1:
+        delta_seen = float(raw_delta)
+        tasks_needed = holdout_size(delta_seen, base=base_rate, k=k_eval)["n_tasks"]
+    verdict_word = (
+        target_verdict if target_result else _verdict_word(results["pass_at_1"], replicated)
+    )
+    if verdict_word == "no_change_detected" and can_prove is not None:
+        line = (
+            f"{n_paired} paired tasks at k={k_eval} can prove a gain of about "
+            f"+{can_prove:.2f} at 80% power"
+        )
+        if tasks_needed is not None and delta_seen is not None:
+            line += (
+                f"; to prove the {delta_seen:+.3f} seen here you need about {tasks_needed} tasks"
+            )
+        warnings.append(line + " (holdout_size).")
     if target_verdict == "target_not_measured":
         warnings.append(f"target {target!r} is not on both row sets")
     groups: dict[str, dict[str, Any]] | None = None
@@ -398,6 +432,8 @@ def delta_report(
         "eval_runs": eval_runs,
         "replicated": replicated,
         "ceiling": ceiling,
+        "detectable_effect": can_prove,
+        "tasks_needed": tasks_needed,
         "proxy": proxy_key,
         "proxy_verdict": proxy_verdict,
         "proxy_delta": proxy_result["delta"] if proxy_result else None,

{whileai-0.55 → whileai-0.56}/whileai/simulations/score/optimize.py

@@ -614,6 +614,105 @@ def _spread_by(prompts: list[str], key) -> list[str]:
     return out
 
 
+def _task_keys_of(tasks: Sequence[Any]) -> list[str]:
+    out: list[str] = []
+    for t in tasks:
+        if isinstance(t, dict):
+            out.append(task_key(t))
+        else:
+            out.append(task_key({"prompt": str(t)}))
+    return out
+
+
+def next_round(
+    prior: Sequence[dict],
+    *,
+    tasks: Sequence[Any] | None = None,
+    lo: float = DEFAULT_BAND[0],
+    hi: float = DEFAULT_BAND[1],
+) -> dict[str, Any]:
+    """The prompt set for the next round, from the last round's graded
+    rollouts.
+
+    A round trained on the file it started from keeps paying for groups
+    that give no gradient: at a 0.65 training reward about half the
+    groups are all-pass or all-fail. The band is the fix the book already
+    names (rlhf-book ch. 7: filter to the 20-80% band; ch. 6, DAPO's
+    dynamic sampling drops groups with no contrast), applied to what the
+    *current* policy does rather than what the base did. ``prior`` is
+    round N's graded rollouts (``simulate(tasks=..., repeats=k)`` on the
+    round-N policy, or the trainer's own sampled rows); each task's pass
+    rate over them decides: inside ``[lo, hi]`` it is kept, above ``hi``
+    it is solved and dropped, below ``lo`` it is unsolved and dropped.
+    ``tasks`` restricts the candidates (rows, task dicts with a
+    ``prompt``, or prompt strings); a task with no prior rollouts is
+    ``unknown`` and kept, since nothing says it is flat.
+
+    Returns ``tasks`` (one representative row per kept task: the prior
+    row, with ``calibration.pass_rate`` and the band), the counts
+    ``kept``, ``dropped_solved``, ``dropped_unsolved``, ``unknown``,
+    ``pass_rates`` per task, ``band``, ``from_policy`` (the policy
+    versions the prior rows came from) and ``prompt_set_sha``: the
+    identity of the kept set, for lineage on the run. Push the kept rows
+    as the next train set with ``parent=`` the last one.
+    """
+    if not 0 <= lo < hi <= 1:
+        raise ValueError("band is 0 <= lo < hi <= 1")
+    labels = _group_label_lists(prior)
+    rates = {key: sum(v) / len(v) for key, v in labels.items() if v}
+    first: dict[str, dict] = {}
+    policies: set[str] = set()
+    for row in prior:
+        if not isinstance(row, dict):
+            continue
+        first.setdefault(task_key(row), row)
+        if row.get("policy_version"):
+            policies.add(str(row["policy_version"]))
+    if tasks is None:
+        candidates = list(rates)
+        given: dict[str, Any] = {}
+    else:
+        given = {}
+        for t in tasks:
+            key = task_key(t) if isinstance(t, dict) else task_key({"prompt": str(t)})
+            given.setdefault(key, t)
+        candidates = list(given)
+    kept: list[dict] = []
+    solved = unsolved = unknown = 0
+    for key in candidates:
+        rate = rates.get(key)
+        if rate is None:
+            unknown += 1
+            rep = given.get(key)
+            rep = dict(rep) if isinstance(rep, dict) else {"prompt": str(rep)}
+            kept.append(rep)
+            continue
+        if rate > hi:
+            solved += 1
+            continue
+        if rate < lo:
+            unsolved += 1
+            continue
+        rep = dict(first.get(key) or given.get(key) or {"prompt": key})
+        cal = dict(rep.get("calibration") or {})
+        cal.update({"pass_rate": round(rate, 4), "n": len(labels[key]), "band": [lo, hi]})
+        rep["calibration"] = cal
+        kept.append(rep)
+    sha = hashlib.sha256("\n".join(sorted(task_key(r) for r in kept)).encode()).hexdigest()[:16]
+    return {
+        "tasks": kept,
+        "kept": len(kept) - unknown,
+        "dropped_solved": solved,
+        "dropped_unsolved": unsolved,
+        "unknown": unknown,
+        "n_prior_tasks": len(rates),
+        "pass_rates": {k: round(v, 4) for k, v in rates.items()},
+        "band": [lo, hi],
+        "from_policy": sorted(policies),
+        "prompt_set_sha": sha,
+    }
+
+
 def select_for_rl(
     rows: Sequence[dict],
     *,
@@ -627,9 +726,18 @@ def select_for_rl(
     endorsed: Sequence[str] = (),
     truncated: str = "drop",
     order: str = "spread",
+    prior: Sequence[dict] | None = None,
 ) -> tuple[list[dict], dict[str, Any]]:
     """Whole mixed groups up to roughly ``target`` rows. Groups never split.
 
+    ``prior`` is the previous round's graded rollouts: tasks the round-N
+    policy already solves (pass rate above ``hi`` on ``prior``) or never
+    solves (below ``lo``) are dropped before anything else, so round N+1
+    trains on what that policy gets right 20-80% of the time rather than
+    on the file round 1 started from (``next_round``; rlhf-book ch. 7).
+    The report's ``prior`` block counts kept, dropped_solved,
+    dropped_unsolved and unknown.
+
     ``truncated`` says what happens to a rollout cut at the token cap
     (rlhf-book ch. 6, DAPO's overlong handling; ch. 7 overlong filtering):
     ``"drop"`` removes it (the default; ``drop_truncated=False`` is the old
@@ -691,6 +799,24 @@ def select_for_rl(
         raise ValueError(f"order must be one of {', '.join(RL_ORDERS)}; got {order!r}")
     if not drop_truncated and truncated == "drop":
         truncated = "keep"
+    prior_report: dict[str, Any] | None = None
+    if prior is not None:
+        plan = next_round(prior, lo=lo, hi=hi)
+        rates = plan["pass_rates"]
+        before_n = len(rows)
+        rows = [
+            r
+            for r in rows
+            if not isinstance(r, dict)
+            or rates.get(task_key(r)) is None
+            or lo <= rates[task_key(r)] <= hi
+        ]
+        prior_report = {
+            k: plan[k]
+            for k in ("kept", "dropped_solved", "dropped_unsolved", "unknown", "from_policy")
+        }
+        prior_report["rows_dropped"] = before_n - len(rows)
+        prior_report["prompt_set_sha"] = plan["prompt_set_sha"]
     penalized = kept_overlong = 0
     if truncated != "drop":
         marked: list[dict] = []
@@ -838,6 +964,7 @@ def select_for_rl(
         "enforce_band": bool(enforce_band),
         "band_groups_dropped": band_report["n_groups_dropped"],
         "band_dropped": {"too_easy": band_report["too_easy"], "too_hard": band_report["too_hard"]},
+        "prior": prior_report,
         "duplicates": dup_report,
         "truncated_dropped": trunc_report["n_dropped"],
         "truncated_policy": truncated,

{whileai-0.55 → whileai-0.56}/whileai/simulations/score/stats.py

@@ -57,6 +57,116 @@ def wilson_interval(successes: int, n: int, *, z: float = 1.96) -> tuple[float,
     return (max(0.0, center - half), min(1.0, center + half))
 
 
+def _z(p: float) -> float:
+    from statistics import NormalDist
+
+    return NormalDist().inv_cdf(p)
+
+
+def _paired_task_sd(base: float, effect: float, k: int) -> float:
+    """Standard deviation of one task's paired difference (after minus
+    before pass rate over ``k`` rollouts each side) when the gain lands
+    uniformly: before at ``base``, after at ``base + effect``."""
+    p = min(1.0, max(0.0, float(base)))
+    q = min(1.0, max(0.0, p + float(effect)))
+    kk = max(1, int(k))
+    return math.sqrt((p * (1 - p) + q * (1 - q)) / kk)
+
+
+def _rows_base_and_k(rows: Sequence[dict]) -> tuple[float, int]:
+    """Mean per-task pass rate and the smallest rollouts-per-task on graded
+    rows: what ``delta_report`` would pair on."""
+    groups: dict[str, list[float]] = {}
+    for row in rows:
+        if not isinstance(row, dict):
+            continue
+        value = _binary(row)
+        if value is None:
+            continue
+        groups.setdefault(task_key(row), []).append(value)
+    if not groups:
+        raise ValueError("rows carry no 0/1 rewards; grade them first, or pass base= and k=")
+    base = _mean([_mean(v) for v in groups.values()])
+    k = min(len(v) for v in groups.values())
+    return base, k
+
+
+def holdout_size(
+    effect: float,
+    *,
+    base: float = 0.6,
+    k: int = 4,
+    power: float = 0.8,
+    alpha: float = 0.05,
+    rows: Sequence[dict] | None = None,
+) -> dict[str, Any]:
+    """How many paired tasks a holdout needs to prove a gain of ``effect``.
+
+    Models the test ``delta_report`` runs: each task's pass rate over
+    ``k`` rollouts on each side, the delta as the mean of the paired
+    differences, the interval from a bootstrap over tasks. A task's
+    difference then has standard deviation
+    ``sqrt((p(1-p) + q(1-q)) / k)`` with ``p = base`` and ``q = base +
+    effect``, and the usual two-sided power calculation gives
+    ``n = ((z_{1-alpha/2} + z_power) * sd / effect) ** 2`` (rlhf-book ch. 16,
+    appendix C: the eval's own variance decides what a delta can mean).
+    It assumes the gain lands uniformly across tasks; a gain concentrated
+    on a few tasks needs more.
+
+    ``rows`` (graded before-side rows) reads ``base`` and ``k`` off the
+    data instead. Returns ``n_tasks`` plus the inputs, ``sd_task``, and
+    ``half_width``: the 95% band on the delta at that ``n``.
+
+    The recipe that asked for this had 140 tasks at k=4 around 0.6: a
+    band of about +-0.06, so a real 3-point gain reads
+    ``no_change_detected`` every round. This says so before training.
+    """
+    if not 0 < float(effect) < 1:
+        raise ValueError(
+            "effect is the gain in pass rate to prove, between 0 and 1 (0.05 = 5 points)"
+        )
+    if not 0 < power < 1 or not 0 < alpha < 1:
+        raise ValueError("power and alpha are probabilities strictly between 0 and 1")
+    if rows is not None:
+        base, k = _rows_base_and_k(rows)
+    sd = _paired_task_sd(base, effect, k)
+    z = _z(1 - alpha / 2) + _z(power)
+    n = math.ceil((z * sd / float(effect)) ** 2) if sd > 0 else 1
+    n = max(n, 2)
+    return {
+        "n_tasks": n,
+        "effect": float(effect),
+        "base": float(base),
+        "k": int(k),
+        "power": float(power),
+        "alpha": float(alpha),
+        "sd_task": round(sd, 4),
+        "half_width": round(_z(1 - alpha / 2) * sd / math.sqrt(n), 4),
+    }
+
+
+def detectable_effect(
+    n_tasks: int,
+    *,
+    base: float = 0.6,
+    k: int = 4,
+    power: float = 0.8,
+    alpha: float = 0.05,
+) -> float | None:
+    """The smallest gain ``n_tasks`` paired tasks can prove at ``power``:
+    ``holdout_size`` solved for the effect (a few fixed-point steps, since
+    the after-side variance depends on it). ``None`` below two tasks."""
+    n = int(n_tasks)
+    if n < 2:
+        return None
+    z = _z(1 - alpha / 2) + _z(power)
+    effect = 0.0
+    for _ in range(12):
+        sd = _paired_task_sd(base, effect, k)
+        effect = z * sd / math.sqrt(n)
+    return round(min(1.0, effect), 4)
+
+
 def bootstrap_ci(
     values: Sequence[float],
     *,

{whileai-0.55 → whileai-0.56/whileai.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: whileai
-Version: 0.55
+Version: 0.56
 Summary: While Python SDK: trace ingestion and verification client, plus agent simulations grounded in your tools and system prompt.
 Author: While
 License: Apache-2.0