shrewd 0.1.0__py3-none-any.whl

shrewd/__init__.py

@@ -0,0 +1,72 @@
+"""Turn LLM judgments into a small, fast, local text model for one fixed task."""
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from shrewd._autotune import autotune as autotune
+    from shrewd.decide import Choice as Choice
+    from shrewd.decide import Noul as Noul
+    from shrewd.decide import Score as Score
+    from shrewd.decisions import Decisions as Decisions
+    from shrewd.evaluate import DistillResult as DistillResult
+    from shrewd.evaluate import Finding as Finding
+    from shrewd.project import Project as Project
+    from shrewd.students import load as load
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "Choice", "Decisions", "DistillResult", "Finding", "Noul", "Project", "Score",
+    "autotune", "load",
+]
+
+
+def __dir__():
+    return sorted(set(globals()) | set(__all__))
+
+
+class _needs_teacher_extra:
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc, tb):
+        if isinstance(exc, ImportError) and exc.name in ("litellm", "gepa"):
+            raise ImportError(
+                f"training needs {exc.name}, which is not installed: "
+                'pip install "shrewd[teacher]" (inference-only installs can skip it)'
+            ) from None
+
+
+# Lazy so that `from shrewd import load` works in inference environments
+# where litellm/gepa are not installed.
+def __getattr__(name):
+    if name == "Project":
+        with _needs_teacher_extra():
+            from shrewd.project import Project
+
+        return Project
+    if name == "autotune":
+        # the submodule is named _autotune so this attribute never collides with it
+        # (a same-named submodule would shadow the function on from-imports)
+        with _needs_teacher_extra():
+            from shrewd._autotune import autotune
+
+        return autotune
+    if name == "load":
+        from shrewd.students import load
+
+        return load
+    if name in ("Choice", "Noul", "Score"):
+        from shrewd import decide
+
+        return getattr(decide, name)
+    if name == "Decisions":
+        with _needs_teacher_extra():
+            from shrewd.decisions import Decisions
+
+        return Decisions
+    if name in ("DistillResult", "Finding"):
+        from shrewd import evaluate
+
+        return getattr(evaluate, name)
+    raise AttributeError(f"module 'shrewd' has no attribute {name!r}")

shrewd/_autotune.py

@@ -0,0 +1,138 @@
+"""Budget-capped controller over the Project pipeline - cheapest teacher first, escalating
+only while the dev-split score falls short of `target` and the next tier fits the
+remaining budget. The locked test set is evaluated once, on the winner.
+"""
+
+import json
+from pathlib import Path
+
+from shrewd import data, evaluate
+from shrewd.optimize import build_seed_prompt
+from shrewd.project import Project
+from shrewd.students import STUDENTS, check_students
+from shrewd.teacher import estimate_calls, resolve_model
+
+# multiplier on the estimated labeling cost of a tier, to leave room for
+# optimize() and the final test labeling before committing to the tier
+ESTIMATE_MARGIN = 1.5
+
+
+def _spent(projects):
+    return sum(s["cost_usd"] for p in projects for s in p._manifest["stages"])
+
+
+def _dev_probe(kept, dev, student_name, classes, seed):
+    """Macro-F1 on gold dev of a student trained on pool rows alone (no dev leak)."""
+    probe = STUDENTS[student_name](seed=seed)
+    probe.fit(kept["text"].tolist(), kept["label"].tolist())
+    return evaluate.compute_metrics(
+        dev["label"], probe.predict(dev["text"].tolist()), classes
+    )["macro_f1"]
+
+
+def autotune(dir, instructions, labels, seed_df, pool_df, budget_usd, target=None,
+             teachers=("anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"),
+             students=("tfidf",), votes=1, optimize_budget=600, seed=42):
+    """Run the escalation ladder under a spending cap and return the winning pipeline.
+
+    One sub-project per teacher tier under `dir` (same seed data, so identical splits). Per
+    tier: optimize -> label -> a free sweep of `students` x min_confidence on the dev split.
+    Escalates while the best dev macro-F1 is below `target` (always if None) and the
+    estimated cost fits.
+
+    Returns {"teacher", "student", "min_confidence", "dir", "dev_macro_f1", "spent_usd",
+    "result", "trail"}. The trail is also written to `dir`/autotune_trail.json. Re-running
+    with the same `dir` resumes from caches.
+    """
+    check_students(students)  # fail on a missing optional extra before any spend
+    parent = Path(dir)
+    parent.mkdir(parents=True, exist_ok=True)
+    confidence_rungs = (0.0,) if votes == 1 else (0.0, 0.67, 1.0)
+    pool_texts = pool_df["text"].astype(str).tolist()
+    trail, projects = [], []
+    best = None
+
+    for model in (resolve_model(m) for m in teachers):
+        tier_dir = parent / model.split("/")[-1]
+        project = Project(
+            tier_dir, instructions=instructions, labels=labels, teacher=model, seed=seed
+        )
+        if not (project.dir / "seed_test.csv").exists():
+            project.add_seed(seed_df)
+
+        # budget guard - price the tier's labeling (seed prompt as proxy) before spending
+        _, estimate = estimate_calls(
+            pool_texts, build_seed_prompt(instructions, labels), model, votes,
+            project._cache(),
+        )
+        remaining = round(budget_usd - _spent(projects), 2)
+        if estimate is None:
+            trail.append({"rung": f"teacher:{model}", "decision":
+                          "no litellm price data, budget guard waived for this tier"})
+        else:
+            planned = round(estimate * ESTIMATE_MARGIN, 2)
+            if planned > remaining:
+                if best is None:
+                    raise ValueError(
+                        f"budget_usd={budget_usd} cannot cover the first tier "
+                        f"({model}: ~${planned}); nothing was spent"
+                    )
+                trail.append({"rung": f"teacher:{model}", "decision":
+                              f"skipped: needs ~${planned}, ${remaining} left"})
+                break
+
+        project.optimize(budget=optimize_budget, target=target)
+        project.label(pool_df, votes=votes)
+        projects.append(project)
+
+        labeled = data.read_csv(project.dir / "pool_labeled.csv")
+        dev = data.read_csv(project.dir / "seed_dev.csv")
+        test = data.read_csv(project.dir / "seed_test.csv")
+        valid, _ = data.usable_pool(labeled, dev, test)
+        classes = list(project.labels)
+        for min_confidence in confidence_rungs:
+            kept = valid[valid["confidence"] >= min_confidence]
+            if kept["label"].nunique() < 2:
+                trail.append({
+                    "rung": f"teacher:{model}", "min_confidence": min_confidence,
+                    "decision": "probe skipped: under two labels left after filtering",
+                })
+                continue
+            for student in students:
+                dev_f1 = _dev_probe(kept, dev, student, classes, seed)
+                trail.append({
+                    "rung": f"teacher:{model}", "student": student,
+                    "min_confidence": min_confidence, "dev_macro_f1": round(dev_f1, 4),
+                    "spent_usd": round(_spent(projects), 2),
+                })
+                if best is None or dev_f1 > best["dev_f1"]:
+                    best = {"project": project, "student": student,
+                            "min_confidence": min_confidence, "dev_f1": dev_f1}
+        if target is not None and best is not None and best["dev_f1"] >= target:
+            trail.append({"rung": f"teacher:{model}",
+                          "decision": f"dev target {target} reached, stopping early"})
+            break
+
+    if best is None:
+        (parent / "autotune_trail.json").write_text(json.dumps(trail, indent=2))
+        raise RuntimeError(
+            "no configuration could be probed: every rung left fewer than two distinct "
+            "labels (the teacher may be collapsing to one class). See "
+            f"{parent / 'autotune_trail.json'} and the per-tier pool_labeled.csv files."
+        )
+    # the single test-set evaluation, on the winning configuration only
+    result = best["project"].distill(
+        student=best["student"], min_confidence=best["min_confidence"]
+    )
+    summary = {
+        "teacher": best["project"].teacher,
+        "student": best["student"],
+        "min_confidence": best["min_confidence"],
+        "dir": str(best["project"].dir),
+        "dev_macro_f1": round(best["dev_f1"], 4),
+        "spent_usd": round(_spent(projects), 2),
+    }
+    trail.append({"rung": "final", **summary,
+                  "test_macro_f1": round(result.metrics["student"]["macro_f1"], 4)})
+    (parent / "autotune_trail.json").write_text(json.dumps(trail, indent=2))
+    return {**summary, "result": result, "trail": trail}

shrewd/acquire.py

@@ -0,0 +1,54 @@
+"""Pick which pool rows are worth a teacher call. The ones the current tfidf probe is least
+sure about, spread across clusters. Measured at 1.1-2.3x fewer labels than file order for
+the same student accuracy. The probe's picks were about as useful to the embedding
+student as random order, so the probe does not commit you to shipping tfidf.
+"""
+
+import numpy as np
+from sklearn.cluster import KMeans
+from sklearn.decomposition import TruncatedSVD
+
+from shrewd.students import TfidfStudent
+
+
+def default_batch(n_pool):
+    return int(min(500, max(50, n_pool // 10)))
+
+
+def fit_probe(train_texts, train_labels, seed):
+    probe = TfidfStudent(seed=seed)
+    probe.fit(train_texts, train_labels)
+    return probe
+
+
+def select(probe, candidates, batch, seed):
+    """Return indices into `candidates` for the next batch.
+
+    Rank by margin between the top two probabilities (small margin = unsure), take the
+    4x most uncertain as a shortlist, cluster the shortlist and keep the most uncertain
+    row from each cluster.
+    """
+    if batch >= len(candidates):
+        return list(range(len(candidates)))
+    proba = probe.predict_proba(candidates)
+    top2 = np.sort(proba, axis=1)[:, -2:]
+    margin = top2[:, 1] - top2[:, 0]
+    order = np.argsort(margin)
+    shortlist = order[: min(len(candidates), 4 * batch)]
+    if len(shortlist) <= batch:
+        return [int(i) for i in shortlist]
+    features = probe._pipe.named_steps["features"].transform([candidates[i] for i in shortlist])
+    dims = min(64, features.shape[1] - 1, len(shortlist) - 1)
+    if dims < 2:
+        return [int(i) for i in order[:batch]]
+    z = TruncatedSVD(dims, random_state=seed).fit_transform(features)
+    clusters = KMeans(n_clusters=batch, n_init=3, random_state=seed).fit_predict(z)
+    picked = []
+    for c in range(batch):
+        members = np.where(clusters == c)[0]
+        if len(members):
+            picked.append(int(shortlist[members[np.argmin(margin[shortlist[members]])]]))
+    if len(picked) < batch:  # empty clusters: top up by uncertainty
+        taken = set(picked)
+        picked += [int(i) for i in order if int(i) not in taken][: batch - len(picked)]
+    return picked

shrewd/calibrate.py

@@ -0,0 +1,375 @@
+"""Post-hoc calibration of a student's probabilities.
+
+With more than two options the calibration map runs on the top score, so the predicted
+option never changes. For yes/no questions the calibrated quantity is P(yes) itself and it
+may cross 0.5: a question with a 3% base rate should not sit at 0.5.
+See `cross_fit_proba` for which rows the calibrator is fit on.
+"""
+
+import numpy as np
+from scipy.optimize import minimize_scalar
+from sklearn.isotonic import IsotonicRegression
+from sklearn.linear_model import LogisticRegression
+from sklearn.model_selection import StratifiedKFold
+
+EPS = 1e-12
+METHODS = ("temperature", "isotonic", "platt", "none")
+
+
+def _logits(proba):
+    return np.log(np.clip(np.asarray(proba, dtype=float), EPS, 1.0))
+
+
+def _softmax(z):
+    z = z - z.max(axis=1, keepdims=True)
+    e = np.exp(z)
+    return e / e.sum(axis=1, keepdims=True)
+
+
+def _onehot(y_idx, k):
+    out = np.zeros((len(y_idx), k))
+    out[np.arange(len(y_idx)), y_idx] = 1.0
+    return out
+
+
+def encode(labels, classes):
+    """Label strings to column indices. Labels outside `classes` come back as -1."""
+    index = {c: i for i, c in enumerate(classes)}
+    return np.array([index.get(str(y), -1) for y in labels])
+
+
+# ---------------------------------------------------------------- scoring rules
+
+
+def nll(proba, y_idx):
+    """Mean negative log likelihood of the true class (log loss)."""
+    proba = np.clip(np.asarray(proba, dtype=float), EPS, 1.0)
+    return float(-np.mean(np.log(proba[np.arange(len(y_idx)), y_idx])))
+
+
+def brier(proba, y_idx):
+    """Multiclass Brier score: mean squared error over the whole probability vector.
+
+    Ranges 0 (perfect) to 2 (confidently wrong). Unlike ECE this is a proper scoring
+    rule, so it can't be gamed by reporting the base rate on every row.
+    """
+    proba = np.asarray(proba, dtype=float)
+    return float(np.mean(((proba - _onehot(y_idx, proba.shape[1])) ** 2).sum(axis=1)))
+
+
+def ece(confidence, correct, bins=10):
+    """Expected calibration error with equal-mass bins, so the number does not swing with the
+    bin count when most rows sit in one confidence band.
+    """
+    confidence = np.asarray(confidence, dtype=float)
+    correct = np.asarray(correct, dtype=float)
+    if len(confidence) == 0:
+        return float("nan")
+    order = np.argsort(confidence, kind="mergesort")
+    conf, corr = confidence[order], correct[order]
+    total = 0.0
+    for chunk in np.array_split(np.arange(len(conf)), min(bins, len(conf))):
+        if len(chunk):
+            total += len(chunk) / len(conf) * abs(conf[chunk].mean() - corr[chunk].mean())
+    return float(total)
+
+
+def reliability_table(confidence, correct, bins=10):
+    """Per-bin stated confidence vs. measured accuracy: the reliability diagram as rows."""
+    confidence = np.asarray(confidence, dtype=float)
+    correct = np.asarray(correct, dtype=float)
+    order = np.argsort(confidence, kind="mergesort")
+    conf, corr = confidence[order], correct[order]
+    rows = []
+    for chunk in np.array_split(np.arange(len(conf)), min(bins, max(len(conf), 1))):
+        if not len(chunk):
+            continue
+        rows.append(
+            {
+                "low": round(float(conf[chunk].min()), 4),
+                "high": round(float(conf[chunk].max()), 4),
+                "n": int(len(chunk)),
+                "stated": round(float(conf[chunk].mean()), 4),
+                "actual": round(float(corr[chunk].mean()), 4),
+            }
+        )
+    return rows
+
+
+def aurc(confidence, correct):
+    """Area under the risk-coverage curve: mean error rate over every prefix of rows
+    sorted most-confident-first. Lower is better. It rewards a confidence signal that
+    puts the mistakes at the bottom, independently of whether the scale is calibrated.
+    """
+    confidence = np.asarray(confidence, dtype=float)
+    correct = np.asarray(correct, dtype=float)
+    if len(confidence) == 0:
+        return float("nan")
+    order = np.argsort(-confidence, kind="mergesort")
+    errors = 1.0 - correct[order]
+    return float(np.mean(np.cumsum(errors) / np.arange(1, len(errors) + 1)))
+
+
+def calibration_metrics(proba, labels, classes, bins=10):
+    """Every calibration number for one set of predictions, as a plain dict."""
+    proba = np.asarray(proba, dtype=float)
+    y_idx = encode(labels, classes)
+    keep = y_idx >= 0
+    proba, y_idx = proba[keep], y_idx[keep]
+    if len(y_idx) == 0:
+        return {}
+    # for a yes/no question the reliability curve that matters runs on P(yes) across
+    # every row, not on the winning side's confidence: with a 3% base rate the latter
+    # is dominated by easy "no"s and looks excellent while P(yes) is off by 10x
+    if len(classes) == 2:
+        confidence, correct = proba[:, 1], (y_idx == 1).astype(float)
+    else:
+        confidence = proba.max(axis=1)
+        correct = (proba.argmax(axis=1) == y_idx).astype(float)
+    accuracy = float((proba.argmax(axis=1) == y_idx).mean())
+    return {
+        "n": int(len(y_idx)),
+        "accuracy": round(accuracy, 4),
+        "mean_confidence": round(float(confidence.mean()), 4),
+        "overconfidence": round(float(confidence.mean() - correct.mean()), 4),
+        "ece": round(ece(confidence, correct, bins), 4),
+        "brier": round(brier(proba, y_idx), 4),
+        "nll": round(nll(proba, y_idx), 4),
+        "aurc": round(aurc(confidence, correct), 4),
+        "reliability": reliability_table(confidence, correct, bins),
+    }
+
+
+# ---------------------------------------------------------------- the calibrator
+
+
+class Calibrator:
+    """A fitted, monotone rescaling of a student's probabilities. Serializes to JSON.
+
+    `temperature` (one scalar), `platt` (two parameters on the log-odds, the right choice
+    for a lopsided yes/no question) or `isotonic` (a step function, needs thousands of rows
+    and collapses on a few dozen).
+    """
+
+    def __init__(self, method, params, n_fit=0, classes=None):
+        self.method = method
+        self.params = params
+        self.n_fit = n_fit
+        self.classes = classes
+
+    # -- fitting ---------------------------------------------------------------
+
+    @classmethod
+    def fit(cls, proba, labels, classes, method="auto", seed=0):
+        """Fit on (probabilities, true labels). `method="auto"` picks by cross-validated
+        log loss among temperature/platt/isotonic/none, which keeps a small or degenerate
+        calibration set from choosing a flexible method it cannot support."""
+        proba = np.asarray(proba, dtype=float)
+        y_idx = encode(labels, classes)
+        keep = y_idx >= 0
+        proba, y_idx = proba[keep], y_idx[keep]
+        if len(y_idx) < 20 or len(np.unique(y_idx)) < 2:
+            return cls("none", {}, n_fit=int(len(y_idx)), classes=list(classes))
+        if method == "auto":
+            method = cls._pick(proba, y_idx, classes, seed)
+        params = cls._fit_params(method, proba, y_idx, len(classes))
+        return cls(method, params, n_fit=int(len(y_idx)), classes=list(classes))
+
+    @staticmethod
+    def _fit_params(method, proba, y_idx, k):
+        """Fit one method's parameters. Multi-class maps the top score and keeps the argmax;
+        two-class maps P(positive) directly and may cross 0.5.
+        """
+        if method == "none":
+            return {}
+        if method == "temperature":
+            return {"temperature": _fit_temperature(_logits(proba), _onehot(y_idx, k))}
+        binary = k == 2
+        if binary:
+            score, target = proba[:, 1], (y_idx == 1).astype(float)
+        else:
+            score, target = proba.max(axis=1), (proba.argmax(axis=1) == y_idx).astype(float)
+        if method == "platt":
+            a, b = _fit_platt(score, target)
+            return {"a": a, "b": b, "binary": binary}
+        if method == "isotonic":
+            iso = IsotonicRegression(out_of_bounds="clip", y_min=0.0, y_max=1.0)
+            iso.fit(score, target)
+            return {
+                "x": [round(float(v), 6) for v in iso.X_thresholds_],
+                "y": [round(float(v), 6) for v in iso.y_thresholds_],
+                "binary": binary,
+            }
+        raise ValueError(f"unknown calibration method: {method!r} (expected one of {METHODS})")
+
+    @classmethod
+    def _pick(cls, proba, y_idx, classes, seed, n_splits=4):
+        """Cross-validated log loss over the calibration rows themselves."""
+        n_splits = max(2, min(n_splits, int(np.bincount(y_idx).min()) if len(y_idx) else 2))
+        try:
+            folds = list(StratifiedKFold(n_splits, shuffle=True, random_state=seed).split(
+                proba, y_idx
+            ))
+        except ValueError:
+            return "temperature"
+        scores = {}
+        for method in METHODS:
+            total, ok = 0.0, True
+            for train, test in folds:
+                try:
+                    params = cls._fit_params(method, proba[train], y_idx[train], len(classes))
+                    held = cls(method, params, classes=list(classes)).transform(proba[test])
+                except (ValueError, FloatingPointError):
+                    ok = False
+                    break
+                total += nll(held, y_idx[test]) * len(test)
+            if ok:
+                scores[method] = total / len(y_idx)
+        if not scores:
+            return "temperature"
+        best = min(scores, key=scores.get)
+        # a tie goes to the simpler method: temperature generalizes off-distribution
+        # better than isotonic and there is no reason to pay for flexibility you don't use
+        for method in METHODS:
+            if method in scores and scores[method] <= scores[best] + 1e-4:
+                return method
+        return best
+
+    # -- applying --------------------------------------------------------------
+
+    def transform(self, proba):
+        """Rescale probabilities. Row-stochastic in, row-stochastic out."""
+        proba = np.clip(np.asarray(proba, dtype=float), EPS, 1.0)
+        if self.method == "none":
+            return proba / proba.sum(axis=1, keepdims=True)
+        if self.method == "temperature":
+            return _softmax(_logits(proba) / self.params["temperature"])
+        binary = bool(self.params.get("binary")) and proba.shape[1] == 2
+        score = proba[:, 1] if binary else proba.max(axis=1)
+        if self.method == "platt":
+            mapped = _apply_platt(score, self.params["a"], self.params["b"])
+        elif self.method == "isotonic":
+            mapped = np.interp(score, self.params["x"], self.params["y"])
+        else:
+            raise ValueError(f"unknown calibration method: {self.method!r}")
+        mapped = np.clip(mapped, EPS, 1.0 - EPS)
+        if binary:
+            return np.column_stack([1.0 - mapped, mapped])
+        return _rescale_top(proba, mapped)
+
+    # -- persistence -----------------------------------------------------------
+
+    def to_dict(self):
+        return {
+            "method": self.method,
+            "params": self.params,
+            "n_fit": self.n_fit,
+            "classes": self.classes,
+        }
+
+    @classmethod
+    def from_dict(cls, blob):
+        return cls(
+            blob["method"], blob.get("params", {}), blob.get("n_fit", 0), blob.get("classes")
+        )
+
+    def __repr__(self):
+        detail = ""
+        if self.method == "temperature":
+            detail = f" T={self.params['temperature']:.3f}"
+        return f"<Calibrator {self.method}{detail} fit on {self.n_fit} rows>"
+
+
+def _rescale_top(proba, new_top):
+    """Set the winning class to `new_top` and spread the rest proportionally. `new_top` is
+    floored where the runner-up would draw level, so the winning option never changes.
+    """
+    out = np.array(proba, dtype=float, copy=True)
+    rows = np.arange(len(out))
+    win = out.argmax(axis=1)
+    top = out[rows, win]
+    rest = out.sum(axis=1) - top
+    other_max = np.where(out.shape[1] > 1, (out - np.eye(out.shape[1])[win] * out).max(axis=1), 0.0)
+    floor = np.where(rest > EPS, other_max / np.maximum(rest + other_max, EPS), 0.0)
+    new_top = np.maximum(new_top, np.minimum(floor + 1e-9, 1.0 - EPS))
+    scale = np.where(rest > EPS, (1.0 - new_top) / np.maximum(rest, EPS), 0.0)
+    out *= scale[:, None]
+    out[rows, win] = new_top
+    total = out.sum(axis=1, keepdims=True)
+    return out / np.maximum(total, EPS)
+
+
+def _fit_temperature(logits, onehot, bounds=(-3.0, 3.0)):
+    """The temperature minimizing log loss. Optimized over log T so it stays positive."""
+
+    def objective(log_t):
+        z = logits / np.exp(log_t)
+        z = z - z.max(axis=1, keepdims=True)
+        log_p = z - np.log(np.exp(z).sum(axis=1, keepdims=True))
+        return -np.mean((onehot * log_p).sum(axis=1))
+
+    result = minimize_scalar(objective, bounds=bounds, method="bounded")
+    return round(float(np.exp(result.x)), 6)
+
+
+def _fit_platt(confidence, correct):
+    """Two-parameter logistic on the top score's log-odds.
+
+    Falls back to the identity map when the calibration rows are all right or all
+    wrong: there is no slope to estimate from one class, and a logistic fit would
+    either refuse outright or run its coefficients off to infinity.
+    """
+    correct = correct.astype(int)
+    if len(np.unique(correct)) < 2:
+        return 1.0, 0.0
+    x = np.log(np.clip(confidence, EPS, 1 - 1e-9) / np.clip(1 - confidence, EPS, 1.0))
+    model = LogisticRegression(C=1e6, solver="lbfgs", max_iter=1000)
+    model.fit(x.reshape(-1, 1), correct)
+    return round(float(model.coef_[0][0]), 6), round(float(model.intercept_[0]), 6)
+
+
+def _apply_platt(confidence, a, b):
+    x = np.log(np.clip(confidence, EPS, 1 - 1e-9) / np.clip(1 - confidence, EPS, 1.0))
+    return 1.0 / (1.0 + np.exp(-(a * x + b)))
+
+
+# ---------------------------------------------------------------- the right rows
+
+
+def cross_fit_proba(make_student, texts, labels, classes, n_splits=5, seed=0, desc=None):
+    """Out-of-fold probabilities over the teacher-labeled pool: K refits, each scoring the
+    rows it did not train on. No API calls.
+
+    Fitting on the student's own training rows gives a temperature near 1 (memorized
+    scores). Fitting on the seed dev split (80-200 easy rows) made two of three datasets
+    worse. The labels are the teacher's, so this calibrates to agreement with the teacher;
+    the report says so when the teacher is the weak link.
+    """
+    texts, labels = list(texts), [str(y) for y in labels]
+    index = {c: i for i, c in enumerate(classes)}
+    y = np.array([index.get(label, -1) for label in labels])
+    keep = np.where(y >= 0)[0]
+    if len(keep) < 40:
+        return None, None
+    counts = np.bincount(y[keep], minlength=len(classes))
+    n_splits = int(max(2, min(n_splits, counts[counts > 0].min())))
+    if n_splits < 2:
+        return None, None
+
+    sub_texts = [texts[i] for i in keep]
+    sub_labels = [labels[i] for i in keep]
+    oof = np.zeros((len(keep), len(classes)))
+    splitter = StratifiedKFold(n_splits, shuffle=True, random_state=seed)
+    folds = list(splitter.split(np.zeros(len(keep)), y[keep]))
+    for fold, (train, test) in enumerate(folds, 1):
+        if desc:
+            print(f"  {desc}: calibration fold {fold}/{len(folds)}", flush=True)
+        student = make_student()
+        student.fit([sub_texts[i] for i in train], [sub_labels[i] for i in train])
+        proba = student.predict_proba([sub_texts[i] for i in test])
+        for j, cls in enumerate(student.classes_):
+            if cls in index:
+                oof[test, index[cls]] = proba[:, j]
+    total = oof.sum(axis=1, keepdims=True)
+    oof = np.divide(oof, total, out=np.full_like(oof, 1.0 / len(classes)), where=total > EPS)
+    return oof, sub_labels