yoro-cache 0.1.1__py3-none-any.whl

bench/__init__.py

@@ -0,0 +1,33 @@
+"""YORO benchmark harness.
+
+A baseline ladder over labelled prompt streams, runnable locally with a mock model
+(`--smoke`, no GPU) or against any OpenAI-compatible endpoint, with per-level
+checkpoint/resume, optional cloud sinks (S3 / CloudWatch / W&B), and a hard
+auto-shutdown budget cap for rented GPUs.
+
+Modules:
+  budget   - BudgetGuard: spend tracking + auto-shutdown before the ceiling.
+  metrics  - per-prompt Outcome, run summary, cross-seed aggregation + significance.
+  ladder   - the five rungs (no-cache / exact / gptcache-semantic / behaviors / YORO).
+  wandb_log- thin W&B logger shim (falls back to stdout if wandb is absent).
+"""
+from .budget import BudgetGuard
+from .metrics import Outcome, summarize, aggregate_seeds, paired_t
+from .ladder import (Strategy, NoCache, ExactCache, SemanticCache, BehaviorsOnly,
+                     YOROStrategy, build_ladder)
+from .wandb_log import WandbLogger
+from .eventlog import EventLog, S3FileSink, CloudWatchSink
+from .checkpoint import Checkpoint
+from .convergence import Convergence, ci_halfwidth
+from .vast import VastCredit, stop_self
+
+__all__ = [
+    "BudgetGuard",
+    "Outcome", "summarize", "aggregate_seeds", "paired_t",
+    "Strategy", "NoCache", "ExactCache", "SemanticCache", "BehaviorsOnly", "YOROStrategy", "build_ladder",
+    "WandbLogger",
+    "EventLog", "S3FileSink", "CloudWatchSink",
+    "Checkpoint",
+    "Convergence", "ci_halfwidth",
+    "VastCredit", "stop_self",
+]

bench/budget.py

@@ -0,0 +1,81 @@
+"""BudgetGuard — the safety feature that makes a multi-day, unattended, $500 rented-GPU
+run safe to leave alone.
+
+It tracks spend = instance $/hr x elapsed + any per-token API cost, and once spend
+crosses a soft fraction of the hard ceiling it fires a provider-specific shutdown hook
+(e.g. `vastai destroy instance <id>`) exactly once. Pair it with frequent checkpointing
+so the auto-shutdown (or a spot preemption) never loses results.
+
+The clock is injectable so the logic is unit-testable without waiting hours.
+"""
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, field
+from typing import Callable, Optional
+
+
+@dataclass
+class BudgetGuard:
+    ceiling_usd: float                                  # HARD cap (e.g. 500)
+    hourly_usd: float                                   # instance price, e.g. Vast.ai spot $/hr
+    shutdown_frac: float = 0.9                          # auto-shutdown at 90% of the ceiling
+    clock: Callable[[], float] = time.time             # injectable for tests
+    on_shutdown: Optional[Callable[[], None]] = None    # provider terminate hook (fired once)
+    token_cost_usd: float = 0.0                         # accrued per-token cost (0 when self-hosting)
+    started_at: Optional[float] = None
+    _stopped: bool = field(default=False, repr=False)
+
+    def __post_init__(self):
+        if self.started_at is None:
+            self.started_at = self.clock()
+
+    def add_token_cost(self, usd: float) -> None:
+        self.token_cost_usd += max(0.0, usd)
+
+    def spent(self) -> float:
+        hours = max(0.0, (self.clock() - self.started_at) / 3600.0)
+        return hours * self.hourly_usd + self.token_cost_usd
+
+    def remaining(self) -> float:
+        return max(0.0, self.ceiling_usd - self.spent())
+
+    def soft_cap(self) -> float:
+        return self.ceiling_usd * self.shutdown_frac
+
+    def should_stop(self) -> bool:
+        return self.spent() >= self.soft_cap()
+
+    def check(self) -> bool:
+        """Call this periodically (e.g. every checkpoint). Returns True once spend crosses
+        the soft cap, firing the shutdown hook exactly once. Idempotent thereafter."""
+        if self._stopped:
+            return True
+        if self.should_stop():
+            self._stopped = True
+            if self.on_shutdown is not None:
+                try:
+                    self.on_shutdown()
+                except Exception:
+                    pass
+            return True
+        return False
+
+    def stop(self) -> None:
+        """Force-stop from an EXTERNAL signal (e.g. low real Vast credit), so a sweep that
+        shares this guard also halts — not just the current level."""
+        self._stopped = True
+
+    @property
+    def stopped(self) -> bool:
+        return self._stopped
+
+    def status(self) -> dict:
+        return {
+            "spent_usd": round(self.spent(), 2),
+            "remaining_usd": round(self.remaining(), 2),
+            "ceiling_usd": self.ceiling_usd,
+            "soft_cap_usd": round(self.soft_cap(), 2),
+            "hourly_usd": self.hourly_usd,
+            "stopped": self._stopped,
+        }

bench/checkpoint.py

@@ -0,0 +1,67 @@
+"""Checkpoint + resume — so a Vast.ai spot preemption or the budget auto-shutdown never
+loses work. A checkpoint is a JSON snapshot of run state (cursor into the prompt stream,
+per-rung outcomes so far, cache contents, accumulated spend). It's written ATOMICALLY
+(temp file + os.replace, so a kill mid-write can't corrupt it) and mirrored to S3, so a
+fresh instance can pull the latest and continue from the cursor.
+
+Usage:
+    ck = Checkpoint("runs/phase0/ckpt.json", s3=("my-bucket", "yoro/phase0/ckpt.json"))
+    state = ck.load() or {"cursor": 0, "outcomes": {}}
+    ...                      # run from state["cursor"], periodically:
+    ck.save(state)
+"""
+from __future__ import annotations
+
+import json
+import os
+import tempfile
+from typing import Optional, Tuple
+
+
+def _s3():
+    try:
+        import boto3
+        return boto3.client("s3")
+    except Exception:
+        return None
+
+
+class Checkpoint:
+    def __init__(self, path: str, s3: Optional[Tuple[str, str]] = None):
+        self.path = path
+        self.s3 = s3                                     # (bucket, key) or None
+        self._c = _s3() if s3 else None
+
+    def save(self, state: dict) -> None:
+        d = os.path.dirname(os.path.abspath(self.path)) or "."
+        os.makedirs(d, exist_ok=True)
+        fd, tmp = tempfile.mkstemp(dir=d, suffix=".tmp")
+        try:
+            with os.fdopen(fd, "w") as f:
+                json.dump(state, f, default=str)
+            os.replace(tmp, self.path)                   # atomic on POSIX
+        finally:
+            if os.path.exists(tmp):
+                os.remove(tmp)
+        if self._c:
+            try:
+                self._c.upload_file(self.path, self.s3[0], self.s3[1])
+            except Exception as e:
+                print(f"[ckpt s3 err {str(e)[:70]}]")
+
+    def load(self) -> Optional[dict]:
+        if os.path.exists(self.path):
+            try:
+                with open(self.path) as f:
+                    return json.load(f)
+            except Exception:
+                pass                                     # fall through to S3
+        if self._c:
+            try:
+                os.makedirs(os.path.dirname(os.path.abspath(self.path)) or ".", exist_ok=True)
+                self._c.download_file(self.s3[0], self.s3[1], self.path)   # boto3 won't mkdir the target dir
+                with open(self.path) as f:
+                    return json.load(f)
+            except Exception:
+                return None
+        return None

bench/convergence.py

@@ -0,0 +1,81 @@
+"""Convergence-based early stop — conclude when the result is statistically SETTLED, not
+when the budget runs out. $500 is the ceiling; convergence is the target.
+
+After each seed we hold the per-seed value of every metric for every rung. We track the
+95% CI half-width of the PRIMARY quantities — YORO's accuracy and hit-rate, and the
+YORO-vs-GPTCache deltas on staleness + brittleness (the headline claims). Once every
+primary CI is tighter than `ci_target` and at least `min_seeds` are in, we stop — usually
+well under `max_seeds`, saving GPU hours. Final p-values still come from scipy on the
+cluster; this is just the stopping rule.
+"""
+from __future__ import annotations
+
+import math
+
+# t_{.975, n-1} for small n (two-sided 95%); falls back to 1.96 for large n.
+_T95 = {2: 12.71, 3: 4.30, 4: 3.18, 5: 2.78, 6: 2.57, 7: 2.45, 8: 2.36, 9: 2.31,
+        10: 2.26, 12: 2.20, 15: 2.14, 20: 2.09, 25: 2.06, 30: 2.05}
+
+
+def _t95(n: int) -> float:
+    if n < 2:
+        return float("inf")
+    for k in sorted(_T95):
+        if n <= k:
+            return _T95[k]
+    return 1.96
+
+
+def ci_halfwidth(vals: list) -> float:
+    n = len(vals)
+    if n < 2:
+        return float("inf")
+    m = sum(vals) / n
+    sd = math.sqrt(sum((v - m) ** 2 for v in vals) / (n - 1))
+    return _t95(n) * sd / math.sqrt(n)
+
+
+class Convergence:
+    def __init__(self, min_seeds: int = 8, max_seeds: int = 30, ci_target: float = 0.02):
+        self.min_seeds = min_seeds
+        self.max_seeds = max_seeds
+        self.ci_target = ci_target
+
+    def primaries(self, per_seed: list) -> dict:
+        """The primary metrics whose CIs gate the early-stop: reuse rate and
+        the staleness/brittleness advantage vs GPTCache. NOTE: model accuracy is deliberately
+        NOT here — it's a property of gpt-oss (±0.05 seed noise), not of YORO, so gating on its
+        0.02-precision would force every level to max_seeds without sharpening the claim. It is
+        still measured per-seed and reported in the aggregate (with its honest CI)."""
+        if not per_seed:
+            return {}
+        rungs = per_seed[0].keys()
+        out = {}
+        if "yoro" in rungs:
+            out["yoro.hit_rate"] = [ps["yoro"]["hit_rate"] for ps in per_seed]
+        if "yoro" in rungs and "gptcache-semantic" in rungs:
+            out["delta.staleness"] = [ps["gptcache-semantic"]["staleness"] - ps["yoro"]["staleness"]
+                                      for ps in per_seed]
+            out["delta.brittleness"] = [ps["gptcache-semantic"]["brittleness"] - ps["yoro"]["brittleness"]
+                                        for ps in per_seed]
+        return out
+
+    def reported(self, per_seed: list) -> dict:
+        """CIs we LOG for transparency but do NOT gate on (model accuracy — gpt-oss noise)."""
+        if not per_seed or "yoro" not in per_seed[0]:
+            return {}
+        return {"yoro.accuracy": [ps["yoro"]["accuracy"] for ps in per_seed]}
+
+    def check(self, per_seed: list) -> tuple:
+        """Returns (stop: bool, info: dict). Stop on convergence OR max_seeds."""
+        n = len(per_seed)
+        if n >= self.max_seeds:
+            return True, {"reason": "max_seeds", "n": n}
+        if n < self.min_seeds:
+            return False, {"reason": "min_seeds_not_reached", "n": n, "need": self.min_seeds}
+        widths = {k: ci_halfwidth(v) for k, v in self.primaries(per_seed).items()}
+        extra = {k: ci_halfwidth(v) for k, v in self.reported(per_seed).items()}   # logged, not gated
+        converged = bool(widths) and all(w <= self.ci_target for w in widths.values())
+        return converged, {"reason": "converged" if converged else "not_yet", "n": n,
+                           "ci_target": self.ci_target, "gated_on": list(widths.keys()),
+                           "ci_halfwidths": {k: round(w, 4) for k, w in {**widths, **extra}.items()}}