tare-cli 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tare/__init__.py +1 -0
- tare/agents.py +672 -0
- tare/calibrate.py +98 -0
- tare/capsule.py +246 -0
- tare/cli.py +312 -0
- tare/cliff.py +183 -0
- tare/dashboard.html +269 -0
- tare/dashboard.py +216 -0
- tare/fake.py +183 -0
- tare/journal.py +35 -0
- tare/judge.py +108 -0
- tare/overview.html +78 -0
- tare/probe.py +273 -0
- tare/recipe.py +102 -0
- tare/room.py +67 -0
- tare/swap.py +219 -0
- tare/trail.py +203 -0
- tare/usage.py +237 -0
- tare_cli-0.2.0.dist-info/METADATA +505 -0
- tare_cli-0.2.0.dist-info/RECORD +23 -0
- tare_cli-0.2.0.dist-info/WHEEL +4 -0
- tare_cli-0.2.0.dist-info/entry_points.txt +2 -0
- tare_cli-0.2.0.dist-info/licenses/LICENSE +21 -0
tare/cliff.py
ADDED
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
"""Cliff: where did a failed run become lost (epic Cliff).
|
|
2
|
+
|
|
3
|
+
The original run fails its check. Tails resumed from each capsule show how often the run
|
|
4
|
+
could still have passed from that point. The baseline is a fresh start (step 0). A step
|
|
5
|
+
is high when its tails pass at least half as often as the baseline, low otherwise.
|
|
6
|
+
Bisection narrows the gap between the last high and the first low step; then more tails
|
|
7
|
+
go to that pair until their Wilson intervals separate or the budget runs out. The result
|
|
8
|
+
is a range, not a single step, unless the evidence says more.
|
|
9
|
+
"""
|
|
10
|
+
import math
|
|
11
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
12
|
+
from dataclasses import dataclass, field
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
from . import capsule as caps
|
|
16
|
+
from . import usage as usages
|
|
17
|
+
from .journal import Journal
|
|
18
|
+
|
|
19
|
+
Z = 1.96 # 95 % intervals
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def wilson(passes: int, n: int) -> tuple[float, float]:
|
|
23
|
+
if n == 0:
|
|
24
|
+
return 0.0, 1.0
|
|
25
|
+
p = passes / n
|
|
26
|
+
denominator = 1 + Z * Z / n
|
|
27
|
+
centre = (p + Z * Z / (2 * n)) / denominator
|
|
28
|
+
half = Z * math.sqrt(p * (1 - p) / n + Z * Z / (4 * n * n)) / denominator
|
|
29
|
+
return max(0.0, centre - half), min(1.0, centre + half)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass
|
|
33
|
+
class Search:
|
|
34
|
+
results: dict[int, list[bool]] = field(default_factory=dict)
|
|
35
|
+
verdict: str = ""
|
|
36
|
+
cliff: tuple[int, int] | None = None # (last high step, first low step)
|
|
37
|
+
separated: bool = False
|
|
38
|
+
exhausted: bool = False
|
|
39
|
+
rebounds: list[int] = field(default_factory=list) # high steps after the first low one
|
|
40
|
+
spent: int = 0
|
|
41
|
+
|
|
42
|
+
def rate(self, step: int) -> float:
|
|
43
|
+
return sum(self.results[step]) / len(self.results[step])
|
|
44
|
+
|
|
45
|
+
def interval(self, step: int) -> tuple[float, float]:
|
|
46
|
+
return wilson(sum(self.results[step]), len(self.results[step]))
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def search(probe, last: int, tails: int, budget: int, gap_below: float = 0.2) -> Search:
|
|
50
|
+
"""probe(step, n) runs n tails from a capsule and returns whether each passed."""
|
|
51
|
+
s = Search()
|
|
52
|
+
|
|
53
|
+
def run(step: int, n: int) -> bool:
|
|
54
|
+
n = min(n, budget - s.spent)
|
|
55
|
+
if n <= 0:
|
|
56
|
+
s.exhausted = True
|
|
57
|
+
return False
|
|
58
|
+
s.results.setdefault(step, []).extend(probe(step, n))
|
|
59
|
+
s.spent += n
|
|
60
|
+
return True
|
|
61
|
+
|
|
62
|
+
run(0, tails)
|
|
63
|
+
# 0/3 still allows a rate of 0.56: sample the baseline until a gap is clear, or it passes once
|
|
64
|
+
while s.rate(0) == 0 and s.interval(0)[1] >= gap_below and run(0, tails):
|
|
65
|
+
pass
|
|
66
|
+
if s.rate(0) == 0:
|
|
67
|
+
n, hi = len(s.results[0]), s.interval(0)[1]
|
|
68
|
+
clear = hi < gap_below
|
|
69
|
+
s.verdict = (f"A fresh start fails too: {n} of {n} failed, so the upper end of the baseline's 95% interval is {hi:.2f}. The task "
|
|
70
|
+
"is too hard for the model (a model gap), so there is no cliff to find." if clear else
|
|
71
|
+
f"No fresh start passed in {n} tails, so the upper end of the baseline's 95% interval is {hi:.2f}. The budget ran "
|
|
72
|
+
"out before a model gap was clear.")
|
|
73
|
+
return s
|
|
74
|
+
run(last, tails)
|
|
75
|
+
unlucky = ("Continued from its last step, the run still passes at least half as often as a fresh start. "
|
|
76
|
+
"The original run was unlucky, not lost.")
|
|
77
|
+
|
|
78
|
+
while True:
|
|
79
|
+
tau = s.rate(0) / 2
|
|
80
|
+
probed = sorted(s.results)
|
|
81
|
+
lows = [step for step in probed if step > 0 and s.rate(step) < tau]
|
|
82
|
+
if not lows:
|
|
83
|
+
s.cliff, s.verdict = None, unlucky
|
|
84
|
+
return s
|
|
85
|
+
low = min(lows)
|
|
86
|
+
high = max(step for step in probed if step < low and s.rate(step) >= tau)
|
|
87
|
+
s.cliff = (high, low)
|
|
88
|
+
if low - high > 1:
|
|
89
|
+
if not run((high + low) // 2, tails):
|
|
90
|
+
break
|
|
91
|
+
continue
|
|
92
|
+
if s.interval(high)[0] > s.interval(low)[1]:
|
|
93
|
+
s.separated = True
|
|
94
|
+
break
|
|
95
|
+
if not (run(high, tails) and run(low, tails)):
|
|
96
|
+
break
|
|
97
|
+
s.rebounds = [step for step in sorted(s.results) if step > s.cliff[1] and s.rate(step) >= s.rate(0) / 2]
|
|
98
|
+
|
|
99
|
+
high, low = s.cliff
|
|
100
|
+
if low - high > 1:
|
|
101
|
+
s.verdict = f"Lost somewhere between step {high} and step {low}. The budget ran out before the range narrowed."
|
|
102
|
+
elif s.separated:
|
|
103
|
+
s.verdict = f"The run became lost at step {low}."
|
|
104
|
+
else:
|
|
105
|
+
s.verdict = (f"Most likely lost at step {low}. The intervals of steps {high} and {low} still overlap, and the "
|
|
106
|
+
"budget ran out.")
|
|
107
|
+
return s
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def cliff(agent, real, project: Path, prompt: str, check: str, out: Path, *, tails: int = 3, budget: int = 30,
|
|
111
|
+
jobs: int = 3, claude_args: list[str] | None = None, env: dict[str, str] | None = None,
|
|
112
|
+
journal: Journal | None = None, gap_below: float = 0.2) -> str:
|
|
113
|
+
"""The whole move: original run, capsules, baseline, search, report. Returns the report."""
|
|
114
|
+
claude_args = claude_args or []
|
|
115
|
+
journal = journal or Journal(None)
|
|
116
|
+
journal("start", kind="cliff", task=prompt, check=check, project=str(project), tails=tails, budget=budget,
|
|
117
|
+
sides={"a": {"agent": agent.name, "args": claude_args, "dir": ".", "billing": usages.billing(agent, real)}})
|
|
118
|
+
journal("phase", phase="original run")
|
|
119
|
+
caps.record(agent, real, project, out, prompt, claude_args, env)
|
|
120
|
+
passed, detail, spent = caps.check_original(agent, out, check)
|
|
121
|
+
header = [f"tare cliff · {agent.name} · {project}", f" task {prompt}", f" check {check}"]
|
|
122
|
+
steps = caps.capsules(out, agent)
|
|
123
|
+
journal("original", side="a", passed=passed, detail=detail, steps=[s.tool for s in steps], **spent)
|
|
124
|
+
if passed:
|
|
125
|
+
return _write(out, header + [f" original passed ({detail}): there is no cliff to find"], journal)
|
|
126
|
+
header.append(f" original failed ({detail}) after {len(steps) - 1} steps")
|
|
127
|
+
if len(steps) == 1:
|
|
128
|
+
return _write(out, header + [" The run made no tool calls, so there is no step to continue from."], journal)
|
|
129
|
+
journal("phase", phase="tails")
|
|
130
|
+
|
|
131
|
+
counts: dict[int, int] = {}
|
|
132
|
+
events = agent.trail(caps.session_lines(out)).events
|
|
133
|
+
|
|
134
|
+
def tail(step: int, i: int) -> caps.Tail:
|
|
135
|
+
native = step == 0 or agent.native_resume
|
|
136
|
+
name = f"{step:04d}-{i}"
|
|
137
|
+
journal("tail", id=name, status="running", step=step, room="a", agent="a", agent_name=agent.name,
|
|
138
|
+
kind="native" if native else "handoff", dir=f"tails/{name}")
|
|
139
|
+
if native:
|
|
140
|
+
result = caps.run_tail(agent, real, out, steps[step], prompt, claude_args, check, i, env)
|
|
141
|
+
else:
|
|
142
|
+
result = caps.run_handoff(agent, real, out / "tails" / name, steps[step], events, prompt,
|
|
143
|
+
claude_args, check, env)
|
|
144
|
+
journal("tail", id=name, status="passed" if result.passed else "failed", detail=result.detail,
|
|
145
|
+
usage=result.usage, check_usage=result.check_usage)
|
|
146
|
+
return result
|
|
147
|
+
|
|
148
|
+
def probe(step: int, n: int) -> list[bool]:
|
|
149
|
+
start = counts.get(step, 0)
|
|
150
|
+
counts[step] = start + n
|
|
151
|
+
with ThreadPoolExecutor(max_workers=jobs) as pool:
|
|
152
|
+
return [t.passed for t in pool.map(lambda i: tail(step, i), range(start, start + n))]
|
|
153
|
+
|
|
154
|
+
s = search(probe, len(steps) - 1, tails, budget, gap_below)
|
|
155
|
+
return _write(out, header + report(s, steps, budget), journal)
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def report(s: Search, steps: list, budget: int) -> list[str]:
|
|
159
|
+
lines = ["", " step what the step did tails pass rate 95% interval"]
|
|
160
|
+
for step in sorted(s.results):
|
|
161
|
+
lo, hi = s.interval(step)
|
|
162
|
+
marker = " <" if s.cliff and step == s.cliff[1] else ""
|
|
163
|
+
lines.append(f" {step:>4} {steps[step].tool[:52]:<52} {len(s.results[step]):>5} {sum(s.results[step]):>4}"
|
|
164
|
+
f" {s.rate(step):.2f} {lo:.2f}-{hi:.2f}{marker}")
|
|
165
|
+
lines += ["", f" verdict {s.verdict}"]
|
|
166
|
+
if s.cliff and s.cliff[1] - s.cliff[0] == 1:
|
|
167
|
+
step = steps[s.cliff[1]]
|
|
168
|
+
changed = caps.changed_files(steps[s.cliff[0]].archive, step.archive)
|
|
169
|
+
lines.append(f" at step {step.step}: {step.tool}")
|
|
170
|
+
lines.append(f" changed {', '.join(changed[:10]) or 'no files (the conversation changed, not the workspace)'}")
|
|
171
|
+
if s.cliff:
|
|
172
|
+
lines.append(" monotone " + ("yes: no step after the cliff passes again" if not s.rebounds else
|
|
173
|
+
f"no: steps {', '.join(map(str, s.rebounds))} pass again after the cliff"))
|
|
174
|
+
lines.append(f" tails {s.spent} of a budget of {budget}")
|
|
175
|
+
return lines
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _write(out: Path, lines: list[str], journal: Journal) -> str:
|
|
179
|
+
text = "\n".join(lines + usages.report(out)) + "\n"
|
|
180
|
+
(out / "report.txt").write_text(text)
|
|
181
|
+
journal("report", text=text)
|
|
182
|
+
journal("phase", phase="done")
|
|
183
|
+
return text
|
tare/dashboard.html
ADDED
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
<!doctype html>
|
|
2
|
+
<html lang="en">
|
|
3
|
+
<head>
|
|
4
|
+
<meta charset="utf-8">
|
|
5
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
6
|
+
<title>tare live</title>
|
|
7
|
+
<style>
|
|
8
|
+
:root {
|
|
9
|
+
--bg: #0e1116; --panel: #161b22; --panel-2: #1c222b; --line: #2a313c; --text: #e6edf3; --muted: #8b96a5;
|
|
10
|
+
--pass: #3fb950; --fail: #f85149; --run: #d29922; --accent: #79c0ff; --threshold: #a371f7;
|
|
11
|
+
--mono: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
|
|
12
|
+
}
|
|
13
|
+
@media (prefers-color-scheme: light) {
|
|
14
|
+
:root { --bg: #f6f8fa; --panel: #ffffff; --panel-2: #f0f3f6; --line: #d0d7de; --text: #1f2328; --muted: #59636e;
|
|
15
|
+
--pass: #1a7f37; --fail: #cf222e; --run: #9a6700; --accent: #0969da; --threshold: #8250df; }
|
|
16
|
+
}
|
|
17
|
+
* { box-sizing: border-box; }
|
|
18
|
+
body { margin: 0; background: var(--bg); color: var(--text); font: 14px/1.45 system-ui, -apple-system, "Segoe UI", sans-serif; }
|
|
19
|
+
main { max-width: 1180px; margin: 0 auto; padding: 20px 16px 60px; }
|
|
20
|
+
header { display: flex; flex-wrap: wrap; align-items: center; gap: 12px; margin-bottom: 16px; }
|
|
21
|
+
h1 { font: 600 22px/1 var(--mono); margin: 0; letter-spacing: .5px; }
|
|
22
|
+
h2 { font-size: 13px; text-transform: uppercase; letter-spacing: .08em; color: var(--muted); margin: 0 0 10px; font-weight: 600; }
|
|
23
|
+
.chip { display: inline-flex; align-items: center; gap: 6px; padding: 3px 10px; border-radius: 999px; background: var(--panel-2);
|
|
24
|
+
border: 1px solid var(--line); font-size: 12px; white-space: nowrap; }
|
|
25
|
+
.dot { width: 8px; height: 8px; border-radius: 50%; background: var(--muted); display: inline-block; flex: none; }
|
|
26
|
+
.dot.pass { background: var(--pass); } .dot.fail { background: var(--fail); } .dot.run { background: var(--run); animation: pulse 1.2s infinite; }
|
|
27
|
+
@keyframes pulse { 50% { opacity: .35; } }
|
|
28
|
+
.muted { color: var(--muted); }
|
|
29
|
+
.grid { display: grid; gap: 14px; grid-template-columns: repeat(auto-fit, minmax(320px, 1fr)); }
|
|
30
|
+
section.panel { background: var(--panel); border: 1px solid var(--line); border-radius: 10px; padding: 14px 16px; margin-bottom: 14px; }
|
|
31
|
+
.kv { display: grid; grid-template-columns: 90px 1fr; gap: 4px 12px; }
|
|
32
|
+
.kv div:nth-child(odd) { color: var(--muted); }
|
|
33
|
+
code, pre, .mono { font-family: var(--mono); font-size: 12.5px; }
|
|
34
|
+
pre { margin: 0; white-space: pre-wrap; word-break: break-word; }
|
|
35
|
+
.steps { display: flex; flex-wrap: wrap; gap: 4px; margin-top: 8px; }
|
|
36
|
+
.step { min-width: 26px; padding: 2px 6px; border-radius: 6px; background: var(--panel-2); border: 1px solid var(--line);
|
|
37
|
+
font: 11px var(--mono); text-align: center; }
|
|
38
|
+
.step.cliff { border-color: var(--fail); color: var(--fail); font-weight: 700; }
|
|
39
|
+
table { width: 100%; border-collapse: collapse; }
|
|
40
|
+
th, td { text-align: left; padding: 6px 8px; border-bottom: 1px solid var(--line); vertical-align: top; }
|
|
41
|
+
th { color: var(--muted); font-weight: 500; font-size: 12px; }
|
|
42
|
+
td.num { font-family: var(--mono); white-space: nowrap; }
|
|
43
|
+
.matrix { display: grid; grid-template-columns: 96px 1fr 1fr; gap: 6px; align-items: stretch; margin: 8px 0; }
|
|
44
|
+
.matrix .head { color: var(--muted); font-size: 12px; align-self: end; }
|
|
45
|
+
.cell { background: var(--panel-2); border: 1px solid var(--line); border-radius: 8px; padding: 8px; min-height: 58px; }
|
|
46
|
+
.cell .big { font: 600 18px var(--mono); }
|
|
47
|
+
.dots { display: flex; gap: 3px; flex-wrap: wrap; margin-top: 4px; }
|
|
48
|
+
.effect { font-family: var(--mono); font-size: 12.5px; }
|
|
49
|
+
.bar { height: 6px; border-radius: 3px; background: var(--panel-2); overflow: hidden; margin-top: 6px; }
|
|
50
|
+
.bar > i { display: block; height: 100%; background: var(--accent); }
|
|
51
|
+
.activity { font-family: var(--mono); font-size: 12px; color: var(--muted); overflow-wrap: anywhere; }
|
|
52
|
+
details summary { cursor: pointer; color: var(--muted); }
|
|
53
|
+
svg text { fill: var(--muted); font: 11px var(--mono); }
|
|
54
|
+
.empty { color: var(--muted); font-style: italic; }
|
|
55
|
+
.stale { color: var(--run); }
|
|
56
|
+
</style>
|
|
57
|
+
</head>
|
|
58
|
+
<body>
|
|
59
|
+
<main>
|
|
60
|
+
<header>
|
|
61
|
+
<a id="back" href="../../" style="display:none;color:var(--accent);text-decoration:none">← all runs</a>
|
|
62
|
+
<h1 id="title">tare</h1>
|
|
63
|
+
<span class="chip" id="phase"><span class="dot"></span><span>connecting</span></span>
|
|
64
|
+
<span class="chip" id="elapsed"></span>
|
|
65
|
+
<span class="muted" id="updated"></span>
|
|
66
|
+
</header>
|
|
67
|
+
<div id="app"><p class="empty">Waiting for the run to start…</p></div>
|
|
68
|
+
</main>
|
|
69
|
+
<script>
|
|
70
|
+
const $ = (s) => document.querySelector(s);
|
|
71
|
+
const esc = (v) => String(v ?? "").replace(/[&<>"']/g, (c) => ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" }[c]));
|
|
72
|
+
const pct = (x) => (x == null ? "–" : x.toFixed(2));
|
|
73
|
+
const dur = (s) => { s = Math.max(0, Math.round(s)); const m = Math.floor(s / 60); return m ? `${m}m ${String(s % 60).padStart(2, "0")}s` : `${s}s`; };
|
|
74
|
+
const dot = (status) => `<span class="dot ${status === "passed" ? "pass" : status === "failed" ? "fail" : status === "running" ? "run" : ""}"></span>`;
|
|
75
|
+
|
|
76
|
+
function recipeCard(s) {
|
|
77
|
+
const r = s.recipe || {};
|
|
78
|
+
const sides = Object.entries(s.sides || {}).map(([k, v]) => {
|
|
79
|
+
const version = r.agents && r.agents[k] ? r.agents[k].version : "";
|
|
80
|
+
const own = v.prompt ? `<div class="muted">prompt: ${esc(v.prompt)}</div>` : "";
|
|
81
|
+
return `<div>${s.kind === "calibrate" ? "side" : "agent"} ${esc(k)}</div><div><b>${esc(v.agent)}</b> ${esc((v.args || []).join(" "))} <span class="muted">${esc(version)}</span>${own}</div>`;
|
|
82
|
+
}).join("");
|
|
83
|
+
const p = s.params || {};
|
|
84
|
+
const params = s.kind === "cliff" ? `${esc(p.tails)} tails per step tried · budget ${esc(p.budget)}`
|
|
85
|
+
: s.kind === "calibrate" ? `${esc(p.tails)} fresh starts per side` : `cuts ${esc((p.cuts || []).join(", "))} · ${esc(p.tails)} tails per cell`;
|
|
86
|
+
return `<section class="panel"><h2>Recipe</h2><div class="kv">
|
|
87
|
+
<div>task</div><div>${s.task ? esc(s.task) : `<span class="muted">a prompt per side</span>`}</div>
|
|
88
|
+
<div>check</div><div>${s.check ? `<code>${esc(s.check)}</code>` : `<span class="muted">none. A run counts as finished when its agent exits 0. Workspaces are kept.</span>`}</div>
|
|
89
|
+
<div>project</div><div><code>${esc(s.project)}</code> <span class="muted">${esc(r.project_digest ? "fingerprint " + r.project_digest.slice(0, 12) : "")}</span></div>
|
|
90
|
+
${sides}
|
|
91
|
+
<div>settings</div><div>${params}</div>
|
|
92
|
+
${r.command ? `<div>repeat</div><div><code>tare rerun ${esc(r.out || "")}</code></div>` : ""}
|
|
93
|
+
</div></section>`;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function probesCard(s) {
|
|
97
|
+
if (!s.probes.length) return "";
|
|
98
|
+
return `<section class="panel"><h2>Probes</h2><div class="grid">${s.probes.map((p) => `
|
|
99
|
+
<div><span class="chip">${dot(p.zero ? "passed" : "failed")} ${esc(p.agent)} · ${p.zero ? "tare: 0.00" : esc((p.text || "").trim().split("\n").pop())}</span>
|
|
100
|
+
<details><summary>reading</summary><pre>${esc(p.text)}</pre></details></div>`).join("")}</div></section>`;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
function originalsCard(s, cliffStep) {
|
|
104
|
+
const lanes = Object.entries(s.sides || {}).map(([k, side]) => {
|
|
105
|
+
const o = s.originals[k];
|
|
106
|
+
if (!o) return `<div><b>${esc(k)} · ${esc(side.agent)}</b> <span class="muted">waiting</span></div>`;
|
|
107
|
+
if (o.running) return `<div><span class="chip">${dot("running")} ${esc(k)} · ${esc(side.agent)} · ${o.count} steps so far</span>
|
|
108
|
+
<div class="activity">${esc(o.activity || "starting…")}</div></div>`;
|
|
109
|
+
const steps = o.steps.slice(1).map((t, i) => `<span class="step ${cliffStep === i + 1 ? "cliff" : ""}" title="${esc(t)}">${i + 1}</span>`).join("");
|
|
110
|
+
return `<div><span class="chip">${dot(o.passed ? "passed" : "failed")} ${esc(k)} · ${esc(side.agent)} · ${o.passed ? "passed" : "failed"} after ${o.steps.length - 1} steps</span>
|
|
111
|
+
<div class="steps">${steps}</div></div>`;
|
|
112
|
+
}).join("");
|
|
113
|
+
return `<section class="panel"><h2>Original ${Object.keys(s.sides || {}).length > 1 ? "runs" : "run"}</h2><div class="grid">${lanes}</div></section>`;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
function cliffCard(s) {
|
|
117
|
+
const rows = s.steps || [];
|
|
118
|
+
if (!rows.length) return "";
|
|
119
|
+
const o = s.originals.a || {};
|
|
120
|
+
const last = Math.max(1, (o.steps ? o.steps.length - 1 : 0), ...rows.map((r) => r.step));
|
|
121
|
+
const base = rows.find((r) => r.step === 0);
|
|
122
|
+
const theta = base && base.rate != null ? base.rate : null;
|
|
123
|
+
const W = 760, H = 220, L = 40, R = 14, T = 12, B = 28;
|
|
124
|
+
const x = (step) => L + (step / last) * (W - L - R), y = (v) => T + (1 - v) * (H - T - B);
|
|
125
|
+
let svg = `<svg viewBox="0 0 ${W} ${H}" width="100%" style="max-width:${W}px;display:block" role="img" aria-label="pass rate by step">`;
|
|
126
|
+
for (const v of [0, .5, 1]) svg += `<line x1="${L}" x2="${W - R}" y1="${y(v)}" y2="${y(v)}" stroke="var(--line)"/><text x="6" y="${y(v) + 4}">${v.toFixed(1)}</text>`;
|
|
127
|
+
if (theta != null) {
|
|
128
|
+
svg += `<line x1="${L}" x2="${W - R}" y1="${y(theta)}" y2="${y(theta)}" stroke="var(--accent)" stroke-dasharray="2 3"/>`;
|
|
129
|
+
svg += `<line x1="${L}" x2="${W - R}" y1="${y(theta / 2)}" y2="${y(theta / 2)}" stroke="var(--threshold)" stroke-dasharray="6 4"/>`;
|
|
130
|
+
svg += `<text x="${L + 6}" y="${y(theta / 2) - 5}">threshold: half the baseline pass rate</text>`;
|
|
131
|
+
}
|
|
132
|
+
for (const r of rows) {
|
|
133
|
+
if (r.rate == null) { svg += `<circle cx="${x(r.step)}" cy="${y(.5)}" r="5" fill="none" stroke="var(--run)"/>`; continue; }
|
|
134
|
+
const high = theta == null || r.rate >= theta / 2, color = high ? "var(--pass)" : "var(--fail)";
|
|
135
|
+
svg += `<line x1="${x(r.step)}" x2="${x(r.step)}" y1="${y(r.hi)}" y2="${y(r.lo)}" stroke="${color}" stroke-width="3" opacity=".45"/>`;
|
|
136
|
+
svg += `<circle cx="${x(r.step)}" cy="${y(r.rate)}" r="5" fill="${color}"/>`;
|
|
137
|
+
svg += `<text x="${x(r.step) - 4}" y="${H - 8}">${r.step}</text>`;
|
|
138
|
+
}
|
|
139
|
+
svg += `</svg>`;
|
|
140
|
+
const table = rows.map((r) => `<tr><td class="num">${r.step}</td><td>${esc(o.steps ? o.steps[r.step] : "")}</td>
|
|
141
|
+
<td class="num">${r.passes}/${r.n}${r.running ? ` <span class="muted">+${r.running} running</span>` : ""}</td>
|
|
142
|
+
<td class="num">${pct(r.rate)}</td><td class="num">${r.n ? `${pct(r.lo)}-${pct(r.hi)}` : "–"}</td></tr>`).join("");
|
|
143
|
+
return `<section class="panel"><h2>Cliff search</h2>${svg}
|
|
144
|
+
<table><thead><tr><th>step</th><th>what the step did</th><th>passed / tails</th><th>rate</th><th>95% interval</th></tr></thead><tbody>${table}</tbody></table></section>`;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
function effect(e, label) {
|
|
148
|
+
if (!e) return `<div class="effect muted">${label}: waiting for every cell</div>`;
|
|
149
|
+
const [v, lo, hi] = e;
|
|
150
|
+
return `<div class="effect">${label}: <b>${v >= 0 ? "+" : ""}${v.toFixed(2)}</b> <span class="muted">[${lo.toFixed(2)}, ${hi.toFixed(2)}]</span></div>`;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
function swapCard(s) {
|
|
154
|
+
const cuts = s.cells || [];
|
|
155
|
+
if (!cuts.length) return "";
|
|
156
|
+
const agentName = (k) => esc((s.sides[k] || {}).agent || k);
|
|
157
|
+
return `<section class="panel"><h2>Swap</h2><div class="grid">${cuts.map((c) => {
|
|
158
|
+
const cell = (r, m) => {
|
|
159
|
+
const v = c.cells[r + m], dots = "<span class='dot pass'></span>".repeat(v.passes) + "<span class='dot fail'></span>".repeat(v.n - v.passes) + "<span class='dot run'></span>".repeat(v.running);
|
|
160
|
+
return `<div class="cell"><div class="big">${v.passes}/${v.n}</div><div class="dots">${dots}</div></div>`;
|
|
161
|
+
};
|
|
162
|
+
const native = Object.entries(c.native).map(([r, v]) => `workspace ${r}, own session: ${v.passes}/${v.n}`).join(" · ");
|
|
163
|
+
return `<div><b>cut ${c.fraction.toFixed(2)}</b> <span class="muted">workspace a at step ${c.steps.a}, workspace b at step ${c.steps.b}</span>
|
|
164
|
+
<div class="matrix"><div></div><div class="head">agent a · ${agentName("a")}</div><div class="head">agent b · ${agentName("b")}</div>
|
|
165
|
+
<div class="head">workspace a</div>${cell("a", "a")}${cell("a", "b")}
|
|
166
|
+
<div class="head">workspace b</div>${cell("b", "a")}${cell("b", "b")}</div>
|
|
167
|
+
${native ? `<div class="muted">${native}</div>` : ""}
|
|
168
|
+
${effect(c.state, "state effect (positive: workspace a better)")}${effect(c.model, "model effect (positive: agent a better)")}</div>`;
|
|
169
|
+
}).join("")}</div></section>`;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
function runsCard(s) {
|
|
173
|
+
const rates = s.rates || {};
|
|
174
|
+
const rows = Object.entries(s.sides || {}).map(([k, side]) => {
|
|
175
|
+
const r = rates[k] || { passes: 0, n: 0, running: 0, times: [] };
|
|
176
|
+
const dots = "<span class='dot pass'></span>".repeat(r.passes) + "<span class='dot fail'></span>".repeat(r.n - r.passes) + "<span class='dot run'></span>".repeat(r.running);
|
|
177
|
+
const times = [...(r.times || [])].sort((a, b) => a - b);
|
|
178
|
+
const median = times.length ? times[Math.floor((times.length - 1) / 2)] : null;
|
|
179
|
+
return `<tr><td class="num">${esc(k)}</td><td><b>${esc(side.agent)}</b> ${esc((side.args || []).join(" "))}<div class="dots">${dots}</div></td>
|
|
180
|
+
<td class="num">${r.passes}/${r.n}</td><td class="num">${median == null ? "–" : `${dur(median)} (${dur(times[0])} to ${dur(times[times.length - 1])})`}</td></tr>`;
|
|
181
|
+
}).join("");
|
|
182
|
+
return `<section class="panel"><h2>Runs · no check</h2><table><thead><tr><th>side</th><th>agent</th><th>finished</th><th>time (median, range)</th></tr></thead><tbody>${rows}</tbody></table></section>`;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
function calibrateCard(s) {
|
|
186
|
+
if (!s.check) return runsCard(s);
|
|
187
|
+
const rates = s.rates || {};
|
|
188
|
+
const judged = Object.values(rates).some((r) => (r.scores || []).length);
|
|
189
|
+
const rows = Object.entries(s.sides || {}).map(([k, side]) => {
|
|
190
|
+
const r = rates[k] || { passes: 0, n: 0, running: 0 };
|
|
191
|
+
const dots = "<span class='dot pass'></span>".repeat(r.passes) + "<span class='dot fail'></span>".repeat(r.n - r.passes) + "<span class='dot run'></span>".repeat(r.running);
|
|
192
|
+
const band = r.n ? `<div class="bar" style="position:relative"><i style="margin-left:${(r.lo * 100).toFixed(1)}%;width:${((r.hi - r.lo) * 100).toFixed(1)}%"></i></div>` : "";
|
|
193
|
+
const scores = r.scores || [];
|
|
194
|
+
const strip = scores.length ? `<svg viewBox="0 0 200 22" width="200" height="22" style="display:block">
|
|
195
|
+
<line x1="0" x2="200" y1="11" y2="11" stroke="var(--line)"/>
|
|
196
|
+
${[0, 50, 100].map((v) => `<line x1="${v * 2}" x2="${v * 2}" y1="6" y2="16" stroke="var(--line)"/>`).join("")}
|
|
197
|
+
${scores.map((v) => `<circle cx="${v * 2}" cy="11" r="4" fill="var(--accent)" opacity=".75"><title>${v}</title></circle>`).join("")}
|
|
198
|
+
${r.mean != null ? `<line x1="${r.mean * 2}" x2="${r.mean * 2}" y1="2" y2="20" stroke="var(--text)" stroke-width="2"/>` : ""}
|
|
199
|
+
</svg><div class="muted mono">${[...scores].sort((a, b) => a - b).join(" ")} · mean ${r.mean.toFixed(1)}</div>` : `<span class="muted">–</span>`;
|
|
200
|
+
return `<tr><td class="num">${esc(k)}</td><td><b>${esc(side.agent)}</b> ${esc((side.args || []).join(" "))}<div class="dots">${dots}</div></td>
|
|
201
|
+
${judged ? `<td>${strip}</td>` : ""}
|
|
202
|
+
<td class="num">${r.passes}/${r.n}</td><td class="num">${pct(r.rate)}</td><td class="num" style="min-width:160px">${r.n ? `${pct(r.lo)}-${pct(r.hi)}` : "–"}${band}</td></tr>`;
|
|
203
|
+
}).join("");
|
|
204
|
+
const title = judged ? "Judge scores and pass rates" : "Pass rates";
|
|
205
|
+
return `<section class="panel"><h2>${title}</h2><table><thead><tr><th>side</th><th>agent</th>${judged ? "<th>judge scores (0 to 100, line = mean)</th>" : ""}<th>pass</th><th>rate</th><th>95% interval</th></tr></thead><tbody>${rows}</tbody></table></section>`;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
function usageCard(s) {
|
|
209
|
+
const u = s.usage || {};
|
|
210
|
+
if (!u.total) return "";
|
|
211
|
+
const n = (x) => (x >= 1e6 ? `${(x / 1e6).toFixed(2)}M` : x >= 1e3 ? `${(x / 1e3).toFixed(1)}k` : `${x}`);
|
|
212
|
+
const money = (c) => (c < 10 ? c.toFixed(3) : c.toFixed(2));
|
|
213
|
+
const cost = (v) => (v.cost_usd == null ? `<span class="muted">unknown</span>`
|
|
214
|
+
: `${v.estimated ? "about " : ""}$${money(v.cost_usd)}${v.unpriced ? ` <span class="muted">(${v.unpriced} of ${v.runs} runs unknown)</span>` : ""}`
|
|
215
|
+
+ (v.estimated ? `<div class="muted">estimated from ${esc(v.estimated)}</div>` : ""));
|
|
216
|
+
const row = (name, v) => `<tr><td>${name}</td><td class="num">${v.runs}</td><td class="num">${n(v.input)} <span class="muted">(${n(v.cached)} cached)</span></td>
|
|
217
|
+
<td class="num">${n(v.output)}</td><td class="num">${cost(v)}</td></tr>`;
|
|
218
|
+
const rows = Object.entries(u.sides || {}).filter(([, v]) => v.usage)
|
|
219
|
+
.map(([k, v]) => row(`<b>${esc(k)}</b> ${esc(v.label)}${v.billing === "subscription"
|
|
220
|
+
? `<div class="muted">subscription login, so the cost is notional</div>` : ""}`, v.usage)).join("")
|
|
221
|
+
+ (u.check ? row("check (the judge)", u.check) : "") + row("<b>total</b>", u.total);
|
|
222
|
+
return `<section class="panel"><h2>Usage</h2><table><thead><tr><th></th><th>runs</th><th>tokens in</th><th>tokens out</th><th>cost</th></tr></thead>
|
|
223
|
+
<tbody>${rows}</tbody></table></section>`;
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
function tailsCard(s) {
|
|
227
|
+
const running = s.tails.filter((t) => t.status === "running"), done = s.tails.filter((t) => t.status !== "running");
|
|
228
|
+
if (!s.tails.length) return "";
|
|
229
|
+
const fraction = (i) => (s.plan && s.plan[i] ? s.plan[i].fraction.toFixed(2) : i);
|
|
230
|
+
const where = (t) => s.kind === "cliff" ? `step ${t.step}` : s.kind === "calibrate" ? `side ${t.side} · fresh start`
|
|
231
|
+
: `cut ${fraction(t.cut)} · workspace ${t.room} · agent ${t.agent}${t.kind === "native" ? " · own session" : ""}`;
|
|
232
|
+
const row = (t) => `<tr><td>${dot(t.status)}</td><td class="num">${esc(where(t))}</td><td>${esc(t.agent_name)}</td>
|
|
233
|
+
<td class="num">${dur((t.ended || s.now) - t.started)}</td><td class="activity">${esc(t.status === "running" ? (t.activity || "starting…") : t.detail || t.status)}</td></tr>`;
|
|
234
|
+
const total = s.kind === "cliff" ? `${done.length} of a budget of ${esc((s.params || {}).budget)}` : `${done.length} done`;
|
|
235
|
+
return `<section class="panel"><h2>${s.kind === "calibrate" ? "Runs" : "Tails"} · ${running.length} running · ${total}</h2>
|
|
236
|
+
<table><thead><tr><th></th><th>where</th><th>agent</th><th>time</th><th>doing / result</th></tr></thead>
|
|
237
|
+
<tbody>${running.map(row).join("")}${done.slice(0, 40).map(row).join("")}</tbody></table></section>`;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
function render(s) {
|
|
241
|
+
$("#title").textContent = s.kind ? `tare ${s.kind}` : "tare";
|
|
242
|
+
document.title = s.kind ? `tare ${s.kind} · ${s.phase}` : "tare live";
|
|
243
|
+
const status = s.phase === "done" ? "passed" : "running";
|
|
244
|
+
$("#phase").innerHTML = `${dot(status)}<span>${esc(s.phase)}</span>`;
|
|
245
|
+
if (s.started) $("#elapsed").textContent = `${dur((s.phase === "done" ? s.updated : s.now) - s.started)}`;
|
|
246
|
+
const quiet = s.updated ? s.now - s.updated : 0;
|
|
247
|
+
$("#updated").innerHTML = s.phase !== "done" && quiet > 300 ? `<span class="stale">no update for ${dur(quiet)}</span>` : "";
|
|
248
|
+
let cliffStep = null;
|
|
249
|
+
if (s.report) { const m = s.report.match(/lost at step (\d+)/); if (m) cliffStep = Number(m[1]); }
|
|
250
|
+
$("#app").innerHTML = (s.kind ? recipeCard(s) : "") + probesCard(s) + (s.kind && s.kind !== "calibrate" ? originalsCard(s, cliffStep) : "")
|
|
251
|
+
+ (s.report ? `<section class="panel"><h2>Report</h2><pre>${esc(s.report)}</pre></section>` : "")
|
|
252
|
+
+ (s.kind === "calibrate" ? calibrateCard(s) : "")
|
|
253
|
+
+ (s.kind === "cliff" ? cliffCard(s) : "") + (s.kind === "swap" ? swapCard(s) : "") + usageCard(s) + tailsCard(s);
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
async function tick() {
|
|
257
|
+
try {
|
|
258
|
+
const r = await fetch("state", { cache: "no-store" }); // relative: under run/<i>/ when tare watch shows several runs
|
|
259
|
+
render(await r.json());
|
|
260
|
+
} catch (e) {
|
|
261
|
+
$("#phase").innerHTML = `${dot("failed")}<span>disconnected</span>`;
|
|
262
|
+
}
|
|
263
|
+
setTimeout(tick, 1000);
|
|
264
|
+
}
|
|
265
|
+
if (location.pathname.startsWith("/run/")) $("#back").style.display = "";
|
|
266
|
+
tick();
|
|
267
|
+
</script>
|
|
268
|
+
</body>
|
|
269
|
+
</html>
|