the-high-council 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +19 -0
- package/CONTRIBUTING.md +41 -0
- package/LICENSE +21 -0
- package/PERSONAS.md +51 -0
- package/README.md +499 -0
- package/TROUBLESHOOTING.md +45 -0
- package/chains/cheap.json +35 -0
- package/chains/gp-judge-v1.json +80 -0
- package/chains/idea-open-c2.json +106 -0
- package/chains/mock-budget.json +15 -0
- package/chains/mock-debate.json +59 -0
- package/chains/mock-external.json +59 -0
- package/chains/mock-open.json +60 -0
- package/chains/mock-proposals.json +57 -0
- package/chains/mock-questions-wait.json +63 -0
- package/chains/mock-questions.json +63 -0
- package/chains/mock-relay.json +42 -0
- package/chains/mock-unanimous.json +42 -0
- package/chains/mock.json +17 -0
- package/chains/plan-auto-mistral.json +101 -0
- package/chains/plan-auto.json +131 -0
- package/chains/plan-cheap.json +82 -0
- package/chains/plan-debate-c2.json +86 -0
- package/chains/plan-debate-open-c2.json +98 -0
- package/chains/plan-debate-roles-c1.json +149 -0
- package/chains/plan-debate.json +116 -0
- package/chains/plan-fast.json +66 -0
- package/chains/plan-proposals-best-of-5.json +98 -0
- package/chains/plan-proposals.json +96 -0
- package/chains/plan-relay.json +83 -0
- package/chains/plan-thorough.json +82 -0
- package/chains/plan-unanimous.json +82 -0
- package/chains/seven-cheap.json +67 -0
- package/chains/seven.json +60 -0
- package/chains/smo-v2-cheap.json +67 -0
- package/chains/smo.json +50 -0
- package/chains/verify.json +40 -0
- package/package.json +55 -0
- package/src/board-export.js +77 -0
- package/src/budget.js +126 -0
- package/src/buys.js +44 -0
- package/src/cache-integrity.js +38 -0
- package/src/chain-lint.js +107 -0
- package/src/chain.js +880 -0
- package/src/cli.js +1031 -0
- package/src/cost-forecast.js +70 -0
- package/src/cost.js +139 -0
- package/src/errors.js +82 -0
- package/src/http-server.js +349 -0
- package/src/integrity.js +48 -0
- package/src/key-redaction.js +97 -0
- package/src/mcp/server.js +319 -0
- package/src/name-lint.js +186 -0
- package/src/partial-deliverable.js +38 -0
- package/src/peer-claim.js +98 -0
- package/src/personas.js +54 -0
- package/src/preflight.js +84 -0
- package/src/pricing.json +120 -0
- package/src/providers.js +268 -0
- package/src/replay.js +40 -0
- package/src/resume-brief.js +114 -0
- package/src/role-diagnostics.js +181 -0
- package/src/roles.js +520 -0
- package/src/schema-version.js +39 -0
- package/src/scope-freeze.js +35 -0
- package/src/seat-role.js +96 -0
- package/src/shape-rounds.js +58 -0
- package/src/spend.js +193 -0
- package/src/stage-contract.js +223 -0
- package/src/stage-submission.js +53 -0
- package/src/tie-break.js +94 -0
- package/src/ui/app.js +139 -0
- package/src/ui/index.html +43 -0
- package/src/ui/parse.js +101 -0
- package/src/ui/server.js +117 -0
- package/src/ui/style.css +37 -0
- package/src/verdict-stats.js +267 -0
- package/src/withdrawal-ledger.js +47 -0
package/src/chain.js
ADDED
|
@@ -0,0 +1,880 @@
|
|
|
1
|
+
import { call, keyFor } from './providers.js';
|
|
2
|
+
import * as R from './roles.js';
|
|
3
|
+
import { costOf, summarise, formatUsd, worstCaseOf, wouldBreach } from './cost.js';
|
|
4
|
+
import { requiredDeliverableSections } from './preflight.js';
|
|
5
|
+
import { withdrawalLedger } from './withdrawal-ledger.js';
|
|
6
|
+
import { applySeatRole } from './seat-role.js';
|
|
7
|
+
import { NO_TIE_BREAK } from './tie-break.js';
|
|
8
|
+
|
|
9
|
+
// v3 §4: the criteria stage's own user prompt, exported so it's testable without running a
|
|
10
|
+
// full chain. Tells the criteria seat what the chain's own contract will require in the
|
|
11
|
+
// deliverable, so it never writes a criterion that conflicts with a section the builder is
|
|
12
|
+
// required to produce - the recurring contract-vs-criteria conflict v2's pre-flight check
|
|
13
|
+
// could only warn about after the fact. Reuses the same required-sections data the pre-flight
|
|
14
|
+
// check already computes - no new state, no second source of truth.
|
|
15
|
+
export function criteriaUserPrompt(request, config) {
|
|
16
|
+
const required = requiredDeliverableSections(config);
|
|
17
|
+
const requiredBlock = required.length
|
|
18
|
+
? `\n\n# Sections this chain always adds\nThe deliverable will also contain: ${required.join(', ')}.\nDo not write a criterion that forbids this section's presence or dictates an ordering it cannot satisfy.`
|
|
19
|
+
: '';
|
|
20
|
+
return `# Request\n\n${request}${requiredBlock}`;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
// A "seat" is one lab's model occupying one slot in the chain.
|
|
24
|
+
// { provider, model, maxTokens?, temperature? }
|
|
25
|
+
|
|
26
|
+
// Qwen3.5-9B (2026-09-10, three real runs against the same task, 3/3 failed identically):
|
|
27
|
+
// when quoting a source phrase verbatim inside a JSON string value, it wraps the quote in
|
|
28
|
+
// markdown-style 'single quotes' but leaves any literal " inside that span unescaped (e.g.
|
|
29
|
+
// `"evidence": "...states: 'no mention of "The Council," no round count...'"` - the inner "
|
|
30
|
+
// terminates the JSON string early). Separately (same evening, a different run): a raw
|
|
31
|
+
// unescaped newline landed inside a string value instead of \n. Both are mechanical, safe to
|
|
32
|
+
// repair by scanning with real string-state tracking rather than a single regex (a regex can't
|
|
33
|
+
// tell "the real closing quote" from "a stray quote mid-sentence" without looking at what comes
|
|
34
|
+
// next). A quote is only treated as ending a string if the next non-whitespace character is a
|
|
35
|
+
// plausible JSON continuation (, : } ]) or end-of-text; otherwise it's escaped in place. This
|
|
36
|
+
// was verified against all three real broken runs (fixed the one that was otherwise-clean; the
|
|
37
|
+
// other two had additional, genuinely garbled model output - e.g. a stray `;` and leaked
|
|
38
|
+
// CSS-looking text - that no mechanical repair should paper over) and regression-checked against
|
|
39
|
+
// every other already-valid critique reply on disk at the time (26 files, zero broken).
|
|
40
|
+
function repairStrayQuotesAndControlChars(text) {
|
|
41
|
+
let out = '';
|
|
42
|
+
let inString = false;
|
|
43
|
+
for (let i = 0; i < text.length; i++) {
|
|
44
|
+
const c = text[i];
|
|
45
|
+
if (c === '\\' && inString) {
|
|
46
|
+
out += c;
|
|
47
|
+
if (i + 1 < text.length) { out += text[i + 1]; i++; }
|
|
48
|
+
continue;
|
|
49
|
+
}
|
|
50
|
+
if (inString && (c === '\n' || c === '\r' || c === '\t')) {
|
|
51
|
+
out += c === '\n' ? '\\n' : c === '\r' ? '\\r' : '\\t';
|
|
52
|
+
continue;
|
|
53
|
+
}
|
|
54
|
+
if (c === '"') {
|
|
55
|
+
if (!inString) {
|
|
56
|
+
inString = true;
|
|
57
|
+
out += c;
|
|
58
|
+
continue;
|
|
59
|
+
}
|
|
60
|
+
let j = i + 1;
|
|
61
|
+
while (j < text.length && /\s/.test(text[j])) j++;
|
|
62
|
+
const next = text[j];
|
|
63
|
+
if (next === undefined || ',:}]'.includes(next)) {
|
|
64
|
+
inString = false;
|
|
65
|
+
out += c;
|
|
66
|
+
} else {
|
|
67
|
+
out += '\\"';
|
|
68
|
+
}
|
|
69
|
+
continue;
|
|
70
|
+
}
|
|
71
|
+
out += c;
|
|
72
|
+
}
|
|
73
|
+
return out;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
// §5 (v3 plan, MISTRAL-3 accepted): a reviser that judges an objection to not
|
|
77
|
+
// be a real defect ends its reply with one or more trailing "DECLINED: <reason>"
|
|
78
|
+
// lines (src/roles.js's reviser system prompt). Those lines are never part of
|
|
79
|
+
// the deliverable a critic grades - they're stripped here and returned
|
|
80
|
+
// separately so the caller can carry them into report.json's `disputes`
|
|
81
|
+
// field instead. Only lines strictly at the end of the reply count: a
|
|
82
|
+
// "DECLINED:"-shaped line does not get pulled out of the middle of the
|
|
83
|
+
// deliverable's own body text.
|
|
84
|
+
export function parseDisputes(text) {
|
|
85
|
+
const lines = text.split('\n');
|
|
86
|
+
let i = lines.length - 1;
|
|
87
|
+
const declined = [];
|
|
88
|
+
while (i >= 0 && /^DECLINED:\s*.+/.test(lines[i].trim())) {
|
|
89
|
+
declined.unshift(lines[i].trim().replace(/^DECLINED:\s*/, ''));
|
|
90
|
+
i--;
|
|
91
|
+
}
|
|
92
|
+
let draftLines = lines.slice(0, i + 1);
|
|
93
|
+
while (draftLines.length && draftLines[draftLines.length - 1].trim() === '') draftLines.pop();
|
|
94
|
+
return { draft: draftLines.join('\n'), disputes: declined };
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
export function parseJson(text) {
|
|
98
|
+
// Models wrap JSON in prose or fences no matter how firmly you ask them not to,
|
|
99
|
+
// and the small ones leave a trailing comma before ] or } (GLM 5.3 Flash,
|
|
100
|
+
// 2026-09-07: a real objection counted as an abstention over one comma).
|
|
101
|
+
const fenced = text.match(/```(?:json)?\s*([\s\S]*?)```/);
|
|
102
|
+
const openFence = text.match(/```(?:json)?\s*([\s\S]*)$/); // closing fence missing
|
|
103
|
+
const first = text.indexOf('{');
|
|
104
|
+
const last = text.lastIndexOf('}');
|
|
105
|
+
const braces = first !== -1 && last > first ? text.slice(first, last + 1) : null;
|
|
106
|
+
const candidates = [text, fenced?.[1], openFence?.[1], braces].filter(Boolean);
|
|
107
|
+
// Repairs, mildest first. Each one targets a slip a real seat has made:
|
|
108
|
+
// - trailing comma before ] or } (GLM 5.3 Flash)
|
|
109
|
+
// - a key written as a bare string, "reason: text" instead of "reason": "text"
|
|
110
|
+
// (Mistral Large, six times in one reply, 2026-09-07)
|
|
111
|
+
// - a stray unescaped quote or raw control char inside a string value (Qwen3.5-9B, 2026-09-10)
|
|
112
|
+
const repairs = [
|
|
113
|
+
c => c,
|
|
114
|
+
c => c.replace(/,\s*([\]}])/g, '$1'),
|
|
115
|
+
c => c.replace(/"([A-Za-z_][A-Za-z0-9_ ]*):\s+([^"]*)"/g, (all, k, v) => `"${k}": "${v.replace(/"/g, '\\"')}"`).replace(/,\s*([\]}])/g, '$1'),
|
|
116
|
+
c => repairStrayQuotesAndControlChars(c).replace(/,\s*([\]}])/g, '$1'),
|
|
117
|
+
];
|
|
118
|
+
for (const repair of repairs) {
|
|
119
|
+
for (const c of candidates) {
|
|
120
|
+
try { return JSON.parse(repair(c.trim())); } catch { /* try the next shape */ }
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
return null;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
// Explains why a critic's reply couldn't be parsed, for the abstention log line. Found 2026-09-10
|
|
127
|
+
// (GP judging run 2026-09-10T20-01-59-545Z, panel-1-glm.md): the old two-way check (token cap vs.
|
|
128
|
+
// "malformed JSON") mislabelled a genuinely empty reply as "malformed JSON - read the saved reply,
|
|
129
|
+
// it may still be an objection" - there is nothing to read, and it is not an objection. The
|
|
130
|
+
// provider's own finish_reason ("stop" in usage, set by providers.js from finish_reason/
|
|
131
|
+
// stop_reason) already says this plainly when it's not a normal completion: OpenRouter/OpenAI-
|
|
132
|
+
// compatible providers return "error" when the upstream model errors out mid-generation (here,
|
|
133
|
+
// GLM 5.3 Flash burned its whole token budget on thinking - usage.thinking === usage.output - and
|
|
134
|
+
// returned no content at all), which is a distinct failure from either truncation or a formatting
|
|
135
|
+
// slip and needs its own label so a later session doesn't chase the wrong bug.
|
|
136
|
+
export function classifyUnreadable(usage, maxTokens) {
|
|
137
|
+
if (usage.stop === 'error') return 'provider returned an error mid-generation (stop: error) - not a truncation or a JSON-formatting problem';
|
|
138
|
+
const cap = maxTokens ?? 8000;
|
|
139
|
+
if (usage.output >= cap * 0.95) return 'hit the token cap, truncated';
|
|
140
|
+
return 'malformed JSON - read the saved reply, it may still be an objection';
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
// Small critics fill the schema literally: every criterion MET, meets:false,
|
|
144
|
+
// and a placeholder failure {criterion:"None"} because the array "had to"
|
|
145
|
+
// hold something (Llama 3.3 70B did exactly this on 2026-09-07, and the
|
|
146
|
+
// harness counted it as an objection). The per-criterion table is the honest
|
|
147
|
+
// signal, so the summary fields are derived from it rather than trusted.
|
|
148
|
+
function shuffle(list) {
|
|
149
|
+
const out = [...list];
|
|
150
|
+
for (let i = out.length - 1; i > 0; i--) {
|
|
151
|
+
const j = Math.floor(Math.random() * (i + 1));
|
|
152
|
+
[out[i], out[j]] = [out[j], out[i]];
|
|
153
|
+
}
|
|
154
|
+
return out;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
// A critic's own text (criterion/problem/fix/verdict_line) is entirely critic-controlled and
|
|
158
|
+
// flows, undelimited until roles.js's own prompt-construction fix, straight into the reviser's
|
|
159
|
+
// prompt and the run history handed to the finalist (docs/security-prompt-injection.md S3, found
|
|
160
|
+
// from cnc-harness's own security review of what a plan-N seat actually runs). A 20 000-token
|
|
161
|
+
// critic reply could otherwise become a 20 000-token "failure" in the next prompt; cap each
|
|
162
|
+
// field here, at parse time, once, rather than trusting every call site downstream to remember.
|
|
163
|
+
const CRITIQUE_FIELD_MAX_CHARS = 2000;
|
|
164
|
+
function capField(value) {
|
|
165
|
+
if (typeof value !== 'string') return value;
|
|
166
|
+
return value.length > CRITIQUE_FIELD_MAX_CHARS
|
|
167
|
+
? `${value.slice(0, CRITIQUE_FIELD_MAX_CHARS)}… [truncated, ${value.length} chars total]`
|
|
168
|
+
: value;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
export function normaliseCritique(critique, log = () => {}) {
|
|
172
|
+
const placeholder = f => !f || !f.criterion || /^(none|n\/?a|-|no failures?)\.?$/i.test(String(f.criterion).trim());
|
|
173
|
+
const failures = (critique.failures || []).filter(f => !placeholder(f))
|
|
174
|
+
.map(f => ({ ...f, criterion: capField(f.criterion), problem: capField(f.problem), fix: capField(f.fix) }));
|
|
175
|
+
const dropped = (critique.failures || []).length - failures.length;
|
|
176
|
+
if (dropped) log(` (ignored ${dropped} placeholder failure entr${dropped === 1 ? 'y' : 'ies'} that named no criterion)`);
|
|
177
|
+
for (const row of Array.isArray(critique.criteria) ? critique.criteria : []) {
|
|
178
|
+
if (String(row.verdict || '').toUpperCase() !== 'FAILED') continue;
|
|
179
|
+
if (failures.some(f => f.criterion === row.criterion)) continue;
|
|
180
|
+
failures.push({ criterion: capField(row.criterion), problem: capField(row.evidence) || 'marked FAILED in the criteria table', fix: '' });
|
|
181
|
+
log(` (added a failure the critic marked FAILED in its table but left out of its failures list)`);
|
|
182
|
+
}
|
|
183
|
+
return { ...critique, failures, verdict_line: capField(critique.verdict_line), meets: failures.length === 0 };
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
// Reads the plan's own "Scope ledger" lines: `<id> - accepted|cut - <reason>`.
|
|
187
|
+
// An id the ledger does not mention is "unaccounted", which is itself a
|
|
188
|
+
// finding about the integrator, not the proposer.
|
|
189
|
+
export function scoreProposals(proposals, plan) {
|
|
190
|
+
const rows = proposals.map(p => {
|
|
191
|
+
const re = new RegExp(`^[\\s*_\`-]*${p.id}[\`*_]*\\s*[-:\u2013\u2014]\\s*\\**(accepted|cut|withdrawn)\\**\\s*[-:\u2013\u2014]?\\s*(.*)$`, 'im');
|
|
192
|
+
const m = plan.match(re);
|
|
193
|
+
return { ...p, status: m ? m[1].toLowerCase() : 'unaccounted', note: m ? m[2].trim() : '' };
|
|
194
|
+
});
|
|
195
|
+
const labs = [...new Set(rows.map(r => r.lab))].map(lab => {
|
|
196
|
+
const mine = rows.filter(r => r.lab === lab);
|
|
197
|
+
const n = st => mine.filter(r => r.status === st).length;
|
|
198
|
+
return { lab, model: mine[0].model, proposed: mine.length, accepted: n('accepted'), cut: n('cut'), withdrawn: n('withdrawn'), unaccounted: n('unaccounted'), built: null };
|
|
199
|
+
});
|
|
200
|
+
return { rows, labs };
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
// A lab's identity is its provider unless the seat names one: three seats on
|
|
204
|
+
// OpenRouter are three labs, and mock chains need two labs on one provider.
|
|
205
|
+
const labOf = seat => seat.lab || seat.provider;
|
|
206
|
+
|
|
207
|
+
// A run pauses at an external seat: the prompt is written for whoever plays
|
|
208
|
+
// that seat (a Claude Code session on the Max plan, a human), and the run
|
|
209
|
+
// resumes from disk once <label>.md exists. Thrown, not returned, so the
|
|
210
|
+
// chain's control flow stays linear.
|
|
211
|
+
export class ExternalPause extends Error {
|
|
212
|
+
constructor(label, system, user) { super(`waiting for external stage ${label}`); this.label = label; this.system = system; this.user = user; }
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
// A run hits its spend ceiling: thrown from invoke() BEFORE the call goes
|
|
216
|
+
// out, so the breaching stage is never paid for. Like ExternalPause this is
|
|
217
|
+
// thrown rather than returned, and for the same reason - the chain's control
|
|
218
|
+
// flow stays linear, and every completed stage is already on disk, so the run
|
|
219
|
+
// is resumable under a higher ceiling.
|
|
220
|
+
export class BudgetExceeded extends Error {
|
|
221
|
+
constructor({ label, seat, spent, cap, projected }) {
|
|
222
|
+
super(`per-run spend cap reached before stage ${label}`);
|
|
223
|
+
this.label = label;
|
|
224
|
+
this.seat = seat;
|
|
225
|
+
this.spent = spent;
|
|
226
|
+
this.cap = cap;
|
|
227
|
+
this.projected = projected;
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
// Per-run cache, set by the CLI: get(label) -> { text, usage, usd, ... } or null.
|
|
232
|
+
let cache = { get: () => null };
|
|
233
|
+
export function setCache(c) { cache = c || { get: () => null }; }
|
|
234
|
+
|
|
235
|
+
// Per-run spend ceiling in USD, set by the CLI. null means no ceiling.
|
|
236
|
+
// `spent` accumulates every stage this run has paid for, including stages
|
|
237
|
+
// replayed from disk on a resume - the ceiling is on the run as a whole, not
|
|
238
|
+
// on one sitting at the terminal.
|
|
239
|
+
let budget = { cap: null, spent: 0 };
|
|
240
|
+
export function setBudget(cap) { budget = { cap: cap ?? null, spent: 0 }; }
|
|
241
|
+
export function budgetState() { return { ...budget, remaining: budget.cap === null ? null : Math.max(0, budget.cap - budget.spent) }; }
|
|
242
|
+
|
|
243
|
+
async function invoke(seat, { system, user, log, label }) {
|
|
244
|
+
const started = Date.now();
|
|
245
|
+
// Resume: a stage that already ran in this run folder is replayed from
|
|
246
|
+
// disk, so a paused-and-resumed run never pays twice.
|
|
247
|
+
const hit = cache.get(label);
|
|
248
|
+
if (hit) {
|
|
249
|
+
budget.spent += hit.usd || 0;
|
|
250
|
+
log(` ${label}: ${hit.provider || seat.provider}/${hit.model || seat.model} - from disk (${hit.usage?.input ?? 0} in, ${hit.usage?.output ?? 0} out, ${formatUsd(hit.usd || 0)} already spent)`);
|
|
251
|
+
return { label, provider: hit.provider || seat.provider, model: hit.model || seat.model, lab: labOf(seat), usage: hit.usage || { input: 0, output: 0 }, usd: hit.usd || 0, priced: true, ms: 0, text: hit.text, cached: true };
|
|
252
|
+
}
|
|
253
|
+
if (seat.provider === 'external') throw new ExternalPause(label, system, user);
|
|
254
|
+
|
|
255
|
+
// The cap is checked here, before the only line in this file that spends
|
|
256
|
+
// money. Anthropic seats are projected at two attempts because invoke()
|
|
257
|
+
// below may pay for the same stage twice (the thinking-disabled retry).
|
|
258
|
+
const maxTokens = seat.maxTokens ?? 8000;
|
|
259
|
+
const projected = worstCaseOf(seat.provider, seat.model, {
|
|
260
|
+
promptChars: (system || '').length + (user || '').length,
|
|
261
|
+
maxTokens,
|
|
262
|
+
retries: seat.provider === 'anthropic' ? 2 : 1,
|
|
263
|
+
}).usd;
|
|
264
|
+
const verdict = wouldBreach({ spent: budget.spent, cap: budget.cap, projected });
|
|
265
|
+
if (verdict.breach) {
|
|
266
|
+
log(` ${label}: STOPPED - ${formatUsd(budget.spent)} spent, this stage could cost up to ${formatUsd(projected)}, ceiling is ${formatUsd(budget.cap)}.`);
|
|
267
|
+
throw new BudgetExceeded({ label, seat: `${seat.provider}/${seat.model}`, spent: budget.spent, cap: budget.cap, projected });
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
const ask = extra => call(seat.provider, {
|
|
271
|
+
model: seat.model,
|
|
272
|
+
system,
|
|
273
|
+
messages: [{ role: 'user', content: user }],
|
|
274
|
+
maxTokens: seat.maxTokens ?? 8000,
|
|
275
|
+
temperature: seat.temperature,
|
|
276
|
+
extra,
|
|
277
|
+
});
|
|
278
|
+
let res = await ask(seat.extra);
|
|
279
|
+
let wasted = 0;
|
|
280
|
+
// Claude's adaptive thinking counts against max_tokens and is not text. On
|
|
281
|
+
// a hard prompt it can spend the whole budget before writing a word - the
|
|
282
|
+
// netcode review on 2026-09-07 did exactly that three stages in a row, and
|
|
283
|
+
// the panel reviewed an empty page twice. The same budget can also cut a
|
|
284
|
+
// reply off mid-sentence (the v2 criteria stage, same day: 1534 thinking
|
|
285
|
+
// tokens, JSON truncated), which is just as unusable. Either way: retry
|
|
286
|
+
// once with thinking off.
|
|
287
|
+
if (seat.provider === 'anthropic' && res.usage.thinking > 0 && res.usage.stop === 'max_tokens') {
|
|
288
|
+
wasted = costOf(res.provider, res.model, res.usage).usd;
|
|
289
|
+
const how = res.text.trim() ? `text cut off - ${res.usage.thinking} of ${res.usage.output} tokens went to thinking` : `empty text - all ${res.usage.output} tokens went to thinking`;
|
|
290
|
+
log(` ${label}: ${how} (stop: max_tokens, ${formatUsd(wasted)} spent); retrying once with thinking disabled.`);
|
|
291
|
+
res = await ask({ ...(seat.extra || {}), thinking: { type: 'disabled' } });
|
|
292
|
+
}
|
|
293
|
+
const cost = costOf(res.provider, res.model, res.usage);
|
|
294
|
+
const stage = {
|
|
295
|
+
label,
|
|
296
|
+
provider: res.provider,
|
|
297
|
+
model: res.model,
|
|
298
|
+
lab: labOf(seat),
|
|
299
|
+
usage: res.usage,
|
|
300
|
+
usd: cost.usd + wasted,
|
|
301
|
+
priced: cost.priced,
|
|
302
|
+
ms: Date.now() - started,
|
|
303
|
+
text: res.text,
|
|
304
|
+
};
|
|
305
|
+
budget.spent += stage.usd;
|
|
306
|
+
const think = res.usage.thinking ? ` (${res.usage.thinking} thinking)` : '';
|
|
307
|
+
const cut = res.usage.stop === 'max_tokens' || res.usage.stop === 'length' ? ' [hit the cap]' : '';
|
|
308
|
+
log(` ${label}: ${res.provider}/${res.model} - ${res.usage.input} in, ${res.usage.output} out${think}${cut}, ${formatUsd(stage.usd)}, ${(stage.ms / 1000).toFixed(1)}s`);
|
|
309
|
+
return stage;
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
export function checkSeats(seats) {
|
|
313
|
+
const missing = [];
|
|
314
|
+
for (const s of seats) {
|
|
315
|
+
if (!s) continue;
|
|
316
|
+
if (!keyFor(s.provider)) missing.push(`${s.provider} (seat ${s.model})`);
|
|
317
|
+
}
|
|
318
|
+
return [...new Set(missing)];
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
/**
|
|
322
|
+
* Run the chain.
|
|
323
|
+
*
|
|
324
|
+
* Two termination modes, both against the same fixed, binary checklist - only
|
|
325
|
+
* how many labs must agree changes, never what they're allowed to ask for:
|
|
326
|
+
*
|
|
327
|
+
* - "first" (default): alternate one critic and a revision until any single
|
|
328
|
+
* critic passes the draft, or the round cap is reached.
|
|
329
|
+
* - "unanimous": every critic seat reviews the SAME draft each round; a round
|
|
330
|
+
* only counts as a pass if every one of them independently signs off with
|
|
331
|
+
* no failures. Otherwise the reviser fixes the union of everything any
|
|
332
|
+
* critic flagged, and the whole panel reviews again next round.
|
|
333
|
+
*
|
|
334
|
+
* Panel topology, unanimous mode only (config.panel):
|
|
335
|
+
* - "independent" (default): each critic sees only the draft. Verdicts are
|
|
336
|
+
* uncorrelated, so disagreement between labs means something.
|
|
337
|
+
* - "relay" (experimental): critics run one after another and each is handed
|
|
338
|
+
* the verdicts before it - the chain as originally sketched. On the one
|
|
339
|
+
* controlled comparison so far (Docs/Runs.md) it caught 0/3 of a real
|
|
340
|
+
* scope leak the blind panel caught 2/3: anchoring on the first seat.
|
|
341
|
+
* Kept for comparison runs. Seat order is randomised per round and earlier
|
|
342
|
+
* verdicts are anonymised, which is the best-evidenced mitigation short of
|
|
343
|
+
* not doing it.
|
|
344
|
+
*
|
|
345
|
+
* The round cap is the whole point either way - an uncapped refine loop
|
|
346
|
+
* drifts, whether that's one critic asked again and again or a panel asked
|
|
347
|
+
* to keep agreeing.
|
|
348
|
+
*/
|
|
349
|
+
export async function runChain({ request: requestIn, config, draft: initialDraft = null, log = console.log, onStage = () => {} }) {
|
|
350
|
+
const stages = [];
|
|
351
|
+
const record = s => { stages.push(s); onStage(s); return s; };
|
|
352
|
+
let request = requestIn;
|
|
353
|
+
|
|
354
|
+
// 0. Questions first (config.questions: { max, wait }). The answers become
|
|
355
|
+
// part of the request before criteria are written, so the checklist is
|
|
356
|
+
// written against the resolved request, not the ambiguous one.
|
|
357
|
+
let questions = null;
|
|
358
|
+
if (config.questions && !initialDraft) {
|
|
359
|
+
const max = config.questions.max ?? 6;
|
|
360
|
+
log('\nStage: questions (what would change the plan?)');
|
|
361
|
+
const qs = record(await invoke(config.seats.questions || config.seats.criteria, {
|
|
362
|
+
system: R.QUESTIONS_SYSTEM,
|
|
363
|
+
user: R.questionsUser({ request, max }),
|
|
364
|
+
log, label: 'questions',
|
|
365
|
+
}));
|
|
366
|
+
const parsed = parseJson(qs.text);
|
|
367
|
+
questions = Array.isArray(parsed?.questions) ? parsed.questions.slice(0, max).filter(q => q && q.question) : [];
|
|
368
|
+
if (!questions.length) log(' no usable questions returned; continuing without.');
|
|
369
|
+
else {
|
|
370
|
+
questions.forEach((q, i) => log(` ${i + 1}. ${q.question}\n default: ${q.default}`));
|
|
371
|
+
const answered = cache.get('answers');
|
|
372
|
+
if (answered) {
|
|
373
|
+
log(` answers: from disk (${answered.text.split(/\s+/).length} words)`);
|
|
374
|
+
request += R.answersSection(questions, answered.text);
|
|
375
|
+
} else if (config.questions.wait === false) {
|
|
376
|
+
log(' wait: false - defaults taken for every question.');
|
|
377
|
+
request += R.answersSection(questions, '');
|
|
378
|
+
} else {
|
|
379
|
+
throw new ExternalPause('answers',
|
|
380
|
+
'Answer the planner\'s questions. Write plain text: one numbered answer per question, in the same order. An answer may be "default" to take the planner\'s own default. Anything else you want the plan to know may follow the numbered answers.',
|
|
381
|
+
`# The request\n\n${request}\n\n# The questions\n\n${R.renderQuestions(questions)}`);
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
const maxRounds = config.maxRounds ?? 2;
|
|
387
|
+
const stopOnPass = config.stopOnPass !== false;
|
|
388
|
+
const open = config.scope === 'open';
|
|
389
|
+
if (open) log('scope: OPEN - every seat may add scope; additions are recorded, the verdict pass cuts');
|
|
390
|
+
|
|
391
|
+
// 1. Acceptance criteria. Written before the deliverable exists, so they
|
|
392
|
+
// describe the request rather than rationalising whatever got built.
|
|
393
|
+
let criteria = config.criteria;
|
|
394
|
+
if (!criteria || criteria.length === 0) {
|
|
395
|
+
log('\nStage: acceptance criteria');
|
|
396
|
+
const s = record(await invoke(config.seats.criteria, {
|
|
397
|
+
system: R.criteriaSystem(open),
|
|
398
|
+
user: criteriaUserPrompt(request, config),
|
|
399
|
+
log, label: 'criteria',
|
|
400
|
+
}));
|
|
401
|
+
const parsed = parseJson(s.text);
|
|
402
|
+
criteria = parsed?.criteria;
|
|
403
|
+
if (!Array.isArray(criteria) || criteria.length === 0) {
|
|
404
|
+
throw new Error('The criteria stage returned no usable criteria. Raw output kept in the run log.');
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
log(`\nAcceptance criteria (${criteria.length}):`);
|
|
408
|
+
criteria.forEach((c, i) => log(` ${i + 1}. ${c}`));
|
|
409
|
+
|
|
410
|
+
// 1b. Proposal stage (optional, config.proposals). A skeleton first, then
|
|
411
|
+
// every proposer seat - blind, concurrently - offers up to N buildable
|
|
412
|
+
// parts against it. The builder integrates and keeps a scope ledger.
|
|
413
|
+
// Authorship stays with one model: the labs propose, they never
|
|
414
|
+
// co-write (Docs/Evidence.md: mixed-author merges lose to one strong
|
|
415
|
+
// author; distinct roles win).
|
|
416
|
+
let proposals = [];
|
|
417
|
+
let dropouts = [];
|
|
418
|
+
const proposalPool = [];
|
|
419
|
+
let skeleton = null;
|
|
420
|
+
let board = null;
|
|
421
|
+
let debate = null;
|
|
422
|
+
if (config.proposals && !initialDraft) {
|
|
423
|
+
const parts = config.proposals.parts ?? 3;
|
|
424
|
+
const perPart = config.proposals.maxTokens ?? 1500;
|
|
425
|
+
log('\nStage: skeleton');
|
|
426
|
+
skeleton = record(await invoke(config.seats.skeleton || config.seats.builder, {
|
|
427
|
+
system: R.SKELETON_SYSTEM,
|
|
428
|
+
user: R.skeletonUser({ request, criteria }),
|
|
429
|
+
log, label: 'skeleton',
|
|
430
|
+
})).text;
|
|
431
|
+
const proposers = config.seats.proposers || config.seats.critics;
|
|
432
|
+
// Best-of-N: `samples` independent attempts per lab, pooled, then a judge
|
|
433
|
+
// keeps the strongest `keep` distinct ones. samples=1 means no judging.
|
|
434
|
+
const samples = config.proposals.samples ?? 1;
|
|
435
|
+
const keep = config.proposals.keep ?? parts;
|
|
436
|
+
log(`\nStage: proposals (${proposers.length} labs, ${samples} attempt(s) of up to ${parts} parts each, blind${samples > 1 ? `; a judge keeps up to ${keep} per lab` : ''})`);
|
|
437
|
+
const results = await Promise.all(proposers.map(async seat => {
|
|
438
|
+
const lines = [];
|
|
439
|
+
const say = m => lines.push(m);
|
|
440
|
+
const capped = { ...seat, maxTokens: Math.min(seat.maxTokens ?? 8000, parts * perPart + 300) };
|
|
441
|
+
const pool = [];
|
|
442
|
+
let unreadable = 0;
|
|
443
|
+
for (let k = 0; k < samples; k++) {
|
|
444
|
+
const st = record(await invoke(capped, {
|
|
445
|
+
system: R.proposerSystem(open),
|
|
446
|
+
user: R.proposerUser({ request, criteria, skeleton, parts }),
|
|
447
|
+
log: say, label: `propose-${labOf(seat)}${samples > 1 ? `-${k + 1}` : ''}`,
|
|
448
|
+
}));
|
|
449
|
+
const parsed = parseJson(st.text);
|
|
450
|
+
const list = Array.isArray(parsed?.proposals) ? parsed.proposals.slice(0, parts) : null;
|
|
451
|
+
if (!list) unreadable++; else pool.push(...list.filter(p => p && p.title).map(p => ({ ...p, attempt: k + 1 })));
|
|
452
|
+
}
|
|
453
|
+
if (unreadable) say(` ${labOf(seat)}/${seat.model}: ${unreadable} of ${samples} attempt(s) unreadable.`);
|
|
454
|
+
// A lab that returns nothing readable drops out of the panel entirely:
|
|
455
|
+
// no proposals, no seat in the debate, no row anyone is likely to miss
|
|
456
|
+
// it from. For a harness whose whole claim is that independent labs
|
|
457
|
+
// disagree on the record, losing a lab to one malformed reply quietly
|
|
458
|
+
// degrades the exact thing being measured - so buy one retry before
|
|
459
|
+
// accepting it. The retry goes through invoke() like any other call, so
|
|
460
|
+
// the spend cap still governs it.
|
|
461
|
+
if (!pool.length) {
|
|
462
|
+
const st = record(await invoke(capped, {
|
|
463
|
+
system: R.proposerSystem(open),
|
|
464
|
+
user: R.proposerUser({ request, criteria, skeleton, parts }),
|
|
465
|
+
log: say, label: `propose-${labOf(seat)}-retry`,
|
|
466
|
+
}));
|
|
467
|
+
const parsed = parseJson(st.text);
|
|
468
|
+
const list2 = Array.isArray(parsed?.proposals) ? parsed.proposals.slice(0, parts) : null;
|
|
469
|
+
if (list2) pool.push(...list2.filter(p => p && p.title).map(p => ({ ...p, attempt: samples + 1 })));
|
|
470
|
+
say(` ${labOf(seat)}/${seat.model}: nothing readable; retried once - ${pool.length ? `recovered ${pool.length} proposal(s)` : 'still nothing'}.`);
|
|
471
|
+
}
|
|
472
|
+
let list = pool;
|
|
473
|
+
let judged = null;
|
|
474
|
+
if (samples > 1 && pool.length > 0) {
|
|
475
|
+
const judgeSeat = config.seats.judge || seat;
|
|
476
|
+
const js = record(await invoke(judgeSeat, {
|
|
477
|
+
system: R.JUDGE_SYSTEM,
|
|
478
|
+
user: R.judgeUser({ request, criteria, skeleton, pool, keep }),
|
|
479
|
+
log: say, label: `judge-${labOf(seat)}`,
|
|
480
|
+
}));
|
|
481
|
+
const jp = parseJson(js.text);
|
|
482
|
+
const picks = Array.isArray(jp?.picks) ? [...new Set(jp.picks.map(Number).filter(n => n >= 1 && n <= pool.length))].slice(0, keep) : null;
|
|
483
|
+
if (picks && picks.length) {
|
|
484
|
+
list = picks.map(n => pool[n - 1]);
|
|
485
|
+
judged = jp.dropped_because || '';
|
|
486
|
+
} else {
|
|
487
|
+
list = pool.slice(0, keep);
|
|
488
|
+
say(` ${labOf(seat)}/${seat.model}: judge reply unreadable - keeping the first ${list.length} of the pool.`);
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
if (!list.length) say(` ${labOf(seat)}/${seat.model}: no proposals.`);
|
|
492
|
+
else say(` ${labOf(seat)}/${seat.model}: ${list.length} proposal(s)${samples > 1 ? ` kept from a pool of ${pool.length}` : ''}${list.map(p => `\n - ${p.title}`).join('')}${judged ? `\n judge: ${judged}` : ''}`);
|
|
493
|
+
// v5 §1 candidate 10: opt-in only, no default (his call, 2026-09-13) -
|
|
494
|
+
// a chain that never sets max_proposals_per_seat behaves exactly as it
|
|
495
|
+
// did before this candidate existed. Only a chain that does set it, and
|
|
496
|
+
// only a seat whose own list still exceeds that cap, pays for one more
|
|
497
|
+
// prompt to fold its own proposals down before the board sees them.
|
|
498
|
+
const cap = Number.isInteger(config.proposals.maxProposalsPerSeat) && config.proposals.maxProposalsPerSeat > 0
|
|
499
|
+
? config.proposals.maxProposalsPerSeat : null;
|
|
500
|
+
if (cap && list.length > cap) {
|
|
501
|
+
const ms = record(await invoke(capped, {
|
|
502
|
+
system: R.PROPOSAL_MERGE_SYSTEM,
|
|
503
|
+
user: R.proposalMergeUser({ request, criteria, skeleton, list, cap }),
|
|
504
|
+
log: say, label: `propose-${labOf(seat)}-merge`,
|
|
505
|
+
}));
|
|
506
|
+
const parsed = parseJson(ms.text);
|
|
507
|
+
const picks = Array.isArray(parsed?.kept) ? [...new Set(parsed.kept.map(Number).filter(n => n >= 1 && n <= list.length))].slice(0, cap) : null;
|
|
508
|
+
const before = list.length;
|
|
509
|
+
list = picks && picks.length ? picks.map(n => list[n - 1]) : list.slice(0, cap);
|
|
510
|
+
say(` ${labOf(seat)}/${seat.model}: folded ${before} proposal(s) down to ${list.length} (cap ${cap})${parsed?.merged_because ? ` - ${parsed.merged_because}` : ''}`);
|
|
511
|
+
}
|
|
512
|
+
return { seat, list, pool, lines };
|
|
513
|
+
}));
|
|
514
|
+
// Labs that ended the stage with nothing, after the retry. Recorded so
|
|
515
|
+
// the run report shows a shrunken roster instead of leaving it to be
|
|
516
|
+
// inferred from a missing scoreboard row.
|
|
517
|
+
dropouts = results.filter(r => !r.list.length).map(r => ({ lab: labOf(r.seat), model: r.seat.model, stage: 'proposals', reason: 'no readable proposals after a retry' }));
|
|
518
|
+
|
|
519
|
+
const seen = {};
|
|
520
|
+
for (const { seat, list, pool, lines } of results) {
|
|
521
|
+
lines.forEach(m => log(m));
|
|
522
|
+
let tag = labOf(seat).toUpperCase().replace(/[^A-Z0-9]/g, '');
|
|
523
|
+
seen[tag] = (seen[tag] || 0) + 1;
|
|
524
|
+
if (seen[tag] > 1) tag += String(seen[tag]);
|
|
525
|
+
list.forEach((p, i) => proposals.push({ id: `${tag}-${i + 1}`, lab: labOf(seat), model: seat.model, ...p }));
|
|
526
|
+
pool.forEach(p => proposalPool.push({ lab: labOf(seat), model: seat.model, ...p, kept: list.includes(p) }));
|
|
527
|
+
}
|
|
528
|
+
log(` ${proposals.length} proposal(s) go to the builder.`);
|
|
529
|
+
if (dropouts.length) {
|
|
530
|
+
log(` !! ROSTER SHRANK: ${dropouts.length} of ${proposers.length} labs produced nothing and are not in this run's debate or scoreboard:`);
|
|
531
|
+
for (const d of dropouts) log(` - ${d.lab}/${d.model} (${d.reason})`);
|
|
532
|
+
log(` Independence is measured across the labs that actually spoke, not the ones the chain names.`);
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
// Debate (config.debate): every lab reads the others' proposals and posts;
|
|
536
|
+
// every author replies. Labs are anonymised in the prompts. The board is
|
|
537
|
+
// what the builder reads instead of bare proposals.
|
|
538
|
+
if (config.debate && proposals.length > 1) {
|
|
539
|
+
const maps = R.anonymise(proposals);
|
|
540
|
+
const labs = [...new Set(proposals.map(p => p.lab))];
|
|
541
|
+
const seatOf = lab => proposers.find(s => labOf(s) === lab);
|
|
542
|
+
log(`\nStage: debate (${labs.length} labs read each other's proposals, anonymised)`);
|
|
543
|
+
const postResults = await Promise.all(labs.map(async lab => {
|
|
544
|
+
const lines = []; const say = m => lines.push(m);
|
|
545
|
+
let posts = [], revisions = [];
|
|
546
|
+
try {
|
|
547
|
+
// v6 §1/§3: role augmentation applies only here, the debate-stage
|
|
548
|
+
// system prompt - never to the panel/critique stage. A seat with
|
|
549
|
+
// no role gets R.DEBATE_SYSTEM back unchanged (applySeatRole is a
|
|
550
|
+
// no-op), which is what the golden-hash compatibility test in
|
|
551
|
+
// test/seat-role.test.js checks.
|
|
552
|
+
const st = record(await invoke(seatOf(lab), {
|
|
553
|
+
system: applySeatRole(R.DEBATE_SYSTEM, seatOf(lab)?.role),
|
|
554
|
+
user: R.debateUser({ request, criteria, skeleton, proposals, lab, maps }),
|
|
555
|
+
log: say, label: `debate-${lab}`,
|
|
556
|
+
}));
|
|
557
|
+
const parsed = parseJson(st.text);
|
|
558
|
+
if (!parsed) say(` ${lab}: unreadable debate reply - no posts counted.`);
|
|
559
|
+
else {
|
|
560
|
+
posts = (parsed.posts || []).map(x => ({ by: lab, on: maps.idFrom[x.on] || x.on, stance: String(x.stance || '').toLowerCase(), text: x.text || '', merge_with: x.merge_with ? (maps.idFrom[x.merge_with] || x.merge_with) : undefined }))
|
|
561
|
+
.filter(x => proposals.some(p => p.id === x.on && p.lab !== lab) && ['support', 'object', 'merge'].includes(x.stance));
|
|
562
|
+
revisions = (parsed.revisions || []).map(r => ({ ...r, id: maps.idFrom[r.id] || r.id })).filter(r => proposals.some(p => p.id === r.id && p.lab === lab));
|
|
563
|
+
const n = st => posts.filter(x => x.stance === st).length;
|
|
564
|
+
say(` ${lab}: ${posts.length} post(s) - ${n('support')} support, ${n('object')} object, ${n('merge')} merge${revisions.length ? `; revised ${revisions.length} of its own` : ''}`);
|
|
565
|
+
}
|
|
566
|
+
} catch (err) {
|
|
567
|
+
say(` ${lab}: no debate reply (${String(err.message).slice(0, 100)}).`);
|
|
568
|
+
}
|
|
569
|
+
return { lines, posts, revisions };
|
|
570
|
+
}));
|
|
571
|
+
const posts = [];
|
|
572
|
+
for (const r of postResults) { r.lines.forEach(m => log(m)); posts.push(...r.posts); for (const rev of r.revisions) { const p = proposals.find(x => x.id === rev.id); if (rev.how) p.how = rev.how; if (rev.acceptance_test) p.acceptance_test = rev.acceptance_test; p.amended = true; } }
|
|
573
|
+
|
|
574
|
+
log(`\nStage: replies (each author answers the posts on its proposals)`);
|
|
575
|
+
const replies = [];
|
|
576
|
+
const replyResults = await Promise.all(labs.map(async lab => {
|
|
577
|
+
const lines = []; const say = m => lines.push(m);
|
|
578
|
+
const mineWithPosts = proposals.filter(p => p.lab === lab && posts.some(x => x.on === p.id));
|
|
579
|
+
if (!mineWithPosts.length) { say(` ${lab}: nothing to answer.`); return { lines, replies: [] }; }
|
|
580
|
+
try {
|
|
581
|
+
const st = record(await invoke(seatOf(lab), {
|
|
582
|
+
system: R.REPLY_SYSTEM,
|
|
583
|
+
user: R.replyUser({ request, proposals, posts, lab, maps }),
|
|
584
|
+
log: say, label: `reply-${lab}`,
|
|
585
|
+
}));
|
|
586
|
+
const parsed = parseJson(st.text);
|
|
587
|
+
if (!parsed) { say(` ${lab}: unreadable reply round - proposals stand as posted.`); return { lines, replies: [] }; }
|
|
588
|
+
const mine = (parsed.replies || []).map(r => ({ ...r, id: maps.idFrom[r.id] || r.id, replaced_by: r.replaced_by ? (maps.idFrom[r.replaced_by] || r.replaced_by) : undefined, action: String(r.action || '').toLowerCase() }))
|
|
589
|
+
.filter(r => mineWithPosts.some(p => p.id === r.id) && ['keep', 'amend', 'withdraw'].includes(r.action));
|
|
590
|
+
const n = a => mine.filter(r => r.action === a).length;
|
|
591
|
+
say(` ${lab}: ${n('keep')} keep, ${n('amend')} amend, ${n('withdraw')} withdraw`);
|
|
592
|
+
return { lines, replies: mine };
|
|
593
|
+
} catch (err) {
|
|
594
|
+
say(` ${lab}: no reply-round answer (${String(err.message).slice(0, 100)}).`);
|
|
595
|
+
return { lines, replies: [] };
|
|
596
|
+
}
|
|
597
|
+
}));
|
|
598
|
+
for (const r of replyResults) { r.lines.forEach(m => log(m)); replies.push(...r.replies); }
|
|
599
|
+
for (const r of replies) {
|
|
600
|
+
const p = proposals.find(x => x.id === r.id);
|
|
601
|
+
if (r.action === 'amend') { if (r.how) p.how = r.how; if (r.acceptance_test) p.acceptance_test = r.acceptance_test; p.amended = true; }
|
|
602
|
+
if (r.action === 'withdraw') { p.withdrawn = true; p.replaced_by = r.replaced_by; }
|
|
603
|
+
}
|
|
604
|
+
board = R.renderBoard(proposals, posts, replies);
|
|
605
|
+
// v6 §4: the field is always present once a debate stage has run, even
|
|
606
|
+
// when no tie ever occurred - an absent field reads as "no tie-break
|
|
607
|
+
// happened" and as "the feature isn't wired up" identically, which is
|
|
608
|
+
// exactly the silent-drop failure class this project was burned by
|
|
609
|
+
// tonight. Phase 3 delivers the arithmetic and this always-present
|
|
610
|
+
// field; no call site in this stage decides pass/fail by vote yet, so
|
|
611
|
+
// debate runs record the no-op result until a future phase wires one.
|
|
612
|
+
debate = { posts, replies, tie_break: NO_TIE_BREAK };
|
|
613
|
+
const w = proposals.filter(p => p.withdrawn).length;
|
|
614
|
+
log(` board: ${posts.length} post(s), ${replies.length} repl${replies.length === 1 ? 'y' : 'ies'}, ${w} proposal(s) withdrawn, ${proposals.filter(p => p.amended).length} amended.`);
|
|
615
|
+
|
|
616
|
+
// v5 §1 candidate 3: a withdrawal chain that cycles or dead-ends
|
|
617
|
+
// (e.g. two labs mutually withdrawing in each other's favour) leaves
|
|
618
|
+
// a section with no surviving owner - silently, unless the next
|
|
619
|
+
// stage is told. Told here rather than aborting the run: the
|
|
620
|
+
// integration stage can still assign or explicitly drop it.
|
|
621
|
+
const ledger = withdrawalLedger(proposals);
|
|
622
|
+
if (ledger.orphanSections.length) {
|
|
623
|
+
log(` WARNING: ${ledger.orphanSections.length} withdrawn proposal(s) have no surviving owner (${ledger.withdrawalCycles} withdrawal cycle(s)): ${ledger.orphanSections.join(', ')}`);
|
|
624
|
+
board += `\n\n# Orphaned withdrawals - no surviving owner\n\nThese proposal ids withdrew in a chain that never reaches a proposal still standing (a cycle, or a dead end): ${ledger.orphanSections.join(', ')}. For each one, either assign the section it covered to something else in the plan, or state explicitly in the Scope ledger that it is dropped and why - do not silently leave it uncovered.`;
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
}
|
|
628
|
+
|
|
629
|
+
// 2. First build - unless a draft was handed in. Re-running a panel on the
|
|
630
|
+
// exact draft an earlier run reviewed is the only fair way to compare
|
|
631
|
+
// panel topologies; a fresh build is a fresh coin toss.
|
|
632
|
+
let draft = initialDraft;
|
|
633
|
+
if (draft) {
|
|
634
|
+
log('\nRound 1: build (skipped - reviewing a draft handed in)');
|
|
635
|
+
} else {
|
|
636
|
+
log('\nRound 1: build');
|
|
637
|
+
draft = record(await invoke(config.seats.builder, {
|
|
638
|
+
system: R.builderSystem(open),
|
|
639
|
+
user: R.builderUser({ request, criteria, proposals, board }),
|
|
640
|
+
log, label: 'build',
|
|
641
|
+
})).text;
|
|
642
|
+
}
|
|
643
|
+
|
|
644
|
+
// 3. Critic / revise rounds, hard-capped.
|
|
645
|
+
const history = [];
|
|
646
|
+
let passed = false;
|
|
647
|
+
let lastCritique = null;
|
|
648
|
+
let signoff = null;
|
|
649
|
+
const disputes = [];
|
|
650
|
+
|
|
651
|
+
if (config.signoff === 'unanimous') {
|
|
652
|
+
for (let round = 1; round <= maxRounds; round++) {
|
|
653
|
+
const relay = config.panel === 'relay';
|
|
654
|
+
log(`\nRound ${round}: panel review (${config.seats.critics.length} labs, ${relay ? 'relay - each sees the verdicts before it' : 'independent'})`);
|
|
655
|
+
const verdicts = [];
|
|
656
|
+
const reviewSeat = async (criticSeat, prior, say) => {
|
|
657
|
+
let cs;
|
|
658
|
+
try {
|
|
659
|
+
cs = record(await invoke(criticSeat, {
|
|
660
|
+
system: R.criticSystem(open),
|
|
661
|
+
user: R.criticUser({ request, criteria, draft, prior }),
|
|
662
|
+
log: say, label: `panel-${round}-${labOf(criticSeat)}`,
|
|
663
|
+
}));
|
|
664
|
+
} catch (err) {
|
|
665
|
+
// A lab that is down (429 after retries, 5xx, network) must not take
|
|
666
|
+
// the panel down with it. It abstains, and the log says why.
|
|
667
|
+
say(` ${labOf(criticSeat)}/${criticSeat.model}: no reply (${String(err.message).slice(0, 120)}) - counted as an abstention.`);
|
|
668
|
+
return { seat: criticSeat, critique: null, abstained: true, error: String(err.message) };
|
|
669
|
+
}
|
|
670
|
+
const parsed = parseJson(cs.text);
|
|
671
|
+
if (!parsed) {
|
|
672
|
+
// An unreadable verdict is an abstention: it neither signs off nor
|
|
673
|
+
// objects, and it cannot block the panel. It used to count as a
|
|
674
|
+
// pass, which would have waved a truncated FAILED straight through.
|
|
675
|
+
const why = classifyUnreadable(cs.usage, criticSeat.maxTokens);
|
|
676
|
+
// v5 §1 candidate 4: the code is prepended, the diagnosis itself
|
|
677
|
+
// is untouched - classifyUnreadable's three distinct reasons are
|
|
678
|
+
// load-bearing (each one traces to a real incident on disk) and
|
|
679
|
+
// this candidate changes what's printed, never what's diagnosed.
|
|
680
|
+
say(` ${labOf(criticSeat)}/${criticSeat.model}: [COUNCIL-E004] unreadable reply (${cs.usage.output} tokens out; ${why}) - counted as an abstention, not a sign-off.`);
|
|
681
|
+
return { seat: criticSeat, critique: null, abstained: true };
|
|
682
|
+
}
|
|
683
|
+
const critique = normaliseCritique(parsed, say);
|
|
684
|
+
say(` ${labOf(criticSeat)}/${criticSeat.model}: ${critique.meets ? 'SIGNED OFF' : `${critique.failures.length} failure(s)`} - ${critique.verdict_line || ''}`);
|
|
685
|
+
critique.failures.forEach(f => say(` FAILED: ${f.criterion} - ${f.problem}`));
|
|
686
|
+
return { seat: criticSeat, critique };
|
|
687
|
+
};
|
|
688
|
+
|
|
689
|
+
if (relay) {
|
|
690
|
+
// Relay seats run one after another, in a fresh random order each
|
|
691
|
+
// round so no lab is always the anchor, and earlier verdicts are
|
|
692
|
+
// handed on anonymised so a seat can't defer to a name.
|
|
693
|
+
for (const criticSeat of shuffle(config.seats.critics)) {
|
|
694
|
+
const prior = verdicts.filter(v => v.critique).map((v, i) => ({ lab: `Reviewer ${String.fromCharCode(65 + i)}`, verdict_line: v.critique.verdict_line, failures: v.critique.failures }));
|
|
695
|
+
verdicts.push(await reviewSeat(criticSeat, prior, log));
|
|
696
|
+
}
|
|
697
|
+
} else {
|
|
698
|
+
// Blind seats see nothing of each other, so they run at the same
|
|
699
|
+
// time. Each seat's log lines are buffered and flushed in seat order,
|
|
700
|
+
// so the run log reads exactly as it did when they ran in sequence.
|
|
701
|
+
const results = await Promise.all(config.seats.critics.map(async criticSeat => {
|
|
702
|
+
const lines = [];
|
|
703
|
+
const verdict = await reviewSeat(criticSeat, [], m => lines.push(m));
|
|
704
|
+
return { verdict, lines };
|
|
705
|
+
}));
|
|
706
|
+
for (const { verdict, lines } of results) {
|
|
707
|
+
lines.forEach(m => log(m));
|
|
708
|
+
verdicts.push(verdict);
|
|
709
|
+
}
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
// Each objection carries the lab that raised it, so the human review
|
|
713
|
+
// step sees who said what rather than an anonymous merged list.
|
|
714
|
+
const voting = verdicts.filter(v => v.critique);
|
|
715
|
+
const abstained = verdicts.length - voting.length;
|
|
716
|
+
const allFailures = voting.flatMap(v => v.critique.failures.map(f => ({ ...f, lab: labOf(v.seat) })));
|
|
717
|
+
const allSignedOff = voting.length > 0 && voting.every(v => v.critique.meets === true);
|
|
718
|
+
lastCritique = { meets: allSignedOff, failures: allFailures };
|
|
719
|
+
// `objections` is why a seat declined, co-located with the decision itself.
|
|
720
|
+
// The same failures also appear flattened in lastCritique.failures, tagged
|
|
721
|
+
// with `lab`, because the reviser and the CLI want the union across the
|
|
722
|
+
// panel. Both are built here from the same `voting` array in one pass, so
|
|
723
|
+
// they cannot drift. Added 2026-09-11: a four-lab panel reading a run
|
|
724
|
+
// folder concluded a holdout's reason was not machine-readable, because
|
|
725
|
+
// signoff carried the verdict and nothing else - the join through
|
|
726
|
+
// lastCritique.failures[].lab existed but was undiscoverable.
|
|
727
|
+
signoff = verdicts.map(v => ({
|
|
728
|
+
provider: labOf(v.seat), model: v.seat.model,
|
|
729
|
+
signedOff: v.abstained ? null : v.critique.meets === true,
|
|
730
|
+
objections: v.abstained ? null : v.critique.failures,
|
|
731
|
+
}));
|
|
732
|
+
|
|
733
|
+
history.push(`## Round ${round} panel\n${verdicts.map(v =>
|
|
734
|
+
`- ${labOf(v.seat)}/${v.seat.model}: ${v.abstained ? 'abstained (unreadable reply)' : v.critique.meets ? 'signed off' : `${v.critique.failures.length} failure(s)`}`
|
|
735
|
+
).join('\n')}`);
|
|
736
|
+
|
|
737
|
+
if (allSignedOff) {
|
|
738
|
+
passed = true;
|
|
739
|
+
log(abstained
|
|
740
|
+
? ` every lab that answered signed off (${abstained} abstained); stopping.`
|
|
741
|
+
: ` every lab on the panel signed off; stopping.`);
|
|
742
|
+
break;
|
|
743
|
+
}
|
|
744
|
+
|
|
745
|
+
if (round === maxRounds) {
|
|
746
|
+
log(` round cap (${maxRounds}) reached without unanimous signoff; open objections go into the report.`);
|
|
747
|
+
break;
|
|
748
|
+
}
|
|
749
|
+
|
|
750
|
+
log(`\nRound ${round}: revise (union of everything any lab flagged)`);
|
|
751
|
+
const reviserSeat = config.seats.reviser || config.seats.builder;
|
|
752
|
+
const revised = record(await invoke(reviserSeat, {
|
|
753
|
+
system: R.reviserSystem(open),
|
|
754
|
+
user: R.reviserUser({ request, criteria, draft, critique: { failures: allFailures }, proposals, board }),
|
|
755
|
+
log, label: `revise-${round}`,
|
|
756
|
+
})).text;
|
|
757
|
+
const parsedRevise = parseDisputes(revised);
|
|
758
|
+
draft = parsedRevise.draft;
|
|
759
|
+
parsedRevise.disputes.forEach(reason => disputes.push({ round, reason }));
|
|
760
|
+
passed = false;
|
|
761
|
+
}
|
|
762
|
+
} else {
|
|
763
|
+
for (let round = 1; round <= maxRounds; round++) {
|
|
764
|
+
const criticSeat = config.seats.critics[(round - 1) % config.seats.critics.length];
|
|
765
|
+
log(`\nRound ${round}: critique`);
|
|
766
|
+
const cs = record(await invoke(criticSeat, {
|
|
767
|
+
system: R.criticSystem(open),
|
|
768
|
+
user: R.criticUser({ request, criteria, draft }),
|
|
769
|
+
log, label: `critique-${round}`,
|
|
770
|
+
}));
|
|
771
|
+
|
|
772
|
+
const parsed = parseJson(cs.text);
|
|
773
|
+
if (!parsed) {
|
|
774
|
+
// Previously logged "treating the round as a pass and stopping" while leaving `passed`
|
|
775
|
+
// at its prior value and `lastCritique` untouched - the log claimed a pass that never
|
|
776
|
+
// happened, and the report came out as passed:false with no recorded reason at all
|
|
777
|
+
// (found for real 2026-09-10: three separate Sophi-A runs stopped exactly here, silent).
|
|
778
|
+
// Deliberately NOT treated as a real pass - a critic's reply we can't parse might be
|
|
779
|
+
// hiding a genuine objection, and this chain's whole value is not letting that slip
|
|
780
|
+
// through. `passed` keeps whatever it already was (false unless an earlier round already
|
|
781
|
+
// passed); what changes is that the report now says why, instead of nothing.
|
|
782
|
+
log(` critic reply could not be parsed as JSON even after repair attempts; stopping ` +
|
|
783
|
+
`without a verdict from this critic - not a pass, not counted as an objection either.`);
|
|
784
|
+
lastCritique = { meets: false, failures: [{
|
|
785
|
+
criterion: '(critic reply unparseable)',
|
|
786
|
+
problem: `${criticSeat.provider}/${criticSeat.model}'s round ${round} reply could not ` +
|
|
787
|
+
`be parsed as JSON even after repair attempts - no verdict was recovered from it.`,
|
|
788
|
+
}] };
|
|
789
|
+
break;
|
|
790
|
+
}
|
|
791
|
+
const critique = normaliseCritique(parsed, log);
|
|
792
|
+
lastCritique = critique;
|
|
793
|
+
const failures = critique.failures;
|
|
794
|
+
log(` verdict: ${critique.meets ? 'MEETS' : `${failures.length} failure(s)`} - ${critique.verdict_line || ''}`);
|
|
795
|
+
failures.forEach(f => log(` FAILED: ${f.criterion} - ${f.problem}`));
|
|
796
|
+
|
|
797
|
+
history.push(`## Round ${round} critique (${criticSeat.provider}/${criticSeat.model})\n${critique.verdict_line || ''}\n${failures.map(f => `- FAILED: ${f.criterion} - ${f.problem}`).join('\n')}`);
|
|
798
|
+
|
|
799
|
+
if (critique.meets === true || failures.length === 0) {
|
|
800
|
+
passed = true;
|
|
801
|
+
if (stopOnPass) {
|
|
802
|
+
log(` a critic from a different lab passed it; stopping early rather than inventing work.`);
|
|
803
|
+
break;
|
|
804
|
+
}
|
|
805
|
+
}
|
|
806
|
+
|
|
807
|
+
if (round === maxRounds) {
|
|
808
|
+
log(` round cap (${maxRounds}) reached; the remaining failures go into the report, not another round.`);
|
|
809
|
+
break;
|
|
810
|
+
}
|
|
811
|
+
|
|
812
|
+
log(`\nRound ${round}: revise`);
|
|
813
|
+
const reviserSeat = config.seats.reviser || config.seats.builder;
|
|
814
|
+
const revised = record(await invoke(reviserSeat, {
|
|
815
|
+
system: R.reviserSystem(open),
|
|
816
|
+
user: R.reviserUser({ request, criteria, draft, critique }),
|
|
817
|
+
log, label: `revise-${round}`,
|
|
818
|
+
})).text;
|
|
819
|
+
const parsedRevise = parseDisputes(revised);
|
|
820
|
+
draft = parsedRevise.draft;
|
|
821
|
+
parsedRevise.disputes.forEach(reason => disputes.push({ round, reason }));
|
|
822
|
+
passed = false;
|
|
823
|
+
}
|
|
824
|
+
}
|
|
825
|
+
|
|
826
|
+
// 4. Optional final edit: strips chain artifacts. Never adds material.
|
|
827
|
+
if (config.seats.finalist) {
|
|
828
|
+
log('\nStage: final edit');
|
|
829
|
+
draft = record(await invoke(config.seats.finalist, {
|
|
830
|
+
system: R.FINALIST_SYSTEM,
|
|
831
|
+
user: R.finalistUser({ request, draft, history: history.join('\n\n') }),
|
|
832
|
+
log, label: 'final',
|
|
833
|
+
})).text;
|
|
834
|
+
}
|
|
835
|
+
|
|
836
|
+
// 5. Handoff (config.handoff): the file a build session reads first.
|
|
837
|
+
let handoff = null;
|
|
838
|
+
if (config.handoff) {
|
|
839
|
+
log('\nStage: handoff');
|
|
840
|
+
handoff = record(await invoke(config.seats.handoff || config.seats.builder, {
|
|
841
|
+
system: R.HANDOFF_SYSTEM,
|
|
842
|
+
user: R.handoffUser({ request, draft, planFile: config.handoffPlanFile || 'PLAN.md' }),
|
|
843
|
+
log, label: 'handoff',
|
|
844
|
+
})).text;
|
|
845
|
+
}
|
|
846
|
+
|
|
847
|
+
// Scoreboard: what each lab proposed and what the plan did with it. The
|
|
848
|
+
// "built" column can only be filled after a build session; it is left to
|
|
849
|
+
// the human on purpose (a part that reads well may not build).
|
|
850
|
+
const scoreboard = proposals.length ? scoreProposals(proposals, draft) : null;
|
|
851
|
+
if (scoreboard) {
|
|
852
|
+
log('\nProposal scoreboard (built column: fill in after the build session)');
|
|
853
|
+
for (const row of scoreboard.labs) log(` ${row.lab.padEnd(12)} proposed ${row.proposed} accepted ${row.accepted} cut ${row.cut} withdrawn ${row.withdrawn} unaccounted ${row.unaccounted}`);
|
|
854
|
+
}
|
|
855
|
+
|
|
856
|
+
const ledger = proposals.length ? withdrawalLedger(proposals) : null;
|
|
857
|
+
|
|
858
|
+
return {
|
|
859
|
+
deliverable: draft,
|
|
860
|
+
criteria,
|
|
861
|
+
questions,
|
|
862
|
+
skeleton,
|
|
863
|
+
proposals,
|
|
864
|
+
proposalPool,
|
|
865
|
+
dropouts,
|
|
866
|
+
board,
|
|
867
|
+
debate,
|
|
868
|
+
handoff,
|
|
869
|
+
scoreboard,
|
|
870
|
+
passed,
|
|
871
|
+
lastCritique,
|
|
872
|
+
signoff,
|
|
873
|
+
disputes,
|
|
874
|
+
history,
|
|
875
|
+
stages,
|
|
876
|
+
totals: summarise(stages),
|
|
877
|
+
orphanSections: ledger?.orphanSections ?? [],
|
|
878
|
+
withdrawalCycles: ledger?.withdrawalCycles ?? 0,
|
|
879
|
+
};
|
|
880
|
+
}
|