pgroast 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +173 -2
- package/bin/pgroast.mjs +19 -0
- package/data/precedents.sql.gz +0 -0
- package/docs/corpus.md +352 -0
- package/docs/design.md +540 -0
- package/docs/eval.md +245 -0
- package/docs/haas-paranoia-notes.md +223 -0
- package/invariants/README.md +15 -0
- package/invariants/nbtree.md +14 -0
- package/package.json +51 -4
- package/prompts/brief.md +32 -0
- package/prompts/committer-paranoia.md +149 -0
- package/prompts/critic.md +56 -0
- package/prompts/enrich.md +35 -0
- package/prompts/pushback-map.md +58 -0
- package/prompts/pushback-reduce.md +72 -0
- package/prompts/reflect.md +20 -0
- package/prompts/relevance.md +20 -0
- package/prompts/roaster-already-exists.md +23 -0
- package/prompts/roaster-common.md +88 -0
- package/prompts/roaster-demand.md +28 -0
- package/prompts/roaster-maintenance-fit.md +22 -0
- package/prompts/roaster-prior-attempts.md +29 -0
- package/prompts/roaster-replication-ondisk.md +24 -0
- package/prompts/roaster-security.md +21 -0
- package/prompts/roaster-standards-compat.md +21 -0
- package/prompts/supervisor.md +31 -0
- package/prompts/synth-self.md +19 -0
- package/prompts/synth.md +140 -0
- package/prompts/tone.md +21 -0
- package/prompts/writing.md +98 -0
- package/sql/schema.sql +45 -0
- package/src/cli.ts +391 -0
- package/src/corpus/cf.ts +220 -0
- package/src/corpus/cli.ts +189 -0
- package/src/corpus/db.ts +119 -0
- package/src/corpus/fetch.ts +104 -0
- package/src/corpus/reverts.ts +133 -0
- package/src/corpus/search.ts +201 -0
- package/src/corpus/util.ts +133 -0
- package/src/corpus/wiki.ts +80 -0
- package/src/eval/cases.ts +118 -0
- package/src/eval/cli.ts +105 -0
- package/src/eval/judge.ts +136 -0
- package/src/eval/report.ts +268 -0
- package/src/eval/run.ts +153 -0
- package/src/eval/score.ts +275 -0
- package/src/roast/doctor.ts +109 -0
- package/src/roast/enrich.ts +335 -0
- package/src/roast/invariants.ts +65 -0
- package/src/roast/llm.ts +601 -0
- package/src/roast/material.ts +133 -0
- package/src/roast/page.ts +266 -0
- package/src/roast/pipeline.ts +981 -0
- package/src/roast/precedents.ts +308 -0
- package/src/roast/pushback.ts +664 -0
- package/src/roast/render.ts +386 -0
- package/src/roast/sanitize.ts +140 -0
- package/src/roast/schema.ts +714 -0
- package/src/roast/tone.ts +216 -0
- package/src/thread/fetch.ts +172 -0
- package/src/thread/load.ts +227 -0
- package/src/thread/local.ts +73 -0
- package/src/thread/parse.ts +243 -0
- package/src/thread/patchprose.ts +143 -0
- package/src/thread/types.ts +120 -0
|
@@ -0,0 +1,981 @@
|
|
|
1
|
+
// The roast pipeline: brief -> retrieval (code) -> roasters (parallel) -> critic -> synthesis
|
|
2
|
+
// -> assembly (code). Everything the model returns passes through src/roast/sanitize.ts.
|
|
3
|
+
|
|
4
|
+
import { readFileSync } from "node:fs";
|
|
5
|
+
import { join } from "node:path";
|
|
6
|
+
import { REPO_ROOT } from "../thread/fetch.ts";
|
|
7
|
+
import type { ProposalInput, Thread } from "../thread/types.ts";
|
|
8
|
+
import { loadInvariantFiles, renderInvariants, selectInvariants, type InvariantFile } from "./invariants.ts";
|
|
9
|
+
import { toneCheck } from "./tone.ts";
|
|
10
|
+
import { ENRICH_SCHEMA, RELEVANCE_SCHEMA, enrichPrecedents, openPgEnrichmentStore, type EnrichmentStore } from "./enrich.ts";
|
|
11
|
+
import type { LlmBackend } from "./llm.ts";
|
|
12
|
+
import { materialRawText, renderMaterial } from "./material.ts";
|
|
13
|
+
import { renderPrecedentsForPrompt, retrievePrecedents, type SearchFn } from "./precedents.ts";
|
|
14
|
+
import { AllowedUrls, emptyReport, filterPrecedentIds, filterThreadRefs, scrubUrls, verifyQuote } from "./sanitize.ts";
|
|
15
|
+
import {
|
|
16
|
+
ANGLES,
|
|
17
|
+
BRIEF_SCHEMA,
|
|
18
|
+
CRITIC_SCHEMA,
|
|
19
|
+
DISCLOSURE,
|
|
20
|
+
LEVELS,
|
|
21
|
+
LEVEL_NAMES,
|
|
22
|
+
REFLECT_SCHEMA,
|
|
23
|
+
ROASTER_SCHEMA,
|
|
24
|
+
SCHEMA_VERSION,
|
|
25
|
+
SUPERVISOR_SCHEMA,
|
|
26
|
+
SYNTH_SCHEMA,
|
|
27
|
+
SYNTH_SELF_SCHEMA,
|
|
28
|
+
TONE_REPAIR_SCHEMA,
|
|
29
|
+
TOUCH_POINTS,
|
|
30
|
+
levelRank,
|
|
31
|
+
type Angle,
|
|
32
|
+
type Brief,
|
|
33
|
+
type CriticOutput,
|
|
34
|
+
type FinalObjection,
|
|
35
|
+
type JsonSchema,
|
|
36
|
+
type Level,
|
|
37
|
+
type Precedent,
|
|
38
|
+
type PriorAttempt,
|
|
39
|
+
type RawObjection,
|
|
40
|
+
type RawPriorAttempt,
|
|
41
|
+
type ReflectOutput,
|
|
42
|
+
type Roast,
|
|
43
|
+
type RoundRecord,
|
|
44
|
+
type SelfAdvice,
|
|
45
|
+
type SupervisorOutput,
|
|
46
|
+
type RoasterOutput,
|
|
47
|
+
type SanitizerReport,
|
|
48
|
+
type SynthOutput,
|
|
49
|
+
type TouchPointResult,
|
|
50
|
+
type Verdict,
|
|
51
|
+
} from "./schema.ts";
|
|
52
|
+
|
|
53
|
+
export interface RunOptions {
|
|
54
|
+
backend: LlmBackend;
|
|
55
|
+
models: { roaster: string; critic: string; synth: string; brief: string };
|
|
56
|
+
/** "self" = the author's own pre-flight check: same pipeline, other framing. */
|
|
57
|
+
mode?: "review" | "self";
|
|
58
|
+
/** Extra rounds after the first pass (default 0). 1 = "What did you forget?", 2 = + "Are you happy with the results?", 3+ = supervisor follow-ups. */
|
|
59
|
+
rounds?: number;
|
|
60
|
+
/** Injected for tests; default = the files under invariants/. */
|
|
61
|
+
invariants?: InvariantFile[];
|
|
62
|
+
/** Recorded in roast.json: "claude" | "codex". */
|
|
63
|
+
backendName?: string;
|
|
64
|
+
hint?: string | null;
|
|
65
|
+
/** Injected precedent search (tests). Default: the corpus adapter. */
|
|
66
|
+
search?: SearchFn;
|
|
67
|
+
/** false = no enrichment and no relevance pass (--no-enrich). */
|
|
68
|
+
enrich?: boolean;
|
|
69
|
+
enrichMax?: number;
|
|
70
|
+
/** undefined = the table in the corpus database; null = no cache. */
|
|
71
|
+
enrichStore?: EnrichmentStore | null;
|
|
72
|
+
enrichLoadThread?: (messageId: string) => Promise<Thread>;
|
|
73
|
+
enrichClosers?: (p: Precedent) => Promise<string[]>;
|
|
74
|
+
/** At most this many precedents are shown to the roasters. */
|
|
75
|
+
precedentLimit?: number;
|
|
76
|
+
promptsDir?: string;
|
|
77
|
+
angles?: readonly Angle[];
|
|
78
|
+
log?: (msg: string) => void;
|
|
79
|
+
now?: () => Date;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// ---------------------------------------------------------------------------------------------
|
|
83
|
+
// Prompts
|
|
84
|
+
|
|
85
|
+
export interface Prompts {
|
|
86
|
+
rubric: string;
|
|
87
|
+
brief: string;
|
|
88
|
+
roasterCommon: string;
|
|
89
|
+
angles: Record<Angle, string>;
|
|
90
|
+
critic: string;
|
|
91
|
+
synth: string;
|
|
92
|
+
enrich: string;
|
|
93
|
+
relevance: string;
|
|
94
|
+
reflect: string;
|
|
95
|
+
supervisor: string;
|
|
96
|
+
synthSelf: string;
|
|
97
|
+
tone: string;
|
|
98
|
+
/** Shared writing rules (prompts/writing.md), added to every stage that writes prose. */
|
|
99
|
+
writing: string;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** The two reflection questions, verbatim from the issue. */
|
|
103
|
+
export const FORGOT_PROMPT = "What did you forget?";
|
|
104
|
+
export const HAPPY_PROMPT = "Are you happy with the results?";
|
|
105
|
+
|
|
106
|
+
export function loadPrompts(dir: string = join(REPO_ROOT, "prompts")): Prompts {
|
|
107
|
+
const read = (f: string) => readFileSync(join(dir, f), "utf8").trim();
|
|
108
|
+
return {
|
|
109
|
+
// Read from disk at run time and included verbatim: the rubric has its own owner.
|
|
110
|
+
rubric: read("committer-paranoia.md"),
|
|
111
|
+
brief: read("brief.md"),
|
|
112
|
+
roasterCommon: read("roaster-common.md"),
|
|
113
|
+
angles: Object.fromEntries(ANGLES.map((a) => [a, read(`roaster-${a}.md`)])) as Record<Angle, string>,
|
|
114
|
+
critic: read("critic.md"),
|
|
115
|
+
synth: read("synth.md"),
|
|
116
|
+
enrich: read("enrich.md"),
|
|
117
|
+
relevance: read("relevance.md"),
|
|
118
|
+
reflect: read("reflect.md"),
|
|
119
|
+
supervisor: read("supervisor.md"),
|
|
120
|
+
synthSelf: read("synth-self.md"),
|
|
121
|
+
tone: read("tone.md"),
|
|
122
|
+
writing: read("writing.md"),
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/** Render a schema as a compact TypeScript-like shape for the prompt. */
|
|
127
|
+
export function describeSchema(s: JsonSchema, indent = ""): string {
|
|
128
|
+
const nul = s.nullable ? " | null" : "";
|
|
129
|
+
switch (s.type) {
|
|
130
|
+
case "string":
|
|
131
|
+
return (s.enum ? s.enum.map((e) => JSON.stringify(e)).join(" | ") : "string") + nul;
|
|
132
|
+
case "integer":
|
|
133
|
+
case "number": {
|
|
134
|
+
const range = s.minimum != null || s.maximum != null ? ` (${s.minimum ?? ""}..${s.maximum ?? ""})` : "";
|
|
135
|
+
return s.type + range + nul;
|
|
136
|
+
}
|
|
137
|
+
case "boolean":
|
|
138
|
+
return "boolean" + nul;
|
|
139
|
+
case "array": {
|
|
140
|
+
const lim = [s.minItems != null ? `min ${s.minItems}` : "", s.maxItems != null ? `max ${s.maxItems}` : ""].filter(Boolean).join(", ");
|
|
141
|
+
return `[ ${describeSchema(s.items, indent)} ]${lim ? ` (${lim} items)` : ""}${nul}`;
|
|
142
|
+
}
|
|
143
|
+
case "object": {
|
|
144
|
+
const inner = Object.entries(s.properties)
|
|
145
|
+
.map(([k, v]) => `${indent} ${JSON.stringify(k)}: ${describeSchema(v, indent + " ")}`)
|
|
146
|
+
.join(",\n");
|
|
147
|
+
return `{\n${inner}\n${indent}}${nul}`;
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
const TOUCH_POINT_LIST = TOUCH_POINTS.map((t) => `- \`${t.id}\`: ${t.label}`).join("\n");
|
|
153
|
+
|
|
154
|
+
export function outputContract(schema: JsonSchema, withTouchPoints: boolean): string {
|
|
155
|
+
return [
|
|
156
|
+
"## Output contract",
|
|
157
|
+
"",
|
|
158
|
+
"One JSON object of this shape (all keys required; no text before or after it, no code fence):",
|
|
159
|
+
"",
|
|
160
|
+
describeSchema(schema),
|
|
161
|
+
...(withTouchPoints ? ["", "Touch point ids:", "", TOUCH_POINT_LIST] : []),
|
|
162
|
+
].join("\n");
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
function withRubric(head: string, rubric: string, tail: string): string {
|
|
166
|
+
return `${head}\n\n---\n\n# Rubric (prompts/committer-paranoia.md, verbatim)\n\n${rubric}\n\n---\n\n${tail}`;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/** Angles that get the subsystem invariants (the critic gets them too). */
|
|
170
|
+
export const INVARIANT_ANGLES: readonly Angle[] = ["replication-ondisk", "maintenance-fit"];
|
|
171
|
+
|
|
172
|
+
const withInvariants = (angle: Angle, invariants: string) => (invariants && INVARIANT_ANGLES.includes(angle) ? `\n\n${invariants}` : "");
|
|
173
|
+
|
|
174
|
+
export function roasterSystem(p: Prompts, angle: Angle, invariants = ""): string {
|
|
175
|
+
return withRubric(p.roasterCommon, p.rubric, `${p.angles[angle]}${withInvariants(angle, invariants)}\n\n${p.writing}\n\n${outputContract(ROASTER_SCHEMA, true)}`);
|
|
176
|
+
}
|
|
177
|
+
export function reflectSystem(p: Prompts, angle: Angle, invariants = ""): string {
|
|
178
|
+
return withRubric(p.roasterCommon, p.rubric, `${p.angles[angle]}${withInvariants(angle, invariants)}\n\n${p.reflect}\n\n${p.writing}\n\n${outputContract(REFLECT_SCHEMA, true)}`);
|
|
179
|
+
}
|
|
180
|
+
export function criticSystem(p: Prompts, invariants = ""): string {
|
|
181
|
+
return withRubric(p.critic, p.rubric, `${invariants ? `${invariants}\n\n` : ""}${p.writing}\n\n${outputContract(CRITIC_SCHEMA, false)}`);
|
|
182
|
+
}
|
|
183
|
+
export function synthSystem(p: Prompts, self = false): string {
|
|
184
|
+
return withRubric(p.synth, p.rubric, `${self ? `${p.synthSelf}\n\n` : ""}${p.writing}\n\n${outputContract(self ? SYNTH_SELF_SCHEMA : SYNTH_SCHEMA, true)}`);
|
|
185
|
+
}
|
|
186
|
+
export function supervisorSystem(p: Prompts): string {
|
|
187
|
+
return `${p.supervisor}\n\n${outputContract(SUPERVISOR_SCHEMA, false)}`;
|
|
188
|
+
}
|
|
189
|
+
export function briefSystem(p: Prompts): string {
|
|
190
|
+
return `${p.brief}\n\n${p.writing}\n\n${outputContract(BRIEF_SCHEMA, false)}`;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
// ---------------------------------------------------------------------------------------------
|
|
194
|
+
// Helpers
|
|
195
|
+
|
|
196
|
+
function reviewDate(p: ProposalInput, now: Date): string {
|
|
197
|
+
if (p.filters.blind == null && !p.filters.before) return now.toISOString().slice(0, 10);
|
|
198
|
+
const last = p.thread.messages[p.thread.messages.length - 1]!.date.slice(0, 10);
|
|
199
|
+
return last;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
function header(p: ProposalInput, now: Date): string {
|
|
203
|
+
const evalMode = p.filters.blind != null || !!p.filters.before;
|
|
204
|
+
return evalMode
|
|
205
|
+
? `Date of this review: ${reviewDate(p, now)} (the date of the last visible message). Nothing after that date exists for you.`
|
|
206
|
+
: `Date of this review: ${reviewDate(p, now)}.`;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
export function slugFor(p: ProposalInput): string {
|
|
210
|
+
const base =
|
|
211
|
+
p.ref.kind === "cf"
|
|
212
|
+
? `cf-${p.ref.id}`
|
|
213
|
+
: p.ref.kind === "local"
|
|
214
|
+
? "self-" + p.ref.name.replace(/\.[A-Za-z0-9]+$/, "").replace(/[^A-Za-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 40)
|
|
215
|
+
: "msg-" +
|
|
216
|
+
p.thread.rootMessageId
|
|
217
|
+
.replace(/@.*$/, "")
|
|
218
|
+
.replace(/[^A-Za-z0-9]+/g, "-")
|
|
219
|
+
.replace(/^-|-$/g, "")
|
|
220
|
+
.slice(0, 40);
|
|
221
|
+
return base + (p.filters.blind != null ? `-blind${p.filters.blind}` : "") + (p.filters.before ? `-before${p.filters.before}` : "");
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
function dedupeBy<T>(xs: T[], key: (x: T) => string): T[] {
|
|
225
|
+
const seen = new Set<string>();
|
|
226
|
+
return xs.filter((x) => (seen.has(key(x)) ? false : (seen.add(key(x)), true)));
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
// ---------------------------------------------------------------------------------------------
|
|
230
|
+
// Assembly (pure code; exported for tests)
|
|
231
|
+
|
|
232
|
+
export interface AssembleArgs {
|
|
233
|
+
synth: SynthOutput;
|
|
234
|
+
objections: FinalObjection[];
|
|
235
|
+
precedents: Precedent[];
|
|
236
|
+
/** Raw text of the visible thread, for quote checks. */
|
|
237
|
+
threadText: string;
|
|
238
|
+
report: SanitizerReport;
|
|
239
|
+
warnings: string[];
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
export interface Assembled {
|
|
243
|
+
verdict: Verdict;
|
|
244
|
+
failing_level: Level | null;
|
|
245
|
+
levels: Roast["levels"];
|
|
246
|
+
objections: FinalObjection[];
|
|
247
|
+
other_surviving: FinalObjection[];
|
|
248
|
+
refuted: FinalObjection[];
|
|
249
|
+
withdrawn: FinalObjection[];
|
|
250
|
+
touch_points: TouchPointResult[];
|
|
251
|
+
prior_attempts: PriorAttempt[];
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
export function resolvePriorAttempts(
|
|
255
|
+
raw: RawPriorAttempt[],
|
|
256
|
+
precedents: Precedent[],
|
|
257
|
+
threadText: string,
|
|
258
|
+
report: SanitizerReport,
|
|
259
|
+
stage: string,
|
|
260
|
+
): PriorAttempt[] {
|
|
261
|
+
const known = new Map(precedents.map((p) => [p.id, p]));
|
|
262
|
+
const out: PriorAttempt[] = [];
|
|
263
|
+
for (const r of raw) {
|
|
264
|
+
const [id] = filterPrecedentIds([r.precedent_id], known, stage, report);
|
|
265
|
+
if (!id || out.some((o) => o.precedent_id === id)) continue;
|
|
266
|
+
const p = known.get(id)!;
|
|
267
|
+
// An AI summary is never a quote source: only corpus text, the verified key sentence, the thread.
|
|
268
|
+
const sources = p.enriched ? [p.key_quote ?? "", threadText] : [p.objection, p.summary, p.title, p.key_quote ?? "", threadText];
|
|
269
|
+
let quote = verifyQuote(r.quote, sources);
|
|
270
|
+
if (r.quote && !quote) report.quotes_dropped.push({ precedent_id: id, quote: r.quote });
|
|
271
|
+
let quote_url: string | null = quote && p.key_quote && verifyQuote(quote, [p.key_quote]) ? p.quote_message_url : null;
|
|
272
|
+
if (!quote && p.key_quote) {
|
|
273
|
+
// Fall back to the sentence that code verified against the precedent's thread.
|
|
274
|
+
quote = p.key_quote;
|
|
275
|
+
quote_url = p.quote_message_url;
|
|
276
|
+
}
|
|
277
|
+
out.push({
|
|
278
|
+
precedent_id: id,
|
|
279
|
+
source_id: p.source_id,
|
|
280
|
+
title: p.title,
|
|
281
|
+
kind: p.kind,
|
|
282
|
+
status: p.status,
|
|
283
|
+
url: p.url,
|
|
284
|
+
thread_urls: p.thread_urls,
|
|
285
|
+
date: p.date,
|
|
286
|
+
first_date: p.first_date,
|
|
287
|
+
who_objected: p.objectors,
|
|
288
|
+
quote,
|
|
289
|
+
quote_url,
|
|
290
|
+
enriched: p.enriched,
|
|
291
|
+
reason: p.objection,
|
|
292
|
+
relation: r.relation,
|
|
293
|
+
what_is_different: r.what_is_different,
|
|
294
|
+
});
|
|
295
|
+
}
|
|
296
|
+
return out;
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
export function assemble(a: AssembleArgs): Assembled {
|
|
300
|
+
const { synth, warnings } = a;
|
|
301
|
+
const survived = a.objections.filter((o) => o.status === "survived");
|
|
302
|
+
const refuted = a.objections.filter((o) => o.status === "refuted");
|
|
303
|
+
const withdrawn = a.objections.filter((o) => o.status === "withdrawn");
|
|
304
|
+
const byId = new Map(survived.map((o) => [o.id, o]));
|
|
305
|
+
|
|
306
|
+
// Ranked list: surviving objections only, no duplicates, at most 5.
|
|
307
|
+
let ranked: FinalObjection[] = [];
|
|
308
|
+
for (const id of synth.ranked_objection_ids) {
|
|
309
|
+
const o = byId.get(id.trim());
|
|
310
|
+
if (!o) {
|
|
311
|
+
warnings.push(`synthesis ranked "${id}", which is not a surviving objection; dropped`);
|
|
312
|
+
continue;
|
|
313
|
+
}
|
|
314
|
+
if (!ranked.includes(o)) ranked.push(o);
|
|
315
|
+
}
|
|
316
|
+
ranked = ranked.slice(0, 5);
|
|
317
|
+
|
|
318
|
+
// Base-up rule (the rubric's operating rule): the first failing level leads, and objections
|
|
319
|
+
// from higher levels are not listed; those levels are reported as "not evaluated".
|
|
320
|
+
let failing: Level | null = synth.failing_level;
|
|
321
|
+
if (ranked.length > 0) {
|
|
322
|
+
const lowest = LEVELS[Math.min(...ranked.map((o) => levelRank(o.level)))]!;
|
|
323
|
+
if (failing == null) {
|
|
324
|
+
// The model listed objections but named no failing level: fine when they are all fixable.
|
|
325
|
+
if (ranked.some((o) => o.severity === "fatal")) {
|
|
326
|
+
failing = LEVELS[Math.min(...ranked.filter((o) => o.severity === "fatal").map((o) => levelRank(o.level)))]!;
|
|
327
|
+
warnings.push(`synthesis gave no failing level but ranked a fatal objection; set to ${failing}`);
|
|
328
|
+
}
|
|
329
|
+
} else if (levelRank(lowest) < levelRank(failing) && ranked.some((o) => o.level === lowest && o.severity === "fatal")) {
|
|
330
|
+
warnings.push(`synthesis named level ${failing} as failing but ranked a fatal objection at level ${lowest}; set to ${lowest}`);
|
|
331
|
+
failing = lowest;
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
if (failing != null) {
|
|
335
|
+
const f = levelRank(failing);
|
|
336
|
+
const kept: FinalObjection[] = [];
|
|
337
|
+
for (const o of ranked) {
|
|
338
|
+
if (levelRank(o.level) > f) {
|
|
339
|
+
o.not_ranked_reason = `level ${o.level} is above the failing level ${failing}: not evaluated (base-up stop rule)`;
|
|
340
|
+
} else kept.push(o);
|
|
341
|
+
}
|
|
342
|
+
// Stable: the failing level leads, then anything the model ranked from below it.
|
|
343
|
+
ranked = [...kept.filter((o) => levelRank(o.level) === f), ...kept.filter((o) => levelRank(o.level) < f)];
|
|
344
|
+
}
|
|
345
|
+
const rankedSet = new Set(ranked);
|
|
346
|
+
const other = survived.filter((o) => !rankedSet.has(o));
|
|
347
|
+
for (const o of other) {
|
|
348
|
+
o.not_ranked_reason ??=
|
|
349
|
+
failing != null && levelRank(o.level) > levelRank(failing)
|
|
350
|
+
? `level ${o.level} is above the failing level ${failing}: not evaluated (base-up stop rule)`
|
|
351
|
+
: "not among the strongest objections";
|
|
352
|
+
}
|
|
353
|
+
const levels = LEVELS.map((level) => ({
|
|
354
|
+
level,
|
|
355
|
+
name: LEVEL_NAMES[level],
|
|
356
|
+
status:
|
|
357
|
+
failing == null || levelRank(level) < levelRank(failing)
|
|
358
|
+
? ("passed" as const)
|
|
359
|
+
: level === failing
|
|
360
|
+
? ("failed" as const)
|
|
361
|
+
: ("not_evaluated" as const),
|
|
362
|
+
}));
|
|
363
|
+
for (const o of ranked) o.not_ranked_reason = null;
|
|
364
|
+
|
|
365
|
+
// Prior attempts.
|
|
366
|
+
const prior = resolvePriorAttempts(synth.prior_attempts, a.precedents, a.threadText, a.report, "synth");
|
|
367
|
+
|
|
368
|
+
// Verdict guards.
|
|
369
|
+
let verdict = synth.verdict;
|
|
370
|
+
const fatalRanked = ranked.filter((o) => o.severity === "fatal");
|
|
371
|
+
if (verdict === "LIKELY-REJECT" && fatalRanked.length === 0 && !prior.some((p) => p.relation === "same_idea")) {
|
|
372
|
+
warnings.push("synthesis said LIKELY-REJECT with no surviving fatal objection and no same-idea prior attempt; downgraded to NEEDS-DESIGN-DISCUSSION");
|
|
373
|
+
verdict = "NEEDS-DESIGN-DISCUSSION";
|
|
374
|
+
}
|
|
375
|
+
if (verdict === "NO-FUNDAMENTAL-OBJECTION" && fatalRanked.length > 0) {
|
|
376
|
+
warnings.push("synthesis said NO-FUNDAMENTAL-OBJECTION but ranked a fatal objection; changed to NEEDS-DESIGN-DISCUSSION");
|
|
377
|
+
verdict = "NEEDS-DESIGN-DISCUSSION";
|
|
378
|
+
}
|
|
379
|
+
// A listed objection at level 1 means desirability or the problem statement is in question,
|
|
380
|
+
// which by the verdict definitions (prompts/synth.md) is at least a design discussion.
|
|
381
|
+
const level1 = ranked.find((o) => o.level === "1a" || o.level === "1b");
|
|
382
|
+
if (level1 && (verdict === "NEEDS-COMMITTER-EYES" || verdict === "NO-FUNDAMENTAL-OBJECTION")) {
|
|
383
|
+
warnings.push(`synthesis said ${verdict} but listed a level ${level1.level} objection (${level1.id}); changed to NEEDS-DESIGN-DISCUSSION`);
|
|
384
|
+
verdict = "NEEDS-DESIGN-DISCUSSION";
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
// Touch points: all 15, once each, in the issue's order.
|
|
388
|
+
const given = new Map(synth.touch_points.map((t) => [t.id, t]));
|
|
389
|
+
const touch: TouchPointResult[] = TOUCH_POINTS.map((t) => {
|
|
390
|
+
const g = given.get(t.id);
|
|
391
|
+
if (!g) warnings.push(`synthesis did not report touch point "${t.id}"; marked as not hit`);
|
|
392
|
+
let hit = g?.hit ?? false;
|
|
393
|
+
let reason = hit ? (g?.reason ?? "").trim() : "";
|
|
394
|
+
// A ranked objection about a touch point means the touch point is hit.
|
|
395
|
+
const src = ranked.find((o) => o.category === t.id || o.touch_points.includes(t.id));
|
|
396
|
+
if (src && !hit) {
|
|
397
|
+
hit = true;
|
|
398
|
+
reason = `see objection ${src.id}: ${src.title}`;
|
|
399
|
+
}
|
|
400
|
+
if (hit && !reason) reason = src ? `see objection ${src.id}: ${src.title}` : "(no reason given)";
|
|
401
|
+
return { id: t.id, label: t.label, hit, reason };
|
|
402
|
+
});
|
|
403
|
+
|
|
404
|
+
return { verdict, failing_level: failing, levels, objections: ranked, other_surviving: other, refuted, withdrawn, touch_points: touch, prior_attempts: prior };
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
// ---------------------------------------------------------------------------------------------
|
|
408
|
+
// The run
|
|
409
|
+
|
|
410
|
+
export async function runRoast(proposal: ProposalInput, opts: RunOptions): Promise<Roast> {
|
|
411
|
+
const t0 = Date.now();
|
|
412
|
+
const log = opts.log ?? ((m: string) => console.error(m));
|
|
413
|
+
const now = (opts.now ?? (() => new Date()))();
|
|
414
|
+
const prompts = loadPrompts(opts.promptsDir);
|
|
415
|
+
const { backend, models } = opts;
|
|
416
|
+
const angles = opts.angles ?? ANGLES;
|
|
417
|
+
const warnings: string[] = [...proposal.warnings];
|
|
418
|
+
const report = emptyReport();
|
|
419
|
+
const evalMode = proposal.filters.blind != null || !!proposal.filters.before;
|
|
420
|
+
|
|
421
|
+
const material = renderMaterial(proposal);
|
|
422
|
+
if (material.omitted) warnings.push(`${material.omitted} long messages in the middle of the thread were left out of the prompts`);
|
|
423
|
+
const threadText = materialRawText(proposal);
|
|
424
|
+
const head = header(proposal, now);
|
|
425
|
+
|
|
426
|
+
// URLs the output may contain, before the precedents are known.
|
|
427
|
+
const allowed = new AllowedUrls();
|
|
428
|
+
allowed.add(proposal.thread.threadUrl);
|
|
429
|
+
for (const m of proposal.thread.messages) allowed.add(m.url);
|
|
430
|
+
if (proposal.cf) allowed.add(proposal.cf.url);
|
|
431
|
+
allowed.addFromText(threadText);
|
|
432
|
+
allowed.addFromText(prompts.rubric);
|
|
433
|
+
|
|
434
|
+
// Subsystem invariants: files whose path prefixes match what the patch touches.
|
|
435
|
+
const touched = [...new Set(proposal.patchProse.flatMap((pp) => pp.files))];
|
|
436
|
+
const invariantFiles = selectInvariants(touched, opts.invariants ?? loadInvariantFiles());
|
|
437
|
+
const invariantsText = renderInvariants(invariantFiles);
|
|
438
|
+
for (const f of invariantFiles) for (const u of f.sourceUrls) allowed.add(u);
|
|
439
|
+
if (invariantFiles.length) log(`invariants: ${invariantFiles.map((f) => f.file).join(", ")}`);
|
|
440
|
+
const selfMode = opts.mode === "self";
|
|
441
|
+
const maxRounds = Math.max(0, Math.floor(opts.rounds ?? 0));
|
|
442
|
+
|
|
443
|
+
// a. Brief
|
|
444
|
+
log("brief ...");
|
|
445
|
+
let brief = await backend.complete<Brief>({
|
|
446
|
+
label: "brief",
|
|
447
|
+
model: models.brief,
|
|
448
|
+
system: briefSystem(prompts),
|
|
449
|
+
prompt: `${head}\n\n${material.text}\n\n# Task\n\nWrite the brief and the search queries. JSON only.`,
|
|
450
|
+
schema: BRIEF_SCHEMA,
|
|
451
|
+
});
|
|
452
|
+
brief = scrubUrls(brief, allowed, "brief", report);
|
|
453
|
+
if (backend.dryRun) {
|
|
454
|
+
// The stub has no usable queries; search by the subject so the later prompts are realistic.
|
|
455
|
+
const subject = proposal.thread.subject.replace(/^\s*(\[[^\]]*\]|re:)\s*/gi, "").trim();
|
|
456
|
+
brief = { ...brief, title: subject, search_queries: dedupeBy([subject, proposal.cf?.title ?? subject], (s) => s) };
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
// b. Retrieval (code)
|
|
460
|
+
const cutoff = proposal.filters.before ?? proposal.threadStart.slice(0, 10);
|
|
461
|
+
log(`precedent search (before ${cutoff}) ...`);
|
|
462
|
+
const retrieved = await retrievePrecedents({
|
|
463
|
+
queries: brief.search_queries,
|
|
464
|
+
hint: opts.hint ?? null,
|
|
465
|
+
before: cutoff,
|
|
466
|
+
includeUndated: !evalMode,
|
|
467
|
+
self: {
|
|
468
|
+
cfId: proposal.cf?.id ?? null,
|
|
469
|
+
messageIds: [proposal.thread.rootMessageId, ...(proposal.cf?.threads.map((t) => t.messageId) ?? [])].filter(Boolean),
|
|
470
|
+
},
|
|
471
|
+
search: opts.search,
|
|
472
|
+
limit: opts.enrich === false ? (opts.precedentLimit ?? 15) : 24,
|
|
473
|
+
log,
|
|
474
|
+
});
|
|
475
|
+
warnings.push(...retrieved.warnings);
|
|
476
|
+
const pool = retrieved.precedents;
|
|
477
|
+
|
|
478
|
+
// b2. Lazy enrichment: read the threads of the top candidates that have no recorded reason.
|
|
479
|
+
let enrichment: Roast["enrichment"] = null;
|
|
480
|
+
if (pool.length > 0 && opts.enrich !== false && !backend.dryRun) {
|
|
481
|
+
log("enriching precedents ...");
|
|
482
|
+
const store = opts.enrichStore === undefined ? await openPgEnrichmentStore() : opts.enrichStore;
|
|
483
|
+
if (!store) warnings.push("the enrichment table could not be opened: precedent summaries are not cached in this run");
|
|
484
|
+
const res = await enrichPrecedents(pool, {
|
|
485
|
+
backend,
|
|
486
|
+
model: models.roaster,
|
|
487
|
+
system: `${prompts.enrich}\n\n${prompts.writing}\n\n${outputContract(ENRICH_SCHEMA, false)}`,
|
|
488
|
+
store,
|
|
489
|
+
max: opts.enrichMax ?? 8,
|
|
490
|
+
evalMode,
|
|
491
|
+
selfMessageIds: [proposal.thread.rootMessageId, ...(proposal.cf?.threads.map((t) => t.messageId) ?? [])].filter(Boolean),
|
|
492
|
+
loadThreadFn: opts.enrichLoadThread,
|
|
493
|
+
closersFn: opts.enrichClosers,
|
|
494
|
+
log,
|
|
495
|
+
});
|
|
496
|
+
if (opts.enrichStore === undefined) await store?.close?.();
|
|
497
|
+
warnings.push(...res.warnings);
|
|
498
|
+
enrichment = { ...res.stats, relevance_pass: "skipped", relevant: pool.length };
|
|
499
|
+
log(`enrichment: ${res.stats.candidates} candidates, ${res.stats.from_cache} from cache, ${res.stats.new_calls} new, ${res.stats.failed} failed`);
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
// b3. Relevance pass: one call, ids only. The roasters see only what it keeps.
|
|
503
|
+
let precedents = pool.slice(0, opts.precedentLimit ?? 15);
|
|
504
|
+
if (pool.length > 0 && opts.enrich !== false) {
|
|
505
|
+
try {
|
|
506
|
+
const rel = await backend.complete<{ relevant_ids: string[] }>({
|
|
507
|
+
label: "relevance",
|
|
508
|
+
model: models.roaster,
|
|
509
|
+
system: `${prompts.relevance}\n\n${outputContract(RELEVANCE_SCHEMA, false)}`,
|
|
510
|
+
prompt: `${head}\n\n# Brief\n\n${JSON.stringify(brief, null, 1)}\n\n# Candidates\n\n${renderPrecedentsForPrompt(pool)}\n\n# Task\n\nReturn the ids of the relevant candidates. JSON only.`,
|
|
511
|
+
schema: RELEVANCE_SCHEMA,
|
|
512
|
+
});
|
|
513
|
+
if (!backend.dryRun) {
|
|
514
|
+
const ids = filterPrecedentIds(rel.relevant_ids, new Map(pool.map((p) => [p.id, p])), "relevance", report);
|
|
515
|
+
for (const p of pool) p.relevant = ids.includes(p.id);
|
|
516
|
+
precedents = ids.map((id) => pool.find((p) => p.id === id)!).slice(0, opts.precedentLimit ?? 15);
|
|
517
|
+
enrichment = { ...(enrichment ?? { candidates: 0, from_cache: 0, new_calls: 0, failed: 0, quotes_verified: 0, quotes_dropped: 0, objectors_dropped: 0 }), relevance_pass: "done", relevant: precedents.length };
|
|
518
|
+
log(`relevance: ${precedents.length} of ${pool.length} candidates kept`);
|
|
519
|
+
}
|
|
520
|
+
} catch (e) {
|
|
521
|
+
warnings.push(`the relevance pass failed (${(e as Error).message.split("\n")[0]}); the review used the top candidates unfiltered`);
|
|
522
|
+
if (enrichment) enrichment.relevance_pass = "failed";
|
|
523
|
+
}
|
|
524
|
+
}
|
|
525
|
+
const knownPrecedents = new Map(precedents.map((p) => [p.id, p]));
|
|
526
|
+
for (const p of precedents) {
|
|
527
|
+
allowed.add(p.url);
|
|
528
|
+
for (const u of p.thread_urls) allowed.add(u);
|
|
529
|
+
allowed.add(p.quote_message_url);
|
|
530
|
+
}
|
|
531
|
+
const precedentText =
|
|
532
|
+
retrieved.backend === "none"
|
|
533
|
+
? "The precedent index is not available in this run: no search was done. Do not cite any earlier thread, commit or decision as a precedent, and do not claim that none exists."
|
|
534
|
+
: precedents.length === 0 && pool.length > 0
|
|
535
|
+
? `The corpus search returned ${pool.length} candidates and none of them is about the same or a similar idea. No precedent found. Do not cite any earlier thread, commit or decision as a precedent.`
|
|
536
|
+
: renderPrecedentsForPrompt(precedents);
|
|
537
|
+
log(`precedents: ${precedents.length} shown to the roasters, ${pool.length} retrieved (backend: ${retrieved.backend})`);
|
|
538
|
+
|
|
539
|
+
const shared = [
|
|
540
|
+
head,
|
|
541
|
+
`# Brief\n\n${JSON.stringify(brief, null, 1)}`,
|
|
542
|
+
`# Material\n\n${material.text}`,
|
|
543
|
+
`# Precedents (retrieved by code; keyword search, so check relevance)\n\n${precedentText}`,
|
|
544
|
+
].join("\n\n");
|
|
545
|
+
|
|
546
|
+
// c. Roasters, independent, in parallel (the backend limits concurrency)
|
|
547
|
+
log(`roasters: ${angles.join(", ")} ...`);
|
|
548
|
+
const visible = new Set(proposal.thread.messages.map((m) => m.index));
|
|
549
|
+
const msgByIndex = new Map(proposal.thread.messages.map((m) => [m.index, m]));
|
|
550
|
+
const roasterNotes: Roast["roaster_notes"] = [];
|
|
551
|
+
const objections: FinalObjection[] = [];
|
|
552
|
+
const rounds: RoundRecord[] = [];
|
|
553
|
+
let roasterPrior: RawPriorAttempt[] = [];
|
|
554
|
+
|
|
555
|
+
/** Sanitise one raw objection and resolve its references. Links come from code. */
|
|
556
|
+
const resolve = (o: RawObjection, stage: string) => {
|
|
557
|
+
const precedent_ids = filterPrecedentIds(o.precedent_ids, knownPrecedents, stage, report);
|
|
558
|
+
const refs = filterThreadRefs(o.thread_message_refs, visible, stage, report);
|
|
559
|
+
return {
|
|
560
|
+
...o,
|
|
561
|
+
precedent_ids,
|
|
562
|
+
thread_message_refs: refs,
|
|
563
|
+
touch_points: [...new Set(o.touch_points)],
|
|
564
|
+
precedents: precedent_ids.map((id) => {
|
|
565
|
+
const p = knownPrecedents.get(id)!;
|
|
566
|
+
return { id, title: p.title, url: p.url, thread_urls: p.thread_urls, date: p.date, status: p.status };
|
|
567
|
+
}),
|
|
568
|
+
thread_refs: refs.map((n) => {
|
|
569
|
+
const m = msgByIndex.get(n)!;
|
|
570
|
+
return { index: n, from: m.from.replace(/\s*<[^>]*>\s*$/, ""), date: m.date, url: m.url };
|
|
571
|
+
}),
|
|
572
|
+
};
|
|
573
|
+
};
|
|
574
|
+
const addObjections = (angle: Angle, raws: RawObjection[], round: number, stage: string): FinalObjection[] => {
|
|
575
|
+
const added: FinalObjection[] = [];
|
|
576
|
+
for (const o of raws) {
|
|
577
|
+
const n = objections.filter((x) => x.angle === angle && x.round === round).length + 1;
|
|
578
|
+
const fo: FinalObjection = {
|
|
579
|
+
...resolve(o, stage),
|
|
580
|
+
id: round === 0 ? `${angle}-${n}` : `${angle}-r${round}-${n}`,
|
|
581
|
+
angle,
|
|
582
|
+
round,
|
|
583
|
+
revised_in_round: null,
|
|
584
|
+
status: "survived",
|
|
585
|
+
withdrawal: null,
|
|
586
|
+
critic: { refutation_kind: "none", reasoning: "", original_severity: null },
|
|
587
|
+
not_ranked_reason: null,
|
|
588
|
+
};
|
|
589
|
+
objections.push(fo);
|
|
590
|
+
added.push(fo);
|
|
591
|
+
}
|
|
592
|
+
return added;
|
|
593
|
+
};
|
|
594
|
+
|
|
595
|
+
const forPrompt = (o: FinalObjection) => ({
|
|
596
|
+
id: o.id,
|
|
597
|
+
angle: o.angle,
|
|
598
|
+
title: o.title,
|
|
599
|
+
level: o.level,
|
|
600
|
+
rubric_item: o.rubric_item,
|
|
601
|
+
category: o.category,
|
|
602
|
+
touch_points: o.touch_points,
|
|
603
|
+
severity: o.severity,
|
|
604
|
+
failure_scenario: o.failure_scenario,
|
|
605
|
+
argument: o.argument,
|
|
606
|
+
precedent_ids: o.precedent_ids,
|
|
607
|
+
thread_message_refs: o.thread_message_refs,
|
|
608
|
+
author_must_show: o.author_must_show,
|
|
609
|
+
});
|
|
610
|
+
const withVerdict = (o: FinalObjection) => ({
|
|
611
|
+
...forPrompt(o),
|
|
612
|
+
round: o.round,
|
|
613
|
+
status: o.status,
|
|
614
|
+
critic_reasoning: o.critic.reasoning,
|
|
615
|
+
severity_before_critic: o.critic.original_severity,
|
|
616
|
+
...(o.withdrawal ? { withdrawn_because: o.withdrawal.reason } : {}),
|
|
617
|
+
});
|
|
618
|
+
|
|
619
|
+
/** d. Critic: tries to refute the given objections; one verdict each. */
|
|
620
|
+
const runCritic = async (targets: FinalObjection[], label: string): Promise<void> => {
|
|
621
|
+
if (targets.length === 0 && !backend.dryRun) return;
|
|
622
|
+
log(`${label} ...`);
|
|
623
|
+
const earlier = objections.filter((o) => !targets.includes(o) && o.status !== "withdrawn");
|
|
624
|
+
const critic = scrubUrls(
|
|
625
|
+
await backend.complete<CriticOutput>({
|
|
626
|
+
label,
|
|
627
|
+
model: models.critic,
|
|
628
|
+
system: criticSystem(prompts, invariantsText),
|
|
629
|
+
prompt: [
|
|
630
|
+
shared,
|
|
631
|
+
...(earlier.length && targets.length ? [`# Objections already checked in earlier rounds (for duplicate detection only; give no verdict on these)\n\n${JSON.stringify(earlier.map((o) => ({ id: o.id, title: o.title, status: o.status })), null, 1)}`] : []),
|
|
632
|
+
`# Objections to refute\n\n${JSON.stringify(targets.map(forPrompt), null, 1)}`,
|
|
633
|
+
`# Task\n\nTry to refute each objection. One verdict per id: ${targets.map((o) => o.id).join(", ") || "(none)"}. JSON only.`,
|
|
634
|
+
].join("\n\n"),
|
|
635
|
+
schema: CRITIC_SCHEMA,
|
|
636
|
+
}),
|
|
637
|
+
allowed,
|
|
638
|
+
label,
|
|
639
|
+
report,
|
|
640
|
+
);
|
|
641
|
+
const verdicts = new Map(critic.verdicts.map((v) => [v.objection_id.trim(), v]));
|
|
642
|
+
for (const o of targets) {
|
|
643
|
+
const v = verdicts.get(o.id);
|
|
644
|
+
if (!v) {
|
|
645
|
+
warnings.push(`the critic gave no verdict on ${o.id}; kept as survived, unchecked`);
|
|
646
|
+
o.critic.reasoning = "(the critic gave no verdict; this objection was not checked)";
|
|
647
|
+
continue;
|
|
648
|
+
}
|
|
649
|
+
o.status = v.verdict;
|
|
650
|
+
o.critic.refutation_kind = v.verdict === "refuted" ? v.refutation_kind : "none";
|
|
651
|
+
o.critic.reasoning = v.reasoning;
|
|
652
|
+
if (v.verdict === "survived" && v.adjusted_severity && v.adjusted_severity !== o.severity) {
|
|
653
|
+
o.critic.original_severity = o.severity;
|
|
654
|
+
o.severity = v.adjusted_severity;
|
|
655
|
+
}
|
|
656
|
+
}
|
|
657
|
+
const ok = targets.filter((o) => o.status === "survived").length;
|
|
658
|
+
log(`${label}: ${ok} survived, ${targets.length - ok} refuted`);
|
|
659
|
+
};
|
|
660
|
+
|
|
661
|
+
// Round 0: the first pass.
|
|
662
|
+
let callsBefore = backend.stats().calls;
|
|
663
|
+
const outputs = await Promise.all(
|
|
664
|
+
angles.map(async (angle) => {
|
|
665
|
+
try {
|
|
666
|
+
const out = await backend.complete<RoasterOutput>({
|
|
667
|
+
label: `roaster-${angle}`,
|
|
668
|
+
model: models.roaster,
|
|
669
|
+
system: roasterSystem(prompts, angle, invariantsText),
|
|
670
|
+
prompt: `${shared}\n\n# Task\n\nRoast this proposal from your angle (${angle}). Zero objections is a valid answer. JSON only.`,
|
|
671
|
+
schema: ROASTER_SCHEMA,
|
|
672
|
+
});
|
|
673
|
+
return { angle, out, error: null as string | null };
|
|
674
|
+
} catch (e) {
|
|
675
|
+
return { angle, out: null, error: (e as Error).message };
|
|
676
|
+
}
|
|
677
|
+
}),
|
|
678
|
+
);
|
|
679
|
+
if (outputs.every((o) => o.error)) throw new Error(`every roaster failed; first error: ${outputs[0]!.error}`);
|
|
680
|
+
|
|
681
|
+
const ran: Angle[] = [];
|
|
682
|
+
for (const { angle, out, error } of outputs) {
|
|
683
|
+
if (!out) {
|
|
684
|
+
warnings.push(`review angle "${angle}" failed and is missing from this review: ${error}`);
|
|
685
|
+
roasterNotes.push({ angle, objections: 0, note: `FAILED: ${error}` });
|
|
686
|
+
continue;
|
|
687
|
+
}
|
|
688
|
+
ran.push(angle);
|
|
689
|
+
const stage = `roaster-${angle}`;
|
|
690
|
+
const clean = scrubUrls(out, allowed, stage, report);
|
|
691
|
+
addObjections(angle, clean.objections, 0, stage);
|
|
692
|
+
if (angle === "prior-attempts") roasterPrior = clean.prior_attempts;
|
|
693
|
+
else if (clean.prior_attempts.length) warnings.push(`review angle "${angle}" returned prior attempts; ignored (only the prior-attempts angle may)`);
|
|
694
|
+
roasterNotes.push({ angle, objections: clean.objections.length, note: clean.no_objection_note });
|
|
695
|
+
}
|
|
696
|
+
log(`objections raised: ${objections.length}`);
|
|
697
|
+
await runCritic([...objections], "critic");
|
|
698
|
+
rounds.push({
|
|
699
|
+
round: 0,
|
|
700
|
+
kind: "initial",
|
|
701
|
+
llm_calls: backend.stats().calls - callsBefore,
|
|
702
|
+
new_objections: objections.length,
|
|
703
|
+
new_survived: objections.filter((o) => o.status === "survived").length,
|
|
704
|
+
revised: 0,
|
|
705
|
+
withdrawn: 0,
|
|
706
|
+
follow_ups: [],
|
|
707
|
+
note: "",
|
|
708
|
+
});
|
|
709
|
+
|
|
710
|
+
// Extra rounds (--rounds N). Same evidence rules, same no-tools guarantee; a round may add nothing.
|
|
711
|
+
const answersOf = (angle: Angle) => ({
|
|
712
|
+
objections: objections.filter((o) => o.angle === angle).map(withVerdict),
|
|
713
|
+
note: roasterNotes.find((n) => n.angle === angle)?.note ?? "",
|
|
714
|
+
});
|
|
715
|
+
for (let round = 1; round <= maxRounds; round++) {
|
|
716
|
+
callsBefore = backend.stats().calls;
|
|
717
|
+
const fresh: FinalObjection[] = [];
|
|
718
|
+
const rec: RoundRecord = { round, kind: round === 1 ? "what_did_you_forget" : round === 2 ? "are_you_happy" : "supervisor", llm_calls: 0, new_objections: 0, new_survived: 0, revised: 0, withdrawn: 0, follow_ups: [], note: "" };
|
|
719
|
+
|
|
720
|
+
if (round <= 2) {
|
|
721
|
+
// Reflection: every roaster that ran gets one short question about its own answer.
|
|
722
|
+
const question = round === 1 ? FORGOT_PROMPT : HAPPY_PROMPT;
|
|
723
|
+
log(`round ${round}: "${question}" ...`);
|
|
724
|
+
const replies = await Promise.all(
|
|
725
|
+
ran.map(async (angle) => {
|
|
726
|
+
try {
|
|
727
|
+
const out = await backend.complete<ReflectOutput>({
|
|
728
|
+
label: `round${round}-${angle}`,
|
|
729
|
+
model: models.roaster,
|
|
730
|
+
system: reflectSystem(prompts, angle, invariantsText),
|
|
731
|
+
prompt: `${shared}\n\n# Your answers so far (angle: ${angle})${round >= 2 ? ", with the critic's verdicts" : ""}\n\n${JSON.stringify(answersOf(angle), null, 1)}\n\n# Question\n\n${question}`,
|
|
732
|
+
schema: REFLECT_SCHEMA,
|
|
733
|
+
});
|
|
734
|
+
return { angle, out };
|
|
735
|
+
} catch (e) {
|
|
736
|
+
warnings.push(`round ${round}: review angle "${angle}" failed: ${(e as Error).message.split("\n")[0]}`);
|
|
737
|
+
return { angle, out: null };
|
|
738
|
+
}
|
|
739
|
+
}),
|
|
740
|
+
);
|
|
741
|
+
for (const { angle, out } of replies) {
|
|
742
|
+
if (!out) continue;
|
|
743
|
+
const stage = `round${round}-${angle}`;
|
|
744
|
+
const clean = scrubUrls(out, allowed, stage, report);
|
|
745
|
+
fresh.push(...addObjections(angle, clean.objections, round, stage));
|
|
746
|
+
if (round >= 2) {
|
|
747
|
+
const own = (id: string) => objections.find((o) => o.id === id.trim() && o.angle === angle && o.status !== "withdrawn");
|
|
748
|
+
for (const w of clean.withdrawn) {
|
|
749
|
+
const o = own(w.objection_id);
|
|
750
|
+
if (!o) {
|
|
751
|
+
warnings.push(`round ${round}: "${angle}" tried to withdraw "${w.objection_id}", which is not an open objection of its own; ignored`);
|
|
752
|
+
continue;
|
|
753
|
+
}
|
|
754
|
+
// Recorded with its reason, never deleted.
|
|
755
|
+
o.status = "withdrawn";
|
|
756
|
+
o.withdrawal = { round, reason: w.reason };
|
|
757
|
+
rec.withdrawn++;
|
|
758
|
+
}
|
|
759
|
+
for (const rv of clean.revised) {
|
|
760
|
+
const o = own(rv.objection_id);
|
|
761
|
+
if (!o) {
|
|
762
|
+
warnings.push(`round ${round}: "${angle}" tried to revise "${rv.objection_id}", which is not an open objection of its own; ignored`);
|
|
763
|
+
continue;
|
|
764
|
+
}
|
|
765
|
+
Object.assign(o, resolve(rv.objection, stage), { revised_in_round: round, status: "survived", critic: { refutation_kind: "none", reasoning: "", original_severity: null } });
|
|
766
|
+
if (!fresh.includes(o)) fresh.push(o);
|
|
767
|
+
rec.revised++;
|
|
768
|
+
}
|
|
769
|
+
} else if (clean.withdrawn.length || clean.revised.length) {
|
|
770
|
+
warnings.push(`round 1: "${angle}" returned revisions or withdrawals; ignored (round 1 only adds)`);
|
|
771
|
+
}
|
|
772
|
+
if (clean.note.trim()) rec.note += `${angle}: ${clean.note.trim()} `;
|
|
773
|
+
}
|
|
774
|
+
} else {
|
|
775
|
+
// Supervisor: names what nobody examined; an empty list ends the loop.
|
|
776
|
+
log(`round ${round}: supervisor ...`);
|
|
777
|
+
let sup: SupervisorOutput;
|
|
778
|
+
try {
|
|
779
|
+
sup = scrubUrls(
|
|
780
|
+
await backend.complete<SupervisorOutput>({
|
|
781
|
+
label: `round${round}-supervisor`,
|
|
782
|
+
model: models.critic,
|
|
783
|
+
system: supervisorSystem(prompts),
|
|
784
|
+
prompt: [
|
|
785
|
+
head,
|
|
786
|
+
`# Brief\n\n${JSON.stringify(brief, null, 1)}`,
|
|
787
|
+
`# Files touched by the patch\n\n${touched.length ? touched.map((f) => `- ${f}`).join("\n") : "(no patch read)"}`,
|
|
788
|
+
`# Objections so far\n\n${JSON.stringify(objections.map(withVerdict), null, 1)}`,
|
|
789
|
+
`# Roaster notes\n\n${roasterNotes.map((n) => `- ${n.angle}: ${n.objections} objection(s) in the first pass. ${n.note}`).join("\n")}`,
|
|
790
|
+
`# Angles you can send back\n\n${ran.join(", ")}`,
|
|
791
|
+
"# Task\n\nName up to 3 uncovered areas, or none. JSON only.",
|
|
792
|
+
].join("\n\n"),
|
|
793
|
+
schema: SUPERVISOR_SCHEMA,
|
|
794
|
+
}),
|
|
795
|
+
allowed,
|
|
796
|
+
`round${round}-supervisor`,
|
|
797
|
+
report,
|
|
798
|
+
);
|
|
799
|
+
} catch (e) {
|
|
800
|
+
warnings.push(`round ${round}: the supervisor failed: ${(e as Error).message.split("\n")[0]}; rounds stopped`);
|
|
801
|
+
break;
|
|
802
|
+
}
|
|
803
|
+
rec.note = sup.coverage_note;
|
|
804
|
+
const follow = sup.follow_ups.filter((f) => ran.includes(f.angle)).slice(0, 3);
|
|
805
|
+
rec.follow_ups = follow.map((f) => ({ angle: f.angle, uncovered_area: f.uncovered_area }));
|
|
806
|
+
const replies = await Promise.all(
|
|
807
|
+
follow.map(async (f, i) => {
|
|
808
|
+
try {
|
|
809
|
+
const out = await backend.complete<RoasterOutput>({
|
|
810
|
+
label: `round${round}-followup${i + 1}-${f.angle}`,
|
|
811
|
+
model: models.roaster,
|
|
812
|
+
system: roasterSystem(prompts, f.angle, invariantsText),
|
|
813
|
+
prompt: `${shared}\n\n# Your answers so far (angle: ${f.angle})\n\n${JSON.stringify(answersOf(f.angle), null, 1)}\n\n# Follow-up from the supervisor\n\nArea not covered: ${f.uncovered_area}\n\n${f.instruction}\n\n# Task\n\nExamine this area only, from your angle (${f.angle}). Return only new objections about it; do not repeat earlier ones. An empty list is a valid answer: say in no_objection_note what you checked. JSON only.`,
|
|
814
|
+
schema: ROASTER_SCHEMA,
|
|
815
|
+
});
|
|
816
|
+
return { f, out };
|
|
817
|
+
} catch (e) {
|
|
818
|
+
warnings.push(`round ${round}: follow-up for "${f.angle}" failed: ${(e as Error).message.split("\n")[0]}`);
|
|
819
|
+
return { f, out: null };
|
|
820
|
+
}
|
|
821
|
+
}),
|
|
822
|
+
);
|
|
823
|
+
for (const { f, out } of replies) {
|
|
824
|
+
if (!out) continue;
|
|
825
|
+
const stage = `round${round}-${f.angle}`;
|
|
826
|
+
fresh.push(...addObjections(f.angle, scrubUrls(out, allowed, stage, report).objections, round, stage));
|
|
827
|
+
}
|
|
828
|
+
if (follow.length === 0) {
|
|
829
|
+
rec.llm_calls = backend.stats().calls - callsBefore;
|
|
830
|
+
rounds.push(rec);
|
|
831
|
+
log(`round ${round}: the supervisor found the coverage adequate; rounds end`);
|
|
832
|
+
break;
|
|
833
|
+
}
|
|
834
|
+
}
|
|
835
|
+
|
|
836
|
+
const added = fresh.filter((o) => o.round === round);
|
|
837
|
+
await runCritic(fresh, `critic-round${round}`);
|
|
838
|
+
rec.new_objections = added.length;
|
|
839
|
+
rec.new_survived = added.filter((o) => o.status === "survived").length;
|
|
840
|
+
rec.llm_calls = backend.stats().calls - callsBefore;
|
|
841
|
+
rounds.push(rec);
|
|
842
|
+
log(`round ${round}: ${rec.new_objections} new (${rec.new_survived} survived), ${rec.revised} revised, ${rec.withdrawn} withdrawn`);
|
|
843
|
+
}
|
|
844
|
+
|
|
845
|
+
// e. Synthesis
|
|
846
|
+
log("synthesis ...");
|
|
847
|
+
const synthMaterial = renderMaterial(proposal, { maxMessages: 3, maxThreadChars: 40_000 });
|
|
848
|
+
const live = objections.filter((o) => o.status !== "withdrawn");
|
|
849
|
+
const synth = scrubUrls(
|
|
850
|
+
await backend.complete<SynthOutput>({
|
|
851
|
+
label: "synth",
|
|
852
|
+
model: models.synth,
|
|
853
|
+
system: synthSystem(prompts, selfMode),
|
|
854
|
+
prompt: [
|
|
855
|
+
head,
|
|
856
|
+
`# Brief\n\n${JSON.stringify(brief, null, 1)}`,
|
|
857
|
+
`# Material (start of the thread and patch prose)\n\n${synthMaterial.text}`,
|
|
858
|
+
`# Precedents (retrieved by code)\n\n${precedentText}`,
|
|
859
|
+
`# Prior attempts proposed by the prior-attempts roaster\n\n${JSON.stringify(roasterPrior, null, 1)}`,
|
|
860
|
+
`# Objections with the critic's verdicts\n\n${JSON.stringify(
|
|
861
|
+
live.map((o) => ({ ...forPrompt(o), critic_verdict: o.status, critic_reasoning: o.critic.reasoning, severity_before_critic: o.critic.original_severity })),
|
|
862
|
+
null,
|
|
863
|
+
1,
|
|
864
|
+
)}`,
|
|
865
|
+
`# Roaster notes\n\n${roasterNotes.map((n) => `- ${n.angle}: ${n.objections} objection(s). ${n.note}`).join("\n")}`,
|
|
866
|
+
`# Task\n\nWrite the synthesis${selfMode ? " for the author (self mode)" : ""}. JSON only.`,
|
|
867
|
+
].join("\n\n"),
|
|
868
|
+
schema: selfMode ? SYNTH_SELF_SCHEMA : SYNTH_SCHEMA,
|
|
869
|
+
}),
|
|
870
|
+
allowed,
|
|
871
|
+
"synth",
|
|
872
|
+
report,
|
|
873
|
+
);
|
|
874
|
+
|
|
875
|
+
const asm = assemble({ synth, objections, precedents, threadText, report, warnings });
|
|
876
|
+
if (retrieved.backend === "none") {
|
|
877
|
+
warnings.push('prior attempts: the corpus was not searched, so an empty list does NOT mean "no precedent found"');
|
|
878
|
+
}
|
|
879
|
+
|
|
880
|
+
// "Not now" is for the case where only demand / accumulation fails; it is not a verdict.
|
|
881
|
+
const notNow = { ...synth.not_now, evidence_that_would_change_it: synth.not_now.applies ? synth.not_now.evidence_that_would_change_it : "" };
|
|
882
|
+
if (notNow.applies && asm.objections.some((o) => o.severity === "fatal")) {
|
|
883
|
+
warnings.push('synthesis set "not now" although a fatal objection is listed; "not now" is meant for the case where only demand or accumulation fails');
|
|
884
|
+
}
|
|
885
|
+
if (notNow.applies && !notNow.evidence_that_would_change_it.trim()) {
|
|
886
|
+
warnings.push('synthesis set "not now" without saying what evidence would change it');
|
|
887
|
+
}
|
|
888
|
+
|
|
889
|
+
// Self mode: advice per surviving objection, in the model's order; unknown ids are dropped.
|
|
890
|
+
const selfAdvice: SelfAdvice[] = [];
|
|
891
|
+
if (selfMode) {
|
|
892
|
+
const surviving = new Set([...asm.objections, ...asm.other_surviving].map((o) => o.id));
|
|
893
|
+
for (const a of synth.self_advice ?? []) {
|
|
894
|
+
const id = a.objection_id.trim();
|
|
895
|
+
if (surviving.has(id) && !selfAdvice.some((x) => x.objection_id === id)) selfAdvice.push({ ...a, objection_id: id });
|
|
896
|
+
else if (!surviving.has(id)) warnings.push(`self advice for "${a.objection_id}", which is not a surviving objection; dropped`);
|
|
897
|
+
}
|
|
898
|
+
}
|
|
899
|
+
|
|
900
|
+
const roast: Roast = {
|
|
901
|
+
schema_version: SCHEMA_VERSION,
|
|
902
|
+
tool: "pgroast/0.1",
|
|
903
|
+
generated_at: now.toISOString(),
|
|
904
|
+
input: {
|
|
905
|
+
arg: proposal.input,
|
|
906
|
+
cf: proposal.cf
|
|
907
|
+
? {
|
|
908
|
+
id: proposal.cf.id,
|
|
909
|
+
url: proposal.cf.url,
|
|
910
|
+
title: proposal.cf.title,
|
|
911
|
+
status: proposal.cf.status,
|
|
912
|
+
authors: proposal.cf.authors,
|
|
913
|
+
redacted: proposal.cf.redacted,
|
|
914
|
+
}
|
|
915
|
+
: null,
|
|
916
|
+
thread_url: proposal.thread.threadUrl,
|
|
917
|
+
subject: proposal.thread.subject,
|
|
918
|
+
thread_start: proposal.threadStart,
|
|
919
|
+
messages_read: proposal.thread.messages.length,
|
|
920
|
+
messages_total: proposal.totalMessages,
|
|
921
|
+
blind: proposal.filters.blind,
|
|
922
|
+
before: proposal.filters.before,
|
|
923
|
+
hint: opts.hint ?? null,
|
|
924
|
+
attachments_read: proposal.patchProse.map((p) => p.attachment.name),
|
|
925
|
+
},
|
|
926
|
+
mode: selfMode ? "self" : "review",
|
|
927
|
+
backend: opts.backendName ?? (backend.dryRun ? "dry-run" : "claude"),
|
|
928
|
+
models,
|
|
929
|
+
verdict: asm.verdict,
|
|
930
|
+
confidence: synth.confidence,
|
|
931
|
+
verdict_reasoning: synth.verdict_reasoning,
|
|
932
|
+
failing_level: asm.failing_level,
|
|
933
|
+
levels: asm.levels,
|
|
934
|
+
brief,
|
|
935
|
+
prior_attempts: asm.prior_attempts,
|
|
936
|
+
objections: asm.objections,
|
|
937
|
+
other_surviving: asm.other_surviving,
|
|
938
|
+
refuted: asm.refuted,
|
|
939
|
+
withdrawn: asm.withdrawn,
|
|
940
|
+
touch_points: asm.touch_points,
|
|
941
|
+
questions: synth.questions.slice(0, 5),
|
|
942
|
+
committability: synth.committability,
|
|
943
|
+
not_now: notNow,
|
|
944
|
+
priority: synth.priority,
|
|
945
|
+
self_advice: selfAdvice,
|
|
946
|
+
provisional_note: synth.provisional_note,
|
|
947
|
+
disclosure: DISCLOSURE,
|
|
948
|
+
precedents_retrieved: pool,
|
|
949
|
+
enrichment,
|
|
950
|
+
precedent_cutoff: cutoff,
|
|
951
|
+
precedent_backend: retrieved.backend,
|
|
952
|
+
roaster_notes: roasterNotes,
|
|
953
|
+
rounds_requested: maxRounds,
|
|
954
|
+
rounds,
|
|
955
|
+
invariants_loaded: invariantFiles.map((f) => f.file),
|
|
956
|
+
warnings,
|
|
957
|
+
tone_lint: { fields_checked: 0, hits_found: 0, sentences_rewritten: 0, repair_call: "not needed", remaining: [] },
|
|
958
|
+
sanitizer: report,
|
|
959
|
+
stats: { llm_calls: 0, wall_ms: 0, cost_usd: null, dry_run: backend.dryRun },
|
|
960
|
+
};
|
|
961
|
+
|
|
962
|
+
// Tone lint over every prose field, before anything is written; one repair call on a hit.
|
|
963
|
+
roast.tone_lint = await toneCheck(roast, {
|
|
964
|
+
backend,
|
|
965
|
+
model: models.synth,
|
|
966
|
+
system: `${prompts.tone}\n\n${prompts.writing}\n\n${outputContract(TONE_REPAIR_SCHEMA, false)}`,
|
|
967
|
+
allowed,
|
|
968
|
+
report,
|
|
969
|
+
log,
|
|
970
|
+
});
|
|
971
|
+
if (roast.tone_lint.hits_found) {
|
|
972
|
+
log(`tone lint: ${roast.tone_lint.hits_found} hit(s), ${roast.tone_lint.sentences_rewritten} sentence(s) rewritten, ${roast.tone_lint.remaining.length} left`);
|
|
973
|
+
}
|
|
974
|
+
if (roast.tone_lint.remaining.length && !backend.dryRun) {
|
|
975
|
+
warnings.push(`tone lint: ${roast.tone_lint.remaining.length} finding(s) remain after the repair call; see tone_lint in roast.json before forwarding this text`);
|
|
976
|
+
}
|
|
977
|
+
|
|
978
|
+
const st = backend.stats();
|
|
979
|
+
roast.stats = { llm_calls: st.calls, wall_ms: Date.now() - t0, cost_usd: st.costUsd, dry_run: backend.dryRun };
|
|
980
|
+
return roast;
|
|
981
|
+
}
|