pgroast 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +173 -2
- package/bin/pgroast.mjs +19 -0
- package/data/precedents.sql.gz +0 -0
- package/docs/corpus.md +352 -0
- package/docs/design.md +540 -0
- package/docs/eval.md +245 -0
- package/docs/haas-paranoia-notes.md +223 -0
- package/invariants/README.md +15 -0
- package/invariants/nbtree.md +14 -0
- package/package.json +51 -4
- package/prompts/brief.md +32 -0
- package/prompts/committer-paranoia.md +149 -0
- package/prompts/critic.md +56 -0
- package/prompts/enrich.md +35 -0
- package/prompts/pushback-map.md +58 -0
- package/prompts/pushback-reduce.md +72 -0
- package/prompts/reflect.md +20 -0
- package/prompts/relevance.md +20 -0
- package/prompts/roaster-already-exists.md +23 -0
- package/prompts/roaster-common.md +88 -0
- package/prompts/roaster-demand.md +28 -0
- package/prompts/roaster-maintenance-fit.md +22 -0
- package/prompts/roaster-prior-attempts.md +29 -0
- package/prompts/roaster-replication-ondisk.md +24 -0
- package/prompts/roaster-security.md +21 -0
- package/prompts/roaster-standards-compat.md +21 -0
- package/prompts/supervisor.md +31 -0
- package/prompts/synth-self.md +19 -0
- package/prompts/synth.md +140 -0
- package/prompts/tone.md +21 -0
- package/prompts/writing.md +98 -0
- package/sql/schema.sql +45 -0
- package/src/cli.ts +391 -0
- package/src/corpus/cf.ts +220 -0
- package/src/corpus/cli.ts +189 -0
- package/src/corpus/db.ts +119 -0
- package/src/corpus/fetch.ts +104 -0
- package/src/corpus/reverts.ts +133 -0
- package/src/corpus/search.ts +201 -0
- package/src/corpus/util.ts +133 -0
- package/src/corpus/wiki.ts +80 -0
- package/src/eval/cases.ts +118 -0
- package/src/eval/cli.ts +105 -0
- package/src/eval/judge.ts +136 -0
- package/src/eval/report.ts +268 -0
- package/src/eval/run.ts +153 -0
- package/src/eval/score.ts +275 -0
- package/src/roast/doctor.ts +109 -0
- package/src/roast/enrich.ts +335 -0
- package/src/roast/invariants.ts +65 -0
- package/src/roast/llm.ts +601 -0
- package/src/roast/material.ts +133 -0
- package/src/roast/page.ts +266 -0
- package/src/roast/pipeline.ts +981 -0
- package/src/roast/precedents.ts +308 -0
- package/src/roast/pushback.ts +664 -0
- package/src/roast/render.ts +386 -0
- package/src/roast/sanitize.ts +140 -0
- package/src/roast/schema.ts +714 -0
- package/src/roast/tone.ts +216 -0
- package/src/thread/fetch.ts +172 -0
- package/src/thread/load.ts +227 -0
- package/src/thread/local.ts +73 -0
- package/src/thread/parse.ts +243 -0
- package/src/thread/patchprose.ts +143 -0
- package/src/thread/types.ts +120 -0
|
@@ -0,0 +1,714 @@
|
|
|
1
|
+
// PG Roast data model: the types of roast.json, the JSON Schemas the model must follow at each
|
|
2
|
+
// stage, and a small validator for them. No dependencies.
|
|
3
|
+
|
|
4
|
+
// ---------------------------------------------------------------------------------------------
|
|
5
|
+
// Constants
|
|
6
|
+
|
|
7
|
+
export const SCHEMA_VERSION = 1;
|
|
8
|
+
|
|
9
|
+
/** Verbatim from the issue. Never produced by the model. */
|
|
10
|
+
export const DISCLOSURE = "Prepared by an AI harness; treat as a devil's advocate, not a verdict.";
|
|
11
|
+
|
|
12
|
+
export const VERDICTS = [
|
|
13
|
+
"LIKELY-REJECT",
|
|
14
|
+
"NEEDS-DESIGN-DISCUSSION",
|
|
15
|
+
"NEEDS-COMMITTER-EYES",
|
|
16
|
+
"NO-FUNDAMENTAL-OBJECTION",
|
|
17
|
+
] as const;
|
|
18
|
+
export type Verdict = (typeof VERDICTS)[number];
|
|
19
|
+
|
|
20
|
+
export const CONFIDENCES = ["low", "medium", "high"] as const;
|
|
21
|
+
export type Confidence = (typeof CONFIDENCES)[number];
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Levels of the review pyramid that PG Roast judges (prompts/committer-paranoia.md), base first.
|
|
25
|
+
* 1a understandability, 1b desirability, 2 architecture and design.
|
|
26
|
+
*/
|
|
27
|
+
export const LEVELS = ["1a", "1b", "2"] as const;
|
|
28
|
+
export type Level = (typeof LEVELS)[number];
|
|
29
|
+
export const LEVEL_NAMES: Record<Level, string> = {
|
|
30
|
+
"1a": "Level 1a: understandability",
|
|
31
|
+
"1b": "Level 1b: desirability",
|
|
32
|
+
"2": "Level 2: architecture and design",
|
|
33
|
+
};
|
|
34
|
+
export const levelRank = (l: Level): number => LEVELS.indexOf(l);
|
|
35
|
+
|
|
36
|
+
/** The 15 touch points of the issue, in the issue's order. */
|
|
37
|
+
export const TOUCH_POINTS = [
|
|
38
|
+
{ id: "on_disk_pg_upgrade", label: "on-disk format / pg_upgrade" },
|
|
39
|
+
{ id: "wal_replication", label: "WAL / replication / standby" },
|
|
40
|
+
{ id: "wire_protocol", label: "wire protocol" },
|
|
41
|
+
{ id: "catalog", label: "catalog changes" },
|
|
42
|
+
{ id: "abi_extension_api", label: "ABI / extension API" },
|
|
43
|
+
{ id: "sql_standard_syntax", label: "SQL-standard conflict or non-standard syntax" },
|
|
44
|
+
{ id: "new_guc", label: "new GUC" },
|
|
45
|
+
{ id: "planner_behaviour", label: "planner behaviour change" },
|
|
46
|
+
{ id: "pg_dump_round_trip", label: "pg_dump / restore round-trip" },
|
|
47
|
+
{ id: "psql_ux", label: "psql UX conventions" },
|
|
48
|
+
{ id: "security_privileges", label: "security / privileges" },
|
|
49
|
+
{ id: "backpatchability", label: "backpatchability" },
|
|
50
|
+
{ id: "maintenance_burden", label: 'maintenance burden ("who maintains this in 10 years")' },
|
|
51
|
+
{ id: "hot_path_performance", label: "performance on the hot path" },
|
|
52
|
+
{ id: "does_not_fit", label: "doesn't fit what Postgres is about (extension / external tool / already solved)" },
|
|
53
|
+
] as const;
|
|
54
|
+
export type TouchPointId = (typeof TOUCH_POINTS)[number]["id"];
|
|
55
|
+
export const TOUCH_POINT_IDS = TOUCH_POINTS.map((t) => t.id) as TouchPointId[];
|
|
56
|
+
|
|
57
|
+
/** Objection categories: a touch point, or one of the non-touch-point classes. */
|
|
58
|
+
export const EXTRA_CATEGORIES = ["problem_statement", "desirability", "demand", "prior_rejection", "design"] as const;
|
|
59
|
+
export const CATEGORIES = [...TOUCH_POINT_IDS, ...EXTRA_CATEGORIES] as const;
|
|
60
|
+
export type Category = (typeof CATEGORIES)[number];
|
|
61
|
+
|
|
62
|
+
export const ANGLES = [
|
|
63
|
+
"prior-attempts",
|
|
64
|
+
"standards-compat",
|
|
65
|
+
"replication-ondisk",
|
|
66
|
+
"security",
|
|
67
|
+
"maintenance-fit",
|
|
68
|
+
"already-exists",
|
|
69
|
+
"demand",
|
|
70
|
+
] as const;
|
|
71
|
+
export type Angle = (typeof ANGLES)[number];
|
|
72
|
+
|
|
73
|
+
export const SEVERITIES = ["fatal", "fixable"] as const;
|
|
74
|
+
export type Severity = (typeof SEVERITIES)[number];
|
|
75
|
+
|
|
76
|
+
export const REFUTATION_KINDS = [
|
|
77
|
+
"none",
|
|
78
|
+
"wrong_facts",
|
|
79
|
+
"precedent_mismatch",
|
|
80
|
+
"outdated_reason",
|
|
81
|
+
"already_handled",
|
|
82
|
+
"not_specific",
|
|
83
|
+
"no_failure_scenario",
|
|
84
|
+
] as const;
|
|
85
|
+
export type RefutationKind = (typeof REFUTATION_KINDS)[number];
|
|
86
|
+
|
|
87
|
+
// ---------------------------------------------------------------------------------------------
|
|
88
|
+
// Stage outputs (what the model returns)
|
|
89
|
+
|
|
90
|
+
export interface Brief {
|
|
91
|
+
title: string;
|
|
92
|
+
problem: string;
|
|
93
|
+
proposed_design: string;
|
|
94
|
+
user_visible_surface: string[];
|
|
95
|
+
author_claims: string[];
|
|
96
|
+
thread_state: string;
|
|
97
|
+
unclear: string[];
|
|
98
|
+
search_queries: string[];
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
export interface RawObjection {
|
|
102
|
+
title: string;
|
|
103
|
+
level: Level;
|
|
104
|
+
/** Item number in the rubric (1-16), or null. */
|
|
105
|
+
rubric_item: number | null;
|
|
106
|
+
category: Category;
|
|
107
|
+
touch_points: TouchPointId[];
|
|
108
|
+
severity: Severity;
|
|
109
|
+
/** The horror story: what concretely goes wrong, for whom, when. */
|
|
110
|
+
failure_scenario: string;
|
|
111
|
+
/** Why this applies to this proposal (facts from the thread / brief). */
|
|
112
|
+
argument: string;
|
|
113
|
+
/** Ids from the retrieved precedent list (P1, P2, ...). */
|
|
114
|
+
precedent_ids: string[];
|
|
115
|
+
/** Indexes of thread messages relied on. */
|
|
116
|
+
thread_message_refs: number[];
|
|
117
|
+
author_must_show: string;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
export interface RawPriorAttempt {
|
|
121
|
+
precedent_id: string;
|
|
122
|
+
relation: "same_idea" | "similar_idea" | "related_only";
|
|
123
|
+
what_is_different: string;
|
|
124
|
+
/** A sentence copied verbatim from the precedent text, or null. Verified by code. */
|
|
125
|
+
quote: string | null;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
export interface RoasterOutput {
|
|
129
|
+
objections: RawObjection[];
|
|
130
|
+
/** Only the prior-attempts roaster fills this. */
|
|
131
|
+
prior_attempts: RawPriorAttempt[];
|
|
132
|
+
/** Why there are no objections from this angle (when the list is empty), else "". */
|
|
133
|
+
no_objection_note: string;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
export interface CriticVerdict {
|
|
137
|
+
objection_id: string;
|
|
138
|
+
verdict: "survived" | "refuted";
|
|
139
|
+
refutation_kind: RefutationKind;
|
|
140
|
+
reasoning: string;
|
|
141
|
+
/** Set when the objection survives but with a different severity. */
|
|
142
|
+
adjusted_severity: Severity | null;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
export interface CriticOutput {
|
|
146
|
+
verdicts: CriticVerdict[];
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
export const OWNER_STATES = ["qualified committer engaged", "silent", "none obvious"] as const;
|
|
150
|
+
export type OwnerState = (typeof OWNER_STATES)[number];
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* "Will not get committed" is not "should be rejected": this is reported apart from the verdict
|
|
154
|
+
* (rubric, last section) and must never raise the verdict to LIKELY-REJECT by itself.
|
|
155
|
+
*/
|
|
156
|
+
export interface Committability {
|
|
157
|
+
/** Who would own the aftermath. */
|
|
158
|
+
owner: OwnerState;
|
|
159
|
+
/** Which kind of committer that is (by area), in one sentence. */
|
|
160
|
+
owner_note: string;
|
|
161
|
+
/** What the owner signs for, and for how long. */
|
|
162
|
+
liability: string;
|
|
163
|
+
/** Any sign in the thread that the author stays for fixes. */
|
|
164
|
+
author_follow_up: string;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/** "Not now / show demand": a visible outcome, not a fifth verdict. */
|
|
168
|
+
export interface NotNow {
|
|
169
|
+
/** True when only the demand / accumulation items (rubric items 3, 5) fail. */
|
|
170
|
+
applies: boolean;
|
|
171
|
+
/** What evidence would change the answer; "" when it does not apply. */
|
|
172
|
+
evidence_that_would_change_it: string;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
export const AUDIENCES = ["most users", "a sizeable minority", "niche", "unclear"] as const;
|
|
176
|
+
export const GRADES = ["low", "medium", "high"] as const;
|
|
177
|
+
export const ACTIONS = [
|
|
178
|
+
"proceed",
|
|
179
|
+
"park",
|
|
180
|
+
"bundle",
|
|
181
|
+
"needs champion",
|
|
182
|
+
"needs design rethink",
|
|
183
|
+
"withdraw and resubmit",
|
|
184
|
+
"fix design first",
|
|
185
|
+
"close: dead or superseded",
|
|
186
|
+
] as const;
|
|
187
|
+
|
|
188
|
+
/** One finding of the tone lint that the repair call did not remove. */
|
|
189
|
+
export interface ToneHit {
|
|
190
|
+
/** Where in roast.json, e.g. "objections[0].argument". */
|
|
191
|
+
path: string;
|
|
192
|
+
rule: string;
|
|
193
|
+
match: string;
|
|
194
|
+
sentence: string;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
export interface ToneLint {
|
|
198
|
+
fields_checked: number;
|
|
199
|
+
/** Hits before the repair call. */
|
|
200
|
+
hits_found: number;
|
|
201
|
+
sentences_rewritten: number;
|
|
202
|
+
repair_call: "not needed" | "done" | "failed" | "skipped";
|
|
203
|
+
remaining: ToneHit[];
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
export const TONE_REPAIR_SCHEMA: JsonSchema = {
|
|
207
|
+
type: "object",
|
|
208
|
+
properties: {
|
|
209
|
+
rewrites: {
|
|
210
|
+
type: "array",
|
|
211
|
+
items: { type: "object", properties: { n: { type: "integer" }, text: { type: "string", minLength: 1 } } },
|
|
212
|
+
},
|
|
213
|
+
},
|
|
214
|
+
};
|
|
215
|
+
|
|
216
|
+
/**
|
|
217
|
+
* Value / priority, apart from the verdict: most stale patches are low-value but net-positive,
|
|
218
|
+
* not wrong. Never an input to the verdict rules.
|
|
219
|
+
*/
|
|
220
|
+
export interface Priority {
|
|
221
|
+
audience: (typeof AUDIENCES)[number];
|
|
222
|
+
maintenance_burden: (typeof GRADES)[number];
|
|
223
|
+
/** From the patch size and the subsystems in the touched-file list. */
|
|
224
|
+
review_cost: (typeof GRADES)[number];
|
|
225
|
+
recommended_action: (typeof ACTIONS)[number];
|
|
226
|
+
reason: string;
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
/** Self mode: what the author should do about one surviving objection before sending. */
|
|
230
|
+
export interface SelfAdvice {
|
|
231
|
+
objection_id: string;
|
|
232
|
+
action: "fix" | "rebut";
|
|
233
|
+
how: string;
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
/** A reflection round's answer of one roaster. */
|
|
237
|
+
export interface ReflectOutput {
|
|
238
|
+
objections: RawObjection[];
|
|
239
|
+
revised: { objection_id: string; objection: RawObjection }[];
|
|
240
|
+
withdrawn: { objection_id: string; reason: string }[];
|
|
241
|
+
note: string;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
export interface SupervisorOutput {
|
|
245
|
+
follow_ups: { angle: Angle; uncovered_area: string; instruction: string }[];
|
|
246
|
+
coverage_note: string;
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
export type RoundKind = "initial" | "what_did_you_forget" | "are_you_happy" | "supervisor";
|
|
250
|
+
|
|
251
|
+
/** One round of the run, for measuring what each round adds. */
|
|
252
|
+
export interface RoundRecord {
|
|
253
|
+
round: number;
|
|
254
|
+
kind: RoundKind;
|
|
255
|
+
llm_calls: number;
|
|
256
|
+
new_objections: number;
|
|
257
|
+
/** Of the new objections, how many survived the critic. */
|
|
258
|
+
new_survived: number;
|
|
259
|
+
revised: number;
|
|
260
|
+
withdrawn: number;
|
|
261
|
+
/** Supervisor rounds: the follow-ups issued. */
|
|
262
|
+
follow_ups: { angle: Angle; uncovered_area: string }[];
|
|
263
|
+
note: string;
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
export interface SynthOutput {
|
|
267
|
+
verdict: Verdict;
|
|
268
|
+
confidence: Confidence;
|
|
269
|
+
verdict_reasoning: string;
|
|
270
|
+
/** Lowest pyramid level that fails, or null when none does. */
|
|
271
|
+
failing_level: Level | null;
|
|
272
|
+
/** 0-5 ids of surviving objections, strongest first. */
|
|
273
|
+
ranked_objection_ids: string[];
|
|
274
|
+
touch_points: { id: TouchPointId; hit: boolean; reason: string }[];
|
|
275
|
+
questions: string[];
|
|
276
|
+
prior_attempts: RawPriorAttempt[];
|
|
277
|
+
committability: Committability;
|
|
278
|
+
not_now: NotNow;
|
|
279
|
+
priority: Priority;
|
|
280
|
+
/** Self mode only. */
|
|
281
|
+
self_advice?: SelfAdvice[];
|
|
282
|
+
/** What could not be judged without reading the code. */
|
|
283
|
+
provisional_note: string;
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
// ---------------------------------------------------------------------------------------------
|
|
287
|
+
// Precedents (normalised by src/roast/precedents.ts; every field comes from the corpus DB)
|
|
288
|
+
|
|
289
|
+
export interface Precedent {
|
|
290
|
+
/** Run-local id shown to the model: P1, P2, ... */
|
|
291
|
+
id: string;
|
|
292
|
+
/** Id in the corpus, e.g. "cf:5253", "revert:<sha>". */
|
|
293
|
+
source_id: string;
|
|
294
|
+
kind: string;
|
|
295
|
+
title: string;
|
|
296
|
+
status: string;
|
|
297
|
+
/** Decision / close date (ISO) or null. */
|
|
298
|
+
date: string | null;
|
|
299
|
+
/** Thread start date (ISO) or null. */
|
|
300
|
+
first_date: string | null;
|
|
301
|
+
url: string;
|
|
302
|
+
thread_urls: string[];
|
|
303
|
+
authors: string;
|
|
304
|
+
objectors: string;
|
|
305
|
+
summary: string;
|
|
306
|
+
objection: string;
|
|
307
|
+
touch_points: string[];
|
|
308
|
+
/** Which queries found it. */
|
|
309
|
+
matched_queries: string[];
|
|
310
|
+
/** True when found by the --hint directed lookup. */
|
|
311
|
+
from_hint: boolean;
|
|
312
|
+
score: number;
|
|
313
|
+
/**
|
|
314
|
+
* True when `objection` (and maybe `summary`, `objectors`) is an AI summary of the linked
|
|
315
|
+
* thread, made at roast time (src/roast/enrich.ts), not text recorded in the corpus.
|
|
316
|
+
*/
|
|
317
|
+
enriched: boolean;
|
|
318
|
+
/** A sentence verified by code to be verbatim in one message of the precedent's thread. */
|
|
319
|
+
key_quote: string | null;
|
|
320
|
+
/** Permalink of that message (built by code). */
|
|
321
|
+
quote_message_url: string | null;
|
|
322
|
+
/** Result of the relevance pass: shown to the roasters or not; null = the pass did not run. */
|
|
323
|
+
relevant: boolean | null;
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
// ---------------------------------------------------------------------------------------------
|
|
327
|
+
// roast.json
|
|
328
|
+
|
|
329
|
+
export interface FinalObjection extends RawObjection {
|
|
330
|
+
/** Run-local id, e.g. "security-1". */
|
|
331
|
+
id: string;
|
|
332
|
+
angle: Angle;
|
|
333
|
+
/** 0 = first pass; 1 = "What did you forget?"; 2 = "Are you happy with the results?"; 3+ = supervisor follow-ups. */
|
|
334
|
+
round: number;
|
|
335
|
+
/** Set when the roaster restated the objection in a later round. */
|
|
336
|
+
revised_in_round: number | null;
|
|
337
|
+
status: "survived" | "refuted" | "withdrawn";
|
|
338
|
+
/** Set when the roaster took the objection back; withdrawn objections are kept, not deleted. */
|
|
339
|
+
withdrawal: { round: number; reason: string } | null;
|
|
340
|
+
critic: {
|
|
341
|
+
refutation_kind: RefutationKind;
|
|
342
|
+
reasoning: string;
|
|
343
|
+
/** Severity the roaster gave, when the critic changed it. */
|
|
344
|
+
original_severity: Severity | null;
|
|
345
|
+
};
|
|
346
|
+
/** Precedents relied on, resolved by code from precedent_ids. Links come from the DB only. */
|
|
347
|
+
precedents: { id: string; title: string; url: string; thread_urls: string[]; date: string | null; status: string }[];
|
|
348
|
+
/** Thread messages relied on, resolved by code. */
|
|
349
|
+
thread_refs: { index: number; from: string; date: string; url: string }[];
|
|
350
|
+
/** Why a surviving objection is not in the ranked list (stop rule, rank cut), else null. */
|
|
351
|
+
not_ranked_reason: string | null;
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
export interface PriorAttempt {
|
|
355
|
+
precedent_id: string;
|
|
356
|
+
source_id: string;
|
|
357
|
+
title: string;
|
|
358
|
+
kind: string;
|
|
359
|
+
status: string;
|
|
360
|
+
/** Link(s): from the corpus DB only. */
|
|
361
|
+
url: string;
|
|
362
|
+
thread_urls: string[];
|
|
363
|
+
date: string | null;
|
|
364
|
+
first_date: string | null;
|
|
365
|
+
/** From the corpus DB; "" when the DB does not record it. */
|
|
366
|
+
who_objected: string;
|
|
367
|
+
/** Verbatim sentence found in the DB text or the precedent's thread, or null = "quote unavailable". */
|
|
368
|
+
quote: string | null;
|
|
369
|
+
/** The message the quote is from, when it was verified against the precedent's thread. */
|
|
370
|
+
quote_url: string | null;
|
|
371
|
+
/** True when the reason shown is an AI summary of the linked thread. */
|
|
372
|
+
enriched: boolean;
|
|
373
|
+
/** Why it was not accepted: corpus text, or the AI summary when `enriched`. */
|
|
374
|
+
reason: string;
|
|
375
|
+
relation: "same_idea" | "similar_idea" | "related_only";
|
|
376
|
+
what_is_different: string;
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
export interface TouchPointResult {
|
|
380
|
+
id: TouchPointId;
|
|
381
|
+
label: string;
|
|
382
|
+
hit: boolean;
|
|
383
|
+
reason: string;
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
export interface SanitizerReport {
|
|
387
|
+
/** URLs in model output that were not in the allowed set, with where they were found. */
|
|
388
|
+
urls_removed: { stage: string; url: string }[];
|
|
389
|
+
/** Precedent ids the model cited that are not in the retrieved list. */
|
|
390
|
+
precedent_ids_dropped: { stage: string; id: string }[];
|
|
391
|
+
/** Thread message indexes the model cited that are not in the visible thread. */
|
|
392
|
+
thread_refs_dropped: { stage: string; index: number }[];
|
|
393
|
+
/** Quotes that were not found verbatim in the precedent text. */
|
|
394
|
+
quotes_dropped: { precedent_id: string; quote: string }[];
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
export interface RoastInputInfo {
|
|
398
|
+
arg: string;
|
|
399
|
+
cf: {
|
|
400
|
+
id: number;
|
|
401
|
+
url: string;
|
|
402
|
+
title: string;
|
|
403
|
+
status: string | null;
|
|
404
|
+
authors: string;
|
|
405
|
+
redacted: boolean;
|
|
406
|
+
} | null;
|
|
407
|
+
thread_url: string;
|
|
408
|
+
subject: string;
|
|
409
|
+
thread_start: string;
|
|
410
|
+
messages_read: number;
|
|
411
|
+
messages_total: number;
|
|
412
|
+
blind: number | null;
|
|
413
|
+
before: string | null;
|
|
414
|
+
hint: string | null;
|
|
415
|
+
attachments_read: string[];
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
export interface Roast {
|
|
419
|
+
schema_version: number;
|
|
420
|
+
tool: string;
|
|
421
|
+
generated_at: string;
|
|
422
|
+
input: RoastInputInfo;
|
|
423
|
+
/** "review" = someone else's proposal; "self" = the author's own pre-flight check. */
|
|
424
|
+
mode: "review" | "self";
|
|
425
|
+
/** "claude" | "codex" | "dry-run" | "test". */
|
|
426
|
+
backend: string;
|
|
427
|
+
models: { roaster: string; critic: string; synth: string; brief: string };
|
|
428
|
+
|
|
429
|
+
verdict: Verdict;
|
|
430
|
+
confidence: Confidence;
|
|
431
|
+
verdict_reasoning: string;
|
|
432
|
+
failing_level: Level | null;
|
|
433
|
+
/** Base-up status of each level: levels above the failing one are not evaluated. */
|
|
434
|
+
levels: { level: Level; name: string; status: "passed" | "failed" | "not_evaluated" }[];
|
|
435
|
+
|
|
436
|
+
brief: Brief;
|
|
437
|
+
/** Empty = "no precedent found". */
|
|
438
|
+
prior_attempts: PriorAttempt[];
|
|
439
|
+
/** The ranked surviving objections (the steelman), strongest first. */
|
|
440
|
+
objections: FinalObjection[];
|
|
441
|
+
/** Survived the critic but not listed in the steelman. */
|
|
442
|
+
other_surviving: FinalObjection[];
|
|
443
|
+
refuted: FinalObjection[];
|
|
444
|
+
/** Taken back by their own roaster in a reflection round, with the reason. */
|
|
445
|
+
withdrawn: FinalObjection[];
|
|
446
|
+
touch_points: TouchPointResult[];
|
|
447
|
+
questions: string[];
|
|
448
|
+
committability: Committability;
|
|
449
|
+
not_now: NotNow;
|
|
450
|
+
priority: Priority;
|
|
451
|
+
/** Self mode: advice per surviving objection; empty otherwise. */
|
|
452
|
+
self_advice: SelfAdvice[];
|
|
453
|
+
provisional_note: string;
|
|
454
|
+
disclosure: string;
|
|
455
|
+
|
|
456
|
+
/** Audit trail. */
|
|
457
|
+
precedents_retrieved: Precedent[];
|
|
458
|
+
/** Precedents dated on or after this day were excluded. */
|
|
459
|
+
precedent_cutoff: string;
|
|
460
|
+
/** Which corpus backend answered; "none" = the search was not done. */
|
|
461
|
+
precedent_backend: string;
|
|
462
|
+
/** Lazy enrichment and relevance pass; null when they did not run. */
|
|
463
|
+
enrichment: {
|
|
464
|
+
candidates: number;
|
|
465
|
+
from_cache: number;
|
|
466
|
+
new_calls: number;
|
|
467
|
+
failed: number;
|
|
468
|
+
quotes_verified: number;
|
|
469
|
+
quotes_dropped: number;
|
|
470
|
+
objectors_dropped: number;
|
|
471
|
+
relevance_pass: "done" | "skipped" | "failed";
|
|
472
|
+
relevant: number;
|
|
473
|
+
} | null;
|
|
474
|
+
roaster_notes: { angle: Angle; objections: number; note: string }[];
|
|
475
|
+
/** --rounds N as requested, and what each round did. */
|
|
476
|
+
rounds_requested: number;
|
|
477
|
+
rounds: RoundRecord[];
|
|
478
|
+
/** Subsystem invariant files given to the roasters (paths relative to the repository). */
|
|
479
|
+
invariants_loaded: string[];
|
|
480
|
+
warnings: string[];
|
|
481
|
+
/** Deterministic tone check over every prose field; what is left after one repair call. */
|
|
482
|
+
tone_lint: ToneLint;
|
|
483
|
+
sanitizer: SanitizerReport;
|
|
484
|
+
stats: { llm_calls: number; wall_ms: number; cost_usd: number | null; dry_run: boolean };
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
// ---------------------------------------------------------------------------------------------
|
|
488
|
+
// JSON Schema subset + validator
|
|
489
|
+
|
|
490
|
+
export type JsonSchema =
|
|
491
|
+
| { type: "string"; enum?: readonly string[]; minLength?: number; nullable?: boolean }
|
|
492
|
+
| { type: "integer" | "number"; minimum?: number; maximum?: number; nullable?: boolean }
|
|
493
|
+
| { type: "boolean"; nullable?: boolean }
|
|
494
|
+
| { type: "array"; items: JsonSchema; minItems?: number; maxItems?: number; nullable?: boolean }
|
|
495
|
+
| {
|
|
496
|
+
type: "object";
|
|
497
|
+
properties: Record<string, JsonSchema>;
|
|
498
|
+
required?: readonly string[];
|
|
499
|
+
nullable?: boolean;
|
|
500
|
+
};
|
|
501
|
+
|
|
502
|
+
/** Returns a list of problems; empty means valid. Unknown object keys are allowed. */
|
|
503
|
+
export function validate(value: unknown, schema: JsonSchema, path = "$"): string[] {
|
|
504
|
+
if (value === null || value === undefined) {
|
|
505
|
+
return schema.nullable && value === null ? [] : [`${path}: expected ${schema.type}, got ${value === null ? "null" : "nothing"}`];
|
|
506
|
+
}
|
|
507
|
+
switch (schema.type) {
|
|
508
|
+
case "string": {
|
|
509
|
+
if (typeof value !== "string") return [`${path}: expected string, got ${typeName(value)}`];
|
|
510
|
+
if (schema.enum && !schema.enum.includes(value)) {
|
|
511
|
+
return [`${path}: "${value}" is not one of ${schema.enum.join(" | ")}`];
|
|
512
|
+
}
|
|
513
|
+
if (schema.minLength && value.trim().length < schema.minLength) return [`${path}: must not be empty`];
|
|
514
|
+
return [];
|
|
515
|
+
}
|
|
516
|
+
case "integer":
|
|
517
|
+
case "number": {
|
|
518
|
+
if (typeof value !== "number" || !Number.isFinite(value)) return [`${path}: expected ${schema.type}, got ${typeName(value)}`];
|
|
519
|
+
if (schema.type === "integer" && !Number.isInteger(value)) return [`${path}: expected integer`];
|
|
520
|
+
if (schema.minimum != null && value < schema.minimum) return [`${path}: below minimum ${schema.minimum}`];
|
|
521
|
+
if (schema.maximum != null && value > schema.maximum) return [`${path}: above maximum ${schema.maximum}`];
|
|
522
|
+
return [];
|
|
523
|
+
}
|
|
524
|
+
case "boolean":
|
|
525
|
+
return typeof value === "boolean" ? [] : [`${path}: expected boolean, got ${typeName(value)}`];
|
|
526
|
+
case "array": {
|
|
527
|
+
if (!Array.isArray(value)) return [`${path}: expected array, got ${typeName(value)}`];
|
|
528
|
+
const errs: string[] = [];
|
|
529
|
+
if (schema.minItems != null && value.length < schema.minItems) errs.push(`${path}: needs at least ${schema.minItems} items, has ${value.length}`);
|
|
530
|
+
if (schema.maxItems != null && value.length > schema.maxItems) errs.push(`${path}: at most ${schema.maxItems} items, has ${value.length}`);
|
|
531
|
+
value.forEach((v, i) => errs.push(...validate(v, schema.items, `${path}[${i}]`)));
|
|
532
|
+
return errs;
|
|
533
|
+
}
|
|
534
|
+
case "object": {
|
|
535
|
+
if (typeof value !== "object" || Array.isArray(value)) return [`${path}: expected object, got ${typeName(value)}`];
|
|
536
|
+
const errs: string[] = [];
|
|
537
|
+
const obj = value as Record<string, unknown>;
|
|
538
|
+
for (const k of schema.required ?? Object.keys(schema.properties)) {
|
|
539
|
+
if (!(k in obj)) errs.push(`${path}.${k}: missing`);
|
|
540
|
+
}
|
|
541
|
+
for (const [k, sub] of Object.entries(schema.properties)) {
|
|
542
|
+
if (k in obj) errs.push(...validate(obj[k], sub, `${path}.${k}`));
|
|
543
|
+
}
|
|
544
|
+
return errs;
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
function typeName(v: unknown): string {
|
|
550
|
+
return Array.isArray(v) ? "array" : typeof v;
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
const str: JsonSchema = { type: "string" };
|
|
554
|
+
const nonEmpty: JsonSchema = { type: "string", minLength: 1 };
|
|
555
|
+
const strList = (min = 0, max?: number): JsonSchema => ({ type: "array", items: nonEmpty, minItems: min, ...(max != null ? { maxItems: max } : {}) });
|
|
556
|
+
|
|
557
|
+
export const BRIEF_SCHEMA: JsonSchema = {
|
|
558
|
+
type: "object",
|
|
559
|
+
properties: {
|
|
560
|
+
title: nonEmpty,
|
|
561
|
+
problem: nonEmpty,
|
|
562
|
+
proposed_design: nonEmpty,
|
|
563
|
+
user_visible_surface: strList(),
|
|
564
|
+
author_claims: strList(),
|
|
565
|
+
thread_state: str,
|
|
566
|
+
unclear: strList(),
|
|
567
|
+
search_queries: strList(5, 8),
|
|
568
|
+
},
|
|
569
|
+
};
|
|
570
|
+
|
|
571
|
+
const OBJECTION_SCHEMA: JsonSchema = {
|
|
572
|
+
type: "object",
|
|
573
|
+
properties: {
|
|
574
|
+
title: nonEmpty,
|
|
575
|
+
level: { type: "string", enum: LEVELS },
|
|
576
|
+
rubric_item: { type: "integer", minimum: 1, maximum: 16, nullable: true },
|
|
577
|
+
category: { type: "string", enum: CATEGORIES },
|
|
578
|
+
touch_points: { type: "array", items: { type: "string", enum: TOUCH_POINT_IDS } },
|
|
579
|
+
severity: { type: "string", enum: SEVERITIES },
|
|
580
|
+
failure_scenario: nonEmpty,
|
|
581
|
+
argument: nonEmpty,
|
|
582
|
+
precedent_ids: { type: "array", items: str },
|
|
583
|
+
thread_message_refs: { type: "array", items: { type: "integer", minimum: 1 } },
|
|
584
|
+
author_must_show: nonEmpty,
|
|
585
|
+
},
|
|
586
|
+
};
|
|
587
|
+
|
|
588
|
+
const PRIOR_ATTEMPT_SCHEMA: JsonSchema = {
|
|
589
|
+
type: "object",
|
|
590
|
+
properties: {
|
|
591
|
+
precedent_id: nonEmpty,
|
|
592
|
+
relation: { type: "string", enum: ["same_idea", "similar_idea", "related_only"] },
|
|
593
|
+
what_is_different: nonEmpty,
|
|
594
|
+
quote: { type: "string", nullable: true },
|
|
595
|
+
},
|
|
596
|
+
};
|
|
597
|
+
|
|
598
|
+
export const ROASTER_SCHEMA: JsonSchema = {
|
|
599
|
+
type: "object",
|
|
600
|
+
properties: {
|
|
601
|
+
objections: { type: "array", items: OBJECTION_SCHEMA, maxItems: 4 },
|
|
602
|
+
prior_attempts: { type: "array", items: PRIOR_ATTEMPT_SCHEMA },
|
|
603
|
+
no_objection_note: str,
|
|
604
|
+
},
|
|
605
|
+
};
|
|
606
|
+
|
|
607
|
+
export const CRITIC_SCHEMA: JsonSchema = {
|
|
608
|
+
type: "object",
|
|
609
|
+
properties: {
|
|
610
|
+
verdicts: {
|
|
611
|
+
type: "array",
|
|
612
|
+
items: {
|
|
613
|
+
type: "object",
|
|
614
|
+
properties: {
|
|
615
|
+
objection_id: nonEmpty,
|
|
616
|
+
verdict: { type: "string", enum: ["survived", "refuted"] },
|
|
617
|
+
refutation_kind: { type: "string", enum: REFUTATION_KINDS },
|
|
618
|
+
reasoning: nonEmpty,
|
|
619
|
+
adjusted_severity: { type: "string", enum: SEVERITIES, nullable: true },
|
|
620
|
+
},
|
|
621
|
+
},
|
|
622
|
+
},
|
|
623
|
+
},
|
|
624
|
+
};
|
|
625
|
+
|
|
626
|
+
export const SYNTH_SCHEMA: JsonSchema = {
|
|
627
|
+
type: "object",
|
|
628
|
+
properties: {
|
|
629
|
+
verdict: { type: "string", enum: VERDICTS },
|
|
630
|
+
confidence: { type: "string", enum: CONFIDENCES },
|
|
631
|
+
verdict_reasoning: nonEmpty,
|
|
632
|
+
failing_level: { type: "string", enum: LEVELS, nullable: true },
|
|
633
|
+
ranked_objection_ids: { type: "array", items: nonEmpty, maxItems: 5 },
|
|
634
|
+
touch_points: {
|
|
635
|
+
type: "array",
|
|
636
|
+
items: {
|
|
637
|
+
type: "object",
|
|
638
|
+
properties: {
|
|
639
|
+
id: { type: "string", enum: TOUCH_POINT_IDS },
|
|
640
|
+
hit: { type: "boolean" },
|
|
641
|
+
reason: str,
|
|
642
|
+
},
|
|
643
|
+
},
|
|
644
|
+
},
|
|
645
|
+
questions: strList(0, 5),
|
|
646
|
+
prior_attempts: { type: "array", items: PRIOR_ATTEMPT_SCHEMA },
|
|
647
|
+
committability: {
|
|
648
|
+
type: "object",
|
|
649
|
+
properties: {
|
|
650
|
+
owner: { type: "string", enum: OWNER_STATES },
|
|
651
|
+
owner_note: str,
|
|
652
|
+
liability: nonEmpty,
|
|
653
|
+
author_follow_up: nonEmpty,
|
|
654
|
+
},
|
|
655
|
+
},
|
|
656
|
+
not_now: {
|
|
657
|
+
type: "object",
|
|
658
|
+
properties: {
|
|
659
|
+
applies: { type: "boolean" },
|
|
660
|
+
evidence_that_would_change_it: str,
|
|
661
|
+
},
|
|
662
|
+
},
|
|
663
|
+
priority: {
|
|
664
|
+
type: "object",
|
|
665
|
+
properties: {
|
|
666
|
+
audience: { type: "string", enum: AUDIENCES },
|
|
667
|
+
maintenance_burden: { type: "string", enum: GRADES },
|
|
668
|
+
review_cost: { type: "string", enum: GRADES },
|
|
669
|
+
recommended_action: { type: "string", enum: ACTIONS },
|
|
670
|
+
reason: nonEmpty,
|
|
671
|
+
},
|
|
672
|
+
},
|
|
673
|
+
provisional_note: str,
|
|
674
|
+
},
|
|
675
|
+
};
|
|
676
|
+
|
|
677
|
+
export const SYNTH_SELF_SCHEMA: JsonSchema = {
|
|
678
|
+
type: "object",
|
|
679
|
+
properties: {
|
|
680
|
+
...(SYNTH_SCHEMA as { properties: Record<string, JsonSchema> }).properties,
|
|
681
|
+
self_advice: {
|
|
682
|
+
type: "array",
|
|
683
|
+
items: {
|
|
684
|
+
type: "object",
|
|
685
|
+
properties: { objection_id: nonEmpty, action: { type: "string", enum: ["fix", "rebut"] }, how: nonEmpty },
|
|
686
|
+
},
|
|
687
|
+
},
|
|
688
|
+
},
|
|
689
|
+
};
|
|
690
|
+
|
|
691
|
+
export const REFLECT_SCHEMA: JsonSchema = {
|
|
692
|
+
type: "object",
|
|
693
|
+
properties: {
|
|
694
|
+
objections: { type: "array", items: OBJECTION_SCHEMA, maxItems: 3 },
|
|
695
|
+
revised: { type: "array", items: { type: "object", properties: { objection_id: nonEmpty, objection: OBJECTION_SCHEMA } } },
|
|
696
|
+
withdrawn: { type: "array", items: { type: "object", properties: { objection_id: nonEmpty, reason: nonEmpty } } },
|
|
697
|
+
note: str,
|
|
698
|
+
},
|
|
699
|
+
};
|
|
700
|
+
|
|
701
|
+
export const SUPERVISOR_SCHEMA: JsonSchema = {
|
|
702
|
+
type: "object",
|
|
703
|
+
properties: {
|
|
704
|
+
follow_ups: {
|
|
705
|
+
type: "array",
|
|
706
|
+
maxItems: 3,
|
|
707
|
+
items: {
|
|
708
|
+
type: "object",
|
|
709
|
+
properties: { angle: { type: "string", enum: ANGLES }, uncovered_area: nonEmpty, instruction: nonEmpty },
|
|
710
|
+
},
|
|
711
|
+
},
|
|
712
|
+
coverage_note: str,
|
|
713
|
+
},
|
|
714
|
+
};
|