substrate-ai 0.21.9 → 0.21.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/acceptance-judge-CKNJdq03.js +4 -0
- package/dist/acceptance-judge-eIZZha1C.js +702 -0
- package/dist/adapter-registry-Z0RlAJY4.js +4 -0
- package/dist/cli/index.js +175 -22
- package/dist/{decision-router-BAPpON_C.js → decision-router-Dyby0fUf.js} +1 -1
- package/dist/{decisions-DMUwdH1O.js → decisions-Cst_r3kF.js} +2 -2
- package/dist/{dist-CO4BbTu3.js → dist-BZGqn-XQ.js} +5 -3
- package/dist/{errors-0OA2nge4.js → errors-xSeGw6pf.js} +2 -2
- package/dist/{experimenter-BTUtFcGu.js → experimenter-hnLiBgcQ.js} +1 -1
- package/dist/health--DIz8sJ9.js +8 -0
- package/dist/{health-BbgLm021.js → health-B867L-8I.js} +3 -3
- package/dist/{index-s7wBSfVT.d.ts → index-Be5MEp2y.d.ts} +2 -1
- package/dist/index.d.ts +13 -2
- package/dist/index.js +2 -2
- package/dist/{interactive-prompt-qTo2iuop.js → interactive-prompt-BvHUcoY4.js} +2 -2
- package/dist/{manifest-read-qsNEHEaF.js → manifest-read-BGzmmY01.js} +268 -5
- package/dist/modules/interactive-prompt/index.js +3 -3
- package/dist/{recovery-engine-BKGBeBnW.js → recovery-engine-dtLHyV4M.js} +2 -2
- package/dist/{routing-Bet3Iu2w.js → routing-BcnHtKAm.js} +2 -2
- package/dist/{run-nda59MmG.js → run-CoZ5kK6c.js} +617 -1169
- package/dist/run-G4HOt6-R.js +15 -0
- package/dist/src/modules/decision-router/index.js +1 -1
- package/dist/src/modules/recovery-engine/index.d.ts +1 -1
- package/dist/src/modules/recovery-engine/index.js +2 -2
- package/dist/{upgrade-Cryksfvg.js → upgrade-BoSsIewI.js} +3 -3
- package/dist/{upgrade-DGpQdA9A.js → upgrade-CNGc12AV.js} +2 -2
- package/dist/version-manager-impl-B9VDdCKb.js +4 -0
- package/dist/{work-graph-repository-DZyJv5pV.js → work-graph-repository-4cKsf8Lf.js} +1 -1
- package/package.json +1 -1
- package/packs/bmad/prompts/acceptance-judge.md +1 -1
- package/dist/adapter-registry-Cr9f2ICC.js +0 -4
- package/dist/health-CLo5vM7O.js +0 -8
- package/dist/run-DXJeZOid.js +0 -14
- package/dist/version-manager-impl-DSK9i3Xk.js +0 -4
- /package/dist/{decisions-CzSIEeGP.js → decisions-BAaxpVD0.js} +0 -0
- /package/dist/{routing-DFxoKHDt.js → routing-DZT5PN3N.js} +0 -0
- /package/dist/{version-manager-impl-qFBiO4Eh.js → version-manager-impl-DMTw551Q.js} +0 -0
|
@@ -0,0 +1,702 @@
|
|
|
1
|
+
import { createLogger } from "./logger-KeHncl-f.js";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { z } from "zod";
|
|
4
|
+
import { readFile } from "node:fs/promises";
|
|
5
|
+
|
|
6
|
+
//#region src/modules/context-compiler/token-counter.ts
|
|
7
|
+
/**
|
|
8
|
+
* Token counter utility for the context-compiler module.
|
|
9
|
+
*
|
|
10
|
+
* Uses a simple heuristic: chars/4, with a 10% upward adjustment for text
|
|
11
|
+
* containing fenced code blocks (triple backticks). Intentionally conservative
|
|
12
|
+
* — better to under-fill a prompt than overflow.
|
|
13
|
+
*/
|
|
14
|
+
const CHARS_PER_TOKEN = 4;
|
|
15
|
+
const CODE_BLOCK_ADJUSTMENT = 1.1;
|
|
16
|
+
const CODE_BLOCK_MARKER = "```";
|
|
17
|
+
/**
|
|
18
|
+
* Approximate the number of tokens in `text`.
|
|
19
|
+
*
|
|
20
|
+
* - Base heuristic: `Math.ceil(text.length / 4)`
|
|
21
|
+
* - If the text contains one or more fenced code blocks (triple backticks),
|
|
22
|
+
* apply a 10% upward multiplier and re-ceil.
|
|
23
|
+
*
|
|
24
|
+
* The approximation is within 15% of actual tokenizer output for typical
|
|
25
|
+
* prompt content.
|
|
26
|
+
*/
|
|
27
|
+
function countTokens(text) {
|
|
28
|
+
if (text.length === 0) return 0;
|
|
29
|
+
const base = text.length / CHARS_PER_TOKEN;
|
|
30
|
+
const hasCodeBlock = text.includes(CODE_BLOCK_MARKER);
|
|
31
|
+
const adjusted = hasCodeBlock ? base * CODE_BLOCK_ADJUSTMENT : base;
|
|
32
|
+
return Math.ceil(adjusted);
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Truncate `text` to approximately `maxTokens` tokens.
|
|
36
|
+
*
|
|
37
|
+
* Returns the original text if it already fits within the budget.
|
|
38
|
+
* Appends a `…` ellipsis suffix when truncation occurs to indicate content
|
|
39
|
+
* was cut off.
|
|
40
|
+
*
|
|
41
|
+
* Note: Because the chars/4 heuristic is an approximation, the truncated
|
|
42
|
+
* result may still slightly exceed or fall short of the exact token target.
|
|
43
|
+
*/
|
|
44
|
+
function truncateToTokens(text, maxTokens) {
|
|
45
|
+
if (maxTokens <= 0) return "";
|
|
46
|
+
if (countTokens(text) <= maxTokens) return text;
|
|
47
|
+
const hasCodeBlock = text.includes(CODE_BLOCK_MARKER);
|
|
48
|
+
const multiplier = hasCodeBlock ? CODE_BLOCK_ADJUSTMENT : 1;
|
|
49
|
+
const targetChars = Math.floor(maxTokens * CHARS_PER_TOKEN / multiplier);
|
|
50
|
+
if (targetChars <= 0) return "";
|
|
51
|
+
const roughTrunc = text.slice(0, targetChars);
|
|
52
|
+
const lastSpace = roughTrunc.lastIndexOf(" ", roughTrunc.length - 1);
|
|
53
|
+
let truncated;
|
|
54
|
+
if (lastSpace > targetChars - 50 && lastSpace > 0) truncated = roughTrunc.slice(0, lastSpace);
|
|
55
|
+
else truncated = roughTrunc;
|
|
56
|
+
return truncated + "…";
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
//#endregion
|
|
60
|
+
//#region src/modules/compiled-workflows/prompt-assembler.ts
|
|
61
|
+
const logger$1 = createLogger("compiled-workflows:prompt-assembler");
|
|
62
|
+
/**
|
|
63
|
+
* Assemble a final prompt from a template and sections map.
|
|
64
|
+
*
|
|
65
|
+
* Steps:
|
|
66
|
+
* 1. Build a map of placeholder → content from sections array
|
|
67
|
+
* 2. Replace {{placeholder}} patterns in the template
|
|
68
|
+
* 3. Estimate total token count
|
|
69
|
+
* 4. If over ceiling, truncate sections in reverse priority order
|
|
70
|
+
* 5. Return assembled prompt with token count and truncation flag
|
|
71
|
+
*
|
|
72
|
+
* @param template - Prompt template with {{placeholder}} markers
|
|
73
|
+
* @param sections - Named sections with content and priority
|
|
74
|
+
* @param tokenCeiling - Hard token ceiling (default: 2200)
|
|
75
|
+
*/
|
|
76
|
+
function assemblePrompt(template, sections, tokenCeiling = 2200) {
|
|
77
|
+
const contentMap = {};
|
|
78
|
+
for (const section of sections) contentMap[section.name] = section.content;
|
|
79
|
+
let prompt = replacePlaceholders(template, contentMap);
|
|
80
|
+
let tokenCount = countTokens(prompt);
|
|
81
|
+
if (tokenCount <= tokenCeiling) return {
|
|
82
|
+
prompt,
|
|
83
|
+
tokenCount,
|
|
84
|
+
truncated: false
|
|
85
|
+
};
|
|
86
|
+
logger$1.warn({
|
|
87
|
+
tokenCount,
|
|
88
|
+
ceiling: tokenCeiling
|
|
89
|
+
}, "Prompt exceeds token ceiling — truncating optional sections");
|
|
90
|
+
const priorityOrder = ["optional", "important"];
|
|
91
|
+
let truncated = false;
|
|
92
|
+
for (const priority of priorityOrder) {
|
|
93
|
+
const sectionsAtPriority = sections.filter((s) => s.priority === priority);
|
|
94
|
+
for (const section of sectionsAtPriority) {
|
|
95
|
+
if (tokenCount <= tokenCeiling) break;
|
|
96
|
+
const overBy = tokenCount - tokenCeiling;
|
|
97
|
+
const currentSectionTokens = countTokens(section.content);
|
|
98
|
+
if (currentSectionTokens === 0) continue;
|
|
99
|
+
const targetSectionTokens = Math.max(0, currentSectionTokens - overBy);
|
|
100
|
+
if (targetSectionTokens === 0) {
|
|
101
|
+
contentMap[section.name] = "";
|
|
102
|
+
logger$1.warn({ sectionName: section.name }, "Section eliminated to fit token budget");
|
|
103
|
+
} else {
|
|
104
|
+
contentMap[section.name] = truncateToTokens(section.content, targetSectionTokens);
|
|
105
|
+
logger$1.warn({
|
|
106
|
+
sectionName: section.name,
|
|
107
|
+
targetSectionTokens
|
|
108
|
+
}, "Section truncated to fit token budget");
|
|
109
|
+
}
|
|
110
|
+
truncated = true;
|
|
111
|
+
prompt = replacePlaceholders(template, contentMap);
|
|
112
|
+
tokenCount = countTokens(prompt);
|
|
113
|
+
}
|
|
114
|
+
if (tokenCount <= tokenCeiling) break;
|
|
115
|
+
}
|
|
116
|
+
if (tokenCount > tokenCeiling) logger$1.warn({
|
|
117
|
+
tokenCount,
|
|
118
|
+
ceiling: tokenCeiling
|
|
119
|
+
}, "Required sections alone exceed token ceiling — returning over-budget prompt");
|
|
120
|
+
return {
|
|
121
|
+
prompt,
|
|
122
|
+
tokenCount,
|
|
123
|
+
truncated
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
/**
|
|
127
|
+
* Replace {{placeholder}} patterns in template with values from contentMap.
|
|
128
|
+
* Missing placeholders are replaced with an empty string.
|
|
129
|
+
*/
|
|
130
|
+
function replacePlaceholders(template, contentMap) {
|
|
131
|
+
return template.replace(/\{\{(\w[\w_-]*)\}\}/g, (_match, key) => {
|
|
132
|
+
return contentMap[key] ?? "";
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
//#endregion
|
|
137
|
+
//#region src/modules/compiled-workflows/schemas.ts
|
|
138
|
+
/**
|
|
139
|
+
* Schema for the YAML output contract of the create-story sub-agent.
|
|
140
|
+
* The agent must emit YAML matching this shape.
|
|
141
|
+
*/
|
|
142
|
+
const CreateStoryResultSchema = z.object({
|
|
143
|
+
result: z.preprocess((val) => val === "failure" ? "failed" : val, z.enum(["success", "failed"])),
|
|
144
|
+
story_file: z.string().optional(),
|
|
145
|
+
story_key: z.string().optional(),
|
|
146
|
+
story_title: z.string().optional()
|
|
147
|
+
});
|
|
148
|
+
/**
|
|
149
|
+
* Coerce a YAML value to a plain string. Agents sometimes emit
|
|
150
|
+
* `ac_failures: [AC7: explanation]` which YAML parses as a mapping
|
|
151
|
+
* `{ AC7: "explanation" }` instead of a string. This flattens it.
|
|
152
|
+
*/
|
|
153
|
+
const coerceToString = z.preprocess((val) => {
|
|
154
|
+
if (typeof val === "string") return val;
|
|
155
|
+
if (val !== null && typeof val === "object") return Object.entries(val).map(([k, v]) => `${k}: ${String(v)}`).join(", ");
|
|
156
|
+
return String(val);
|
|
157
|
+
}, z.string());
|
|
158
|
+
/**
|
|
159
|
+
* Schema for the YAML output contract of the dev-story sub-agent.
|
|
160
|
+
* The agent must emit YAML matching this shape.
|
|
161
|
+
*/
|
|
162
|
+
const DevStoryResultSchema = z.object({
|
|
163
|
+
result: z.preprocess((val) => val === "failure" ? "failed" : val, z.enum(["success", "failed"])),
|
|
164
|
+
ac_met: z.array(coerceToString).default([]),
|
|
165
|
+
ac_failures: z.array(coerceToString).default([]),
|
|
166
|
+
files_modified: z.array(z.string()).default([]),
|
|
167
|
+
tests: z.preprocess((val) => {
|
|
168
|
+
if (typeof val === "string") {
|
|
169
|
+
const lower = val.toLowerCase();
|
|
170
|
+
if (lower.includes("fail")) return "fail";
|
|
171
|
+
return "pass";
|
|
172
|
+
}
|
|
173
|
+
if (val !== null && typeof val === "object") {
|
|
174
|
+
const obj = val;
|
|
175
|
+
if (typeof obj.fail === "number" && obj.fail > 0) return "fail";
|
|
176
|
+
return "pass";
|
|
177
|
+
}
|
|
178
|
+
if (typeof val === "number") return val > 0 ? "fail" : "pass";
|
|
179
|
+
return "pass";
|
|
180
|
+
}, z.enum(["pass", "fail"])),
|
|
181
|
+
notes: z.string().optional()
|
|
182
|
+
});
|
|
183
|
+
/**
|
|
184
|
+
* Coerce a YAML value to a number. Agents sometimes emit line numbers
|
|
185
|
+
* as strings (`"42"` instead of `42`). This handles the conversion.
|
|
186
|
+
*/
|
|
187
|
+
const coerceOptionalNumber = z.preprocess((val) => typeof val === "string" ? Number(val) : val, z.number().optional());
|
|
188
|
+
const coerceNumber = z.preprocess((val) => {
|
|
189
|
+
if (typeof val === "string") return Number(val);
|
|
190
|
+
if (Array.isArray(val)) return val.length;
|
|
191
|
+
return val;
|
|
192
|
+
}, z.number());
|
|
193
|
+
/**
|
|
194
|
+
* Schema for a single issue in the code review output.
|
|
195
|
+
*/
|
|
196
|
+
const CodeReviewIssueSchema = z.object({
|
|
197
|
+
severity: z.enum([
|
|
198
|
+
"blocker",
|
|
199
|
+
"major",
|
|
200
|
+
"minor"
|
|
201
|
+
]),
|
|
202
|
+
description: z.string(),
|
|
203
|
+
file: z.string().optional(),
|
|
204
|
+
line: coerceOptionalNumber
|
|
205
|
+
});
|
|
206
|
+
/**
|
|
207
|
+
* Schema for a single entry in the AC verification checklist emitted by
|
|
208
|
+
* the code-review sub-agent. Each entry maps an acceptance criterion to a
|
|
209
|
+
* status determination backed by concrete evidence from the diff.
|
|
210
|
+
*/
|
|
211
|
+
const AcChecklistEntrySchema = z.object({
|
|
212
|
+
ac_id: z.string().default(""),
|
|
213
|
+
status: z.enum([
|
|
214
|
+
"met",
|
|
215
|
+
"not_met",
|
|
216
|
+
"partial"
|
|
217
|
+
]).default("met"),
|
|
218
|
+
evidence: z.string().default("")
|
|
219
|
+
});
|
|
220
|
+
/**
|
|
221
|
+
* Compute the verdict from the issue list using deterministic rules.
|
|
222
|
+
*
|
|
223
|
+
* The agent reports issues with severities; the pipeline computes the
|
|
224
|
+
* verdict. This decouples model-routing cost decisions (MAJOR_REWORK
|
|
225
|
+
* triggers opus) from agent judgment, and scales naturally with story
|
|
226
|
+
* size — severity classification is per-issue, not per-story.
|
|
227
|
+
*
|
|
228
|
+
* Rules:
|
|
229
|
+
* - Any blocker → NEEDS_MAJOR_REWORK (security, data loss, architectural breakage)
|
|
230
|
+
* - Any major or minor issues → NEEDS_MINOR_FIXES (fixable by sonnet with guidance)
|
|
231
|
+
* - No issues → SHIP_IT
|
|
232
|
+
*
|
|
233
|
+
* Note: LGTM_WITH_NOTES is handled in the transform (not here) because it
|
|
234
|
+
* requires knowledge of the agent's original verdict.
|
|
235
|
+
*/
|
|
236
|
+
function computeVerdict(issueList) {
|
|
237
|
+
const hasBlocker = issueList.some((i) => i.severity === "blocker");
|
|
238
|
+
if (hasBlocker) return "NEEDS_MAJOR_REWORK";
|
|
239
|
+
if (issueList.length > 0) return "NEEDS_MINOR_FIXES";
|
|
240
|
+
return "SHIP_IT";
|
|
241
|
+
}
|
|
242
|
+
/**
|
|
243
|
+
* Schema for the YAML output contract of the code-review sub-agent.
|
|
244
|
+
*
|
|
245
|
+
* The agent must emit YAML with verdict, issues count, issue_list, and
|
|
246
|
+
* optionally an ac_checklist mapping each AC to its implementation status.
|
|
247
|
+
* Example:
|
|
248
|
+
* verdict: NEEDS_MINOR_FIXES
|
|
249
|
+
* issues: 3
|
|
250
|
+
* issue_list:
|
|
251
|
+
* - severity: minor
|
|
252
|
+
* description: "Missing error handling in createFoo()"
|
|
253
|
+
* file: "src/modules/foo/foo.ts"
|
|
254
|
+
* line: 42
|
|
255
|
+
* ac_checklist:
|
|
256
|
+
* - ac_id: AC1
|
|
257
|
+
* status: met
|
|
258
|
+
* evidence: "Implemented in src/modules/foo/foo.ts:createFoo()"
|
|
259
|
+
* - ac_id: AC2
|
|
260
|
+
* status: not_met
|
|
261
|
+
* evidence: "No implementation found for getConstraints()"
|
|
262
|
+
* notes: "Generally clean implementation."
|
|
263
|
+
*
|
|
264
|
+
* The transform:
|
|
265
|
+
* 1. Auto-corrects the issues count from issue_list length.
|
|
266
|
+
* 2. Auto-injects a major issue for each not_met AC not already flagged.
|
|
267
|
+
* 3. Recomputes the verdict from the (possibly augmented) issue list.
|
|
268
|
+
* The agent's original verdict is preserved as `agentVerdict` for logging.
|
|
269
|
+
*/
|
|
270
|
+
const CodeReviewResultSchema = z.object({
|
|
271
|
+
verdict: z.enum([
|
|
272
|
+
"SHIP_IT",
|
|
273
|
+
"NEEDS_MINOR_FIXES",
|
|
274
|
+
"NEEDS_MAJOR_REWORK",
|
|
275
|
+
"LGTM_WITH_NOTES"
|
|
276
|
+
]),
|
|
277
|
+
issues: coerceNumber,
|
|
278
|
+
issue_list: z.array(CodeReviewIssueSchema),
|
|
279
|
+
ac_checklist: z.array(AcChecklistEntrySchema).default([]),
|
|
280
|
+
notes: z.string().optional()
|
|
281
|
+
}).transform((data) => {
|
|
282
|
+
const augmentedIssueList = [...data.issue_list];
|
|
283
|
+
for (const entry of data.ac_checklist) if (entry.status === "not_met") {
|
|
284
|
+
const acIdPattern = new RegExp(`\\b${entry.ac_id}\\b`);
|
|
285
|
+
const alreadyFlagged = augmentedIssueList.some((issue) => (issue.severity === "major" || issue.severity === "blocker") && acIdPattern.test(issue.description));
|
|
286
|
+
if (!alreadyFlagged) augmentedIssueList.push({
|
|
287
|
+
severity: "major",
|
|
288
|
+
description: `${entry.ac_id} not implemented: ${entry.evidence}`,
|
|
289
|
+
file: ""
|
|
290
|
+
});
|
|
291
|
+
}
|
|
292
|
+
const computedVerdict = computeVerdict(augmentedIssueList);
|
|
293
|
+
const finalVerdict = data.verdict === "LGTM_WITH_NOTES" && computedVerdict === "SHIP_IT" ? "LGTM_WITH_NOTES" : computedVerdict;
|
|
294
|
+
return {
|
|
295
|
+
...data,
|
|
296
|
+
issue_list: augmentedIssueList,
|
|
297
|
+
issues: augmentedIssueList.length,
|
|
298
|
+
agentVerdict: data.verdict,
|
|
299
|
+
verdict: finalVerdict
|
|
300
|
+
};
|
|
301
|
+
});
|
|
302
|
+
/**
|
|
303
|
+
* Schema for the YAML output contract of the test-plan sub-agent.
|
|
304
|
+
*
|
|
305
|
+
* The agent must emit YAML with result, test_files, test_categories, and coverage_notes.
|
|
306
|
+
* Example:
|
|
307
|
+
* result: success
|
|
308
|
+
* test_files:
|
|
309
|
+
* - src/modules/foo/__tests__/foo.test.ts
|
|
310
|
+
* test_categories:
|
|
311
|
+
* - unit
|
|
312
|
+
* - integration
|
|
313
|
+
* coverage_notes: "AC1 covered by foo.test.ts"
|
|
314
|
+
*/
|
|
315
|
+
const TestPlanResultSchema = z.object({
|
|
316
|
+
result: z.preprocess((val) => val === "failure" ? "failed" : val, z.enum(["success", "failed"])),
|
|
317
|
+
test_files: z.array(z.string()).default([]),
|
|
318
|
+
test_categories: z.array(z.string()).default([]),
|
|
319
|
+
coverage_notes: z.string().default("")
|
|
320
|
+
});
|
|
321
|
+
/**
|
|
322
|
+
* Schema for a single coverage gap identified during test expansion analysis.
|
|
323
|
+
*/
|
|
324
|
+
const CoverageGapSchema = z.object({
|
|
325
|
+
ac_ref: z.string(),
|
|
326
|
+
description: z.string(),
|
|
327
|
+
gap_type: z.enum([
|
|
328
|
+
"missing-e2e",
|
|
329
|
+
"missing-integration",
|
|
330
|
+
"unit-only"
|
|
331
|
+
])
|
|
332
|
+
});
|
|
333
|
+
/**
|
|
334
|
+
* Schema for a single suggested test generated during test expansion analysis.
|
|
335
|
+
*/
|
|
336
|
+
const SuggestedTestSchema = z.object({
|
|
337
|
+
test_name: z.string(),
|
|
338
|
+
test_type: z.enum([
|
|
339
|
+
"e2e",
|
|
340
|
+
"integration",
|
|
341
|
+
"unit"
|
|
342
|
+
]),
|
|
343
|
+
description: z.string(),
|
|
344
|
+
target_ac: z.string().optional()
|
|
345
|
+
});
|
|
346
|
+
/**
|
|
347
|
+
* Schema for the YAML output contract of the test-expansion sub-agent.
|
|
348
|
+
*
|
|
349
|
+
* The agent must emit YAML with expansion_priority, coverage_gaps, and suggested_tests.
|
|
350
|
+
* Example:
|
|
351
|
+
* expansion_priority: medium
|
|
352
|
+
* coverage_gaps:
|
|
353
|
+
* - ac_ref: AC1
|
|
354
|
+
* description: "Happy path not exercised at module boundary"
|
|
355
|
+
* gap_type: missing-integration
|
|
356
|
+
* suggested_tests:
|
|
357
|
+
* - test_name: "runFoo integration happy path"
|
|
358
|
+
* test_type: integration
|
|
359
|
+
* description: "Test runFoo with real DB to verify AC1 end-to-end"
|
|
360
|
+
* target_ac: AC1
|
|
361
|
+
* notes: "Unit coverage is solid but integration layer is untested."
|
|
362
|
+
*/
|
|
363
|
+
const TestExpansionResultSchema = z.object({
|
|
364
|
+
expansion_priority: z.preprocess((val) => [
|
|
365
|
+
"low",
|
|
366
|
+
"medium",
|
|
367
|
+
"high"
|
|
368
|
+
].includes(val) ? val : "low", z.enum([
|
|
369
|
+
"low",
|
|
370
|
+
"medium",
|
|
371
|
+
"high"
|
|
372
|
+
])),
|
|
373
|
+
coverage_gaps: z.array(CoverageGapSchema).default([]),
|
|
374
|
+
suggested_tests: z.array(SuggestedTestSchema).default([]),
|
|
375
|
+
notes: z.string().optional()
|
|
376
|
+
});
|
|
377
|
+
/**
|
|
378
|
+
* Inline mirror of RuntimeProbeSchema from @substrate-ai/sdlc.
|
|
379
|
+
*
|
|
380
|
+
* Inlined here instead of imported to avoid triggering Vitest's mock validation
|
|
381
|
+
* error in orchestrator tests that mock @substrate-ai/sdlc without including
|
|
382
|
+
* RuntimeProbeListSchema. The canonical definition lives in
|
|
383
|
+
* packages/sdlc/src/verification/probes/types.ts — keep in sync when that
|
|
384
|
+
* schema changes.
|
|
385
|
+
*/
|
|
386
|
+
const InlineRuntimeProbeSchema = z.object({
|
|
387
|
+
name: z.string().min(1, "probe name is required"),
|
|
388
|
+
sandbox: z.enum(["host", "twin"]),
|
|
389
|
+
command: z.string().min(1, "probe command is required"),
|
|
390
|
+
timeout_ms: z.number().int().positive().optional(),
|
|
391
|
+
description: z.string().optional(),
|
|
392
|
+
expect_stdout_no_regex: z.array(z.string().min(1)).optional(),
|
|
393
|
+
expect_stdout_regex: z.array(z.string().min(1)).optional()
|
|
394
|
+
});
|
|
395
|
+
const InlineRuntimeProbeListSchema = z.array(InlineRuntimeProbeSchema);
|
|
396
|
+
/**
|
|
397
|
+
* Schema for the YAML output contract of the probe-author sub-agent.
|
|
398
|
+
*
|
|
399
|
+
* The agent emits a yaml block containing result and probes list.
|
|
400
|
+
* The probes field is validated against the RuntimeProbe shape from @substrate-ai/sdlc
|
|
401
|
+
* (inlined to avoid module mock conflicts in orchestrator unit tests).
|
|
402
|
+
*
|
|
403
|
+
* Example:
|
|
404
|
+
* result: success
|
|
405
|
+
* probes:
|
|
406
|
+
* - name: my-probe
|
|
407
|
+
* sandbox: host
|
|
408
|
+
* command: echo "hello"
|
|
409
|
+
*/
|
|
410
|
+
const ProbeAuthorResultSchema = z.object({
|
|
411
|
+
result: z.preprocess((val) => val === "failure" ? "failed" : val, z.enum(["success", "failed"])),
|
|
412
|
+
probes: InlineRuntimeProbeListSchema
|
|
413
|
+
});
|
|
414
|
+
/**
|
|
415
|
+
* Per-end-state verdict from the acceptance judge.
|
|
416
|
+
*
|
|
417
|
+
* UNREACHABLE is first-class and distinct from FAIL: the walk could not even
|
|
418
|
+
* be attempted because the affordance does not exist in the rendered surface
|
|
419
|
+
* — it is what the income-sources UJ-2 failure WAS. Evidence is MANDATORY:
|
|
420
|
+
* a verdict that cannot cite the artifact region it grounds on is
|
|
421
|
+
* schema-invalid (naive prose judging is banned by construction — the
|
|
422
|
+
* documented LLM-judge biases make ungrounded verdicts worthless).
|
|
423
|
+
*/
|
|
424
|
+
const AcceptanceJudgeVerdictSchema = z.object({
|
|
425
|
+
end_state_id: z.string().min(1),
|
|
426
|
+
verdict: z.preprocess((val) => typeof val === "string" ? val.toUpperCase() : val, z.enum([
|
|
427
|
+
"PASS",
|
|
428
|
+
"FAIL",
|
|
429
|
+
"UNREACHABLE"
|
|
430
|
+
])),
|
|
431
|
+
evidence: z.object({
|
|
432
|
+
artifact: z.string().min(1),
|
|
433
|
+
excerpt: z.string().min(1)
|
|
434
|
+
}),
|
|
435
|
+
reasoning: z.string().optional()
|
|
436
|
+
});
|
|
437
|
+
/**
|
|
438
|
+
* Schema for the YAML output contract of the acceptance-judge sub-agent.
|
|
439
|
+
*/
|
|
440
|
+
const AcceptanceJudgeResultSchema = z.object({
|
|
441
|
+
result: z.preprocess((val) => val === "failure" ? "failed" : val, z.enum(["success", "failed"])),
|
|
442
|
+
verdicts: z.array(AcceptanceJudgeVerdictSchema).default([]),
|
|
443
|
+
error: z.string().optional()
|
|
444
|
+
});
|
|
445
|
+
|
|
446
|
+
//#endregion
|
|
447
|
+
//#region src/modules/compiled-workflows/token-ceiling.ts
|
|
448
|
+
/**
|
|
449
|
+
* Default token ceilings for each compiled workflow.
|
|
450
|
+
* These match the hardcoded constants previously defined inline in each workflow.
|
|
451
|
+
*/
|
|
452
|
+
const TOKEN_CEILING_DEFAULTS = {
|
|
453
|
+
"create-story": 5e4,
|
|
454
|
+
"dev-story": 4e5,
|
|
455
|
+
"code-review": 5e5,
|
|
456
|
+
"test-plan": 1e5,
|
|
457
|
+
"test-expansion": 2e5,
|
|
458
|
+
"probe-author": 5e4,
|
|
459
|
+
"acceptance-judge": 1e5
|
|
460
|
+
};
|
|
461
|
+
/**
|
|
462
|
+
* Resolve the effective token ceiling for a workflow type.
|
|
463
|
+
*
|
|
464
|
+
* Returns the ceiling from `tokenCeilings` config if present and valid,
|
|
465
|
+
* otherwise falls back to the hardcoded default.
|
|
466
|
+
*
|
|
467
|
+
* @param workflowType - One of: 'create-story', 'dev-story', 'code-review', 'test-plan', 'test-expansion'
|
|
468
|
+
* @param tokenCeilings - Optional per-workflow overrides from parsed config
|
|
469
|
+
* @returns `{ ceiling: number, source: 'config' | 'default' }`
|
|
470
|
+
*/
|
|
471
|
+
function getTokenCeiling(workflowType, tokenCeilings) {
|
|
472
|
+
if (tokenCeilings !== void 0) {
|
|
473
|
+
const configured = tokenCeilings[workflowType];
|
|
474
|
+
if (configured !== void 0) return {
|
|
475
|
+
ceiling: configured,
|
|
476
|
+
source: "config"
|
|
477
|
+
};
|
|
478
|
+
}
|
|
479
|
+
const defaultValue = TOKEN_CEILING_DEFAULTS[workflowType] ?? 0;
|
|
480
|
+
return {
|
|
481
|
+
ceiling: defaultValue,
|
|
482
|
+
source: "default"
|
|
483
|
+
};
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
//#endregion
|
|
487
|
+
//#region src/modules/compiled-workflows/acceptance-judge.ts
|
|
488
|
+
const logger = createLogger("compiled-workflows:acceptance-judge");
|
|
489
|
+
/** Cap per-artifact content injected into the prompt (chars). */
|
|
490
|
+
const ARTIFACT_CONTENT_CAP = 16e3;
|
|
491
|
+
function renderEndStates(journey) {
|
|
492
|
+
return journey.end_states.map((es) => `- id: ${es.id}\n given: ${es.given}\n walk: ${es.walk}\n then: ${es.then}`).join("\n");
|
|
493
|
+
}
|
|
494
|
+
async function readArtifactContents(artifactsDir, artifacts) {
|
|
495
|
+
const map = new Map();
|
|
496
|
+
for (const rel of artifacts) try {
|
|
497
|
+
map.set(rel, await readFile(join(artifactsDir, rel), "utf-8"));
|
|
498
|
+
} catch {}
|
|
499
|
+
return map;
|
|
500
|
+
}
|
|
501
|
+
function renderArtifactContents(contents) {
|
|
502
|
+
const blocks = [];
|
|
503
|
+
for (const [rel, content] of contents) {
|
|
504
|
+
const truncated = content.length > ARTIFACT_CONTENT_CAP;
|
|
505
|
+
blocks.push(`--- ${rel}${truncated ? ` (truncated to first ${String(ARTIFACT_CONTENT_CAP)} chars)` : ""} ---\n` + content.slice(0, ARTIFACT_CONTENT_CAP));
|
|
506
|
+
}
|
|
507
|
+
return blocks.join("\n\n");
|
|
508
|
+
}
|
|
509
|
+
/** Validate verdict completeness: every end-state exactly once, no unknown ids. */
|
|
510
|
+
function validateVerdictCoverage(journey, verdicts) {
|
|
511
|
+
const expected = new Set(journey.end_states.map((es) => es.id));
|
|
512
|
+
const seen = new Set();
|
|
513
|
+
for (const v of verdicts) {
|
|
514
|
+
if (!expected.has(v.end_state_id)) return `verdict for unknown end-state id "${v.end_state_id}"`;
|
|
515
|
+
if (seen.has(v.end_state_id)) return `duplicate verdict for end-state "${v.end_state_id}"`;
|
|
516
|
+
seen.add(v.end_state_id);
|
|
517
|
+
}
|
|
518
|
+
const missing = [...expected].filter((id) => !seen.has(id));
|
|
519
|
+
if (missing.length > 0) return `missing verdict(s) for end-state(s): ${missing.join(", ")}`;
|
|
520
|
+
return void 0;
|
|
521
|
+
}
|
|
522
|
+
/**
|
|
523
|
+
* Normalize for substring grounding: strip HTML tags, decode a few common
|
|
524
|
+
* entities, collapse whitespace, lowercase. Tag-stripping is essential — a
|
|
525
|
+
* judge legitimately quotes RENDERED (visible) text, which in HTML is often
|
|
526
|
+
* split across tags (e.g. `<a>Grade 1</a> <a>Grade 2</a>` reads as
|
|
527
|
+
* "Grade 1 Grade 2"); without stripping, a correct PASS citation fails to
|
|
528
|
+
* ground. A fabricated injection excerpt still won't appear in the
|
|
529
|
+
* tag-stripped real content, so the anti-injection property holds.
|
|
530
|
+
*/
|
|
531
|
+
function normalizeForGrounding(s) {
|
|
532
|
+
return s.replace(/<[^>]*>/g, " ").replace(/ /gi, " ").replace(/&/gi, "&").replace(/</gi, "<").replace(/>/gi, ">").replace(/"/gi, "\"").replace(/'/gi, "'").replace(/\s+/g, " ").trim().toLowerCase();
|
|
533
|
+
}
|
|
534
|
+
/**
|
|
535
|
+
* A5.1 F7 (red-team): a PASS excerpt must be a VERBATIM substring of the named
|
|
536
|
+
* artifact — a deterministic check (no LLM) that a hostile render cannot
|
|
537
|
+
* satisfy the citation requirement by fabricating an excerpt alongside an
|
|
538
|
+
* injected "PASS". Grounding is scoped to PASS ONLY: that is exactly where the
|
|
539
|
+
* "mark everything PASS" injection lives, and it is the only verdict that makes
|
|
540
|
+
* a POSITIVE claim ("the affordance exists — here it is") which must be
|
|
541
|
+
* citable. FAIL and UNREACHABLE are NEGATIVE findings (the observable is wrong
|
|
542
|
+
* or absent) whose excerpt legitimately DESCRIBES an absence rather than
|
|
543
|
+
* quoting present text — requiring a verbatim substring there is a category
|
|
544
|
+
* error and false-positives correct judgments. A fabricated FAIL/UNREACHABLE
|
|
545
|
+
* cannot help an attacker (it blocks their own story), so leaving them
|
|
546
|
+
* unground-checked costs no security.
|
|
547
|
+
* Returns a problem string for the FIRST ungrounded PASS citation, or undefined.
|
|
548
|
+
*/
|
|
549
|
+
function validateEvidenceGrounding(verdicts, artifactContents) {
|
|
550
|
+
for (const v of verdicts) {
|
|
551
|
+
if (v.verdict !== "PASS") continue;
|
|
552
|
+
const content = artifactContents.get(v.evidence.artifact);
|
|
553
|
+
if (content === void 0) return `verdict ${v.end_state_id} cites artifact "${v.evidence.artifact}" which is not in the rendered set`;
|
|
554
|
+
const haystack = ` ${normalizeForGrounding(content)} ${content.toLowerCase().replace(/\s+/g, " ")} `;
|
|
555
|
+
const tokens = normalizeForGrounding(v.evidence.excerpt).split(" ").map((t) => t.replace(/[^a-z0-9]/g, "")).filter((t) => t.length >= 4);
|
|
556
|
+
if (tokens.length < 2) return `verdict ${v.end_state_id} cites too thin an excerpt to ground ("${v.evidence.excerpt}") — quote a substantive span (several words) of the artifact`;
|
|
557
|
+
const present = tokens.filter((t) => haystack.includes(t)).length;
|
|
558
|
+
const overlap = present / tokens.length;
|
|
559
|
+
if (overlap < .6) return `verdict ${v.end_state_id} excerpt does not appear in ${v.evidence.artifact} (only ${String(present)}/${String(tokens.length)} tokens present — fabricated citation?)`;
|
|
560
|
+
}
|
|
561
|
+
return void 0;
|
|
562
|
+
}
|
|
563
|
+
async function runAcceptanceJudge(deps, params) {
|
|
564
|
+
const { journey, artifactsDir, artifacts, storyKey } = params;
|
|
565
|
+
let template;
|
|
566
|
+
try {
|
|
567
|
+
template = await deps.pack.getPrompt("acceptance-judge");
|
|
568
|
+
} catch (err) {
|
|
569
|
+
return {
|
|
570
|
+
result: "failed",
|
|
571
|
+
error: `Failed to retrieve acceptance-judge prompt: ${err instanceof Error ? err.message : String(err)}`,
|
|
572
|
+
tokenUsage: {
|
|
573
|
+
input: 0,
|
|
574
|
+
output: 0
|
|
575
|
+
}
|
|
576
|
+
};
|
|
577
|
+
}
|
|
578
|
+
const artifactContentMap = await readArtifactContents(artifactsDir, artifacts);
|
|
579
|
+
const artifactContents = renderArtifactContents(artifactContentMap);
|
|
580
|
+
const buildPrompt = (correctivePreamble) => {
|
|
581
|
+
const { prompt } = assemblePrompt(template, [
|
|
582
|
+
{
|
|
583
|
+
name: "journey_id",
|
|
584
|
+
content: journey.id,
|
|
585
|
+
priority: "required"
|
|
586
|
+
},
|
|
587
|
+
{
|
|
588
|
+
name: "journey_title",
|
|
589
|
+
content: journey.title,
|
|
590
|
+
priority: "required"
|
|
591
|
+
},
|
|
592
|
+
{
|
|
593
|
+
name: "end_states",
|
|
594
|
+
content: renderEndStates(journey),
|
|
595
|
+
priority: "required"
|
|
596
|
+
},
|
|
597
|
+
{
|
|
598
|
+
name: "artifact_manifest",
|
|
599
|
+
content: artifacts.length > 0 ? artifacts.map((a) => `- ${a}`).join("\n") : "(no artifacts)",
|
|
600
|
+
priority: "required"
|
|
601
|
+
},
|
|
602
|
+
{
|
|
603
|
+
name: "artifact_contents",
|
|
604
|
+
content: artifactContents,
|
|
605
|
+
priority: "required"
|
|
606
|
+
}
|
|
607
|
+
], getTokenCeiling("acceptance-judge", deps.tokenCeilings).ceiling);
|
|
608
|
+
return correctivePreamble !== void 0 ? `${correctivePreamble}\n\n${prompt}` : prompt;
|
|
609
|
+
};
|
|
610
|
+
let totalTokens = {
|
|
611
|
+
input: 0,
|
|
612
|
+
output: 0
|
|
613
|
+
};
|
|
614
|
+
let lastProblem = "unknown";
|
|
615
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
616
|
+
const prompt = attempt === 0 ? buildPrompt() : buildPrompt(`PREVIOUS ATTEMPT REJECTED: ${lastProblem}. Emit ONLY the YAML block per the Output Contract — every end-state exactly once, every verdict with evidence {artifact, excerpt}.`);
|
|
617
|
+
const handle = deps.dispatcher.dispatch({
|
|
618
|
+
prompt,
|
|
619
|
+
agent: deps.agentId ?? "claude-code",
|
|
620
|
+
taskType: "acceptance-judge",
|
|
621
|
+
outputSchema: AcceptanceJudgeResultSchema,
|
|
622
|
+
maxTurns: 30,
|
|
623
|
+
workingDirectory: artifactsDir,
|
|
624
|
+
...deps.otlpEndpoint !== void 0 ? { otlpEndpoint: deps.otlpEndpoint } : {},
|
|
625
|
+
...storyKey !== void 0 ? { storyKey } : {}
|
|
626
|
+
});
|
|
627
|
+
let dispatchResult;
|
|
628
|
+
try {
|
|
629
|
+
dispatchResult = await handle.result;
|
|
630
|
+
} catch (err) {
|
|
631
|
+
return {
|
|
632
|
+
result: "failed",
|
|
633
|
+
error: `Dispatch error: ${err instanceof Error ? err.message : String(err)}`,
|
|
634
|
+
tokenUsage: totalTokens
|
|
635
|
+
};
|
|
636
|
+
}
|
|
637
|
+
totalTokens = {
|
|
638
|
+
input: totalTokens.input + dispatchResult.tokenEstimate.input,
|
|
639
|
+
output: totalTokens.output + dispatchResult.tokenEstimate.output
|
|
640
|
+
};
|
|
641
|
+
if (dispatchResult.status === "failed" || dispatchResult.status === "timeout") return {
|
|
642
|
+
result: "failed",
|
|
643
|
+
error: `Dispatch status: ${dispatchResult.status}. ${dispatchResult.parseError ?? ""}`.trim(),
|
|
644
|
+
tokenUsage: totalTokens
|
|
645
|
+
};
|
|
646
|
+
const parsed = AcceptanceJudgeResultSchema.safeParse(dispatchResult.parsed);
|
|
647
|
+
if (!parsed.success) {
|
|
648
|
+
if (process.env.SUBSTRATE_DEBUG === "acceptance-judge") process.stderr.write(`[judge-debug] status=${dispatchResult.status} exit=${String(dispatchResult.exitCode)} parseError=${String(dispatchResult.parseError)} outputLen=${String(dispatchResult.output?.length ?? 0)}\n[judge-debug] output-tail: ${(dispatchResult.output ?? "").slice(-800)}\n`);
|
|
649
|
+
lastProblem = `output failed schema validation (${parsed.error.issues.map((i) => `${i.path.join(".")}: ${i.message}`).join("; ").slice(0, 500)})`;
|
|
650
|
+
logger.warn({
|
|
651
|
+
journeyId: journey.id,
|
|
652
|
+
storyKey,
|
|
653
|
+
attempt,
|
|
654
|
+
lastProblem
|
|
655
|
+
}, "A2.1: judge output invalid");
|
|
656
|
+
continue;
|
|
657
|
+
}
|
|
658
|
+
if (parsed.data.result === "failed") return {
|
|
659
|
+
result: "failed",
|
|
660
|
+
error: "acceptance-judge-refused",
|
|
661
|
+
details: parsed.data.error ?? "judge reported failure without a reason",
|
|
662
|
+
tokenUsage: totalTokens
|
|
663
|
+
};
|
|
664
|
+
const coverageProblem = validateVerdictCoverage(journey, parsed.data.verdicts);
|
|
665
|
+
if (coverageProblem !== void 0) {
|
|
666
|
+
lastProblem = coverageProblem;
|
|
667
|
+
logger.warn({
|
|
668
|
+
journeyId: journey.id,
|
|
669
|
+
storyKey,
|
|
670
|
+
attempt,
|
|
671
|
+
coverageProblem
|
|
672
|
+
}, "A2.1: judge verdicts incomplete");
|
|
673
|
+
continue;
|
|
674
|
+
}
|
|
675
|
+
const groundingWarning = validateEvidenceGrounding(parsed.data.verdicts, artifactContentMap);
|
|
676
|
+
if (groundingWarning !== void 0) logger.warn({
|
|
677
|
+
journeyId: journey.id,
|
|
678
|
+
storyKey,
|
|
679
|
+
groundingWarning
|
|
680
|
+
}, "A5.1 F7: judge PASS citation weakly grounded (advisory — not blocking)");
|
|
681
|
+
logger.info({
|
|
682
|
+
journeyId: journey.id,
|
|
683
|
+
storyKey,
|
|
684
|
+
verdicts: parsed.data.verdicts.map((v) => `${v.end_state_id}=${v.verdict}`)
|
|
685
|
+
}, "A2.1: acceptance judge verdicts accepted");
|
|
686
|
+
return {
|
|
687
|
+
result: "success",
|
|
688
|
+
verdicts: parsed.data.verdicts,
|
|
689
|
+
tokenUsage: totalTokens
|
|
690
|
+
};
|
|
691
|
+
}
|
|
692
|
+
return {
|
|
693
|
+
result: "failed",
|
|
694
|
+
error: "acceptance-judge-invalid",
|
|
695
|
+
details: `judge produced invalid output twice — last problem: ${lastProblem}`,
|
|
696
|
+
tokenUsage: totalTokens
|
|
697
|
+
};
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
//#endregion
|
|
701
|
+
export { CodeReviewResultSchema, CreateStoryResultSchema, DevStoryResultSchema, ProbeAuthorResultSchema, TestExpansionResultSchema, TestPlanResultSchema, assemblePrompt, countTokens, getTokenCeiling, runAcceptanceJudge, truncateToTokens, validateEvidenceGrounding, validateVerdictCoverage };
|
|
702
|
+
//# sourceMappingURL=acceptance-judge-eIZZha1C.js.map
|