specdojo 0.3.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/dist/catalog-build-references.js +144 -0
- package/dist/catalog-build-render.js +144 -0
- package/dist/catalog-build.js +13 -290
- package/dist/catalog-generate.js +6 -9
- package/dist/catalog-plan-generate.js +2 -2
- package/dist/catalog-plan.js +6 -6
- package/dist/catalog-scaffold.js +4 -4
- package/dist/catalog.js +2 -2
- package/dist/dashboard-exec-records.js +105 -0
- package/dist/dashboard-routines.js +255 -0
- package/dist/dashboard-shared.js +14 -0
- package/dist/dashboard-timeline.js +175 -0
- package/dist/dashboard.js +10 -544
- package/dist/deliverable-trash.js +3 -3
- package/dist/doc-index.js +3 -3
- package/dist/exec-commands.js +557 -0
- package/dist/exec-core.js +457 -0
- package/dist/exec-plans-core.js +693 -0
- package/dist/exec-plans-generation.js +206 -0
- package/dist/exec-plans-storage.js +131 -0
- package/dist/exec-plans.js +3 -1018
- package/dist/exec-repo-integration-git.js +23 -0
- package/dist/exec-repo-integration-sync.js +264 -0
- package/dist/exec-repo-integration.js +4 -281
- package/dist/exec-reporter-output.js +206 -0
- package/dist/exec-reporter.js +2 -206
- package/dist/exec-results.js +22 -15
- package/dist/exec-run-agent-pipeline.js +256 -0
- package/dist/exec-run-agent.js +531 -0
- package/dist/exec-run-in-place.js +541 -0
- package/dist/exec-run-job.js +717 -0
- package/dist/exec-run-lifecycle-commands.js +522 -0
- package/dist/exec-run-mode-shared.js +140 -0
- package/dist/exec-run-register-command.js +244 -0
- package/dist/exec-run-register-core.js +386 -0
- package/dist/exec-run-register-mode.js +206 -0
- package/dist/exec-run-register-resume.js +393 -0
- package/dist/exec-run-register-state.js +322 -0
- package/dist/exec-run-register-worktree.js +509 -0
- package/dist/exec-run-schedule-mode.js +294 -0
- package/dist/exec-run-schedule-prepare.js +353 -0
- package/dist/exec-run-schedule-task.js +572 -0
- package/dist/exec-run-shared.js +399 -0
- package/dist/exec-run.js +11 -6182
- package/dist/exec-shared.js +2 -2
- package/dist/exec-trial-commands.js +558 -0
- package/dist/exec-trial-core.js +371 -0
- package/dist/exec-trial.js +2 -921
- package/dist/exec-worktree-ops-commit.js +707 -0
- package/dist/exec-worktree-ops-lifecycle.js +292 -0
- package/dist/exec-worktree-ops-merge.js +303 -0
- package/dist/exec-worktree-ops.js +3 -1292
- package/dist/exec-worktree.js +32 -4
- package/dist/exec.js +2 -1006
- package/dist/frontmatter-namespace.js +2 -2
- package/dist/grade-apply.js +734 -0
- package/dist/grade-command-apply.js +250 -0
- package/dist/grade-command-inspect.js +138 -0
- package/dist/grade-command-plan.js +101 -0
- package/dist/grade-command-shared.js +68 -0
- package/dist/grade-core.js +307 -0
- package/dist/grade-markdown.js +274 -0
- package/dist/grade-pipeline-state.js +184 -0
- package/dist/grade-plan.js +507 -0
- package/dist/grade-result.js +5 -5
- package/dist/grade-target-discovery.js +265 -0
- package/dist/grade-targets.js +432 -0
- package/dist/grade.js +14 -3188
- package/dist/init-command.js +7 -7
- package/dist/job-definition.js +670 -0
- package/dist/job.js +9 -673
- package/dist/kata-command.js +2 -2
- package/dist/register-command-types.js +1 -0
- package/dist/register-core.js +363 -0
- package/dist/register-create-commands.js +245 -0
- package/dist/register-events.js +4 -4
- package/dist/register-item.js +6 -6
- package/dist/register-maintenance-commands.js +187 -0
- package/dist/register-migration.js +268 -0
- package/dist/register-renumber.js +314 -0
- package/dist/register-transition-commands.js +248 -0
- package/dist/register-update.js +230 -0
- package/dist/register-views.js +256 -0
- package/dist/register.js +61 -2069
- package/dist/review-plan.js +3 -3
- package/dist/routine-definition.js +401 -0
- package/dist/routine.js +6 -402
- package/dist/schedule-approach.js +2 -2
- package/dist/schedule-strategy-generate-document.js +273 -0
- package/dist/schedule-strategy-generate-scope.js +199 -0
- package/dist/schedule-strategy-generate.js +9 -473
- package/dist/schedule.js +4 -4
- package/dist/specdojo-config.js +3 -4
- package/dist/viewpoint-document-kinds-check.js +3 -3
- package/dist/yaml-pages.js +4 -4
- package/dist/yaml.js +28 -0
- package/docs/ja/specdojo/guides/exec-config-guide.md +6 -6
- package/docs/ja/specdojo/guides/orchestrator-operation-guide.md +20 -18
- package/docs/ja/specdojo/guides/routine-operation-guide.md +1 -1
- package/package.json +4 -5
- package/templates/antigravity/README.md +3 -3
- package/templates/antigravity/exec-defaults-snippet.yaml +6 -4
- package/templates/init-exec-defaults.yaml +3 -1
|
@@ -0,0 +1,371 @@
|
|
|
1
|
+
import { spawn } from "node:child_process";
|
|
2
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
3
|
+
import { existsSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
|
4
|
+
import { dirname, join, relative, resolve, sep } from "node:path";
|
|
5
|
+
import { agentProtectedConfigViolation, captureAgentProtectedConfigSnapshot, changedAgentProtectedConfigPaths, } from "./exec-agent-protected-config.js";
|
|
6
|
+
import { agentGitStateViolation, captureAgentGitStateSnapshot, changedAgentGitStateFields, } from "./exec-agent-git-state.js";
|
|
7
|
+
import { ensureDir, safeSlug } from "./exec-shared.js";
|
|
8
|
+
import { gitEnvironment, gitResult, worktreeNameFromTaskId } from "./exec-worktree.js";
|
|
9
|
+
import { getProjectSchedulePath, loadConfig, specdojoRootDir, } from "./specdojo-config.js";
|
|
10
|
+
export const TRIAL_PROMPT_SUFFIX = `
|
|
11
|
+
|
|
12
|
+
---
|
|
13
|
+
|
|
14
|
+
# Agent comparison trial instructions
|
|
15
|
+
|
|
16
|
+
This is an isolated comparison trial. Edit and validate the artifacts required by the plan, but
|
|
17
|
+
do not update task lifecycle state or create/update the result file. End the final response with
|
|
18
|
+
exactly one machine-readable executor evidence envelope as documented by SpecDojo.
|
|
19
|
+
`;
|
|
20
|
+
export function sha256(value) {
|
|
21
|
+
return createHash("sha256").update(value).digest("hex");
|
|
22
|
+
}
|
|
23
|
+
function normalizePlanPathCandidate(value) {
|
|
24
|
+
const candidate = value
|
|
25
|
+
.trim()
|
|
26
|
+
.replace(/^\.\//u, "")
|
|
27
|
+
.replace(/:\d+(?::\d+)?$/u, "")
|
|
28
|
+
.replace(/[.,;]$/u, "");
|
|
29
|
+
if (!candidate ||
|
|
30
|
+
candidate.startsWith("/") ||
|
|
31
|
+
candidate.includes("://") ||
|
|
32
|
+
/[\s<>{}*?$]/u.test(candidate) ||
|
|
33
|
+
candidate === ".env" ||
|
|
34
|
+
candidate.startsWith("secrets/")) {
|
|
35
|
+
return null;
|
|
36
|
+
}
|
|
37
|
+
if (/^(?:\.specdojo|docs|scripts|src|tests|tools)\//u.test(candidate) ||
|
|
38
|
+
/^(?:lefthook\.yml|package\.json|tsconfig[^/]*\.json|vitest[^/]*\.ts)$/u.test(candidate)) {
|
|
39
|
+
return candidate;
|
|
40
|
+
}
|
|
41
|
+
return null;
|
|
42
|
+
}
|
|
43
|
+
export function extractPlanRepoPaths(plan) {
|
|
44
|
+
const candidates = [];
|
|
45
|
+
for (const match of plan.matchAll(/`([^`\n]+)`/gu))
|
|
46
|
+
candidates.push(match[1]);
|
|
47
|
+
for (const match of plan.matchAll(/\]\(([^)\s]+)(?:\s+[^)]*)?\)/gu))
|
|
48
|
+
candidates.push(match[1]);
|
|
49
|
+
return [
|
|
50
|
+
...new Set(candidates
|
|
51
|
+
.map(normalizePlanPathCandidate)
|
|
52
|
+
.filter((candidate) => candidate !== null)),
|
|
53
|
+
].sort();
|
|
54
|
+
}
|
|
55
|
+
function objectExistsAtCommit(repoRoot, commit, path) {
|
|
56
|
+
return gitResult(repoRoot, ["cat-file", "-e", `${commit}:${path}`]).status === 0;
|
|
57
|
+
}
|
|
58
|
+
export function buildTrialBaseValidation(params) {
|
|
59
|
+
const referencedPaths = extractPlanRepoPaths(params.planContent).map((path) => ({
|
|
60
|
+
path,
|
|
61
|
+
exists_at_base: objectExistsAtCommit(params.repoRoot, params.resolvedCommit, path),
|
|
62
|
+
}));
|
|
63
|
+
const missingReferencedPaths = referencedPaths
|
|
64
|
+
.filter((entry) => !entry.exists_at_base)
|
|
65
|
+
.map((entry) => entry.path);
|
|
66
|
+
const planPathExistsAtBase = objectExistsAtCommit(params.repoRoot, params.resolvedCommit, params.planPath);
|
|
67
|
+
const warnings = [];
|
|
68
|
+
const baseIsAncestorOfHead = gitResult(params.repoRoot, [
|
|
69
|
+
"merge-base",
|
|
70
|
+
"--is-ancestor",
|
|
71
|
+
params.resolvedCommit,
|
|
72
|
+
params.headCommit,
|
|
73
|
+
]).status === 0;
|
|
74
|
+
if (!baseIsAncestorOfHead) {
|
|
75
|
+
warnings.push("The base commit is not an ancestor of HEAD; the current HEAD cannot be used as a linear reference result.");
|
|
76
|
+
}
|
|
77
|
+
if (!planPathExistsAtBase) {
|
|
78
|
+
warnings.push("The plan file is not present at the base commit; the current plan content will still be supplied to every trial.");
|
|
79
|
+
}
|
|
80
|
+
if (missingReferencedPaths.length > 0) {
|
|
81
|
+
warnings.push(`Referenced paths missing at the base commit: ${missingReferencedPaths.join(", ")}`);
|
|
82
|
+
}
|
|
83
|
+
return {
|
|
84
|
+
requested_revision: params.requestedRevision,
|
|
85
|
+
resolved_commit: params.resolvedCommit,
|
|
86
|
+
head_commit_at_start: params.headCommit,
|
|
87
|
+
base_is_ancestor_of_head: baseIsAncestorOfHead,
|
|
88
|
+
defaulted_to_head: params.defaultedToHead,
|
|
89
|
+
plan_path_exists_at_base: planPathExistsAtBase,
|
|
90
|
+
compatible: baseIsAncestorOfHead && missingReferencedPaths.length === 0,
|
|
91
|
+
referenced_paths: referencedPaths,
|
|
92
|
+
missing_referenced_paths: missingReferencedPaths,
|
|
93
|
+
warnings,
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
export function classifyReporterFailure(result, reason) {
|
|
97
|
+
if (result === "rate_limit")
|
|
98
|
+
return "rate_limit";
|
|
99
|
+
return reason.startsWith("reporter output invalid after")
|
|
100
|
+
? "invalid_output"
|
|
101
|
+
: "invocation_failure";
|
|
102
|
+
}
|
|
103
|
+
export function hasValidExecutorEnvelope(stdout) {
|
|
104
|
+
const matches = [
|
|
105
|
+
...stdout.matchAll(/<specdojo_executor_evidence>\s*([\s\S]*?)\s*<\/specdojo_executor_evidence>/gu),
|
|
106
|
+
];
|
|
107
|
+
const raw = matches.at(-1)?.[1];
|
|
108
|
+
if (!raw)
|
|
109
|
+
return false;
|
|
110
|
+
try {
|
|
111
|
+
const value = JSON.parse(raw);
|
|
112
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
113
|
+
return false;
|
|
114
|
+
const report = value;
|
|
115
|
+
return typeof report.final_message === "string" && Array.isArray(report.validations);
|
|
116
|
+
}
|
|
117
|
+
catch {
|
|
118
|
+
return false;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
export function repoRelative(repoRoot, path) {
|
|
122
|
+
const value = relative(repoRoot, path);
|
|
123
|
+
if (!value || value === ".." || value.startsWith(`..${sep}`)) {
|
|
124
|
+
throw new Error(`Path is outside repository root: ${path}`);
|
|
125
|
+
}
|
|
126
|
+
return value.split(sep).join("/");
|
|
127
|
+
}
|
|
128
|
+
export function comparisonDir(executionPath, comparisonId) {
|
|
129
|
+
return join(executionPath, "exec", "trials", safeSlug(comparisonId));
|
|
130
|
+
}
|
|
131
|
+
export function comparisonPath(executionPath, comparisonId) {
|
|
132
|
+
return join(comparisonDir(executionPath, comparisonId), "comparison.json");
|
|
133
|
+
}
|
|
134
|
+
export function writeComparison(path, value) {
|
|
135
|
+
ensureDir(dirname(path));
|
|
136
|
+
const temporary = `${path}.${process.pid}.${randomUUID()}.tmp`;
|
|
137
|
+
writeFileSync(temporary, `${JSON.stringify(value, null, 2)}\n`, "utf8");
|
|
138
|
+
renameSync(temporary, path);
|
|
139
|
+
}
|
|
140
|
+
export function readComparison(path) {
|
|
141
|
+
if (!existsSync(path))
|
|
142
|
+
throw new Error(`Comparison not found: ${path}`);
|
|
143
|
+
return JSON.parse(readFileSync(path, "utf8"));
|
|
144
|
+
}
|
|
145
|
+
export function configuredWorktreeBase(schedulePath) {
|
|
146
|
+
const { config } = loadConfig();
|
|
147
|
+
if (!config)
|
|
148
|
+
return undefined;
|
|
149
|
+
const rootDir = specdojoRootDir();
|
|
150
|
+
for (const project of Object.values(config.projects)) {
|
|
151
|
+
if (resolve(rootDir, getProjectSchedulePath(project)) === schedulePath) {
|
|
152
|
+
return project.run?.worktree_base;
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
return undefined;
|
|
156
|
+
}
|
|
157
|
+
export function parsePositiveInteger(value, flag) {
|
|
158
|
+
if (!value || !/^\d+$/.test(value) || Number(value) < 1) {
|
|
159
|
+
throw new Error(`${flag} must be a positive integer: ${value ?? ""}`);
|
|
160
|
+
}
|
|
161
|
+
return Number(value);
|
|
162
|
+
}
|
|
163
|
+
export function parseRating(value, flag) {
|
|
164
|
+
if (value === undefined)
|
|
165
|
+
return undefined;
|
|
166
|
+
if (!/^[1-5]$/.test(value))
|
|
167
|
+
throw new Error(`${flag} must be an integer from 1 to 5.`);
|
|
168
|
+
return Number(value);
|
|
169
|
+
}
|
|
170
|
+
export function resolveMember(roster, nickname, stageRole) {
|
|
171
|
+
const member = roster?.members.find((candidate) => candidate.type === "agent" && candidate.nickname === nickname);
|
|
172
|
+
if (!member)
|
|
173
|
+
throw new Error(`Agent nickname not found in agent-members.yaml: ${nickname}`);
|
|
174
|
+
if (member.disabled)
|
|
175
|
+
throw new Error(`Agent is disabled in agent-members.yaml: ${nickname}`);
|
|
176
|
+
if ((stageRole === "reporter" && member.stage_role !== "reporter") ||
|
|
177
|
+
(stageRole === "executor" && member.stage_role === "reporter")) {
|
|
178
|
+
throw new Error(`Agent must have stage_role: ${stageRole}: ${nickname}`);
|
|
179
|
+
}
|
|
180
|
+
return member;
|
|
181
|
+
}
|
|
182
|
+
export function spawnAgent(command, prompt, cwd, env) {
|
|
183
|
+
return new Promise((resolveOutcome) => {
|
|
184
|
+
const protectedConfigBefore = captureAgentProtectedConfigSnapshot(cwd);
|
|
185
|
+
const gitStateBefore = captureAgentGitStateSnapshot(cwd);
|
|
186
|
+
const child = spawn(command, { cwd, env, shell: true, stdio: ["pipe", "pipe", "pipe"] });
|
|
187
|
+
let stdout = "";
|
|
188
|
+
let stderr = "";
|
|
189
|
+
let settled = false;
|
|
190
|
+
child.stdin.on("error", () => undefined);
|
|
191
|
+
child.stdout.setEncoding("utf8");
|
|
192
|
+
child.stderr.setEncoding("utf8");
|
|
193
|
+
child.stdout.on("data", (chunk) => {
|
|
194
|
+
stdout += chunk;
|
|
195
|
+
process.stdout.write(chunk);
|
|
196
|
+
});
|
|
197
|
+
child.stderr.on("data", (chunk) => {
|
|
198
|
+
stderr += chunk;
|
|
199
|
+
process.stderr.write(chunk);
|
|
200
|
+
});
|
|
201
|
+
const finish = (outcome) => {
|
|
202
|
+
if (settled)
|
|
203
|
+
return;
|
|
204
|
+
settled = true;
|
|
205
|
+
const protectedChanges = changedAgentProtectedConfigPaths(cwd, protectedConfigBefore);
|
|
206
|
+
if (protectedChanges.length > 0) {
|
|
207
|
+
outcome.stderr += `${agentProtectedConfigViolation(protectedChanges)}\n`;
|
|
208
|
+
outcome.exitCode = 1;
|
|
209
|
+
}
|
|
210
|
+
const gitStateChanges = changedAgentGitStateFields(cwd, gitStateBefore);
|
|
211
|
+
if (gitStateChanges.length > 0) {
|
|
212
|
+
outcome.stderr += `${agentGitStateViolation(gitStateChanges)}\n`;
|
|
213
|
+
outcome.exitCode = 1;
|
|
214
|
+
}
|
|
215
|
+
resolveOutcome(outcome);
|
|
216
|
+
};
|
|
217
|
+
child.once("error", (error) => {
|
|
218
|
+
stderr += `${error.message}\n`;
|
|
219
|
+
finish({ exitCode: null, stdout, stderr });
|
|
220
|
+
});
|
|
221
|
+
child.once("close", (exitCode) => finish({ exitCode, stdout, stderr }));
|
|
222
|
+
child.stdin.end(prompt);
|
|
223
|
+
});
|
|
224
|
+
}
|
|
225
|
+
export function worktreeEnvironment(repoRoot, worktreePath, schedulePath, executionPath) {
|
|
226
|
+
return {
|
|
227
|
+
...gitEnvironment(),
|
|
228
|
+
SPECDOJO_SCHEDULE_PATH: resolve(worktreePath, repoRelative(repoRoot, schedulePath)),
|
|
229
|
+
SPECDOJO_EXECUTION_PATH: resolve(worktreePath, repoRelative(repoRoot, executionPath)),
|
|
230
|
+
};
|
|
231
|
+
}
|
|
232
|
+
function blankTrial(params) {
|
|
233
|
+
const trialId = safeSlug(params.trialId ?? params.agent);
|
|
234
|
+
const worktreeTaskId = `${params.projectId}:trial:${params.comparisonId}:${trialId}`;
|
|
235
|
+
const name = worktreeNameFromTaskId(worktreeTaskId);
|
|
236
|
+
return {
|
|
237
|
+
trial_id: trialId,
|
|
238
|
+
agent: params.agent,
|
|
239
|
+
reporter_agent: params.reporter ?? null,
|
|
240
|
+
worktree: resolve(params.worktreeBase, name),
|
|
241
|
+
branch: `exec/${name}`,
|
|
242
|
+
status: "pending",
|
|
243
|
+
started_at: null,
|
|
244
|
+
completed_at: null,
|
|
245
|
+
duration_ms: null,
|
|
246
|
+
executor: {
|
|
247
|
+
exit_code: null,
|
|
248
|
+
attempts: 0,
|
|
249
|
+
duration_ms: null,
|
|
250
|
+
structured_output: false,
|
|
251
|
+
validations_reported: 0,
|
|
252
|
+
validations_passed: 0,
|
|
253
|
+
validations_failed: 0,
|
|
254
|
+
validations_not_run: 0,
|
|
255
|
+
validations_requires_human: 0,
|
|
256
|
+
files_changed: 0,
|
|
257
|
+
evidence_ref: null,
|
|
258
|
+
final_message: "",
|
|
259
|
+
validations: [],
|
|
260
|
+
changes: [],
|
|
261
|
+
diff_summary: "",
|
|
262
|
+
},
|
|
263
|
+
parent_validation: {
|
|
264
|
+
configured_ids: [...(params.parentValidationIds ?? [])],
|
|
265
|
+
status: params.parentValidationIds?.length ? "not_run" : "not_configured",
|
|
266
|
+
duration_ms: null,
|
|
267
|
+
validations_passed: 0,
|
|
268
|
+
validations_failed: 0,
|
|
269
|
+
validations_not_run: 0,
|
|
270
|
+
},
|
|
271
|
+
reporter: {
|
|
272
|
+
status: "not_run",
|
|
273
|
+
structured_output: false,
|
|
274
|
+
format_attempts: 0,
|
|
275
|
+
format_retries: 0,
|
|
276
|
+
outcome: null,
|
|
277
|
+
output: null,
|
|
278
|
+
failure_category: null,
|
|
279
|
+
failure_reason: "",
|
|
280
|
+
},
|
|
281
|
+
subjective: {
|
|
282
|
+
judgment_quality: null,
|
|
283
|
+
writing_quality: null,
|
|
284
|
+
scope_adherence: null,
|
|
285
|
+
notes: "",
|
|
286
|
+
},
|
|
287
|
+
};
|
|
288
|
+
}
|
|
289
|
+
export function buildComparisonRecord(params) {
|
|
290
|
+
const reporterMode = params.pairs ? "paired" : params.reporter ? "shared" : "none";
|
|
291
|
+
const definitions = params.pairs ??
|
|
292
|
+
(params.agents ?? []).map((agent) => ({ executor: agent, reporter: params.reporter }));
|
|
293
|
+
const trialIds = definitions.map(({ executor, reporter }) => safeSlug(reporterMode === "paired" ? `${executor}-${reporter}` : executor));
|
|
294
|
+
if (new Set(trialIds).size !== trialIds.length) {
|
|
295
|
+
throw new Error("Trial executor/reporter selections must produce distinct trial ids.");
|
|
296
|
+
}
|
|
297
|
+
const baseValidation = params.baseValidation ??
|
|
298
|
+
{
|
|
299
|
+
requested_revision: "HEAD",
|
|
300
|
+
resolved_commit: params.baseCommit,
|
|
301
|
+
head_commit_at_start: params.baseCommit,
|
|
302
|
+
base_is_ancestor_of_head: true,
|
|
303
|
+
defaulted_to_head: true,
|
|
304
|
+
plan_path_exists_at_base: true,
|
|
305
|
+
compatible: true,
|
|
306
|
+
referenced_paths: [],
|
|
307
|
+
missing_referenced_paths: [],
|
|
308
|
+
warnings: [],
|
|
309
|
+
};
|
|
310
|
+
return {
|
|
311
|
+
schema_version: 1,
|
|
312
|
+
comparison_id: params.comparisonId,
|
|
313
|
+
project_id: params.projectId,
|
|
314
|
+
task_id: params.taskId,
|
|
315
|
+
status: "active",
|
|
316
|
+
created_at: params.createdAt,
|
|
317
|
+
completed_at: null,
|
|
318
|
+
base_commit: params.baseCommit,
|
|
319
|
+
base: baseValidation,
|
|
320
|
+
reporter_mode: reporterMode,
|
|
321
|
+
plan: {
|
|
322
|
+
path: params.planPath,
|
|
323
|
+
sha256: sha256(params.planContent),
|
|
324
|
+
prompt_sha256: sha256(params.prompt),
|
|
325
|
+
},
|
|
326
|
+
selected_trial_id: null,
|
|
327
|
+
trials: definitions.map(({ executor, reporter }, index) => blankTrial({
|
|
328
|
+
projectId: params.projectId,
|
|
329
|
+
comparisonId: params.comparisonId,
|
|
330
|
+
agent: executor,
|
|
331
|
+
trialId: trialIds[index],
|
|
332
|
+
reporter,
|
|
333
|
+
worktreeBase: params.worktreeBase,
|
|
334
|
+
parentValidationIds: params.parentValidationIds,
|
|
335
|
+
})),
|
|
336
|
+
agent_selection: {
|
|
337
|
+
policy: "manual",
|
|
338
|
+
target: "agent-members.yaml",
|
|
339
|
+
rationale: "A single comparison does not automatically change agent assignment. A human reviews objective and subjective results, then updates agent-members.yaml priority/capabilities when repeated evidence supports the change.",
|
|
340
|
+
},
|
|
341
|
+
};
|
|
342
|
+
}
|
|
343
|
+
export function updateObjectiveMetrics(trial, evidence, structuredOutput) {
|
|
344
|
+
trial.executor.structured_output = structuredOutput;
|
|
345
|
+
trial.executor.files_changed = evidence.diff_summary.files_changed;
|
|
346
|
+
trial.executor.final_message = evidence.final_message;
|
|
347
|
+
trial.executor.validations = evidence.validations;
|
|
348
|
+
trial.executor.changes = evidence.changes;
|
|
349
|
+
trial.executor.diff_summary = evidence.diff_summary.summary;
|
|
350
|
+
const executorValidations = evidence.validations.filter((validation) => validation.source !== "runner");
|
|
351
|
+
trial.executor.validations_reported = executorValidations.length;
|
|
352
|
+
trial.executor.validations_passed = 0;
|
|
353
|
+
trial.executor.validations_failed = 0;
|
|
354
|
+
trial.executor.validations_not_run = 0;
|
|
355
|
+
trial.executor.validations_requires_human = 0;
|
|
356
|
+
for (const validation of executorValidations) {
|
|
357
|
+
if (validation.status === "passed")
|
|
358
|
+
trial.executor.validations_passed++;
|
|
359
|
+
else if (validation.status === "failed")
|
|
360
|
+
trial.executor.validations_failed++;
|
|
361
|
+
else if (validation.status === "not_run")
|
|
362
|
+
trial.executor.validations_not_run++;
|
|
363
|
+
else if (validation.status === "requires_human") {
|
|
364
|
+
trial.executor.validations_requires_human++;
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
const parentValidations = evidence.validations.filter((validation) => validation.source === "runner");
|
|
368
|
+
trial.parent_validation.validations_passed = parentValidations.filter((validation) => validation.status === "passed").length;
|
|
369
|
+
trial.parent_validation.validations_failed = parentValidations.filter((validation) => validation.status === "failed").length;
|
|
370
|
+
trial.parent_validation.validations_not_run = parentValidations.filter((validation) => validation.status === "not_run").length;
|
|
371
|
+
}
|