specdojo 0.3.0 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/CHANGELOG.md +18 -0
  2. package/dist/catalog-build-references.js +144 -0
  3. package/dist/catalog-build-render.js +144 -0
  4. package/dist/catalog-build.js +13 -290
  5. package/dist/catalog-generate.js +6 -9
  6. package/dist/catalog-plan-generate.js +2 -2
  7. package/dist/catalog-plan.js +6 -6
  8. package/dist/catalog-scaffold.js +4 -4
  9. package/dist/catalog.js +2 -2
  10. package/dist/dashboard-exec-records.js +105 -0
  11. package/dist/dashboard-routines.js +255 -0
  12. package/dist/dashboard-shared.js +14 -0
  13. package/dist/dashboard-timeline.js +175 -0
  14. package/dist/dashboard.js +10 -544
  15. package/dist/deliverable-trash.js +3 -3
  16. package/dist/doc-index.js +3 -3
  17. package/dist/exec-commands.js +557 -0
  18. package/dist/exec-core.js +457 -0
  19. package/dist/exec-plans-core.js +693 -0
  20. package/dist/exec-plans-generation.js +206 -0
  21. package/dist/exec-plans-storage.js +131 -0
  22. package/dist/exec-plans.js +3 -1018
  23. package/dist/exec-repo-integration-git.js +23 -0
  24. package/dist/exec-repo-integration-sync.js +264 -0
  25. package/dist/exec-repo-integration.js +4 -281
  26. package/dist/exec-reporter-output.js +206 -0
  27. package/dist/exec-reporter.js +2 -206
  28. package/dist/exec-results.js +22 -15
  29. package/dist/exec-run-agent-pipeline.js +256 -0
  30. package/dist/exec-run-agent.js +531 -0
  31. package/dist/exec-run-in-place.js +541 -0
  32. package/dist/exec-run-job.js +717 -0
  33. package/dist/exec-run-lifecycle-commands.js +522 -0
  34. package/dist/exec-run-mode-shared.js +140 -0
  35. package/dist/exec-run-register-command.js +244 -0
  36. package/dist/exec-run-register-core.js +386 -0
  37. package/dist/exec-run-register-mode.js +206 -0
  38. package/dist/exec-run-register-resume.js +393 -0
  39. package/dist/exec-run-register-state.js +322 -0
  40. package/dist/exec-run-register-worktree.js +509 -0
  41. package/dist/exec-run-schedule-mode.js +294 -0
  42. package/dist/exec-run-schedule-prepare.js +353 -0
  43. package/dist/exec-run-schedule-task.js +572 -0
  44. package/dist/exec-run-shared.js +399 -0
  45. package/dist/exec-run.js +11 -6182
  46. package/dist/exec-shared.js +2 -2
  47. package/dist/exec-trial-commands.js +558 -0
  48. package/dist/exec-trial-core.js +371 -0
  49. package/dist/exec-trial.js +2 -921
  50. package/dist/exec-worktree-ops-commit.js +707 -0
  51. package/dist/exec-worktree-ops-lifecycle.js +292 -0
  52. package/dist/exec-worktree-ops-merge.js +303 -0
  53. package/dist/exec-worktree-ops.js +3 -1292
  54. package/dist/exec-worktree.js +32 -4
  55. package/dist/exec.js +2 -1006
  56. package/dist/frontmatter-namespace.js +2 -2
  57. package/dist/grade-apply.js +734 -0
  58. package/dist/grade-command-apply.js +250 -0
  59. package/dist/grade-command-inspect.js +138 -0
  60. package/dist/grade-command-plan.js +101 -0
  61. package/dist/grade-command-shared.js +68 -0
  62. package/dist/grade-core.js +307 -0
  63. package/dist/grade-markdown.js +274 -0
  64. package/dist/grade-pipeline-state.js +184 -0
  65. package/dist/grade-plan.js +507 -0
  66. package/dist/grade-result.js +5 -5
  67. package/dist/grade-target-discovery.js +265 -0
  68. package/dist/grade-targets.js +432 -0
  69. package/dist/grade.js +14 -3188
  70. package/dist/init-command.js +7 -7
  71. package/dist/job-definition.js +670 -0
  72. package/dist/job.js +9 -673
  73. package/dist/kata-command.js +2 -2
  74. package/dist/register-command-types.js +1 -0
  75. package/dist/register-core.js +363 -0
  76. package/dist/register-create-commands.js +245 -0
  77. package/dist/register-events.js +4 -4
  78. package/dist/register-item.js +6 -6
  79. package/dist/register-maintenance-commands.js +187 -0
  80. package/dist/register-migration.js +268 -0
  81. package/dist/register-renumber.js +314 -0
  82. package/dist/register-transition-commands.js +248 -0
  83. package/dist/register-update.js +230 -0
  84. package/dist/register-views.js +256 -0
  85. package/dist/register.js +61 -2069
  86. package/dist/review-plan.js +3 -3
  87. package/dist/routine-definition.js +401 -0
  88. package/dist/routine.js +6 -402
  89. package/dist/schedule-approach.js +2 -2
  90. package/dist/schedule-strategy-generate-document.js +273 -0
  91. package/dist/schedule-strategy-generate-scope.js +199 -0
  92. package/dist/schedule-strategy-generate.js +9 -473
  93. package/dist/schedule.js +4 -4
  94. package/dist/specdojo-config.js +3 -4
  95. package/dist/viewpoint-document-kinds-check.js +3 -3
  96. package/dist/yaml-pages.js +4 -4
  97. package/dist/yaml.js +28 -0
  98. package/docs/ja/specdojo/guides/exec-config-guide.md +6 -6
  99. package/docs/ja/specdojo/guides/orchestrator-operation-guide.md +20 -18
  100. package/docs/ja/specdojo/guides/routine-operation-guide.md +1 -1
  101. package/package.json +4 -5
  102. package/templates/antigravity/README.md +3 -3
  103. package/templates/antigravity/exec-defaults-snippet.yaml +6 -4
  104. package/templates/init-exec-defaults.yaml +3 -1
@@ -1,7 +1,7 @@
1
1
  import { randomBytes } from "node:crypto";
2
2
  import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
3
3
  import { basename, join, relative } from "node:path";
4
- import yaml from "js-yaml";
4
+ import { load } from "./yaml.js";
5
5
  import { specdojoRootDir } from "./specdojo-config.js";
6
6
  export function nowUtcIsoSeconds() {
7
7
  return new Date().toISOString().replace(/\.\d{3}Z$/, "Z");
@@ -202,7 +202,7 @@ export function writeJson(path, data) {
202
202
  writeFileSync(path, JSON.stringify(data, null, 2) + "\n", "utf8");
203
203
  }
204
204
  export function readYaml(path) {
205
- return yaml.load(readFileSync(path, "utf8"));
205
+ return load(readFileSync(path, "utf8"));
206
206
  }
207
207
  export function isSchYamlFilename(path) {
208
208
  const name = basename(path);
@@ -0,0 +1,558 @@
1
+ import { randomUUID } from "node:crypto";
2
+ import { copyFileSync, existsSync, readFileSync, writeFileSync } from "node:fs";
3
+ import { basename, dirname, join, resolve } from "node:path";
4
+ import { defaultExecDefaultsPath, loadExecDefaultsConfig, resolveMemberCommand, resolveRateLimitDetection, } from "./exec-agent-config.js";
5
+ import { agentProtectedConfigViolation, isAgentProtectedConfigPath, } from "./exec-agent-protected-config.js";
6
+ import { recordExecutorEvidence, recordParentValidationFailureOutput, } from "./exec-evidence.js";
7
+ import { parentValidationLabels, } from "./exec-parent-validation.js";
8
+ import { activateResolvedProjectPaths, resolveProjectPaths } from "./exec-project.js";
9
+ import { buildExecutorPrompt, isRateLimitError, loadRosterForExecutionPath, runConfiguredParentValidationBatch, } from "./exec-run.js";
10
+ import { parsePlanTaskIdentity } from "./exec-plans.js";
11
+ import { deriveReporterFacts, evidenceForReporter, runReporterWithFormatRetry, } from "./exec-reporter.js";
12
+ import { ensureDir, safeSlug } from "./exec-shared.js";
13
+ import { ensureExecWorktree, execBranchExists, findExecWorktree, gitOutput, gitResult, resolveWorktreeBase, } from "./exec-worktree.js";
14
+ import { specdojoRootDir } from "./specdojo-config.js";
15
+ import { TRIAL_PROMPT_SUFFIX, buildComparisonRecord, buildTrialBaseValidation, classifyReporterFailure, comparisonDir, comparisonPath, configuredWorktreeBase, hasValidExecutorEnvelope, parsePositiveInteger, parseRating, readComparison, repoRelative, resolveMember, spawnAgent, updateObjectiveMetrics, worktreeEnvironment, writeComparison, } from "./exec-trial-core.js";
16
+ async function runOneTrial(params) {
17
+ const { trial, worktree } = params;
18
+ const startedAt = new Date();
19
+ trial.status = "running";
20
+ trial.started_at = startedAt.toISOString();
21
+ writeComparison(params.recordPath, params.record);
22
+ const outcome = await spawnAgent(params.executorCommand, params.prompt, worktree.path, worktreeEnvironment(params.repoRoot, worktree.path, params.schedulePath, params.executionPath));
23
+ const executorCompletedAt = new Date();
24
+ trial.executor.duration_ms = executorCompletedAt.getTime() - startedAt.getTime();
25
+ trial.parent_validation.configured_ids = [...params.parentValidationIds];
26
+ let parentValidations = [];
27
+ let parentValidationFailureOutputs = [];
28
+ if (params.parentValidationIds.length === 0) {
29
+ trial.parent_validation.status = "not_configured";
30
+ }
31
+ else if (outcome.exitCode !== 0) {
32
+ trial.parent_validation.status = "not_run";
33
+ }
34
+ else {
35
+ const parentValidationStartedAt = Date.now();
36
+ const parentValidationBatch = await runConfiguredParentValidationBatch(params.execDefaults, worktree.path, {
37
+ label: `${params.record.task_id} (${trial.trial_id})`,
38
+ });
39
+ parentValidations = parentValidationBatch.validations;
40
+ parentValidationFailureOutputs = parentValidationBatch.failureOutputs;
41
+ trial.parent_validation.duration_ms = Date.now() - parentValidationStartedAt;
42
+ trial.parent_validation.status = parentValidations.some((validation) => validation.status !== "passed")
43
+ ? "failed"
44
+ : "passed";
45
+ }
46
+ const runId = `${safeSlug(params.record.comparison_id)}-${trial.trial_id}`;
47
+ const evidenceRecorded = recordExecutorEvidence({
48
+ repoRoot: params.repoRoot,
49
+ worktreePath: worktree.path,
50
+ executionPath: params.executionPath,
51
+ taskId: params.record.task_id,
52
+ runId,
53
+ actor: trial.agent,
54
+ status: outcome.exitCode === 0 ? "succeeded" : "failed",
55
+ startedAt: startedAt.toISOString(),
56
+ completedAt: executorCompletedAt.toISOString(),
57
+ exitCode: outcome.exitCode,
58
+ attempts: 1,
59
+ stdout: outcome.stdout,
60
+ stderr: outcome.stderr,
61
+ parentValidations,
62
+ });
63
+ evidenceRecorded.evidence = recordParentValidationFailureOutput({
64
+ worktreePath: worktree.path,
65
+ evidencePath: evidenceRecorded.evidencePath,
66
+ evidence: evidenceRecorded.evidence,
67
+ failureOutputs: parentValidationFailureOutputs,
68
+ });
69
+ const centralizedEvidenceDir = join(comparisonDir(params.executionPath, params.record.comparison_id), "trials", trial.trial_id);
70
+ ensureDir(centralizedEvidenceDir);
71
+ const centralizedEvidencePath = join(centralizedEvidenceDir, "evidence.json");
72
+ evidenceRecorded.evidence.log_refs = evidenceRecorded.evidence.log_refs.map((ref) => {
73
+ const logName = basename(ref.path);
74
+ const centralizedLogPath = join(centralizedEvidenceDir, logName);
75
+ copyFileSync(join(dirname(evidenceRecorded.evidencePath), logName), centralizedLogPath);
76
+ return { ...ref, path: repoRelative(params.repoRoot, centralizedLogPath) };
77
+ });
78
+ writeFileSync(centralizedEvidencePath, `${JSON.stringify(evidenceRecorded.evidence, null, 2)}\n`, "utf8");
79
+ trial.executor.exit_code = outcome.exitCode;
80
+ trial.executor.attempts = 1;
81
+ trial.executor.evidence_ref = repoRelative(params.repoRoot, centralizedEvidencePath);
82
+ updateObjectiveMetrics(trial, evidenceRecorded.evidence, hasValidExecutorEnvelope(outcome.stdout));
83
+ let reporterSucceeded = true;
84
+ if (outcome.exitCode === 0 && params.reporterCommand) {
85
+ const reporterEvidence = evidenceForReporter(evidenceRecorded.evidence, {
86
+ repoRoot: params.repoRoot,
87
+ worktreePath: worktree.path,
88
+ executionPath: params.executionPath,
89
+ });
90
+ const reporter = await runReporterWithFormatRetry({
91
+ plan: params.plan,
92
+ evidence: reporterEvidence,
93
+ mode: parsePlanTaskIdentity(params.plan)?.mode ?? "edit",
94
+ invoke: async (reporterPrompt) => {
95
+ const reporterOutcome = await spawnAgent(params.reporterCommand, reporterPrompt, worktree.path, worktreeEnvironment(params.repoRoot, worktree.path, params.schedulePath, params.executionPath));
96
+ const result = reporterOutcome.exitCode === 0
97
+ ? "success"
98
+ : isRateLimitError(reporterOutcome.exitCode, `${reporterOutcome.stdout}\n${reporterOutcome.stderr}`, params.reporterRateLimitDetection)
99
+ ? "rate_limit"
100
+ : "failure";
101
+ return {
102
+ result,
103
+ stdout: reporterOutcome.stdout,
104
+ stderr: reporterOutcome.stderr,
105
+ };
106
+ },
107
+ });
108
+ trial.reporter.format_attempts = reporter.formatAttempts;
109
+ trial.reporter.format_retries = Math.max(0, reporter.formatAttempts - 1);
110
+ if (reporter.result === "success") {
111
+ const facts = deriveReporterFacts(reporterEvidence, {
112
+ mode: parsePlanTaskIdentity(params.plan)?.mode ?? "edit",
113
+ });
114
+ trial.reporter.structured_output = true;
115
+ trial.reporter.outcome = facts.outcome;
116
+ trial.reporter.output = reporter.output;
117
+ reporterSucceeded = facts.outcome === "complete";
118
+ if (reporterSucceeded) {
119
+ trial.reporter.status = "succeeded";
120
+ }
121
+ else {
122
+ trial.reporter.status = "blocked";
123
+ trial.reporter.failure_category = "evidence_blocked";
124
+ trial.reporter.failure_reason = facts.block_reason;
125
+ }
126
+ }
127
+ else {
128
+ trial.reporter.status = "failed";
129
+ trial.reporter.failure_category = classifyReporterFailure(reporter.result, reporter.reason);
130
+ trial.reporter.failure_reason = reporter.reason;
131
+ reporterSucceeded = false;
132
+ }
133
+ }
134
+ const completedAt = new Date();
135
+ trial.completed_at = completedAt.toISOString();
136
+ trial.duration_ms = completedAt.getTime() - startedAt.getTime();
137
+ trial.status =
138
+ outcome.exitCode === 0 &&
139
+ trial.executor.validations_failed === 0 &&
140
+ trial.parent_validation.status !== "failed" &&
141
+ trial.parent_validation.status !== "not_run" &&
142
+ reporterSucceeded
143
+ ? "succeeded"
144
+ : "failed";
145
+ writeComparison(params.recordPath, params.record);
146
+ }
147
+ async function runPool(items, parallel) {
148
+ let next = 0;
149
+ const workers = Array.from({ length: Math.min(parallel, items.length) }, async () => {
150
+ while (next < items.length) {
151
+ const index = next++;
152
+ await items[index]();
153
+ }
154
+ });
155
+ await Promise.all(workers);
156
+ }
157
+ function parseTrialPair(value) {
158
+ const separator = value.indexOf("=");
159
+ const executor = value.slice(0, separator).trim();
160
+ const reporter = value.slice(separator + 1).trim();
161
+ if (separator < 1 || !executor || !reporter || reporter.includes("=")) {
162
+ throw new Error(`--pair must use <executor>=<reporter>: ${value}`);
163
+ }
164
+ return { executor, reporter };
165
+ }
166
+ function resolveTrialDefinitions(opts) {
167
+ const agentValues = (opts.agent ?? []).map((value) => value.trim()).filter(Boolean);
168
+ const pairValues = (opts.pair ?? []).map((value) => value.trim()).filter(Boolean);
169
+ if (agentValues.length > 0 && pairValues.length > 0) {
170
+ throw new Error("Use either --agent or --pair, not both.");
171
+ }
172
+ if (pairValues.length > 0) {
173
+ if (opts.reporterBy)
174
+ throw new Error("--pair cannot be combined with --reporter-by.");
175
+ const pairs = pairValues.map(parseTrialPair);
176
+ const keys = pairs.map((pair) => `${pair.executor}\0${pair.reporter}`);
177
+ if (new Set(keys).size < 2 || new Set(keys).size !== keys.length) {
178
+ throw new Error("exec trial run requires at least two distinct --pair values.");
179
+ }
180
+ return { agents: [...new Set(pairs.map((pair) => pair.executor))], pairs };
181
+ }
182
+ const agents = [...new Set(agentValues)];
183
+ if (agents.length < 2) {
184
+ throw new Error("exec trial run requires at least two distinct --agent values or --pair values.");
185
+ }
186
+ return { agents };
187
+ }
188
+ function resolveBaseCommit(repoRoot, revision) {
189
+ const result = gitResult(repoRoot, [
190
+ "rev-parse",
191
+ "--verify",
192
+ "--end-of-options",
193
+ `${revision}^{commit}`,
194
+ ]);
195
+ if (result.status !== 0)
196
+ throw new Error(`Trial base is not a commit: ${revision}`);
197
+ return String(result.stdout).trim();
198
+ }
199
+ async function runTrials(opts) {
200
+ const paths = resolveProjectPaths({ project: opts.project });
201
+ activateResolvedProjectPaths(paths);
202
+ const repoRoot = specdojoRootDir();
203
+ const projectId = paths.projectId ?? opts.project?.trim();
204
+ if (!projectId)
205
+ throw new Error("exec trial run requires a project id.");
206
+ const planPath = resolve(opts.plan);
207
+ if (!existsSync(planPath))
208
+ throw new Error(`Plan not found: ${planPath}`);
209
+ const planContent = readFileSync(planPath, "utf8");
210
+ const identity = parsePlanTaskIdentity(planContent);
211
+ if (!identity)
212
+ throw new Error("Trial plan must have SpecDojo task_id/mode/project_id frontmatter.");
213
+ if (identity.projectId && identity.projectId !== projectId) {
214
+ throw new Error(`Plan project_id ${identity.projectId} does not match --project ${projectId}.`);
215
+ }
216
+ const { agents, pairs } = resolveTrialDefinitions(opts);
217
+ const parallel = parsePositiveInteger(opts.parallel ?? "1", "--parallel");
218
+ const comparisonId = safeSlug(opts.comparison?.trim() ||
219
+ `${identity.taskId}-${new Date().toISOString()}-${randomUUID().slice(0, 8)}`);
220
+ const recordPath = comparisonPath(paths.executionPath, comparisonId);
221
+ if (existsSync(recordPath))
222
+ throw new Error(`Comparison already exists: ${comparisonId}`);
223
+ const worktreeBase = resolveWorktreeBase(repoRoot, opts.worktreeBase, configuredWorktreeBase(paths.schedulePath));
224
+ const roster = loadRosterForExecutionPath(paths.executionPath);
225
+ const defaults = loadExecDefaultsConfig(opts.execDefaults ?? defaultExecDefaultsPath(), paths.executionPath);
226
+ const executorCommands = new Map();
227
+ for (const agent of agents) {
228
+ const command = resolveMemberCommand(defaults, resolveMember(roster, agent, "executor"));
229
+ if (!command)
230
+ throw new Error(`Agent command could not be resolved: ${agent}`);
231
+ executorCommands.set(agent, command);
232
+ }
233
+ const reporterCommands = new Map();
234
+ const reporterRateLimitDetections = new Map();
235
+ if (opts.reporterBy) {
236
+ const reporterMember = resolveMember(roster, opts.reporterBy, "reporter");
237
+ const reporterCommand = resolveMemberCommand(defaults, reporterMember);
238
+ if (!reporterCommand)
239
+ throw new Error(`Reporter command could not be resolved: ${opts.reporterBy}`);
240
+ reporterCommands.set(opts.reporterBy, reporterCommand);
241
+ reporterRateLimitDetections.set(opts.reporterBy, resolveRateLimitDetection(defaults, reporterMember.provider));
242
+ }
243
+ for (const reporter of new Set(pairs?.map((pair) => pair.reporter) ?? [])) {
244
+ const reporterMember = resolveMember(roster, reporter, "reporter");
245
+ const reporterCommand = resolveMemberCommand(defaults, reporterMember);
246
+ if (!reporterCommand)
247
+ throw new Error(`Reporter command could not be resolved: ${reporter}`);
248
+ reporterCommands.set(reporter, reporterCommand);
249
+ reporterRateLimitDetections.set(reporter, resolveRateLimitDetection(defaults, reporterMember.provider));
250
+ }
251
+ const parentValidationEntries = defaults.pipeline?.parent_validations ?? [];
252
+ // Labels are `<id>` for the project repository and `<repo>:<id>` for a product repository.
253
+ const parentValidationIds = parentValidationLabels(parentValidationEntries);
254
+ const prompt = `${buildExecutorPrompt(planContent, parentValidationEntries).trimEnd()}${TRIAL_PROMPT_SUFFIX}`;
255
+ const requestedRevision = opts.base?.trim() || "HEAD";
256
+ const headCommit = resolveBaseCommit(repoRoot, "HEAD");
257
+ const baseCommit = resolveBaseCommit(repoRoot, requestedRevision);
258
+ const planRepoPath = repoRelative(repoRoot, planPath);
259
+ const baseValidation = buildTrialBaseValidation({
260
+ repoRoot,
261
+ requestedRevision,
262
+ resolvedCommit: baseCommit,
263
+ headCommit,
264
+ defaultedToHead: !opts.base?.trim(),
265
+ planPath: planRepoPath,
266
+ planContent,
267
+ });
268
+ const record = buildComparisonRecord({
269
+ comparisonId,
270
+ projectId,
271
+ taskId: identity.taskId,
272
+ createdAt: new Date().toISOString(),
273
+ baseCommit,
274
+ baseValidation,
275
+ planPath: planRepoPath,
276
+ planContent,
277
+ prompt,
278
+ agents,
279
+ pairs,
280
+ reporter: opts.reporterBy,
281
+ worktreeBase,
282
+ parentValidationIds,
283
+ });
284
+ if (opts.dryRun) {
285
+ process.stdout.write(`${JSON.stringify(record, null, 2)}\n`);
286
+ return;
287
+ }
288
+ writeComparison(recordPath, record);
289
+ const prepared = new Map();
290
+ for (const trial of record.trials) {
291
+ const worktreeTaskId = `${projectId}:trial:${comparisonId}:${trial.trial_id}`;
292
+ if (findExecWorktree(repoRoot, worktreeTaskId) || execBranchExists(repoRoot, worktreeTaskId)) {
293
+ throw new Error(`Trial worktree or branch already exists: ${trial.branch}`);
294
+ }
295
+ const worktree = ensureExecWorktree({
296
+ repoRoot,
297
+ worktreeBase,
298
+ taskId: worktreeTaskId,
299
+ startPoint: record.base_commit,
300
+ });
301
+ prepared.set(trial.trial_id, worktree);
302
+ trial.status = "prepared";
303
+ writeComparison(recordPath, record);
304
+ }
305
+ await runPool(record.trials.map((trial) => async () => {
306
+ const worktree = prepared.get(trial.trial_id);
307
+ if (!worktree)
308
+ throw new Error(`Prepared worktree missing: ${trial.trial_id}`);
309
+ await runOneTrial({
310
+ record,
311
+ recordPath,
312
+ trial,
313
+ worktree,
314
+ plan: planContent,
315
+ prompt,
316
+ executorCommand: executorCommands.get(trial.agent),
317
+ reporterCommand: trial.reporter_agent
318
+ ? reporterCommands.get(trial.reporter_agent)
319
+ : undefined,
320
+ reporterRateLimitDetection: trial.reporter_agent
321
+ ? reporterRateLimitDetections.get(trial.reporter_agent)
322
+ : undefined,
323
+ parentValidationIds,
324
+ execDefaults: defaults,
325
+ repoRoot,
326
+ schedulePath: paths.schedulePath,
327
+ executionPath: paths.executionPath,
328
+ });
329
+ }), parallel);
330
+ process.stdout.write(`Comparison recorded: ${repoRelative(repoRoot, recordPath)}\n`);
331
+ }
332
+ function resolveRecord(opts) {
333
+ const paths = resolveProjectPaths({ project: opts.project });
334
+ activateResolvedProjectPaths(paths);
335
+ const path = comparisonPath(paths.executionPath, opts.comparison);
336
+ return { record: readComparison(path), path, repoRoot: specdojoRootDir() };
337
+ }
338
+ function findTrial(record, selector) {
339
+ const matches = record.trials.filter((trial) => trial.trial_id === selector || trial.agent === selector);
340
+ if (matches.length !== 1)
341
+ throw new Error(`Trial not found or ambiguous: ${selector}`);
342
+ return matches[0];
343
+ }
344
+ function statusTrials(opts) {
345
+ const { record } = resolveRecord(opts);
346
+ process.stdout.write(`Comparison ${record.comparison_id} [${record.status}] task=${record.task_id}\n`);
347
+ process.stdout.write(`plan sha256=${record.plan.sha256} prompt sha256=${record.plan.prompt_sha256}\n`);
348
+ process.stdout.write(`base=${record.base_commit} requested=${record.base?.requested_revision ?? "(legacy)"} head_reference=${record.base?.head_commit_at_start ?? "unknown"} compatible=${record.base?.compatible ?? "unknown"} reporter_mode=${record.reporter_mode ?? "shared-or-none"}\n`);
349
+ for (const warning of record.base?.warnings ?? []) {
350
+ process.stdout.write(`base warning: ${warning}\n`);
351
+ }
352
+ for (const trial of record.trials) {
353
+ const reporterStatus = trial.reporter.status ?? (trial.reporter.structured_output ? "succeeded" : "not_run");
354
+ const parentValidation = trial.parent_validation ?? {
355
+ status: "not_configured",
356
+ duration_ms: null,
357
+ validations_passed: 0,
358
+ validations_failed: 0,
359
+ validations_not_run: 0,
360
+ };
361
+ process.stdout.write(`${trial.trial_id}\t${trial.status}\ttime(executor/parent/total)=${trial.executor.duration_ms ?? "-"}/${parentValidation.duration_ms ?? "-"}/${trial.duration_ms ?? "-"}ms\tfiles=${trial.executor.files_changed}\texecutor-validations(reported/passed/failed/not_run/requires_human)=${trial.executor.validations_reported ?? trial.executor.validations.length}/${trial.executor.validations_passed}/${trial.executor.validations_failed}/${trial.executor.validations_not_run}/${trial.executor.validations_requires_human ?? 0}\tparent-validations(status/passed/failed/not_run)=${parentValidation.status}/${parentValidation.validations_passed}/${parentValidation.validations_failed}/${parentValidation.validations_not_run}\treporter=${reporterStatus}/${trial.reporter.format_attempts}/${trial.reporter.failure_category ?? "-"}\n`);
362
+ }
363
+ }
364
+ function rateTrial(opts) {
365
+ const { record, path } = resolveRecord(opts);
366
+ const trial = findTrial(record, opts.trial);
367
+ const judgment = parseRating(opts.judgmentQuality, "--judgment-quality");
368
+ const writing = parseRating(opts.writingQuality, "--writing-quality");
369
+ const scope = parseRating(opts.scopeAdherence, "--scope-adherence");
370
+ if (judgment !== undefined)
371
+ trial.subjective.judgment_quality = judgment;
372
+ if (writing !== undefined)
373
+ trial.subjective.writing_quality = writing;
374
+ if (scope !== undefined)
375
+ trial.subjective.scope_adherence = scope;
376
+ if (opts.notes !== undefined)
377
+ trial.subjective.notes = opts.notes.trim();
378
+ writeComparison(path, record);
379
+ process.stdout.write(`Rated trial: ${trial.trial_id}\n`);
380
+ }
381
+ function changedPaths(worktreePath) {
382
+ const output = gitOutput(worktreePath, [
383
+ "status",
384
+ "--porcelain=v1",
385
+ "-z",
386
+ "--untracked-files=all",
387
+ ]);
388
+ const paths = [];
389
+ for (const record of output.split("\0").filter(Boolean)) {
390
+ if (record.length >= 4)
391
+ paths.push(record.slice(3));
392
+ }
393
+ return paths;
394
+ }
395
+ function removeTrialWorktree(repoRoot, trial, deleteUnmerged) {
396
+ const registered = findExecWorktree(repoRoot, trial.branch.slice("exec/".length));
397
+ const worktreePath = registered?.path ?? trial.worktree;
398
+ if (existsSync(worktreePath)) {
399
+ gitOutput(repoRoot, ["worktree", "remove", "--force", worktreePath]);
400
+ }
401
+ if (gitResult(repoRoot, ["show-ref", "--verify", "--quiet", `refs/heads/${trial.branch}`])
402
+ .status === 0) {
403
+ gitOutput(repoRoot, ["branch", deleteUnmerged ? "-D" : "-d", trial.branch]);
404
+ }
405
+ }
406
+ function commitSelectedTrial(repoRoot, record, trial) {
407
+ const protectedPaths = changedPaths(trial.worktree).filter(isAgentProtectedConfigPath);
408
+ if (protectedPaths.length > 0)
409
+ throw new Error(agentProtectedConfigViolation(protectedPaths));
410
+ gitOutput(trial.worktree, ["add", "-A"]);
411
+ const staged = gitResult(trial.worktree, ["diff", "--cached", "--quiet"]);
412
+ if (staged.status === 1) {
413
+ const commit = gitResult(trial.worktree, [
414
+ "commit",
415
+ "-m",
416
+ `feat(trial): adopt ${record.task_id} from ${trial.agent}`,
417
+ ]);
418
+ if (commit.status !== 0) {
419
+ gitResult(trial.worktree, ["reset", "--quiet"]);
420
+ throw new Error(`Trial commit failed: ${commit.stderr || commit.stdout}`);
421
+ }
422
+ }
423
+ else if (staged.status !== 0) {
424
+ throw new Error("Failed to inspect selected trial changes.");
425
+ }
426
+ const ahead = Number(gitOutput(trial.worktree, ["rev-list", "--count", `${record.base_commit}..HEAD`]).trim());
427
+ if (!Number.isFinite(ahead) || ahead < 1)
428
+ throw new Error("Selected trial has no changes to adopt.");
429
+ }
430
+ function adoptTrial(opts) {
431
+ const { record, path, repoRoot } = resolveRecord(opts);
432
+ if (record.status !== "active")
433
+ throw new Error(`Comparison is already ${record.status}.`);
434
+ const selected = findTrial(record, opts.trial);
435
+ if (selected.status !== "succeeded")
436
+ throw new Error(`Only a succeeded trial can be adopted: ${selected.trial_id}`);
437
+ if (opts.dryRun) {
438
+ process.stdout.write(`[dry-run] adopt ${selected.branch}; discard ${record.trials.length - 1} other trial(s)\n`);
439
+ return;
440
+ }
441
+ commitSelectedTrial(repoRoot, record, selected);
442
+ try {
443
+ gitOutput(repoRoot, ["merge", "--no-ff", "--no-edit", selected.branch]);
444
+ }
445
+ catch (error) {
446
+ gitResult(repoRoot, ["merge", "--abort"]);
447
+ throw error;
448
+ }
449
+ removeTrialWorktree(repoRoot, selected, false);
450
+ selected.status = "adopted";
451
+ for (const trial of record.trials) {
452
+ if (trial === selected)
453
+ continue;
454
+ removeTrialWorktree(repoRoot, trial, true);
455
+ trial.status = "discarded";
456
+ }
457
+ record.status = "adopted";
458
+ record.selected_trial_id = selected.trial_id;
459
+ record.completed_at = new Date().toISOString();
460
+ writeComparison(path, record);
461
+ process.stdout.write(`Adopted trial ${selected.trial_id}; discarded remaining trial worktrees.\n`);
462
+ }
463
+ function discardTrials(opts) {
464
+ const { record, path, repoRoot } = resolveRecord(opts);
465
+ if (record.status !== "active")
466
+ throw new Error(`Comparison is already ${record.status}.`);
467
+ for (const trial of record.trials) {
468
+ removeTrialWorktree(repoRoot, trial, true);
469
+ trial.status = "discarded";
470
+ }
471
+ record.status = "discarded";
472
+ record.completed_at = new Date().toISOString();
473
+ writeComparison(path, record);
474
+ process.stdout.write(`Discarded all trial worktrees; comparison record retained.\n`);
475
+ }
476
+ function commandError(error) {
477
+ process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
478
+ process.exitCode = 1;
479
+ }
480
+ export function registerExecTrialCommands(exec) {
481
+ const trial = exec.command("trial").description("Compare multiple agents on one immutable plan");
482
+ trial
483
+ .command("run")
484
+ .requiredOption("--project <projectId>", "Project id")
485
+ .requiredOption("--plan <path>", "Existing SpecDojo plan used unchanged by every trial")
486
+ .option("--agent <nicknames...>", "Two or more executor agent nicknames")
487
+ .option("--pair <executor=reporter...>", "Two or more executor/reporter pairs (alternative to --agent)")
488
+ .option("--reporter-by <nickname>", "Shared reporter agent used to assess structured output")
489
+ .option("--base <revision>", "Commit used as the trial worktree start point (default: HEAD)")
490
+ .option("--comparison <id>", "Stable comparison id (generated when omitted)")
491
+ .option("--parallel <n>", "Maximum concurrent trials", "1")
492
+ .option("--worktree-base <path>", "Override worktree base directory")
493
+ .option("--exec-defaults <path>", "Path to exec-defaults.yaml")
494
+ .option("--dry-run", "Print the planned comparison without creating worktrees", false)
495
+ .action(async (opts) => {
496
+ try {
497
+ await runTrials(opts);
498
+ }
499
+ catch (error) {
500
+ commandError(error);
501
+ }
502
+ });
503
+ trial
504
+ .command("status")
505
+ .requiredOption("--project <projectId>", "Project id")
506
+ .requiredOption("--comparison <id>", "Comparison id")
507
+ .action((opts) => {
508
+ try {
509
+ statusTrials(opts);
510
+ }
511
+ catch (error) {
512
+ commandError(error);
513
+ }
514
+ });
515
+ trial
516
+ .command("rate")
517
+ .requiredOption("--project <projectId>", "Project id")
518
+ .requiredOption("--comparison <id>", "Comparison id")
519
+ .requiredOption("--trial <idOrAgent>", "Trial id or agent nickname")
520
+ .option("--judgment-quality <1-5>", "Human judgment-quality rating")
521
+ .option("--writing-quality <1-5>", "Human writing-quality rating")
522
+ .option("--scope-adherence <1-5>", "Human scope-adherence rating")
523
+ .option("--notes <text>", "Human evaluation notes")
524
+ .action((opts) => {
525
+ try {
526
+ rateTrial(opts);
527
+ }
528
+ catch (error) {
529
+ commandError(error);
530
+ }
531
+ });
532
+ trial
533
+ .command("adopt")
534
+ .requiredOption("--project <projectId>", "Project id")
535
+ .requiredOption("--comparison <id>", "Comparison id")
536
+ .requiredOption("--trial <idOrAgent>", "Succeeded trial to merge")
537
+ .option("--dry-run", "Print adopt/discard actions", false)
538
+ .action((opts) => {
539
+ try {
540
+ adoptTrial(opts);
541
+ }
542
+ catch (error) {
543
+ commandError(error);
544
+ }
545
+ });
546
+ trial
547
+ .command("discard")
548
+ .requiredOption("--project <projectId>", "Project id")
549
+ .requiredOption("--comparison <id>", "Comparison id")
550
+ .action((opts) => {
551
+ try {
552
+ discardTrials(opts);
553
+ }
554
+ catch (error) {
555
+ commandError(error);
556
+ }
557
+ });
558
+ }