specdojo 0.2.1 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/agents/specdojo-orchestrator.md +165 -0
- package/CHANGELOG.md +298 -0
- package/README.md +92 -42
- package/dist/agent-response.js +20 -0
- package/dist/agent.js +4 -4
- package/dist/build-command.js +28 -8
- package/dist/catalog-build-references.js +144 -0
- package/dist/catalog-build-render.js +144 -0
- package/dist/catalog-build.js +13 -290
- package/dist/catalog-generate.js +6 -9
- package/dist/catalog-plan-generate.js +2 -2
- package/dist/catalog-plan.js +6 -6
- package/dist/catalog-scaffold.js +4 -4
- package/dist/catalog.js +2 -2
- package/dist/dashboard-exec-records.js +105 -0
- package/dist/dashboard-routines.js +255 -0
- package/dist/dashboard-shared.js +14 -0
- package/dist/dashboard-timeline.js +175 -0
- package/dist/dashboard.js +31 -444
- package/dist/deliverable-trash.js +3 -3
- package/dist/devcontainer-command.js +43 -0
- package/dist/devcontainer-scaffold.js +165 -0
- package/dist/doc-index.js +3 -3
- package/dist/exec-agent-config.js +107 -9
- package/dist/exec-agent-git-state.js +18 -0
- package/dist/exec-agent-protected-config.js +46 -8
- package/dist/exec-commands.js +557 -0
- package/dist/exec-core.js +457 -0
- package/dist/exec-evidence.js +254 -11
- package/dist/exec-handoff.js +336 -0
- package/dist/exec-parent-validation.js +317 -43
- package/dist/exec-pipeline-state.js +74 -9
- package/dist/exec-plans-core.js +693 -0
- package/dist/exec-plans-generation.js +206 -0
- package/dist/exec-plans-storage.js +131 -0
- package/dist/exec-plans.js +3 -1016
- package/dist/exec-protection-handoff.js +43 -2
- package/dist/exec-provider-scaffold.js +247 -15
- package/dist/exec-register-resume.js +65 -19
- package/dist/exec-register.js +9 -3
- package/dist/exec-repo-integration-git.js +23 -0
- package/dist/exec-repo-integration-sync.js +264 -0
- package/dist/exec-repo-integration.js +534 -0
- package/dist/exec-repo-trace.js +182 -0
- package/dist/exec-reporter-output.js +206 -0
- package/dist/exec-reporter.js +525 -248
- package/dist/exec-results.js +103 -71
- package/dist/exec-run-agent-pipeline.js +256 -0
- package/dist/exec-run-agent.js +531 -0
- package/dist/exec-run-in-place.js +541 -0
- package/dist/exec-run-job.js +717 -0
- package/dist/exec-run-lifecycle-commands.js +522 -0
- package/dist/exec-run-lock.js +110 -107
- package/dist/exec-run-mode-shared.js +140 -0
- package/dist/exec-run-register-command.js +244 -0
- package/dist/exec-run-register-core.js +386 -0
- package/dist/exec-run-register-mode.js +206 -0
- package/dist/exec-run-register-resume.js +393 -0
- package/dist/exec-run-register-state.js +322 -0
- package/dist/exec-run-register-worktree.js +509 -0
- package/dist/exec-run-schedule-mode.js +294 -0
- package/dist/exec-run-schedule-prepare.js +353 -0
- package/dist/exec-run-schedule-task.js +572 -0
- package/dist/exec-run-shared.js +399 -0
- package/dist/exec-run.js +11 -4953
- package/dist/exec-shared.js +9 -2
- package/dist/exec-slot-lock.js +242 -0
- package/dist/exec-task-repos.js +415 -0
- package/dist/exec-trial-commands.js +558 -0
- package/dist/exec-trial-core.js +371 -0
- package/dist/exec-trial.js +2 -895
- package/dist/exec-worktree-command.js +87 -43
- package/dist/exec-worktree-ops-commit.js +707 -0
- package/dist/exec-worktree-ops-lifecycle.js +292 -0
- package/dist/exec-worktree-ops-merge.js +303 -0
- package/dist/exec-worktree-ops.js +3 -847
- package/dist/exec-worktree.js +146 -31
- package/dist/exec.js +2 -999
- package/dist/frontmatter-namespace.js +2 -2
- package/dist/grade-apply.js +734 -0
- package/dist/grade-command-apply.js +250 -0
- package/dist/grade-command-inspect.js +138 -0
- package/dist/grade-command-plan.js +101 -0
- package/dist/grade-command-shared.js +68 -0
- package/dist/grade-core.js +307 -0
- package/dist/grade-markdown.js +274 -0
- package/dist/grade-pipeline-state.js +184 -0
- package/dist/grade-plan.js +507 -0
- package/dist/grade-result.js +133 -5
- package/dist/grade-target-discovery.js +265 -0
- package/dist/grade-targets.js +432 -0
- package/dist/grade.js +14 -2451
- package/dist/init-command.js +443 -0
- package/dist/job-definition.js +670 -0
- package/dist/job.js +64 -578
- package/dist/kata-command.js +2 -2
- package/dist/orchestrator.js +213 -0
- package/dist/register-command-types.js +1 -0
- package/dist/register-core.js +363 -0
- package/dist/register-create-commands.js +245 -0
- package/dist/register-event-merge.js +169 -0
- package/dist/register-events.js +4 -4
- package/dist/register-item.js +13 -7
- package/dist/register-maintenance-commands.js +187 -0
- package/dist/register-migration.js +268 -0
- package/dist/register-renumber.js +314 -0
- package/dist/register-transition-commands.js +248 -0
- package/dist/register-update.js +230 -0
- package/dist/register-views.js +256 -0
- package/dist/register.js +175 -1975
- package/dist/review-grade.js +252 -0
- package/dist/review-plan.js +114 -4
- package/dist/review-types.js +40 -1
- package/dist/routine-definition.js +401 -0
- package/dist/routine.js +73 -402
- package/dist/schedule-approach.js +2 -2
- package/dist/schedule-build.js +2 -2
- package/dist/schedule-strategy-generate-document.js +273 -0
- package/dist/schedule-strategy-generate-scope.js +199 -0
- package/dist/schedule-strategy-generate.js +9 -473
- package/dist/schedule.js +19 -6
- package/dist/specdojo-config.js +465 -20
- package/dist/specdojo-gitignore.js +135 -0
- package/dist/specdojo.js +6 -0
- package/dist/viewpoint-document-kinds-check.js +128 -0
- package/dist/yaml-pages.js +4 -4
- package/dist/yaml.js +28 -0
- package/docs/ja/specdojo/defaults/pm-review-viewpoints.yaml +348 -90
- package/docs/ja/specdojo/exec-templates/xep-common-conventions-template.md +51 -17
- package/docs/ja/specdojo/exec-templates/xep-fully-guided-template.md +7 -5
- package/docs/ja/specdojo/exec-templates/xrp-freeform-template.md +29 -31
- package/docs/ja/specdojo/exec-templates/xrp-fully-guided-template.md +34 -55
- package/docs/ja/specdojo/exec-templates/xrp-job-template.md +54 -0
- package/docs/ja/specdojo/exec-templates/xrp-recipe-guided-template.md +28 -37
- package/docs/ja/specdojo/exec-templates/xrp-recipe-maintenance-template.md +29 -34
- package/docs/ja/specdojo/exec-templates/xrp-retrofit-template.md +30 -30
- package/docs/ja/specdojo/exec-templates/xrp-rulebook-maintenance-template.md +29 -34
- package/docs/ja/specdojo/exec-templates/xrp-sample-maintenance-template.md +29 -34
- package/docs/ja/specdojo/exec-templates/xrp-template-maintenance-template.md +29 -34
- package/docs/ja/specdojo/exec-templates/xrp-template.md +29 -28
- package/docs/ja/specdojo/exec-templates/xrr-template.md +11 -9
- package/docs/ja/specdojo/guides/docs-editing-guide.md +20 -0
- package/docs/ja/specdojo/guides/docs-structure-guide.md +103 -70
- package/docs/ja/specdojo/guides/exec-config-guide.md +93 -49
- package/docs/ja/specdojo/guides/exec-operation-guide.md +124 -27
- package/docs/ja/specdojo/guides/exec-worktree-guide.md +40 -11
- package/docs/ja/specdojo/guides/kata-guide.md +4 -4
- package/docs/ja/specdojo/guides/orchestrator-operation-guide.md +45 -22
- package/docs/ja/specdojo/guides/plan-result-lifecycle-guide.md +1 -1
- package/docs/ja/specdojo/guides/practice-system-composition-guide.md +13 -13
- package/docs/ja/specdojo/guides/quick-start-guide.md +260 -30
- package/docs/ja/specdojo/guides/register-operation-guide.md +31 -14
- package/docs/ja/specdojo/guides/release-v0-3-0-migration-guide.md +264 -0
- package/docs/ja/specdojo/guides/remote-host-development-guide.md +22 -1
- package/docs/ja/specdojo/guides/review-guide.md +195 -158
- package/docs/ja/specdojo/guides/routine-operation-guide.md +75 -10
- package/docs/ja/specdojo/guides/ryu-guide.md +32 -30
- package/docs/ja/specdojo/guides/schedule-design-guide.md +10 -10
- package/docs/ja/specdojo/guides/schedule-operation-guide.md +1 -1
- package/docs/ja/specdojo/guides/specdojo-overview-guide.md +58 -33
- package/docs/ja/specdojo/guides/use-case-guide.md +3 -3
- package/docs/ja/specdojo/guides/waza-guide.md +15 -2
- package/docs/ja/specdojo/index.md +2 -2
- package/docs/ja/specdojo/philosophy/specdojo-philosophy.md +5 -5
- package/docs/ja/specdojo/recipes/bps-recipe.md +213 -0
- package/docs/ja/specdojo/recipes/pm-members-recipe.md +52 -117
- package/docs/ja/specdojo/references/command-reference.md +335 -92
- package/docs/ja/specdojo/references/deliverables-reference.md +1 -1
- package/docs/ja/specdojo/references/directory-layout-reference.md +13 -9
- package/docs/ja/specdojo/references/specdojo-config-reference.md +107 -24
- package/docs/ja/specdojo/rulebooks/bps-rulebook.md +132 -220
- package/docs/ja/specdojo/rulebooks/br-rulebook.md +6 -44
- package/docs/ja/specdojo/rulebooks/cdfd-rulebook.md +2 -0
- package/docs/ja/specdojo/rulebooks/dmd-rulebook.md +0 -87
- package/docs/ja/specdojo/rulebooks/gl-rulebook.md +61 -39
- package/docs/ja/specdojo/rulebooks/ifx-cmd-rulebook.md +0 -87
- package/docs/ja/specdojo/rulebooks/ifx-index-rulebook.md +1 -33
- package/docs/ja/specdojo/rulebooks/opd-rulebook.md +0 -137
- package/docs/ja/specdojo/rulebooks/opr-rulebook.md +0 -247
- package/docs/ja/specdojo/rulebooks/otp-rulebook.md +0 -83
- package/docs/ja/specdojo/rulebooks/pjr-rulebook.md +4 -2
- package/docs/ja/specdojo/rulebooks/pm-members-rulebook.md +78 -150
- package/docs/ja/specdojo/rulebooks/pm-roles-rulebook.md +5 -4
- package/docs/ja/specdojo/rulebooks/sch-rulebook.md +1 -1
- package/docs/ja/specdojo/rulebooks/stsd-mermaid-rulebook.md +0 -11
- package/docs/ja/specdojo/rulebooks/stsd-rulebook.md +0 -61
- package/docs/ja/specdojo/rulebooks/tml-rulebook.md +2 -47
- package/docs/ja/specdojo/samples/bps-sample.md +74 -21
- package/docs/ja/specdojo/samples/gl-sample.yaml +17 -1
- package/docs/ja/specdojo/samples/ifx-index-sample.yaml +28 -0
- package/docs/ja/specdojo/samples/pm-members-sample.yaml +2 -67
- package/docs/ja/specdojo/samples/pm-organization-sample.md +8 -7
- package/docs/ja/specdojo/samples/pm-raci-sample.md +1 -1
- package/docs/ja/specdojo/samples/tml-sample.yaml +40 -0
- package/docs/ja/specdojo/standards/document-metadata-standard.md +21 -16
- package/docs/ja/specdojo/standards/id-and-file-naming-standard.md +14 -14
- package/docs/ja/specdojo/standards/job-definition-standard.md +37 -17
- package/docs/ja/specdojo/standards/people-and-organization-definition-standard.md +37 -32
- package/docs/ja/specdojo/standards/philosophy-authoring-standard.md +1 -1
- package/docs/ja/specdojo/standards/rulebook-authoring-standard.md +21 -11
- package/docs/ja/specdojo/templates/agent-members-template.yaml +25 -0
- package/docs/ja/specdojo/templates/bps-template.md +91 -0
- package/docs/ja/specdojo/templates/pm-members-template.yaml +2 -59
- package/docs/specdojo/schemas/v1/agent-members.schema.yaml +76 -0
- package/docs/specdojo/schemas/v1/exec-common.schema.yaml +4 -2
- package/docs/specdojo/schemas/v1/exec-defaults.schema.yaml +55 -2
- package/docs/specdojo/schemas/v1/exec-evidence.schema.yaml +93 -6
- package/docs/specdojo/schemas/v1/exec-reporter-output.schema.yaml +27 -78
- package/docs/specdojo/schemas/v1/gl.schema.yaml +87 -0
- package/docs/specdojo/schemas/v1/grade-result.schema.yaml +33 -0
- package/docs/specdojo/schemas/v1/job-run.schema.yaml +27 -0
- package/docs/specdojo/schemas/v1/job.schema.yaml +25 -0
- package/docs/specdojo/schemas/v1/pipeline-state.schema.yaml +49 -1
- package/docs/specdojo/schemas/v1/pm-members.schema.yaml +9 -169
- package/docs/specdojo/schemas/v1/pm-review-viewpoints.schema.yaml +100 -10
- package/docs/specdojo/schemas/v1/register-item-frontmatter.schema.yaml +13 -0
- package/docs/specdojo/schemas/v1/routine-run.schema.yaml +3 -0
- package/docs/specdojo/schemas/v1/routine.schema.yaml +7 -0
- package/docs/specdojo/schemas/v1/specdojo-config.schema.yaml +110 -0
- package/package.json +17 -12
- package/templates/antigravity/README.md +29 -4
- package/templates/antigravity/{pm-members-snippet.yaml → agent-members-snippet.yaml} +8 -2
- package/templates/antigravity/exec-defaults-snippet.yaml +10 -7
- package/templates/antigravity/orchestrator.md +18 -7
- package/templates/antigravity/settings.global.json +55 -0
- package/templates/claude/agents/claude-executor.md +14 -0
- package/templates/claude/agents/claude-reporter.md +16 -0
- package/templates/claude/agents/specdojo-orchestrator.md +18 -7
- package/templates/claude/settings.edit.json +27 -1
- package/templates/codex/README.md +5 -4
- package/templates/codex/agents/specdojo-orchestrator.toml +18 -7
- package/templates/codex/orchestrator.md +18 -7
- package/templates/copilot/README.md +7 -5
- package/templates/copilot/{pm-members-snippet.yaml → agent-members-snippet.yaml} +4 -2
- package/templates/devcontainer/devcontainer.json +27 -0
- package/templates/devcontainer/emacs/init.el +88 -0
- package/templates/devcontainer/post-create.sh +62 -0
- package/templates/init-agent-members.yaml +200 -0
- package/templates/init-exec-defaults.yaml +74 -0
- package/templates/opencode/agents/specdojo-orchestrator.md +18 -7
- package/docs/ja/specdojo/exec-templates/xrp-viewpoint-detail-template.md +0 -7
- package/docs/ja/specdojo/exec-templates/xrr-viewpoint-detail-template.md +0 -7
|
@@ -0,0 +1,558 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { copyFileSync, existsSync, readFileSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { basename, dirname, join, resolve } from "node:path";
|
|
4
|
+
import { defaultExecDefaultsPath, loadExecDefaultsConfig, resolveMemberCommand, resolveRateLimitDetection, } from "./exec-agent-config.js";
|
|
5
|
+
import { agentProtectedConfigViolation, isAgentProtectedConfigPath, } from "./exec-agent-protected-config.js";
|
|
6
|
+
import { recordExecutorEvidence, recordParentValidationFailureOutput, } from "./exec-evidence.js";
|
|
7
|
+
import { parentValidationLabels, } from "./exec-parent-validation.js";
|
|
8
|
+
import { activateResolvedProjectPaths, resolveProjectPaths } from "./exec-project.js";
|
|
9
|
+
import { buildExecutorPrompt, isRateLimitError, loadRosterForExecutionPath, runConfiguredParentValidationBatch, } from "./exec-run.js";
|
|
10
|
+
import { parsePlanTaskIdentity } from "./exec-plans.js";
|
|
11
|
+
import { deriveReporterFacts, evidenceForReporter, runReporterWithFormatRetry, } from "./exec-reporter.js";
|
|
12
|
+
import { ensureDir, safeSlug } from "./exec-shared.js";
|
|
13
|
+
import { ensureExecWorktree, execBranchExists, findExecWorktree, gitOutput, gitResult, resolveWorktreeBase, } from "./exec-worktree.js";
|
|
14
|
+
import { specdojoRootDir } from "./specdojo-config.js";
|
|
15
|
+
import { TRIAL_PROMPT_SUFFIX, buildComparisonRecord, buildTrialBaseValidation, classifyReporterFailure, comparisonDir, comparisonPath, configuredWorktreeBase, hasValidExecutorEnvelope, parsePositiveInteger, parseRating, readComparison, repoRelative, resolveMember, spawnAgent, updateObjectiveMetrics, worktreeEnvironment, writeComparison, } from "./exec-trial-core.js";
|
|
16
|
+
async function runOneTrial(params) {
|
|
17
|
+
const { trial, worktree } = params;
|
|
18
|
+
const startedAt = new Date();
|
|
19
|
+
trial.status = "running";
|
|
20
|
+
trial.started_at = startedAt.toISOString();
|
|
21
|
+
writeComparison(params.recordPath, params.record);
|
|
22
|
+
const outcome = await spawnAgent(params.executorCommand, params.prompt, worktree.path, worktreeEnvironment(params.repoRoot, worktree.path, params.schedulePath, params.executionPath));
|
|
23
|
+
const executorCompletedAt = new Date();
|
|
24
|
+
trial.executor.duration_ms = executorCompletedAt.getTime() - startedAt.getTime();
|
|
25
|
+
trial.parent_validation.configured_ids = [...params.parentValidationIds];
|
|
26
|
+
let parentValidations = [];
|
|
27
|
+
let parentValidationFailureOutputs = [];
|
|
28
|
+
if (params.parentValidationIds.length === 0) {
|
|
29
|
+
trial.parent_validation.status = "not_configured";
|
|
30
|
+
}
|
|
31
|
+
else if (outcome.exitCode !== 0) {
|
|
32
|
+
trial.parent_validation.status = "not_run";
|
|
33
|
+
}
|
|
34
|
+
else {
|
|
35
|
+
const parentValidationStartedAt = Date.now();
|
|
36
|
+
const parentValidationBatch = await runConfiguredParentValidationBatch(params.execDefaults, worktree.path, {
|
|
37
|
+
label: `${params.record.task_id} (${trial.trial_id})`,
|
|
38
|
+
});
|
|
39
|
+
parentValidations = parentValidationBatch.validations;
|
|
40
|
+
parentValidationFailureOutputs = parentValidationBatch.failureOutputs;
|
|
41
|
+
trial.parent_validation.duration_ms = Date.now() - parentValidationStartedAt;
|
|
42
|
+
trial.parent_validation.status = parentValidations.some((validation) => validation.status !== "passed")
|
|
43
|
+
? "failed"
|
|
44
|
+
: "passed";
|
|
45
|
+
}
|
|
46
|
+
const runId = `${safeSlug(params.record.comparison_id)}-${trial.trial_id}`;
|
|
47
|
+
const evidenceRecorded = recordExecutorEvidence({
|
|
48
|
+
repoRoot: params.repoRoot,
|
|
49
|
+
worktreePath: worktree.path,
|
|
50
|
+
executionPath: params.executionPath,
|
|
51
|
+
taskId: params.record.task_id,
|
|
52
|
+
runId,
|
|
53
|
+
actor: trial.agent,
|
|
54
|
+
status: outcome.exitCode === 0 ? "succeeded" : "failed",
|
|
55
|
+
startedAt: startedAt.toISOString(),
|
|
56
|
+
completedAt: executorCompletedAt.toISOString(),
|
|
57
|
+
exitCode: outcome.exitCode,
|
|
58
|
+
attempts: 1,
|
|
59
|
+
stdout: outcome.stdout,
|
|
60
|
+
stderr: outcome.stderr,
|
|
61
|
+
parentValidations,
|
|
62
|
+
});
|
|
63
|
+
evidenceRecorded.evidence = recordParentValidationFailureOutput({
|
|
64
|
+
worktreePath: worktree.path,
|
|
65
|
+
evidencePath: evidenceRecorded.evidencePath,
|
|
66
|
+
evidence: evidenceRecorded.evidence,
|
|
67
|
+
failureOutputs: parentValidationFailureOutputs,
|
|
68
|
+
});
|
|
69
|
+
const centralizedEvidenceDir = join(comparisonDir(params.executionPath, params.record.comparison_id), "trials", trial.trial_id);
|
|
70
|
+
ensureDir(centralizedEvidenceDir);
|
|
71
|
+
const centralizedEvidencePath = join(centralizedEvidenceDir, "evidence.json");
|
|
72
|
+
evidenceRecorded.evidence.log_refs = evidenceRecorded.evidence.log_refs.map((ref) => {
|
|
73
|
+
const logName = basename(ref.path);
|
|
74
|
+
const centralizedLogPath = join(centralizedEvidenceDir, logName);
|
|
75
|
+
copyFileSync(join(dirname(evidenceRecorded.evidencePath), logName), centralizedLogPath);
|
|
76
|
+
return { ...ref, path: repoRelative(params.repoRoot, centralizedLogPath) };
|
|
77
|
+
});
|
|
78
|
+
writeFileSync(centralizedEvidencePath, `${JSON.stringify(evidenceRecorded.evidence, null, 2)}\n`, "utf8");
|
|
79
|
+
trial.executor.exit_code = outcome.exitCode;
|
|
80
|
+
trial.executor.attempts = 1;
|
|
81
|
+
trial.executor.evidence_ref = repoRelative(params.repoRoot, centralizedEvidencePath);
|
|
82
|
+
updateObjectiveMetrics(trial, evidenceRecorded.evidence, hasValidExecutorEnvelope(outcome.stdout));
|
|
83
|
+
let reporterSucceeded = true;
|
|
84
|
+
if (outcome.exitCode === 0 && params.reporterCommand) {
|
|
85
|
+
const reporterEvidence = evidenceForReporter(evidenceRecorded.evidence, {
|
|
86
|
+
repoRoot: params.repoRoot,
|
|
87
|
+
worktreePath: worktree.path,
|
|
88
|
+
executionPath: params.executionPath,
|
|
89
|
+
});
|
|
90
|
+
const reporter = await runReporterWithFormatRetry({
|
|
91
|
+
plan: params.plan,
|
|
92
|
+
evidence: reporterEvidence,
|
|
93
|
+
mode: parsePlanTaskIdentity(params.plan)?.mode ?? "edit",
|
|
94
|
+
invoke: async (reporterPrompt) => {
|
|
95
|
+
const reporterOutcome = await spawnAgent(params.reporterCommand, reporterPrompt, worktree.path, worktreeEnvironment(params.repoRoot, worktree.path, params.schedulePath, params.executionPath));
|
|
96
|
+
const result = reporterOutcome.exitCode === 0
|
|
97
|
+
? "success"
|
|
98
|
+
: isRateLimitError(reporterOutcome.exitCode, `${reporterOutcome.stdout}\n${reporterOutcome.stderr}`, params.reporterRateLimitDetection)
|
|
99
|
+
? "rate_limit"
|
|
100
|
+
: "failure";
|
|
101
|
+
return {
|
|
102
|
+
result,
|
|
103
|
+
stdout: reporterOutcome.stdout,
|
|
104
|
+
stderr: reporterOutcome.stderr,
|
|
105
|
+
};
|
|
106
|
+
},
|
|
107
|
+
});
|
|
108
|
+
trial.reporter.format_attempts = reporter.formatAttempts;
|
|
109
|
+
trial.reporter.format_retries = Math.max(0, reporter.formatAttempts - 1);
|
|
110
|
+
if (reporter.result === "success") {
|
|
111
|
+
const facts = deriveReporterFacts(reporterEvidence, {
|
|
112
|
+
mode: parsePlanTaskIdentity(params.plan)?.mode ?? "edit",
|
|
113
|
+
});
|
|
114
|
+
trial.reporter.structured_output = true;
|
|
115
|
+
trial.reporter.outcome = facts.outcome;
|
|
116
|
+
trial.reporter.output = reporter.output;
|
|
117
|
+
reporterSucceeded = facts.outcome === "complete";
|
|
118
|
+
if (reporterSucceeded) {
|
|
119
|
+
trial.reporter.status = "succeeded";
|
|
120
|
+
}
|
|
121
|
+
else {
|
|
122
|
+
trial.reporter.status = "blocked";
|
|
123
|
+
trial.reporter.failure_category = "evidence_blocked";
|
|
124
|
+
trial.reporter.failure_reason = facts.block_reason;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
else {
|
|
128
|
+
trial.reporter.status = "failed";
|
|
129
|
+
trial.reporter.failure_category = classifyReporterFailure(reporter.result, reporter.reason);
|
|
130
|
+
trial.reporter.failure_reason = reporter.reason;
|
|
131
|
+
reporterSucceeded = false;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
const completedAt = new Date();
|
|
135
|
+
trial.completed_at = completedAt.toISOString();
|
|
136
|
+
trial.duration_ms = completedAt.getTime() - startedAt.getTime();
|
|
137
|
+
trial.status =
|
|
138
|
+
outcome.exitCode === 0 &&
|
|
139
|
+
trial.executor.validations_failed === 0 &&
|
|
140
|
+
trial.parent_validation.status !== "failed" &&
|
|
141
|
+
trial.parent_validation.status !== "not_run" &&
|
|
142
|
+
reporterSucceeded
|
|
143
|
+
? "succeeded"
|
|
144
|
+
: "failed";
|
|
145
|
+
writeComparison(params.recordPath, params.record);
|
|
146
|
+
}
|
|
147
|
+
async function runPool(items, parallel) {
|
|
148
|
+
let next = 0;
|
|
149
|
+
const workers = Array.from({ length: Math.min(parallel, items.length) }, async () => {
|
|
150
|
+
while (next < items.length) {
|
|
151
|
+
const index = next++;
|
|
152
|
+
await items[index]();
|
|
153
|
+
}
|
|
154
|
+
});
|
|
155
|
+
await Promise.all(workers);
|
|
156
|
+
}
|
|
157
|
+
function parseTrialPair(value) {
|
|
158
|
+
const separator = value.indexOf("=");
|
|
159
|
+
const executor = value.slice(0, separator).trim();
|
|
160
|
+
const reporter = value.slice(separator + 1).trim();
|
|
161
|
+
if (separator < 1 || !executor || !reporter || reporter.includes("=")) {
|
|
162
|
+
throw new Error(`--pair must use <executor>=<reporter>: ${value}`);
|
|
163
|
+
}
|
|
164
|
+
return { executor, reporter };
|
|
165
|
+
}
|
|
166
|
+
function resolveTrialDefinitions(opts) {
|
|
167
|
+
const agentValues = (opts.agent ?? []).map((value) => value.trim()).filter(Boolean);
|
|
168
|
+
const pairValues = (opts.pair ?? []).map((value) => value.trim()).filter(Boolean);
|
|
169
|
+
if (agentValues.length > 0 && pairValues.length > 0) {
|
|
170
|
+
throw new Error("Use either --agent or --pair, not both.");
|
|
171
|
+
}
|
|
172
|
+
if (pairValues.length > 0) {
|
|
173
|
+
if (opts.reporterBy)
|
|
174
|
+
throw new Error("--pair cannot be combined with --reporter-by.");
|
|
175
|
+
const pairs = pairValues.map(parseTrialPair);
|
|
176
|
+
const keys = pairs.map((pair) => `${pair.executor}\0${pair.reporter}`);
|
|
177
|
+
if (new Set(keys).size < 2 || new Set(keys).size !== keys.length) {
|
|
178
|
+
throw new Error("exec trial run requires at least two distinct --pair values.");
|
|
179
|
+
}
|
|
180
|
+
return { agents: [...new Set(pairs.map((pair) => pair.executor))], pairs };
|
|
181
|
+
}
|
|
182
|
+
const agents = [...new Set(agentValues)];
|
|
183
|
+
if (agents.length < 2) {
|
|
184
|
+
throw new Error("exec trial run requires at least two distinct --agent values or --pair values.");
|
|
185
|
+
}
|
|
186
|
+
return { agents };
|
|
187
|
+
}
|
|
188
|
+
function resolveBaseCommit(repoRoot, revision) {
|
|
189
|
+
const result = gitResult(repoRoot, [
|
|
190
|
+
"rev-parse",
|
|
191
|
+
"--verify",
|
|
192
|
+
"--end-of-options",
|
|
193
|
+
`${revision}^{commit}`,
|
|
194
|
+
]);
|
|
195
|
+
if (result.status !== 0)
|
|
196
|
+
throw new Error(`Trial base is not a commit: ${revision}`);
|
|
197
|
+
return String(result.stdout).trim();
|
|
198
|
+
}
|
|
199
|
+
async function runTrials(opts) {
|
|
200
|
+
const paths = resolveProjectPaths({ project: opts.project });
|
|
201
|
+
activateResolvedProjectPaths(paths);
|
|
202
|
+
const repoRoot = specdojoRootDir();
|
|
203
|
+
const projectId = paths.projectId ?? opts.project?.trim();
|
|
204
|
+
if (!projectId)
|
|
205
|
+
throw new Error("exec trial run requires a project id.");
|
|
206
|
+
const planPath = resolve(opts.plan);
|
|
207
|
+
if (!existsSync(planPath))
|
|
208
|
+
throw new Error(`Plan not found: ${planPath}`);
|
|
209
|
+
const planContent = readFileSync(planPath, "utf8");
|
|
210
|
+
const identity = parsePlanTaskIdentity(planContent);
|
|
211
|
+
if (!identity)
|
|
212
|
+
throw new Error("Trial plan must have SpecDojo task_id/mode/project_id frontmatter.");
|
|
213
|
+
if (identity.projectId && identity.projectId !== projectId) {
|
|
214
|
+
throw new Error(`Plan project_id ${identity.projectId} does not match --project ${projectId}.`);
|
|
215
|
+
}
|
|
216
|
+
const { agents, pairs } = resolveTrialDefinitions(opts);
|
|
217
|
+
const parallel = parsePositiveInteger(opts.parallel ?? "1", "--parallel");
|
|
218
|
+
const comparisonId = safeSlug(opts.comparison?.trim() ||
|
|
219
|
+
`${identity.taskId}-${new Date().toISOString()}-${randomUUID().slice(0, 8)}`);
|
|
220
|
+
const recordPath = comparisonPath(paths.executionPath, comparisonId);
|
|
221
|
+
if (existsSync(recordPath))
|
|
222
|
+
throw new Error(`Comparison already exists: ${comparisonId}`);
|
|
223
|
+
const worktreeBase = resolveWorktreeBase(repoRoot, opts.worktreeBase, configuredWorktreeBase(paths.schedulePath));
|
|
224
|
+
const roster = loadRosterForExecutionPath(paths.executionPath);
|
|
225
|
+
const defaults = loadExecDefaultsConfig(opts.execDefaults ?? defaultExecDefaultsPath(), paths.executionPath);
|
|
226
|
+
const executorCommands = new Map();
|
|
227
|
+
for (const agent of agents) {
|
|
228
|
+
const command = resolveMemberCommand(defaults, resolveMember(roster, agent, "executor"));
|
|
229
|
+
if (!command)
|
|
230
|
+
throw new Error(`Agent command could not be resolved: ${agent}`);
|
|
231
|
+
executorCommands.set(agent, command);
|
|
232
|
+
}
|
|
233
|
+
const reporterCommands = new Map();
|
|
234
|
+
const reporterRateLimitDetections = new Map();
|
|
235
|
+
if (opts.reporterBy) {
|
|
236
|
+
const reporterMember = resolveMember(roster, opts.reporterBy, "reporter");
|
|
237
|
+
const reporterCommand = resolveMemberCommand(defaults, reporterMember);
|
|
238
|
+
if (!reporterCommand)
|
|
239
|
+
throw new Error(`Reporter command could not be resolved: ${opts.reporterBy}`);
|
|
240
|
+
reporterCommands.set(opts.reporterBy, reporterCommand);
|
|
241
|
+
reporterRateLimitDetections.set(opts.reporterBy, resolveRateLimitDetection(defaults, reporterMember.provider));
|
|
242
|
+
}
|
|
243
|
+
for (const reporter of new Set(pairs?.map((pair) => pair.reporter) ?? [])) {
|
|
244
|
+
const reporterMember = resolveMember(roster, reporter, "reporter");
|
|
245
|
+
const reporterCommand = resolveMemberCommand(defaults, reporterMember);
|
|
246
|
+
if (!reporterCommand)
|
|
247
|
+
throw new Error(`Reporter command could not be resolved: ${reporter}`);
|
|
248
|
+
reporterCommands.set(reporter, reporterCommand);
|
|
249
|
+
reporterRateLimitDetections.set(reporter, resolveRateLimitDetection(defaults, reporterMember.provider));
|
|
250
|
+
}
|
|
251
|
+
const parentValidationEntries = defaults.pipeline?.parent_validations ?? [];
|
|
252
|
+
// Labels are `<id>` for the project repository and `<repo>:<id>` for a product repository.
|
|
253
|
+
const parentValidationIds = parentValidationLabels(parentValidationEntries);
|
|
254
|
+
const prompt = `${buildExecutorPrompt(planContent, parentValidationEntries).trimEnd()}${TRIAL_PROMPT_SUFFIX}`;
|
|
255
|
+
const requestedRevision = opts.base?.trim() || "HEAD";
|
|
256
|
+
const headCommit = resolveBaseCommit(repoRoot, "HEAD");
|
|
257
|
+
const baseCommit = resolveBaseCommit(repoRoot, requestedRevision);
|
|
258
|
+
const planRepoPath = repoRelative(repoRoot, planPath);
|
|
259
|
+
const baseValidation = buildTrialBaseValidation({
|
|
260
|
+
repoRoot,
|
|
261
|
+
requestedRevision,
|
|
262
|
+
resolvedCommit: baseCommit,
|
|
263
|
+
headCommit,
|
|
264
|
+
defaultedToHead: !opts.base?.trim(),
|
|
265
|
+
planPath: planRepoPath,
|
|
266
|
+
planContent,
|
|
267
|
+
});
|
|
268
|
+
const record = buildComparisonRecord({
|
|
269
|
+
comparisonId,
|
|
270
|
+
projectId,
|
|
271
|
+
taskId: identity.taskId,
|
|
272
|
+
createdAt: new Date().toISOString(),
|
|
273
|
+
baseCommit,
|
|
274
|
+
baseValidation,
|
|
275
|
+
planPath: planRepoPath,
|
|
276
|
+
planContent,
|
|
277
|
+
prompt,
|
|
278
|
+
agents,
|
|
279
|
+
pairs,
|
|
280
|
+
reporter: opts.reporterBy,
|
|
281
|
+
worktreeBase,
|
|
282
|
+
parentValidationIds,
|
|
283
|
+
});
|
|
284
|
+
if (opts.dryRun) {
|
|
285
|
+
process.stdout.write(`${JSON.stringify(record, null, 2)}\n`);
|
|
286
|
+
return;
|
|
287
|
+
}
|
|
288
|
+
writeComparison(recordPath, record);
|
|
289
|
+
const prepared = new Map();
|
|
290
|
+
for (const trial of record.trials) {
|
|
291
|
+
const worktreeTaskId = `${projectId}:trial:${comparisonId}:${trial.trial_id}`;
|
|
292
|
+
if (findExecWorktree(repoRoot, worktreeTaskId) || execBranchExists(repoRoot, worktreeTaskId)) {
|
|
293
|
+
throw new Error(`Trial worktree or branch already exists: ${trial.branch}`);
|
|
294
|
+
}
|
|
295
|
+
const worktree = ensureExecWorktree({
|
|
296
|
+
repoRoot,
|
|
297
|
+
worktreeBase,
|
|
298
|
+
taskId: worktreeTaskId,
|
|
299
|
+
startPoint: record.base_commit,
|
|
300
|
+
});
|
|
301
|
+
prepared.set(trial.trial_id, worktree);
|
|
302
|
+
trial.status = "prepared";
|
|
303
|
+
writeComparison(recordPath, record);
|
|
304
|
+
}
|
|
305
|
+
await runPool(record.trials.map((trial) => async () => {
|
|
306
|
+
const worktree = prepared.get(trial.trial_id);
|
|
307
|
+
if (!worktree)
|
|
308
|
+
throw new Error(`Prepared worktree missing: ${trial.trial_id}`);
|
|
309
|
+
await runOneTrial({
|
|
310
|
+
record,
|
|
311
|
+
recordPath,
|
|
312
|
+
trial,
|
|
313
|
+
worktree,
|
|
314
|
+
plan: planContent,
|
|
315
|
+
prompt,
|
|
316
|
+
executorCommand: executorCommands.get(trial.agent),
|
|
317
|
+
reporterCommand: trial.reporter_agent
|
|
318
|
+
? reporterCommands.get(trial.reporter_agent)
|
|
319
|
+
: undefined,
|
|
320
|
+
reporterRateLimitDetection: trial.reporter_agent
|
|
321
|
+
? reporterRateLimitDetections.get(trial.reporter_agent)
|
|
322
|
+
: undefined,
|
|
323
|
+
parentValidationIds,
|
|
324
|
+
execDefaults: defaults,
|
|
325
|
+
repoRoot,
|
|
326
|
+
schedulePath: paths.schedulePath,
|
|
327
|
+
executionPath: paths.executionPath,
|
|
328
|
+
});
|
|
329
|
+
}), parallel);
|
|
330
|
+
process.stdout.write(`Comparison recorded: ${repoRelative(repoRoot, recordPath)}\n`);
|
|
331
|
+
}
|
|
332
|
+
function resolveRecord(opts) {
|
|
333
|
+
const paths = resolveProjectPaths({ project: opts.project });
|
|
334
|
+
activateResolvedProjectPaths(paths);
|
|
335
|
+
const path = comparisonPath(paths.executionPath, opts.comparison);
|
|
336
|
+
return { record: readComparison(path), path, repoRoot: specdojoRootDir() };
|
|
337
|
+
}
|
|
338
|
+
function findTrial(record, selector) {
|
|
339
|
+
const matches = record.trials.filter((trial) => trial.trial_id === selector || trial.agent === selector);
|
|
340
|
+
if (matches.length !== 1)
|
|
341
|
+
throw new Error(`Trial not found or ambiguous: ${selector}`);
|
|
342
|
+
return matches[0];
|
|
343
|
+
}
|
|
344
|
+
function statusTrials(opts) {
|
|
345
|
+
const { record } = resolveRecord(opts);
|
|
346
|
+
process.stdout.write(`Comparison ${record.comparison_id} [${record.status}] task=${record.task_id}\n`);
|
|
347
|
+
process.stdout.write(`plan sha256=${record.plan.sha256} prompt sha256=${record.plan.prompt_sha256}\n`);
|
|
348
|
+
process.stdout.write(`base=${record.base_commit} requested=${record.base?.requested_revision ?? "(legacy)"} head_reference=${record.base?.head_commit_at_start ?? "unknown"} compatible=${record.base?.compatible ?? "unknown"} reporter_mode=${record.reporter_mode ?? "shared-or-none"}\n`);
|
|
349
|
+
for (const warning of record.base?.warnings ?? []) {
|
|
350
|
+
process.stdout.write(`base warning: ${warning}\n`);
|
|
351
|
+
}
|
|
352
|
+
for (const trial of record.trials) {
|
|
353
|
+
const reporterStatus = trial.reporter.status ?? (trial.reporter.structured_output ? "succeeded" : "not_run");
|
|
354
|
+
const parentValidation = trial.parent_validation ?? {
|
|
355
|
+
status: "not_configured",
|
|
356
|
+
duration_ms: null,
|
|
357
|
+
validations_passed: 0,
|
|
358
|
+
validations_failed: 0,
|
|
359
|
+
validations_not_run: 0,
|
|
360
|
+
};
|
|
361
|
+
process.stdout.write(`${trial.trial_id}\t${trial.status}\ttime(executor/parent/total)=${trial.executor.duration_ms ?? "-"}/${parentValidation.duration_ms ?? "-"}/${trial.duration_ms ?? "-"}ms\tfiles=${trial.executor.files_changed}\texecutor-validations(reported/passed/failed/not_run/requires_human)=${trial.executor.validations_reported ?? trial.executor.validations.length}/${trial.executor.validations_passed}/${trial.executor.validations_failed}/${trial.executor.validations_not_run}/${trial.executor.validations_requires_human ?? 0}\tparent-validations(status/passed/failed/not_run)=${parentValidation.status}/${parentValidation.validations_passed}/${parentValidation.validations_failed}/${parentValidation.validations_not_run}\treporter=${reporterStatus}/${trial.reporter.format_attempts}/${trial.reporter.failure_category ?? "-"}\n`);
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
function rateTrial(opts) {
|
|
365
|
+
const { record, path } = resolveRecord(opts);
|
|
366
|
+
const trial = findTrial(record, opts.trial);
|
|
367
|
+
const judgment = parseRating(opts.judgmentQuality, "--judgment-quality");
|
|
368
|
+
const writing = parseRating(opts.writingQuality, "--writing-quality");
|
|
369
|
+
const scope = parseRating(opts.scopeAdherence, "--scope-adherence");
|
|
370
|
+
if (judgment !== undefined)
|
|
371
|
+
trial.subjective.judgment_quality = judgment;
|
|
372
|
+
if (writing !== undefined)
|
|
373
|
+
trial.subjective.writing_quality = writing;
|
|
374
|
+
if (scope !== undefined)
|
|
375
|
+
trial.subjective.scope_adherence = scope;
|
|
376
|
+
if (opts.notes !== undefined)
|
|
377
|
+
trial.subjective.notes = opts.notes.trim();
|
|
378
|
+
writeComparison(path, record);
|
|
379
|
+
process.stdout.write(`Rated trial: ${trial.trial_id}\n`);
|
|
380
|
+
}
|
|
381
|
+
function changedPaths(worktreePath) {
|
|
382
|
+
const output = gitOutput(worktreePath, [
|
|
383
|
+
"status",
|
|
384
|
+
"--porcelain=v1",
|
|
385
|
+
"-z",
|
|
386
|
+
"--untracked-files=all",
|
|
387
|
+
]);
|
|
388
|
+
const paths = [];
|
|
389
|
+
for (const record of output.split("\0").filter(Boolean)) {
|
|
390
|
+
if (record.length >= 4)
|
|
391
|
+
paths.push(record.slice(3));
|
|
392
|
+
}
|
|
393
|
+
return paths;
|
|
394
|
+
}
|
|
395
|
+
function removeTrialWorktree(repoRoot, trial, deleteUnmerged) {
|
|
396
|
+
const registered = findExecWorktree(repoRoot, trial.branch.slice("exec/".length));
|
|
397
|
+
const worktreePath = registered?.path ?? trial.worktree;
|
|
398
|
+
if (existsSync(worktreePath)) {
|
|
399
|
+
gitOutput(repoRoot, ["worktree", "remove", "--force", worktreePath]);
|
|
400
|
+
}
|
|
401
|
+
if (gitResult(repoRoot, ["show-ref", "--verify", "--quiet", `refs/heads/${trial.branch}`])
|
|
402
|
+
.status === 0) {
|
|
403
|
+
gitOutput(repoRoot, ["branch", deleteUnmerged ? "-D" : "-d", trial.branch]);
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
function commitSelectedTrial(repoRoot, record, trial) {
|
|
407
|
+
const protectedPaths = changedPaths(trial.worktree).filter(isAgentProtectedConfigPath);
|
|
408
|
+
if (protectedPaths.length > 0)
|
|
409
|
+
throw new Error(agentProtectedConfigViolation(protectedPaths));
|
|
410
|
+
gitOutput(trial.worktree, ["add", "-A"]);
|
|
411
|
+
const staged = gitResult(trial.worktree, ["diff", "--cached", "--quiet"]);
|
|
412
|
+
if (staged.status === 1) {
|
|
413
|
+
const commit = gitResult(trial.worktree, [
|
|
414
|
+
"commit",
|
|
415
|
+
"-m",
|
|
416
|
+
`feat(trial): adopt ${record.task_id} from ${trial.agent}`,
|
|
417
|
+
]);
|
|
418
|
+
if (commit.status !== 0) {
|
|
419
|
+
gitResult(trial.worktree, ["reset", "--quiet"]);
|
|
420
|
+
throw new Error(`Trial commit failed: ${commit.stderr || commit.stdout}`);
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
else if (staged.status !== 0) {
|
|
424
|
+
throw new Error("Failed to inspect selected trial changes.");
|
|
425
|
+
}
|
|
426
|
+
const ahead = Number(gitOutput(trial.worktree, ["rev-list", "--count", `${record.base_commit}..HEAD`]).trim());
|
|
427
|
+
if (!Number.isFinite(ahead) || ahead < 1)
|
|
428
|
+
throw new Error("Selected trial has no changes to adopt.");
|
|
429
|
+
}
|
|
430
|
+
function adoptTrial(opts) {
|
|
431
|
+
const { record, path, repoRoot } = resolveRecord(opts);
|
|
432
|
+
if (record.status !== "active")
|
|
433
|
+
throw new Error(`Comparison is already ${record.status}.`);
|
|
434
|
+
const selected = findTrial(record, opts.trial);
|
|
435
|
+
if (selected.status !== "succeeded")
|
|
436
|
+
throw new Error(`Only a succeeded trial can be adopted: ${selected.trial_id}`);
|
|
437
|
+
if (opts.dryRun) {
|
|
438
|
+
process.stdout.write(`[dry-run] adopt ${selected.branch}; discard ${record.trials.length - 1} other trial(s)\n`);
|
|
439
|
+
return;
|
|
440
|
+
}
|
|
441
|
+
commitSelectedTrial(repoRoot, record, selected);
|
|
442
|
+
try {
|
|
443
|
+
gitOutput(repoRoot, ["merge", "--no-ff", "--no-edit", selected.branch]);
|
|
444
|
+
}
|
|
445
|
+
catch (error) {
|
|
446
|
+
gitResult(repoRoot, ["merge", "--abort"]);
|
|
447
|
+
throw error;
|
|
448
|
+
}
|
|
449
|
+
removeTrialWorktree(repoRoot, selected, false);
|
|
450
|
+
selected.status = "adopted";
|
|
451
|
+
for (const trial of record.trials) {
|
|
452
|
+
if (trial === selected)
|
|
453
|
+
continue;
|
|
454
|
+
removeTrialWorktree(repoRoot, trial, true);
|
|
455
|
+
trial.status = "discarded";
|
|
456
|
+
}
|
|
457
|
+
record.status = "adopted";
|
|
458
|
+
record.selected_trial_id = selected.trial_id;
|
|
459
|
+
record.completed_at = new Date().toISOString();
|
|
460
|
+
writeComparison(path, record);
|
|
461
|
+
process.stdout.write(`Adopted trial ${selected.trial_id}; discarded remaining trial worktrees.\n`);
|
|
462
|
+
}
|
|
463
|
+
function discardTrials(opts) {
|
|
464
|
+
const { record, path, repoRoot } = resolveRecord(opts);
|
|
465
|
+
if (record.status !== "active")
|
|
466
|
+
throw new Error(`Comparison is already ${record.status}.`);
|
|
467
|
+
for (const trial of record.trials) {
|
|
468
|
+
removeTrialWorktree(repoRoot, trial, true);
|
|
469
|
+
trial.status = "discarded";
|
|
470
|
+
}
|
|
471
|
+
record.status = "discarded";
|
|
472
|
+
record.completed_at = new Date().toISOString();
|
|
473
|
+
writeComparison(path, record);
|
|
474
|
+
process.stdout.write(`Discarded all trial worktrees; comparison record retained.\n`);
|
|
475
|
+
}
|
|
476
|
+
function commandError(error) {
|
|
477
|
+
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
|
|
478
|
+
process.exitCode = 1;
|
|
479
|
+
}
|
|
480
|
+
export function registerExecTrialCommands(exec) {
|
|
481
|
+
const trial = exec.command("trial").description("Compare multiple agents on one immutable plan");
|
|
482
|
+
trial
|
|
483
|
+
.command("run")
|
|
484
|
+
.requiredOption("--project <projectId>", "Project id")
|
|
485
|
+
.requiredOption("--plan <path>", "Existing SpecDojo plan used unchanged by every trial")
|
|
486
|
+
.option("--agent <nicknames...>", "Two or more executor agent nicknames")
|
|
487
|
+
.option("--pair <executor=reporter...>", "Two or more executor/reporter pairs (alternative to --agent)")
|
|
488
|
+
.option("--reporter-by <nickname>", "Shared reporter agent used to assess structured output")
|
|
489
|
+
.option("--base <revision>", "Commit used as the trial worktree start point (default: HEAD)")
|
|
490
|
+
.option("--comparison <id>", "Stable comparison id (generated when omitted)")
|
|
491
|
+
.option("--parallel <n>", "Maximum concurrent trials", "1")
|
|
492
|
+
.option("--worktree-base <path>", "Override worktree base directory")
|
|
493
|
+
.option("--exec-defaults <path>", "Path to exec-defaults.yaml")
|
|
494
|
+
.option("--dry-run", "Print the planned comparison without creating worktrees", false)
|
|
495
|
+
.action(async (opts) => {
|
|
496
|
+
try {
|
|
497
|
+
await runTrials(opts);
|
|
498
|
+
}
|
|
499
|
+
catch (error) {
|
|
500
|
+
commandError(error);
|
|
501
|
+
}
|
|
502
|
+
});
|
|
503
|
+
trial
|
|
504
|
+
.command("status")
|
|
505
|
+
.requiredOption("--project <projectId>", "Project id")
|
|
506
|
+
.requiredOption("--comparison <id>", "Comparison id")
|
|
507
|
+
.action((opts) => {
|
|
508
|
+
try {
|
|
509
|
+
statusTrials(opts);
|
|
510
|
+
}
|
|
511
|
+
catch (error) {
|
|
512
|
+
commandError(error);
|
|
513
|
+
}
|
|
514
|
+
});
|
|
515
|
+
trial
|
|
516
|
+
.command("rate")
|
|
517
|
+
.requiredOption("--project <projectId>", "Project id")
|
|
518
|
+
.requiredOption("--comparison <id>", "Comparison id")
|
|
519
|
+
.requiredOption("--trial <idOrAgent>", "Trial id or agent nickname")
|
|
520
|
+
.option("--judgment-quality <1-5>", "Human judgment-quality rating")
|
|
521
|
+
.option("--writing-quality <1-5>", "Human writing-quality rating")
|
|
522
|
+
.option("--scope-adherence <1-5>", "Human scope-adherence rating")
|
|
523
|
+
.option("--notes <text>", "Human evaluation notes")
|
|
524
|
+
.action((opts) => {
|
|
525
|
+
try {
|
|
526
|
+
rateTrial(opts);
|
|
527
|
+
}
|
|
528
|
+
catch (error) {
|
|
529
|
+
commandError(error);
|
|
530
|
+
}
|
|
531
|
+
});
|
|
532
|
+
trial
|
|
533
|
+
.command("adopt")
|
|
534
|
+
.requiredOption("--project <projectId>", "Project id")
|
|
535
|
+
.requiredOption("--comparison <id>", "Comparison id")
|
|
536
|
+
.requiredOption("--trial <idOrAgent>", "Succeeded trial to merge")
|
|
537
|
+
.option("--dry-run", "Print adopt/discard actions", false)
|
|
538
|
+
.action((opts) => {
|
|
539
|
+
try {
|
|
540
|
+
adoptTrial(opts);
|
|
541
|
+
}
|
|
542
|
+
catch (error) {
|
|
543
|
+
commandError(error);
|
|
544
|
+
}
|
|
545
|
+
});
|
|
546
|
+
trial
|
|
547
|
+
.command("discard")
|
|
548
|
+
.requiredOption("--project <projectId>", "Project id")
|
|
549
|
+
.requiredOption("--comparison <id>", "Comparison id")
|
|
550
|
+
.action((opts) => {
|
|
551
|
+
try {
|
|
552
|
+
discardTrials(opts);
|
|
553
|
+
}
|
|
554
|
+
catch (error) {
|
|
555
|
+
commandError(error);
|
|
556
|
+
}
|
|
557
|
+
});
|
|
558
|
+
}
|