switchboard-cli 0.1.0-alpha.4 → 0.1.0-alpha.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/README.md +121 -65
- package/bin/switchboard.js +75 -0
- package/dist/index.cjs +14371 -0
- package/package.json +45 -8
- package/bin/switchboard.mjs +0 -49
- package/src/autoagent/boundary-check.spec.ts +0 -77
- package/src/autoagent/boundary-check.ts +0 -102
- package/src/autoagent/loop.ts +0 -327
- package/src/autoagent/results.spec.ts +0 -73
- package/src/autoagent/results.ts +0 -68
- package/src/autoagent/runner.spec.ts +0 -20
- package/src/autoagent/runner.ts +0 -92
- package/src/autoagent/types.ts +0 -64
- package/src/commands/audit-codex.ts +0 -266
- package/src/commands/autoagent.ts +0 -108
- package/src/commands/calibrate.ts +0 -70
- package/src/commands/compile.ts +0 -117
- package/src/commands/evaluate.ts +0 -103
- package/src/commands/ingest.ts +0 -250
- package/src/commands/init.ts +0 -133
- package/src/commands/packet.ts +0 -408
- package/src/commands/receipt.ts +0 -336
- package/src/commands/run-claude.ts +0 -355
- package/src/index.ts +0 -47
- package/src/lib/draft-return.ts +0 -278
- package/src/lib/drift-guard.ts +0 -105
- package/src/lib/errors.ts +0 -61
- package/src/lib/output.ts +0 -43
- package/src/lib/paths.ts +0 -125
- package/src/lib/proof.ts +0 -262
- package/src/lib/transport.ts +0 -276
- package/src/lib/yaml-io.ts +0 -62
- package/src/store/filesystem-store.ts +0 -326
package/src/autoagent/results.ts
DELETED
|
@@ -1,68 +0,0 @@
|
|
|
1
|
-
import { appendFileSync, readFileSync, existsSync, mkdirSync } from "fs";
|
|
2
|
-
import { dirname } from "path";
|
|
3
|
-
import type { IterationResult } from "./types";
|
|
4
|
-
|
|
5
|
-
const HEADER = "iteration\tcommit\tscore\tprev_score\ttests\tstatus\tdescription";
|
|
6
|
-
|
|
7
|
-
export class ResultsLedger {
|
|
8
|
-
private path: string;
|
|
9
|
-
|
|
10
|
-
constructor(path: string) {
|
|
11
|
-
this.path = path;
|
|
12
|
-
const dir = dirname(path);
|
|
13
|
-
if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
|
|
14
|
-
}
|
|
15
|
-
|
|
16
|
-
append(result: IterationResult): void {
|
|
17
|
-
if (!existsSync(this.path)) {
|
|
18
|
-
appendFileSync(this.path, HEADER + "\n", "utf-8");
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
const line = [
|
|
22
|
-
result.iteration,
|
|
23
|
-
result.commit,
|
|
24
|
-
result.score.toFixed(4),
|
|
25
|
-
result.prevScore.toFixed(4),
|
|
26
|
-
result.testSummary,
|
|
27
|
-
result.status,
|
|
28
|
-
result.description,
|
|
29
|
-
].join("\t");
|
|
30
|
-
|
|
31
|
-
appendFileSync(this.path, line + "\n", "utf-8");
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
readAll(): IterationResult[] {
|
|
35
|
-
if (!existsSync(this.path)) return [];
|
|
36
|
-
|
|
37
|
-
const lines = readFileSync(this.path, "utf-8").trim().split("\n");
|
|
38
|
-
if (lines.length <= 1) return [];
|
|
39
|
-
|
|
40
|
-
return lines.slice(1).filter(Boolean).map(line => {
|
|
41
|
-
const parts = line.split("\t");
|
|
42
|
-
return {
|
|
43
|
-
iteration: parseInt(parts[0]!, 10),
|
|
44
|
-
commit: parts[1]!,
|
|
45
|
-
score: parseFloat(parts[2]!),
|
|
46
|
-
prevScore: parseFloat(parts[3]!),
|
|
47
|
-
testsPass: true,
|
|
48
|
-
testSummary: parts[4]!,
|
|
49
|
-
status: parts[5]! as IterationResult["status"],
|
|
50
|
-
description: parts[6]!,
|
|
51
|
-
filesChanged: [],
|
|
52
|
-
timestamp: 0,
|
|
53
|
-
durationMs: 0,
|
|
54
|
-
};
|
|
55
|
-
});
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
getBest(): IterationResult | null {
|
|
59
|
-
const kept = this.readAll().filter(r => r.status === "keep");
|
|
60
|
-
if (kept.length === 0) return null;
|
|
61
|
-
return kept.reduce((best, r) => r.score > best.score ? r : best);
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
format(): string {
|
|
65
|
-
if (!existsSync(this.path)) return "No results yet.";
|
|
66
|
-
return readFileSync(this.path, "utf-8");
|
|
67
|
-
}
|
|
68
|
-
}
|
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
import { describe, it, expect } from "vitest";
|
|
2
|
-
import { runTests, runScore, getChangedFiles, getShortCommit } from "./runner";
|
|
3
|
-
|
|
4
|
-
describe("runner", () => {
|
|
5
|
-
it("runTests returns pass/fail and summary", async () => {
|
|
6
|
-
const result = await runTests("echo '3 tests passed'", process.cwd());
|
|
7
|
-
expect(result.passed).toBe(true);
|
|
8
|
-
expect(result.output).toContain("3 tests passed");
|
|
9
|
-
});
|
|
10
|
-
|
|
11
|
-
it("runTests detects failure from non-zero exit", async () => {
|
|
12
|
-
const result = await runTests("exit 1", process.cwd());
|
|
13
|
-
expect(result.passed).toBe(false);
|
|
14
|
-
});
|
|
15
|
-
|
|
16
|
-
it("getShortCommit returns 7-char hash", async () => {
|
|
17
|
-
const hash = await getShortCommit(process.cwd());
|
|
18
|
-
expect(hash.length).toBe(7);
|
|
19
|
-
});
|
|
20
|
-
});
|
package/src/autoagent/runner.ts
DELETED
|
@@ -1,92 +0,0 @@
|
|
|
1
|
-
import { execSync } from "child_process";
|
|
2
|
-
import type { ScoreResult } from "./types";
|
|
3
|
-
|
|
4
|
-
// ---------------------------------------------------------------------------
|
|
5
|
-
// Test runner — executes project test command
|
|
6
|
-
// ---------------------------------------------------------------------------
|
|
7
|
-
|
|
8
|
-
export async function runTests(
|
|
9
|
-
command: string,
|
|
10
|
-
cwd: string,
|
|
11
|
-
): Promise<{ passed: boolean; output: string }> {
|
|
12
|
-
try {
|
|
13
|
-
const output = execSync(command, {
|
|
14
|
-
cwd,
|
|
15
|
-
encoding: "utf-8",
|
|
16
|
-
timeout: 300_000, // 5 min max
|
|
17
|
-
stdio: ["pipe", "pipe", "pipe"],
|
|
18
|
-
});
|
|
19
|
-
return { passed: true, output: output.slice(-500) };
|
|
20
|
-
} catch (err: any) {
|
|
21
|
-
const output = (err.stdout ?? "") + (err.stderr ?? "");
|
|
22
|
-
return { passed: false, output: output.slice(-500) };
|
|
23
|
-
}
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
// ---------------------------------------------------------------------------
|
|
27
|
-
// Score runner — executes project score function
|
|
28
|
-
// ---------------------------------------------------------------------------
|
|
29
|
-
|
|
30
|
-
export async function runScore(
|
|
31
|
-
command: string,
|
|
32
|
-
cwd: string,
|
|
33
|
-
): Promise<ScoreResult> {
|
|
34
|
-
try {
|
|
35
|
-
const output = execSync(command, {
|
|
36
|
-
cwd,
|
|
37
|
-
encoding: "utf-8",
|
|
38
|
-
timeout: 600_000, // 10 min max for scoring (may run engine)
|
|
39
|
-
stdio: ["pipe", "pipe", "pipe"],
|
|
40
|
-
});
|
|
41
|
-
|
|
42
|
-
// Score function should output JSON to stdout
|
|
43
|
-
const jsonMatch = output.match(/\{[\s\S]*"score"[\s\S]*\}/);
|
|
44
|
-
if (!jsonMatch) {
|
|
45
|
-
return { score: 0, breakdown: {}, summary: "Score function produced no JSON output" };
|
|
46
|
-
}
|
|
47
|
-
return JSON.parse(jsonMatch[0]) as ScoreResult;
|
|
48
|
-
} catch (err: any) {
|
|
49
|
-
return { score: 0, breakdown: {}, summary: `Score error: ${(err.message ?? "").slice(0, 200)}` };
|
|
50
|
-
}
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
// ---------------------------------------------------------------------------
|
|
54
|
-
// Git helpers
|
|
55
|
-
// ---------------------------------------------------------------------------
|
|
56
|
-
|
|
57
|
-
export async function getChangedFiles(cwd: string): Promise<string[]> {
|
|
58
|
-
try {
|
|
59
|
-
const output = execSync("git diff --name-only HEAD~1 HEAD", {
|
|
60
|
-
cwd,
|
|
61
|
-
encoding: "utf-8",
|
|
62
|
-
});
|
|
63
|
-
return output.trim().split("\n").filter(Boolean);
|
|
64
|
-
} catch {
|
|
65
|
-
// If no previous commit or error, check working tree
|
|
66
|
-
try {
|
|
67
|
-
const output = execSync("git diff --name-only", { cwd, encoding: "utf-8" });
|
|
68
|
-
return output.trim().split("\n").filter(Boolean);
|
|
69
|
-
} catch {
|
|
70
|
-
return [];
|
|
71
|
-
}
|
|
72
|
-
}
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
export async function getShortCommit(cwd: string): Promise<string> {
|
|
76
|
-
try {
|
|
77
|
-
return execSync("git rev-parse --short HEAD", { cwd, encoding: "utf-8" }).trim();
|
|
78
|
-
} catch {
|
|
79
|
-
return "0000000";
|
|
80
|
-
}
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
export async function gitRevert(cwd: string): Promise<void> {
|
|
84
|
-
execSync("git revert --no-commit HEAD", { cwd, encoding: "utf-8" });
|
|
85
|
-
execSync('git commit -m "autoagent: discard — reverted last iteration"', { cwd, encoding: "utf-8" });
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
export async function gitCommit(cwd: string, message: string): Promise<string> {
|
|
89
|
-
execSync("git add -A", { cwd, encoding: "utf-8" });
|
|
90
|
-
execSync(`git commit -m "${message.replace(/"/g, '\\"')}" --allow-empty`, { cwd, encoding: "utf-8" });
|
|
91
|
-
return getShortCommit(cwd);
|
|
92
|
-
}
|
package/src/autoagent/types.ts
DELETED
|
@@ -1,64 +0,0 @@
|
|
|
1
|
-
// ---------------------------------------------------------------------------
|
|
2
|
-
// AutoAgent types
|
|
3
|
-
// ---------------------------------------------------------------------------
|
|
4
|
-
|
|
5
|
-
export type AutoAgentTier = "code" | "harness" | "full";
|
|
6
|
-
|
|
7
|
-
export type IterationStatus = "keep" | "discard" | "crash" | "boundary-violation";
|
|
8
|
-
|
|
9
|
-
export interface AutoAgentConfig {
|
|
10
|
-
tier: AutoAgentTier;
|
|
11
|
-
maxIterations: number; // 0 = unlimited
|
|
12
|
-
maxTimeMs: number; // 0 = unlimited
|
|
13
|
-
scoreTarget: number; // 0 = no target
|
|
14
|
-
testCommand: string; // default: "pnpm test"
|
|
15
|
-
scoreCommand: string; // default: "npx tsx .switchboard/autoagent/score.ts"
|
|
16
|
-
receiptEveryN: number; // default: 10
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
export interface ScoreResult {
|
|
20
|
-
score: number;
|
|
21
|
-
breakdown: Record<string, number>;
|
|
22
|
-
summary: string;
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
export interface IterationResult {
|
|
26
|
-
iteration: number;
|
|
27
|
-
commit: string;
|
|
28
|
-
score: number;
|
|
29
|
-
prevScore: number;
|
|
30
|
-
testsPass: boolean;
|
|
31
|
-
testSummary: string;
|
|
32
|
-
status: IterationStatus;
|
|
33
|
-
description: string;
|
|
34
|
-
filesChanged: string[];
|
|
35
|
-
timestamp: number;
|
|
36
|
-
durationMs: number;
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
export interface AutoAgentState {
|
|
40
|
-
startedAt: number;
|
|
41
|
-
iteration: number;
|
|
42
|
-
bestScore: number;
|
|
43
|
-
bestCommit: string;
|
|
44
|
-
currentScore: number;
|
|
45
|
-
results: IterationResult[];
|
|
46
|
-
status: "running" | "stopped" | "target-reached" | "max-iterations" | "max-time" | "tests-failing";
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
export interface BoundarySpec {
|
|
50
|
-
tier: AutoAgentTier;
|
|
51
|
-
modifiable: string[]; // glob patterns of files that can be modified
|
|
52
|
-
frozen: string[]; // files that can NEVER be modified
|
|
53
|
-
contractNonGoals: string[];
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
export const DEFAULT_CONFIG: AutoAgentConfig = {
|
|
57
|
-
tier: "code",
|
|
58
|
-
maxIterations: 0,
|
|
59
|
-
maxTimeMs: 0,
|
|
60
|
-
scoreTarget: 0,
|
|
61
|
-
testCommand: "pnpm test",
|
|
62
|
-
scoreCommand: "npx tsx .switchboard/autoagent/score.ts",
|
|
63
|
-
receiptEveryN: 10,
|
|
64
|
-
};
|
|
@@ -1,266 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* switchboard audit codex
|
|
3
|
-
*
|
|
4
|
-
* Generates a Codex audit packet from the latest dispatch + return + receipt.
|
|
5
|
-
* This is a packet-generation surface — it prepares audit context for Codex
|
|
6
|
-
* but does not execute the Codex transport.
|
|
7
|
-
*
|
|
8
|
-
* Uses existing @switchboard/core: shouldRequireAudit, computeAuditResult,
|
|
9
|
-
* formatDispatchPacket.
|
|
10
|
-
*
|
|
11
|
-
* Does not require the hosted web app or Supabase.
|
|
12
|
-
*/
|
|
13
|
-
|
|
14
|
-
import { randomBytes } from "crypto";
|
|
15
|
-
import { existsSync } from "fs";
|
|
16
|
-
import { join } from "path";
|
|
17
|
-
import type { Command } from "commander";
|
|
18
|
-
import {
|
|
19
|
-
shouldRequireAudit,
|
|
20
|
-
computeAuditResult,
|
|
21
|
-
noAuditRequired,
|
|
22
|
-
parseStructuredReturn,
|
|
23
|
-
formatDispatchPacket,
|
|
24
|
-
type Surface,
|
|
25
|
-
} from "@switchboard/core";
|
|
26
|
-
import { requireRepoRoot, packetsDir, returnsDir, sbDir } from "../lib/paths";
|
|
27
|
-
import { readMarkdown, readYaml } from "../lib/yaml-io";
|
|
28
|
-
import {
|
|
29
|
-
loadContract,
|
|
30
|
-
loadCurrentState,
|
|
31
|
-
loadReceiptByDispatch,
|
|
32
|
-
loadLatestReceipt,
|
|
33
|
-
loadSpec,
|
|
34
|
-
writePacket,
|
|
35
|
-
} from "../store/filesystem-store";
|
|
36
|
-
import * as out from "../lib/output";
|
|
37
|
-
import { withCliErrors } from "../lib/errors";
|
|
38
|
-
import { detectCodexTransport, executeCodexTransport } from "../lib/transport";
|
|
39
|
-
import { captureTranscriptProof, buildAndWriteProofManifest } from "../lib/proof";
|
|
40
|
-
|
|
41
|
-
export function registerAuditCodexCommand(program: Command): void {
|
|
42
|
-
program
|
|
43
|
-
.command("audit")
|
|
44
|
-
.argument("<surface>", "Audit surface: codex")
|
|
45
|
-
.description("Generate an audit packet for Codex review")
|
|
46
|
-
.action(withCliErrors(async (surfaceArg: string) => {
|
|
47
|
-
if (surfaceArg !== "codex") {
|
|
48
|
-
out.fail(`Wave 2 audit only supports 'codex'. Got: "${surfaceArg}"`);
|
|
49
|
-
process.exitCode = 1;
|
|
50
|
-
return;
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
const repoRoot = requireRepoRoot();
|
|
54
|
-
|
|
55
|
-
out.heading("switchboard audit codex");
|
|
56
|
-
|
|
57
|
-
// 1. Load state
|
|
58
|
-
const contract = loadContract(repoRoot);
|
|
59
|
-
const current = loadCurrentState(repoRoot);
|
|
60
|
-
const spec = loadSpec(repoRoot) ?? "";
|
|
61
|
-
|
|
62
|
-
// 2. Anchor to latest-ingest.yaml — the governed ingest pointer.
|
|
63
|
-
// This ensures the audit targets the exact dispatch that was
|
|
64
|
-
// ingested, not a stale handoff or un-ingested dispatch.
|
|
65
|
-
const pointerPath = join(sbDir(repoRoot), "working", "latest-ingest.yaml");
|
|
66
|
-
const pointer = readYaml<{
|
|
67
|
-
dispatch_id?: string;
|
|
68
|
-
has_proof_manifest?: boolean;
|
|
69
|
-
proof_manifest_id?: string;
|
|
70
|
-
}>(pointerPath);
|
|
71
|
-
|
|
72
|
-
if (!pointer?.dispatch_id) {
|
|
73
|
-
out.fail("No ingested dispatch found. Run the full loop first:");
|
|
74
|
-
out.bullet("switchboard run claude → switchboard ingest → switchboard audit codex");
|
|
75
|
-
process.exitCode = 1;
|
|
76
|
-
return;
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
const dispatchId = pointer.dispatch_id;
|
|
80
|
-
out.section("Auditing dispatch", dispatchId);
|
|
81
|
-
|
|
82
|
-
// 3. Load the return data for the ingested dispatch
|
|
83
|
-
const returnPath = join(returnsDir(repoRoot), `${dispatchId}-return.yaml`);
|
|
84
|
-
const rawReturn = readMarkdown(returnPath);
|
|
85
|
-
|
|
86
|
-
if (!rawReturn) {
|
|
87
|
-
out.fail(`No return data for dispatch ${dispatchId}. The ingest pointer exists but the return file is missing.`);
|
|
88
|
-
process.exitCode = 1;
|
|
89
|
-
return;
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
const report = parseStructuredReturn(rawReturn);
|
|
93
|
-
out.success(`Loaded return: ${report.status}, ${report.changed_files.length} file(s)`);
|
|
94
|
-
|
|
95
|
-
// 4. Run audit check
|
|
96
|
-
const auditCheck = shouldRequireAudit({
|
|
97
|
-
report,
|
|
98
|
-
complexityMode: null,
|
|
99
|
-
taskType: "implementation-request",
|
|
100
|
-
});
|
|
101
|
-
|
|
102
|
-
const audit = auditCheck.required
|
|
103
|
-
? computeAuditResult({ report, triggers: auditCheck.triggers, returnId: dispatchId })
|
|
104
|
-
: noAuditRequired(dispatchId);
|
|
105
|
-
|
|
106
|
-
out.section("Audit required", audit.audit_required ? "yes" : "no");
|
|
107
|
-
out.section("Outcome", audit.outcome);
|
|
108
|
-
|
|
109
|
-
if (audit.findings.length > 0) {
|
|
110
|
-
for (const f of audit.findings) {
|
|
111
|
-
const icon = f.severity === "blocker" ? "BLOCKED" : "CAUTION";
|
|
112
|
-
out.bullet(`[${icon}] ${f.check}: ${f.detail}`);
|
|
113
|
-
}
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
// 5. Load receipt for this dispatch by scanning receipt files for matching dispatch_id
|
|
117
|
-
let receipt = loadReceiptByDispatch(repoRoot, dispatchId);
|
|
118
|
-
if (!receipt) {
|
|
119
|
-
// Fallback to latest, but warn if it belongs to a different dispatch
|
|
120
|
-
receipt = loadLatestReceipt(repoRoot);
|
|
121
|
-
if (receipt && (receipt as any).dispatch_id !== dispatchId) {
|
|
122
|
-
out.warn(`Latest receipt belongs to dispatch ${(receipt as any).dispatch_id}, not ${dispatchId}. Using for context only.`);
|
|
123
|
-
}
|
|
124
|
-
}
|
|
125
|
-
|
|
126
|
-
// 6. Generate the Codex audit packet
|
|
127
|
-
const auditDispatchId = `codex-audit-${randomBytes(6).toString("hex")}`;
|
|
128
|
-
const auditPromptSections: string[] = [];
|
|
129
|
-
|
|
130
|
-
auditPromptSections.push("# Codex Audit Packet\n");
|
|
131
|
-
auditPromptSections.push(`**Project:** ${contract.title}`);
|
|
132
|
-
auditPromptSections.push(`**Dispatch:** ${dispatchId}`);
|
|
133
|
-
auditPromptSections.push(`**Return status:** ${report.status}`);
|
|
134
|
-
auditPromptSections.push(`**Objective met:** ${report.objective_met}\n`);
|
|
135
|
-
|
|
136
|
-
auditPromptSections.push("## Audit Findings\n");
|
|
137
|
-
if (audit.findings.length === 0) {
|
|
138
|
-
auditPromptSections.push("No audit findings.\n");
|
|
139
|
-
} else {
|
|
140
|
-
for (const f of audit.findings) {
|
|
141
|
-
auditPromptSections.push(`- **[${f.severity}]** ${f.check}: ${f.detail}`);
|
|
142
|
-
if (f.files_affected.length > 0) {
|
|
143
|
-
auditPromptSections.push(` Files: ${f.files_affected.join(", ")}`);
|
|
144
|
-
}
|
|
145
|
-
}
|
|
146
|
-
}
|
|
147
|
-
|
|
148
|
-
auditPromptSections.push("\n## Changed Files\n");
|
|
149
|
-
for (const f of report.changed_files) {
|
|
150
|
-
auditPromptSections.push(`- ${f}`);
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
auditPromptSections.push("\n## Verification Runs\n");
|
|
154
|
-
if (report.verification_run.length === 0) {
|
|
155
|
-
auditPromptSections.push("None reported.\n");
|
|
156
|
-
} else {
|
|
157
|
-
for (const v of report.verification_run) {
|
|
158
|
-
auditPromptSections.push(`- ${v}`);
|
|
159
|
-
}
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
auditPromptSections.push("\n## Evidence\n");
|
|
163
|
-
if (report.evidence.length === 0) {
|
|
164
|
-
auditPromptSections.push("None reported.\n");
|
|
165
|
-
} else {
|
|
166
|
-
for (const e of report.evidence) {
|
|
167
|
-
auditPromptSections.push(`- ${e}`);
|
|
168
|
-
}
|
|
169
|
-
}
|
|
170
|
-
|
|
171
|
-
auditPromptSections.push("\n## Return Summary\n");
|
|
172
|
-
auditPromptSections.push(report.summary);
|
|
173
|
-
|
|
174
|
-
if (receipt) {
|
|
175
|
-
auditPromptSections.push("\n## Receipt Context\n");
|
|
176
|
-
auditPromptSections.push(`- Receipt ID: ${receipt.receipt_id}`);
|
|
177
|
-
const verdict = (receipt as any).issuance_assessment?.verdict ?? (receipt as any).trust_posture ?? "unknown";
|
|
178
|
-
auditPromptSections.push(`- Verdict: ${verdict}`);
|
|
179
|
-
auditPromptSections.push(`- Trust posture: ${(receipt as any).trust_posture ?? "unknown"}`);
|
|
180
|
-
auditPromptSections.push(`- Residuals: ${(receipt as any).residual_count ?? 0}`);
|
|
181
|
-
}
|
|
182
|
-
|
|
183
|
-
auditPromptSections.push("\n---\n");
|
|
184
|
-
auditPromptSections.push("## Audit Instructions\n");
|
|
185
|
-
auditPromptSections.push("Review the changed files against the audit findings.");
|
|
186
|
-
auditPromptSections.push("Verify that sensitive systems (auth, payment, migration) have adequate testing.");
|
|
187
|
-
auditPromptSections.push("Report any additional concerns not caught by the pattern-based audit.");
|
|
188
|
-
|
|
189
|
-
const auditContent = auditPromptSections.join("\n");
|
|
190
|
-
|
|
191
|
-
// 7. Write the audit packet
|
|
192
|
-
const timestamp = new Date().toISOString().replace(/[:.]/g, "-").slice(0, 19);
|
|
193
|
-
const filename = `codex-audit-${timestamp}-${auditDispatchId.slice(12)}.md`;
|
|
194
|
-
|
|
195
|
-
const packetContent = [
|
|
196
|
-
"---",
|
|
197
|
-
`dispatch_id: ${auditDispatchId}`,
|
|
198
|
-
`surface: codex`,
|
|
199
|
-
`audit_for_dispatch: ${dispatchId}`,
|
|
200
|
-
`generated_at: ${new Date().toISOString()}`,
|
|
201
|
-
"---\n",
|
|
202
|
-
auditContent,
|
|
203
|
-
].join("\n");
|
|
204
|
-
|
|
205
|
-
const packetPath = writePacket(repoRoot, filename, packetContent);
|
|
206
|
-
out.fileWritten("Codex audit packet", packetPath);
|
|
207
|
-
|
|
208
|
-
out.heading("Audit packet ready");
|
|
209
|
-
out.section("Audit dispatch", auditDispatchId);
|
|
210
|
-
out.section("Systems audited", audit.systems_audited.join(", ") || "baseline only");
|
|
211
|
-
|
|
212
|
-
// 8. Attempt automatic Codex transport
|
|
213
|
-
const transport = detectCodexTransport();
|
|
214
|
-
out.section("Transport", `${transport.status} — ${transport.detail}`);
|
|
215
|
-
|
|
216
|
-
if (transport.status !== "ready") {
|
|
217
|
-
if (transport.status === "no-cli") {
|
|
218
|
-
out.warn("Codex CLI not found — falling back to manual audit.");
|
|
219
|
-
out.info("To enable automatic audit:");
|
|
220
|
-
out.bullet(" npm install -g @openai/codex");
|
|
221
|
-
} else if (transport.status === "cli-error") {
|
|
222
|
-
out.warn(`Codex CLI found but not invocable: ${transport.detail}`);
|
|
223
|
-
} else {
|
|
224
|
-
out.warn(`Transport not ready: ${transport.detail}`);
|
|
225
|
-
}
|
|
226
|
-
out.bullet("Send the audit packet to Codex for independent review");
|
|
227
|
-
return;
|
|
228
|
-
}
|
|
229
|
-
|
|
230
|
-
// Transport ready — execute via Codex CLI (read-only review)
|
|
231
|
-
const auditTitle = `Switchboard audit for dispatch ${dispatchId} — ${contract.title}`;
|
|
232
|
-
out.info("Executing audit via Codex CLI (codex review --uncommitted)...");
|
|
233
|
-
const result = executeCodexTransport(transport.cli_path!, auditTitle, repoRoot);
|
|
234
|
-
|
|
235
|
-
// Capture audit transcript as proof and rebuild manifest
|
|
236
|
-
captureTranscriptProof(repoRoot, dispatchId, result.transcript, "codex-audit", result.duration_ms);
|
|
237
|
-
const updatedManifest = buildAndWriteProofManifest(repoRoot, dispatchId);
|
|
238
|
-
out.info(`Proof manifest rebuilt: ${updatedManifest.artifacts.length} artifact(s), ${updatedManifest.provenance_summary.captured_count} captured`);
|
|
239
|
-
|
|
240
|
-
if (result.success) {
|
|
241
|
-
out.success(`Codex audit completed (${result.duration_ms}ms)`);
|
|
242
|
-
|
|
243
|
-
// Write the Codex response alongside the packet
|
|
244
|
-
const responseFilename = `codex-audit-response-${auditDispatchId.slice(12)}.md`;
|
|
245
|
-
writePacket(repoRoot, responseFilename, [
|
|
246
|
-
"---",
|
|
247
|
-
`audit_dispatch_id: ${auditDispatchId}`,
|
|
248
|
-
`source_dispatch: ${dispatchId}`,
|
|
249
|
-
`completed_at: ${new Date().toISOString()}`,
|
|
250
|
-
`duration_ms: ${result.duration_ms}`,
|
|
251
|
-
"---\n",
|
|
252
|
-
"# Codex Audit Response\n",
|
|
253
|
-
result.transcript,
|
|
254
|
-
].join("\n"));
|
|
255
|
-
out.fileWritten("Codex audit response", responseFilename);
|
|
256
|
-
out.info("Audit transcript captured as proof artifact.");
|
|
257
|
-
} else {
|
|
258
|
-
out.warn(`Codex audit exited with code ${result.exit_code} (${result.duration_ms}ms)`);
|
|
259
|
-
if (result.error) {
|
|
260
|
-
out.info(`Error: ${result.error.slice(0, 200)}`);
|
|
261
|
-
}
|
|
262
|
-
out.info("Partial transcript captured. Review the audit packet manually.");
|
|
263
|
-
out.bullet("Send the audit packet to Codex for independent review");
|
|
264
|
-
}
|
|
265
|
-
}));
|
|
266
|
-
}
|
|
@@ -1,108 +0,0 @@
|
|
|
1
|
-
import type { Command } from "commander";
|
|
2
|
-
import { join } from "path";
|
|
3
|
-
import { existsSync } from "fs";
|
|
4
|
-
import { requireRepoRoot, sbDir } from "../lib/paths";
|
|
5
|
-
import { loadContract } from "../store/filesystem-store";
|
|
6
|
-
import { AutoAgentLoop } from "../autoagent/loop";
|
|
7
|
-
import { ResultsLedger } from "../autoagent/results";
|
|
8
|
-
import type { AutoAgentTier } from "../autoagent/types";
|
|
9
|
-
import * as out from "../lib/output";
|
|
10
|
-
import { withCliErrors } from "../lib/errors";
|
|
11
|
-
|
|
12
|
-
export function registerAutoAgentCommand(program: Command): void {
|
|
13
|
-
program
|
|
14
|
-
.command("autoagent")
|
|
15
|
-
.description("Run governed autonomous improvement loop")
|
|
16
|
-
.option("--tier <tier>", "Autonomy tier: code, harness, full", "code")
|
|
17
|
-
.option("--max-iterations <n>", "Stop after N iterations", "0")
|
|
18
|
-
.option("--max-time <duration>", "Stop after duration (e.g., 4h, 30m)", "0")
|
|
19
|
-
.option("--score-target <n>", "Stop when score reaches threshold", "0")
|
|
20
|
-
.option("--results", "Show results.tsv")
|
|
21
|
-
.option("--rollback", "Rollback to best-scoring iteration")
|
|
22
|
-
.action(withCliErrors(async (opts) => {
|
|
23
|
-
const repoRoot = requireRepoRoot();
|
|
24
|
-
|
|
25
|
-
out.heading("switchboard autoagent");
|
|
26
|
-
|
|
27
|
-
// Show results mode
|
|
28
|
-
if (opts.results) {
|
|
29
|
-
const ledger = new ResultsLedger(join(sbDir(repoRoot), "autoagent", "results.tsv"));
|
|
30
|
-
console.log(ledger.format());
|
|
31
|
-
return;
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
// Validate required files
|
|
35
|
-
const autoagentDir = join(sbDir(repoRoot), "autoagent");
|
|
36
|
-
if (!existsSync(join(autoagentDir, "program.md"))) {
|
|
37
|
-
out.fail("Missing .switchboard/autoagent/program.md");
|
|
38
|
-
out.info("Write a directive describing what the autoagent should improve.");
|
|
39
|
-
process.exitCode = 1;
|
|
40
|
-
return;
|
|
41
|
-
}
|
|
42
|
-
if (!existsSync(join(autoagentDir, "score.ts"))) {
|
|
43
|
-
out.fail("Missing .switchboard/autoagent/score.ts");
|
|
44
|
-
out.info("Write a score function: export async function score(): Promise<ScoreResult>");
|
|
45
|
-
process.exitCode = 1;
|
|
46
|
-
return;
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
// Load contract for boundaries
|
|
50
|
-
const contract = loadContract(repoRoot);
|
|
51
|
-
|
|
52
|
-
// Parse config
|
|
53
|
-
const tier = opts.tier as AutoAgentTier;
|
|
54
|
-
if (!["code", "harness", "full"].includes(tier)) {
|
|
55
|
-
out.fail(`Invalid tier: ${tier}. Use: code, harness, full`);
|
|
56
|
-
process.exitCode = 1;
|
|
57
|
-
return;
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
const maxTimeMs = parseTime(opts.maxTime);
|
|
61
|
-
|
|
62
|
-
out.section("Tier", tier);
|
|
63
|
-
out.section("Max iterations", opts.maxIterations === "0" ? "unlimited" : opts.maxIterations);
|
|
64
|
-
out.section("Max time", maxTimeMs === 0 ? "unlimited" : opts.maxTime);
|
|
65
|
-
out.section("Score target", opts.scoreTarget === "0" ? "none" : opts.scoreTarget);
|
|
66
|
-
|
|
67
|
-
// Create and run the loop
|
|
68
|
-
const loop = new AutoAgentLoop(repoRoot, {
|
|
69
|
-
tier,
|
|
70
|
-
maxIterations: parseInt(opts.maxIterations, 10),
|
|
71
|
-
maxTimeMs,
|
|
72
|
-
scoreTarget: parseFloat(opts.scoreTarget),
|
|
73
|
-
}, contract);
|
|
74
|
-
|
|
75
|
-
// Handle SIGINT
|
|
76
|
-
process.on("SIGINT", () => {
|
|
77
|
-
out.warn("Stopping autoagent loop...");
|
|
78
|
-
loop.stop();
|
|
79
|
-
});
|
|
80
|
-
|
|
81
|
-
const finalState = await loop.run();
|
|
82
|
-
|
|
83
|
-
// Print summary
|
|
84
|
-
out.heading("AutoAgent Complete");
|
|
85
|
-
out.section("Iterations", `${finalState.iteration}`);
|
|
86
|
-
out.section("Best score", `${finalState.bestScore.toFixed(4)} (commit: ${finalState.bestCommit})`);
|
|
87
|
-
out.section("Final score", `${finalState.currentScore.toFixed(4)}`);
|
|
88
|
-
out.section("Status", finalState.status);
|
|
89
|
-
|
|
90
|
-
const kept = finalState.results.filter(r => r.status === "keep").length;
|
|
91
|
-
const discarded = finalState.results.filter(r => r.status === "discard").length;
|
|
92
|
-
out.section("Kept/Discarded", `${kept}/${discarded}`);
|
|
93
|
-
}));
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
function parseTime(input: string): number {
|
|
97
|
-
if (input === "0") return 0;
|
|
98
|
-
const match = input.match(/^(\d+)(h|m|s)$/);
|
|
99
|
-
if (!match) return 0;
|
|
100
|
-
const [, num, unit] = match;
|
|
101
|
-
const n = parseInt(num!, 10);
|
|
102
|
-
switch (unit) {
|
|
103
|
-
case "h": return n * 3600_000;
|
|
104
|
-
case "m": return n * 60_000;
|
|
105
|
-
case "s": return n * 1000;
|
|
106
|
-
default: return 0;
|
|
107
|
-
}
|
|
108
|
-
}
|
|
@@ -1,70 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* switchboard calibrate
|
|
3
|
-
*
|
|
4
|
-
* Runs the two-layer calibration harness against the governance pipeline.
|
|
5
|
-
* Layer A: internal pipeline with full trace access.
|
|
6
|
-
* Layer B: independent black-box scorer with zero core imports.
|
|
7
|
-
*
|
|
8
|
-
* Produces an append-only calibration ledger with seeds, traces,
|
|
9
|
-
* verdicts, diffs, diagnosis cards, and escalation queue.
|
|
10
|
-
*/
|
|
11
|
-
|
|
12
|
-
import type { Command } from "commander";
|
|
13
|
-
import { resolve } from "path";
|
|
14
|
-
import { runCalibration } from "../../calibration/skill/calibrate";
|
|
15
|
-
import * as out from "../lib/output";
|
|
16
|
-
import { withCliErrors } from "../lib/errors";
|
|
17
|
-
|
|
18
|
-
export function registerCalibrateCommand(program: Command): void {
|
|
19
|
-
program
|
|
20
|
-
.command("calibrate")
|
|
21
|
-
.description("Run two-layer calibration harness against the governance pipeline")
|
|
22
|
-
.option("--blind <count>", "Number of blind seeds (default: 5)")
|
|
23
|
-
.option("--adversarial <count>", "Number of adversarial seeds (default: 3)")
|
|
24
|
-
.option("--replay <count>", "Number of replay seeds (default: 2)")
|
|
25
|
-
.option("--adversarial-only [count]", "Run only adversarial seeds")
|
|
26
|
-
.option("--replay-only", "Run only replay seeds")
|
|
27
|
-
.option("--retest", "Re-test seeds from open diagnosis cards")
|
|
28
|
-
.option("--review", "Review pending escalations")
|
|
29
|
-
.option("--trends", "Show calibration trends over time")
|
|
30
|
-
.action(withCliErrors(async (opts) => {
|
|
31
|
-
out.heading("switchboard calibrate");
|
|
32
|
-
|
|
33
|
-
// Build args string from commander options
|
|
34
|
-
const args = buildArgsString(opts);
|
|
35
|
-
|
|
36
|
-
// Use the CLI package root as the project root for calibration
|
|
37
|
-
// (calibration ledger lives inside the package, not the user's project)
|
|
38
|
-
const calibrationRoot = resolve(__dirname, "..", "..");
|
|
39
|
-
|
|
40
|
-
const result = await runCalibration(args, calibrationRoot);
|
|
41
|
-
|
|
42
|
-
console.log(result.report);
|
|
43
|
-
|
|
44
|
-
if (result.warnings.length > 0) {
|
|
45
|
-
console.log("");
|
|
46
|
-
out.heading("Self-monitoring warnings");
|
|
47
|
-
for (const w of result.warnings) {
|
|
48
|
-
out.warn(w);
|
|
49
|
-
}
|
|
50
|
-
}
|
|
51
|
-
}));
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
function buildArgsString(opts: Record<string, unknown>): string {
|
|
55
|
-
const parts: string[] = [];
|
|
56
|
-
|
|
57
|
-
if (opts["blind"]) parts.push(`--blind ${opts["blind"]}`);
|
|
58
|
-
if (opts["adversarial"]) parts.push(`--adversarial ${opts["adversarial"]}`);
|
|
59
|
-
if (opts["replay"]) parts.push(`--replay ${opts["replay"]}`);
|
|
60
|
-
if (opts["adversarialOnly"]) {
|
|
61
|
-
const count = typeof opts["adversarialOnly"] === "string" ? opts["adversarialOnly"] : "";
|
|
62
|
-
parts.push(`--adversarial-only ${count}`.trim());
|
|
63
|
-
}
|
|
64
|
-
if (opts["replayOnly"]) parts.push("--replay-only");
|
|
65
|
-
if (opts["retest"]) parts.push("--retest");
|
|
66
|
-
if (opts["review"]) parts.push("--review");
|
|
67
|
-
if (opts["trends"]) parts.push("--trends");
|
|
68
|
-
|
|
69
|
-
return parts.join(" ");
|
|
70
|
-
}
|