switchboard-cli 0.1.0-alpha.4 → 0.1.0-alpha.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/README.md +121 -65
- package/bin/switchboard.js +75 -0
- package/dist/index.cjs +14371 -0
- package/package.json +45 -8
- package/bin/switchboard.mjs +0 -49
- package/src/autoagent/boundary-check.spec.ts +0 -77
- package/src/autoagent/boundary-check.ts +0 -102
- package/src/autoagent/loop.ts +0 -327
- package/src/autoagent/results.spec.ts +0 -73
- package/src/autoagent/results.ts +0 -68
- package/src/autoagent/runner.spec.ts +0 -20
- package/src/autoagent/runner.ts +0 -92
- package/src/autoagent/types.ts +0 -64
- package/src/commands/audit-codex.ts +0 -266
- package/src/commands/autoagent.ts +0 -108
- package/src/commands/calibrate.ts +0 -70
- package/src/commands/compile.ts +0 -117
- package/src/commands/evaluate.ts +0 -103
- package/src/commands/ingest.ts +0 -250
- package/src/commands/init.ts +0 -133
- package/src/commands/packet.ts +0 -408
- package/src/commands/receipt.ts +0 -336
- package/src/commands/run-claude.ts +0 -355
- package/src/index.ts +0 -47
- package/src/lib/draft-return.ts +0 -278
- package/src/lib/drift-guard.ts +0 -105
- package/src/lib/errors.ts +0 -61
- package/src/lib/output.ts +0 -43
- package/src/lib/paths.ts +0 -125
- package/src/lib/proof.ts +0 -262
- package/src/lib/transport.ts +0 -276
- package/src/lib/yaml-io.ts +0 -62
- package/src/store/filesystem-store.ts +0 -326
package/package.json
CHANGED
|
@@ -1,16 +1,53 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "switchboard-cli",
|
|
3
|
-
"version": "0.1.0-alpha.
|
|
4
|
-
"description": "Switchboard CLI
|
|
3
|
+
"version": "0.1.0-alpha.5",
|
|
4
|
+
"description": "Switchboard CLI \u2014 portable governance substrate for AI workflows",
|
|
5
5
|
"license": "MIT",
|
|
6
|
-
"
|
|
7
|
-
"
|
|
8
|
-
|
|
9
|
-
|
|
6
|
+
"main": "./dist/index.cjs",
|
|
7
|
+
"bin": {
|
|
8
|
+
"sb": "./bin/switchboard.js",
|
|
9
|
+
"switchboard": "./bin/switchboard.js"
|
|
10
|
+
},
|
|
11
|
+
"files": [
|
|
12
|
+
"bin/",
|
|
13
|
+
"dist/",
|
|
14
|
+
"calibration/",
|
|
15
|
+
"README.md",
|
|
16
|
+
"CHANGELOG.md"
|
|
17
|
+
],
|
|
18
|
+
"engines": {
|
|
19
|
+
"node": ">=18"
|
|
20
|
+
},
|
|
21
|
+
"publishConfig": {
|
|
22
|
+
"access": "public"
|
|
23
|
+
},
|
|
24
|
+
"keywords": [
|
|
25
|
+
"switchboard",
|
|
26
|
+
"governance",
|
|
27
|
+
"ai",
|
|
28
|
+
"cli",
|
|
29
|
+
"receipts",
|
|
30
|
+
"trust",
|
|
31
|
+
"calibration",
|
|
32
|
+
"evaluation"
|
|
33
|
+
],
|
|
34
|
+
"scripts": {
|
|
35
|
+
"build": "esbuild src/index.ts --bundle --platform=node --format=cjs --target=node18 --outfile=dist/index.cjs --external:commander --external:yaml --external:zod --log-level=warning",
|
|
36
|
+
"prepublishOnly": "pnpm run build",
|
|
37
|
+
"lint": "echo 'linted cli'",
|
|
38
|
+
"typecheck": "tsc --noEmit",
|
|
39
|
+
"test": "vitest run",
|
|
40
|
+
"dev": "tsx src/index.ts"
|
|
41
|
+
},
|
|
10
42
|
"dependencies": {
|
|
11
43
|
"commander": "^13.1.0",
|
|
12
|
-
"tsx": "^4.19.3",
|
|
13
44
|
"yaml": "^2.7.0",
|
|
14
45
|
"zod": "^3.24.2"
|
|
46
|
+
},
|
|
47
|
+
"devDependencies": {
|
|
48
|
+
"@switchboard/core": "workspace:*",
|
|
49
|
+
"@switchboard/projections": "workspace:*",
|
|
50
|
+
"esbuild": "^0.28.1",
|
|
51
|
+
"tsx": "^4.19.3"
|
|
15
52
|
}
|
|
16
|
-
}
|
|
53
|
+
}
|
package/bin/switchboard.mjs
DELETED
|
@@ -1,49 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
|
|
3
|
-
/**
|
|
4
|
-
* Switchboard CLI bin shim.
|
|
5
|
-
*
|
|
6
|
-
* Spawns tsx on the real TypeScript entrypoint so the CLI is invocable
|
|
7
|
-
* without a build step. Works from any cwd — resolves paths relative
|
|
8
|
-
* to this file, not the caller.
|
|
9
|
-
*/
|
|
10
|
-
|
|
11
|
-
import { execFileSync } from "node:child_process";
|
|
12
|
-
import { fileURLToPath } from "node:url";
|
|
13
|
-
import { dirname, resolve } from "node:path";
|
|
14
|
-
|
|
15
|
-
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
16
|
-
const entry = resolve(__dirname, "..", "src", "index.ts");
|
|
17
|
-
|
|
18
|
-
// Resolve tsx from this package's own node_modules first, then fall
|
|
19
|
-
// back to a global/hoisted location.
|
|
20
|
-
const tsxPaths = [
|
|
21
|
-
resolve(__dirname, "..", "node_modules", ".bin", "tsx"),
|
|
22
|
-
"tsx",
|
|
23
|
-
];
|
|
24
|
-
|
|
25
|
-
let tsxBin;
|
|
26
|
-
for (const candidate of tsxPaths) {
|
|
27
|
-
try {
|
|
28
|
-
execFileSync(candidate, ["--version"], { stdio: "ignore" });
|
|
29
|
-
tsxBin = candidate;
|
|
30
|
-
break;
|
|
31
|
-
} catch {
|
|
32
|
-
// try next
|
|
33
|
-
}
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
if (!tsxBin) {
|
|
37
|
-
console.error("switchboard: could not find tsx. Run `pnpm install` in the monorepo root.");
|
|
38
|
-
process.exit(1);
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
try {
|
|
42
|
-
execFileSync(tsxBin, [entry, ...process.argv.slice(2)], {
|
|
43
|
-
stdio: "inherit",
|
|
44
|
-
cwd: process.cwd(),
|
|
45
|
-
});
|
|
46
|
-
} catch (err) {
|
|
47
|
-
// execFileSync throws on non-zero exit; the child already printed output
|
|
48
|
-
process.exit(err.status ?? 1);
|
|
49
|
-
}
|
|
@@ -1,77 +0,0 @@
|
|
|
1
|
-
import { describe, it, expect } from "vitest";
|
|
2
|
-
import { buildBoundarySpec, checkBoundaryViolation } from "./boundary-check";
|
|
3
|
-
import type { AutoAgentTier } from "./types";
|
|
4
|
-
|
|
5
|
-
describe("buildBoundarySpec", () => {
|
|
6
|
-
const contract = {
|
|
7
|
-
task_clauses: {
|
|
8
|
-
scope_out: ["Live trading", "Multi-exchange"],
|
|
9
|
-
},
|
|
10
|
-
non_goals: ["Not a portfolio manager", "Not live-trading by default"],
|
|
11
|
-
};
|
|
12
|
-
|
|
13
|
-
it("code tier allows src/ only", () => {
|
|
14
|
-
const spec = buildBoundarySpec("code", contract as any);
|
|
15
|
-
expect(spec.modifiable).toContain("src/**");
|
|
16
|
-
expect(spec.frozen).toContain(".switchboard/**");
|
|
17
|
-
expect(spec.frozen).toContain("package.json");
|
|
18
|
-
});
|
|
19
|
-
|
|
20
|
-
it("harness tier allows src/ and config", () => {
|
|
21
|
-
const spec = buildBoundarySpec("harness", contract as any);
|
|
22
|
-
expect(spec.modifiable).toContain("src/**");
|
|
23
|
-
expect(spec.modifiable).toContain("*.config.*");
|
|
24
|
-
});
|
|
25
|
-
|
|
26
|
-
it("full tier allows most files", () => {
|
|
27
|
-
const spec = buildBoundarySpec("full", contract as any);
|
|
28
|
-
expect(spec.modifiable).toContain("src/**");
|
|
29
|
-
expect(spec.modifiable).toContain("package.json");
|
|
30
|
-
});
|
|
31
|
-
|
|
32
|
-
it("all tiers freeze contract and autoagent config", () => {
|
|
33
|
-
for (const tier of ["code", "harness", "full"] as AutoAgentTier[]) {
|
|
34
|
-
const spec = buildBoundarySpec(tier, contract as any);
|
|
35
|
-
expect(spec.frozen).toContain(".switchboard/project/contract.yaml");
|
|
36
|
-
expect(spec.frozen).toContain(".switchboard/autoagent/program.md");
|
|
37
|
-
expect(spec.frozen).toContain(".switchboard/autoagent/score.ts");
|
|
38
|
-
}
|
|
39
|
-
});
|
|
40
|
-
});
|
|
41
|
-
|
|
42
|
-
describe("checkBoundaryViolation", () => {
|
|
43
|
-
it("returns null when files are within boundary", () => {
|
|
44
|
-
const spec = {
|
|
45
|
-
tier: "code" as const,
|
|
46
|
-
modifiable: ["src/**"],
|
|
47
|
-
frozen: [".switchboard/**"],
|
|
48
|
-
contractNonGoals: [],
|
|
49
|
-
};
|
|
50
|
-
const result = checkBoundaryViolation(["src/index.ts", "src/lib/utils.ts"], spec);
|
|
51
|
-
expect(result).toBeNull();
|
|
52
|
-
});
|
|
53
|
-
|
|
54
|
-
it("returns violation when frozen file is touched", () => {
|
|
55
|
-
const spec = {
|
|
56
|
-
tier: "code" as const,
|
|
57
|
-
modifiable: ["src/**"],
|
|
58
|
-
frozen: [".switchboard/**", "package.json"],
|
|
59
|
-
contractNonGoals: [],
|
|
60
|
-
};
|
|
61
|
-
const result = checkBoundaryViolation(["src/index.ts", "package.json"], spec);
|
|
62
|
-
expect(result).not.toBeNull();
|
|
63
|
-
expect(result).toContain("package.json");
|
|
64
|
-
});
|
|
65
|
-
|
|
66
|
-
it("returns violation when file is outside modifiable scope", () => {
|
|
67
|
-
const spec = {
|
|
68
|
-
tier: "code" as const,
|
|
69
|
-
modifiable: ["src/**"],
|
|
70
|
-
frozen: [".switchboard/**"],
|
|
71
|
-
contractNonGoals: [],
|
|
72
|
-
};
|
|
73
|
-
const result = checkBoundaryViolation(["scripts/deploy.sh"], spec);
|
|
74
|
-
expect(result).not.toBeNull();
|
|
75
|
-
expect(result).toContain("scripts/deploy.sh");
|
|
76
|
-
});
|
|
77
|
-
});
|
|
@@ -1,102 +0,0 @@
|
|
|
1
|
-
import type { AutoAgentTier, BoundarySpec } from "./types";
|
|
2
|
-
|
|
3
|
-
// ---------------------------------------------------------------------------
|
|
4
|
-
// Build boundary spec from tier + contract
|
|
5
|
-
// ---------------------------------------------------------------------------
|
|
6
|
-
|
|
7
|
-
export function buildBoundarySpec(
|
|
8
|
-
tier: AutoAgentTier,
|
|
9
|
-
contract: { task_clauses?: { scope_out?: string[] }; non_goals?: string[] },
|
|
10
|
-
): BoundarySpec {
|
|
11
|
-
// Files that are ALWAYS frozen regardless of tier
|
|
12
|
-
const alwaysFrozen = [
|
|
13
|
-
".switchboard/project/contract.yaml",
|
|
14
|
-
".switchboard/autoagent/program.md",
|
|
15
|
-
".switchboard/autoagent/score.ts",
|
|
16
|
-
".switchboard/autoagent/results.tsv",
|
|
17
|
-
"**/*.spec.ts",
|
|
18
|
-
"**/*.test.ts",
|
|
19
|
-
];
|
|
20
|
-
|
|
21
|
-
let modifiable: string[];
|
|
22
|
-
let frozen: string[];
|
|
23
|
-
|
|
24
|
-
switch (tier) {
|
|
25
|
-
case "code":
|
|
26
|
-
modifiable = ["src/**"];
|
|
27
|
-
frozen = [...alwaysFrozen, ".switchboard/**", "package.json", "*.config.*", "tsconfig.json"];
|
|
28
|
-
break;
|
|
29
|
-
|
|
30
|
-
case "harness":
|
|
31
|
-
modifiable = ["src/**", "*.config.*", "tsconfig.json"];
|
|
32
|
-
frozen = [...alwaysFrozen, ".switchboard/**", "package.json"];
|
|
33
|
-
break;
|
|
34
|
-
|
|
35
|
-
case "full":
|
|
36
|
-
modifiable = ["src/**", "*.config.*", "tsconfig.json", "package.json"];
|
|
37
|
-
frozen = [...alwaysFrozen, ".switchboard/project/**", ".switchboard/autoagent/**"];
|
|
38
|
-
break;
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
const contractNonGoals = contract.non_goals ?? [];
|
|
42
|
-
|
|
43
|
-
return { tier, modifiable, frozen, contractNonGoals };
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
// ---------------------------------------------------------------------------
|
|
47
|
-
// Check if changed files violate boundary
|
|
48
|
-
// ---------------------------------------------------------------------------
|
|
49
|
-
|
|
50
|
-
export function checkBoundaryViolation(
|
|
51
|
-
changedFiles: string[],
|
|
52
|
-
spec: BoundarySpec,
|
|
53
|
-
): string | null {
|
|
54
|
-
const violations: string[] = [];
|
|
55
|
-
|
|
56
|
-
for (const file of changedFiles) {
|
|
57
|
-
// Check frozen files
|
|
58
|
-
if (matchesAny(file, spec.frozen)) {
|
|
59
|
-
violations.push(file);
|
|
60
|
-
continue;
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
// Check if file is within modifiable scope
|
|
64
|
-
if (!matchesAny(file, spec.modifiable)) {
|
|
65
|
-
violations.push(file);
|
|
66
|
-
}
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
if (violations.length === 0) return null;
|
|
70
|
-
return `Boundary violation: ${violations.join(", ")} are outside the ${spec.tier} tier scope`;
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
// ---------------------------------------------------------------------------
|
|
74
|
-
// Simple glob matching (** = any path, * = any segment)
|
|
75
|
-
// ---------------------------------------------------------------------------
|
|
76
|
-
|
|
77
|
-
function matchesAny(file: string, patterns: string[]): boolean {
|
|
78
|
-
return patterns.some(pattern => matchGlob(file, pattern));
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
function matchGlob(file: string, pattern: string): boolean {
|
|
82
|
-
// Exact match
|
|
83
|
-
if (file === pattern) return true;
|
|
84
|
-
|
|
85
|
-
// ** matches any path depth
|
|
86
|
-
if (pattern.endsWith("/**")) {
|
|
87
|
-
const prefix = pattern.slice(0, -3);
|
|
88
|
-
return file.startsWith(prefix + "/") || file === prefix;
|
|
89
|
-
}
|
|
90
|
-
if (pattern.startsWith("**/")) {
|
|
91
|
-
const suffix = pattern.slice(3);
|
|
92
|
-
return file.endsWith(suffix) || file.includes("/" + suffix);
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
// * matches within a single segment
|
|
96
|
-
if (pattern.startsWith("*.")) {
|
|
97
|
-
const ext = pattern.slice(1);
|
|
98
|
-
return file.endsWith(ext) || file.includes(ext + ".");
|
|
99
|
-
}
|
|
100
|
-
|
|
101
|
-
return false;
|
|
102
|
-
}
|
package/src/autoagent/loop.ts
DELETED
|
@@ -1,327 +0,0 @@
|
|
|
1
|
-
import { execSync } from "child_process";
|
|
2
|
-
import { readFileSync, existsSync, writeFileSync, mkdirSync } from "fs";
|
|
3
|
-
import { join } from "path";
|
|
4
|
-
import type { AutoAgentConfig, AutoAgentState, IterationResult, BoundarySpec, ScoreResult } from "./types";
|
|
5
|
-
import { DEFAULT_CONFIG } from "./types";
|
|
6
|
-
import { buildBoundarySpec, checkBoundaryViolation } from "./boundary-check";
|
|
7
|
-
import { ResultsLedger } from "./results";
|
|
8
|
-
import { runTests, runScore, getChangedFiles, getShortCommit, gitRevert, gitCommit } from "./runner";
|
|
9
|
-
import * as out from "../lib/output";
|
|
10
|
-
|
|
11
|
-
// ---------------------------------------------------------------------------
|
|
12
|
-
// AutoAgent Loop — the core iteration engine
|
|
13
|
-
// ---------------------------------------------------------------------------
|
|
14
|
-
|
|
15
|
-
export class AutoAgentLoop {
|
|
16
|
-
private config: AutoAgentConfig;
|
|
17
|
-
private repoRoot: string;
|
|
18
|
-
private state: AutoAgentState;
|
|
19
|
-
private ledger: ResultsLedger;
|
|
20
|
-
private boundary: BoundarySpec;
|
|
21
|
-
private programMd: string;
|
|
22
|
-
private stopped = false;
|
|
23
|
-
|
|
24
|
-
constructor(repoRoot: string, config: Partial<AutoAgentConfig>, contract: any) {
|
|
25
|
-
this.repoRoot = repoRoot;
|
|
26
|
-
this.config = { ...DEFAULT_CONFIG, ...config };
|
|
27
|
-
this.ledger = new ResultsLedger(join(repoRoot, ".switchboard", "autoagent", "results.tsv"));
|
|
28
|
-
this.boundary = buildBoundarySpec(this.config.tier, contract);
|
|
29
|
-
|
|
30
|
-
const programPath = join(repoRoot, ".switchboard", "autoagent", "program.md");
|
|
31
|
-
if (!existsSync(programPath)) {
|
|
32
|
-
throw new Error("Missing .switchboard/autoagent/program.md — write the directive first.");
|
|
33
|
-
}
|
|
34
|
-
this.programMd = readFileSync(programPath, "utf-8");
|
|
35
|
-
|
|
36
|
-
// Write boundary spec for subagent reference
|
|
37
|
-
const boundaryPath = join(repoRoot, ".switchboard", "autoagent", "boundaries.md");
|
|
38
|
-
mkdirSync(join(repoRoot, ".switchboard", "autoagent"), { recursive: true });
|
|
39
|
-
writeFileSync(boundaryPath, formatBoundaryDoc(this.boundary), "utf-8");
|
|
40
|
-
|
|
41
|
-
this.state = {
|
|
42
|
-
startedAt: Date.now(),
|
|
43
|
-
iteration: 0,
|
|
44
|
-
bestScore: 0,
|
|
45
|
-
bestCommit: "",
|
|
46
|
-
currentScore: 0,
|
|
47
|
-
results: [],
|
|
48
|
-
status: "running",
|
|
49
|
-
};
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
stop(): void {
|
|
53
|
-
this.stopped = true;
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
getState(): AutoAgentState {
|
|
57
|
-
return { ...this.state };
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
async run(): Promise<AutoAgentState> {
|
|
61
|
-
// Run initial score to establish baseline
|
|
62
|
-
out.info("Establishing baseline score...");
|
|
63
|
-
const baselineScore = await runScore(this.config.scoreCommand, this.repoRoot);
|
|
64
|
-
this.state.bestScore = baselineScore.score;
|
|
65
|
-
this.state.currentScore = baselineScore.score;
|
|
66
|
-
this.state.bestCommit = await getShortCommit(this.repoRoot);
|
|
67
|
-
out.success(`Baseline score: ${baselineScore.score.toFixed(4)} — ${baselineScore.summary}`);
|
|
68
|
-
|
|
69
|
-
while (!this.shouldStop()) {
|
|
70
|
-
this.state.iteration++;
|
|
71
|
-
out.heading(`Iteration ${this.state.iteration}`);
|
|
72
|
-
|
|
73
|
-
const iterStart = Date.now();
|
|
74
|
-
|
|
75
|
-
try {
|
|
76
|
-
const result = await this.runIteration();
|
|
77
|
-
result.durationMs = Date.now() - iterStart;
|
|
78
|
-
this.state.results.push(result);
|
|
79
|
-
this.ledger.append(result);
|
|
80
|
-
|
|
81
|
-
if (result.status === "keep") {
|
|
82
|
-
out.success(`KEEP: ${result.description} (${result.prevScore.toFixed(4)} → ${result.score.toFixed(4)})`);
|
|
83
|
-
this.state.currentScore = result.score;
|
|
84
|
-
if (result.score > this.state.bestScore) {
|
|
85
|
-
this.state.bestScore = result.score;
|
|
86
|
-
this.state.bestCommit = result.commit;
|
|
87
|
-
}
|
|
88
|
-
} else {
|
|
89
|
-
out.warn(`DISCARD: ${result.description} (${result.status})`);
|
|
90
|
-
}
|
|
91
|
-
} catch (err: any) {
|
|
92
|
-
out.fail(`Iteration error: ${err.message}`);
|
|
93
|
-
this.ledger.append({
|
|
94
|
-
iteration: this.state.iteration,
|
|
95
|
-
commit: await getShortCommit(this.repoRoot),
|
|
96
|
-
score: this.state.currentScore,
|
|
97
|
-
prevScore: this.state.currentScore,
|
|
98
|
-
testsPass: false,
|
|
99
|
-
testSummary: "error",
|
|
100
|
-
status: "crash",
|
|
101
|
-
description: err.message.slice(0, 100),
|
|
102
|
-
filesChanged: [],
|
|
103
|
-
timestamp: Date.now(),
|
|
104
|
-
durationMs: Date.now() - iterStart,
|
|
105
|
-
});
|
|
106
|
-
}
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
return this.state;
|
|
110
|
-
}
|
|
111
|
-
|
|
112
|
-
private async runIteration(): Promise<IterationResult> {
|
|
113
|
-
const prevScore = this.state.currentScore;
|
|
114
|
-
|
|
115
|
-
// 1. DIAGNOSE — read previous results, find what to improve
|
|
116
|
-
const diagnosis = this.buildDiagnosis();
|
|
117
|
-
|
|
118
|
-
// 2. MODIFY — dispatch subagent to make one change
|
|
119
|
-
out.info("Dispatching modification subagent...");
|
|
120
|
-
const modPrompt = this.buildModifyPrompt(diagnosis);
|
|
121
|
-
|
|
122
|
-
// Execute modification via a shell command that invokes Claude
|
|
123
|
-
// For now, write the prompt to a file and let the caller handle dispatch
|
|
124
|
-
const promptPath = join(this.repoRoot, ".switchboard", "autoagent", "current-prompt.md");
|
|
125
|
-
writeFileSync(promptPath, modPrompt, "utf-8");
|
|
126
|
-
|
|
127
|
-
// The actual modification happens via the subagent dispatched by the CLI command
|
|
128
|
-
// For the loop module, we simulate by checking if files changed
|
|
129
|
-
out.info("Subagent should modify files based on the prompt at:");
|
|
130
|
-
out.info(promptPath);
|
|
131
|
-
out.info("Waiting for changes...");
|
|
132
|
-
|
|
133
|
-
// In practice, the CLI command's action handler dispatches the Agent tool
|
|
134
|
-
// and this loop continues after the agent returns. For now, return a
|
|
135
|
-
// placeholder that the CLI command fills in.
|
|
136
|
-
return {
|
|
137
|
-
iteration: this.state.iteration,
|
|
138
|
-
commit: "",
|
|
139
|
-
score: 0,
|
|
140
|
-
prevScore,
|
|
141
|
-
testsPass: false,
|
|
142
|
-
testSummary: "",
|
|
143
|
-
status: "discard",
|
|
144
|
-
description: "placeholder — filled by CLI orchestrator",
|
|
145
|
-
filesChanged: [],
|
|
146
|
-
timestamp: Date.now(),
|
|
147
|
-
durationMs: 0,
|
|
148
|
-
};
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
// Called by the CLI command after the subagent completes
|
|
152
|
-
async evaluateChanges(description: string): Promise<IterationResult> {
|
|
153
|
-
const prevScore = this.state.currentScore;
|
|
154
|
-
const iterStart = Date.now();
|
|
155
|
-
|
|
156
|
-
// 3. BOUNDARY CHECK
|
|
157
|
-
const changedFiles = await getChangedFiles(this.repoRoot);
|
|
158
|
-
const violation = checkBoundaryViolation(changedFiles, this.boundary);
|
|
159
|
-
if (violation) {
|
|
160
|
-
out.fail(`Boundary violation: ${violation}`);
|
|
161
|
-
await gitRevert(this.repoRoot);
|
|
162
|
-
return {
|
|
163
|
-
iteration: this.state.iteration,
|
|
164
|
-
commit: await getShortCommit(this.repoRoot),
|
|
165
|
-
score: prevScore,
|
|
166
|
-
prevScore,
|
|
167
|
-
testsPass: false,
|
|
168
|
-
testSummary: "boundary-violation",
|
|
169
|
-
status: "boundary-violation",
|
|
170
|
-
description: violation.slice(0, 100),
|
|
171
|
-
filesChanged: changedFiles,
|
|
172
|
-
timestamp: Date.now(),
|
|
173
|
-
durationMs: Date.now() - iterStart,
|
|
174
|
-
};
|
|
175
|
-
}
|
|
176
|
-
|
|
177
|
-
// 4. TEST — hard gate
|
|
178
|
-
out.info("Running tests...");
|
|
179
|
-
const testResult = await runTests(this.config.testCommand, this.repoRoot);
|
|
180
|
-
if (!testResult.passed) {
|
|
181
|
-
out.fail("Tests failed — discarding");
|
|
182
|
-
await gitRevert(this.repoRoot);
|
|
183
|
-
return {
|
|
184
|
-
iteration: this.state.iteration,
|
|
185
|
-
commit: await getShortCommit(this.repoRoot),
|
|
186
|
-
score: prevScore,
|
|
187
|
-
prevScore,
|
|
188
|
-
testsPass: false,
|
|
189
|
-
testSummary: "FAIL",
|
|
190
|
-
status: "discard",
|
|
191
|
-
description: `tests failed: ${description}`,
|
|
192
|
-
filesChanged: changedFiles,
|
|
193
|
-
timestamp: Date.now(),
|
|
194
|
-
durationMs: Date.now() - iterStart,
|
|
195
|
-
};
|
|
196
|
-
}
|
|
197
|
-
|
|
198
|
-
// 5. SCORE
|
|
199
|
-
out.info("Running score function...");
|
|
200
|
-
const scoreResult = await runScore(this.config.scoreCommand, this.repoRoot);
|
|
201
|
-
|
|
202
|
-
// 6. DECIDE — keep or discard
|
|
203
|
-
const improved = scoreResult.score > prevScore;
|
|
204
|
-
const equal = Math.abs(scoreResult.score - prevScore) < 0.0001;
|
|
205
|
-
const simpler = changedFiles.length === 0; // equal score + no extra code = simpler
|
|
206
|
-
const keep = improved || (equal && simpler);
|
|
207
|
-
|
|
208
|
-
const commit = await gitCommit(
|
|
209
|
-
this.repoRoot,
|
|
210
|
-
`autoagent: iteration ${this.state.iteration} — ${keep ? "keep" : "discard"}: ${description}`,
|
|
211
|
-
);
|
|
212
|
-
|
|
213
|
-
if (!keep) {
|
|
214
|
-
await gitRevert(this.repoRoot);
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
return {
|
|
218
|
-
iteration: this.state.iteration,
|
|
219
|
-
commit,
|
|
220
|
-
score: scoreResult.score,
|
|
221
|
-
prevScore,
|
|
222
|
-
testsPass: true,
|
|
223
|
-
testSummary: "PASS",
|
|
224
|
-
status: keep ? "keep" : "discard",
|
|
225
|
-
description,
|
|
226
|
-
filesChanged: changedFiles,
|
|
227
|
-
timestamp: Date.now(),
|
|
228
|
-
durationMs: Date.now() - iterStart,
|
|
229
|
-
};
|
|
230
|
-
}
|
|
231
|
-
|
|
232
|
-
private buildDiagnosis(): string {
|
|
233
|
-
const results = this.ledger.readAll();
|
|
234
|
-
if (results.length === 0) {
|
|
235
|
-
return "This is the first iteration. No previous results to diagnose.";
|
|
236
|
-
}
|
|
237
|
-
|
|
238
|
-
const kept = results.filter(r => r.status === "keep");
|
|
239
|
-
const discarded = results.filter(r => r.status === "discard");
|
|
240
|
-
const lines: string[] = [];
|
|
241
|
-
|
|
242
|
-
lines.push(`Previous iterations: ${results.length} (${kept.length} kept, ${discarded.length} discarded)`);
|
|
243
|
-
lines.push(`Current score: ${this.state.currentScore.toFixed(4)}`);
|
|
244
|
-
lines.push(`Best score: ${this.state.bestScore.toFixed(4)}`);
|
|
245
|
-
|
|
246
|
-
if (discarded.length > 0) {
|
|
247
|
-
lines.push("\nRecent discarded (avoid repeating):");
|
|
248
|
-
for (const r of discarded.slice(-3)) {
|
|
249
|
-
lines.push(` - ${r.description} (score: ${r.score.toFixed(4)})`);
|
|
250
|
-
}
|
|
251
|
-
}
|
|
252
|
-
|
|
253
|
-
if (kept.length > 0) {
|
|
254
|
-
lines.push("\nRecent kept (build on these):");
|
|
255
|
-
for (const r of kept.slice(-3)) {
|
|
256
|
-
lines.push(` - ${r.description} (score: ${r.score.toFixed(4)})`);
|
|
257
|
-
}
|
|
258
|
-
}
|
|
259
|
-
|
|
260
|
-
return lines.join("\n");
|
|
261
|
-
}
|
|
262
|
-
|
|
263
|
-
private buildModifyPrompt(diagnosis: string): string {
|
|
264
|
-
return `# AutoAgent Modification Request
|
|
265
|
-
|
|
266
|
-
## Directive
|
|
267
|
-
${this.programMd}
|
|
268
|
-
|
|
269
|
-
## Current State
|
|
270
|
-
${diagnosis}
|
|
271
|
-
|
|
272
|
-
## Boundary Rules (TIER: ${this.config.tier})
|
|
273
|
-
You may ONLY modify files matching these patterns:
|
|
274
|
-
${this.boundary.modifiable.map(p => ` - ${p}`).join("\n")}
|
|
275
|
-
|
|
276
|
-
You must NOT modify:
|
|
277
|
-
${this.boundary.frozen.map(p => ` - ${p}`).join("\n")}
|
|
278
|
-
|
|
279
|
-
## Contract Non-Goals (do NOT introduce these)
|
|
280
|
-
${this.boundary.contractNonGoals.map(g => ` - ${g}`).join("\n")}
|
|
281
|
-
|
|
282
|
-
## Instructions
|
|
283
|
-
1. Make ONE targeted change that you believe will improve the score.
|
|
284
|
-
2. Do not make broad refactors — one hypothesis, one modification.
|
|
285
|
-
3. Commit your change with a descriptive message.
|
|
286
|
-
4. Describe what you changed and why in 1-2 sentences.
|
|
287
|
-
`;
|
|
288
|
-
}
|
|
289
|
-
|
|
290
|
-
private shouldStop(): boolean {
|
|
291
|
-
if (this.stopped) {
|
|
292
|
-
this.state.status = "stopped";
|
|
293
|
-
return true;
|
|
294
|
-
}
|
|
295
|
-
if (this.config.maxIterations > 0 && this.state.iteration >= this.config.maxIterations) {
|
|
296
|
-
this.state.status = "max-iterations";
|
|
297
|
-
return true;
|
|
298
|
-
}
|
|
299
|
-
if (this.config.maxTimeMs > 0 && (Date.now() - this.state.startedAt) >= this.config.maxTimeMs) {
|
|
300
|
-
this.state.status = "max-time";
|
|
301
|
-
return true;
|
|
302
|
-
}
|
|
303
|
-
if (this.config.scoreTarget > 0 && this.state.currentScore >= this.config.scoreTarget) {
|
|
304
|
-
this.state.status = "target-reached";
|
|
305
|
-
return true;
|
|
306
|
-
}
|
|
307
|
-
return false;
|
|
308
|
-
}
|
|
309
|
-
}
|
|
310
|
-
|
|
311
|
-
// ---------------------------------------------------------------------------
|
|
312
|
-
// Format boundary doc for subagent
|
|
313
|
-
// ---------------------------------------------------------------------------
|
|
314
|
-
|
|
315
|
-
function formatBoundaryDoc(spec: BoundarySpec): string {
|
|
316
|
-
return `# AutoAgent Boundary Spec (Tier: ${spec.tier})
|
|
317
|
-
|
|
318
|
-
## Modifiable Files
|
|
319
|
-
${spec.modifiable.map(p => `- ${p}`).join("\n")}
|
|
320
|
-
|
|
321
|
-
## Frozen Files (NEVER modify)
|
|
322
|
-
${spec.frozen.map(p => `- ${p}`).join("\n")}
|
|
323
|
-
|
|
324
|
-
## Contract Non-Goals
|
|
325
|
-
${spec.contractNonGoals.map(g => `- ${g}`).join("\n")}
|
|
326
|
-
`;
|
|
327
|
-
}
|
|
@@ -1,73 +0,0 @@
|
|
|
1
|
-
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
|
2
|
-
import { ResultsLedger } from "./results";
|
|
3
|
-
import { existsSync, unlinkSync, readFileSync } from "fs";
|
|
4
|
-
import type { IterationResult } from "./types";
|
|
5
|
-
|
|
6
|
-
const TEST_PATH = "/tmp/autoagent-test-results.tsv";
|
|
7
|
-
|
|
8
|
-
describe("ResultsLedger", () => {
|
|
9
|
-
let ledger: ResultsLedger;
|
|
10
|
-
|
|
11
|
-
beforeEach(() => {
|
|
12
|
-
if (existsSync(TEST_PATH)) unlinkSync(TEST_PATH);
|
|
13
|
-
ledger = new ResultsLedger(TEST_PATH);
|
|
14
|
-
});
|
|
15
|
-
|
|
16
|
-
afterEach(() => {
|
|
17
|
-
if (existsSync(TEST_PATH)) unlinkSync(TEST_PATH);
|
|
18
|
-
});
|
|
19
|
-
|
|
20
|
-
it("creates file with header on first append", () => {
|
|
21
|
-
const result: IterationResult = {
|
|
22
|
-
iteration: 1,
|
|
23
|
-
commit: "a1b2c3d",
|
|
24
|
-
score: 0.72,
|
|
25
|
-
prevScore: 0.65,
|
|
26
|
-
testsPass: true,
|
|
27
|
-
testSummary: "26/26",
|
|
28
|
-
status: "keep",
|
|
29
|
-
description: "raised min APR threshold",
|
|
30
|
-
filesChanged: ["src/config/strategy.ts"],
|
|
31
|
-
timestamp: Date.now(),
|
|
32
|
-
durationMs: 45000,
|
|
33
|
-
};
|
|
34
|
-
|
|
35
|
-
ledger.append(result);
|
|
36
|
-
|
|
37
|
-
const content = readFileSync(TEST_PATH, "utf-8");
|
|
38
|
-
const lines = content.trim().split("\n");
|
|
39
|
-
|
|
40
|
-
expect(lines[0]).toContain("iteration");
|
|
41
|
-
expect(lines[1]).toContain("a1b2c3d");
|
|
42
|
-
expect(lines[1]).toContain("0.72");
|
|
43
|
-
expect(lines[1]).toContain("keep");
|
|
44
|
-
});
|
|
45
|
-
|
|
46
|
-
it("reads all results back", () => {
|
|
47
|
-
ledger.append({
|
|
48
|
-
iteration: 1, commit: "aaa", score: 0.5, prevScore: 0, testsPass: true,
|
|
49
|
-
testSummary: "ok", status: "keep", description: "first",
|
|
50
|
-
filesChanged: [], timestamp: Date.now(), durationMs: 1000,
|
|
51
|
-
});
|
|
52
|
-
ledger.append({
|
|
53
|
-
iteration: 2, commit: "bbb", score: 0.6, prevScore: 0.5, testsPass: true,
|
|
54
|
-
testSummary: "ok", status: "keep", description: "second",
|
|
55
|
-
filesChanged: [], timestamp: Date.now(), durationMs: 2000,
|
|
56
|
-
});
|
|
57
|
-
|
|
58
|
-
const results = ledger.readAll();
|
|
59
|
-
expect(results).toHaveLength(2);
|
|
60
|
-
expect(results[0]!.commit).toBe("aaa");
|
|
61
|
-
expect(results[1]!.score).toBe(0.6);
|
|
62
|
-
});
|
|
63
|
-
|
|
64
|
-
it("getBest returns highest scoring keep result", () => {
|
|
65
|
-
ledger.append({ iteration: 1, commit: "aaa", score: 0.5, prevScore: 0, testsPass: true, testSummary: "ok", status: "keep", description: "first", filesChanged: [], timestamp: Date.now(), durationMs: 1000 });
|
|
66
|
-
ledger.append({ iteration: 2, commit: "bbb", score: 0.8, prevScore: 0.5, testsPass: true, testSummary: "ok", status: "keep", description: "best", filesChanged: [], timestamp: Date.now(), durationMs: 1000 });
|
|
67
|
-
ledger.append({ iteration: 3, commit: "ccc", score: 0.6, prevScore: 0.8, testsPass: true, testSummary: "ok", status: "discard", description: "worse", filesChanged: [], timestamp: Date.now(), durationMs: 1000 });
|
|
68
|
-
|
|
69
|
-
const best = ledger.getBest();
|
|
70
|
-
expect(best?.commit).toBe("bbb");
|
|
71
|
-
expect(best?.score).toBe(0.8);
|
|
72
|
-
});
|
|
73
|
-
});
|