vally-claudecodecli 0.0.0-stage → 0.1.0

package/dist/executor.js

@@ -0,0 +1,397 @@
+import { mergeEnv, redactSecrets, stimulusPrompts } from "@microsoft/vally";
+import { spawn as nodeSpawn } from "node:child_process";
+import { createWriteStream } from "node:fs";
+import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { attachmentBlocks, buildClaudeArgs, userMessage } from "./args.js";
+import { parseConfig, providerEnv } from "./config.js";
+import { ClaudeProcess } from "./process.js";
+import { simulationSettings } from "./simulation.js";
+import { buildTrajectory } from "./trajectory.js";
+export const EXECUTOR_NAME = "claude-code-cli";
+/** Bound on each teardown step (interrupt settle, exit after closing stdin, SIGTERM grace). */
+const CLEANUP_TIMEOUT_MS = 5_000;
+const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+/** Result subtypes where Claude Code stopped on a limit it was given; the run is still gradable. */
+const LIMIT_RESULT_SUBTYPES = new Set(["error_max_turns", "error_max_budget_usd"]);
+class ExecutorDeadlineExceededError extends Error {
+    retryable = false;
+}
+/** Runs prompts through the Claude Code CLI (`claude -p`) and returns Vally trajectories. */
+export class ClaudeCodeExecutor {
+    name = EXECUTOR_NAME;
+    supportsPreparedWorkspace = true;
+    supportsMultiTurn = true;
+    supportsTurnCompletion = true;
+    supportsSimulation = true;
+    supportsAttachments = true;
+    supportsEnvVars = true;
+    active = new Set();
+    spawn;
+    hookScript;
+    cleanupTimeoutMs;
+    constructor(options = {}) {
+        this.spawn = options.spawn ?? nodeSpawn;
+        this.hookScript = options.hookScript ?? fileURLToPath(new URL("./hook.js", import.meta.url));
+        this.cleanupTimeoutMs = options.cleanupTimeoutMs ?? CLEANUP_TIMEOUT_MS;
+    }
+    validateConfig(config) {
+        parseConfig(config);
+    }
+    async shutdown() {
+        await Promise.all([...this.active].map((proc) => proc.kill(this.cleanupTimeoutMs)));
+    }
+    async execute(stimulus, options) {
+        const config = parseConfig(options.executorConfig);
+        let reasoningEffort;
+        if (options.reasoningEffort !== undefined) {
+            reasoningEffort = options.reasoningEffort.trim();
+            if (!reasoningEffort)
+                throw new Error("Reasoning effort must be a non-empty string");
+        }
+        const command = config.claudePath ?? "claude";
+        const prompts = stimulusPrompts(stimulus);
+        const startedAt = new Date();
+        const provider = config.provider ? providerEnv(config.provider) : undefined;
+        const secrets = [...(provider?.secrets ?? []), ...Object.values(options.env ?? {})];
+        const runDir = await mkdtemp(path.join(tmpdir(), "vally-claude-code-"));
+        const artifacts = await openArtifacts(options.sessionLog?.executorArtifactsDir);
+        let proc;
+        try {
+            const skillsDir = await linkSkills(runDir, options.skills);
+            let settings;
+            const overrides = stimulus.simulation?.tool_overrides;
+            if (overrides && Object.keys(overrides).length > 0) {
+                const overridesFile = path.join(runDir, "simulation.json");
+                await writeFile(overridesFile, JSON.stringify(overrides), "utf8");
+                settings = simulationSettings(this.hookScript, overridesFile);
+            }
+            const attachments = stimulus.attachments?.length ? await attachmentBlocks(stimulus.attachments) : [];
+            const sessionLogId = options.sessionLog?.sessionID;
+            const args = buildClaudeArgs({
+                config,
+                model: options.model,
+                reasoningEffort,
+                mcpServers: options.mcpServers,
+                resumeSessionId: options.sessionID,
+                sessionId: sessionLogId && UUID.test(sessionLogId) ? sessionLogId : undefined,
+                skillsDir,
+                settings,
+            });
+            proc = new ClaudeProcess(this.spawn(command, args, { cwd: options.workDir, env: buildEnv(options, provider), stdio: "pipe" }), (line, raw) => {
+                artifacts.stream?.write(`${raw}\n`);
+                options.onRawEvent?.(line);
+            }, (chunk) => artifacts.stderr?.write(chunk));
+            this.active.add(proc);
+            const run = await new TurnRunner(proc, stimulus, options, prompts, attachments, this.cleanupTimeoutMs).run();
+            const completedAt = new Date();
+            const { trajectory, atif } = await buildTrajectory({
+                lines: proc.lines,
+                stimulus,
+                prompts,
+                workDir: options.workDir,
+                executor: this.name,
+                requestedModel: options.model,
+                skills: options.skills,
+                startedAt,
+                completedAt,
+                endReason: run.endReason,
+                softErrors: run.softErrors,
+            });
+            if (artifacts.dir) {
+                await writeFile(path.join(artifacts.dir, "trajectory.atif.json"), `${JSON.stringify(atif, null, 2)}\n`, "utf8");
+            }
+            return trajectory;
+        }
+        catch (error) {
+            if (error instanceof Error) {
+                const errno = error;
+                if (errno.code === "ENOENT" && errno.syscall?.startsWith("spawn")) {
+                    error.message =
+                        `Claude Code executable '${command}' was not found. ` +
+                            `Install Claude Code or set defaults.executor.config.claudePath.`;
+                }
+                error.message = redactSecrets(error.message, secrets);
+            }
+            throw error;
+        }
+        finally {
+            if (proc) {
+                if (!proc.hasExited)
+                    await proc.kill(this.cleanupTimeoutMs);
+                this.active.delete(proc);
+            }
+            await artifacts.close();
+            await rm(runDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 }).catch(() => { });
+        }
+    }
+}
+/** Drives one Claude Code session through the configured turns. */
+class TurnRunner {
+    proc;
+    stimulus;
+    options;
+    prompts;
+    attachments;
+    cleanupTimeoutMs;
+    softErrors = [];
+    notified = new Set();
+    expiresAt;
+    stopReason;
+    stopped;
+    signalStop;
+    budgetRemaining;
+    budgetTimer;
+    budgetStartedAt;
+    constructor(proc, stimulus, options, prompts, attachments, cleanupTimeoutMs) {
+        this.proc = proc;
+        this.stimulus = stimulus;
+        this.options = options;
+        this.prompts = prompts;
+        this.attachments = attachments;
+        this.cleanupTimeoutMs = cleanupTimeoutMs;
+        this.expiresAt = { value: performance.now() + options.timeout };
+        this.stopped = new Promise((resolve) => {
+            this.signalStop = () => resolve({ kind: "stop" });
+        });
+        const budget = options.maxAgentDurationMs;
+        if (budget !== undefined && budget > 0 && Number.isFinite(budget))
+            this.budgetRemaining = budget;
+        const maxIterations = stimulus.simulation?.max_iterations;
+        if (maxIterations !== undefined) {
+            // Each root model response counts as one iteration.
+            const responses = new Set();
+            proc.onLine((line) => {
+                if (line.type !== "assistant" || line.parent_tool_use_id || !line.message?.id)
+                    return;
+                responses.add(line.message.id);
+                if (responses.size > maxIterations)
+                    this.requestStop("simulation_cap");
+            });
+        }
+    }
+    async run() {
+        const { proc, prompts } = this;
+        let activeTurn = 0;
+        let pendingResult;
+        this.resumeBudget();
+        try {
+            for (let turn = 0; turn < prompts.length && !this.stopReason; turn++) {
+                const stage = `turn ${turn + 1} of ${prompts.length}`;
+                activeTurn = turn;
+                const content = [...(turn === 0 ? this.attachments : []), { type: "text", text: prompts[turn] }];
+                proc.send(userMessage(content));
+                pendingResult = proc.nextResult();
+                const outcome = await this.waitFor(pendingResult, stage);
+                if (outcome.kind === "stop")
+                    break;
+                if (outcome.kind === "deadline")
+                    throw this.timeoutError(stage);
+                if (outcome.kind === "exit")
+                    throw this.exitError(outcome.exit, stage);
+                pendingResult = undefined;
+                if (outcome.line.is_error) {
+                    if (this.stopReason)
+                        break;
+                    const message = resultError(outcome.line);
+                    if (!LIMIT_RESULT_SUBTYPES.has(outcome.line.subtype ?? "")) {
+                        throw Object.assign(new Error(`Claude Code failed during ${stage} of stimulus '${this.stimulus.name}': ${message}`), { retryable: isRetryableStatus(outcome.line.api_error_status) });
+                    }
+                    this.softErrors.push(message);
+                    break;
+                }
+                if (turn < prompts.length - 1)
+                    await this.notify(turn, "completed");
+            }
+            if (this.stopReason) {
+                // Interrupt the active turn, give it a moment to settle, then end the session.
+                let settled = pendingResult === undefined;
+                if (!settled && !proc.hasExited) {
+                    proc.interrupt();
+                    settled = await settleWithin(pendingResult, this.cleanupTimeoutMs);
+                }
+                const closed = await proc.close(this.cleanupTimeoutMs);
+                await this.notify(activeTurn, settled && closed ? "interrupted" : "unavailable", true);
+                return { endReason: this.stopReason, softErrors: this.softErrors };
+            }
+            // Claude Code can keep working after its last result (e.g. reacting to finished
+            // background tasks), so the final turn completes only when the process exits.
+            proc.endInput();
+            const stage = `turn ${prompts.length} of ${prompts.length}`;
+            const ended = await this.waitFor(undefined, stage);
+            if (ended.kind === "deadline")
+                throw this.timeoutError(stage);
+            if (ended.kind === "exit" && ended.exit.error)
+                throw this.exitError(ended.exit, stage);
+            if (ended.kind === "stop") {
+                proc.interrupt();
+                const closed = await proc.close(this.cleanupTimeoutMs);
+                await this.notify(activeTurn, closed ? "interrupted" : "unavailable", true);
+                return { endReason: this.stopReason, softErrors: this.softErrors };
+            }
+            await this.notify(activeTurn, "completed", true);
+            return { endReason: "completed", softErrors: this.softErrors };
+        }
+        finally {
+            clearTimeout(this.budgetTimer);
+            this.budgetTimer = undefined;
+        }
+    }
+    requestStop(reason) {
+        if (this.stopReason)
+            return;
+        this.stopReason = reason;
+        this.signalStop();
+    }
+    /** Waits for a result (or, without one, for the process to exit), a stop request, or the deadline. */
+    async waitFor(result, stage) {
+        const remaining = this.expiresAt.value - performance.now();
+        if (remaining <= 0)
+            throw this.timeoutError(stage);
+        let timer;
+        try {
+            return await Promise.race([
+                ...(result ? [result.then((line) => ({ kind: "result", line }))] : []),
+                this.proc.exited.then((exit) => ({ kind: "exit", exit })),
+                this.stopped,
+                new Promise((resolve) => {
+                    timer = setTimeout(() => resolve({ kind: "deadline" }), Math.ceil(remaining));
+                }),
+            ]);
+        }
+        finally {
+            clearTimeout(timer);
+        }
+    }
+    async notify(turn, status, final = false) {
+        if (!this.options.onTurnComplete || this.notified.has(turn))
+            return;
+        this.notified.add(turn);
+        this.pauseBudget();
+        const callbackStartedAt = performance.now();
+        try {
+            await this.options.onTurnComplete({ turn, status, ...(final && { final: true }) });
+        }
+        finally {
+            // Host-side work doesn't count against the agent's budgets.
+            this.expiresAt.value += performance.now() - callbackStartedAt;
+            this.resumeBudget();
+        }
+    }
+    pauseBudget() {
+        if (this.budgetTimer === undefined || this.budgetStartedAt === undefined || this.budgetRemaining === undefined)
+            return;
+        clearTimeout(this.budgetTimer);
+        this.budgetTimer = undefined;
+        this.budgetRemaining = Math.max(0, this.budgetRemaining - (performance.now() - this.budgetStartedAt));
+        this.budgetStartedAt = undefined;
+    }
+    resumeBudget() {
+        if (this.stopReason || this.budgetRemaining === undefined || this.budgetTimer !== undefined)
+            return;
+        this.budgetStartedAt = performance.now();
+        this.budgetTimer = setTimeout(() => {
+            this.budgetTimer = undefined;
+            this.budgetRemaining = 0;
+            this.requestStop("agent_timeout");
+        }, this.budgetRemaining);
+    }
+    timeoutError(stage) {
+        return new ExecutorDeadlineExceededError(`Stimulus '${this.stimulus.name}' timed out after ${this.options.timeout} ms during ${stage}`);
+    }
+    exitError(exit, stage) {
+        const how = exit.error
+            ? exit.error.message
+            : `exited unexpectedly with ${exit.signal ? `signal ${exit.signal}` : `code ${exit.code}`}`;
+        const stderr = this.proc.stderr.trim();
+        const error = new Error(`Claude Code ${how} during ${stage} of stimulus '${this.stimulus.name}'${stderr ? `:\n${stderr}` : ""}`, exit.error ? { cause: exit.error } : undefined);
+        if (exit.error) {
+            const { code, syscall } = exit.error;
+            Object.assign(error, { code, syscall });
+        }
+        return error;
+    }
+}
+function resultError(result) {
+    const parts = [
+        typeof result.result === "string" && result.result ? result.result : undefined,
+        ...(Array.isArray(result.errors) ? result.errors : []),
+    ].filter(Boolean);
+    return parts.length ? parts.join("; ") : `${result.subtype ?? "error"} (${result.terminal_reason ?? "unknown reason"})`;
+}
+function isRetryableStatus(status) {
+    return typeof status === "number" && (status === 408 || status === 429 || status >= 500);
+}
+async function settleWithin(promise, ms) {
+    let timer;
+    try {
+        return await Promise.race([
+            promise.then(() => true),
+            new Promise((resolve) => {
+                timer = setTimeout(() => resolve(false), ms);
+            }),
+        ]);
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
+export function buildEnv(options, provider) {
+    const inherited = { ...process.env };
+    if (provider) {
+        for (const name of provider.unset)
+            delete inherited[name];
+        Object.assign(inherited, provider.set);
+    }
+    const env = mergeEnv(inherited, options.env);
+    const trace = options.traceContext;
+    if (trace?.traceparent) {
+        env.TRACEPARENT = trace.traceparent;
+        if (trace.tracestate)
+            env.TRACESTATE = trace.tracestate;
+        else
+            delete env.TRACESTATE;
+    }
+    return env;
+}
+/**
+ * Exposes Vally skills to Claude Code without touching the workspace: each skill
+ * directory is linked under `<dir>/.claude/skills/<name>`, and `<dir>` is passed
+ * with `--add-dir`.
+ */
+async function linkSkills(runDir, skills) {
+    const withPaths = skills?.filter((skill) => skill.path) ?? [];
+    if (withPaths.length === 0)
+        return undefined;
+    const root = path.join(runDir, "skills");
+    const skillsDir = path.join(root, ".claude", "skills");
+    await mkdir(skillsDir, { recursive: true });
+    const names = new Set();
+    for (const skill of withPaths) {
+        const source = path.dirname(skill.path);
+        const name = (skill.name || path.basename(source)).replace(/[^A-Za-z0-9._-]/g, "-");
+        if (names.has(name))
+            throw new Error(`Two skills resolve to the same name '${name}'`);
+        names.add(name);
+        await symlink(source, path.join(skillsDir, name), "junction");
+    }
+    return root;
+}
+async function openArtifacts(dir) {
+    if (!dir)
+        return { close: async () => { } };
+    await mkdir(dir, { recursive: true });
+    const stream = createWriteStream(path.join(dir, "claude-code.jsonl"));
+    const stderr = createWriteStream(path.join(dir, "claude-code.stderr.log"));
+    const end = (file) => new Promise((resolve) => file.end(resolve));
+    return {
+        dir,
+        stream,
+        stderr,
+        close: async () => {
+            await Promise.all([end(stream), end(stderr)]);
+        },
+    };
+}

package/dist/hook.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dist/hook.js

@@ -0,0 +1,21 @@
+#!/usr/bin/env node
+// Claude Code PreToolUse hook that answers simulated tool calls.
+// Usage: node hook.js <overrides.json>; the hook event arrives as JSON on stdin.
+import { readFileSync } from "node:fs";
+import { hookResponse } from "./simulation.js";
+const chunks = [];
+process.stdin.on("data", (chunk) => chunks.push(chunk));
+process.stdin.on("end", () => {
+    try {
+        const event = JSON.parse(Buffer.concat(chunks).toString("utf8"));
+        const overrides = JSON.parse(readFileSync(process.argv[2], "utf8"));
+        const response = event.tool_name ? hookResponse(event.tool_name, event.tool_input, overrides) : undefined;
+        if (response)
+            process.stdout.write(JSON.stringify(response));
+    }
+    catch (error) {
+        // Exit code 2 blocks the call, so a broken simulation never lets the real tool run.
+        process.stderr.write(`vally simulation hook failed: ${error instanceof Error ? error.message : String(error)}`);
+        process.exitCode = 2;
+    }
+});

package/dist/index.d.ts

@@ -0,0 +1,5 @@
+import type { ExecutorRegistry } from "@microsoft/vally";
+export declare function registerExecutors(registry: ExecutorRegistry): void;
+export { ClaudeCodeExecutor, EXECUTOR_NAME } from "./executor.js";
+export type { ClaudeCodeExecutorOptions, SpawnFn } from "./executor.js";
+export type { ClaudeCodeExecutorConfig, ClaudeCodeProviderConfig, PermissionMode, ProviderType, SettingSource, } from "./config.js";

package/dist/index.js

@@ -0,0 +1,5 @@
+import { ClaudeCodeExecutor } from "./executor.js";
+export function registerExecutors(registry) {
+    registry.register(new ClaudeCodeExecutor());
+}
+export { ClaudeCodeExecutor, EXECUTOR_NAME } from "./executor.js";

package/dist/process.d.ts

@@ -0,0 +1,32 @@
+import type { ChildProcessWithoutNullStreams } from "node:child_process";
+import type { StreamLine } from "./trajectory.js";
+export interface ProcessExit {
+    code: number | null;
+    signal: NodeJS.Signals | null;
+    error?: Error;
+}
+/** Wraps a Claude Code process speaking stream-json on stdin/stdout. */
+export declare class ClaudeProcess {
+    private readonly child;
+    readonly lines: StreamLine[];
+    readonly exited: Promise<ProcessExit>;
+    private stderrTail;
+    private readonly results;
+    private resultWaiters;
+    private readonly listeners;
+    private exit?;
+    private interrupts;
+    constructor(child: ChildProcessWithoutNullStreams, onLine: (line: StreamLine, raw: string) => void, onStderr: (chunk: string) => void);
+    onLine(listener: (line: StreamLine) => void): void;
+    /** Closes stdin and waits for a clean exit, killing the process if it doesn't exit in time. */
+    close(timeoutMs: number): Promise<boolean>;
+    get stderr(): string;
+    get hasExited(): boolean;
+    /** Resolves with the next `result` line, i.e. the end of the current agent turn. */
+    nextResult(): Promise<StreamLine>;
+    send(message: string): void;
+    interrupt(): void;
+    endInput(): void;
+    /** Terminates the process, escalating to SIGKILL after `graceMs`. */
+    kill(graceMs: number): Promise<ProcessExit>;
+}

package/dist/process.js

@@ -0,0 +1,119 @@
+import { createInterface } from "node:readline";
+import { interruptRequest } from "./args.js";
+const STDERR_TAIL_BYTES = 16 * 1024;
+/** Wraps a Claude Code process speaking stream-json on stdin/stdout. */
+export class ClaudeProcess {
+    child;
+    lines = [];
+    exited;
+    stderrTail = "";
+    results = [];
+    resultWaiters = [];
+    listeners = [];
+    exit;
+    interrupts = 0;
+    constructor(child, onLine, onStderr) {
+        this.child = child;
+        this.exited = new Promise((resolve) => {
+            let spawnError;
+            child.once("error", (error) => {
+                spawnError = error;
+                // A process that failed to spawn never emits 'close'.
+                if (child.pid === undefined)
+                    resolve((this.exit = { code: null, signal: null, error }));
+            });
+            child.once("close", (code, signal) => resolve((this.exit = { code, signal, error: spawnError })));
+        });
+        // Writes after the process exits fail with EPIPE; the exit is reported separately.
+        child.stdin.on("error", () => { });
+        child.stderr.setEncoding("utf8");
+        child.stderr.on("data", (chunk) => {
+            this.stderrTail = (this.stderrTail + chunk).slice(-STDERR_TAIL_BYTES);
+            onStderr(chunk);
+        });
+        createInterface({ input: child.stdout, crlfDelay: Infinity }).on("line", (raw) => {
+            if (!raw.trim())
+                return;
+            let line;
+            try {
+                line = JSON.parse(raw);
+            }
+            catch {
+                onStderr(`${raw}\n`);
+                return;
+            }
+            if (typeof line !== "object" || line === null || typeof line.type !== "string")
+                return;
+            this.lines.push(line);
+            onLine(line, raw);
+            for (const listener of this.listeners)
+                listener(line);
+            if (line.type === "result") {
+                const waiter = this.resultWaiters.shift();
+                if (waiter)
+                    waiter(line);
+                else
+                    this.results.push(line);
+            }
+        });
+    }
+    onLine(listener) {
+        this.listeners.push(listener);
+    }
+    /** Closes stdin and waits for a clean exit, killing the process if it doesn't exit in time. */
+    async close(timeoutMs) {
+        this.endInput();
+        let timer;
+        const exited = await Promise.race([
+            this.exited.then(() => true),
+            new Promise((resolve) => {
+                timer = setTimeout(() => resolve(false), timeoutMs);
+            }),
+        ]);
+        clearTimeout(timer);
+        if (!exited)
+            await this.kill(timeoutMs);
+        return exited;
+    }
+    get stderr() {
+        return this.stderrTail;
+    }
+    get hasExited() {
+        return this.exit !== undefined;
+    }
+    /** Resolves with the next `result` line, i.e. the end of the current agent turn. */
+    nextResult() {
+        const queued = this.results.shift();
+        if (queued)
+            return Promise.resolve(queued);
+        return new Promise((resolve) => this.resultWaiters.push(resolve));
+    }
+    send(message) {
+        this.child.stdin.write(message);
+    }
+    interrupt() {
+        this.send(interruptRequest(`vally-interrupt-${++this.interrupts}`));
+    }
+    endInput() {
+        if (!this.child.stdin.writableEnded)
+            this.child.stdin.end();
+    }
+    /** Terminates the process, escalating to SIGKILL after `graceMs`. */
+    async kill(graceMs) {
+        if (this.exit)
+            return this.exit;
+        this.child.kill("SIGTERM");
+        let timer;
+        const exited = await Promise.race([
+            this.exited,
+            new Promise((resolve) => {
+                timer = setTimeout(() => resolve(undefined), graceMs);
+            }),
+        ]);
+        clearTimeout(timer);
+        if (exited)
+            return exited;
+        this.child.kill("SIGKILL");
+        return this.exited;
+    }
+}

package/dist/simulation.d.ts

@@ -0,0 +1,20 @@
+import type { SimulationConfig } from "@microsoft/vally";
+type Overrides = NonNullable<SimulationConfig["tool_overrides"]>;
+export interface SimulatedResult {
+    output: string;
+    isError: boolean;
+    shell: boolean;
+}
+/**
+ * Decides the simulated result for a Claude Code tool call, or `null` to run the
+ * real tool. Shell overrides (`bash`/`shell`/`powershell`) apply to Claude's
+ * `Bash` and `PowerShell` tools; other keys match Claude tool names exactly.
+ */
+export declare function resolveToolOverride(toolName: string, args: unknown, overrides: Overrides | undefined): SimulatedResult | null;
+/** A shell command that prints the canned output and exits with the matching status. */
+export declare function renderShellCommand(toolName: string, result: SimulatedResult): string;
+/** The PreToolUse hook response for a tool call, or `undefined` to let it run unchanged. */
+export declare function hookResponse(toolName: string, toolInput: unknown, overrides: Overrides | undefined): Record<string, unknown> | undefined;
+/** `--settings` JSON that routes every tool call through the simulation hook. */
+export declare function simulationSettings(hookScript: string, overridesFile: string): string;
+export {};