vally-claudecodecli 0.0.0-stage → 0.1.0

package/dist/simulation.js

@@ -0,0 +1,121 @@
+/** Vally's shell override keys; they alias one another. */
+const SHELL_KEYS = ["bash", "shell", "powershell"];
+/** Claude Code's shell tools and the Vally key each answers to first. */
+const CLAUDE_SHELL_TOOLS = { Bash: "bash", PowerShell: "powershell" };
+function isPatternOverride(value) {
+    return typeof value === "object" && value !== null && "patterns" in value;
+}
+function normalizeFixed(value) {
+    if (typeof value === "string")
+        return { output: value, isError: false };
+    return { output: typeof value.output === "string" ? value.output : String(value.output ?? ""), isError: value.is_error === true };
+}
+/** Builds a RegExp, honoring a leading inline-flags group such as `(?i)`. */
+function toRegExp(source, defaultFlags) {
+    const inline = /^\(\?([ims]+)\)/.exec(source);
+    return inline ? new RegExp(source.slice(inline[0].length), inline[1]) : new RegExp(source, defaultFlags);
+}
+function resolveKey(toolName, overrides) {
+    const has = (key) => Object.prototype.hasOwnProperty.call(overrides, key);
+    if (has(toolName))
+        return toolName;
+    const shellKey = CLAUDE_SHELL_TOOLS[toolName];
+    if (!shellKey)
+        return undefined;
+    return [shellKey, ...SHELL_KEYS].find(has);
+}
+function matches(pattern, args, shell) {
+    if (pattern.match === undefined && pattern.args === undefined && pattern.pattern === undefined)
+        return false;
+    if (pattern.match !== undefined) {
+        const command = typeof args.command === "string" ? args.command : "";
+        if (!shell || !toRegExp(pattern.match, "i").test(command))
+            return false;
+    }
+    for (const [name, source] of Object.entries(pattern.args ?? {})) {
+        const value = args[name];
+        if (typeof value !== "string" || !toRegExp(source).test(value))
+            return false;
+    }
+    if (pattern.pattern !== undefined) {
+        let serialized;
+        try {
+            serialized = JSON.stringify(args);
+        }
+        catch {
+            return false;
+        }
+        if (!toRegExp(pattern.pattern).test(serialized))
+            return false;
+    }
+    return true;
+}
+/**
+ * Decides the simulated result for a Claude Code tool call, or `null` to run the
+ * real tool. Shell overrides (`bash`/`shell`/`powershell`) apply to Claude's
+ * `Bash` and `PowerShell` tools; other keys match Claude tool names exactly.
+ */
+export function resolveToolOverride(toolName, args, overrides) {
+    if (!overrides)
+        return null;
+    const key = resolveKey(toolName, overrides);
+    if (key === undefined)
+        return null;
+    const shell = toolName in CLAUDE_SHELL_TOOLS;
+    const record = typeof args === "object" && args !== null && !Array.isArray(args) ? args : {};
+    const value = overrides[key];
+    if (isPatternOverride(value)) {
+        for (const pattern of value.patterns) {
+            if (matches(pattern, record, shell))
+                return { output: pattern.output, isError: pattern.is_error === true, shell };
+        }
+        return value.default === undefined ? null : { ...normalizeFixed(value.default), shell };
+    }
+    return { ...normalizeFixed(value), shell };
+}
+/** A shell command that prints the canned output and exits with the matching status. */
+export function renderShellCommand(toolName, result) {
+    const exitCode = result.isError ? 1 : 0;
+    if (toolName === "PowerShell") {
+        // Base64 keeps PowerShell's many quote characters from terminating the literal.
+        const b64 = Buffer.from(result.output, "utf8").toString("base64");
+        const stream = result.isError ? "Error" : "Out";
+        return `[Console]::${stream}.Write([Text.Encoding]::UTF8.GetString([Convert]::FromBase64String('${b64}'))); exit ${exitCode}`;
+    }
+    const escaped = result.output.replace(/'/g, `'\\''`);
+    return `printf '%s' '${escaped}'${result.isError ? " >&2" : ""}; exit ${exitCode}`;
+}
+/** The PreToolUse hook response for a tool call, or `undefined` to let it run unchanged. */
+export function hookResponse(toolName, toolInput, overrides) {
+    const result = resolveToolOverride(toolName, toolInput, overrides);
+    if (!result)
+        return undefined;
+    if (result.shell) {
+        return {
+            hookSpecificOutput: {
+                hookEventName: "PreToolUse",
+                permissionDecision: "allow",
+                updatedInput: { ...toolInput, command: renderShellCommand(toolName, result) },
+            },
+        };
+    }
+    // Claude Code can't return a canned result for non-shell tools; denying the call surfaces
+    // the output to the model as the tool's error, which is what Vally allows for these tools.
+    return {
+        hookSpecificOutput: {
+            hookEventName: "PreToolUse",
+            permissionDecision: "deny",
+            permissionDecisionReason: result.output,
+        },
+    };
+}
+function quote(value) {
+    return `"${value.replace(/(["\\$`])/g, "\\$1")}"`;
+}
+/** `--settings` JSON that routes every tool call through the simulation hook. */
+export function simulationSettings(hookScript, overridesFile) {
+    const command = [process.execPath, hookScript, overridesFile].map(quote).join(" ");
+    return JSON.stringify({
+        hooks: { PreToolUse: [{ matcher: "*", hooks: [{ type: "command", command }] }] },
+    });
+}

package/dist/trajectory.d.ts

@@ -0,0 +1,48 @@
+import type { AtifTrajectory, Skill, Stimulus, Trajectory, TrajectoryEndReason, TrajectoryEvent } from "@microsoft/vally";
+/** A parsed Claude Code stream-json line. Only the fields the executor reads are typed. */
+export interface StreamLine {
+    type: string;
+    subtype?: string;
+    session_id?: string;
+    model?: string;
+    parent_tool_use_id?: string | null;
+    isReplay?: boolean;
+    is_error?: boolean;
+    result?: string;
+    errors?: string[];
+    api_error_status?: number | null;
+    terminal_reason?: string;
+    message?: {
+        id?: string;
+        content?: unknown;
+    };
+    [key: string]: unknown;
+}
+export interface BuildTrajectoryInput {
+    lines: StreamLine[];
+    stimulus: Stimulus;
+    prompts: string[];
+    workDir: string;
+    executor: string;
+    requestedModel?: string;
+    skills?: Skill[];
+    startedAt: Date;
+    completedAt: Date;
+    endReason: TrajectoryEndReason;
+    /** Error results the agent hit but that still produced a gradable run (e.g. budget exhausted). */
+    softErrors: string[];
+    sessionID?: string;
+}
+export declare function toAtif(lines: StreamLine[]): Promise<AtifTrajectory>;
+/**
+ * Stamps each event with the configured turn it belongs to. Turns advance only on
+ * a root user message matching the next configured prompt, so other user
+ * messages (interrupt notices, subagent prompts) don't shift the count.
+ */
+export declare function stampTurns(events: TrajectoryEvent[], prompts: string[]): void;
+/** The final answer: the last root `result` text, falling back to the last assistant message. */
+export declare function finalOutput(lines: StreamLine[], events: TrajectoryEvent[]): string;
+export declare function buildTrajectory(input: BuildTrajectoryInput): Promise<{
+    trajectory: Trajectory;
+    atif: AtifTrajectory;
+}>;

package/dist/trajectory.js

@@ -0,0 +1,86 @@
+import { computeMetrics, fromAtif, validateAtifTrajectory } from "@microsoft/vally";
+import { convert } from "atifact";
+import { resolveToolOverride } from "./simulation.js";
+export async function toAtif(lines) {
+    const content = lines.map((line) => JSON.stringify(line)).join("\n");
+    let trajectory;
+    try {
+        ({ trajectory } = await convert(content, { format: "claude-code-jsonl", sourceName: "claude-code stream-json" }));
+    }
+    catch (error) {
+        throw new Error("Failed to convert the Claude Code log to ATIF", { cause: error });
+    }
+    return validateAtifTrajectory(trajectory);
+}
+/**
+ * Stamps each event with the configured turn it belongs to. Turns advance only on
+ * a root user message matching the next configured prompt, so other user
+ * messages (interrupt notices, subagent prompts) don't shift the count.
+ */
+export function stampTurns(events, prompts) {
+    let turn = 0;
+    for (const event of events) {
+        if (event.type === "user_message" &&
+            event.agentId == null &&
+            turn + 1 < prompts.length &&
+            event.data.content.trim() === prompts[turn + 1].trim()) {
+            turn++;
+        }
+        event.turn = turn;
+    }
+}
+/** The final answer: the last root `result` text, falling back to the last assistant message. */
+export function finalOutput(lines, events) {
+    for (let i = lines.length - 1; i >= 0; i--) {
+        const line = lines[i];
+        if (line.type === "result" && typeof line.result === "string" && !line.is_error)
+            return line.result;
+    }
+    for (let i = events.length - 1; i >= 0; i--) {
+        const event = events[i];
+        if (event.type === "assistant_message" && event.agentId == null && event.data.content)
+            return event.data.content;
+    }
+    return "";
+}
+export async function buildTrajectory(input) {
+    const atif = await toAtif(input.lines);
+    const trajectory = fromAtif(atif, {
+        stimulusName: input.stimulus.name,
+        stimulusPrompt: input.stimulus.prompt,
+        workDir: input.workDir,
+    });
+    const events = trajectory.events;
+    stampTurns(events, input.prompts);
+    const overrides = input.stimulus.simulation?.tool_overrides;
+    const skillPaths = new Map(input.skills?.flatMap((skill) => (skill.path ? [[skill.name, skill.path]] : [])));
+    for (const event of events) {
+        if (event.type === "tool_call" && resolveToolOverride(event.data.toolName, event.data.arguments, overrides)) {
+            event.data.simulated = true;
+        }
+        if (event.type === "skill_activation" && !event.data.path) {
+            const skillPath = skillPaths.get(event.data.name);
+            if (skillPath)
+                event.data.path = skillPath;
+        }
+    }
+    const lastTurn = events.at(-1)?.turn ?? 0;
+    for (const message of input.softErrors) {
+        events.push({ type: "error", timestamp: input.completedAt, turn: lastTurn, data: { message } });
+    }
+    const init = input.lines.find((line) => line.type === "system" && line.subtype === "init");
+    const metrics = computeMetrics(events);
+    trajectory.output = finalOutput(input.lines, events);
+    trajectory.stimulus = input.stimulus;
+    trajectory.endReason = input.endReason;
+    trajectory.metrics = { ...metrics, wallTimeMs: input.completedAt.getTime() - input.startedAt.getTime() };
+    trajectory.metadata = {
+        ...trajectory.metadata,
+        model: init?.model ?? trajectory.metadata.model ?? input.requestedModel ?? "unknown",
+        executor: input.executor,
+        sessionID: input.sessionID ?? init?.session_id ?? trajectory.metadata.sessionID,
+        startedAt: input.startedAt,
+        completedAt: input.completedAt,
+    };
+    return { trajectory, atif };
+}

package/package.json

@@ -1,6 +1,76 @@
 {
   "name": "vally-claudecodecli",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.0",
+  "description": "Vally executor plugin that evaluates the Claude Code CLI",
+  "keywords": [
+    "claude-code",
+    "claude",
+    "anthropic",
+    "claude-skills",
+    "cli",
+    "agent-evaluation",
+    "ai-evaluation",
+    "llm-evaluation",
+    "evals",
+    "evaluation",
+    "benchmarking",
+    "testing",
+    "ai-agents",
+    "agentic-ai",
+    "coding-agent",
+    "agent-skills",
+    "mcp",
+    "model-context-protocol",
+    "atif",
+    "vally",
+    "executor",
+    "developer-tools"
+  ],
+  "homepage": "https://github.com/waldekmastykarz/vally-claudecodecli#readme",
+  "bugs": {
+    "url": "https://github.com/waldekmastykarz/vally-claudecodecli/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/waldekmastykarz/vally-claudecodecli.git"
+  },
+  "license": "MIT",
+  "author": "Waldek Mastykarz",
+  "type": "module",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "default": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "clean": "node --eval \"require('node:fs').rmSync('dist', { recursive: true, force: true })\"",
+    "build": "npm run clean && tsc -p tsconfig.build.json",
+    "prepack": "npm run build",
+    "test": "vitest run",
+    "typecheck": "tsc -p tsconfig.json --noEmit"
+  },
+  "engines": {
+    "node": ">=22.15.0"
+  },
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org/"
+  },
+  "peerDependencies": {
+    "@microsoft/vally": ">=0.21.0"
+  },
+  "dependencies": {
+    "atifact": "^0.14.0"
+  },
+  "devDependencies": {
+    "@microsoft/vally": "^0.21.0",
+    "@types/node": "^24.19.2",
+    "typescript": "^7.0.2",
+    "vitest": "^5.1.0"
+  }
+}