praxis-agent 0.1.0

package/dist/plugins/claude-plugin-eval-graders.js

@@ -0,0 +1,137 @@
+import { readFile } from 'node:fs/promises';
+import { minimatch } from 'minimatch';
+import { resolveContainedPath } from './claude-plugin-eval-schema.js';
+function subset(actual, expected) {
+    if (expected && typeof expected === 'object' && !Array.isArray(expected)) {
+        if (!actual || typeof actual !== 'object' || Array.isArray(actual))
+            return false;
+        return Object.entries(expected).every(([key, value]) => subset(actual[key], value));
+    }
+    return Object.is(actual, expected);
+}
+function matches(event, match) {
+    const selected = typeof match === 'string' ? { tool: match } : match;
+    return (event.type === 'tool-call' &&
+        event.tool === selected.tool &&
+        (!selected.input_match || subset(event.input, selected.input_match)));
+}
+async function focusText(focus, artifacts) {
+    if (focus === 'last_message')
+        return artifacts.lastMessage;
+    if (focus === 'trace')
+        return artifacts.trace.map((item) => JSON.stringify(item)).join('\n');
+    if (focus === 'files') {
+        const { glob } = await import('node:fs/promises');
+        const names = [];
+        for await (const path of glob('**/*', {
+            cwd: artifacts.cwd,
+            exclude: ['node_modules/**', '.git/**'],
+        }))
+            names.push(path);
+        return names.sort().join('\n');
+    }
+    return readFile(await resolveContainedPath(artifacts.cwd, focus.path, 'Grader focus'), 'utf8');
+}
+async function freeGrade(grader, artifacts) {
+    let passed = false;
+    let evidence = '';
+    if (grader.type === 'regex') {
+        const target = await focusText(grader.target, artifacts);
+        const expression = new RegExp(grader.pattern, grader.flags.includes('g') ? grader.flags : `${grader.flags}g`);
+        const count = [...target.matchAll(expression)].length;
+        passed =
+            grader.match === 'contains'
+                ? count > 0
+                : grader.match === 'not_contains'
+                    ? count === 0
+                    : count === Number(grader.match.slice(6));
+        evidence = `matches=${count}`;
+    }
+    else if (grader.type === 'tool_used') {
+        const count = artifacts.trace.filter((event) => matches(event, {
+            tool: grader.tool,
+            ...(grader.input_match ? { input_match: grader.input_match } : {}),
+        })).length;
+        passed = count >= grader.min && count <= grader.max;
+        evidence = `uses=${count}`;
+    }
+    else if (grader.type === 'tool_order') {
+        const before = artifacts.trace.findIndex((event) => matches(event, grader.before));
+        const after = artifacts.trace.findIndex((event, index) => index > before && matches(event, grader.after));
+        passed = before >= 0 && after > before;
+        evidence = `before=${before},after=${after}`;
+    }
+    else {
+        const { glob } = await import('node:fs/promises');
+        let count = 0;
+        for await (const path of glob('**/*', { cwd: artifacts.cwd })) {
+            if (!minimatch(path, grader.path))
+                continue;
+            await resolveContainedPath(artifacts.cwd, path, 'file_exists match');
+            count += 1;
+        }
+        passed = grader.exists ? count > 0 : count === 0;
+        evidence = `files=${count}`;
+    }
+    return {
+        name: grader.name,
+        passed,
+        weight: grader.weight,
+        explanation: passed ? 'passed' : 'failed',
+        evidence,
+        ...(grader.arm === 'with-only' ? { with_only: true } : {}),
+    };
+}
+export async function gradeClaudePluginEvalRun(options) {
+    const results = [];
+    let skippedPaidGraders = false;
+    let judgeCostUsd = 0;
+    for (const item of options.case.graders) {
+        if (options.arm === 'without' && item.arm === 'with-only')
+            continue;
+        if (item.type !== 'llm' && item.type !== 'baseline') {
+            results.push(await freeGrade(item, options.artifacts));
+            continue;
+        }
+        if (options.skipPaid) {
+            skippedPaidGraders = true;
+            continue;
+        }
+        if (!options.judge)
+            throw new Error(`Paid grader ${item.name} requires an eval judge`);
+        const judge = options.judge;
+        const focus = item.type === 'llm'
+            ? await focusText(item.focus, options.artifacts)
+            : options.artifacts.lastMessage;
+        const baseline = item.type === 'baseline'
+            ? await readFile(await resolveContainedPath(options.case.dir, item.baseline_file, 'Baseline file'), 'utf8')
+            : undefined;
+        const votes = await Promise.all(Array.from({ length: 3 }, () => judge.vote({
+            criteria: item.criteria,
+            focus,
+            ...(baseline === undefined ? {} : { baseline }),
+            model: options.judgeModel,
+            ...(options.signal ? { signal: options.signal } : {}),
+        })));
+        const passed = votes.filter((vote) => vote.passed).length >= 2;
+        judgeCostUsd += votes.reduce((sum, vote) => sum + vote.costUsd, 0);
+        results.push({
+            name: item.name,
+            passed,
+            weight: item.weight,
+            explanation: votes.find((vote) => vote.explanation)?.explanation ??
+                (passed ? 'passed' : 'failed'),
+            judge_votes: votes.map((vote) => vote.passed),
+            ...(item.arm === 'with-only' ? { with_only: true } : {}),
+        });
+    }
+    const scored = results;
+    const total = scored.reduce((sum, item) => sum + item.weight, 0);
+    const score = total === 0
+        ? 0
+        : scored
+            .filter((item) => item.passed)
+            .reduce((sum, item) => sum + item.weight, 0) / total;
+    return { score, graders: results, skippedPaidGraders, judgeCostUsd };
+}
+//# sourceMappingURL=claude-plugin-eval-graders.js.map

package/dist/plugins/claude-plugin-eval-graders.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"claude-plugin-eval-graders.js","sourceRoot":"","sources":["../../src/plugins/claude-plugin-eval-graders.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAE3C,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AAQrC,OAAO,EAAE,oBAAoB,EAAE,MAAM,gCAAgC,CAAA;AAgCrE,SAAS,MAAM,CAAC,MAAe,EAAE,QAAiB;IAChD,IAAI,QAAQ,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QACzE,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;YAChE,OAAO,KAAK,CAAA;QACd,OAAO,MAAM,CAAC,OAAO,CAAC,QAAmC,CAAC,CAAC,KAAK,CAC9D,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,MAAM,CAAE,MAAkC,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAC1E,CAAA;IACH,CAAC;IACD,OAAO,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAA;AACpC,CAAC;AAED,SAAS,OAAO,CAAC,KAAqB,EAAE,KAAoB;IAC1D,MAAM,QAAQ,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,CAAA;IACpE,OAAO,CACL,KAAK,CAAC,IAAI,KAAK,WAAW;QAC1B,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC,IAAI;QAC5B,CAAC,CAAC,QAAQ,CAAC,WAAW,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC,CACrE,CAAA;AACH,CAAC;AAED,KAAK,UAAU,SAAS,CACtB,KAAgB,EAChB,SAA2B;IAE3B,IAAI,KAAK,KAAK,cAAc;QAAE,OAAO,SAAS,CAAC,WAAW,CAAA;IAC1D,IAAI,KAAK,KAAK,OAAO;QACnB,OAAO,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACvE,IAAI,KAAK,KAAK,OAAO,EAAE,CAAC;QACtB,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAA;QACjD,MAAM,KAAK,GAAa,EAAE,CAAA;QAC1B,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,IAAI,CAAC,MAAM,EAAE;YACpC,GAAG,EAAE,SAAS,CAAC,GAAG;YAClB,OAAO,EAAE,CAAC,iBAAiB,EAAE,SAAS,CAAC;SACxC,CAAC;YACA,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAClB,OAAO,KAAK,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAChC,CAAC;IACD,OAAO,QAAQ,CACb,MAAM,oBAAoB,CAAC,SAAS,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,EAAE,cAAc,CAAC,EACrE,MAAM,CACP,CAAA;AACH,CAAC;AAED,KAAK,UAAU,SAAS,CACtB,MAAqE,EACrE,SAA2B;IAE3B,IAAI,MAAM,GAAG,KAAK,CAAA;IAClB,IAAI,QAAQ,GAAG,EAAE,CAAA;IACjB,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAA;QACxD,MAAM,UAAU,GAAG,IAAI,MAAM,CAC3B,MAAM,CAAC,OAAO,EACd,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,GAAG,CAC/D,CAAA;QACD,MAAM,KAAK,GAAG,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,CAAA;QACrD,MAAM;YACJ,MAAM,CAAC,KAAK,KAAK,UAAU;gBACzB,CAAC,CAAC,KAAK,GAAG,CAAC;gBACX,CAAC,CAAC,MAAM,CAAC,KAAK,KAAK,cAAc;oBAC/B,CAAC,CAAC,KAAK,KAAK,CAAC;oBACb,CAAC,CAAC,KAAK,KAAK,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;QAC/C,QAAQ,GAAG,WAAW,KAAK,EAAE,CAAA;IAC/B,CAAC;SAAM,IAAI,MAAM,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QACvC,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAC7C,OAAO,CAAC,KAAK,EAAE;YACb,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACnE,CAAC,CACH,CAAC,MAAM,CAAA;QACR,MAAM,GAAG,KAAK,IAAI,MAAM,CAAC,GAAG,IAAI,KAAK,IAAI,MAAM,CAAC,GAAG,CAAA;QACnD,QAAQ,GAAG,QAAQ,KAAK,EAAE,CAAA;IAC5B,CAAC;SAAM,IAAI,MAAM,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;QACxC,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,EAAE,CACjD,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,CAC9B,CAAA;QACD,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CACrC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,MAAM,IAAI,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,CACjE,CAAA;QACD,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,KAAK,GAAG,MAAM,CAAA;QACtC,QAAQ,GAAG,UAAU,MAAM,UAAU,KAAK,EAAE,CAAA;IAC9C,CAAC;SAAM,CAAC;QACN,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAA;QACjD,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,GAAG,EAAE,SAAS,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC;YAC9D,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC;gBAAE,SAAQ;YAC3C,MAAM,oBAAoB,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,mBAAmB,CAAC,CAAA;YACpE,KAAK,IAAI,CAAC,CAAA;QACZ,CAAC;QACD,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAA;QAChD,QAAQ,GAAG,SAAS,KAAK,EAAE,CAAA;IAC7B,CAAC;IACD,OAAO;QACL,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,MAAM;QACN,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;QACzC,QAAQ;QACR,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,WAAW,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC3D,CAAA;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAAC,OAQ9C;IAMC,MAAM,OAAO,GAAuB,EAAE,CAAA;IACtC,IAAI,kBAAkB,GAAG,KAAK,CAAA;IAC9B,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;QACxC,IAAI,OAAO,CAAC,GAAG,KAAK,SAAS,IAAI,IAAI,CAAC,GAAG,KAAK,WAAW;YAAE,SAAQ;QACnE,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YACpD,OAAO,CAAC,IAAI,CAAC,MAAM,SAAS,CAAC,IAAI,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC,CAAA;YACtD,SAAQ;QACV,CAAC;QACD,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACrB,kBAAkB,GAAG,IAAI,CAAA;YACzB,SAAQ;QACV,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,KAAK;YAChB,MAAM,IAAI,KAAK,CAAC,eAAe,IAAI,CAAC,IAAI,yBAAyB,CAAC,CAAA;QACpE,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAA;QAC3B,MAAM,KAAK,GACT,IAAI,CAAC,IAAI,KAAK,KAAK;YACjB,CAAC,CAAC,MAAM,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,SAAS,CAAC;YAChD,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,WAAW,CAAA;QACnC,MAAM,QAAQ,GACZ,IAAI,CAAC,IAAI,KAAK,UAAU;YACtB,CAAC,CAAC,MAAM,QAAQ,CACZ,MAAM,oBAAoB,CACxB,OAAO,CAAC,IAAI,CAAC,GAAG,EAChB,IAAI,CAAC,aAAa,EAClB,eAAe,CAChB,EACD,MAAM,CACP;YACH,CAAC,CAAC,SAAS,CAAA;QACf,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,CAC7B,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,GAAG,EAAE,CAC7B,KAAK,CAAC,IAAI,CAAC;YACT,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,KAAK;YACL,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC;YAC/C,KAAK,EAAE,OAAO,CAAC,UAAU;YACzB,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACtD,CAAC,CACH,CACF,CAAA;QACD,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,IAAI,CAAC,CAAA;QAC9D,YAAY,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAA;QAClE,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,MAAM;YACN,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,WAAW,EACT,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,WAAW;gBACnD,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC;YAChC,WAAW,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC;YAC7C,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,WAAW,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACzD,CAAC,CAAA;IACJ,CAAC;IACD,MAAM,MAAM,GAAG,OAAO,CAAA;IACtB,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAA;IAChE,MAAM,KAAK,GACT,KAAK,KAAK,CAAC;QACT,CAAC,CAAC,CAAC;QACH,CAAC,CAAC,MAAM;aACH,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC;aAC7B,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,KAAK,CAAA;IAC5D,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,kBAAkB,EAAE,YAAY,EAAE,CAAA;AACtE,CAAC"}

package/dist/plugins/claude-plugin-eval-runner.d.ts

@@ -0,0 +1,50 @@
+import type { RuntimeEvent } from '../core/runtime.js';
+import type { ClaudePluginEvalCase } from './claude-plugin-eval-schema.js';
+import type { EvalRunArtifacts } from './claude-plugin-eval-graders.js';
+export declare const DEFAULT_EVAL_ALLOWED_TOOLS: readonly ["Read", "Glob", "Grep", "Skill"];
+export interface PluginEvalRuntime {
+    run(prompt: string, signal: AbortSignal): Promise<{
+        text: string;
+        turns: number;
+        costUsd?: number;
+    }>;
+    close?(): Promise<void>;
+}
+export interface PluginEvalRuntimeFactory {
+    create(options: {
+        cwd: string;
+        configRoot: string;
+        home: string;
+        model?: string;
+        maxTurns: number;
+        pluginDirectories: readonly string[];
+        allowedTools: readonly string[];
+        appendSystemPrompt?: string;
+        historyFile?: string;
+        addDirs: readonly string[];
+        env: Readonly<Record<string, string>>;
+        eventSink(event: RuntimeEvent): void;
+    }): Promise<PluginEvalRuntime>;
+}
+export interface EvalSingleRunResult {
+    text: string;
+    turns: number;
+    costUsd: number;
+    costKnown: boolean;
+    artifacts: EvalRunArtifacts;
+    tracePath: string;
+    error: string | null;
+    termination: 'timeout' | 'interrupted' | null;
+    tempRoot?: string;
+}
+export declare function resolveEvalAllowedTools(requested: readonly string[], grants: readonly string[]): string[];
+export declare function runClaudePluginEvalOnce(options: {
+    case: ClaudePluginEvalCase;
+    factory: PluginEvalRuntimeFactory;
+    pluginDirectories: readonly string[];
+    model?: string;
+    allowTools?: readonly string[];
+    scaffold: boolean;
+    signal?: AbortSignal;
+}): Promise<EvalSingleRunResult>;
+//# sourceMappingURL=claude-plugin-eval-runner.d.ts.map

package/dist/plugins/claude-plugin-eval-runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"claude-plugin-eval-runner.d.ts","sourceRoot":"","sources":["../../src/plugins/claude-plugin-eval-runner.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAA;AAKtD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,gCAAgC,CAAA;AAE1E,OAAO,KAAK,EACV,gBAAgB,EAEjB,MAAM,iCAAiC,CAAA;AAExC,eAAO,MAAM,0BAA0B,4CAK7B,CAAA;AAEV,MAAM,WAAW,iBAAiB;IAChC,GAAG,CACD,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,WAAW,GAClB,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;IAC7D,KAAK,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;CACxB;AACD,MAAM,WAAW,wBAAwB;IACvC,MAAM,CAAC,OAAO,EAAE;QACd,GAAG,EAAE,MAAM,CAAA;QACX,UAAU,EAAE,MAAM,CAAA;QAClB,IAAI,EAAE,MAAM,CAAA;QACZ,KAAK,CAAC,EAAE,MAAM,CAAA;QACd,QAAQ,EAAE,MAAM,CAAA;QAChB,iBAAiB,EAAE,SAAS,MAAM,EAAE,CAAA;QACpC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAA;QAC/B,kBAAkB,CAAC,EAAE,MAAM,CAAA;QAC3B,WAAW,CAAC,EAAE,MAAM,CAAA;QACpB,OAAO,EAAE,SAAS,MAAM,EAAE,CAAA;QAC1B,GAAG,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;QACrC,SAAS,CAAC,KAAK,EAAE,YAAY,GAAG,IAAI,CAAA;KACrC,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAA;CAC/B;AAED,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb,OAAO,EAAE,MAAM,CAAA;IACf,SAAS,EAAE,OAAO,CAAA;IAClB,SAAS,EAAE,gBAAgB,CAAA;IAC3B,SAAS,EAAE,MAAM,CAAA;IACjB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;IACpB,WAAW,EAAE,SAAS,GAAG,aAAa,GAAG,IAAI,CAAA;IAC7C,QAAQ,CAAC,EAAE,MAAM,CAAA;CAClB;AAwCD,wBAAgB,uBAAuB,CACrC,SAAS,EAAE,SAAS,MAAM,EAAE,EAC5B,MAAM,EAAE,SAAS,MAAM,EAAE,GACxB,MAAM,EAAE,CAQV;AAqDD,wBAAsB,uBAAuB,CAAC,OAAO,EAAE;IACrD,IAAI,EAAE,oBAAoB,CAAA;IAC1B,OAAO,EAAE,wBAAwB,CAAA;IACjC,iBAAiB,EAAE,SAAS,MAAM,EAAE,CAAA;IACpC,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,UAAU,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IAC9B,QAAQ,EAAE,OAAO,CAAA;IACjB,MAAM,CAAC,EAAE,WAAW,CAAA;CACrB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAiH/B"}

package/dist/plugins/claude-plugin-eval-runner.js

@@ -0,0 +1,189 @@
+import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { BoundedProcessRunner, joinedProcessOutput, } from '../platform/bounded-process-runner.js';
+import { resolveContainedPath } from './claude-plugin-eval-schema.js';
+export const DEFAULT_EVAL_ALLOWED_TOOLS = [
+    'Read',
+    'Glob',
+    'Grep',
+    'Skill',
+];
+function abortError() {
+    return new DOMException('Eval run timed out', 'AbortError');
+}
+function toolName(rule) {
+    const separator = rule.indexOf('(');
+    return separator < 0 ? rule : rule.slice(0, separator);
+}
+function gatedTool(rule) {
+    const name = toolName(rule);
+    return (['Bash', 'Write', 'Edit', 'NotebookEdit', 'WebFetch', 'WebSearch'].includes(name) || name.startsWith('mcp__'));
+}
+function operatorGrants(requested, grants) {
+    const name = toolName(requested);
+    const wildcardMatch = (pattern) => {
+        const parts = pattern.split('*');
+        if (parts.length === 1)
+            return false;
+        if (!requested.startsWith(parts[0] ?? ''))
+            return false;
+        let cursor = parts[0]?.length ?? 0;
+        for (const part of parts.slice(1, -1)) {
+            const index = requested.indexOf(part, cursor);
+            if (index < 0)
+                return false;
+            cursor = index + part.length;
+        }
+        const last = parts.at(-1) ?? '';
+        return last.length === 0 || requested.slice(cursor).endsWith(last);
+    };
+    return grants.some((grant) => grant === requested || grant === name || wildcardMatch(grant));
+}
+export function resolveEvalAllowedTools(requested, grants) {
+    const selected = requested.length ? requested : DEFAULT_EVAL_ALLOWED_TOOLS;
+    for (const rule of selected)
+        if (gatedTool(rule) && !operatorGrants(rule, grants))
+            throw new Error(`Eval case requests gated tool ${rule}; grant it with --allow-tools`);
+    return [...new Set(selected)];
+}
+async function scaffold(caseDef, cwd, home, signal) {
+    if (!caseDef.context.scaffoldScript)
+        return;
+    const script = await resolveContainedPath(caseDef.dir, caseDef.context.scaffoldScript, 'Scaffold script');
+    const result = await new BoundedProcessRunner({
+        cwd,
+        maxOutputBytes: 16 * 1024,
+    }).run({
+        command: '/bin/bash',
+        args: [script],
+        timeoutMs: caseDef.execution.timeoutSeconds * 1000,
+        signal,
+        env: {
+            ...caseDef.execution.env,
+            HOME: home,
+            USERPROFILE: home,
+        },
+    });
+    if (result.timedOut)
+        throw new Error('Scaffold timed out');
+    if (result.code !== 0)
+        throw new Error(`Scaffold failed (${result.code}): ${joinedProcessOutput(result)}`);
+}
+function traceEvent(event) {
+    if (event.type === 'tool-call')
+        return {
+            type: 'tool-call',
+            tool: event.call.name,
+            input: event.call.input,
+            id: event.call.id,
+        };
+    if (event.type === 'tool-result')
+        return {
+            type: 'tool-result',
+            callId: event.callId,
+            content: event.content,
+            isError: event.isError,
+        };
+    return event;
+}
+export async function runClaudePluginEvalOnce(options) {
+    const tempRoot = await mkdtemp(join(tmpdir(), 'praxis-eval-'));
+    const cwd = join(tempRoot, 'cwd');
+    const configRoot = join(tempRoot, 'config');
+    const home = join(tempRoot, 'home');
+    const out = join(tempRoot, 'out');
+    try {
+        await Promise.all([mkdir(cwd), mkdir(configRoot), mkdir(home), mkdir(out)]);
+    }
+    catch (error) {
+        await rm(tempRoot, { recursive: true, force: true });
+        throw error;
+    }
+    const trace = [];
+    const tracePath = join(out, 'trace.jsonl');
+    const controller = new AbortController();
+    const forwardAbort = () => controller.abort(options.signal?.reason);
+    options.signal?.addEventListener('abort', forwardAbort, { once: true });
+    if (options.signal?.aborted)
+        forwardAbort();
+    const timer = setTimeout(() => controller.abort(abortError()), options.case.execution.timeoutSeconds * 1000);
+    let runtime;
+    let text = '';
+    let turns = 0;
+    let costUsd = 0;
+    let costKnown = false;
+    let error = null;
+    let termination = null;
+    try {
+        if (options.scaffold)
+            await scaffold(options.case, cwd, home, controller.signal);
+        const addDirs = await Promise.all(options.case.context.addDirs.map((path) => resolveContainedPath(options.case.dir, path, 'Add directory')));
+        const historyFile = options.case.context.historyFile
+            ? await resolveContainedPath(options.case.dir, options.case.context.historyFile, 'History file')
+            : undefined;
+        runtime = await options.factory.create({
+            cwd,
+            configRoot,
+            home,
+            ...((options.model ?? options.case.execution.model)
+                ? { model: options.model ?? options.case.execution.model }
+                : {}),
+            maxTurns: options.case.execution.maxTurns,
+            pluginDirectories: options.pluginDirectories,
+            allowedTools: resolveEvalAllowedTools(options.case.execution.allowedTools, options.allowTools ?? []),
+            ...(options.case.execution.appendSystemPrompt
+                ? { appendSystemPrompt: options.case.execution.appendSystemPrompt }
+                : {}),
+            ...(historyFile ? { historyFile } : {}),
+            addDirs,
+            env: options.case.execution.env,
+            eventSink: (event) => trace.push(traceEvent(event)),
+        });
+        const result = await runtime.run(options.case.execution.prompt ?? '', controller.signal);
+        text = result.text;
+        turns = result.turns;
+        costKnown = result.costUsd !== undefined;
+        costUsd = result.costUsd ?? 0;
+    }
+    catch (caught) {
+        if (options.signal?.aborted) {
+            termination = 'interrupted';
+            error = 'Eval run interrupted';
+        }
+        else if (controller.signal.aborted) {
+            termination = 'timeout';
+            error = 'Eval run timed out';
+        }
+        else
+            error = caught instanceof Error ? caught.message : String(caught);
+    }
+    finally {
+        clearTimeout(timer);
+        options.signal?.removeEventListener('abort', forwardAbort);
+        try {
+            await runtime?.close?.();
+        }
+        catch (caught) {
+            error ??= `Runtime cleanup failed: ${caught instanceof Error ? caught.message : String(caught)}`;
+        }
+        try {
+            await writeFile(tracePath, trace.map((item) => JSON.stringify(item)).join('\n'));
+        }
+        catch (caught) {
+            error ??= `Trace write failed: ${caught instanceof Error ? caught.message : String(caught)}`;
+        }
+    }
+    return {
+        text,
+        turns,
+        costUsd,
+        costKnown,
+        artifacts: { lastMessage: text, trace, cwd },
+        tracePath,
+        error,
+        termination,
+        tempRoot,
+    };
+}
+//# sourceMappingURL=claude-plugin-eval-runner.js.map

package/dist/plugins/claude-plugin-eval-runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"claude-plugin-eval-runner.js","sourceRoot":"","sources":["../../src/plugins/claude-plugin-eval-runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAChE,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAA;AAChC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAGhC,OAAO,EACL,oBAAoB,EACpB,mBAAmB,GACpB,MAAM,uCAAuC,CAAA;AAE9C,OAAO,EAAE,oBAAoB,EAAE,MAAM,gCAAgC,CAAA;AAMrE,MAAM,CAAC,MAAM,0BAA0B,GAAG;IACxC,MAAM;IACN,MAAM;IACN,MAAM;IACN,OAAO;CACC,CAAA;AAsCV,SAAS,UAAU;IACjB,OAAO,IAAI,YAAY,CAAC,oBAAoB,EAAE,YAAY,CAAC,CAAA;AAC7D,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY;IAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;IACnC,OAAO,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAA;AACxD,CAAC;AAED,SAAS,SAAS,CAAC,IAAY;IAC7B,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAA;IAC3B,OAAO,CACL,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC,QAAQ,CACzE,IAAI,CACL,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAC9B,CAAA;AACH,CAAC;AAED,SAAS,cAAc,CAAC,SAAiB,EAAE,MAAyB;IAClE,MAAM,IAAI,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAA;IAChC,MAAM,aAAa,GAAG,CAAC,OAAe,EAAW,EAAE;QACjD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;QAChC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,KAAK,CAAA;QACpC,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAAE,OAAO,KAAK,CAAA;QACvD,IAAI,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAA;QAClC,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACtC,MAAM,KAAK,GAAG,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;YAC7C,IAAI,KAAK,GAAG,CAAC;gBAAE,OAAO,KAAK,CAAA;YAC3B,MAAM,GAAG,KAAK,GAAG,IAAI,CAAC,MAAM,CAAA;QAC9B,CAAC;QACD,MAAM,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;QAC/B,OAAO,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;IACpE,CAAC,CAAA;IACD,OAAO,MAAM,CAAC,IAAI,CAChB,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,IAAI,aAAa,CAAC,KAAK,CAAC,CACzE,CAAA;AACH,CAAC;AAED,MAAM,UAAU,uBAAuB,CACrC,SAA4B,EAC5B,MAAyB;IAEzB,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,0BAA0B,CAAA;IAC1E,KAAK,MAAM,IAAI,IAAI,QAAQ;QACzB,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,MAAM,CAAC;YAClD,MAAM,IAAI,KAAK,CACb,iCAAiC,IAAI,+BAA+B,CACrE,CAAA;IACL,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAA;AAC/B,CAAC;AAED,KAAK,UAAU,QAAQ,CACrB,OAA6B,EAC7B,GAAW,EACX,IAAY,EACZ,MAAmB;IAEnB,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,cAAc;QAAE,OAAM;IAC3C,MAAM,MAAM,GAAG,MAAM,oBAAoB,CACvC,OAAO,CAAC,GAAG,EACX,OAAO,CAAC,OAAO,CAAC,cAAc,EAC9B,iBAAiB,CAClB,CAAA;IACD,MAAM,MAAM,GAAG,MAAM,IAAI,oBAAoB,CAAC;QAC5C,GAAG;QACH,cAAc,EAAE,EAAE,GAAG,IAAI;KAC1B,CAAC,CAAC,GAAG,CAAC;QACL,OAAO,EAAE,WAAW;QACpB,IAAI,EAAE,CAAC,MAAM,CAAC;QACd,SAAS,EAAE,OAAO,CAAC,SAAS,CAAC,cAAc,GAAG,IAAI;QAClD,MAAM;QACN,GAAG,EAAE;YACH,GAAG,OAAO,CAAC,SAAS,CAAC,GAAG;YACxB,IAAI,EAAE,IAAI;YACV,WAAW,EAAE,IAAI;SAClB;KACF,CAAC,CAAA;IACF,IAAI,MAAM,CAAC,QAAQ;QAAE,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAA;IAC1D,IAAI,MAAM,CAAC,IAAI,KAAK,CAAC;QACnB,MAAM,IAAI,KAAK,CACb,oBAAoB,MAAM,CAAC,IAAI,MAAM,mBAAmB,CAAC,MAAM,CAAC,EAAE,CACnE,CAAA;AACL,CAAC;AAED,SAAS,UAAU,CAAC,KAAmB;IACrC,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW;QAC5B,OAAO;YACL,IAAI,EAAE,WAAW;YACjB,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI;YACrB,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK;YACvB,EAAE,EAAE,KAAK,CAAC,IAAI,CAAC,EAAE;SAClB,CAAA;IACH,IAAI,KAAK,CAAC,IAAI,KAAK,aAAa;QAC9B,OAAO;YACL,IAAI,EAAE,aAAa;YACnB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,OAAO,EAAE,KAAK,CAAC,OAAO;SACvB,CAAA;IACH,OAAO,KAAkC,CAAA;AAC3C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,OAQ7C;IACC,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,cAAc,CAAC,CAAC,CAAA;IAC9D,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAA;IACjC,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAA;IAC3C,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;IACnC,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAA;IACjC,IAAI,CAAC;QACH,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,UAAU,CAAC,EAAE,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;IAC7E,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;QACpD,MAAM,KAAK,CAAA;IACb,CAAC;IACD,MAAM,KAAK,GAAqB,EAAE,CAAA;IAClC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,aAAa,CAAC,CAAA;IAC1C,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAA;IACxC,MAAM,YAAY,GAAG,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IACnE,OAAO,CAAC,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,YAAY,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAA;IACvE,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;QAAE,YAAY,EAAE,CAAA;IAC3C,MAAM,KAAK,GAAG,UAAU,CACtB,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC,EACpC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,cAAc,GAAG,IAAI,CAC7C,CAAA;IACD,IAAI,OAAsC,CAAA;IAC1C,IAAI,IAAI,GAAG,EAAE,CAAA;IACb,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,SAAS,GAAG,KAAK,CAAA;IACrB,IAAI,KAAK,GAAkB,IAAI,CAAA;IAC/B,IAAI,WAAW,GAAuC,IAAI,CAAA;IAC1D,IAAI,CAAC;QACH,IAAI,OAAO,CAAC,QAAQ;YAClB,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC,MAAM,CAAC,CAAA;QAC5D,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC/B,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CACxC,oBAAoB,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,eAAe,CAAC,CAC9D,CACF,CAAA;QACD,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW;YAClD,CAAC,CAAC,MAAM,oBAAoB,CACxB,OAAO,CAAC,IAAI,CAAC,GAAG,EAChB,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,EAChC,cAAc,CACf;YACH,CAAC,CAAC,SAAS,CAAA;QACb,OAAO,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC;YACrC,GAAG;YACH,UAAU;YACV,IAAI;YACJ,GAAG,CAAC,CAAC,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC;gBACjD,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE;gBAC1D,CAAC,CAAC,EAAE,CAAC;YACP,QAAQ,EAAE,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ;YACzC,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;YAC5C,YAAY,EAAE,uBAAuB,CACnC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,YAAY,EACnC,OAAO,CAAC,UAAU,IAAI,EAAE,CACzB;YACD,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,kBAAkB;gBAC3C,CAAC,CAAC,EAAE,kBAAkB,EAAE,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,kBAAkB,EAAE;gBACnE,CAAC,CAAC,EAAE,CAAC;YACP,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACvC,OAAO;YACP,GAAG,EAAE,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG;YAC/B,SAAS,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;SACpD,CAAC,CAAA;QACF,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9B,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,IAAI,EAAE,EACnC,UAAU,CAAC,MAAM,CAClB,CAAA;QACD,IAAI,GAAG,MAAM,CAAC,IAAI,CAAA;QAClB,KAAK,GAAG,MAAM,CAAC,KAAK,CAAA;QACpB,SAAS,GAAG,MAAM,CAAC,OAAO,KAAK,SAAS,CAAA;QACxC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,CAAC,CAAA;IAC/B,CAAC;IAAC,OAAO,MAAM,EAAE,CAAC;QAChB,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;YAC5B,WAAW,GAAG,aAAa,CAAA;YAC3B,KAAK,GAAG,sBAAsB,CAAA;QAChC,CAAC;aAAM,IAAI,UAAU,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACrC,WAAW,GAAG,SAAS,CAAA;YACvB,KAAK,GAAG,oBAAoB,CAAA;QAC9B,CAAC;;YAAM,KAAK,GAAG,MAAM,YAAY,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;IAC1E,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAA;QACnB,OAAO,CAAC,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,YAAY,CAAC,CAAA;QAC1D,IAAI,CAAC;YACH,MAAM,OAAO,EAAE,KAAK,EAAE,EAAE,CAAA;QAC1B,CAAC;QAAC,OAAO,MAAM,EAAE,CAAC;YAChB,KAAK,KAAK,2BACR,MAAM,YAAY,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAC1D,EAAE,CAAA;QACJ,CAAC;QACD,IAAI,CAAC;YACH,MAAM,SAAS,CACb,SAAS,EACT,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CACrD,CAAA;QACH,CAAC;QAAC,OAAO,MAAM,EAAE,CAAC;YAChB,KAAK,KAAK,uBACR,MAAM,YAAY,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAC1D,EAAE,CAAA;QACJ,CAAC;IACH,CAAC;IACD,OAAO;QACL,IAAI;QACJ,KAAK;QACL,OAAO;QACP,SAAS;QACT,SAAS,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE;QAC5C,SAAS;QACT,KAAK;QACL,WAAW;QACX,QAAQ;KACT,CAAA;AACH,CAAC"}

package/dist/plugins/claude-plugin-eval-schema.d.ts

@@ -0,0 +1,80 @@
+export declare const EVAL_SCHEMA_VERSION = "1.0";
+export declare const MAX_EVAL_FILE_BYTES: number;
+export declare const MAX_EVAL_GRADERS = 256;
+export declare const MAX_EVAL_ARRAY_ITEMS = 256;
+export declare const MAX_EVAL_STRING_LENGTH: number;
+export declare const MAX_EVAL_OBJECT_DEPTH = 16;
+export declare const MAX_EVAL_OBJECT_NODES = 4096;
+export type EvalFocus = 'last_message' | 'trace' | 'files' | {
+    source: 'file';
+    path: string;
+};
+export type EvalArm = 'with-only' | 'both';
+interface EvalGraderBase {
+    name: string;
+    weight: number;
+    arm?: EvalArm;
+}
+export type ClaudePluginEvalGrader = (EvalGraderBase & {
+    type: 'regex';
+    target: EvalFocus;
+    pattern: string;
+    flags: string;
+    match: 'contains' | 'not_contains' | `count:${number}`;
+}) | (EvalGraderBase & {
+    type: 'tool_order';
+    before: EvalToolMatch;
+    after: EvalToolMatch;
+}) | (EvalGraderBase & {
+    type: 'tool_used';
+    tool: string;
+    input_match?: Record<string, unknown>;
+    min: number;
+    max: number;
+}) | (EvalGraderBase & {
+    type: 'file_exists';
+    path: string;
+    exists: boolean;
+}) | (EvalGraderBase & {
+    type: 'llm';
+    criteria: string;
+    focus: EvalFocus;
+}) | (EvalGraderBase & {
+    type: 'baseline';
+    baseline_file: string;
+    criteria: string;
+});
+export type EvalToolMatch = string | {
+    tool: string;
+    input_match?: Record<string, unknown>;
+};
+export interface ClaudePluginEvalCase {
+    schemaVersion: string;
+    name: string;
+    description?: string;
+    tags: readonly string[];
+    plugins?: readonly string[];
+    context: {
+        scaffoldScript?: string;
+        historyFile?: string;
+        addDirs: readonly string[];
+    };
+    execution: {
+        prompt?: string;
+        maxTurns: number;
+        timeoutSeconds: number;
+        model?: string;
+        allowedTools: readonly string[];
+        appendSystemPrompt?: string;
+        env: Readonly<Record<string, string>>;
+    };
+    runs: number;
+    graders: readonly ClaudePluginEvalGrader[];
+    expectedOutcome?: string;
+    dir: string;
+    source: 'case_yaml' | 'prose' | 'mixed';
+}
+export declare function loadClaudePluginEvalCase(caseDir: string): Promise<ClaudePluginEvalCase>;
+export declare function resolveContainedPath(root: string, candidate: string, label: string): Promise<string>;
+export {};
+//# sourceMappingURL=claude-plugin-eval-schema.d.ts.map

package/dist/plugins/claude-plugin-eval-schema.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"claude-plugin-eval-schema.d.ts","sourceRoot":"","sources":["../../src/plugins/claude-plugin-eval-schema.ts"],"names":[],"mappings":"AAKA,eAAO,MAAM,mBAAmB,QAAQ,CAAA;AACxC,eAAO,MAAM,mBAAmB,QAAc,CAAA;AAC9C,eAAO,MAAM,gBAAgB,MAAM,CAAA;AACnC,eAAO,MAAM,oBAAoB,MAAM,CAAA;AACvC,eAAO,MAAM,sBAAsB,QAAY,CAAA;AAC/C,eAAO,MAAM,qBAAqB,KAAK,CAAA;AACvC,eAAO,MAAM,qBAAqB,OAAO,CAAA;AAIzC,MAAM,MAAM,SAAS,GACnB,cAAc,GAAG,OAAO,GAAG,OAAO,GAAG;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAA;AACvE,MAAM,MAAM,OAAO,GAAG,WAAW,GAAG,MAAM,CAAA;AAE1C,UAAU,cAAc;IACtB,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,MAAM,CAAA;IACd,GAAG,CAAC,EAAE,OAAO,CAAA;CACd;AAED,MAAM,MAAM,sBAAsB,GAC9B,CAAC,cAAc,GAAG;IAChB,IAAI,EAAE,OAAO,CAAA;IACb,MAAM,EAAE,SAAS,CAAA;IACjB,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;IACb,KAAK,EAAE,UAAU,GAAG,cAAc,GAAG,SAAS,MAAM,EAAE,CAAA;CACvD,CAAC,GACF,CAAC,cAAc,GAAG;IAChB,IAAI,EAAE,YAAY,CAAA;IAClB,MAAM,EAAE,aAAa,CAAA;IACrB,KAAK,EAAE,aAAa,CAAA;CACrB,CAAC,GACF,CAAC,cAAc,GAAG;IAChB,IAAI,EAAE,WAAW,CAAA;IACjB,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IACrC,GAAG,EAAE,MAAM,CAAA;IACX,GAAG,EAAE,MAAM,CAAA;CACZ,CAAC,GACF,CAAC,cAAc,GAAG;IAAE,IAAI,EAAE,aAAa,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,OAAO,CAAA;CAAE,CAAC,GACzE,CAAC,cAAc,GAAG;IAAE,IAAI,EAAE,KAAK,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,SAAS,CAAA;CAAE,CAAC,GACtE,CAAC,cAAc,GAAG;IAChB,IAAI,EAAE,UAAU,CAAA;IAChB,aAAa,EAAE,MAAM,CAAA;IACrB,QAAQ,EAAE,MAAM,CAAA;CACjB,CAAC,CAAA;AAEN,MAAM,MAAM,aAAa,GACrB,MAAM,GACN;IACE,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACtC,CAAA;AAEL,MAAM,WAAW,oBAAoB;IACnC,aAAa,EAAE,MAAM,CAAA;IACrB,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,IAAI,EAAE,SAAS,MAAM,EAAE,CAAA;IACvB,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IAC3B,OAAO,EAAE;QACP,cAAc,CAAC,EAAE,MAAM,CAAA;QACvB,WAAW,CAAC,EAAE,MAAM,CAAA;QACpB,OAAO,EAAE,SAAS,MAAM,EAAE,CAAA;KAC3B,CAAA;IACD,SAAS,EAAE;QACT,MAAM,CAAC,EAAE,MAAM,CAAA;QACf,QAAQ,EAAE,MAAM,CAAA;QAChB,cAAc,EAAE,MAAM,CAAA;QACtB,KAAK,CAAC,EAAE,MAAM,CAAA;QACd,YAAY,EAAE,SAAS,MAAM,EAAE,CAAA;QAC/B,kBAAkB,CAAC,EAAE,MAAM,CAAA;QAC3B,GAAG,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;KACtC,CAAA;IACD,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,SAAS,sBAAsB,EAAE,CAAA;IAC1C,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,GAAG,EAAE,MAAM,CAAA;IACX,MAAM,EAAE,WAAW,GAAG,OAAO,GAAG,OAAO,CAAA;CACxC;AAiaD,wBAAsB,wBAAwB,CAC5C,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,oBAAoB,CAAC,CAiF/B;AAED,wBAAsB,oBAAoB,CACxC,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,MAAM,CAAC,CAMjB"}