vskill 0.5.11 → 0.5.13

package/dist/eval/integration-runner.d.ts

@@ -0,0 +1,12 @@
+import type { IntegrationRunResult, IntegrationEvalCase, IntegrationRunOpts } from "./integration-types.js";
+export declare function checkPlaywright(): void;
+export declare function runIntegrationCase(evalCase: IntegrationEvalCase, opts: IntegrationRunOpts): Promise<IntegrationRunResult>;
+export declare function promptConfirmation(platform: string, actions: string[]): Promise<boolean>;
+/**
+ * Check if this is the first run against a platform (no history file).
+ */
+export declare function isFirstRun(skillDir: string): boolean;
+/**
+ * Record a run in the integration history file.
+ */
+export declare function recordRun(skillDir: string, result: IntegrationRunResult): void;

package/dist/eval/integration-runner.js

@@ -0,0 +1,303 @@
+// ---------------------------------------------------------------------------
+// integration-runner.ts -- 5-phase browser-based integration test runner
+//
+// Phases: Preflight -> Connect -> Execute -> Verify -> Cleanup
+// ---------------------------------------------------------------------------
+import { randomUUID } from "node:crypto";
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { createRequire } from "node:module";
+import { resolveAllCredentials } from "./credential-resolver.js";
+import { resolveProfile } from "./chrome-profile.js";
+import { PlatformRateLimiter } from "./rate-limiter.js";
+import { judgeAssertion } from "./judge.js";
+import { createLlmClient } from "./llm.js";
+// ---------------------------------------------------------------------------
+// SIGINT cleanup state
+// ---------------------------------------------------------------------------
+let cleanupRegistered = false;
+let cleanupFn = null;
+let cleanupDone = false;
+function registerSigintHandler(fn) {
+    cleanupFn = fn;
+    cleanupDone = false;
+    if (!cleanupRegistered) {
+        process.on("SIGINT", sigintHandler);
+        cleanupRegistered = true;
+    }
+}
+function deregisterSigintHandler() {
+    process.removeListener("SIGINT", sigintHandler);
+    cleanupRegistered = false;
+    cleanupFn = null;
+}
+async function sigintHandler() {
+    if (cleanupDone)
+        return;
+    cleanupDone = true;
+    console.log("\nSIGINT received — running cleanup...");
+    if (cleanupFn) {
+        try {
+            await cleanupFn();
+            console.log("Cleanup complete, exiting.");
+        }
+        catch (err) {
+            console.error("Cleanup failed:", err.message);
+        }
+    }
+    process.exit(0);
+}
+// ---------------------------------------------------------------------------
+// Playwright lazy check
+// ---------------------------------------------------------------------------
+export function checkPlaywright() {
+    try {
+        const require = createRequire(import.meta.url);
+        require.resolve("playwright");
+    }
+    catch {
+        throw new Error("Playwright is required for integration tests. Install it with:\n" +
+            "  npm install --save-dev playwright && npx playwright install chromium");
+    }
+}
+// ---------------------------------------------------------------------------
+// Main runner
+// ---------------------------------------------------------------------------
+export async function runIntegrationCase(evalCase, opts) {
+    const runId = opts.runId ?? randomUUID().slice(0, 8).toUpperCase();
+    const testPrefix = `[VSKILL-TEST-${runId}]`;
+    const phases = [];
+    const testArtifactIds = [runId];
+    let browser = null;
+    let context = null;
+    // Register cleanup for SIGINT
+    registerSigintHandler(async () => {
+        await runCleanup(evalCase, browser, testArtifactIds);
+    });
+    try {
+        // -----------------------------------------------------------------------
+        // Phase 1: PREFLIGHT
+        // -----------------------------------------------------------------------
+        const preflightResult = await runPhase("preflight", async () => {
+            // Check credentials
+            if (evalCase.requiredCredentials?.length) {
+                const statuses = resolveAllCredentials(evalCase.requiredCredentials, opts.skillDir);
+                const missing = statuses.filter((s) => s.status === "missing");
+                if (missing.length > 0) {
+                    throw new Error(`Missing credentials: ${missing.map((m) => m.name).join(", ")}. ` +
+                        `Set them with: vskill credentials set <KEY>`);
+                }
+            }
+            // Check Chrome profile
+            const profileName = evalCase.requirements?.chromeProfile;
+            const profilePath = evalCase.requirements?.chromeProfilePath;
+            if (profileName && !profilePath) {
+                resolveProfile(profileName);
+            }
+            // Check Playwright
+            if (!opts.dryRun) {
+                checkPlaywright();
+            }
+        });
+        phases.push(preflightResult);
+        if (preflightResult.status === "fail") {
+            // Abort remaining phases
+            for (const p of ["connect", "execute", "verify", "cleanup"]) {
+                phases.push({ phase: p, status: "skipped" });
+            }
+            return buildResult(evalCase, runId, phases, testArtifactIds, !!opts.dryRun);
+        }
+        // -----------------------------------------------------------------------
+        // Phase 2: CONNECT
+        // -----------------------------------------------------------------------
+        const connectResult = await runPhase("connect", async () => {
+            if (opts.dryRun) {
+                console.log(`[DRY RUN] Would launch browser with profile: ${evalCase.requirements?.chromeProfile ?? "default"}`);
+                return;
+            }
+            const profileName = evalCase.requirements?.chromeProfile;
+            const profilePath = evalCase.requirements?.chromeProfilePath ?? (profileName ? resolveProfile(profileName) : undefined);
+            const pw = await import("playwright");
+            if (profilePath) {
+                context = await pw.chromium.launchPersistentContext(profilePath, {
+                    headless: false,
+                    args: ["--disable-blink-features=AutomationControlled"],
+                });
+                browser = null; // persistent context manages its own browser
+            }
+            else {
+                browser = await pw.chromium.launch({ headless: false });
+                context = await browser.newContext();
+            }
+        });
+        phases.push(connectResult);
+        if (connectResult.status === "fail") {
+            phases.push({ phase: "execute", status: "skipped" });
+            phases.push({ phase: "verify", status: "skipped" });
+            phases.push(await runPhase("cleanup", () => runCleanup(evalCase, browser ?? context, testArtifactIds)));
+            return buildResult(evalCase, runId, phases, testArtifactIds, !!opts.dryRun);
+        }
+        // -----------------------------------------------------------------------
+        // Phase 3: EXECUTE
+        // -----------------------------------------------------------------------
+        let generatedOutput = "";
+        const executeResult = await runPhase("execute", async () => {
+            const platform = evalCase.requirements?.platform;
+            // Rate limiting
+            if (platform && !opts.dryRun) {
+                const rateLimiter = new PlatformRateLimiter(evalCase.requirements?.rateLimit
+                    ? { [platform]: evalCase.requirements.rateLimit }
+                    : undefined);
+                await rateLimiter.acquire(platform);
+            }
+            // Build prompt with test prefix
+            const promptWithPrefix = `${testPrefix}\n\nIMPORTANT: All content you create or post MUST include the prefix "${testPrefix}" for identification and cleanup.\n\n${evalCase.prompt}`;
+            if (opts.dryRun) {
+                console.log(`[DRY RUN] Would execute LLM with prompt:\n${promptWithPrefix.slice(0, 200)}...`);
+                generatedOutput = `[DRY RUN] Simulated output for: ${evalCase.name}`;
+                return;
+            }
+            // Create LLM client and generate
+            const client = createLlmClient();
+            const skillMdPath = join(opts.skillDir, "SKILL.md");
+            let systemPrompt = "You are executing an integration test. Follow the instructions precisely.";
+            if (existsSync(skillMdPath)) {
+                systemPrompt = readFileSync(skillMdPath, "utf-8");
+            }
+            const result = await client.generate(systemPrompt, promptWithPrefix);
+            generatedOutput = result.text;
+        });
+        phases.push(executeResult);
+        if (executeResult.status === "fail") {
+            phases.push({ phase: "verify", status: "skipped" });
+            phases.push(await runPhase("cleanup", () => runCleanup(evalCase, browser ?? context, testArtifactIds)));
+            return buildResult(evalCase, runId, phases, testArtifactIds, !!opts.dryRun);
+        }
+        // -----------------------------------------------------------------------
+        // Phase 4: VERIFY
+        // -----------------------------------------------------------------------
+        const verifyResult = await runPhase("verify", async () => {
+            if (!evalCase.assertions?.length)
+                return;
+            const client = createLlmClient();
+            const results = await Promise.all(evalCase.assertions.map((assertion) => judgeAssertion(generatedOutput, assertion, client)));
+            const failed = results.filter((r) => !r.pass);
+            if (failed.length > 0) {
+                throw new Error(`${failed.length} assertion(s) failed:\n` +
+                    failed.map((f) => `  - ${f.text}: ${f.reasoning}`).join("\n"));
+            }
+        });
+        phases.push(verifyResult);
+        // -----------------------------------------------------------------------
+        // Phase 5: CLEANUP
+        // -----------------------------------------------------------------------
+        const cleanupResult = await runPhase("cleanup", () => runCleanup(evalCase, browser ?? context, testArtifactIds));
+        phases.push(cleanupResult);
+        return buildResult(evalCase, runId, phases, testArtifactIds, !!opts.dryRun);
+    }
+    finally {
+        deregisterSigintHandler();
+    }
+}
+// ---------------------------------------------------------------------------
+// Phase executor
+// ---------------------------------------------------------------------------
+async function runPhase(phase, fn) {
+    const start = Date.now();
+    try {
+        await fn();
+        return { phase, status: "pass", durationMs: Date.now() - start };
+    }
+    catch (err) {
+        return {
+            phase,
+            status: "fail",
+            durationMs: Date.now() - start,
+            errorMessage: err.message,
+        };
+    }
+}
+// ---------------------------------------------------------------------------
+// Cleanup
+// ---------------------------------------------------------------------------
+async function runCleanup(evalCase, browserOrContext, _testArtifactIds) {
+    // Run cleanup actions defined in the eval case
+    if (evalCase.cleanup?.length) {
+        for (const action of evalCase.cleanup) {
+            try {
+                if (action.execute) {
+                    await action.execute();
+                }
+            }
+            catch (err) {
+                // Log but do not throw — test result stands independently
+                console.error(`Cleanup action "${action.description}" failed:`, err.message);
+            }
+        }
+    }
+    // Close browser
+    if (browserOrContext) {
+        try {
+            await browserOrContext.close();
+        }
+        catch {
+            // Browser may already be closed
+        }
+    }
+}
+// ---------------------------------------------------------------------------
+// Confirmation prompt
+// ---------------------------------------------------------------------------
+export async function promptConfirmation(platform, actions) {
+    // Skip in CI
+    if (process.env.CI === "true")
+        return true;
+    const { createInterface } = await import("node:readline");
+    const rl = createInterface({ input: process.stdin, output: process.stdout });
+    const actionList = actions.map((a) => `  - ${a}`).join("\n");
+    const question = `\nThis will perform the following actions on ${platform}:\n${actionList}\n\nProceed? (y/N) `;
+    return new Promise((resolve) => {
+        rl.question(question, (answer) => {
+            rl.close();
+            resolve(answer.toLowerCase() === "y" || answer.toLowerCase() === "yes");
+        });
+    });
+}
+/**
+ * Check if this is the first run against a platform (no history file).
+ */
+export function isFirstRun(skillDir) {
+    return !existsSync(join(skillDir, "evals", ".integration-history.json"));
+}
+/**
+ * Record a run in the integration history file.
+ */
+export function recordRun(skillDir, result) {
+    const historyPath = join(skillDir, "evals", ".integration-history.json");
+    let history = [];
+    if (existsSync(historyPath)) {
+        try {
+            history = JSON.parse(readFileSync(historyPath, "utf-8"));
+        }
+        catch {
+            history = [];
+        }
+    }
+    history.push(result);
+    writeFileSync(historyPath, JSON.stringify(history, null, 2), "utf-8");
+}
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+function buildResult(evalCase, runId, phases, testArtifactIds, dryRun) {
+    const overallPass = phases.every((p) => p.status === "pass" || p.status === "skipped");
+    return {
+        evalId: String(evalCase.id),
+        runId,
+        phases,
+        overallPass,
+        testArtifactIds,
+        dryRun,
+    };
+}
+//# sourceMappingURL=integration-runner.js.map

package/dist/eval/integration-runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"integration-runner.js","sourceRoot":"","sources":["../../src/eval/integration-runner.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,yEAAyE;AACzE,EAAE;AACF,+DAA+D;AAC/D,8EAA8E;AAE9E,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAClE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAqB,qBAAqB,EAAE,MAAM,0BAA0B,CAAC;AACpF,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAE5C,OAAO,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAU3C,8EAA8E;AAC9E,uBAAuB;AACvB,8EAA8E;AAC9E,IAAI,iBAAiB,GAAG,KAAK,CAAC;AAC9B,IAAI,SAAS,GAAiC,IAAI,CAAC;AACnD,IAAI,WAAW,GAAG,KAAK,CAAC;AAExB,SAAS,qBAAqB,CAAC,EAAuB;IACpD,SAAS,GAAG,EAAE,CAAC;IACf,WAAW,GAAG,KAAK,CAAC;IACpB,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACvB,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;QACpC,iBAAiB,GAAG,IAAI,CAAC;IAC3B,CAAC;AACH,CAAC;AAED,SAAS,uBAAuB;IAC9B,OAAO,CAAC,cAAc,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;IAChD,iBAAiB,GAAG,KAAK,CAAC;IAC1B,SAAS,GAAG,IAAI,CAAC;AACnB,CAAC;AAED,KAAK,UAAU,aAAa;IAC1B,IAAI,WAAW;QAAE,OAAO;IACxB,WAAW,GAAG,IAAI,CAAC;IACnB,OAAO,CAAC,GAAG,CAAC,wCAAwC,CAAC,CAAC;IACtD,IAAI,SAAS,EAAE,CAAC;QACd,IAAI,CAAC;YACH,MAAM,SAAS,EAAE,CAAC;YAClB,OAAO,CAAC,GAAG,CAAC,4BAA4B,CAAC,CAAC;QAC5C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,iBAAiB,EAAG,GAAa,CAAC,OAAO,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;IACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,8EAA8E;AAC9E,wBAAwB;AACxB,8EAA8E;AAE9E,MAAM,UAAU,eAAe;IAC7B,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC/C,OAAO,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;IAChC,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CACb,kEAAkE;YAClE,wEAAwE,CACzE,CAAC;IACJ,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,cAAc;AACd,8EAA8E;AAE9E,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,QAA6B,EAC7B,IAAwB;IAExB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;IACnE,MAAM,UAAU,GAAG,gBAAgB,KAAK,GAAG,CAAC;IAC5C,MAAM,MAAM,GAAkB,EAAE,CAAC;IACjC,MAAM,eAAe,GAAa,CAAC,KAAK,CAAC,CAAC;IAC1C,IAAI,OAAO,GAAQ,IAAI,CAAC;IACxB,IAAI,OAAO,GAAQ,IAAI,CAAC;IAExB,8BAA8B;IAC9B,qBAAqB,CAAC,KAAK,IAAI,EAAE;QAC/B,MAAM,UAAU,CAAC,QAAQ,EAAE,OAAO,EAAE,eAAe,CAAC,CAAC;IACvD,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,0EAA0E;QAC1E,qBAAqB;QACrB,0EAA0E;QAC1E,MAAM,eAAe,GAAG,MAAM,QAAQ,CAAC,WAAW,EAAE,KAAK,IAAI,EAAE;YAC7D,oBAAoB;YACpB,IAAI,QAAQ,CAAC,mBAAmB,EAAE,MAAM,EAAE,CAAC;gBACzC,MAAM,QAAQ,GAAG,qBAAqB,CAAC,QAAQ,CAAC,mBAAmB,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACpF,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC;gBAC/D,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACvB,MAAM,IAAI,KAAK,CACb,wBAAwB,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI;wBACjE,6CAA6C,CAC9C,CAAC;gBACJ,CAAC;YACH,CAAC;YAED,uBAAuB;YACvB,MAAM,WAAW,GAAG,QAAQ,CAAC,YAAY,EAAE,aAAa,CAAC;YACzD,MAAM,WAAW,GAAG,QAAQ,CAAC,YAAY,EAAE,iBAAiB,CAAC;YAC7D,IAAI,WAAW,IAAI,CAAC,WAAW,EAAE,CAAC;gBAChC,cAAc,CAAC,WAAW,CAAC,CAAC;YAC9B,CAAC;YAED,mBAAmB;YACnB,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACjB,eAAe,EAAE,CAAC;YACpB,CAAC;QACH,CAAC,CAAC,CAAC;QACH,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAE7B,IAAI,eAAe,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;YACtC,yBAAyB;YACzB,KAAK,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,CAAuB,EAAE,CAAC;gBAClF,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;YAC/C,CAAC;YACD,OAAO,WAAW,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC9E,CAAC;QAED,0EAA0E;QAC1E,mBAAmB;QACnB,0EAA0E;QAC1E,MAAM,aAAa,GAAG,MAAM,QAAQ,CAAC,SAAS,EAAE,KAAK,IAAI,EAAE;YACzD,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBAChB,OAAO,CAAC,GAAG,CAAC,gDAAgD,QAAQ,CAAC,YAAY,EAAE,aAAa,IAAI,SAAS,EAAE,CAAC,CAAC;gBACjH,OAAO;YACT,CAAC;YAED,MAAM,WAAW,GAAG,QAAQ,CAAC,YAAY,EAAE,aAAa,CAAC;YACzD,MAAM,WAAW,GAAG,QAAQ,CAAC,YAAY,EAAE,iBAAiB,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;YAExH,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC;YACtC,IAAI,WAAW,EAAE,CAAC;gBAChB,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,uBAAuB,CAAC,WAAW,EAAE;oBAC/D,QAAQ,EAAE,KAAK;oBACf,IAAI,EAAE,CAAC,+CAA+C,CAAC;iBACxD,CAAC,CAAC;gBACH,OAAO,GAAG,IAAI,CAAC,CAAC,6CAA6C;YAC/D,CAAC;iBAAM,CAAC;gBACN,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;gBACxD,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;YACvC,CAAC;QACH,CAAC,CAAC,CAAC;QACH,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAE3B,IAAI,aAAa,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;YACpC,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;YACrD,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;YACpD,MAAM,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,UAAU,CAAC,QAAQ,EAAE,OAAO,IAAI,OAAO,EAAE,eAAe,CAAC,CAAC,CAAC,CAAC;YACxG,OAAO,WAAW,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC9E,CAAC;QAED,0EAA0E;QAC1E,mBAAmB;QACnB,0EAA0E;QAC1E,IAAI,eAAe,GAAG,EAAE,CAAC;QACzB,MAAM,aAAa,GAAG,MAAM,QAAQ,CAAC,SAAS,EAAE,KAAK,IAAI,EAAE;YACzD,MAAM,QAAQ,GAAG,QAAQ,CAAC,YAAY,EAAE,QAAQ,CAAC;YAEjD,gBAAgB;YAChB,IAAI,QAAQ,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;gBAC7B,MAAM,WAAW,GAAG,IAAI,mBAAmB,CACzC,QAAQ,CAAC,YAAY,EAAE,SAAS;oBAC9B,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,EAAE,QAAQ,CAAC,YAAY,CAAC,SAAS,EAAE;oBACjD,CAAC,CAAC,SAAS,CACd,CAAC;gBACF,MAAM,WAAW,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YACtC,CAAC;YAED,gCAAgC;YAChC,MAAM,gBAAgB,GAAG,GAAG,UAAU,0EAA0E,UAAU,wCAAwC,QAAQ,CAAC,MAAM,EAAE,CAAC;YAEpL,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBAChB,OAAO,CAAC,GAAG,CAAC,6CAA6C,gBAAgB,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;gBAC9F,eAAe,GAAG,mCAAmC,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACrE,OAAO;YACT,CAAC;YAED,iCAAiC;YACjC,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;YACpD,IAAI,YAAY,GAAG,2EAA2E,CAAC;YAC/F,IAAI,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC5B,YAAY,GAAG,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;YACpD,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,YAAY,EAAE,gBAAgB,CAAC,CAAC;YACrE,eAAe,GAAG,MAAM,CAAC,IAAI,CAAC;QAChC,CAAC,CAAC,CAAC;QACH,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAE3B,IAAI,aAAa,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;YACpC,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;YACpD,MAAM,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,UAAU,CAAC,QAAQ,EAAE,OAAO,IAAI,OAAO,EAAE,eAAe,CAAC,CAAC,CAAC,CAAC;YACxG,OAAO,WAAW,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC9E,CAAC;QAED,0EAA0E;QAC1E,kBAAkB;QAClB,0EAA0E;QAC1E,MAAM,YAAY,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,KAAK,IAAI,EAAE;YACvD,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,MAAM;gBAAE,OAAO;YAEzC,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;YACjC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC/B,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CACpC,cAAc,CAAC,eAAe,EAAE,SAAsB,EAAE,MAAM,CAAC,CAChE,CACF,CAAC;YAEF,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAC9C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtB,MAAM,IAAI,KAAK,CACb,GAAG,MAAM,CAAC,MAAM,yBAAyB;oBACzC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAC9D,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;QACH,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAE1B,0EAA0E;QAC1E,mBAAmB;QACnB,0EAA0E;QAC1E,MAAM,aAAa,GAAG,MAAM,QAAQ,CAAC,SAAS,EAAE,GAAG,EAAE,CACnD,UAAU,CAAC,QAAQ,EAAE,OAAO,IAAI,OAAO,EAAE,eAAe,CAAC,CAC1D,CAAC;QACF,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAE3B,OAAO,WAAW,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC9E,CAAC;YAAS,CAAC;QACT,uBAAuB,EAAE,CAAC;IAC5B,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,iBAAiB;AACjB,8EAA8E;AAE9E,KAAK,UAAU,QAAQ,CACrB,KAAuB,EACvB,EAAuB;IAEvB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,IAAI,CAAC;QACH,MAAM,EAAE,EAAE,CAAC;QACX,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE,CAAC;IACnE,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO;YACL,KAAK;YACL,MAAM,EAAE,MAAM;YACd,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;YAC9B,YAAY,EAAG,GAAa,CAAC,OAAO;SACrC,CAAC;IACJ,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,UAAU;AACV,8EAA8E;AAE9E,KAAK,UAAU,UAAU,CACvB,QAA6B,EAC7B,gBAAqB,EACrB,gBAA0B;IAE1B,+CAA+C;IAC/C,IAAI,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;QAC7B,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;YACtC,IAAI,CAAC;gBACH,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;oBACnB,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;gBACzB,CAAC;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,0DAA0D;gBAC1D,OAAO,CAAC,KAAK,CAAC,mBAAmB,MAAM,CAAC,WAAW,WAAW,EAAG,GAAa,CAAC,OAAO,CAAC,CAAC;YAC1F,CAAC;QACH,CAAC;IACH,CAAC;IAED,gBAAgB;IAChB,IAAI,gBAAgB,EAAE,CAAC;QACrB,IAAI,CAAC;YACH,MAAM,gBAAgB,CAAC,KAAK,EAAE,CAAC;QACjC,CAAC;QAAC,MAAM,CAAC;YACP,gCAAgC;QAClC,CAAC;IACH,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,sBAAsB;AACtB,8EAA8E;AAE9E,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,QAAgB,EAChB,OAAiB;IAEjB,aAAa;IACb,IAAI,OAAO,CAAC,GAAG,CAAC,EAAE,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC;IAE3C,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,eAAe,CAAC,CAAC;IAC1D,MAAM,EAAE,GAAG,eAAe,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAE7E,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7D,MAAM,QAAQ,GAAG,gDAAgD,QAAQ,MAAM,UAAU,qBAAqB,CAAC;IAE/G,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,MAAM,EAAE,EAAE;YAC/B,EAAE,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,GAAG,IAAI,MAAM,CAAC,WAAW,EAAE,KAAK,KAAK,CAAC,CAAC;QAC1E,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,UAAU,CAAC,QAAgB;IACzC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,2BAA2B,CAAC,CAAC,CAAC;AAC3E,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,SAAS,CAAC,QAAgB,EAAE,MAA4B;IACtE,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,2BAA2B,CAAC,CAAC;IACzE,IAAI,OAAO,GAA2B,EAAE,CAAC;IACzC,IAAI,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAC5B,IAAI,CAAC;YACH,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC;QAC3D,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,GAAG,EAAE,CAAC;QACf,CAAC;IACH,CAAC;IACD,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACrB,aAAa,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;AACxE,CAAC;AAED,8EAA8E;AAC9E,UAAU;AACV,8EAA8E;AAE9E,SAAS,WAAW,CAClB,QAA6B,EAC7B,KAAa,EACb,MAAqB,EACrB,eAAyB,EACzB,MAAe;IAEf,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC;IACvF,OAAO;QACL,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC3B,KAAK;QACL,MAAM;QACN,WAAW;QACX,eAAe;QACf,MAAM;KACP,CAAC;AACJ,CAAC"}

package/dist/eval/integration-types.d.ts

@@ -0,0 +1,65 @@
+export type IntegrationPhase = "preflight" | "connect" | "execute" | "verify" | "cleanup";
+export interface PhaseResult {
+    phase: IntegrationPhase;
+    status: "pass" | "fail" | "skipped";
+    durationMs?: number;
+    errorMessage?: string;
+}
+export interface IntegrationRunResult {
+    evalId: string;
+    runId: string;
+    phases: PhaseResult[];
+    overallPass: boolean;
+    testArtifactIds: string[];
+    dryRun: boolean;
+}
+export interface PlatformRateLimit {
+    requestsPerMinute: number;
+}
+export interface IntegrationRequirements {
+    chromeProfile?: string;
+    chromeProfilePath?: string;
+    platform?: string;
+    rateLimit?: PlatformRateLimit;
+}
+export interface CleanupAction {
+    type: "delete_post" | "remove_artifact" | "custom";
+    description: string;
+    execute?: () => Promise<void>;
+}
+export interface IntegrationEvalCase {
+    id: number | string;
+    name: string;
+    prompt: string;
+    expected_output: string;
+    assertions: Array<{
+        id: string;
+        text: string;
+        type: string;
+    }>;
+    testType: "integration";
+    requiredCredentials?: string[];
+    requirements?: IntegrationRequirements;
+    cleanup?: CleanupAction[];
+}
+export interface IntegrationRunOpts {
+    dryRun?: boolean;
+    confirm?: boolean;
+    skillDir: string;
+    runId?: string;
+}
+/** Default rate limits per platform (requests per minute). */
+export declare const DEFAULT_RATE_LIMITS: Record<string, PlatformRateLimit>;
+export declare const DEFAULT_RATE_LIMIT: PlatformRateLimit;
+export declare const VALID_CLEANUP_ACTIONS: readonly ["delete_post", "remove_artifact", "custom"];
+export type CleanupActionType = typeof VALID_CLEANUP_ACTIONS[number];
+export interface EvalCleanupSchema {
+    action: CleanupActionType;
+    platform?: string;
+    identifier?: string;
+    description?: string;
+}
+export interface EvalRequirementsSchema {
+    chromeProfile?: string;
+    platform?: string;
+}

package/dist/eval/integration-types.js

@@ -0,0 +1,18 @@
+// ---------------------------------------------------------------------------
+// integration-types.ts -- types for the integration test runner
+// ---------------------------------------------------------------------------
+/** Default rate limits per platform (requests per minute). */
+export const DEFAULT_RATE_LIMITS = {
+    x: { requestsPerMinute: 3 },
+    twitter: { requestsPerMinute: 3 },
+    linkedin: { requestsPerMinute: 2 },
+    slack: { requestsPerMinute: 10 },
+    instagram: { requestsPerMinute: 5 },
+    facebook: { requestsPerMinute: 5 },
+};
+export const DEFAULT_RATE_LIMIT = { requestsPerMinute: 10 };
+// ---------------------------------------------------------------------------
+// Schema types for eval generation validation (US-003, US-004)
+// ---------------------------------------------------------------------------
+export const VALID_CLEANUP_ACTIONS = ["delete_post", "remove_artifact", "custom"];
+//# sourceMappingURL=integration-types.js.map

package/dist/eval/integration-types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"integration-types.js","sourceRoot":"","sources":["../../src/eval/integration-types.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,gEAAgE;AAChE,8EAA8E;AAwD9E,8DAA8D;AAC9D,MAAM,CAAC,MAAM,mBAAmB,GAAsC;IACpE,CAAC,EAAE,EAAE,iBAAiB,EAAE,CAAC,EAAE;IAC3B,OAAO,EAAE,EAAE,iBAAiB,EAAE,CAAC,EAAE;IACjC,QAAQ,EAAE,EAAE,iBAAiB,EAAE,CAAC,EAAE;IAClC,KAAK,EAAE,EAAE,iBAAiB,EAAE,EAAE,EAAE;IAChC,SAAS,EAAE,EAAE,iBAAiB,EAAE,CAAC,EAAE;IACnC,QAAQ,EAAE,EAAE,iBAAiB,EAAE,CAAC,EAAE;CACnC,CAAC;AAEF,MAAM,CAAC,MAAM,kBAAkB,GAAsB,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC;AAE/E,8EAA8E;AAC9E,+DAA+D;AAC/D,8EAA8E;AAE9E,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,aAAa,EAAE,iBAAiB,EAAE,QAAQ,CAAU,CAAC"}

package/dist/eval/judge-cache.d.ts

@@ -0,0 +1,29 @@
+import type { AssertionResult } from "./judge.js";
+export interface CacheEntry {
+    pass: boolean;
+    reasoning: string;
+    cachedAt: string;
+    judgeModel: string;
+}
+export interface CacheData {
+    version: number;
+    entries: Record<string, CacheEntry>;
+}
+export declare class JudgeCache {
+    private readonly skillDir;
+    private data;
+    private dirty;
+    private readonly cachePath;
+    constructor(skillDir: string);
+    private load;
+    static computeKey(assertionText: string, output: string, judgeModel: string): string;
+    getOrCompute(assertionText: string, output: string, judgeModel: string, compute: () => Promise<AssertionResult>): Promise<AssertionResult>;
+    has(assertionText: string, output: string, judgeModel: string): boolean;
+    get size(): number;
+    /**
+     * Persist cache to disk. Call after all operations are complete.
+     * Also ensures .judge-cache.json is in .gitignore.
+     */
+    flush(): void;
+    private ensureGitignore;
+}

package/dist/eval/judge-cache.js

@@ -0,0 +1,109 @@
+// ---------------------------------------------------------------------------
+// judge-cache.ts -- SHA-256 content-hash cache for judge results
+// ---------------------------------------------------------------------------
+import { createHash } from "node:crypto";
+import { readFileSync, writeFileSync, mkdirSync, existsSync, unlinkSync, appendFileSync } from "node:fs";
+import { join, dirname } from "node:path";
+export class JudgeCache {
+    skillDir;
+    data;
+    dirty = false;
+    cachePath;
+    constructor(skillDir) {
+        this.skillDir = skillDir;
+        this.cachePath = join(skillDir, "evals", ".judge-cache.json");
+        this.data = this.load();
+    }
+    load() {
+        try {
+            if (existsSync(this.cachePath)) {
+                const raw = readFileSync(this.cachePath, "utf-8");
+                const parsed = JSON.parse(raw);
+                if (parsed && typeof parsed === "object" && parsed.version === 1 && typeof parsed.entries === "object") {
+                    return parsed;
+                }
+            }
+        }
+        catch {
+            // Corruption recovery: delete corrupted file and start fresh
+            try {
+                if (existsSync(this.cachePath)) {
+                    unlinkSync(this.cachePath);
+                    console.warn(`[judge-cache] Corrupted cache file deleted: ${this.cachePath}`);
+                }
+            }
+            catch {
+                // ignore deletion failure
+            }
+        }
+        return { version: 1, entries: {} };
+    }
+    static computeKey(assertionText, output, judgeModel) {
+        return createHash("sha256")
+            .update(`${assertionText}||${output}||${judgeModel}`)
+            .digest("hex");
+    }
+    async getOrCompute(assertionText, output, judgeModel, compute) {
+        const key = JudgeCache.computeKey(assertionText, output, judgeModel);
+        const cached = this.data.entries[key];
+        if (cached) {
+            // Return cached result, reconstructing the AssertionResult shape
+            return {
+                id: "", // caller overwrites this
+                text: assertionText,
+                pass: cached.pass,
+                reasoning: cached.reasoning,
+            };
+        }
+        const result = await compute();
+        // Store in cache
+        this.data.entries[key] = {
+            pass: result.pass,
+            reasoning: result.reasoning,
+            cachedAt: new Date().toISOString(),
+            judgeModel,
+        };
+        this.dirty = true;
+        return result;
+    }
+    has(assertionText, output, judgeModel) {
+        const key = JudgeCache.computeKey(assertionText, output, judgeModel);
+        return key in this.data.entries;
+    }
+    get size() {
+        return Object.keys(this.data.entries).length;
+    }
+    /**
+     * Persist cache to disk. Call after all operations are complete.
+     * Also ensures .judge-cache.json is in .gitignore.
+     */
+    flush() {
+        if (!this.dirty)
+            return;
+        const dir = dirname(this.cachePath);
+        mkdirSync(dir, { recursive: true });
+        writeFileSync(this.cachePath, JSON.stringify(this.data, null, 2), "utf-8");
+        this.dirty = false;
+        // T-009: Ensure .judge-cache.json is in .gitignore
+        this.ensureGitignore();
+    }
+    ensureGitignore() {
+        const gitignorePath = join(this.skillDir, ".gitignore");
+        const pattern = "evals/.judge-cache.json";
+        try {
+            if (existsSync(gitignorePath)) {
+                const content = readFileSync(gitignorePath, "utf-8");
+                if (content.includes(pattern))
+                    return;
+                appendFileSync(gitignorePath, `\n${pattern}\n`);
+            }
+            else {
+                writeFileSync(gitignorePath, `${pattern}\n`, "utf-8");
+            }
+        }
+        catch {
+            // Non-critical — don't fail the run for gitignore issues
+        }
+    }
+}
+//# sourceMappingURL=judge-cache.js.map

package/dist/eval/judge-cache.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge-cache.js","sourceRoot":"","sources":["../../src/eval/judge-cache.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,iEAAiE;AACjE,8EAA8E;AAE9E,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,SAAS,EAAE,UAAU,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AACzG,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAe1C,MAAM,OAAO,UAAU;IAKQ;IAJrB,IAAI,CAAY;IAChB,KAAK,GAAG,KAAK,CAAC;IACL,SAAS,CAAS;IAEnC,YAA6B,QAAgB;QAAhB,aAAQ,GAAR,QAAQ,CAAQ;QAC3C,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,mBAAmB,CAAC,CAAC;QAC9D,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC1B,CAAC;IAEO,IAAI;QACV,IAAI,CAAC;YACH,IAAI,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC/B,MAAM,GAAG,GAAG,YAAY,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;gBAClD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAC/B,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,OAAO,KAAK,CAAC,IAAI,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;oBACvG,OAAO,MAAmB,CAAC;gBAC7B,CAAC;YACH,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,6DAA6D;YAC7D,IAAI,CAAC;gBACH,IAAI,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;oBAC/B,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBAC3B,OAAO,CAAC,IAAI,CAAC,+CAA+C,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;gBAChF,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,0BAA0B;YAC5B,CAAC;QACH,CAAC;QACD,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;IACrC,CAAC;IAED,MAAM,CAAC,UAAU,CAAC,aAAqB,EAAE,MAAc,EAAE,UAAkB;QACzE,OAAO,UAAU,CAAC,QAAQ,CAAC;aACxB,MAAM,CAAC,GAAG,aAAa,KAAK,MAAM,KAAK,UAAU,EAAE,CAAC;aACpD,MAAM,CAAC,KAAK,CAAC,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,YAAY,CAChB,aAAqB,EACrB,MAAc,EACd,UAAkB,EAClB,OAAuC;QAEvC,MAAM,GAAG,GAAG,UAAU,CAAC,UAAU,CAAC,aAAa,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;QAErE,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACtC,IAAI,MAAM,EAAE,CAAC;YACX,iEAAiE;YACjE,OAAO;gBACL,EAAE,EAAE,EAAE,EAAE,yBAAyB;gBACjC,IAAI,EAAE,aAAa;gBACnB,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,SAAS,EAAE,MAAM,CAAC,SAAS;aAC5B,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,OAAO,EAAE,CAAC;QAE/B,iBAAiB;QACjB,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG;YACvB,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,QAAQ,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAClC,UAAU;SACX,CAAC;QACF,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QAElB,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,GAAG,CAAC,aAAqB,EAAE,MAAc,EAAE,UAAkB;QAC3D,MAAM,GAAG,GAAG,UAAU,CAAC,UAAU,CAAC,aAAa,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;QACrE,OAAO,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC;IAClC,CAAC;IAED,IAAI,IAAI;QACN,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;IAC/C,CAAC;IAED;;;OAGG;IACH,KAAK;QACH,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,OAAO;QAExB,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACpC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACpC,aAAa,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;QAC3E,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QAEnB,mDAAmD;QACnD,IAAI,CAAC,eAAe,EAAE,CAAC;IACzB,CAAC;IAEO,eAAe;QACrB,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QACxD,MAAM,OAAO,GAAG,yBAAyB,CAAC;QAE1C,IAAI,CAAC;YACH,IAAI,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;gBAC9B,MAAM,OAAO,GAAG,YAAY,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;gBACrD,IAAI,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC;oBAAE,OAAO;gBACtC,cAAc,CAAC,aAAa,EAAE,KAAK,OAAO,IAAI,CAAC,CAAC;YAClD,CAAC;iBAAM,CAAC;gBACN,aAAa,CAAC,aAAa,EAAE,GAAG,OAAO,IAAI,EAAE,OAAO,CAAC,CAAC;YACxD,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,yDAAyD;QAC3D,CAAC;IACH,CAAC;CACF"}

package/dist/eval/judge.d.ts

@@ -8,4 +8,4 @@ export interface AssertionResult {
     reasoning: string;
 }
 export declare function buildJudgeSystemPrompt(mcpDeps?: McpDependency[]): string;
-export declare function judgeAssertion(output: string, assertion: Assertion, client: LlmClient, mcpDeps?: McpDependency[]): Promise<AssertionResult>;
+export declare function judgeAssertion(output: string, assertion: Assertion, client: LlmClient, judgeClientOrMcpDeps?: LlmClient | McpDependency[], mcpDeps?: McpDependency[]): Promise<AssertionResult>;

package/dist/eval/judge.js

@@ -17,8 +17,25 @@ When evaluating assertions:
 
 Respond with ONLY a JSON object: { "pass": boolean, "reasoning": "brief explanation" }`;
 }
-export async function judgeAssertion(output, assertion, client, mcpDeps) {
-    const systemPrompt = buildJudgeSystemPrompt(mcpDeps);
+export async function judgeAssertion(output, assertion, client, judgeClientOrMcpDeps, mcpDeps) {
+    // Support both old signature (client, mcpDeps?) and new (client, judgeClient?, mcpDeps?)
+    let effectiveJudgeClient;
+    let effectiveMcpDeps;
+    if (Array.isArray(judgeClientOrMcpDeps)) {
+        // Old-style call: judgeAssertion(output, assertion, client, mcpDeps)
+        effectiveJudgeClient = client;
+        effectiveMcpDeps = judgeClientOrMcpDeps;
+    }
+    else if (judgeClientOrMcpDeps && typeof judgeClientOrMcpDeps === "object" && "generate" in judgeClientOrMcpDeps) {
+        // New-style call: judgeAssertion(output, assertion, client, judgeClient, mcpDeps?)
+        effectiveJudgeClient = judgeClientOrMcpDeps;
+        effectiveMcpDeps = mcpDeps;
+    }
+    else {
+        effectiveJudgeClient = client;
+        effectiveMcpDeps = mcpDeps;
+    }
+    const systemPrompt = buildJudgeSystemPrompt(effectiveMcpDeps);
     const userPrompt = `## LLM Output
 ${output}
 
@@ -26,7 +43,7 @@ ${output}
 ${assertion.text}
 
 Does the LLM output satisfy this assertion? Respond with JSON only: { "pass": boolean, "reasoning": "..." }`;
-    const { text: raw } = await client.generate(systemPrompt, userPrompt);
+    const { text: raw } = await effectiveJudgeClient.generate(systemPrompt, userPrompt);
     const parsed = parseJudgeResponse(raw);
     return {
         id: assertion.id,

package/dist/eval/judge.js.map

@@ -1 +1 @@
-{"version":3,"file":"judge.js","sourceRoot":"","sources":["../../src/eval/judge.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,4BAA4B;AAC5B,8EAA8E;AAa9E,MAAM,YAAY,GAAG,qNAAqN,CAAC;AAE3O,MAAM,UAAU,sBAAsB,CAAC,OAAyB;IAC9D,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACrC,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE3D,OAAO;;oIAE2H,UAAU;;;;;;uFAMvD,CAAC;AACxF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,MAAc,EACd,SAAoB,EACpB,MAAiB,EACjB,OAAyB;IAEzB,MAAM,YAAY,GAAG,sBAAsB,CAAC,OAAO,CAAC,CAAC;IAErD,MAAM,UAAU,GAAG;EACnB,MAAM;;;EAGN,SAAS,CAAC,IAAI;;4GAE4F,CAAC;IAE3G,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;IAEtE,MAAM,MAAM,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC;IAEvC,OAAO;QACL,EAAE,EAAE,SAAS,CAAC,EAAE;QAChB,IAAI,EAAE,SAAS,CAAC,IAAI;QACpB,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,SAAS,EAAE,MAAM,CAAC,SAAS;KAC5B,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,GAAW;IACrC,4CAA4C;IAC5C,MAAM,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;IACjE,MAAM,OAAO,GAAG,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IAEjD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACnC,IAAI,OAAO,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;QACxC,CAAC;QACD,OAAO;YACL,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,SAAS,EAAE,OAAO,MAAM,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE;SACxE,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CACb,sEAAsE,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAC1F,CAAC;IACJ,CAAC;AACH,CAAC"}
+{"version":3,"file":"judge.js","sourceRoot":"","sources":["../../src/eval/judge.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,4BAA4B;AAC5B,8EAA8E;AAa9E,MAAM,YAAY,GAAG,qNAAqN,CAAC;AAE3O,MAAM,UAAU,sBAAsB,CAAC,OAAyB;IAC9D,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACrC,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE3D,OAAO;;oIAE2H,UAAU;;;;;;uFAMvD,CAAC;AACxF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,MAAc,EACd,SAAoB,EACpB,MAAiB,EACjB,oBAAkD,EAClD,OAAyB;IAEzB,yFAAyF;IACzF,IAAI,oBAA+B,CAAC;IACpC,IAAI,gBAA6C,CAAC;IAElD,IAAI,KAAK,CAAC,OAAO,CAAC,oBAAoB,CAAC,EAAE,CAAC;QACxC,qEAAqE;QACrE,oBAAoB,GAAG,MAAM,CAAC;QAC9B,gBAAgB,GAAG,oBAAoB,CAAC;IAC1C,CAAC;SAAM,IAAI,oBAAoB,IAAI,OAAO,oBAAoB,KAAK,QAAQ,IAAI,UAAU,IAAI,oBAAoB,EAAE,CAAC;QAClH,mFAAmF;QACnF,oBAAoB,GAAG,oBAAoB,CAAC;QAC5C,gBAAgB,GAAG,OAAO,CAAC;IAC7B,CAAC;SAAM,CAAC;QACN,oBAAoB,GAAG,MAAM,CAAC;QAC9B,gBAAgB,GAAG,OAAO,CAAC;IAC7B,CAAC;IAED,MAAM,YAAY,GAAG,sBAAsB,CAAC,gBAAgB,CAAC,CAAC;IAE9D,MAAM,UAAU,GAAG;EACnB,MAAM;;;EAGN,SAAS,CAAC,IAAI;;4GAE4F,CAAC;IAE3G,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,MAAM,oBAAoB,CAAC,QAAQ,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;IAEpF,MAAM,MAAM,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC;IAEvC,OAAO;QACL,EAAE,EAAE,SAAS,CAAC,EAAE;QAChB,IAAI,EAAE,SAAS,CAAC,IAAI;QACpB,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,SAAS,EAAE,MAAM,CAAC,SAAS;KAC5B,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,GAAW;IACrC,4CAA4C;IAC5C,MAAM,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;IACjE,MAAM,OAAO,GAAG,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IAEjD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACnC,IAAI,OAAO,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;QACxC,CAAC;QACD,OAAO;YACL,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,SAAS,EAAE,OAAO,MAAM,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE;SACxE,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CACb,sEAAsE,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAC1F,CAAC;IACJ,CAAC;AACH,CAAC"}

package/dist/eval/llm.d.ts

@@ -3,12 +3,13 @@ export interface GenerateResult {
     durationMs: number;
     inputTokens: number | null;
     outputTokens: number | null;
+    cost: number | null;
 }
 export interface LlmClient {
     generate(systemPrompt: string, userPrompt: string): Promise<GenerateResult>;
     readonly model: string;
 }
-export type ProviderName = "anthropic" | "claude-cli" | "codex-cli" | "gemini-cli" | "ollama";
+export type ProviderName = "anthropic" | "claude-cli" | "codex-cli" | "gemini-cli" | "ollama" | "openrouter";
 export interface LlmOverrides {
     provider?: ProviderName;
     model?: string;