vskill 0.5.11 → 0.5.13

package/dist/eval-server/benchmark-runner.js

@@ -6,58 +6,137 @@ import { judgeAssertion } from "../eval/judge.js";
 import { writeHistoryEntry } from "../eval/benchmark-history.js";
 import { sendSSE, sendSSEDone, withHeartbeat } from "./sse-helpers.js";
 import { classifyError } from "./error-classifier.js";
+import { Semaphore } from "./concurrency.js";
+// ---------------------------------------------------------------------------
+// CLI providers that spawn child processes — default concurrency 1
+// ---------------------------------------------------------------------------
+const CLI_PROVIDERS = new Set(["claude-cli", "codex-cli", "gemini-cli"]);
+function defaultConcurrency(provider) {
+    return CLI_PROVIDERS.has(provider) ? 1 : 5;
+}
+// ---------------------------------------------------------------------------
+// 429 retry helper — exponential backoff (1s, 2s, 4s), max 3 retries
+// ---------------------------------------------------------------------------
+function is429(err) {
+    if (err instanceof Error) {
+        const msg = err.message;
+        if (msg.includes("429") || msg.includes("rate limit") || msg.includes("Rate limit"))
+            return true;
+        if ("status" in err && err.status === 429)
+            return true;
+    }
+    return false;
+}
+async function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+// ---------------------------------------------------------------------------
+// Weak-model judge warning heuristic
+// ---------------------------------------------------------------------------
+const MODEL_STRENGTH = {
+    haiku: 1,
+    flash: 1,
+    "mini": 1,
+    sonnet: 2,
+    pro: 2,
+    opus: 3,
+};
+function estimateStrength(model) {
+    const lower = model.toLowerCase();
+    for (const [key, strength] of Object.entries(MODEL_STRENGTH)) {
+        if (lower.includes(key))
+            return strength;
+    }
+    return 2; // unknown = medium
+}
+export function checkWeakJudgeWarning(generatorModel, judgeModel) {
+    const genStrength = estimateStrength(generatorModel);
+    const judgeStrength = estimateStrength(judgeModel);
+    if (judgeStrength < genStrength) {
+        return `Warning: Judge model "${judgeModel}" appears weaker than generator "${generatorModel}". Judge scores may be less reliable.`;
+    }
+    return null;
+}
 export async function runSingleCaseSSE(opts) {
-    const { res, evalCase, systemPrompt, client, isAborted, totalCases = 1, provider } = opts;
-    sendSSE(res, "case_start", {
+    const { res, evalCase, systemPrompt, client, judgeClient, judgeCache, isAborted, totalCases = 1, provider } = opts;
+    const effectiveJudgeClient = judgeClient ?? client;
+    let sequence = 0;
+    const emitSSE = (event, data) => {
+        sendSSE(res, event, { ...data, caseId: evalCase.id, sequence: sequence++ });
+    };
+    emitSSE("case_start", {
         eval_id: evalCase.id,
         eval_name: evalCase.name,
         total: totalCases,
     });
-    sendSSE(res, "progress", {
+    emitSSE("progress", {
         eval_id: evalCase.id,
         phase: "generating",
         message: `Generating LLM response for "${evalCase.name}"...`,
     });
     try {
-        const genResult = await withHeartbeat(res, evalCase.id, "generating", `Generating LLM response for "${evalCase.name}"...`, () => client.generate(systemPrompt, evalCase.prompt));
+        // Retry with exponential backoff on 429
+        let genResult;
+        const MAX_RETRIES = 3;
+        const RETRY_DELAYS = [1000, 2000, 4000];
+        for (let attempt = 0;; attempt++) {
+            try {
+                genResult = await withHeartbeat(res, evalCase.id, "generating", `Generating LLM response for "${evalCase.name}"...`, () => client.generate(systemPrompt, evalCase.prompt));
+                break;
+            }
+            catch (err) {
+                if (is429(err) && attempt < MAX_RETRIES) {
+                    const delayMs = RETRY_DELAYS[attempt];
+                    emitSSE("warning", {
+                        eval_id: evalCase.id,
+                        message: `Rate limited (429). Retrying in ${delayMs / 1000}s (attempt ${attempt + 1}/${MAX_RETRIES})...`,
+                        attempt: attempt + 1,
+                        delayMs,
+                    });
+                    await sleep(delayMs);
+                    continue;
+                }
+                throw err;
+            }
+        }
         const totalTokens = genResult.inputTokens != null && genResult.outputTokens != null
             ? genResult.inputTokens + genResult.outputTokens
             : null;
-        sendSSE(res, "output_ready", {
+        emitSSE("output_ready", {
             eval_id: evalCase.id,
             output: genResult.text,
             durationMs: genResult.durationMs,
             tokens: totalTokens,
         });
-        sendSSE(res, "progress", {
+        emitSSE("progress", {
             eval_id: evalCase.id,
             phase: "judging",
             message: `Evaluating ${evalCase.assertions.length} assertion${evalCase.assertions.length !== 1 ? "s" : ""}...`,
             total: evalCase.assertions.length,
         });
-        const assertionResults = [];
-        for (let ai = 0; ai < evalCase.assertions.length; ai++) {
-            const assertion = evalCase.assertions[ai];
-            sendSSE(res, "progress", {
-                eval_id: evalCase.id,
-                phase: "judging_assertion",
-                message: `Evaluating assertion ${ai + 1}/${evalCase.assertions.length}: "${assertion.text.slice(0, 60)}${assertion.text.length > 60 ? "..." : ""}"`,
-                current: ai + 1,
-                total: evalCase.assertions.length,
-                assertion_id: assertion.id,
-            });
-            if (isAborted())
-                break;
-            const result = await judgeAssertion(genResult.text, assertion, client);
-            assertionResults.push(result);
-            sendSSE(res, "assertion_result", {
+        // T-003: Parallelize intra-case assertion judges via Promise.all
+        const assertionResults = await Promise.all(evalCase.assertions.map(async (assertion, ai) => {
+            if (isAborted()) {
+                return { id: assertion.id, text: assertion.text, pass: false, reasoning: "aborted" };
+            }
+            const judgeCall = () => judgeAssertion(genResult.text, assertion, client, effectiveJudgeClient);
+            let result;
+            if (judgeCache) {
+                result = await judgeCache.getOrCompute(assertion.text, genResult.text, effectiveJudgeClient.model, judgeCall);
+            }
+            else {
+                result = await judgeCall();
+            }
+            emitSSE("assertion_result", {
                 eval_id: evalCase.id,
                 assertion_id: result.id,
                 text: result.text,
                 pass: result.pass,
                 reasoning: result.reasoning,
+                assertion_index: ai,
             });
-        }
+            return result;
+        }));
         const passRate = assertionResults.length > 0
             ? assertionResults.filter((a) => a.pass).length / assertionResults.length
             : 0;
@@ -75,7 +154,7 @@ export async function runSingleCaseSSE(opts) {
             output: genResult.text,
             assertions: assertionResults,
         };
-        sendSSE(res, "case_complete", {
+        emitSSE("case_complete", {
             eval_id: evalCase.id,
             status,
             pass_rate: passRate,
@@ -95,7 +174,7 @@ export async function runSingleCaseSSE(opts) {
             pass_rate: 0,
             assertions: [],
         };
-        sendSSE(res, "case_complete", {
+        emitSSE("case_complete", {
             eval_id: evalCase.id,
             status: "error",
             error_message: errorMsg,
@@ -127,10 +206,11 @@ export function assembleBulkResult(cases, meta) {
     };
 }
 export async function runBenchmarkSSE(opts) {
-    const { res, skillDir, skillName, systemPrompt, runType, provider, evalCases: allCases, filterIds, client, isAborted, } = opts;
+    const { res, skillDir, skillName, systemPrompt, runType, provider, evalCases: allCases, filterIds, client, judgeClient, judgeCache, isAborted, concurrency: concurrencyOverride, } = opts;
     const evalCases = filterIds
         ? allCases.filter((e) => filterIds.has(e.id))
         : allCases;
+    const concurrency = concurrencyOverride ?? defaultConcurrency(provider);
     // Emit duration estimate so the UI can warn about long runs
     const totalAssertions = evalCases.reduce((s, e) => s + e.assertions.length, 0);
     const estimate = estimateDurationSec(provider, evalCases.length, totalAssertions);
@@ -140,29 +220,65 @@ export async function runBenchmarkSSE(opts) {
         estimatedMinSec: estimate.minSec,
         estimatedMaxSec: estimate.maxSec,
         estimatedLabel: estimate.label,
+        concurrency,
     });
-    const cases = [];
-    for (const evalCase of evalCases) {
+    // Emit weak-model judge warning if applicable
+    if (judgeClient) {
+        const warning = checkWeakJudgeWarning(client.model, judgeClient.model);
+        if (warning) {
+            console.warn(warning);
+            sendSSE(res, "warning", { message: warning });
+        }
+    }
+    // T-001: Parallel case execution via Semaphore + Promise.allSettled
+    const semaphore = new Semaphore(concurrency);
+    const caseTasks = evalCases.map((evalCase) => async () => {
         if (isAborted())
-            break;
-        const benchCase = await runSingleCaseSSE({
-            res,
-            evalCase,
-            systemPrompt,
-            client,
-            isAborted,
-            totalCases: evalCases.length,
-            provider,
-        });
-        cases.push(benchCase);
+            return null;
+        await semaphore.acquire();
+        try {
+            return await runSingleCaseSSE({
+                res,
+                evalCase,
+                systemPrompt,
+                client,
+                judgeClient,
+                judgeCache,
+                isAborted,
+                totalCases: evalCases.length,
+                provider,
+            });
+        }
+        finally {
+            semaphore.release();
+        }
+    });
+    const settled = await Promise.allSettled(caseTasks.map((fn) => fn()));
+    const cases = [];
+    for (const result of settled) {
+        if (result.status === "fulfilled" && result.value != null) {
+            cases.push(result.value);
+        }
+        else if (result.status === "rejected") {
+            // Shouldn't normally happen since runSingleCaseSSE catches errors,
+            // but handle as safety net
+            cases.push({
+                eval_id: -1,
+                eval_name: "unknown",
+                status: "error",
+                error_message: result.reason instanceof Error ? result.reason.message : String(result.reason),
+                pass_rate: 0,
+                assertions: [],
+            });
+        }
     }
-    const result = assembleBulkResult(cases, { model: client.model, skillName, runType, provider });
+    const bulkResult = assembleBulkResult(cases, { model: client.model, skillName, runType, provider });
     if (!isAborted()) {
         // Save history for bulk runs (single-case runs save via per-case endpoint)
         if (!filterIds) {
-            await writeHistoryEntry(skillDir, result);
+            await writeHistoryEntry(skillDir, bulkResult);
         }
-        sendSSEDone(res, result);
+        sendSSEDone(res, bulkResult);
     }
 }
 //# sourceMappingURL=benchmark-runner.js.map

package/dist/eval-server/benchmark-runner.js.map

@@ -1 +1 @@
-{"version":3,"file":"benchmark-runner.js","sourceRoot":"","sources":["../../src/eval-server/benchmark-runner.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,sEAAsE;AACtE,8EAA8E;AAM9E,OAAO,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AACrD,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AACjE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACvE,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAgBtD,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,IAA0B;IAC/D,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,EAAE,SAAS,EAAE,UAAU,GAAG,CAAC,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC;IAE1F,OAAO,CAAC,GAAG,EAAE,YAAY,EAAE;QACzB,OAAO,EAAE,QAAQ,CAAC,EAAE;QACpB,SAAS,EAAE,QAAQ,CAAC,IAAI;QACxB,KAAK,EAAE,UAAU;KAClB,CAAC,CAAC;IAEH,OAAO,CAAC,GAAG,EAAE,UAAU,EAAE;QACvB,OAAO,EAAE,QAAQ,CAAC,EAAE;QACpB,KAAK,EAAE,YAAY;QACnB,OAAO,EAAE,gCAAgC,QAAQ,CAAC,IAAI,MAAM;KAC7D,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,MAAM,aAAa,CACnC,GAAG,EAAE,QAAQ,CAAC,EAAE,EAAE,YAAY,EAC9B,gCAAgC,QAAQ,CAAC,IAAI,MAAM,EACnD,GAAG,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC,CACrD,CAAC;QACF,MAAM,WAAW,GAAG,SAAS,CAAC,WAAW,IAAI,IAAI,IAAI,SAAS,CAAC,YAAY,IAAI,IAAI;YACjF,CAAC,CAAC,SAAS,CAAC,WAAW,GAAG,SAAS,CAAC,YAAY;YAChD,CAAC,CAAC,IAAI,CAAC;QAET,OAAO,CAAC,GAAG,EAAE,cAAc,EAAE;YAC3B,OAAO,EAAE,QAAQ,CAAC,EAAE;YACpB,MAAM,EAAE,SAAS,CAAC,IAAI;YACtB,UAAU,EAAE,SAAS,CAAC,UAAU;YAChC,MAAM,EAAE,WAAW;SACpB,CAAC,CAAC;QAEH,OAAO,CAAC,GAAG,EAAE,UAAU,EAAE;YACvB,OAAO,EAAE,QAAQ,CAAC,EAAE;YACpB,KAAK,EAAE,SAAS;YAChB,OAAO,EAAE,cAAc,QAAQ,CAAC,UAAU,CAAC,MAAM,aAAa,QAAQ,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK;YAC9G,KAAK,EAAE,QAAQ,CAAC,UAAU,CAAC,MAAM;SAClC,CAAC,CAAC;QAEH,MAAM,gBAAgB,GAA+B,EAAE,CAAC;QAExD,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,QAAQ,CAAC,UAAU,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC;YACvD,MAAM,SAAS,GAAG,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;YAE1C,OAAO,CAAC,GAAG,EAAE,UAAU,EAAE;gBACvB,OAAO,EAAE,QAAQ,CAAC,EAAE;gBACpB,KAAK,EAAE,mBAAmB;gBAC1B,OAAO,EAAE,wBAAwB,EAAE,GAAG,CAAC,IAAI,QAAQ,CAAC,UAAU,CAAC,MAAM,MAAM,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG;gBACnJ,OAAO,EAAE,EAAE,GAAG,CAAC;gBACf,KAAK,EAAE,QAAQ,CAAC,UAAU,CAAC,MAAM;gBACjC,YAAY,EAAE,SAAS,CAAC,EAAE;aAC3B,CAAC,CAAC;YACH,IAAI,SAAS,EAAE;gBAAE,MAAM;YACvB,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,SAAS,CAAC,IAAI,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;YACvE,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC9B,OAAO,CAAC,GAAG,EAAE,kBAAkB,EAAE;gBAC/B,OAAO,EAAE,QAAQ,CAAC,EAAE;gBACpB,YAAY,EAAE,MAAM,CAAC,EAAE;gBACvB,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,SAAS,EAAE,MAAM,CAAC,SAAS;aAC5B,CAAC,CAAC;QACL,CAAC;QAED,MAAM,QAAQ,GACZ,gBAAgB,CAAC,MAAM,GAAG,CAAC;YACzB,CAAC,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,gBAAgB,CAAC,MAAM;YACzE,CAAC,CAAC,CAAC,CAAC;QACR,MAAM,MAAM,GAAG,gBAAgB,CAAC,MAAM,GAAG,CAAC,IAAI,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;QAEtG,MAAM,SAAS,GAAkB;YAC/B,OAAO,EAAE,QAAQ,CAAC,EAAE;YACpB,SAAS,EAAE,QAAQ,CAAC,IAAI;YACxB,MAAM,EAAE,MAAyB;YACjC,aAAa,EAAE,IAAI;YACnB,SAAS,EAAE,QAAQ;YACnB,UAAU,EAAE,SAAS,CAAC,UAAU;YAChC,MAAM,EAAE,WAAW;YACnB,WAAW,EAAE,SAAS,CAAC,WAAW;YAClC,YAAY,EAAE,SAAS,CAAC,YAAY;YACpC,MAAM,EAAE,SAAS,CAAC,IAAI;YACtB,UAAU,EAAE,gBAAgB;SAC7B,CAAC;QAEF,OAAO,CAAC,GAAG,EAAE,eAAe,EAAE;YAC5B,OAAO,EAAE,QAAQ,CAAC,EAAE;YACpB,MAAM;YACN,SAAS,EAAE,QAAQ;YACnB,UAAU,EAAE,SAAS,CAAC,UAAU;YAChC,MAAM,EAAE,WAAW;SACpB,CAAC,CAAC;QAEH,OAAO,SAAS,CAAC;IACnB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,QAAQ,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAClE,MAAM,UAAU,GAAG,aAAa,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;QAChD,MAAM,SAAS,GAAkB;YAC/B,OAAO,EAAE,QAAQ,CAAC,EAAE;YACpB,SAAS,EAAE,QAAQ,CAAC,IAAI;YACxB,MAAM,EAAE,OAAO;YACf,aAAa,EAAE,QAAQ;YACvB,SAAS,EAAE,CAAC;YACZ,UAAU,EAAE,EAAE;SACf,CAAC;QACF,OAAO,CAAC,GAAG,EAAE,eAAe,EAAE;YAC5B,OAAO,EAAE,QAAQ,CAAC,EAAE;YACpB,MAAM,EAAE,OAAO;YACf,aAAa,EAAE,QAAQ;YACvB,gBAAgB,EAAE,UAAU;SAC7B,CAAC,CAAC;QACH,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,oDAAoD;AACpD,8EAA8E;AAE9E,MAAM,UAAU,kBAAkB,CAChC,KAAsB,EACtB,IAA+F;IAE/F,MAAM,eAAe,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC3E,MAAM,gBAAgB,GAAG,KAAK,CAAC,MAAM,CACnC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EACvD,CAAC,CACF,CAAC;IACF,MAAM,eAAe,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC3E,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,IAAI,IAAI,CAAC,CAAC;IAE3D,OAAO;QACL,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,UAAU,EAAE,IAAI,CAAC,SAAS;QAC1B,KAAK;QACL,iBAAiB,EAAE,eAAe,GAAG,CAAC,CAAC,CAAC,CAAC,gBAAgB,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;QAC/E,IAAI,EAAE,IAAI,CAAC,OAAO;QAClB,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,eAAe;QACf,gBAAgB,EAAE,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI;QACxF,iBAAiB,EAAE,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI;QAC1F,KAAK,EAAE,MAAM;KACd,CAAC;AACJ,CAAC;AAmBD,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,IAAyB;IAC7D,MAAM,EACJ,GAAG,EAAE,QAAQ,EAAE,SAAS,EAAE,YAAY,EAAE,OAAO,EAAE,QAAQ,EACzD,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,GAClD,GAAG,IAAI,CAAC;IAET,MAAM,SAAS,GAAG,SAAS;QACzB,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC7C,CAAC,CAAC,QAAQ,CAAC;IAEb,4DAA4D;IAC5D,MAAM,eAAe,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC/E,MAAM,QAAQ,GAAG,mBAAmB,CAAC,QAAwB,EAAE,SAAS,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IAClG,OAAO,CAAC,GAAG,EAAE,UAAU,EAAE;QACvB,UAAU,EAAE,SAAS,CAAC,MAAM;QAC5B,eAAe;QACf,eAAe,EAAE,QAAQ,CAAC,MAAM;QAChC,eAAe,EAAE,QAAQ,CAAC,MAAM;QAChC,cAAc,EAAE,QAAQ,CAAC,KAAK;KAC/B,CAAC,CAAC;IAEH,MAAM,KAAK,GAAoB,EAAE,CAAC;IAElC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,IAAI,SAAS,EAAE;YAAE,MAAM;QAEvB,MAAM,SAAS,GAAG,MAAM,gBAAgB,CAAC;YACvC,GAAG;YACH,QAAQ;YACR,YAAY;YACZ,MAAM;YACN,SAAS;YACT,UAAU,EAAE,SAAS,CAAC,MAAM;YAC5B,QAAQ;SACT,CAAC,CAAC;QACH,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACxB,CAAC;IAED,MAAM,MAAM,GAAG,kBAAkB,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;IAEhG,IAAI,CAAC,SAAS,EAAE,EAAE,CAAC;QACjB,2EAA2E;QAC3E,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,iBAAiB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAC5C,CAAC;QACD,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IAC3B,CAAC;AACH,CAAC"}
+{"version":3,"file":"benchmark-runner.js","sourceRoot":"","sources":["../../src/eval-server/benchmark-runner.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,sEAAsE;AACtE,8EAA8E;AAM9E,OAAO,EAAE,mBAAmB,EAAmB,MAAM,gBAAgB,CAAC;AACtE,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAElD,OAAO,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AACjE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACvE,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACtD,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAE7C,8EAA8E;AAC9E,mEAAmE;AACnE,8EAA8E;AAE9E,MAAM,aAAa,GAAG,IAAI,GAAG,CAAS,CAAC,YAAY,EAAE,WAAW,EAAE,YAAY,CAAC,CAAC,CAAC;AAEjF,SAAS,kBAAkB,CAAC,QAAgB;IAC1C,OAAO,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC7C,CAAC;AAED,8EAA8E;AAC9E,qEAAqE;AACrE,8EAA8E;AAE9E,SAAS,KAAK,CAAC,GAAY;IACzB,IAAI,GAAG,YAAY,KAAK,EAAE,CAAC;QACzB,MAAM,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC;QACxB,IAAI,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC;YAAE,OAAO,IAAI,CAAC;QACjG,IAAI,QAAQ,IAAI,GAAG,IAAK,GAAW,CAAC,MAAM,KAAK,GAAG;YAAE,OAAO,IAAI,CAAC;IAClE,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,KAAK,UAAU,KAAK,CAAC,EAAU;IAC7B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,8EAA8E;AAC9E,qCAAqC;AACrC,8EAA8E;AAE9E,MAAM,cAAc,GAA2B;IAC7C,KAAK,EAAE,CAAC;IACR,KAAK,EAAE,CAAC;IACR,MAAM,EAAE,CAAC;IACT,MAAM,EAAE,CAAC;IACT,GAAG,EAAE,CAAC;IACN,IAAI,EAAE,CAAC;CACR,CAAC;AAEF,SAAS,gBAAgB,CAAC,KAAa;IACrC,MAAM,KAAK,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;IAClC,KAAK,MAAM,CAAC,GAAG,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC;QAC7D,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC;YAAE,OAAO,QAAQ,CAAC;IAC3C,CAAC;IACD,OAAO,CAAC,CAAC,CAAC,mBAAmB;AAC/B,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,cAAsB,EAAE,UAAkB;IAC9E,MAAM,WAAW,GAAG,gBAAgB,CAAC,cAAc,CAAC,CAAC;IACrD,MAAM,aAAa,GAAG,gBAAgB,CAAC,UAAU,CAAC,CAAC;IACnD,IAAI,aAAa,GAAG,WAAW,EAAE,CAAC;QAChC,OAAO,yBAAyB,UAAU,oCAAoC,cAAc,uCAAuC,CAAC;IACtI,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAkBD,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,IAA0B;IAC/D,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,SAAS,EAAE,UAAU,GAAG,CAAC,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC;IACnH,MAAM,oBAAoB,GAAG,WAAW,IAAI,MAAM,CAAC;IACnD,IAAI,QAAQ,GAAG,CAAC,CAAC;IAEjB,MAAM,OAAO,GAAG,CAAC,KAAa,EAAE,IAA6B,EAAE,EAAE;QAC/D,OAAO,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAC;IAC9E,CAAC,CAAC;IAEF,OAAO,CAAC,YAAY,EAAE;QACpB,OAAO,EAAE,QAAQ,CAAC,EAAE;QACpB,SAAS,EAAE,QAAQ,CAAC,IAAI;QACxB,KAAK,EAAE,UAAU;KAClB,CAAC,CAAC;IAEH,OAAO,CAAC,UAAU,EAAE;QAClB,OAAO,EAAE,QAAQ,CAAC,EAAE;QACpB,KAAK,EAAE,YAAY;QACnB,OAAO,EAAE,gCAAgC,QAAQ,CAAC,IAAI,MAAM;KAC7D,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,wCAAwC;QACxC,IAAI,SAAS,CAAC;QACd,MAAM,WAAW,GAAG,CAAC,CAAC;QACtB,MAAM,YAAY,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;QACxC,KAAK,IAAI,OAAO,GAAG,CAAC,GAAI,OAAO,EAAE,EAAE,CAAC;YAClC,IAAI,CAAC;gBACH,SAAS,GAAG,MAAM,aAAa,CAC7B,GAAG,EAAE,QAAQ,CAAC,EAAE,EAAE,YAAY,EAC9B,gCAAgC,QAAQ,CAAC,IAAI,MAAM,EACnD,GAAG,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC,CACrD,CAAC;gBACF,MAAM;YACR,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,KAAK,CAAC,GAAG,CAAC,IAAI,OAAO,GAAG,WAAW,EAAE,CAAC;oBACxC,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;oBACtC,OAAO,CAAC,SAAS,EAAE;wBACjB,OAAO,EAAE,QAAQ,CAAC,EAAE;wBACpB,OAAO,EAAE,mCAAmC,OAAO,GAAG,IAAI,cAAc,OAAO,GAAG,CAAC,IAAI,WAAW,MAAM;wBACxG,OAAO,EAAE,OAAO,GAAG,CAAC;wBACpB,OAAO;qBACR,CAAC,CAAC;oBACH,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;oBACrB,SAAS;gBACX,CAAC;gBACD,MAAM,GAAG,CAAC;YACZ,CAAC;QACH,CAAC;QAED,MAAM,WAAW,GAAG,SAAS,CAAC,WAAW,IAAI,IAAI,IAAI,SAAS,CAAC,YAAY,IAAI,IAAI;YACjF,CAAC,CAAC,SAAS,CAAC,WAAW,GAAG,SAAS,CAAC,YAAY;YAChD,CAAC,CAAC,IAAI,CAAC;QAET,OAAO,CAAC,cAAc,EAAE;YACtB,OAAO,EAAE,QAAQ,CAAC,EAAE;YACpB,MAAM,EAAE,SAAS,CAAC,IAAI;YACtB,UAAU,EAAE,SAAS,CAAC,UAAU;YAChC,MAAM,EAAE,WAAW;SACpB,CAAC,CAAC;QAEH,OAAO,CAAC,UAAU,EAAE;YAClB,OAAO,EAAE,QAAQ,CAAC,EAAE;YACpB,KAAK,EAAE,SAAS;YAChB,OAAO,EAAE,cAAc,QAAQ,CAAC,UAAU,CAAC,MAAM,aAAa,QAAQ,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK;YAC9G,KAAK,EAAE,QAAQ,CAAC,UAAU,CAAC,MAAM;SAClC,CAAC,CAAC;QAEH,iEAAiE;QACjE,MAAM,gBAAgB,GAA+B,MAAM,OAAO,CAAC,GAAG,CACpE,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,EAAE,EAAE;YAC9C,IAAI,SAAS,EAAE,EAAE,CAAC;gBAChB,OAAO,EAAE,EAAE,EAAE,SAAS,CAAC,EAAE,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC;YACvF,CAAC;YAED,MAAM,SAAS,GAAG,GAAG,EAAE,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,oBAAoB,CAAC,CAAC;YAEhG,IAAI,MAAM,CAAC;YACX,IAAI,UAAU,EAAE,CAAC;gBACf,MAAM,GAAG,MAAM,UAAU,CAAC,YAAY,CACpC,SAAS,CAAC,IAAI,EACd,SAAS,CAAC,IAAI,EACd,oBAAoB,CAAC,KAAK,EAC1B,SAAS,CACV,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,MAAM,GAAG,MAAM,SAAS,EAAE,CAAC;YAC7B,CAAC;YAED,OAAO,CAAC,kBAAkB,EAAE;gBAC1B,OAAO,EAAE,QAAQ,CAAC,EAAE;gBACpB,YAAY,EAAE,MAAM,CAAC,EAAE;gBACvB,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,eAAe,EAAE,EAAE;aACpB,CAAC,CAAC;YAEH,OAAO,MAAM,CAAC;QAChB,CAAC,CAAC,CACH,CAAC;QAEF,MAAM,QAAQ,GACZ,gBAAgB,CAAC,MAAM,GAAG,CAAC;YACzB,CAAC,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,gBAAgB,CAAC,MAAM;YACzE,CAAC,CAAC,CAAC,CAAC;QACR,MAAM,MAAM,GAAG,gBAAgB,CAAC,MAAM,GAAG,CAAC,IAAI,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;QAEtG,MAAM,SAAS,GAAkB;YAC/B,OAAO,EAAE,QAAQ,CAAC,EAAE;YACpB,SAAS,EAAE,QAAQ,CAAC,IAAI;YACxB,MAAM,EAAE,MAAyB;YACjC,aAAa,EAAE,IAAI;YACnB,SAAS,EAAE,QAAQ;YACnB,UAAU,EAAE,SAAS,CAAC,UAAU;YAChC,MAAM,EAAE,WAAW;YACnB,WAAW,EAAE,SAAS,CAAC,WAAW;YAClC,YAAY,EAAE,SAAS,CAAC,YAAY;YACpC,MAAM,EAAE,SAAS,CAAC,IAAI;YACtB,UAAU,EAAE,gBAAgB;SAC7B,CAAC;QAEF,OAAO,CAAC,eAAe,EAAE;YACvB,OAAO,EAAE,QAAQ,CAAC,EAAE;YACpB,MAAM;YACN,SAAS,EAAE,QAAQ;YACnB,UAAU,EAAE,SAAS,CAAC,UAAU;YAChC,MAAM,EAAE,WAAW;SACpB,CAAC,CAAC;QAEH,OAAO,SAAS,CAAC;IACnB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,QAAQ,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAClE,MAAM,UAAU,GAAG,aAAa,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;QAChD,MAAM,SAAS,GAAkB;YAC/B,OAAO,EAAE,QAAQ,CAAC,EAAE;YACpB,SAAS,EAAE,QAAQ,CAAC,IAAI;YACxB,MAAM,EAAE,OAAO;YACf,aAAa,EAAE,QAAQ;YACvB,SAAS,EAAE,CAAC;YACZ,UAAU,EAAE,EAAE;SACf,CAAC;QACF,OAAO,CAAC,eAAe,EAAE;YACvB,OAAO,EAAE,QAAQ,CAAC,EAAE;YACpB,MAAM,EAAE,OAAO;YACf,aAAa,EAAE,QAAQ;YACvB,gBAAgB,EAAE,UAAU;SAC7B,CAAC,CAAC;QACH,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,oDAAoD;AACpD,8EAA8E;AAE9E,MAAM,UAAU,kBAAkB,CAChC,KAAsB,EACtB,IAA+F;IAE/F,MAAM,eAAe,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC3E,MAAM,gBAAgB,GAAG,KAAK,CAAC,MAAM,CACnC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EACvD,CAAC,CACF,CAAC;IACF,MAAM,eAAe,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC3E,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,IAAI,IAAI,CAAC,CAAC;IAE3D,OAAO;QACL,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,UAAU,EAAE,IAAI,CAAC,SAAS;QAC1B,KAAK;QACL,iBAAiB,EAAE,eAAe,GAAG,CAAC,CAAC,CAAC,CAAC,gBAAgB,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;QAC/E,IAAI,EAAE,IAAI,CAAC,OAAO;QAClB,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,eAAe;QACf,gBAAgB,EAAE,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI;QACxF,iBAAiB,EAAE,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI;QAC1F,KAAK,EAAE,MAAM;KACd,CAAC;AACJ,CAAC;AAsBD,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,IAAyB;IAC7D,MAAM,EACJ,GAAG,EAAE,QAAQ,EAAE,SAAS,EAAE,YAAY,EAAE,OAAO,EAAE,QAAQ,EACzD,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,SAAS,EAC1E,WAAW,EAAE,mBAAmB,GACjC,GAAG,IAAI,CAAC;IAET,MAAM,SAAS,GAAG,SAAS;QACzB,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC7C,CAAC,CAAC,QAAQ,CAAC;IAEb,MAAM,WAAW,GAAG,mBAAmB,IAAI,kBAAkB,CAAC,QAAQ,CAAC,CAAC;IAExE,4DAA4D;IAC5D,MAAM,eAAe,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC/E,MAAM,QAAQ,GAAG,mBAAmB,CAAC,QAAwB,EAAE,SAAS,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IAClG,OAAO,CAAC,GAAG,EAAE,UAAU,EAAE;QACvB,UAAU,EAAE,SAAS,CAAC,MAAM;QAC5B,eAAe;QACf,eAAe,EAAE,QAAQ,CAAC,MAAM;QAChC,eAAe,EAAE,QAAQ,CAAC,MAAM;QAChC,cAAc,EAAE,QAAQ,CAAC,KAAK;QAC9B,WAAW;KACZ,CAAC,CAAC;IAEH,8CAA8C;IAC9C,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,OAAO,GAAG,qBAAqB,CAAC,MAAM,CAAC,KAAK,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC;QACvE,IAAI,OAAO,EAAE,CAAC;YACZ,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACtB,OAAO,CAAC,GAAG,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IAED,oEAAoE;IACpE,MAAM,SAAS,GAAG,IAAI,SAAS,CAAC,WAAW,CAAC,CAAC;IAE7C,MAAM,SAAS,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,KAAK,IAAI,EAAE;QACvD,IAAI,SAAS,EAAE;YAAE,OAAO,IAAI,CAAC;QAC7B,MAAM,SAAS,CAAC,OAAO,EAAE,CAAC;QAC1B,IAAI,CAAC;YACH,OAAO,MAAM,gBAAgB,CAAC;gBAC5B,GAAG;gBACH,QAAQ;gBACR,YAAY;gBACZ,MAAM;gBACN,WAAW;gBACX,UAAU;gBACV,SAAS;gBACT,UAAU,EAAE,SAAS,CAAC,MAAM;gBAC5B,QAAQ;aACT,CAAC,CAAC;QACL,CAAC;gBAAS,CAAC;YACT,SAAS,CAAC,OAAO,EAAE,CAAC;QACtB,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;IAEtE,MAAM,KAAK,GAAoB,EAAE,CAAC;IAClC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,IAAI,MAAM,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC;YAC1D,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC3B,CAAC;aAAM,IAAI,MAAM,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;YACxC,mEAAmE;YACnE,2BAA2B;YAC3B,KAAK,CAAC,IAAI,CAAC;gBACT,OAAO,EAAE,CAAC,CAAC;gBACX,SAAS,EAAE,SAAS;gBACpB,MAAM,EAAE,OAAO;gBACf,aAAa,EAAE,MAAM,CAAC,MAAM,YAAY,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC;gBAC7F,SAAS,EAAE,CAAC;gBACZ,UAAU,EAAE,EAAE;aACf,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,kBAAkB,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;IAEpG,IAAI,CAAC,SAAS,EAAE,EAAE,CAAC;QACjB,2EAA2E;QAC3E,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,iBAAiB,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QAChD,CAAC;QACD,WAAW,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;IAC/B,CAAC;AACH,CAAC"}

package/dist/eval-server/concurrency.d.ts

@@ -1,13 +1 @@
-export declare const DEFAULT_CONCURRENCY = 3;
-export declare class Semaphore {
-    private readonly limit;
-    private running;
-    private queue;
-    constructor(limit?: number);
-    acquire(): Promise<void>;
-    release(): void;
-    get available(): number;
-    get pending(): number;
-    get active(): number;
-}
-export declare function getSkillSemaphore(skillKey: string, limit?: number): Semaphore;
+export { Semaphore, DEFAULT_CONCURRENCY, getSkillSemaphore } from "../eval/concurrency.js";

package/dist/eval-server/concurrency.js

@@ -1,52 +1,6 @@
 // ---------------------------------------------------------------------------
-// concurrency.ts -- cooperative semaphore for limiting concurrent LLM calls
+// concurrency.ts -- re-exports from shared eval/concurrency.ts
+// Kept for backward compatibility with eval-server/ imports
 // ---------------------------------------------------------------------------
-export const DEFAULT_CONCURRENCY = 3;
-export class Semaphore {
-    limit;
-    running = 0;
-    queue = [];
-    constructor(limit = DEFAULT_CONCURRENCY) {
-        this.limit = limit;
-        if (limit < 1)
-            throw new Error("Semaphore limit must be >= 1");
-    }
-    async acquire() {
-        if (this.running < this.limit) {
-            this.running++;
-            return;
-        }
-        return new Promise((resolve) => this.queue.push(() => {
-            this.running++;
-            resolve();
-        }));
-    }
-    release() {
-        if (this.running <= 0)
-            return; // idempotent — no underflow
-        this.running--;
-        const next = this.queue.shift();
-        if (next)
-            next();
-    }
-    get available() {
-        return Math.max(0, this.limit - this.running);
-    }
-    get pending() {
-        return this.queue.length;
-    }
-    get active() {
-        return this.running;
-    }
-}
-// Per-skill semaphore registry — ensures one semaphore per skill
-const registry = new Map();
-export function getSkillSemaphore(skillKey, limit = DEFAULT_CONCURRENCY) {
-    let sem = registry.get(skillKey);
-    if (!sem) {
-        sem = new Semaphore(limit);
-        registry.set(skillKey, sem);
-    }
-    return sem;
-}
+export { Semaphore, DEFAULT_CONCURRENCY, getSkillSemaphore } from "../eval/concurrency.js";
 //# sourceMappingURL=concurrency.js.map

package/dist/eval-server/concurrency.js.map

@@ -1 +1 @@
-{"version":3,"file":"concurrency.js","sourceRoot":"","sources":["../../src/eval-server/concurrency.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,4EAA4E;AAC5E,8EAA8E;AAE9E,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,CAAC;AAErC,MAAM,OAAO,SAAS;IAIS;IAHrB,OAAO,GAAG,CAAC,CAAC;IACZ,KAAK,GAAsB,EAAE,CAAC;IAEtC,YAA6B,QAAgB,mBAAmB;QAAnC,UAAK,GAAL,KAAK,CAA8B;QAC9D,IAAI,KAAK,GAAG,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;IACjE,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;YAC9B,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,OAAO;QACT,CAAC;QACD,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE;YACzD,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC,CAAC,CAAC;IACN,CAAC;IAED,OAAO;QACL,IAAI,IAAI,CAAC,OAAO,IAAI,CAAC;YAAE,OAAO,CAAC,4BAA4B;QAC3D,IAAI,CAAC,OAAO,EAAE,CAAC;QACf,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QAChC,IAAI,IAAI;YAAE,IAAI,EAAE,CAAC;IACnB,CAAC;IAED,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC;IAChD,CAAC;IAED,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;IAC3B,CAAC;IAED,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;CACF;AAED,iEAAiE;AACjE,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAqB,CAAC;AAE9C,MAAM,UAAU,iBAAiB,CAAC,QAAgB,EAAE,KAAK,GAAG,mBAAmB;IAC7E,IAAI,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACjC,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,GAAG,GAAG,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC;QAC3B,QAAQ,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IAC9B,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}
+{"version":3,"file":"concurrency.js","sourceRoot":"","sources":["../../src/eval-server/concurrency.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,+DAA+D;AAC/D,4DAA4D;AAC5D,8EAA8E;AAE9E,OAAO,EAAE,SAAS,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC"}

package/dist/eval-server/eval-server.js

@@ -11,6 +11,8 @@ import { registerRoutes } from "./api-routes.js";
 import { registerImproveRoutes } from "./improve-routes.js";
 import { registerModelCompareRoutes } from "./model-compare-routes.js";
 import { registerSkillCreateRoutes } from "./skill-create-routes.js";
+import { registerSweepRoutes } from "./sweep-routes.js";
+import { registerIntegrationRoutes } from "./integration-routes.js";
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 const MIME_TYPES = {
@@ -35,6 +37,8 @@ export async function startEvalServer(opts) {
     registerImproveRoutes(router, root);
     registerModelCompareRoutes(router, root);
     registerSkillCreateRoutes(router, root);
+    registerSweepRoutes(router, root);
+    registerIntegrationRoutes(router, root);
     // Static asset directory
     const staticDir = path.resolve(__dirname, "../eval-ui");
     const server = http.createServer(async (req, res) => {

package/dist/eval-server/eval-server.js.map

@@ -1 +1 @@
-{"version":3,"file":"eval-server.js","sourceRoot":"","sources":["../../src/eval-server/eval-server.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,gDAAgD;AAChD,8EAA8E;AAE9E,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAC5D,OAAO,EAAE,0BAA0B,EAAE,MAAM,2BAA2B,CAAC;AACvE,OAAO,EAAE,yBAAyB,EAAE,MAAM,0BAA0B,CAAC;AAErE,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAClD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;AAE3C,MAAM,UAAU,GAA2B;IACzC,OAAO,EAAE,WAAW;IACpB,KAAK,EAAE,wBAAwB;IAC/B,MAAM,EAAE,UAAU;IAClB,OAAO,EAAE,kBAAkB;IAC3B,MAAM,EAAE,WAAW;IACnB,MAAM,EAAE,eAAe;IACvB,MAAM,EAAE,cAAc;IACtB,OAAO,EAAE,WAAW;IACpB,QAAQ,EAAE,YAAY;IACtB,MAAM,EAAE,UAAU;IAClB,MAAM,EAAE,kBAAkB;IAC1B,OAAO,EAAE,YAAY;CACtB,CAAC;AAQF,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,IAAuB;IAC3D,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;IAC5B,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC;IAE5B,sBAAsB;IACtB,cAAc,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;IAC/C,qBAAqB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IACpC,0BAA0B,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IACzC,yBAAyB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAExC,yBAAyB;IACzB,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;IAExD,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE;QAClD,wBAAwB;QACxB,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC7B,MAAM,SAAS,GAAG,GAAG,CAAC,GAAG,EAAE,UAAU,CAAC,OAAO,CAAC,CAAC;YAC/C,IAAI,SAAS,IAAK,MAAc,CAAC,OAAO,EAAE,CAAC;gBACxC,MAAc,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;gBAClC,OAAO;YACT,CAAC;YACD,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YACnB,GAAG,CAAC,GAAG,EAAE,CAAC;YACV,OAAO;QACT,CAAC;QAED,uBAAuB;QACvB,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QAC9C,IAAI,OAAO;YAAE,OAAO;QAEpB,6BAA6B;QAC7B,IAAI,GAAG,CAAC,GAAG,EAAE,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YACjC,QAAQ,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;YAChD,OAAO;QACT,CAAC;QAED,qBAAqB;QACrB,MAAM,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,SAAS,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,GAAG,EAAE;YACvB,OAAO,CAAC,GAAG,CAAC,sCAAsC,IAAI,IAAI,CAAC,CAAC;YAC5D,OAAO,CAAC,MAAM,CAAC,CAAC;QAClB,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,WAAW,CACxB,GAAyB,EACzB,GAAwB,EACxB,SAAiB;IAEjB,IAAI,OAAO,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,GAAG,EAAE,kBAAkB,CAAC,CAAC,QAAQ,CAAC;IACnE,IAAI,OAAO,KAAK,GAAG;QAAE,OAAO,GAAG,aAAa,CAAC;IAE7C,mCAAmC;IACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC;IACtE,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IAEhD,kCAAkC;IAClC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QACpC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QACnB,GAAG,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACrB,OAAO;IACT,CAAC;IAED,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACnC,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;YAClB,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YACnC,MAAM,WAAW,GAAG,UAAU,CAAC,GAAG,CAAC,IAAI,0BAA0B,CAAC;YAClE,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,WAAW,EAAE,CAAC,CAAC;YACpD,EAAE,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACxC,OAAO;QACT,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,gCAAgC;IAClC,CAAC;IAED,0DAA0D;IAC1D,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;IACrD,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QACpD,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,WAAW,EAAE,CAAC,CAAC;QACpD,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACnB,CAAC;IAAC,MAAM,CAAC;QACP,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QACnB,GAAG,CAAC,GAAG,CAAC,+CAA+C,CAAC,CAAC;IAC3D,CAAC;AACH,CAAC"}
+{"version":3,"file":"eval-server.js","sourceRoot":"","sources":["../../src/eval-server/eval-server.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,gDAAgD;AAChD,8EAA8E;AAE9E,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAC5D,OAAO,EAAE,0BAA0B,EAAE,MAAM,2BAA2B,CAAC;AACvE,OAAO,EAAE,yBAAyB,EAAE,MAAM,0BAA0B,CAAC;AACrE,OAAO,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,EAAE,yBAAyB,EAAE,MAAM,yBAAyB,CAAC;AAEpE,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAClD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;AAE3C,MAAM,UAAU,GAA2B;IACzC,OAAO,EAAE,WAAW;IACpB,KAAK,EAAE,wBAAwB;IAC/B,MAAM,EAAE,UAAU;IAClB,OAAO,EAAE,kBAAkB;IAC3B,MAAM,EAAE,WAAW;IACnB,MAAM,EAAE,eAAe;IACvB,MAAM,EAAE,cAAc;IACtB,OAAO,EAAE,WAAW;IACpB,QAAQ,EAAE,YAAY;IACtB,MAAM,EAAE,UAAU;IAClB,MAAM,EAAE,kBAAkB;IAC1B,OAAO,EAAE,YAAY;CACtB,CAAC;AAQF,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,IAAuB;IAC3D,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;IAC5B,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC;IAE5B,sBAAsB;IACtB,cAAc,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;IAC/C,qBAAqB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IACpC,0BAA0B,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IACzC,yBAAyB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IACxC,mBAAmB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAClC,yBAAyB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAExC,yBAAyB;IACzB,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;IAExD,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE;QAClD,wBAAwB;QACxB,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC7B,MAAM,SAAS,GAAG,GAAG,CAAC,GAAG,EAAE,UAAU,CAAC,OAAO,CAAC,CAAC;YAC/C,IAAI,SAAS,IAAK,MAAc,CAAC,OAAO,EAAE,CAAC;gBACxC,MAAc,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;gBAClC,OAAO;YACT,CAAC;YACD,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YACnB,GAAG,CAAC,GAAG,EAAE,CAAC;YACV,OAAO;QACT,CAAC;QAED,uBAAuB;QACvB,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QAC9C,IAAI,OAAO;YAAE,OAAO;QAEpB,6BAA6B;QAC7B,IAAI,GAAG,CAAC,GAAG,EAAE,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YACjC,QAAQ,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;YAChD,OAAO;QACT,CAAC;QAED,qBAAqB;QACrB,MAAM,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,SAAS,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,GAAG,EAAE;YACvB,OAAO,CAAC,GAAG,CAAC,sCAAsC,IAAI,IAAI,CAAC,CAAC;YAC5D,OAAO,CAAC,MAAM,CAAC,CAAC;QAClB,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,WAAW,CACxB,GAAyB,EACzB,GAAwB,EACxB,SAAiB;IAEjB,IAAI,OAAO,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,GAAG,EAAE,kBAAkB,CAAC,CAAC,QAAQ,CAAC;IACnE,IAAI,OAAO,KAAK,GAAG;QAAE,OAAO,GAAG,aAAa,CAAC;IAE7C,mCAAmC;IACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC;IACtE,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IAEhD,kCAAkC;IAClC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QACpC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QACnB,GAAG,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACrB,OAAO;IACT,CAAC;IAED,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACnC,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;YAClB,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YACnC,MAAM,WAAW,GAAG,UAAU,CAAC,GAAG,CAAC,IAAI,0BAA0B,CAAC;YAClE,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,WAAW,EAAE,CAAC,CAAC;YACpD,EAAE,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACxC,OAAO;QACT,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,gCAAgC;IAClC,CAAC;IAED,0DAA0D;IAC1D,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;IACrD,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QACpD,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,WAAW,EAAE,CAAC,CAAC;QACpD,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACnB,CAAC;IAAC,MAAM,CAAC;QACP,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QACnB,GAAG,CAAC,GAAG,CAAC,+CAA+C,CAAC,CAAC;IAC3D,CAAC;AACH,CAAC"}

package/dist/eval-server/integration-routes.d.ts

@@ -0,0 +1,2 @@
+import type { Router } from "./router.js";
+export declare function registerIntegrationRoutes(router: Router, root: string): void;

package/dist/eval-server/integration-routes.js

@@ -0,0 +1,100 @@
+// ---------------------------------------------------------------------------
+// integration-routes.ts -- API routes for integration tests and credentials
+// ---------------------------------------------------------------------------
+import { sendJson, readBody } from "./router.js";
+import { initSSE, sendSSE, sendSSEDone } from "./sse-helpers.js";
+import { resolveSkillDir } from "./skill-resolver.js";
+import { loadAndValidateEvals } from "../eval/schema.js";
+import { resolveAllCredentials } from "../eval/credential-resolver.js";
+import { runIntegrationCase, recordRun } from "../eval/integration-runner.js";
+export function registerIntegrationRoutes(router, root) {
+    // -------------------------------------------------------------------------
+    // POST /api/skills/:plugin/:skill/integration-run -- SSE integration test
+    // -------------------------------------------------------------------------
+    router.post("/api/skills/:plugin/:skill/integration-run", async (req, res, params) => {
+        const skillDir = resolveSkillDir(root, params.plugin, params.skill);
+        const body = (await readBody(req));
+        initSSE(res, req);
+        try {
+            // Load evals and filter for integration tests
+            const evalsFile = loadAndValidateEvals(skillDir);
+            let integrationCases = evalsFile.evals.filter((e) => e.testType === "integration");
+            // Filter by eval_ids if provided
+            if (body.eval_ids?.length) {
+                const ids = new Set(body.eval_ids);
+                integrationCases = integrationCases.filter((e) => ids.has(e.id));
+            }
+            if (integrationCases.length === 0) {
+                sendSSEDone(res, { status: "no_cases", message: "No integration test cases found" });
+                return;
+            }
+            for (const evalCase of integrationCases) {
+                const integrationCase = {
+                    ...evalCase,
+                    testType: "integration",
+                    cleanup: evalCase.cleanup?.map((c) => ({
+                        type: c.action,
+                        description: c.description ?? c.action,
+                    })),
+                };
+                sendSSE(res, "preflight_start", { evalId: evalCase.id, name: evalCase.name });
+                const result = await runIntegrationCase(integrationCase, {
+                    skillDir,
+                    dryRun: body.dryRun,
+                    confirm: body.confirm,
+                });
+                // Emit phase events
+                for (const phase of result.phases) {
+                    sendSSE(res, phase.phase, {
+                        evalId: evalCase.id,
+                        phase: phase.phase,
+                        status: phase.status,
+                        durationMs: phase.durationMs,
+                        error: phase.errorMessage,
+                    });
+                }
+                // Record run
+                recordRun(skillDir, result);
+            }
+            sendSSEDone(res, { status: "complete" });
+        }
+        catch (err) {
+            sendSSE(res, "error", { error: err.message });
+            sendSSEDone(res, { status: "error", error: err.message });
+        }
+    });
+    // -------------------------------------------------------------------------
+    // GET /api/credentials/:plugin/:skill -- credential status
+    // -------------------------------------------------------------------------
+    router.get("/api/credentials/:plugin/:skill", async (req, res, params) => {
+        const skillDir = resolveSkillDir(root, params.plugin, params.skill);
+        try {
+            // Load evals and collect all requiredCredentials from integration tests
+            const evalsFile = loadAndValidateEvals(skillDir);
+            const allCreds = new Set();
+            for (const evalCase of evalsFile.evals) {
+                if (evalCase.testType === "integration" && evalCase.requiredCredentials) {
+                    for (const cred of evalCase.requiredCredentials) {
+                        allCreds.add(cred);
+                    }
+                }
+            }
+            const names = [...allCreds].sort();
+            if (names.length === 0) {
+                sendJson(res, { credentials: [] }, 200, req);
+                return;
+            }
+            const statuses = resolveAllCredentials(names, skillDir);
+            const credentials = statuses.map((s) => ({
+                name: s.name,
+                status: s.status,
+                ...(s.source ? { source: s.source } : {}),
+            }));
+            sendJson(res, { credentials }, 200, req);
+        }
+        catch (err) {
+            sendJson(res, { error: err.message, credentials: [] }, 400, req);
+        }
+    });
+}
+//# sourceMappingURL=integration-routes.js.map

package/dist/eval-server/integration-routes.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"integration-routes.js","sourceRoot":"","sources":["../../src/eval-server/integration-routes.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,4EAA4E;AAC5E,8EAA8E;AAI9E,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AACjE,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AACzD,OAAO,EAAE,qBAAqB,EAAE,MAAM,gCAAgC,CAAC;AACvE,OAAO,EAAE,kBAAkB,EAAc,SAAS,EAAsB,MAAM,+BAA+B,CAAC;AAG9G,MAAM,UAAU,yBAAyB,CAAC,MAAc,EAAE,IAAY;IACpE,4EAA4E;IAC5E,0EAA0E;IAC1E,4EAA4E;IAC5E,MAAM,CAAC,IAAI,CAAC,4CAA4C,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE;QACnF,MAAM,QAAQ,GAAG,eAAe,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;QACpE,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,GAAG,CAAC,CAIhC,CAAC;QAEF,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QAElB,IAAI,CAAC;YACH,8CAA8C;YAC9C,MAAM,SAAS,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAC;YACjD,IAAI,gBAAgB,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,aAAa,CAAC,CAAC;YAEnF,iCAAiC;YACjC,IAAI,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC;gBAC1B,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACnC,gBAAgB,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACnE,CAAC;YAED,IAAI,gBAAgB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAClC,WAAW,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,iCAAiC,EAAE,CAAC,CAAC;gBACrF,OAAO;YACT,CAAC;YAED,KAAK,MAAM,QAAQ,IAAI,gBAAgB,EAAE,CAAC;gBACxC,MAAM,eAAe,GAAwB;oBAC3C,GAAG,QAAQ;oBACX,QAAQ,EAAE,aAAa;oBACvB,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;wBACrC,IAAI,EAAE,CAAC,CAAC,MAAM;wBACd,WAAW,EAAE,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC,MAAM;qBACvC,CAAC,CAAC;iBACJ,CAAC;gBAEF,OAAO,CAAC,GAAG,EAAE,iBAAiB,EAAE,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;gBAE9E,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,eAAe,EAAE;oBACvD,QAAQ;oBACR,MAAM,EAAE,IAAI,CAAC,MAAM;oBACnB,OAAO,EAAE,IAAI,CAAC,OAAO;iBACtB,CAAC,CAAC;gBAEH,oBAAoB;gBACpB,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;oBAClC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,KAAK,EAAE;wBACxB,MAAM,EAAE,QAAQ,CAAC,EAAE;wBACnB,KAAK,EAAE,KAAK,CAAC,KAAK;wBAClB,MAAM,EAAE,KAAK,CAAC,MAAM;wBACpB,UAAU,EAAE,KAAK,CAAC,UAAU;wBAC5B,KAAK,EAAE,KAAK,CAAC,YAAY;qBAC1B,CAAC,CAAC;gBACL,CAAC;gBAED,aAAa;gBACb,SAAS,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAC9B,CAAC;YAED,WAAW,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;QAC3C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,KAAK,EAAG,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;YACzD,WAAW,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAG,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;QACvE,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,4EAA4E;IAC5E,2DAA2D;IAC3D,4EAA4E;IAC5E,MAAM,CAAC,GAAG,CAAC,iCAAiC,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE;QACvE,MAAM,QAAQ,GAAG,eAAe,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;QAEpE,IAAI,CAAC;YACH,wEAAwE;YACxE,MAAM,SAAS,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAC;YACjD,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;YACnC,KAAK,MAAM,QAAQ,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;gBACvC,IAAI,QAAQ,CAAC,QAAQ,KAAK,aAAa,IAAI,QAAQ,CAAC,mBAAmB,EAAE,CAAC;oBACxE,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,mBAAmB,EAAE,CAAC;wBAChD,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;oBACrB,CAAC;gBACH,CAAC;YACH,CAAC;YAED,MAAM,KAAK,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC;YACnC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACvB,QAAQ,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;gBAC7C,OAAO;YACT,CAAC;YAED,MAAM,QAAQ,GAAG,qBAAqB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;YACxD,MAAM,WAAW,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACvC,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,MAAM,EAAE,CAAC,CAAC,MAAM;gBAChB,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC1C,CAAC,CAAC,CAAC;YAEJ,QAAQ,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;QAC3C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,QAAQ,CAAC,GAAG,EAAE,EAAE,KAAK,EAAG,GAAa,CAAC,OAAO,EAAE,WAAW,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;QAC9E,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC"}