vskill 0.5.12 → 0.5.13

package/dist/eval/llm.js

@@ -49,6 +49,7 @@ export function estimateDurationSec(provider, totalCases, totalAssertions) {
         "codex-cli": [8, 20],
         "gemini-cli": [8, 20],
         "ollama": [5, 30],
+        "openrouter": [4, 15],
     };
     const [lo, hi] = perCall[provider] ?? [5, 20];
     const totalCalls = totalCases + totalAssertions; // 1 generate + N judges
@@ -72,8 +73,10 @@ export function createLlmClient(overrides) {
             return createGeminiCliClient(modelOverride);
         case "ollama":
             return createOllamaClient(modelOverride);
+        case "openrouter":
+            return createOpenRouterClient(modelOverride);
         default:
-            throw new Error(`Unknown VSKILL_EVAL_PROVIDER: "${provider}". Use "claude-cli", "codex-cli", "gemini-cli", "anthropic", or "ollama".`);
+            throw new Error(`Unknown VSKILL_EVAL_PROVIDER: "${provider}". Use "claude-cli", "codex-cli", "gemini-cli", "anthropic", "ollama", or "openrouter".`);
     }
 }
 // ---------------------------------------------------------------------------
@@ -112,6 +115,7 @@ function createAnthropicClient(modelOverride) {
                     durationMs,
                     inputTokens: response.usage?.input_tokens ?? null,
                     outputTokens: response.usage?.output_tokens ?? null,
+                    cost: null,
                 };
             }
             finally {
@@ -182,7 +186,7 @@ function createCliClient(config) {
                 });
                 proc.stdin.end(combinedPrompt);
             });
-            return { text, durationMs: Date.now() - start, inputTokens: null, outputTokens: null };
+            return { text, durationMs: Date.now() - start, inputTokens: null, outputTokens: null, cost: null };
         },
     };
 }
@@ -266,6 +270,54 @@ function createOllamaClient(modelOverride) {
                 durationMs: Date.now() - start,
                 inputTokens: data.prompt_eval_count ?? null,
                 outputTokens: data.eval_count ?? null,
+                cost: null,
+            };
+        },
+    };
+}
+// ---------------------------------------------------------------------------
+// Provider: OpenRouter (100+ models via single API key)
+// ---------------------------------------------------------------------------
+function createOpenRouterClient(modelOverride) {
+    const apiKey = process.env.OPENROUTER_API_KEY;
+    if (!apiKey) {
+        throw new Error("OPENROUTER_API_KEY is not set. Visit https://openrouter.ai/keys to get your key, then run:\n  export OPENROUTER_API_KEY=<your-key>");
+    }
+    const model = modelOverride || process.env.VSKILL_EVAL_MODEL || "anthropic/claude-sonnet-4";
+    return {
+        model,
+        async generate(systemPrompt, userPrompt) {
+            const start = Date.now();
+            const response = await fetch("https://openrouter.ai/api/v1/chat/completions", {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    "Authorization": `Bearer ${apiKey}`,
+                    "HTTP-Referer": "https://verified-skill.com",
+                    "X-Title": "vSkill Studio",
+                },
+                body: JSON.stringify({
+                    model,
+                    messages: [
+                        { role: "system", content: systemPrompt },
+                        { role: "user", content: userPrompt },
+                    ],
+                    max_tokens: 4096,
+                }),
+                signal: AbortSignal.timeout(getTimeoutMs()),
+            });
+            if (!response.ok) {
+                const error = await response.text();
+                throw new Error(`OpenRouter request failed (${response.status}): ${error}`);
+            }
+            const data = (await response.json());
+            const text = data.choices?.[0]?.message?.content || "";
+            return {
+                text,
+                durationMs: Date.now() - start,
+                inputTokens: data.usage?.prompt_tokens ?? null,
+                outputTokens: data.usage?.completion_tokens ?? null,
+                cost: data.usage?.total_cost ?? null,
             };
         },
     };

package/dist/eval/llm.js.map

@@ -1 +1 @@
-{"version":3,"file":"llm.js","sourceRoot":"","sources":["../../src/eval/llm.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,+EAA+E;AAC/E,EAAE;AACF,uDAAuD;AACvD,wEAAwE;AACxE,iFAAiF;AACjF,mEAAmE;AACnE,8DAA8D;AAC9D,uEAAuE;AACvE,EAAE;AACF,uDAAuD;AACvD,iFAAiF;AACjF,2EAA2E;AAC3E,EAAE;AACF,mEAAmE;AACnE,oEAAoE;AACpE,EAAE;AACF,iDAAiD;AACjD,+DAA+D;AAC/D,4EAA4E;AAC5E,iFAAiF;AACjF,4DAA4D;AAC5D,mDAAmD;AACnD,8EAA8E;AAE9E,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAgB5E,SAAS,cAAc;IACrB,OAAO,YAAY,CAAC;AACtB,CAAC;AAED,oFAAoF;AACpF,SAAS,YAAY;IACnB,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC;IAC/C,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACrC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,CAAC;YAAE,OAAO,OAAO,GAAG,IAAI,CAAC;IAC5D,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAOD;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CACjC,QAAsB,EACtB,UAAkB,EAClB,eAAuB;IAEvB,+CAA+C;IAC/C,MAAM,OAAO,GAA2C;QACtD,YAAY,EAAG,CAAC,EAAE,EAAE,EAAE,CAAC;QACvB,WAAW,EAAI,CAAC,CAAC,EAAE,EAAE,CAAC;QACtB,WAAW,EAAI,CAAC,CAAC,EAAE,EAAE,CAAC;QACtB,YAAY,EAAG,CAAC,CAAC,EAAE,EAAE,CAAC;QACtB,QAAQ,EAAO,CAAC,CAAC,EAAE,EAAE,CAAC;KACvB,CAAC;IACF,MAAM,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC9C,MAAM,UAAU,GAAG,UAAU,GAAG,eAAe,CAAC,CAAC,wBAAwB;IACzE,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC;IAE3C,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC;IACxE,MAAM,KAAK,GAAG,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;IAChF,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;AACnC,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,SAAwB;IACtD,MAAM,QAAQ,GAAG,CAAC,SAAS,EAAE,QAAQ,IAAI,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,cAAc,EAAE,CAAiB,CAAC;IAC/G,MAAM,aAAa,GAAG,SAAS,EAAE,KAAK,CAAC;IACvC,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,WAAW;YACd,OAAO,qBAAqB,CAAC,aAAa,CAAC,CAAC;QAC9C,KAAK,YAAY;YACf,OAAO,qBAAqB,CAAC,aAAa,CAAC,CAAC;QAC9C,KAAK,WAAW;YACd,OAAO,oBAAoB,CAAC,aAAa,CAAC,CAAC;QAC7C,KAAK,YAAY;YACf,OAAO,qBAAqB,CAAC,aAAa,CAAC,CAAC;QAC9C,KAAK,QAAQ;YACX,OAAO,kBAAkB,CAAC,aAAa,CAAC,CAAC;QAC3C;YACE,MAAM,IAAI,KAAK,CACb,kCAAkC,QAAQ,2EAA2E,CACtH,CAAC;IACN,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,0BAA0B;AAC1B,8EAA8E;AAC9E,SAAS,qBAAqB,CAAC,aAAsB;IACnD,MAAM,aAAa,GAAG,mBAAmB,CAAC;IAE1C,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;IAC7C,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CACb,sRAAsR,CACvR,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,aAAa,CAAC;IAC9E,IAAI,cAAc,GAAQ,IAAI,CAAC;IAE/B,OAAO;QACL,KAAK;QACL,KAAK,CAAC,QAAQ,CAAC,YAAoB,EAAE,UAAkB;YACrD,IAAI,CAAC,cAAc,EAAE,CAAC;gBACpB,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,CAAC;gBACjE,cAAc,GAAG,IAAI,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;YAC7C,CAAC;YAED,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;YACzC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,YAAY,EAAE,CAAC,CAAC;YACrE,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACzB,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,QAAQ,CAAC,MAAM,CACnD;oBACE,KAAK;oBACL,MAAM,EAAE,YAAY;oBACpB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;oBACjD,UAAU,EAAE,IAAI;iBACjB,EACD,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAC9B,CAAC;gBACF,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;gBAEtC,MAAM,SAAS,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;gBACvE,MAAM,IAAI,GAAG,SAAS,IAAI,MAAM,IAAI,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;gBACpE,OAAO;oBACL,IAAI;oBACJ,UAAU;oBACV,WAAW,EAAE,QAAQ,CAAC,KAAK,EAAE,YAAY,IAAI,IAAI;oBACjD,YAAY,EAAE,QAAQ,CAAC,KAAK,EAAE,aAAa,IAAI,IAAI;iBACpD,CAAC;YACJ,CAAC;oBAAS,CAAC;gBACT,YAAY,CAAC,OAAO,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC;AAiBD,SAAS,eAAe,CAAC,MAAiB;IACxC,uEAAuE;IACvE,MAAM,cAAc,GAAG,gBAAgB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACvD,gFAAgF;IAChF,MAAM,kBAAkB,GAAG,YAAY,EAAE,CAAC;IAE1C,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,YAAY;QAC1B,KAAK,CAAC,QAAQ,CAAC,YAAoB,EAAE,UAAkB;YACrD,MAAM,cAAc,GAAG,GAAG,YAAY,OAAO,UAAU,EAAE,CAAC;YAC1D,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAEzB,MAAM,IAAI,GAAG,MAAM,IAAI,OAAO,CAAS,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBACzD,mEAAmE;gBACnE,IAAI,GAA2B,CAAC;gBAChC,IAAI,MAAM,CAAC,cAAc,EAAE,CAAC;oBAC1B,GAAG,GAAG,EAAE,CAAC;oBACT,MAAM,MAAM,GAAG,MAAM,CAAC,cAAc,CAAC;oBACrC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;wBACjD,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;4BAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;oBAC3D,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,GAAG,GAAG,EAAE,GAAG,OAAO,CAAC,GAA6B,EAAE,CAAC;gBACrD,CAAC;gBACD,GAAG,CAAC,IAAI,GAAG,kBAAkB,CAAC;gBAE9B,sEAAsE;gBACtE,MAAM,UAAU,GAAG,OAAO,CAAC,QAAQ,KAAK,OAAO,IAAI,eAAe,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;gBAExF,MAAM,IAAI,GAAG,KAAK,CAAC,cAAc,EAAE,MAAM,CAAC,IAAI,EAAE;oBAC9C,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;oBAC/B,GAAG;oBACH,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBACvC,CAAC,CAAC;gBAEH,IAAI,MAAM,GAAG,EAAE,CAAC;gBAChB,IAAI,MAAM,GAAG,EAAE,CAAC;gBAChB,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAS,EAAE,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;gBACnE,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAS,EAAE,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;gBAEnE,MAAM,SAAS,GAAG,YAAY,EAAE,CAAC;gBACjC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;oBAC5B,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBACrB,MAAM,CAAC,IAAI,KAAK,CAAC,GAAG,MAAM,CAAC,IAAI,wBAAwB,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC;gBAC/E,CAAC,EAAE,SAAS,CAAC,CAAC;gBAEd,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAA0B,EAAE,EAAE;oBAC9C,YAAY,CAAC,KAAK,CAAC,CAAC;oBACpB,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;wBAC1B,MAAM,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;oBACxC,CAAC;yBAAM,CAAC;wBACN,MAAM,CAAC,IAAI,KAAK,CAAC,GAAG,MAAM,CAAC,IAAI,gBAAgB,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;oBACjE,CAAC;gBACH,CAAC,CAAC,CAAC;gBAEH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;oBACxB,YAAY,CAAC,KAAK,CAAC,CAAC;oBACpB,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;wBACf,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;oBACzB,CAAC;yBAAM,CAAC;wBACN,MAAM,MAAM,GAAG,CAAC,MAAM,IAAI,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;wBAChD,MAAM,CAAC,IAAI,KAAK,CACd,GAAG,MAAM,CAAC,IAAI,yBAAyB,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAC5E,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC,CAAC,CAAC;gBAEH,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;YACjC,CAAC,CAAC,CAAC;YAEH,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;QACzF,CAAC;KACF,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,4EAA4E;AAC5E,uEAAuE;AACvE,8EAA8E;AAC9E,SAAS,qBAAqB,CAAC,aAAsB;IACnD,MAAM,KAAK,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,QAAQ,CAAC;IACzE,OAAO,eAAe,CAAC;QACrB,MAAM,EAAE,QAAQ;QAChB,IAAI,EAAE,QAAQ;QACd,IAAI,EAAE,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC;QAC9B,YAAY,EAAE,UAAU,KAAK,EAAE;QAC/B,cAAc,EAAE,QAAQ;QACxB,WAAW,EACT,qJAAqJ;KACxJ,CAAC,CAAC;AACL,CAAC;AAED,8EAA8E;AAC9E,iFAAiF;AACjF,8EAA8E;AAC9E,SAAS,oBAAoB,CAAC,aAAsB;IAClD,MAAM,KAAK,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,SAAS,CAAC;IAC1E,OAAO,eAAe,CAAC;QACrB,MAAM,EAAE,OAAO;QACf,IAAI,EAAE,OAAO;QACb,IAAI,EAAE,CAAC,MAAM,EAAE,SAAS,EAAE,KAAK,CAAC;QAChC,YAAY,EAAE,SAAS,KAAK,EAAE;QAC9B,WAAW,EACT,4IAA4I;KAC/I,CAAC,CAAC;AACL,CAAC;AAED,8EAA8E;AAC9E,iFAAiF;AACjF,kFAAkF;AAClF,8EAA8E;AAC9E,SAAS,qBAAqB,CAAC,aAAsB;IACnD,MAAM,KAAK,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,gBAAgB,CAAC;IACjF,OAAO,eAAe,CAAC;QACrB,MAAM,EAAE,QAAQ;QAChB,IAAI,EAAE,QAAQ;QACd,IAAI,EAAE,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC;QAC9B,YAAY,EAAE,KAAK;QACnB,WAAW,EACT,kJAAkJ;KACrJ,CAAC,CAAC;AACL,CAAC;AAED,8EAA8E;AAC9E,qDAAqD;AACrD,8EAA8E;AAC9E,SAAS,kBAAkB,CAAC,aAAsB;IAChD,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,wBAAwB,CAAC;IACxE,MAAM,KAAK,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,aAAa,CAAC;IAE9E,OAAO;QACL,KAAK;QACL,KAAK,CAAC,QAAQ,CAAC,YAAoB,EAAE,UAAkB;YACrD,MAAM,UAAU,GAAG,GAAG,YAAY,OAAO,UAAU,EAAE,CAAC;YACtD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAEzB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,OAAO,eAAe,EAAE;gBACtD,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;gBAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,KAAK;oBACL,MAAM,EAAE,UAAU;oBAClB,MAAM,EAAE,KAAK;oBACb,OAAO,EAAE;wBACP,WAAW,EAAE,IAAI;wBACjB,WAAW,EAAE,GAAG;qBACjB;iBACF,CAAC;gBACF,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC;aAC5C,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACpC,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;oBAC3D,MAAM,IAAI,KAAK,CACb,iBAAiB,KAAK,8CAA8C,KAAK,EAAE,CAC5E,CAAC;gBACJ,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,0BAA0B,KAAK,EAAE,CAAC,CAAC;YACrD,CAAC;YAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAIlC,CAAC;YACF,OAAO;gBACL,IAAI,EAAE,IAAI,CAAC,QAAQ,IAAI,EAAE;gBACzB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;gBAC9B,WAAW,EAAE,IAAI,CAAC,iBAAiB,IAAI,IAAI;gBAC3C,YAAY,EAAE,IAAI,CAAC,UAAU,IAAI,IAAI;aACtC,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC"}
+{"version":3,"file":"llm.js","sourceRoot":"","sources":["../../src/eval/llm.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,+EAA+E;AAC/E,EAAE;AACF,uDAAuD;AACvD,wEAAwE;AACxE,iFAAiF;AACjF,mEAAmE;AACnE,8DAA8D;AAC9D,uEAAuE;AACvE,EAAE;AACF,uDAAuD;AACvD,iFAAiF;AACjF,2EAA2E;AAC3E,EAAE;AACF,mEAAmE;AACnE,oEAAoE;AACpE,EAAE;AACF,iDAAiD;AACjD,+DAA+D;AAC/D,4EAA4E;AAC5E,iFAAiF;AACjF,4DAA4D;AAC5D,mDAAmD;AACnD,8EAA8E;AAE9E,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAiB5E,SAAS,cAAc;IACrB,OAAO,YAAY,CAAC;AACtB,CAAC;AAED,oFAAoF;AACpF,SAAS,YAAY;IACnB,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC;IAC/C,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACrC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,CAAC;YAAE,OAAO,OAAO,GAAG,IAAI,CAAC;IAC5D,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAOD;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CACjC,QAAsB,EACtB,UAAkB,EAClB,eAAuB;IAEvB,+CAA+C;IAC/C,MAAM,OAAO,GAA2C;QACtD,YAAY,EAAG,CAAC,EAAE,EAAE,EAAE,CAAC;QACvB,WAAW,EAAI,CAAC,CAAC,EAAE,EAAE,CAAC;QACtB,WAAW,EAAI,CAAC,CAAC,EAAE,EAAE,CAAC;QACtB,YAAY,EAAG,CAAC,CAAC,EAAE,EAAE,CAAC;QACtB,QAAQ,EAAO,CAAC,CAAC,EAAE,EAAE,CAAC;QACtB,YAAY,EAAG,CAAC,CAAC,EAAE,EAAE,CAAC;KACvB,CAAC;IACF,MAAM,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC9C,MAAM,UAAU,GAAG,UAAU,GAAG,eAAe,CAAC,CAAC,wBAAwB;IACzE,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC;IAE3C,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC;IACxE,MAAM,KAAK,GAAG,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;IAChF,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;AACnC,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,SAAwB;IACtD,MAAM,QAAQ,GAAG,CAAC,SAAS,EAAE,QAAQ,IAAI,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,cAAc,EAAE,CAAiB,CAAC;IAC/G,MAAM,aAAa,GAAG,SAAS,EAAE,KAAK,CAAC;IACvC,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,WAAW;YACd,OAAO,qBAAqB,CAAC,aAAa,CAAC,CAAC;QAC9C,KAAK,YAAY;YACf,OAAO,qBAAqB,CAAC,aAAa,CAAC,CAAC;QAC9C,KAAK,WAAW;YACd,OAAO,oBAAoB,CAAC,aAAa,CAAC,CAAC;QAC7C,KAAK,YAAY;YACf,OAAO,qBAAqB,CAAC,aAAa,CAAC,CAAC;QAC9C,KAAK,QAAQ;YACX,OAAO,kBAAkB,CAAC,aAAa,CAAC,CAAC;QAC3C,KAAK,YAAY;YACf,OAAO,sBAAsB,CAAC,aAAa,CAAC,CAAC;QAC/C;YACE,MAAM,IAAI,KAAK,CACb,kCAAkC,QAAQ,yFAAyF,CACpI,CAAC;IACN,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,0BAA0B;AAC1B,8EAA8E;AAC9E,SAAS,qBAAqB,CAAC,aAAsB;IACnD,MAAM,aAAa,GAAG,mBAAmB,CAAC;IAE1C,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;IAC7C,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CACb,sRAAsR,CACvR,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,aAAa,CAAC;IAC9E,IAAI,cAAc,GAAQ,IAAI,CAAC;IAE/B,OAAO;QACL,KAAK;QACL,KAAK,CAAC,QAAQ,CAAC,YAAoB,EAAE,UAAkB;YACrD,IAAI,CAAC,cAAc,EAAE,CAAC;gBACpB,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,CAAC;gBACjE,cAAc,GAAG,IAAI,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;YAC7C,CAAC;YAED,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;YACzC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,YAAY,EAAE,CAAC,CAAC;YACrE,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACzB,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,QAAQ,CAAC,MAAM,CACnD;oBACE,KAAK;oBACL,MAAM,EAAE,YAAY;oBACpB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;oBACjD,UAAU,EAAE,IAAI;iBACjB,EACD,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAC9B,CAAC;gBACF,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;gBAEtC,MAAM,SAAS,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;gBACvE,MAAM,IAAI,GAAG,SAAS,IAAI,MAAM,IAAI,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;gBACpE,OAAO;oBACL,IAAI;oBACJ,UAAU;oBACV,WAAW,EAAE,QAAQ,CAAC,KAAK,EAAE,YAAY,IAAI,IAAI;oBACjD,YAAY,EAAE,QAAQ,CAAC,KAAK,EAAE,aAAa,IAAI,IAAI;oBACnD,IAAI,EAAE,IAAI;iBACX,CAAC;YACJ,CAAC;oBAAS,CAAC;gBACT,YAAY,CAAC,OAAO,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC;AAiBD,SAAS,eAAe,CAAC,MAAiB;IACxC,uEAAuE;IACvE,MAAM,cAAc,GAAG,gBAAgB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACvD,gFAAgF;IAChF,MAAM,kBAAkB,GAAG,YAAY,EAAE,CAAC;IAE1C,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,YAAY;QAC1B,KAAK,CAAC,QAAQ,CAAC,YAAoB,EAAE,UAAkB;YACrD,MAAM,cAAc,GAAG,GAAG,YAAY,OAAO,UAAU,EAAE,CAAC;YAC1D,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAEzB,MAAM,IAAI,GAAG,MAAM,IAAI,OAAO,CAAS,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBACzD,mEAAmE;gBACnE,IAAI,GAA2B,CAAC;gBAChC,IAAI,MAAM,CAAC,cAAc,EAAE,CAAC;oBAC1B,GAAG,GAAG,EAAE,CAAC;oBACT,MAAM,MAAM,GAAG,MAAM,CAAC,cAAc,CAAC;oBACrC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;wBACjD,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;4BAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;oBAC3D,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,GAAG,GAAG,EAAE,GAAG,OAAO,CAAC,GAA6B,EAAE,CAAC;gBACrD,CAAC;gBACD,GAAG,CAAC,IAAI,GAAG,kBAAkB,CAAC;gBAE9B,sEAAsE;gBACtE,MAAM,UAAU,GAAG,OAAO,CAAC,QAAQ,KAAK,OAAO,IAAI,eAAe,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;gBAExF,MAAM,IAAI,GAAG,KAAK,CAAC,cAAc,EAAE,MAAM,CAAC,IAAI,EAAE;oBAC9C,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;oBAC/B,GAAG;oBACH,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBACvC,CAAC,CAAC;gBAEH,IAAI,MAAM,GAAG,EAAE,CAAC;gBAChB,IAAI,MAAM,GAAG,EAAE,CAAC;gBAChB,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAS,EAAE,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;gBACnE,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAS,EAAE,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;gBAEnE,MAAM,SAAS,GAAG,YAAY,EAAE,CAAC;gBACjC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;oBAC5B,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBACrB,MAAM,CAAC,IAAI,KAAK,CAAC,GAAG,MAAM,CAAC,IAAI,wBAAwB,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC;gBAC/E,CAAC,EAAE,SAAS,CAAC,CAAC;gBAEd,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAA0B,EAAE,EAAE;oBAC9C,YAAY,CAAC,KAAK,CAAC,CAAC;oBACpB,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;wBAC1B,MAAM,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;oBACxC,CAAC;yBAAM,CAAC;wBACN,MAAM,CAAC,IAAI,KAAK,CAAC,GAAG,MAAM,CAAC,IAAI,gBAAgB,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;oBACjE,CAAC;gBACH,CAAC,CAAC,CAAC;gBAEH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;oBACxB,YAAY,CAAC,KAAK,CAAC,CAAC;oBACpB,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;wBACf,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;oBACzB,CAAC;yBAAM,CAAC;wBACN,MAAM,MAAM,GAAG,CAAC,MAAM,IAAI,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;wBAChD,MAAM,CAAC,IAAI,KAAK,CACd,GAAG,MAAM,CAAC,IAAI,yBAAyB,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAC5E,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC,CAAC,CAAC;gBAEH,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;YACjC,CAAC,CAAC,CAAC;YAEH,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QACrG,CAAC;KACF,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,4EAA4E;AAC5E,uEAAuE;AACvE,8EAA8E;AAC9E,SAAS,qBAAqB,CAAC,aAAsB;IACnD,MAAM,KAAK,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,QAAQ,CAAC;IACzE,OAAO,eAAe,CAAC;QACrB,MAAM,EAAE,QAAQ;QAChB,IAAI,EAAE,QAAQ;QACd,IAAI,EAAE,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC;QAC9B,YAAY,EAAE,UAAU,KAAK,EAAE;QAC/B,cAAc,EAAE,QAAQ;QACxB,WAAW,EACT,qJAAqJ;KACxJ,CAAC,CAAC;AACL,CAAC;AAED,8EAA8E;AAC9E,iFAAiF;AACjF,8EAA8E;AAC9E,SAAS,oBAAoB,CAAC,aAAsB;IAClD,MAAM,KAAK,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,SAAS,CAAC;IAC1E,OAAO,eAAe,CAAC;QACrB,MAAM,EAAE,OAAO;QACf,IAAI,EAAE,OAAO;QACb,IAAI,EAAE,CAAC,MAAM,EAAE,SAAS,EAAE,KAAK,CAAC;QAChC,YAAY,EAAE,SAAS,KAAK,EAAE;QAC9B,WAAW,EACT,4IAA4I;KAC/I,CAAC,CAAC;AACL,CAAC;AAED,8EAA8E;AAC9E,iFAAiF;AACjF,kFAAkF;AAClF,8EAA8E;AAC9E,SAAS,qBAAqB,CAAC,aAAsB;IACnD,MAAM,KAAK,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,gBAAgB,CAAC;IACjF,OAAO,eAAe,CAAC;QACrB,MAAM,EAAE,QAAQ;QAChB,IAAI,EAAE,QAAQ;QACd,IAAI,EAAE,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC;QAC9B,YAAY,EAAE,KAAK;QACnB,WAAW,EACT,kJAAkJ;KACrJ,CAAC,CAAC;AACL,CAAC;AAED,8EAA8E;AAC9E,qDAAqD;AACrD,8EAA8E;AAC9E,SAAS,kBAAkB,CAAC,aAAsB;IAChD,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,wBAAwB,CAAC;IACxE,MAAM,KAAK,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,aAAa,CAAC;IAE9E,OAAO;QACL,KAAK;QACL,KAAK,CAAC,QAAQ,CAAC,YAAoB,EAAE,UAAkB;YACrD,MAAM,UAAU,GAAG,GAAG,YAAY,OAAO,UAAU,EAAE,CAAC;YACtD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAEzB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,OAAO,eAAe,EAAE;gBACtD,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;gBAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,KAAK;oBACL,MAAM,EAAE,UAAU;oBAClB,MAAM,EAAE,KAAK;oBACb,OAAO,EAAE;wBACP,WAAW,EAAE,IAAI;wBACjB,WAAW,EAAE,GAAG;qBACjB;iBACF,CAAC;gBACF,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC;aAC5C,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACpC,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;oBAC3D,MAAM,IAAI,KAAK,CACb,iBAAiB,KAAK,8CAA8C,KAAK,EAAE,CAC5E,CAAC;gBACJ,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,0BAA0B,KAAK,EAAE,CAAC,CAAC;YACrD,CAAC;YAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAIlC,CAAC;YACF,OAAO;gBACL,IAAI,EAAE,IAAI,CAAC,QAAQ,IAAI,EAAE;gBACzB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;gBAC9B,WAAW,EAAE,IAAI,CAAC,iBAAiB,IAAI,IAAI;gBAC3C,YAAY,EAAE,IAAI,CAAC,UAAU,IAAI,IAAI;gBACrC,IAAI,EAAE,IAAI;aACX,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,wDAAwD;AACxD,8EAA8E;AAC9E,SAAS,sBAAsB,CAAC,aAAsB;IACpD,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC;IAC9C,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CACb,oIAAoI,CACrI,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,2BAA2B,CAAC;IAE5F,OAAO;QACL,KAAK;QACL,KAAK,CAAC,QAAQ,CAAC,YAAoB,EAAE,UAAkB;YACrD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAEzB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,+CAA+C,EAAE;gBAC5E,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,eAAe,EAAE,UAAU,MAAM,EAAE;oBACnC,cAAc,EAAE,4BAA4B;oBAC5C,SAAS,EAAE,eAAe;iBAC3B;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,KAAK;oBACL,QAAQ,EAAE;wBACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,EAAE;wBACzC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE;qBACtC;oBACD,UAAU,EAAE,IAAI;iBACjB,CAAC;gBACF,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC;aAC5C,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACpC,MAAM,IAAI,KAAK,CAAC,8BAA8B,QAAQ,CAAC,MAAM,MAAM,KAAK,EAAE,CAAC,CAAC;YAC9E,CAAC;YAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAOlC,CAAC;YAEF,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,IAAI,EAAE,CAAC;YACvD,OAAO;gBACL,IAAI;gBACJ,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;gBAC9B,WAAW,EAAE,IAAI,CAAC,KAAK,EAAE,aAAa,IAAI,IAAI;gBAC9C,YAAY,EAAE,IAAI,CAAC,KAAK,EAAE,iBAAiB,IAAI,IAAI;gBACnD,IAAI,EAAE,IAAI,CAAC,KAAK,EAAE,UAAU,IAAI,IAAI;aACrC,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC"}

package/dist/eval/prompt-builder.d.ts

@@ -1,4 +1,14 @@
 import type { EvalsFile } from "./schema.js";
+import type { McpDependency } from "./mcp-detector.js";
+export interface BrowserRequirements {
+    hasBrowser: boolean;
+    suggestedProfile?: string;
+}
+export declare function detectBrowserRequirements(skillContent: string): BrowserRequirements;
+export declare function detectPlatformTargets(skillContent: string): string[];
+export declare const CREDENTIAL_HINTS: Record<string, string[]>;
+export declare function buildIntegrationEvalPrompt(skillContent: string, mcpDeps: McpDependency[], browserReqs: BrowserRequirements, platforms: string[]): string;
+export declare function parseGeneratedIntegrationEvals(raw: string): EvalsFile["evals"];
 export declare function buildEvalSystemPrompt(skillContent: string): string;
 export declare function buildBaselineSystemPrompt(): string;
 export declare function buildEvalInitPrompt(skillContent: string): string;

package/dist/eval/prompt-builder.js

@@ -101,6 +101,173 @@ function detectBashTools(skillContent) {
     const tools = allowedToolsMatch[1].toLowerCase();
     return tools.includes("bash") || tools.includes("read") || tools.includes("write") || tools.includes("edit");
 }
+export function detectBrowserRequirements(skillContent) {
+    const hasBash = detectBashTools(skillContent);
+    if (!hasBash)
+        return { hasBrowser: false };
+    const body = skillContent.replace(/^---\s*\n[\s\S]*?\n---/, "");
+    const browserPattern = /\b(browser|chrome|chromium|playwright|puppeteer|selenium|chrome[\s-]?profile)\b/i;
+    if (browserPattern.test(body)) {
+        return { hasBrowser: true, suggestedProfile: "Default" };
+    }
+    return { hasBrowser: false };
+}
+const PLATFORM_KEYWORDS = {
+    twitter: "Twitter",
+    "x.com": "Twitter",
+    "\\bx\\b": "Twitter",
+    linkedin: "LinkedIn",
+    slack: "Slack",
+    instagram: "Instagram",
+    youtube: "YouTube",
+    tiktok: "TikTok",
+    reddit: "Reddit",
+    facebook: "Facebook",
+    discord: "Discord",
+    telegram: "Telegram",
+    threads: "Threads",
+    "dev.to": "dev.to",
+};
+export function detectPlatformTargets(skillContent) {
+    const body = skillContent.replace(/^---\s*\n[\s\S]*?\n---/, "");
+    const detected = new Set();
+    for (const [keyword, platform] of Object.entries(PLATFORM_KEYWORDS)) {
+        const regex = new RegExp(keyword, "i");
+        if (regex.test(body)) {
+            detected.add(platform);
+        }
+    }
+    return Array.from(detected);
+}
+// ---------------------------------------------------------------------------
+// Credential hints (MCP server → env var names)
+// ---------------------------------------------------------------------------
+export const CREDENTIAL_HINTS = {
+    Slack: ["SLACK_BOT_TOKEN", "SLACK_USER_TOKEN"],
+    GitHub: ["GITHUB_TOKEN"],
+    Linear: ["LINEAR_API_KEY"],
+    "Google Workspace": ["GOOGLE_APPLICATION_CREDENTIALS"],
+    Notion: ["NOTION_API_KEY"],
+    Jira: ["JIRA_API_TOKEN", "JIRA_EMAIL"],
+    Confluence: ["CONFLUENCE_API_TOKEN"],
+    Figma: ["FIGMA_ACCESS_TOKEN"],
+    Sentry: ["SENTRY_AUTH_TOKEN"],
+};
+// ---------------------------------------------------------------------------
+// Integration eval prompt builder
+// ---------------------------------------------------------------------------
+const INTEGRATION_SCHEMA_REFERENCE = `
+## Integration evals.json Schema Extension
+
+Integration test cases extend the base schema with these additional fields:
+
+{
+  "id": <number>,
+  "name": "<string>",
+  "prompt": "<string>",
+  "expected_output": "<string>",
+  "testType": "integration",
+  "requiredCredentials": ["<ENV_VAR_NAME>", ...],
+  "requirements": {
+    "chromeProfile": "<string, Chrome profile name for browser-based tests>",
+    "platform": "<string, target platform name>"
+  },
+  "cleanup": [
+    {
+      "action": "delete_post" | "remove_artifact" | "custom",
+      "platform": "<string>",
+      "identifier": "{POSTED_ID}",
+      "description": "<what this cleanup step does>"
+    }
+  ],
+  "assertions": [...]
+}
+
+Every integration eval MUST have testType: "integration".
+`;
+export function buildIntegrationEvalPrompt(skillContent, mcpDeps, browserReqs, platforms) {
+    let credentialSection = "";
+    if (mcpDeps.length > 0) {
+        const creds = [];
+        for (const dep of mcpDeps) {
+            const hints = CREDENTIAL_HINTS[dep.server];
+            if (hints) {
+                creds.push(`- **${dep.server}**: ${hints.join(", ")}`);
+            }
+        }
+        if (creds.length > 0) {
+            credentialSection = `\n## Required Credentials\n\nPopulate \`requiredCredentials\` with these environment variable names:\n${creds.join("\n")}\n`;
+        }
+    }
+    let browserSection = "";
+    if (browserReqs.hasBrowser) {
+        browserSection = `\n## Browser Requirements\n\nThis skill uses browser automation. Each integration test case MUST include:\n- \`requirements.chromeProfile\`: "${browserReqs.suggestedProfile || "Default"}"\n\nThe integration test runner will launch Chrome with this profile's logged-in sessions.\n`;
+    }
+    let platformSection = "";
+    if (platforms.length > 0) {
+        const platformList = platforms.map((p) => `- ${p}`).join("\n");
+        platformSection = `\n## Platform Targets\n\nDetected platforms:\n${platformList}\n\nFor each platform-targeting test case, include a \`cleanup\` block:\n- action: "delete_post" for social media posts\n- action: "remove_artifact" for created resources\n- action: "custom" for other cleanup needs\n- platform: the target platform name\n- identifier: "{POSTED_ID}" (placeholder replaced at runtime)\n- description: what the cleanup step does\n`;
+    }
+    return `You are an expert eval generator for AI skills. Your task is to create INTEGRATION test cases for the skill described below.
+
+## Skill Content (SKILL.md)
+
+${skillContent}
+
+${INTEGRATION_SCHEMA_REFERENCE}
+
+${BEST_PRACTICES}
+${credentialSection}${browserSection}${platformSection}
+## Instructions
+
+Generate integration eval cases that test the skill against real external services. Each case must have:
+- \`testType: "integration"\`
+- \`requiredCredentials\` array with needed env vars
+- \`requirements\` object with chromeProfile (if browser-based) and platform
+- \`cleanup\` array with cleanup actions for any created artifacts
+- At least 2 objectively verifiable assertions per case
+
+Output ONLY the JSON inside a \`\`\`json code fence. The JSON should be an array of eval case objects (NOT the full evals.json wrapper — just the array of cases).`;
+}
+// ---------------------------------------------------------------------------
+// Integration eval response parser
+// ---------------------------------------------------------------------------
+export function parseGeneratedIntegrationEvals(raw) {
+    const match = raw.match(/```(?:json)?\s*\n?([\s\S]*?)\n?```/);
+    if (!match) {
+        throw new Error("No JSON code block found in LLM response. Expected ```json ... ``` fence.");
+    }
+    let parsed;
+    try {
+        parsed = JSON.parse(match[1]);
+    }
+    catch (err) {
+        throw new Error(`Invalid JSON in code block: ${err.message}`);
+    }
+    // Accept both array and wrapped object
+    const cases = Array.isArray(parsed) ? parsed : parsed.evals;
+    if (!Array.isArray(cases) || cases.length === 0) {
+        throw new Error("Expected a non-empty array of integration eval cases");
+    }
+    return cases.map((c) => ({
+        id: c.id,
+        name: c.name || `integration-case-${c.id}`,
+        prompt: c.prompt,
+        expected_output: c.expected_output || "",
+        files: Array.isArray(c.files) ? c.files : [],
+        assertions: Array.isArray(c.assertions)
+            ? c.assertions.map((a) => ({
+                id: a.id,
+                text: a.text,
+                type: a.type || "boolean",
+            }))
+            : [],
+        testType: "integration",
+        requiredCredentials: Array.isArray(c.requiredCredentials) ? c.requiredCredentials : undefined,
+        requirements: c.requirements || undefined,
+        cleanup: Array.isArray(c.cleanup) ? c.cleanup : undefined,
+    }));
+}
 export function buildEvalSystemPrompt(skillContent) {
     if (!skillContent)
         return "You are a helpful AI assistant.";

package/dist/eval/prompt-builder.js.map

@@ -1 +1 @@
-{"version":3,"file":"prompt-builder.js","sourceRoot":"","sources":["../../src/eval/prompt-builder.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,qDAAqD;AACrD,8EAA8E;AAG9E,OAAO,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAE1D,8EAA8E;AAC9E,8BAA8B;AAC9B,8EAA8E;AAE9E,MAAM,gBAAgB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;CA0BxB,CAAC;AAEF,8EAA8E;AAC9E,+CAA+C;AAC/C,8EAA8E;AAE9E,MAAM,aAAa,GAAG;;;;;;;;;;;;;;;;;;;;;CAqBrB,CAAC;AAEF,8EAA8E;AAC9E,4BAA4B;AAC5B,8EAA8E;AAE9E,MAAM,cAAc,GAAG;;;;;;;;;;;;;;;;;;;;CAoBtB,CAAC;AAEF,8EAA8E;AAC9E,4CAA4C;AAC5C,8EAA8E;AAE9E;;;;GAIG;AACH,SAAS,eAAe,CAAC,YAAoB;IAC3C,2EAA2E;IAC3E,MAAM,gBAAgB,GAAG,YAAY,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;IACxE,IAAI,CAAC,gBAAgB;QAAE,OAAO,KAAK,CAAC;IACpC,MAAM,WAAW,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;IACxC,MAAM,iBAAiB,GAAG,WAAW,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;IACzE,IAAI,CAAC,iBAAiB;QAAE,OAAO,KAAK,CAAC;IACrC,MAAM,KAAK,GAAG,iBAAiB,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;IACjD,OAAO,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;AAC/G,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,YAAoB;IACxD,IAAI,CAAC,YAAY;QAAE,OAAO,iCAAiC,CAAC;IAE5D,MAAM,OAAO,GAAG,qBAAqB,CAAC,YAAY,CAAC,CAAC;IACpD,MAAM,YAAY,GAAG,eAAe,CAAC,YAAY,CAAC,CAAC;IAEnD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;QAC1C,OAAO,iEAAiE,YAAY,EAAE,CAAC;IACzF,CAAC;IAED,IAAI,gBAAgB,GAAG,EAAE,CAAC;IAE1B,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,QAAQ,GAAG,OAAO;aACrB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;aAC7D,IAAI,CAAC,IAAI,CAAC,CAAC;QAEd,gBAAgB,IAAI,wNAAwN,QAAQ,qhBAAqhB,CAAC;IAC5wB,CAAC;IAED,IAAI,YAAY,EAAE,CAAC;QACjB,gBAAgB,IAAI,06BAA06B,CAAC;IACj8B,CAAC;IAED,OAAO,iEAAiE,YAAY,UAAU,gBAAgB,EAAE,CAAC;AACnH,CAAC;AAED,MAAM,UAAU,yBAAyB;IACvC,OAAO,iCAAiC,CAAC;AAC3C,CAAC;AAED,8EAA8E;AAC9E,iEAAiE;AACjE,8EAA8E;AAE9E,MAAM,UAAU,mBAAmB,CAAC,YAAoB;IACtD,MAAM,OAAO,GAAG,qBAAqB,CAAC,YAAY,CAAC,CAAC;IAEpD,IAAI,UAAU,GAAG,EAAE,CAAC;IACpB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,QAAQ,GAAG,OAAO;aACrB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;aAC7D,IAAI,CAAC,IAAI,CAAC,CAAC;QAEd,UAAU,GAAG;;;;;;EAMf,QAAQ;;;;;;;CAOT,CAAC;IACA,CAAC;IAED,OAAO;;;;EAIP,YAAY;;EAEZ,gBAAgB;;EAEhB,aAAa;;EAEb,cAAc;EACd,UAAU;;;+OAGmO,CAAC;AAChP,CAAC;AAED,8EAA8E;AAC9E,kBAAkB;AAClB,8EAA8E;AAE9E,MAAM,UAAU,mBAAmB,CAAC,GAAW;IAC7C,+CAA+C;IAC/C,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,oCAAoC,CAAC,CAAC;IAC9D,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CACb,2EAA2E,CAC5E,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;IACzB,IAAI,MAAW,CAAC;IAChB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CACb,+BAAgC,GAAa,CAAC,OAAO,EAAE,CACxD,CAAC;IACJ,CAAC;IAED,0BAA0B;IAC1B,mBAAmB,CAAC,MAAM,CAAC,CAAC;IAE5B,YAAY;IACZ,OAAO;QACL,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC;YACnC,EAAE,EAAE,CAAC,CAAC,EAAE;YACR,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,eAAe,EAAE,CAAC,CAAC,eAAe;YAClC,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;YAC5C,UAAU,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC;gBACxC,EAAE,EAAE,CAAC,CAAC,EAAE;gBACR,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,IAAI,EAAE,CAAC,CAAC,IAAI,IAAI,SAAS;aAC1B,CAAC,CAAC;SACJ,CAAC,CAAC;KACJ,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB,CAAC,MAAW;IACtC,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,IAAI,OAAO,MAAM,CAAC,UAAU,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;QAChE,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IACpC,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9D,MAAM,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;IAC9C,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QAChC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC1B,IAAI,CAAC,CAAC,CAAC,MAAM;gBAAE,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;YACzD,IAAI,CAAC,CAAC,CAAC,IAAI;gBAAE,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC;YACrD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC9D,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,+BAA+B,CAAC,CAAC;YACzD,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,4BAA4B,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACnE,CAAC;AACH,CAAC"}
+{"version":3,"file":"prompt-builder.js","sourceRoot":"","sources":["../../src/eval/prompt-builder.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,qDAAqD;AACrD,8EAA8E;AAI9E,OAAO,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAE1D,8EAA8E;AAC9E,8BAA8B;AAC9B,8EAA8E;AAE9E,MAAM,gBAAgB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;CA0BxB,CAAC;AAEF,8EAA8E;AAC9E,+CAA+C;AAC/C,8EAA8E;AAE9E,MAAM,aAAa,GAAG;;;;;;;;;;;;;;;;;;;;;CAqBrB,CAAC;AAEF,8EAA8E;AAC9E,4BAA4B;AAC5B,8EAA8E;AAE9E,MAAM,cAAc,GAAG;;;;;;;;;;;;;;;;;;;;CAoBtB,CAAC;AAEF,8EAA8E;AAC9E,4CAA4C;AAC5C,8EAA8E;AAE9E;;;;GAIG;AACH,SAAS,eAAe,CAAC,YAAoB;IAC3C,2EAA2E;IAC3E,MAAM,gBAAgB,GAAG,YAAY,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;IACxE,IAAI,CAAC,gBAAgB;QAAE,OAAO,KAAK,CAAC;IACpC,MAAM,WAAW,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;IACxC,MAAM,iBAAiB,GAAG,WAAW,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;IACzE,IAAI,CAAC,iBAAiB;QAAE,OAAO,KAAK,CAAC;IACrC,MAAM,KAAK,GAAG,iBAAiB,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;IACjD,OAAO,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;AAC/G,CAAC;AAWD,MAAM,UAAU,yBAAyB,CAAC,YAAoB;IAC5D,MAAM,OAAO,GAAG,eAAe,CAAC,YAAY,CAAC,CAAC;IAC9C,IAAI,CAAC,OAAO;QAAE,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;IAE3C,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,wBAAwB,EAAE,EAAE,CAAC,CAAC;IAChE,MAAM,cAAc,GAAG,kFAAkF,CAAC;IAC1G,IAAI,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9B,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,gBAAgB,EAAE,SAAS,EAAE,CAAC;IAC3D,CAAC;IACD,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;AAC/B,CAAC;AAED,MAAM,iBAAiB,GAA2B;IAChD,OAAO,EAAE,SAAS;IAClB,OAAO,EAAE,SAAS;IAClB,SAAS,EAAE,SAAS;IACpB,QAAQ,EAAE,UAAU;IACpB,KAAK,EAAE,OAAO;IACd,SAAS,EAAE,WAAW;IACtB,OAAO,EAAE,SAAS;IAClB,MAAM,EAAE,QAAQ;IAChB,MAAM,EAAE,QAAQ;IAChB,QAAQ,EAAE,UAAU;IACpB,OAAO,EAAE,SAAS;IAClB,QAAQ,EAAE,UAAU;IACpB,OAAO,EAAE,SAAS;IAClB,QAAQ,EAAE,QAAQ;CACnB,CAAC;AAEF,MAAM,UAAU,qBAAqB,CAAC,YAAoB;IACxD,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,wBAAwB,EAAE,EAAE,CAAC,CAAC;IAChE,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,KAAK,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC,EAAE,CAAC;QACpE,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QACvC,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACrB,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;AAC9B,CAAC;AAED,8EAA8E;AAC9E,gDAAgD;AAChD,8EAA8E;AAE9E,MAAM,CAAC,MAAM,gBAAgB,GAA6B;IACxD,KAAK,EAAE,CAAC,iBAAiB,EAAE,kBAAkB,CAAC;IAC9C,MAAM,EAAE,CAAC,cAAc,CAAC;IACxB,MAAM,EAAE,CAAC,gBAAgB,CAAC;IAC1B,kBAAkB,EAAE,CAAC,gCAAgC,CAAC;IACtD,MAAM,EAAE,CAAC,gBAAgB,CAAC;IAC1B,IAAI,EAAE,CAAC,gBAAgB,EAAE,YAAY,CAAC;IACtC,UAAU,EAAE,CAAC,sBAAsB,CAAC;IACpC,KAAK,EAAE,CAAC,oBAAoB,CAAC;IAC7B,MAAM,EAAE,CAAC,mBAAmB,CAAC;CAC9B,CAAC;AAEF,8EAA8E;AAC9E,kCAAkC;AAClC,8EAA8E;AAE9E,MAAM,4BAA4B,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA4BpC,CAAC;AAEF,MAAM,UAAU,0BAA0B,CACxC,YAAoB,EACpB,OAAwB,EACxB,WAAgC,EAChC,SAAmB;IAEnB,IAAI,iBAAiB,GAAG,EAAE,CAAC;IAC3B,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;YAC1B,MAAM,KAAK,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAC3C,IAAI,KAAK,EAAE,CAAC;gBACV,KAAK,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,MAAM,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACzD,CAAC;QACH,CAAC;QACD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrB,iBAAiB,GAAG,yGAAyG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;QACpJ,CAAC;IACH,CAAC;IAED,IAAI,cAAc,GAAG,EAAE,CAAC;IACxB,IAAI,WAAW,CAAC,UAAU,EAAE,CAAC;QAC3B,cAAc,GAAG,iJAAiJ,WAAW,CAAC,gBAAgB,IAAI,SAAS,+FAA+F,CAAC;IAC7S,CAAC;IAED,IAAI,eAAe,GAAG,EAAE,CAAC;IACzB,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,MAAM,YAAY,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/D,eAAe,GAAG,iDAAiD,YAAY,0WAA0W,CAAC;IAC5b,CAAC;IAED,OAAO;;;;EAIP,YAAY;;EAEZ,4BAA4B;;EAE5B,cAAc;EACd,iBAAiB,GAAG,cAAc,GAAG,eAAe;;;;;;;;;;mKAU6G,CAAC;AACpK,CAAC;AAED,8EAA8E;AAC9E,mCAAmC;AACnC,8EAA8E;AAE9E,MAAM,UAAU,8BAA8B,CAAC,GAAW;IACxD,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,oCAAoC,CAAC,CAAC;IAC9D,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CACb,2EAA2E,CAC5E,CAAC;IACJ,CAAC;IAED,IAAI,MAAW,CAAC;IAChB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAChC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CACb,+BAAgC,GAAa,CAAC,OAAO,EAAE,CACxD,CAAC;IACJ,CAAC;IAED,uCAAuC;IACvC,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;IAC5D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChD,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;IAC1E,CAAC;IAED,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC;QAC5B,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,IAAI,EAAE,CAAC,CAAC,IAAI,IAAI,oBAAoB,CAAC,CAAC,EAAE,EAAE;QAC1C,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,eAAe,EAAE,CAAC,CAAC,eAAe,IAAI,EAAE;QACxC,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;QAC5C,UAAU,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC;YACrC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC;gBAC5B,EAAE,EAAE,CAAC,CAAC,EAAE;gBACR,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,IAAI,EAAE,CAAC,CAAC,IAAI,IAAI,SAAS;aAC1B,CAAC,CAAC;YACL,CAAC,CAAC,EAAE;QACN,QAAQ,EAAE,aAAsB;QAChC,mBAAmB,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,SAAS;QAC7F,YAAY,EAAE,CAAC,CAAC,YAAY,IAAI,SAAS;QACzC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;KAC1D,CAAC,CAAC,CAAC;AACN,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,YAAoB;IACxD,IAAI,CAAC,YAAY;QAAE,OAAO,iCAAiC,CAAC;IAE5D,MAAM,OAAO,GAAG,qBAAqB,CAAC,YAAY,CAAC,CAAC;IACpD,MAAM,YAAY,GAAG,eAAe,CAAC,YAAY,CAAC,CAAC;IAEnD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;QAC1C,OAAO,iEAAiE,YAAY,EAAE,CAAC;IACzF,CAAC;IAED,IAAI,gBAAgB,GAAG,EAAE,CAAC;IAE1B,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,QAAQ,GAAG,OAAO;aACrB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;aAC7D,IAAI,CAAC,IAAI,CAAC,CAAC;QAEd,gBAAgB,IAAI,wNAAwN,QAAQ,qhBAAqhB,CAAC;IAC5wB,CAAC;IAED,IAAI,YAAY,EAAE,CAAC;QACjB,gBAAgB,IAAI,06BAA06B,CAAC;IACj8B,CAAC;IAED,OAAO,iEAAiE,YAAY,UAAU,gBAAgB,EAAE,CAAC;AACnH,CAAC;AAED,MAAM,UAAU,yBAAyB;IACvC,OAAO,iCAAiC,CAAC;AAC3C,CAAC;AAED,8EAA8E;AAC9E,iEAAiE;AACjE,8EAA8E;AAE9E,MAAM,UAAU,mBAAmB,CAAC,YAAoB;IACtD,MAAM,OAAO,GAAG,qBAAqB,CAAC,YAAY,CAAC,CAAC;IAEpD,IAAI,UAAU,GAAG,EAAE,CAAC;IACpB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,QAAQ,GAAG,OAAO;aACrB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;aAC7D,IAAI,CAAC,IAAI,CAAC,CAAC;QAEd,UAAU,GAAG;;;;;;EAMf,QAAQ;;;;;;;CAOT,CAAC;IACA,CAAC;IAED,OAAO;;;;EAIP,YAAY;;EAEZ,gBAAgB;;EAEhB,aAAa;;EAEb,cAAc;EACd,UAAU;;;+OAGmO,CAAC;AAChP,CAAC;AAED,8EAA8E;AAC9E,kBAAkB;AAClB,8EAA8E;AAE9E,MAAM,UAAU,mBAAmB,CAAC,GAAW;IAC7C,+CAA+C;IAC/C,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,oCAAoC,CAAC,CAAC;IAC9D,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CACb,2EAA2E,CAC5E,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;IACzB,IAAI,MAAW,CAAC;IAChB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CACb,+BAAgC,GAAa,CAAC,OAAO,EAAE,CACxD,CAAC;IACJ,CAAC;IAED,0BAA0B;IAC1B,mBAAmB,CAAC,MAAM,CAAC,CAAC;IAE5B,YAAY;IACZ,OAAO;QACL,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC;YACnC,EAAE,EAAE,CAAC,CAAC,EAAE;YACR,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,eAAe,EAAE,CAAC,CAAC,eAAe;YAClC,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;YAC5C,UAAU,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC;gBACxC,EAAE,EAAE,CAAC,CAAC,EAAE;gBACR,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,IAAI,EAAE,CAAC,CAAC,IAAI,IAAI,SAAS;aAC1B,CAAC,CAAC;SACJ,CAAC,CAAC;KACJ,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB,CAAC,MAAW;IACtC,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,IAAI,OAAO,MAAM,CAAC,UAAU,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;QAChE,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IACpC,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9D,MAAM,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;IAC9C,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QAChC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC1B,IAAI,CAAC,CAAC,CAAC,MAAM;gBAAE,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;YACzD,IAAI,CAAC,CAAC,CAAC,IAAI;gBAAE,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC;YACrD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC9D,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,+BAA+B,CAAC,CAAC;YACzD,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,4BAA4B,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACnE,CAAC;AACH,CAAC"}

package/dist/eval/rate-limiter.d.ts

@@ -0,0 +1,20 @@
+import type { PlatformRateLimit } from "./integration-types.js";
+/**
+ * Per-platform token bucket rate limiter for integration tests.
+ * Ensures requests to external platforms stay within configured limits.
+ */
+export declare class PlatformRateLimiter {
+    private buckets;
+    private overrides;
+    constructor(overrides?: Record<string, PlatformRateLimit>);
+    /**
+     * Acquire a token for the given platform. Waits if no tokens available.
+     */
+    acquire(platform: string): Promise<void>;
+    /**
+     * Get rate limit config for a platform.
+     */
+    getLimit(platform: string): PlatformRateLimit;
+    private getBucket;
+    private refill;
+}

package/dist/eval/rate-limiter.js

@@ -0,0 +1,62 @@
+// ---------------------------------------------------------------------------
+// rate-limiter.ts -- per-platform token bucket rate limiter
+// ---------------------------------------------------------------------------
+import { DEFAULT_RATE_LIMITS, DEFAULT_RATE_LIMIT } from "./integration-types.js";
+/**
+ * Per-platform token bucket rate limiter for integration tests.
+ * Ensures requests to external platforms stay within configured limits.
+ */
+export class PlatformRateLimiter {
+    buckets = new Map();
+    overrides;
+    constructor(overrides) {
+        this.overrides = overrides ?? {};
+    }
+    /**
+     * Acquire a token for the given platform. Waits if no tokens available.
+     */
+    async acquire(platform) {
+        const bucket = this.getBucket(platform);
+        this.refill(bucket);
+        if (bucket.tokens >= 1) {
+            bucket.tokens -= 1;
+            return;
+        }
+        // Wait until a token is available
+        const waitMs = bucket.refillIntervalMs - (Date.now() - bucket.lastRefill);
+        await new Promise((resolve) => setTimeout(resolve, Math.max(0, waitMs)));
+        this.refill(bucket);
+        bucket.tokens = Math.max(0, bucket.tokens - 1);
+    }
+    /**
+     * Get rate limit config for a platform.
+     */
+    getLimit(platform) {
+        return this.overrides[platform] ?? DEFAULT_RATE_LIMITS[platform] ?? DEFAULT_RATE_LIMIT;
+    }
+    getBucket(platform) {
+        let bucket = this.buckets.get(platform);
+        if (!bucket) {
+            const limit = this.getLimit(platform);
+            const refillIntervalMs = (60_000 / limit.requestsPerMinute);
+            bucket = {
+                tokens: limit.requestsPerMinute,
+                maxTokens: limit.requestsPerMinute,
+                refillIntervalMs,
+                lastRefill: Date.now(),
+            };
+            this.buckets.set(platform, bucket);
+        }
+        return bucket;
+    }
+    refill(bucket) {
+        const now = Date.now();
+        const elapsed = now - bucket.lastRefill;
+        const tokensToAdd = Math.floor(elapsed / bucket.refillIntervalMs);
+        if (tokensToAdd > 0) {
+            bucket.tokens = Math.min(bucket.maxTokens, bucket.tokens + tokensToAdd);
+            bucket.lastRefill = now;
+        }
+    }
+}
+//# sourceMappingURL=rate-limiter.js.map

package/dist/eval/rate-limiter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rate-limiter.js","sourceRoot":"","sources":["../../src/eval/rate-limiter.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,4DAA4D;AAC5D,8EAA8E;AAG9E,OAAO,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AASjF;;;GAGG;AACH,MAAM,OAAO,mBAAmB;IACtB,OAAO,GAAG,IAAI,GAAG,EAAuB,CAAC;IACzC,SAAS,CAAoC;IAErD,YAAY,SAA6C;QACvD,IAAI,CAAC,SAAS,GAAG,SAAS,IAAI,EAAE,CAAC;IACnC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO,CAAC,QAAgB;QAC5B,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QACxC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAEpB,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACvB,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC;YACnB,OAAO;QACT,CAAC;QAED,kCAAkC;QAClC,MAAM,MAAM,GAAG,MAAM,CAAC,gBAAgB,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;QAC1E,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;QAC/E,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACpB,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACjD,CAAC;IAED;;OAEG;IACH,QAAQ,CAAC,QAAgB;QACvB,OAAO,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,mBAAmB,CAAC,QAAQ,CAAC,IAAI,kBAAkB,CAAC;IACzF,CAAC;IAEO,SAAS,CAAC,QAAgB;QAChC,IAAI,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACxC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YACtC,MAAM,gBAAgB,GAAG,CAAC,MAAM,GAAG,KAAK,CAAC,iBAAiB,CAAC,CAAC;YAC5D,MAAM,GAAG;gBACP,MAAM,EAAE,KAAK,CAAC,iBAAiB;gBAC/B,SAAS,EAAE,KAAK,CAAC,iBAAiB;gBAClC,gBAAgB;gBAChB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE;aACvB,CAAC;YACF,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACrC,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,MAAM,CAAC,MAAmB;QAChC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,OAAO,GAAG,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC;QACxC,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,gBAAgB,CAAC,CAAC;QAClE,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;YACpB,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,MAAM,GAAG,WAAW,CAAC,CAAC;YACxE,MAAM,CAAC,UAAU,GAAG,GAAG,CAAC;QAC1B,CAAC;IACH,CAAC;CACF"}

package/dist/eval/schema.d.ts

@@ -1,3 +1,4 @@
+import type { EvalCleanupSchema, EvalRequirementsSchema } from "./integration-types.js";
 export interface Assertion {
     id: string;
     text: string;
@@ -12,6 +13,8 @@ export interface EvalCase {
     assertions: Assertion[];
     testType?: "unit" | "integration";
     requiredCredentials?: string[];
+    requirements?: EvalRequirementsSchema;
+    cleanup?: EvalCleanupSchema[];
 }
 export interface EvalsFile {
     skill_name: string;
@@ -21,8 +24,19 @@ export interface ValidationError {
     path: string;
     message: string;
 }
+export interface ValidationWarning {
+    path: string;
+    message: string;
+}
 export declare class EvalValidationError extends Error {
     errors: ValidationError[];
     constructor(errors: ValidationError[]);
 }
+export interface ValidatedEvalsResult {
+    evalsFile: EvalsFile;
+    warnings: ValidationWarning[];
+}
 export declare function loadAndValidateEvals(skillDir: string): EvalsFile;
+export declare function loadAndValidateEvals(skillDir: string, opts: {
+    returnWarnings: true;
+}): ValidatedEvalsResult;

package/dist/eval/schema.js

@@ -3,6 +3,7 @@
 // ---------------------------------------------------------------------------
 import { readFileSync, existsSync } from "node:fs";
 import { join } from "node:path";
+import { VALID_CLEANUP_ACTIONS } from "./integration-types.js";
 export class EvalValidationError extends Error {
     errors;
     constructor(errors) {
@@ -12,10 +13,13 @@ export class EvalValidationError extends Error {
         this.errors = errors;
     }
 }
-// ---------------------------------------------------------------------------
-// Validator
-// ---------------------------------------------------------------------------
-export function loadAndValidateEvals(skillDir) {
+export function loadAndValidateEvals(skillDir, opts) {
+    const result = _loadAndValidateEvalsInternal(skillDir);
+    if (opts?.returnWarnings)
+        return result;
+    return result.evalsFile;
+}
+function _loadAndValidateEvalsInternal(skillDir) {
     const filePath = join(skillDir, "evals", "evals.json");
     if (!existsSync(filePath)) {
         throw new EvalValidationError([
@@ -124,6 +128,50 @@ export function loadAndValidateEvals(skillDir) {
     if (errors.length > 0) {
         throw new EvalValidationError(errors);
     }
+    // Integration-specific validation (warnings + cleanup action errors)
+    const warnings = [];
+    const integrationErrors = [];
+    for (let i = 0; i < parsed.evals.length; i++) {
+        const evalCase = parsed.evals[i];
+        const prefix = `evals[${i}]`;
+        if (evalCase.testType !== "integration")
+            continue;
+        // AC-US4-01: Missing requiredCredentials → warning (not error)
+        if (!Array.isArray(evalCase.requiredCredentials) || evalCase.requiredCredentials.length === 0) {
+            warnings.push({
+                path: `${prefix}.requiredCredentials`,
+                message: "integration test case has no requiredCredentials — some integration tests may not need credentials, but verify this is intentional",
+            });
+        }
+        // AC-US4-02: Invalid cleanup actions → error
+        if (Array.isArray(evalCase.cleanup)) {
+            for (let j = 0; j < evalCase.cleanup.length; j++) {
+                const cleanup = evalCase.cleanup[j];
+                if (cleanup.action && !VALID_CLEANUP_ACTIONS.includes(cleanup.action)) {
+                    integrationErrors.push({
+                        path: `${prefix}.cleanup[${j}].action`,
+                        message: `invalid cleanup action "${cleanup.action}" — allowed values: ${VALID_CLEANUP_ACTIONS.join(", ")}`,
+                    });
+                }
+            }
+        }
+        // AC-US4-03: Prose assertions heuristic → warning
+        if (Array.isArray(evalCase.assertions)) {
+            const outcomeVerbs = /\b(returns|contains|exists|equals|includes|matches|starts|ends|has|is|shows|displays|produces|emits|creates|deletes|responds|status)\b/i;
+            for (let j = 0; j < evalCase.assertions.length; j++) {
+                const assertion = evalCase.assertions[j];
+                if (typeof assertion.text === "string" && assertion.text.length > 100 && !outcomeVerbs.test(assertion.text)) {
+                    warnings.push({
+                        path: `${prefix}.assertions[${j}].text`,
+                        message: "assertion looks like LLM-judged prose — consider using API status codes or resource existence assertions for integration tests",
+                    });
+                }
+            }
+        }
+    }
+    if (integrationErrors.length > 0) {
+        throw new EvalValidationError(integrationErrors);
+    }
     // Normalize: default files to [], testType to "unit"
     const evals = parsed.evals.map((e) => ({
         id: e.id,
@@ -138,7 +186,9 @@ export function loadAndValidateEvals(skillDir) {
         })),
         testType: e.testType === "integration" ? "integration" : "unit",
         ...(Array.isArray(e.requiredCredentials) ? { requiredCredentials: e.requiredCredentials } : {}),
+        ...(e.requirements ? { requirements: e.requirements } : {}),
+        ...(Array.isArray(e.cleanup) ? { cleanup: e.cleanup } : {}),
     }));
-    return { skill_name: parsed.skill_name, evals };
+    return { evalsFile: { skill_name: parsed.skill_name, evals }, warnings };
 }
 //# sourceMappingURL=schema.js.map

package/dist/eval/schema.js.map

@@ -1 +1 @@
-{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/eval/schema.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,+BAA+B;AAC/B,8EAA8E;AAE9E,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAiCjC,MAAM,OAAO,mBAAoB,SAAQ,KAAK;IAC5C,MAAM,CAAoB;IAE1B,YAAY,MAAyB;QACnC,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpE,KAAK,CAAC,2BAA2B,GAAG,EAAE,CAAC,CAAC;QACxC,IAAI,CAAC,IAAI,GAAG,qBAAqB,CAAC;QAClC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;CACF;AAED,8EAA8E;AAC9E,YAAY;AACZ,8EAA8E;AAE9E,MAAM,UAAU,oBAAoB,CAAC,QAAgB;IACnD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC;IAEvD,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,mBAAmB,CAAC;YAC5B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,qBAAqB,EAAE;SACnD,CAAC,CAAC;IACL,CAAC;IAED,MAAM,GAAG,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAE5C,IAAI,MAAW,CAAC;IAChB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC3B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,SAAS,GAAG,GAAkB,CAAC;QACrC,MAAM,IAAI,mBAAmB,CAAC;YAC5B;gBACE,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,qBAAqB,SAAS,CAAC,OAAO,EAAE;aAClD;SACF,CAAC,CAAC;IACL,CAAC;IAED,MAAM,MAAM,GAAsB,EAAE,CAAC;IAErC,IAAI,OAAO,MAAM,CAAC,UAAU,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;QAChE,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,uBAAuB,EAAE,CAAC,CAAC;IACxE,CAAC;IAED,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QACjC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,sBAAsB,EAAE,CAAC,CAAC;QAChE,MAAM,IAAI,mBAAmB,CAAC,MAAM,CAAC,CAAC;IACxC,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7C,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACjC,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC;QAE7B,IAAI,OAAO,QAAQ,CAAC,EAAE,KAAK,QAAQ,EAAE,CAAC;YACpC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,MAAM,KAAK,EAAE,OAAO,EAAE,uBAAuB,EAAE,CAAC,CAAC;QAC1E,CAAC;QAED,qEAAqE;QACrE,IAAI,CAAC,OAAO,QAAQ,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,OAAO,QAAQ,CAAC,EAAE,KAAK,QAAQ,EAAE,CAAC;YAC7F,QAAQ,CAAC,IAAI,GAAG,QAAQ,QAAQ,CAAC,EAAE,EAAE,CAAC;QACxC,CAAC;QACD,IAAI,OAAO,QAAQ,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YACxD,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,GAAG,MAAM,OAAO;gBACtB,OAAO,EAAE,uBAAuB;aACjC,CAAC,CAAC;QACL,CAAC;QAED,IAAI,OAAO,QAAQ,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;YAC5D,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,GAAG,MAAM,SAAS;gBACxB,OAAO,EAAE,uBAAuB;aACjC,CAAC,CAAC;QACL,CAAC;QACD,IACE,OAAO,QAAQ,CAAC,eAAe,KAAK,QAAQ;YAC5C,CAAC,QAAQ,CAAC,eAAe,EACzB,CAAC;YACD,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,GAAG,MAAM,kBAAkB;gBACjC,OAAO,EAAE,uBAAuB;aACjC,CAAC,CAAC;QACL,CAAC;QAED,iGAAiG;QACjG,2FAA2F;QAC3F,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;YAChF,MAAM,MAAM,GAAG,OAAO,QAAQ,CAAC,EAAE,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YACrE,QAAQ,CAAC,UAAU,GAAI,QAAQ,CAAC,YAAyB,CAAC,GAAG,CAAC,CAAC,IAAY,EAAE,CAAS,EAAE,EAAE,CAAC,CAAC;gBAC1F,EAAE,EAAE,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,EAAE;gBACxB,IAAI;gBACJ,IAAI,EAAE,SAAS;aAChB,CAAC,CAAC,CAAC;QACN,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,GAAG,MAAM,aAAa;gBAC5B,OAAO,EAAE,qIAAqI;aAC/I,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QAED,IAAI,QAAQ,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrC,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,GAAG,MAAM,aAAa;gBAC5B,OAAO,EAAE,gCAAgC;aAC1C,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QAED,oCAAoC;QACpC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;QAClC,KAAK,MAAM,SAAS,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC;YAC5C,IAAI,OAAO,SAAS,CAAC,EAAE,KAAK,QAAQ,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC;gBACtD,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,GAAG,MAAM,kBAAkB;oBACjC,OAAO,EAAE,uBAAuB;iBACjC,CAAC,CAAC;YACL,CAAC;YACD,IAAI,OAAO,SAAS,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;gBAC1D,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,GAAG,MAAM,oBAAoB;oBACnC,OAAO,EAAE,uBAAuB;iBACjC,CAAC,CAAC;YACL,CAAC;YACD,IAAI,SAAS,CAAC,EAAE,IAAI,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,CAAC;gBAC9C,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,GAAG,MAAM,aAAa;oBAC5B,OAAO,EAAE,2BAA2B,SAAS,CAAC,EAAE,EAAE;iBACnD,CAAC,CAAC;YACL,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,IAAI,mBAAmB,CAAC,MAAM,CAAC,CAAC;IACxC,CAAC;IAED,qDAAqD;IACrD,MAAM,KAAK,GAAe,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC;QACtD,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,eAAe,EAAE,CAAC,CAAC,eAAe;QAClC,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;QAC5C,UAAU,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC;YACxC,EAAE,EAAE,CAAC,CAAC,EAAE;YACR,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,IAAI,EAAE,CAAC,CAAC,IAAI,IAAI,SAAS;SAC1B,CAAC,CAAC;QACH,QAAQ,EAAE,CAAC,CAAC,QAAQ,KAAK,aAAa,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM;QAC/D,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC,EAAE,mBAAmB,EAAE,CAAC,CAAC,mBAAmB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAChG,CAAC,CAAC,CAAC;IAEJ,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,KAAK,EAAE,CAAC;AAClD,CAAC"}
+{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/eval/schema.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,+BAA+B;AAC/B,8EAA8E;AAE9E,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,qBAAqB,EAAE,MAAM,wBAAwB,CAAC;AAyC/D,MAAM,OAAO,mBAAoB,SAAQ,KAAK;IAC5C,MAAM,CAAoB;IAE1B,YAAY,MAAyB;QACnC,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpE,KAAK,CAAC,2BAA2B,GAAG,EAAE,CAAC,CAAC;QACxC,IAAI,CAAC,IAAI,GAAG,qBAAqB,CAAC;QAClC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;CACF;AAaD,MAAM,UAAU,oBAAoB,CAAC,QAAgB,EAAE,IAAmC;IACxF,MAAM,MAAM,GAAG,6BAA6B,CAAC,QAAQ,CAAC,CAAC;IACvD,IAAI,IAAI,EAAE,cAAc;QAAE,OAAO,MAAM,CAAC;IACxC,OAAO,MAAM,CAAC,SAAS,CAAC;AAC1B,CAAC;AAED,SAAS,6BAA6B,CAAC,QAAgB;IACrD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC;IAEvD,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,mBAAmB,CAAC;YAC5B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,qBAAqB,EAAE;SACnD,CAAC,CAAC;IACL,CAAC;IAED,MAAM,GAAG,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAE5C,IAAI,MAAW,CAAC;IAChB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC3B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,SAAS,GAAG,GAAkB,CAAC;QACrC,MAAM,IAAI,mBAAmB,CAAC;YAC5B;gBACE,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,qBAAqB,SAAS,CAAC,OAAO,EAAE;aAClD;SACF,CAAC,CAAC;IACL,CAAC;IAED,MAAM,MAAM,GAAsB,EAAE,CAAC;IAErC,IAAI,OAAO,MAAM,CAAC,UAAU,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;QAChE,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,uBAAuB,EAAE,CAAC,CAAC;IACxE,CAAC;IAED,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QACjC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,sBAAsB,EAAE,CAAC,CAAC;QAChE,MAAM,IAAI,mBAAmB,CAAC,MAAM,CAAC,CAAC;IACxC,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7C,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACjC,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC;QAE7B,IAAI,OAAO,QAAQ,CAAC,EAAE,KAAK,QAAQ,EAAE,CAAC;YACpC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,MAAM,KAAK,EAAE,OAAO,EAAE,uBAAuB,EAAE,CAAC,CAAC;QAC1E,CAAC;QAED,qEAAqE;QACrE,IAAI,CAAC,OAAO,QAAQ,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,OAAO,QAAQ,CAAC,EAAE,KAAK,QAAQ,EAAE,CAAC;YAC7F,QAAQ,CAAC,IAAI,GAAG,QAAQ,QAAQ,CAAC,EAAE,EAAE,CAAC;QACxC,CAAC;QACD,IAAI,OAAO,QAAQ,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YACxD,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,GAAG,MAAM,OAAO;gBACtB,OAAO,EAAE,uBAAuB;aACjC,CAAC,CAAC;QACL,CAAC;QAED,IAAI,OAAO,QAAQ,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;YAC5D,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,GAAG,MAAM,SAAS;gBACxB,OAAO,EAAE,uBAAuB;aACjC,CAAC,CAAC;QACL,CAAC;QACD,IACE,OAAO,QAAQ,CAAC,eAAe,KAAK,QAAQ;YAC5C,CAAC,QAAQ,CAAC,eAAe,EACzB,CAAC;YACD,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,GAAG,MAAM,kBAAkB;gBACjC,OAAO,EAAE,uBAAuB;aACjC,CAAC,CAAC;QACL,CAAC;QAED,iGAAiG;QACjG,2FAA2F;QAC3F,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;YAChF,MAAM,MAAM,GAAG,OAAO,QAAQ,CAAC,EAAE,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YACrE,QAAQ,CAAC,UAAU,GAAI,QAAQ,CAAC,YAAyB,CAAC,GAAG,CAAC,CAAC,IAAY,EAAE,CAAS,EAAE,EAAE,CAAC,CAAC;gBAC1F,EAAE,EAAE,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,EAAE;gBACxB,IAAI;gBACJ,IAAI,EAAE,SAAS;aAChB,CAAC,CAAC,CAAC;QACN,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,GAAG,MAAM,aAAa;gBAC5B,OAAO,EAAE,qIAAqI;aAC/I,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QAED,IAAI,QAAQ,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrC,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,GAAG,MAAM,aAAa;gBAC5B,OAAO,EAAE,gCAAgC;aAC1C,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QAED,oCAAoC;QACpC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;QAClC,KAAK,MAAM,SAAS,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC;YAC5C,IAAI,OAAO,SAAS,CAAC,EAAE,KAAK,QAAQ,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC;gBACtD,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,GAAG,MAAM,kBAAkB;oBACjC,OAAO,EAAE,uBAAuB;iBACjC,CAAC,CAAC;YACL,CAAC;YACD,IAAI,OAAO,SAAS,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;gBAC1D,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,GAAG,MAAM,oBAAoB;oBACnC,OAAO,EAAE,uBAAuB;iBACjC,CAAC,CAAC;YACL,CAAC;YACD,IAAI,SAAS,CAAC,EAAE,IAAI,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,CAAC;gBAC9C,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,GAAG,MAAM,aAAa;oBAC5B,OAAO,EAAE,2BAA2B,SAAS,CAAC,EAAE,EAAE;iBACnD,CAAC,CAAC;YACL,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,IAAI,mBAAmB,CAAC,MAAM,CAAC,CAAC;IACxC,CAAC;IAED,qEAAqE;IACrE,MAAM,QAAQ,GAAwB,EAAE,CAAC;IACzC,MAAM,iBAAiB,GAAsB,EAAE,CAAC;IAEhD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7C,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACjC,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC;QAE7B,IAAI,QAAQ,CAAC,QAAQ,KAAK,aAAa;YAAE,SAAS;QAElD,+DAA+D;QAC/D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC,IAAI,QAAQ,CAAC,mBAAmB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC9F,QAAQ,CAAC,IAAI,CAAC;gBACZ,IAAI,EAAE,GAAG,MAAM,sBAAsB;gBACrC,OAAO,EAAE,oIAAoI;aAC9I,CAAC,CAAC;QACL,CAAC;QAED,6CAA6C;QAC7C,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YACpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACjD,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;gBACpC,IAAI,OAAO,CAAC,MAAM,IAAI,CAAE,qBAA2C,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;oBAC7F,iBAAiB,CAAC,IAAI,CAAC;wBACrB,IAAI,EAAE,GAAG,MAAM,YAAY,CAAC,UAAU;wBACtC,OAAO,EAAE,2BAA2B,OAAO,CAAC,MAAM,uBAAuB,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;qBAC5G,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;QACH,CAAC;QAED,kDAAkD;QAClD,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;YACvC,MAAM,YAAY,GAAG,yIAAyI,CAAC;YAC/J,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACpD,MAAM,SAAS,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;gBACzC,IAAI,OAAO,SAAS,CAAC,IAAI,KAAK,QAAQ,IAAI,SAAS,CAAC,IAAI,CAAC,MAAM,GAAG,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC5G,QAAQ,CAAC,IAAI,CAAC;wBACZ,IAAI,EAAE,GAAG,MAAM,eAAe,CAAC,QAAQ;wBACvC,OAAO,EAAE,gIAAgI;qBAC1I,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,iBAAiB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjC,MAAM,IAAI,mBAAmB,CAAC,iBAAiB,CAAC,CAAC;IACnD,CAAC;IAED,qDAAqD;IACrD,MAAM,KAAK,GAAe,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC;QACtD,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,eAAe,EAAE,CAAC,CAAC,eAAe;QAClC,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;QAC5C,UAAU,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC;YACxC,EAAE,EAAE,CAAC,CAAC,EAAE;YACR,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,IAAI,EAAE,CAAC,CAAC,IAAI,IAAI,SAAS;SAC1B,CAAC,CAAC;QACH,QAAQ,EAAE,CAAC,CAAC,QAAQ,KAAK,aAAa,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM;QAC/D,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC,EAAE,mBAAmB,EAAE,CAAC,CAAC,mBAAmB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/F,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3D,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC5D,CAAC,CAAC,CAAC;IAEJ,OAAO,EAAE,SAAS,EAAE,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,CAAC;AAC3E,CAAC"}