vskill 0.5.84 → 0.5.85

package/README.md

@@ -361,6 +361,7 @@ The eval system supports multiple LLM providers. Switch between them in the eval
 | **Claude CLI** | Sonnet, Opus, Haiku | Claude Max/Pro subscription + `claude` CLI installed |
 | **Anthropic API** | Claude Sonnet 4.6, Opus 4.6, Haiku 4.5 | `ANTHROPIC_API_KEY` env var |
 | **Ollama** | Any locally installed model | Ollama running at `localhost:11434` |
+| **LM Studio** | Any model loaded in LM Studio | LM Studio running at `localhost:1234` (no API key needed) |
 
 ```bash
 # Use Anthropic API with Opus
@@ -371,6 +372,12 @@ VSKILL_EVAL_PROVIDER=ollama VSKILL_EVAL_MODEL=qwen2.5:32b npx vskill eval run my
 
 # Custom Ollama server
 OLLAMA_BASE_URL=http://gpu-server:11434 VSKILL_EVAL_PROVIDER=ollama npx vskill eval run my-skill
+
+# Use LM Studio with a locally loaded model (no API key required)
+VSKILL_EVAL_PROVIDER=lm-studio VSKILL_EVAL_MODEL=qwen2.5-coder-7b npx vskill eval run my-skill
+
+# Custom LM Studio endpoint
+LM_STUDIO_BASE_URL=http://lan-box:1234/v1 VSKILL_EVAL_PROVIDER=lm-studio npx vskill eval run my-skill
 ```
 
 **Which model for what?**
@@ -378,6 +385,7 @@ OLLAMA_BASE_URL=http://gpu-server:11434 VSKILL_EVAL_PROVIDER=ollama npx vskill e
 - **Skill creation/improvement**: Claude (Sonnet or Opus) produces the best SKILL.md refinements. Other models like Gemini and Codex can create skills too — they understand the SKILL.md format — but output quality may vary. See Anthropic's [Skill Creator](https://github.com/anthropics/skills/tree/main/skills/skill-creator) for the reference methodology.
 - **Benchmarks & A/B comparisons**: Use any model. Cross-model testing reveals whether your skill helps weaker models, and whether base model improvements have made a capability uplift skill unnecessary.
 - **Ollama**: Free, local, no API key. Useful for rapid iteration and validating cross-model portability.
+- **LM Studio**: Free, local, OpenAI-compatible server. Defaults to `http://localhost:1234/v1`; override with `LM_STUDIO_BASE_URL`. LM Studio ignores the API key, so vskill sends a dummy `Bearer lm-studio` token automatically.
 
 ### Platform integration
 
@@ -399,6 +407,38 @@ vskill info remotion-best-practices # skill details
 
 <br/>
 
+## SKILL.md Spec Compliance
+
+vskill emits SKILL.md files that conform to the canonical specification at **[agentskills.io/specification](https://agentskills.io/specification)**.
+
+The spec requires `tags` and `target-agents` to live **under a `metadata:` block**, not at the top level of the frontmatter:
+
+```yaml
+---
+name: my-skill
+description: "..."
+version: 1.0.0
+metadata:
+  tags:
+    - devtools
+    - cli
+  target-agents:
+    - claude-code
+    - cursor
+---
+```
+
+### Validation
+
+- **CI gate** — `npm run lint:skills-spec` walks the repo for every `SKILL.md` and blocks drift. Uses the external `skills-ref` CLI when available; otherwise falls back to a built-in check that enforces the `tags` / `target-agents` nesting rule.
+- **Post-creation (Studio)** — the `interpretValidatorResult` / `formatValidatorReport` helpers in `src/eval-server/skill-create-routes.ts` wrap `skills-ref validate`. Warn-only by default (spec drift prints a `Validation warnings` block, skill file stays on disk); `strict: true` flips the outcome to a blocking error with exit code 1. A missing `skills-ref` binary is non-blocking by design — CI is the enforcement line.
+
+### Migration note for downstream consumers
+
+If you previously read `frontmatter.tags` or `frontmatter['target-agents']` at the top level, read them under `frontmatter.metadata.tags` / `frontmatter.metadata['target-agents']` instead. All emitters migrated in lockstep; there is no period where vskill emits mixed shapes.
+
+<br/>
+
 ## Contributing
 
 Submit your skill for verification:

package/dist/eval/llm.d.ts

@@ -11,7 +11,7 @@ export interface LlmClient {
     generate(systemPrompt: string, userPrompt: string): Promise<GenerateResult>;
     readonly model: string;
 }
-export type ProviderName = "anthropic" | "claude-cli" | "codex-cli" | "gemini-cli" | "ollama" | "openrouter";
+export type ProviderName = "anthropic" | "claude-cli" | "codex-cli" | "gemini-cli" | "lm-studio" | "ollama" | "openrouter";
 export interface LlmOverrides {
     provider?: ProviderName;
     model?: string;

package/dist/eval/llm.js

@@ -50,6 +50,7 @@ export function estimateDurationSec(provider, totalCases, totalAssertions) {
         "codex-cli": [8, 20],
         "gemini-cli": [8, 20],
         "ollama": [5, 30],
+        "lm-studio": [5, 30],
         "openrouter": [4, 15],
     };
     const [lo, hi] = perCall[provider] ?? [5, 20];
@@ -74,10 +75,12 @@ export function createLlmClient(overrides) {
             return createGeminiCliClient(modelOverride);
         case "ollama":
             return createOllamaClient(modelOverride);
+        case "lm-studio":
+            return createLmStudioClient(modelOverride);
         case "openrouter":
             return createOpenRouterClient(modelOverride);
         default:
-            throw new Error(`Unknown VSKILL_EVAL_PROVIDER: "${provider}". Use "claude-cli", "codex-cli", "gemini-cli", "anthropic", "ollama", or "openrouter".`);
+            throw new Error(`Unknown VSKILL_EVAL_PROVIDER: "${provider}". Use "claude-cli", "codex-cli", "gemini-cli", "anthropic", "ollama", "lm-studio", or "openrouter".`);
     }
 }
 // ---------------------------------------------------------------------------
@@ -366,4 +369,65 @@ function createOpenRouterClient(modelOverride) {
         },
     };
 }
+// ---------------------------------------------------------------------------
+// Provider: LM Studio (local OpenAI-compatible server — free, no API key)
+//
+// Mirrors the OpenRouter adapter at the HTTP level because LM Studio exposes
+// the OpenAI chat completions surface verbatim at `<base>/chat/completions`.
+// Three constants differ from OpenRouter:
+//   - base URL: process.env.LM_STUDIO_BASE_URL ?? "http://localhost:1234/v1"
+//   - API key:  the literal string "lm-studio" (LM Studio ignores the token)
+//   - billing:  "free" (local inference)
+//
+// Env overrides:
+//   - LM_STUDIO_BASE_URL    — custom endpoint (e.g. a LAN machine)
+//   - VSKILL_EVAL_MODEL     — the model id loaded in LM Studio
+//
+// Note: LM Studio does not enforce the Bearer token, so the dummy value
+// "lm-studio" is safe to hardcode. Forks that do enforce it can override via
+// a future LM_STUDIO_API_KEY env var (not wired in this increment per
+// ADR-0677-02).
+// ---------------------------------------------------------------------------
+function createLmStudioClient(modelOverride) {
+    const baseUrl = process.env.LM_STUDIO_BASE_URL || "http://localhost:1234/v1";
+    const model = modelOverride || process.env.VSKILL_EVAL_MODEL || "local-model";
+    return {
+        model,
+        async generate(systemPrompt, userPrompt) {
+            const start = Date.now();
+            const response = await fetch(`${baseUrl}/chat/completions`, {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    "Authorization": "Bearer lm-studio",
+                },
+                body: JSON.stringify({
+                    model,
+                    messages: [
+                        { role: "system", content: systemPrompt },
+                        { role: "user", content: userPrompt },
+                    ],
+                    max_tokens: 4096,
+                }),
+                signal: AbortSignal.timeout(getTimeoutMs()),
+            });
+            if (!response.ok) {
+                const errorBody = await response.text();
+                throw new Error(`LM Studio request failed (${response.status}): ${errorBody.slice(0, 200)}`);
+            }
+            const data = (await response.json());
+            const text = data.choices?.[0]?.message?.content || "";
+            const inputTokens = data.usage?.prompt_tokens ?? null;
+            const outputTokens = data.usage?.completion_tokens ?? null;
+            return {
+                text,
+                durationMs: Date.now() - start,
+                inputTokens,
+                outputTokens,
+                cost: 0,
+                billingMode: "free",
+            };
+        },
+    };
+}
 //# sourceMappingURL=llm.js.map

package/dist/eval/llm.js.map

@@ -1 +1 @@
-{"version":3,"file":"llm.js","sourceRoot":"","sources":["../../src/eval/llm.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,+EAA+E;AAC/E,EAAE;AACF,uDAAuD;AACvD,wEAAwE;AACxE,iFAAiF;AACjF,mEAAmE;AACnE,8DAA8D;AAC9D,uEAAuE;AACvE,EAAE;AACF,uDAAuD;AACvD,iFAAiF;AACjF,2EAA2E;AAC3E,EAAE;AACF,mEAAmE;AACnE,oEAAoE;AACpE,EAAE;AACF,iDAAiD;AACjD,+DAA+D;AAC/D,4EAA4E;AAC5E,iFAAiF;AACjF,4DAA4D;AAC5D,mDAAmD;AACnD,8EAA8E;AAE9E,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC5E,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAoB7D,SAAS,cAAc;IACrB,OAAO,YAAY,CAAC;AACtB,CAAC;AAED,oFAAoF;AACpF,SAAS,YAAY;IACnB,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC;IAC/C,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACrC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,CAAC;YAAE,OAAO,OAAO,GAAG,IAAI,CAAC;IAC5D,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAOD;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CACjC,QAAsB,EACtB,UAAkB,EAClB,eAAuB;IAEvB,+CAA+C;IAC/C,MAAM,OAAO,GAA2C;QACtD,YAAY,EAAG,CAAC,EAAE,EAAE,EAAE,CAAC;QACvB,WAAW,EAAI,CAAC,CAAC,EAAE,EAAE,CAAC;QACtB,WAAW,EAAI,CAAC,CAAC,EAAE,EAAE,CAAC;QACtB,YAAY,EAAG,CAAC,CAAC,EAAE,EAAE,CAAC;QACtB,QAAQ,EAAO,CAAC,CAAC,EAAE,EAAE,CAAC;QACtB,YAAY,EAAG,CAAC,CAAC,EAAE,EAAE,CAAC;KACvB,CAAC;IACF,MAAM,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC9C,MAAM,UAAU,GAAG,UAAU,GAAG,eAAe,CAAC,CAAC,wBAAwB;IACzE,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC;IAE3C,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC;IACxE,MAAM,KAAK,GAAG,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;IAChF,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;AACnC,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,SAAwB;IACtD,MAAM,QAAQ,GAAG,CAAC,SAAS,EAAE,QAAQ,IAAI,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,cAAc,EAAE,CAAiB,CAAC;IAC/G,MAAM,aAAa,GAAG,SAAS,EAAE,KAAK,CAAC;IACvC,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,WAAW;YACd,OAAO,qBAAqB,CAAC,aAAa,CAAC,CAAC;QAC9C,KAAK,YAAY;YACf,OAAO,qBAAqB,CAAC,aAAa,CAAC,CAAC;QAC9C,KAAK,WAAW;YACd,OAAO,oBAAoB,CAAC,aAAa,CAAC,CAAC;QAC7C,KAAK,YAAY;YACf,OAAO,qBAAqB,CAAC,aAAa,CAAC,CAAC;QAC9C,KAAK,QAAQ;YACX,OAAO,kBAAkB,CAAC,aAAa,CAAC,CAAC;QAC3C,KAAK,YAAY;YACf,OAAO,sBAAsB,CAAC,aAAa,CAAC,CAAC;QAC/C;YACE,MAAM,IAAI,KAAK,CACb,kCAAkC,QAAQ,yFAAyF,CACpI,CAAC;IACN,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,gEAAgE;AAChE,qEAAqE;AACrE,yEAAyE;AACzE,8EAA8E;AAC9E,MAAM,oBAAoB,GAA2B;IACnD,eAAe,EAAE,QAAQ;IACzB,aAAa,EAAE,MAAM;IACrB,cAAc,EAAE,OAAO;IACvB,mBAAmB,EAAE,QAAQ;IAC7B,0BAA0B,EAAE,QAAQ;IACpC,iBAAiB,EAAE,MAAM;IACzB,iBAAiB,EAAE,MAAM;IACzB,wBAAwB,EAAE,MAAM;IAChC,2BAA2B,EAAE,OAAO;CACrC,CAAC;AAEF,MAAM,mBAAmB,GAA2B;IAClD,QAAQ,EAAE,mBAAmB;IAC7B,MAAM,EAAE,iBAAiB;IACzB,OAAO,EAAE,2BAA2B;IACpC,eAAe,EAAE,mBAAmB;IACpC,aAAa,EAAE,iBAAiB;IAChC,cAAc,EAAE,2BAA2B;CAC5C,CAAC;AAEF,SAAS,uBAAuB,CAAC,KAAa;IAC5C,OAAO,oBAAoB,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC;AAC9C,CAAC;AAED,SAAS,uBAAuB,CAAC,KAAa;IAC5C,OAAO,mBAAmB,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC;AAC7C,CAAC;AAED,8EAA8E;AAC9E,0BAA0B;AAC1B,8EAA8E;AAC9E,SAAS,qBAAqB,CAAC,aAAsB;IACnD,MAAM,aAAa,GAAG,mBAAmB,CAAC;IAE1C,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;IAC7C,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CACb,sRAAsR,CACvR,CAAC;IACJ,CAAC;IAED,MAAM,GAAG,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,aAAa,CAAC;IAC5E,MAAM,KAAK,GAAG,uBAAuB,CAAC,GAAG,CAAC,CAAC;IAC3C,IAAI,cAAc,GAAQ,IAAI,CAAC;IAE/B,OAAO;QACL,KAAK;QACL,KAAK,CAAC,QAAQ,CAAC,YAAoB,EAAE,UAAkB;YACrD,IAAI,CAAC,cAAc,EAAE,CAAC;gBACpB,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,CAAC;gBACjE,cAAc,GAAG,IAAI,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;YAC7C,CAAC;YAED,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;YACzC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,YAAY,EAAE,CAAC,CAAC;YACrE,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACzB,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,QAAQ,CAAC,MAAM,CACnD;oBACE,KAAK;oBACL,MAAM,EAAE,YAAY;oBACpB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;oBACjD,UAAU,EAAE,IAAI;iBACjB,EACD,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAC9B,CAAC;gBACF,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;gBAEtC,MAAM,SAAS,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;gBACvE,MAAM,IAAI,GAAG,SAAS,IAAI,MAAM,IAAI,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;gBACpE,MAAM,WAAW,GAAG,QAAQ,CAAC,KAAK,EAAE,YAAY,IAAI,IAAI,CAAC;gBACzD,MAAM,YAAY,GAAG,QAAQ,CAAC,KAAK,EAAE,aAAa,IAAI,IAAI,CAAC;gBAC3D,OAAO;oBACL,IAAI;oBACJ,UAAU;oBACV,WAAW;oBACX,YAAY;oBACZ,IAAI,EAAE,aAAa,CAAC,WAAW,EAAE,KAAK,EAAE,WAAW,EAAE,YAAY,CAAC;oBAClE,WAAW,EAAE,cAAc,CAAC,WAAW,CAAC;iBACzC,CAAC;YACJ,CAAC;oBAAS,CAAC;gBACT,YAAY,CAAC,OAAO,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC;AAkBD,SAAS,eAAe,CAAC,MAAiB;IACxC,uEAAuE;IACvE,MAAM,cAAc,GAAG,gBAAgB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACvD,gFAAgF;IAChF,MAAM,kBAAkB,GAAG,YAAY,EAAE,CAAC;IAE1C,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,YAAY;QAC1B,KAAK,CAAC,QAAQ,CAAC,YAAoB,EAAE,UAAkB;YACrD,MAAM,cAAc,GAAG,GAAG,YAAY,OAAO,UAAU,EAAE,CAAC;YAC1D,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAEzB,MAAM,IAAI,GAAG,MAAM,IAAI,OAAO,CAAS,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBACzD,mEAAmE;gBACnE,IAAI,GAA2B,CAAC;gBAChC,IAAI,MAAM,CAAC,cAAc,EAAE,CAAC;oBAC1B,GAAG,GAAG,EAAE,CAAC;oBACT,MAAM,MAAM,GAAG,MAAM,CAAC,cAAc,CAAC;oBACrC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;wBACjD,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;4BAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;oBAC3D,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,GAAG,GAAG,EAAE,GAAG,OAAO,CAAC,GAA6B,EAAE,CAAC;gBACrD,CAAC;gBACD,GAAG,CAAC,IAAI,GAAG,kBAAkB,CAAC;gBAE9B,sEAAsE;gBACtE,MAAM,UAAU,GAAG,OAAO,CAAC,QAAQ,KAAK,OAAO,IAAI,eAAe,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;gBAExF,MAAM,IAAI,GAAG,KAAK,CAAC,cAAc,EAAE,MAAM,CAAC,IAAI,EAAE;oBAC9C,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;oBAC/B,GAAG;oBACH,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBACvC,CAAC,CAAC;gBAEH,IAAI,MAAM,GAAG,EAAE,CAAC;gBAChB,IAAI,MAAM,GAAG,EAAE,CAAC;gBAChB,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAS,EAAE,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;gBACnE,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAS,EAAE,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;gBAEnE,MAAM,SAAS,GAAG,YAAY,EAAE,CAAC;gBACjC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;oBAC5B,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBACrB,MAAM,CAAC,IAAI,KAAK,CAAC,GAAG,MAAM,CAAC,IAAI,wBAAwB,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC;gBAC/E,CAAC,EAAE,SAAS,CAAC,CAAC;gBAEd,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAA0B,EAAE,EAAE;oBAC9C,YAAY,CAAC,KAAK,CAAC,CAAC;oBACpB,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;wBAC1B,MAAM,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;oBACxC,CAAC;yBAAM,CAAC;wBACN,MAAM,CAAC,IAAI,KAAK,CAAC,GAAG,MAAM,CAAC,IAAI,gBAAgB,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;oBACjE,CAAC;gBACH,CAAC,CAAC,CAAC;gBAEH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;oBACxB,YAAY,CAAC,KAAK,CAAC,CAAC;oBACpB,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;wBACf,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;oBACzB,CAAC;yBAAM,CAAC;wBACN,MAAM,MAAM,GAAG,CAAC,MAAM,IAAI,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;wBAChD,MAAM,CAAC,IAAI,KAAK,CACd,GAAG,MAAM,CAAC,IAAI,yBAAyB,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAC5E,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC,CAAC,CAAC;gBAEH,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;YACjC,CAAC,CAAC,CAAC;YAEH,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE,cAAc,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QACnJ,CAAC;KACF,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,4EAA4E;AAC5E,uEAAuE;AACvE,8EAA8E;AAC9E,SAAS,qBAAqB,CAAC,aAAsB;IACnD,MAAM,GAAG,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,QAAQ,CAAC;IACvE,MAAM,KAAK,GAAG,uBAAuB,CAAC,GAAG,CAAC,CAAC;IAC3C,OAAO,eAAe,CAAC;QACrB,MAAM,EAAE,QAAQ;QAChB,IAAI,EAAE,QAAQ;QACd,IAAI,EAAE,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC;QAC9B,YAAY,EAAE,KAAK;QACnB,cAAc,EAAE,QAAQ;QACxB,QAAQ,EAAE,YAAY;QACtB,WAAW,EACT,qJAAqJ;KACxJ,CAAC,CAAC;AACL,CAAC;AAED,8EAA8E;AAC9E,iFAAiF;AACjF,8EAA8E;AAC9E,SAAS,oBAAoB,CAAC,aAAsB;IAClD,MAAM,KAAK,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,SAAS,CAAC;IAC1E,OAAO,eAAe,CAAC;QACrB,MAAM,EAAE,OAAO;QACf,IAAI,EAAE,OAAO;QACb,IAAI,EAAE,CAAC,MAAM,EAAE,SAAS,EAAE,KAAK,CAAC;QAChC,YAAY,EAAE,SAAS,KAAK,EAAE;QAC9B,QAAQ,EAAE,WAAW;QACrB,WAAW,EACT,4IAA4I;KAC/I,CAAC,CAAC;AACL,CAAC;AAED,8EAA8E;AAC9E,iFAAiF;AACjF,kFAAkF;AAClF,8EAA8E;AAC9E,SAAS,qBAAqB,CAAC,aAAsB;IACnD,MAAM,KAAK,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,gBAAgB,CAAC;IACjF,OAAO,eAAe,CAAC;QACrB,MAAM,EAAE,QAAQ;QAChB,IAAI,EAAE,QAAQ;QACd,IAAI,EAAE,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC;QAC9B,YAAY,EAAE,KAAK;QACnB,QAAQ,EAAE,YAAY;QACtB,WAAW,EACT,kJAAkJ;KACrJ,CAAC,CAAC;AACL,CAAC;AAED,8EAA8E;AAC9E,qDAAqD;AACrD,8EAA8E;AAC9E,SAAS,kBAAkB,CAAC,aAAsB;IAChD,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,wBAAwB,CAAC;IACxE,MAAM,KAAK,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,aAAa,CAAC;IAE9E,OAAO;QACL,KAAK;QACL,KAAK,CAAC,QAAQ,CAAC,YAAoB,EAAE,UAAkB;YACrD,MAAM,UAAU,GAAG,GAAG,YAAY,OAAO,UAAU,EAAE,CAAC;YACtD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAEzB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,OAAO,eAAe,EAAE;gBACtD,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;gBAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,KAAK;oBACL,MAAM,EAAE,UAAU;oBAClB,MAAM,EAAE,KAAK;oBACb,OAAO,EAAE;wBACP,WAAW,EAAE,IAAI;wBACjB,WAAW,EAAE,GAAG;qBACjB;iBACF,CAAC;gBACF,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC;aAC5C,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACpC,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;oBAC3D,MAAM,IAAI,KAAK,CACb,iBAAiB,KAAK,8CAA8C,KAAK,EAAE,CAC5E,CAAC;gBACJ,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,0BAA0B,KAAK,EAAE,CAAC,CAAC;YACrD,CAAC;YAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAIlC,CAAC;YACF,OAAO;gBACL,IAAI,EAAE,IAAI,CAAC,QAAQ,IAAI,EAAE;gBACzB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;gBAC9B,WAAW,EAAE,IAAI,CAAC,iBAAiB,IAAI,IAAI;gBAC3C,YAAY,EAAE,IAAI,CAAC,UAAU,IAAI,IAAI;gBACrC,IAAI,EAAE,CAAC;gBACP,WAAW,EAAE,MAAe;aAC7B,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,wDAAwD;AACxD,8EAA8E;AAC9E,SAAS,sBAAsB,CAAC,aAAsB;IACpD,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC;IAC9C,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CACb,oIAAoI,CACrI,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,2BAA2B,CAAC;IAE5F,OAAO;QACL,KAAK;QACL,KAAK,CAAC,QAAQ,CAAC,YAAoB,EAAE,UAAkB;YACrD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAEzB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,+CAA+C,EAAE;gBAC5E,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,eAAe,EAAE,UAAU,MAAM,EAAE;oBACnC,cAAc,EAAE,4BAA4B;oBAC5C,SAAS,EAAE,eAAe;iBAC3B;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,KAAK;oBACL,QAAQ,EAAE;wBACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,EAAE;wBACzC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE;qBACtC;oBACD,UAAU,EAAE,IAAI;iBACjB,CAAC;gBACF,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC;aAC5C,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACpC,MAAM,IAAI,KAAK,CAAC,8BAA8B,QAAQ,CAAC,MAAM,MAAM,KAAK,EAAE,CAAC,CAAC;YAC9E,CAAC;YAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAOlC,CAAC;YAEF,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,IAAI,EAAE,CAAC;YACvD,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,EAAE,aAAa,IAAI,IAAI,CAAC;YACtD,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,EAAE,iBAAiB,IAAI,IAAI,CAAC;YAC3D,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,EAAE,UAAU,IAAI,IAAI,CAAC;YAC/C,OAAO;gBACL,IAAI;gBACJ,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;gBAC9B,WAAW;gBACX,YAAY;gBACZ,IAAI,EAAE,OAAO,IAAI,aAAa,CAAC,YAAY,EAAE,KAAK,EAAE,WAAW,EAAE,YAAY,CAAC;gBAC9E,WAAW,EAAE,WAAoB;aAClC,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC"}
+{"version":3,"file":"llm.js","sourceRoot":"","sources":["../../src/eval/llm.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,+EAA+E;AAC/E,EAAE;AACF,uDAAuD;AACvD,wEAAwE;AACxE,iFAAiF;AACjF,mEAAmE;AACnE,8DAA8D;AAC9D,uEAAuE;AACvE,EAAE;AACF,uDAAuD;AACvD,iFAAiF;AACjF,2EAA2E;AAC3E,EAAE;AACF,mEAAmE;AACnE,oEAAoE;AACpE,EAAE;AACF,iDAAiD;AACjD,+DAA+D;AAC/D,4EAA4E;AAC5E,iFAAiF;AACjF,4DAA4D;AAC5D,mDAAmD;AACnD,8EAA8E;AAE9E,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC5E,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAoB7D,SAAS,cAAc;IACrB,OAAO,YAAY,CAAC;AACtB,CAAC;AAED,oFAAoF;AACpF,SAAS,YAAY;IACnB,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC;IAC/C,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACrC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,CAAC;YAAE,OAAO,OAAO,GAAG,IAAI,CAAC;IAC5D,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAOD;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CACjC,QAAsB,EACtB,UAAkB,EAClB,eAAuB;IAEvB,+CAA+C;IAC/C,MAAM,OAAO,GAA2C;QACtD,YAAY,EAAG,CAAC,EAAE,EAAE,EAAE,CAAC;QACvB,WAAW,EAAI,CAAC,CAAC,EAAE,EAAE,CAAC;QACtB,WAAW,EAAI,CAAC,CAAC,EAAE,EAAE,CAAC;QACtB,YAAY,EAAG,CAAC,CAAC,EAAE,EAAE,CAAC;QACtB,QAAQ,EAAO,CAAC,CAAC,EAAE,EAAE,CAAC;QACtB,WAAW,EAAI,CAAC,CAAC,EAAE,EAAE,CAAC;QACtB,YAAY,EAAG,CAAC,CAAC,EAAE,EAAE,CAAC;KACvB,CAAC;IACF,MAAM,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC9C,MAAM,UAAU,GAAG,UAAU,GAAG,eAAe,CAAC,CAAC,wBAAwB;IACzE,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC;IAE3C,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC;IACxE,MAAM,KAAK,GAAG,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;IAChF,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;AACnC,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,SAAwB;IACtD,MAAM,QAAQ,GAAG,CAAC,SAAS,EAAE,QAAQ,IAAI,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,cAAc,EAAE,CAAiB,CAAC;IAC/G,MAAM,aAAa,GAAG,SAAS,EAAE,KAAK,CAAC;IACvC,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,WAAW;YACd,OAAO,qBAAqB,CAAC,aAAa,CAAC,CAAC;QAC9C,KAAK,YAAY;YACf,OAAO,qBAAqB,CAAC,aAAa,CAAC,CAAC;QAC9C,KAAK,WAAW;YACd,OAAO,oBAAoB,CAAC,aAAa,CAAC,CAAC;QAC7C,KAAK,YAAY;YACf,OAAO,qBAAqB,CAAC,aAAa,CAAC,CAAC;QAC9C,KAAK,QAAQ;YACX,OAAO,kBAAkB,CAAC,aAAa,CAAC,CAAC;QAC3C,KAAK,WAAW;YACd,OAAO,oBAAoB,CAAC,aAAa,CAAC,CAAC;QAC7C,KAAK,YAAY;YACf,OAAO,sBAAsB,CAAC,aAAa,CAAC,CAAC;QAC/C;YACE,MAAM,IAAI,KAAK,CACb,kCAAkC,QAAQ,sGAAsG,CACjJ,CAAC;IACN,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,gEAAgE;AAChE,qEAAqE;AACrE,yEAAyE;AACzE,8EAA8E;AAC9E,MAAM,oBAAoB,GAA2B;IACnD,eAAe,EAAE,QAAQ;IACzB,aAAa,EAAE,MAAM;IACrB,cAAc,EAAE,OAAO;IACvB,mBAAmB,EAAE,QAAQ;IAC7B,0BAA0B,EAAE,QAAQ;IACpC,iBAAiB,EAAE,MAAM;IACzB,iBAAiB,EAAE,MAAM;IACzB,wBAAwB,EAAE,MAAM;IAChC,2BAA2B,EAAE,OAAO;CACrC,CAAC;AAEF,MAAM,mBAAmB,GAA2B;IAClD,QAAQ,EAAE,mBAAmB;IAC7B,MAAM,EAAE,iBAAiB;IACzB,OAAO,EAAE,2BAA2B;IACpC,eAAe,EAAE,mBAAmB;IACpC,aAAa,EAAE,iBAAiB;IAChC,cAAc,EAAE,2BAA2B;CAC5C,CAAC;AAEF,SAAS,uBAAuB,CAAC,KAAa;IAC5C,OAAO,oBAAoB,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC;AAC9C,CAAC;AAED,SAAS,uBAAuB,CAAC,KAAa;IAC5C,OAAO,mBAAmB,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC;AAC7C,CAAC;AAED,8EAA8E;AAC9E,0BAA0B;AAC1B,8EAA8E;AAC9E,SAAS,qBAAqB,CAAC,aAAsB;IACnD,MAAM,aAAa,GAAG,mBAAmB,CAAC;IAE1C,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;IAC7C,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CACb,sRAAsR,CACvR,CAAC;IACJ,CAAC;IAED,MAAM,GAAG,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,aAAa,CAAC;IAC5E,MAAM,KAAK,GAAG,uBAAuB,CAAC,GAAG,CAAC,CAAC;IAC3C,IAAI,cAAc,GAAQ,IAAI,CAAC;IAE/B,OAAO;QACL,KAAK;QACL,KAAK,CAAC,QAAQ,CAAC,YAAoB,EAAE,UAAkB;YACrD,IAAI,CAAC,cAAc,EAAE,CAAC;gBACpB,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,CAAC;gBACjE,cAAc,GAAG,IAAI,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;YAC7C,CAAC;YAED,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;YACzC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,YAAY,EAAE,CAAC,CAAC;YACrE,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACzB,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,QAAQ,CAAC,MAAM,CACnD;oBACE,KAAK;oBACL,MAAM,EAAE,YAAY;oBACpB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;oBACjD,UAAU,EAAE,IAAI;iBACjB,EACD,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAC9B,CAAC;gBACF,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;gBAEtC,MAAM,SAAS,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;gBACvE,MAAM,IAAI,GAAG,SAAS,IAAI,MAAM,IAAI,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;gBACpE,MAAM,WAAW,GAAG,QAAQ,CAAC,KAAK,EAAE,YAAY,IAAI,IAAI,CAAC;gBACzD,MAAM,YAAY,GAAG,QAAQ,CAAC,KAAK,EAAE,aAAa,IAAI,IAAI,CAAC;gBAC3D,OAAO;oBACL,IAAI;oBACJ,UAAU;oBACV,WAAW;oBACX,YAAY;oBACZ,IAAI,EAAE,aAAa,CAAC,WAAW,EAAE,KAAK,EAAE,WAAW,EAAE,YAAY,CAAC;oBAClE,WAAW,EAAE,cAAc,CAAC,WAAW,CAAC;iBACzC,CAAC;YACJ,CAAC;oBAAS,CAAC;gBACT,YAAY,CAAC,OAAO,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC;AAkBD,SAAS,eAAe,CAAC,MAAiB;IACxC,uEAAuE;IACvE,MAAM,cAAc,GAAG,gBAAgB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACvD,gFAAgF;IAChF,MAAM,kBAAkB,GAAG,YAAY,EAAE,CAAC;IAE1C,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,YAAY;QAC1B,KAAK,CAAC,QAAQ,CAAC,YAAoB,EAAE,UAAkB;YACrD,MAAM,cAAc,GAAG,GAAG,YAAY,OAAO,UAAU,EAAE,CAAC;YAC1D,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAEzB,MAAM,IAAI,GAAG,MAAM,IAAI,OAAO,CAAS,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBACzD,mEAAmE;gBACnE,IAAI,GAA2B,CAAC;gBAChC,IAAI,MAAM,CAAC,cAAc,EAAE,CAAC;oBAC1B,GAAG,GAAG,EAAE,CAAC;oBACT,MAAM,MAAM,GAAG,MAAM,CAAC,cAAc,CAAC;oBACrC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;wBACjD,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;4BAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;oBAC3D,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,GAAG,GAAG,EAAE,GAAG,OAAO,CAAC,GAA6B,EAAE,CAAC;gBACrD,CAAC;gBACD,GAAG,CAAC,IAAI,GAAG,kBAAkB,CAAC;gBAE9B,sEAAsE;gBACtE,MAAM,UAAU,GAAG,OAAO,CAAC,QAAQ,KAAK,OAAO,IAAI,eAAe,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;gBAExF,MAAM,IAAI,GAAG,KAAK,CAAC,cAAc,EAAE,MAAM,CAAC,IAAI,EAAE;oBAC9C,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;oBAC/B,GAAG;oBACH,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBACvC,CAAC,CAAC;gBAEH,IAAI,MAAM,GAAG,EAAE,CAAC;gBAChB,IAAI,MAAM,GAAG,EAAE,CAAC;gBAChB,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAS,EAAE,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;gBACnE,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAS,EAAE,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;gBAEnE,MAAM,SAAS,GAAG,YAAY,EAAE,CAAC;gBACjC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;oBAC5B,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBACrB,MAAM,CAAC,IAAI,KAAK,CAAC,GAAG,MAAM,CAAC,IAAI,wBAAwB,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC;gBAC/E,CAAC,EAAE,SAAS,CAAC,CAAC;gBAEd,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAA0B,EAAE,EAAE;oBAC9C,YAAY,CAAC,KAAK,CAAC,CAAC;oBACpB,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;wBAC1B,MAAM,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;oBACxC,CAAC;yBAAM,CAAC;wBACN,MAAM,CAAC,IAAI,KAAK,CAAC,GAAG,MAAM,CAAC,IAAI,gBAAgB,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;oBACjE,CAAC;gBACH,CAAC,CAAC,CAAC;gBAEH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;oBACxB,YAAY,CAAC,KAAK,CAAC,CAAC;oBACpB,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;wBACf,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;oBACzB,CAAC;yBAAM,CAAC;wBACN,MAAM,MAAM,GAAG,CAAC,MAAM,IAAI,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;wBAChD,MAAM,CAAC,IAAI,KAAK,CACd,GAAG,MAAM,CAAC,IAAI,yBAAyB,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAC5E,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC,CAAC,CAAC;gBAEH,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;YACjC,CAAC,CAAC,CAAC;YAEH,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE,cAAc,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QACnJ,CAAC;KACF,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,4EAA4E;AAC5E,uEAAuE;AACvE,8EAA8E;AAC9E,SAAS,qBAAqB,CAAC,aAAsB;IACnD,MAAM,GAAG,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,QAAQ,CAAC;IACvE,MAAM,KAAK,GAAG,uBAAuB,CAAC,GAAG,CAAC,CAAC;IAC3C,OAAO,eAAe,CAAC;QACrB,MAAM,EAAE,QAAQ;QAChB,IAAI,EAAE,QAAQ;QACd,IAAI,EAAE,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC;QAC9B,YAAY,EAAE,KAAK;QACnB,cAAc,EAAE,QAAQ;QACxB,QAAQ,EAAE,YAAY;QACtB,WAAW,EACT,qJAAqJ;KACxJ,CAAC,CAAC;AACL,CAAC;AAED,8EAA8E;AAC9E,iFAAiF;AACjF,8EAA8E;AAC9E,SAAS,oBAAoB,CAAC,aAAsB;IAClD,MAAM,KAAK,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,SAAS,CAAC;IAC1E,OAAO,eAAe,CAAC;QACrB,MAAM,EAAE,OAAO;QACf,IAAI,EAAE,OAAO;QACb,IAAI,EAAE,CAAC,MAAM,EAAE,SAAS,EAAE,KAAK,CAAC;QAChC,YAAY,EAAE,SAAS,KAAK,EAAE;QAC9B,QAAQ,EAAE,WAAW;QACrB,WAAW,EACT,4IAA4I;KAC/I,CAAC,CAAC;AACL,CAAC;AAED,8EAA8E;AAC9E,iFAAiF;AACjF,kFAAkF;AAClF,8EAA8E;AAC9E,SAAS,qBAAqB,CAAC,aAAsB;IACnD,MAAM,KAAK,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,gBAAgB,CAAC;IACjF,OAAO,eAAe,CAAC;QACrB,MAAM,EAAE,QAAQ;QAChB,IAAI,EAAE,QAAQ;QACd,IAAI,EAAE,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC;QAC9B,YAAY,EAAE,KAAK;QACnB,QAAQ,EAAE,YAAY;QACtB,WAAW,EACT,kJAAkJ;KACrJ,CAAC,CAAC;AACL,CAAC;AAED,8EAA8E;AAC9E,qDAAqD;AACrD,8EAA8E;AAC9E,SAAS,kBAAkB,CAAC,aAAsB;IAChD,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,wBAAwB,CAAC;IACxE,MAAM,KAAK,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,aAAa,CAAC;IAE9E,OAAO;QACL,KAAK;QACL,KAAK,CAAC,QAAQ,CAAC,YAAoB,EAAE,UAAkB;YACrD,MAAM,UAAU,GAAG,GAAG,YAAY,OAAO,UAAU,EAAE,CAAC;YACtD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAEzB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,OAAO,eAAe,EAAE;gBACtD,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;gBAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,KAAK;oBACL,MAAM,EAAE,UAAU;oBAClB,MAAM,EAAE,KAAK;oBACb,OAAO,EAAE;wBACP,WAAW,EAAE,IAAI;wBACjB,WAAW,EAAE,GAAG;qBACjB;iBACF,CAAC;gBACF,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC;aAC5C,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACpC,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;oBAC3D,MAAM,IAAI,KAAK,CACb,iBAAiB,KAAK,8CAA8C,KAAK,EAAE,CAC5E,CAAC;gBACJ,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,0BAA0B,KAAK,EAAE,CAAC,CAAC;YACrD,CAAC;YAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAIlC,CAAC;YACF,OAAO;gBACL,IAAI,EAAE,IAAI,CAAC,QAAQ,IAAI,EAAE;gBACzB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;gBAC9B,WAAW,EAAE,IAAI,CAAC,iBAAiB,IAAI,IAAI;gBAC3C,YAAY,EAAE,IAAI,CAAC,UAAU,IAAI,IAAI;gBACrC,IAAI,EAAE,CAAC;gBACP,WAAW,EAAE,MAAe;aAC7B,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,wDAAwD;AACxD,8EAA8E;AAC9E,SAAS,sBAAsB,CAAC,aAAsB;IACpD,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC;IAC9C,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CACb,oIAAoI,CACrI,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,2BAA2B,CAAC;IAE5F,OAAO;QACL,KAAK;QACL,KAAK,CAAC,QAAQ,CAAC,YAAoB,EAAE,UAAkB;YACrD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAEzB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,+CAA+C,EAAE;gBAC5E,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,eAAe,EAAE,UAAU,MAAM,EAAE;oBACnC,cAAc,EAAE,4BAA4B;oBAC5C,SAAS,EAAE,eAAe;iBAC3B;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,KAAK;oBACL,QAAQ,EAAE;wBACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,EAAE;wBACzC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE;qBACtC;oBACD,UAAU,EAAE,IAAI;iBACjB,CAAC;gBACF,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC;aAC5C,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACpC,MAAM,IAAI,KAAK,CAAC,8BAA8B,QAAQ,CAAC,MAAM,MAAM,KAAK,EAAE,CAAC,CAAC;YAC9E,CAAC;YAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAOlC,CAAC;YAEF,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,IAAI,EAAE,CAAC;YACvD,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,EAAE,aAAa,IAAI,IAAI,CAAC;YACtD,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,EAAE,iBAAiB,IAAI,IAAI,CAAC;YAC3D,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,EAAE,UAAU,IAAI,IAAI,CAAC;YAC/C,OAAO;gBACL,IAAI;gBACJ,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;gBAC9B,WAAW;gBACX,YAAY;gBACZ,IAAI,EAAE,OAAO,IAAI,aAAa,CAAC,YAAY,EAAE,KAAK,EAAE,WAAW,EAAE,YAAY,CAAC;gBAC9E,WAAW,EAAE,WAAoB;aAClC,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,0EAA0E;AAC1E,EAAE;AACF,6EAA6E;AAC7E,6EAA6E;AAC7E,0CAA0C;AAC1C,6EAA6E;AAC7E,6EAA6E;AAC7E,yCAAyC;AACzC,EAAE;AACF,iBAAiB;AACjB,mEAAmE;AACnE,+DAA+D;AAC/D,EAAE;AACF,wEAAwE;AACxE,6EAA6E;AAC7E,sEAAsE;AACtE,gBAAgB;AAChB,8EAA8E;AAC9E,SAAS,oBAAoB,CAAC,aAAsB;IAClD,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,0BAA0B,CAAC;IAC7E,MAAM,KAAK,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,aAAa,CAAC;IAE9E,OAAO;QACL,KAAK;QACL,KAAK,CAAC,QAAQ,CAAC,YAAoB,EAAE,UAAkB;YACrD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAEzB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,OAAO,mBAAmB,EAAE;gBAC1D,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,eAAe,EAAE,kBAAkB;iBACpC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,KAAK;oBACL,QAAQ,EAAE;wBACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,EAAE;wBACzC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE;qBACtC;oBACD,UAAU,EAAE,IAAI;iBACjB,CAAC;gBACF,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC;aAC5C,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACxC,MAAM,IAAI,KAAK,CACb,6BAA6B,QAAQ,CAAC,MAAM,MAAM,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAC5E,CAAC;YACJ,CAAC;YAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAMlC,CAAC;YAEF,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,IAAI,EAAE,CAAC;YACvD,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,EAAE,aAAa,IAAI,IAAI,CAAC;YACtD,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,EAAE,iBAAiB,IAAI,IAAI,CAAC;YAC3D,OAAO;gBACL,IAAI;gBACJ,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;gBAC9B,WAAW;gBACX,YAAY;gBACZ,IAAI,EAAE,CAAC;gBACP,WAAW,EAAE,MAAe;aAC7B,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC"}

package/dist/eval-server/api-routes.d.ts

@@ -1,4 +1,5 @@
 import type { Router } from "./router.js";
+import type { ProviderName } from "../eval/llm.js";
 import type { AgentDefinition } from "../agents/agents-registry.js";
 export interface InstalledAgentEntry {
     id: string;
@@ -50,4 +51,17 @@ export declare function deriveSourceAgent(skillDir: string, root: string, origin
  * response never fails because of a single bad skill.
  */
 export declare function buildSkillMetadata(skillDir: string, origin: "source" | "installed", root: string): SkillMetadataFields;
+interface ModelOption {
+    id: string;
+    label: string;
+}
+/** Test hook: clear all probe caches so the next detectAvailableProviders() re-probes. */
+export declare function resetDetectionCache(): void;
+export declare function detectAvailableProviders(): Promise<Array<{
+    id: ProviderName;
+    label: string;
+    available: boolean;
+    models: ModelOption[];
+}>>;
 export declare function registerRoutes(router: Router, root: string, projectName?: string): void;
+export {};

package/dist/eval-server/api-routes.js

@@ -299,16 +299,34 @@ const PROVIDER_MODELS = {
         { id: "meta-llama/llama-3.1-70b-instruct", label: "Llama 3.1 70B" },
         { id: "google/gemini-2.5-pro", label: "Gemini 2.5 Pro (via OpenRouter)" },
     ],
+    // LM Studio's default model list is empty because the actual list depends on
+    // what models the user has loaded. The probe at probeLmStudio() populates
+    // this dynamically from GET /v1/models.
+    "lm-studio": [],
 };
 // ---------------------------------------------------------------------------
-// Ollama detection cache — avoids 500ms+ probe on every /api/config request.
-// Without this, page load blocks on a 2s timeout when Ollama is not running.
+// Local provider detection caches — avoid 500ms+ probes on every /api/config
+// request. Without the caches, page load blocks on the timeout when the
+// local server is not running. TTL is 30s to balance freshness with latency.
+//
+// Both Ollama and LM Studio share the same TTL and silent-failure semantics
+// (probe → non-2xx / throw → `available: false`, no log above debug).
+//
+// Follow-up (out of scope for 0677): Ollama's upstream standard env var is
+// OLLAMA_HOST but this codebase uses OLLAMA_BASE_URL. Do not change here;
+// tracked separately.
 // ---------------------------------------------------------------------------
+const PROBE_CACHE_TTL = 30_000; // re-probe every 30s
 let ollamaCache = null;
-const OLLAMA_CACHE_TTL = 30_000; // re-probe every 30s
+let lmStudioCache = null;
+/** Test hook: clear all probe caches so the next detectAvailableProviders() re-probes. */
+export function resetDetectionCache() {
+    ollamaCache = null;
+    lmStudioCache = null;
+}
 async function probeOllama() {
     const now = Date.now();
-    if (ollamaCache && now - ollamaCache.ts < OLLAMA_CACHE_TTL) {
+    if (ollamaCache && now - ollamaCache.ts < PROBE_CACHE_TTL) {
         return ollamaCache;
     }
     let models = PROVIDER_MODELS["ollama"];
@@ -328,7 +346,35 @@ async function probeOllama() {
     ollamaCache = { available, models, ts: now };
     return ollamaCache;
 }
-async function detectAvailableProviders() {
+// ---------------------------------------------------------------------------
+// probeLmStudio — hits GET <base>/models to detect LM Studio and populate the
+// model list from the server's loaded models. Mirrors the Ollama pattern:
+// 500ms AbortSignal timeout, 30s in-memory cache, silent failure on any
+// exception. Base URL is overridable via LM_STUDIO_BASE_URL.
+// ---------------------------------------------------------------------------
+async function probeLmStudio() {
+    const now = Date.now();
+    if (lmStudioCache && now - lmStudioCache.ts < PROBE_CACHE_TTL) {
+        return lmStudioCache;
+    }
+    let models = PROVIDER_MODELS["lm-studio"];
+    let available = false;
+    try {
+        const baseUrl = process.env.LM_STUDIO_BASE_URL || "http://localhost:1234/v1";
+        const resp = await fetch(`${baseUrl}/models`, { signal: AbortSignal.timeout(500) });
+        if (resp.ok) {
+            available = true;
+            const data = await resp.json();
+            if (data.data?.length) {
+                models = data.data.map((m) => ({ id: m.id, label: m.id }));
+            }
+        }
+    }
+    catch { /* lm studio not running */ }
+    lmStudioCache = { available, models, ts: now };
+    return lmStudioCache;
+}
+export async function detectAvailableProviders() {
     const providers = [];
     // Claude CLI — always available for the eval server (runs in a separate terminal)
     providers.push({
@@ -351,14 +397,21 @@ async function detectAvailableProviders() {
         available: !!process.env.OPENROUTER_API_KEY,
         models: PROVIDER_MODELS["openrouter"],
     });
-    // Ollama — cached probe (500ms timeout, refreshes every 30s)
-    const ollama = await probeOllama();
+    // Local providers (Ollama + LM Studio) — cached probes fired in parallel so
+    // total detection time stays ≤ 550ms even if both time out.
+    const [ollama, lmStudio] = await Promise.all([probeOllama(), probeLmStudio()]);
     providers.push({
         id: "ollama",
         label: "Ollama (local, free)",
         available: ollama.available,
         models: ollama.models,
     });
+    providers.push({
+        id: "lm-studio",
+        label: "LM Studio (local, free)",
+        available: lmStudio.available,
+        models: lmStudio.models,
+    });
     return providers;
 }
 export function registerRoutes(router, root, projectName) {