pi-bifrost 0.1.0

package/tests/integration.test.mjs

@@ -0,0 +1,157 @@
+import { describe, it, before } from "node:test";
+import assert from "node:assert/strict";
+import { spawn } from "node:child_process";
+import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const EXTENSION_PATH = join(__dirname, "..", "index.ts");
+
+const PI_ARGS = [
+  "-e",
+  EXTENSION_PATH,
+  "--approve",
+  "--no-session",
+  "--print",
+  "-p",
+];
+
+async function runPi(command, cwd = process.cwd()) {
+  return new Promise((resolve, reject) => {
+    const child = spawn("pi", [...PI_ARGS, command], {
+      stdio: ["ignore", "pipe", "pipe"],
+      env: process.env,
+      cwd,
+    });
+
+    let stdout = "";
+    let stderr = "";
+    child.stdout.setEncoding("utf8");
+    child.stderr.setEncoding("utf8");
+    child.stdout.on("data", (chunk) => {
+      stdout += chunk;
+    });
+    child.stderr.on("data", (chunk) => {
+      stderr += chunk;
+    });
+
+    const timer = setTimeout(() => {
+      child.kill("SIGTERM");
+      const error = new Error(`pi timed out: ${command}`);
+      error.stdout = stdout;
+      error.stderr = stderr;
+      reject(error);
+    }, 120_000);
+
+    child.on("error", (err) => {
+      clearTimeout(timer);
+      reject(err);
+    });
+
+    child.on("close", (code) => {
+      clearTimeout(timer);
+      if (code !== 0) {
+        const error = new Error(`pi exited ${code}: ${command}`);
+        error.stdout = stdout;
+        error.stderr = stderr;
+        reject(error);
+      } else {
+        resolve({ stdout, stderr });
+      }
+    });
+  });
+}
+
+function combined(output) {
+  return `${output.stderr}\n${output.stdout}`;
+}
+
+describe("bifrost integration", { timeout: 300_000, concurrency: 1 }, () => {
+  before(async () => {
+    await runPi("/bifrost cache clear");
+  });
+
+  it("reports classifier status", async () => {
+    const out = combined(await runPi("/bifrost classifier status"));
+    assert.ok(out.includes("enabled=true"));
+    assert.ok(out.includes("opencode/mimo-v2.5-free"));
+    assert.ok(out.includes("endpoint=registry"));
+  });
+
+  it("classifies hello as economical", async () => {
+    const out = combined(await runPi("/bifrost preview hello"));
+    assert.ok(out.includes("source:    classifier"));
+    assert.ok(out.includes("category:  economical"));
+  });
+
+  it("classifies architecture prompt as frontier", async () => {
+    const out = combined(await runPi("/bifrost preview plan the architecture"));
+    assert.ok(out.includes("source:    classifier"));
+    assert.ok(out.includes("category:  frontier"));
+  });
+
+  it("caches classifications", async () => {
+    await runPi("/bifrost cache clear");
+    let out = combined(await runPi("/bifrost cache stats"));
+    assert.ok(out.includes("cache: 0 entries"));
+
+    // Run a live prompt. Routing to the local economical model may fail,
+    // but the classifier result is still cached before routing.
+    try {
+      await runPi("format this file");
+    } catch {
+      // acceptable; routing failure is not the concern of this test
+    }
+
+    out = combined(await runPi("/bifrost cache stats"));
+    assert.ok(out.includes("cache: 1 entries"));
+
+    out = combined(await runPi("/bifrost preview format this file"));
+    assert.ok(out.includes("source:    cache"));
+  });
+
+  it("falls back to regex when classifier is disabled in config", async () => {
+    const tempDir = mkdtempSync(join(tmpdir(), "bifrost-test-"));
+    mkdirSync(join(tempDir, ".pi"), { recursive: true });
+    writeFileSync(
+      join(tempDir, ".pi", "bifrost.json"),
+      JSON.stringify({ classifier: { enabled: false } }),
+    );
+
+    try {
+      const out = combined(
+        await runPi("/bifrost preview lint this file", tempDir),
+      );
+      assert.ok(out.includes("source:    regex"));
+    } finally {
+      rmSync(tempDir, { recursive: true, force: true });
+    }
+  });
+
+  it("classifies without endpoint when model is in pi registry", async () => {
+    const tempDir = mkdtempSync(join(tmpdir(), "bifrost-test-"));
+    mkdirSync(join(tempDir, ".pi"), { recursive: true });
+    writeFileSync(
+      join(tempDir, ".pi", "bifrost.json"),
+      JSON.stringify({
+        classifier: { model: "lmstudio/qwen/qwen3-vl-8b" },
+        models: {
+          economical: "lmstudio/qwen/qwen3-vl-8b",
+          frontier: "openai-codex/gpt-5.4",
+        },
+      }),
+    );
+
+    try {
+      const out = combined(
+        await runPi("/bifrost preview hello", tempDir),
+      );
+      assert.ok(out.includes("source:    classifier"));
+      assert.ok(out.includes("category:  economical"));
+    } finally {
+      rmSync(tempDir, { recursive: true, force: true });
+    }
+  });
+});

package/tests/pipeline.test.ts

@@ -0,0 +1,269 @@
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import type { ClassifierModel } from "../classifier.ts";
+import { createPipeline, type PipelineDeps } from "../classification-pipeline.ts";
+
+function makeModel(provider: string, id: string): ClassifierModel {
+  return {
+    kind: "registry",
+    model: {
+      provider,
+      id,
+      name: id,
+      api: "openai-completions",
+      baseUrl: "http://localhost:1234/v1",
+      reasoning: false,
+      input: ["text"],
+      cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
+      contextWindow: 128000,
+      maxTokens: 4096,
+    },
+  } as ClassifierModel;
+}
+
+function deps(overrides: Partial<PipelineDeps> = {}): PipelineDeps {
+  return {
+    cacheLookup: () => undefined,
+    classifierModels: [],
+    classifyWithLLM: async () => undefined,
+    regexRules: [],
+    defaultTier: undefined,
+    tiers: ["frontier", "economical"],
+    ...overrides,
+  };
+}
+
+describe("classification-pipeline", () => {
+  describe("unclassified", () => {
+    it("returns unclassified when no tiers configured", async () => {
+      const p = createPipeline(deps({ tiers: [] }));
+      const r = await p.classify("hello");
+      assert.equal(r.kind, "unclassified");
+    });
+
+    it("returns unclassified when nothing matches and no default", async () => {
+      const p = createPipeline(deps({ defaultTier: undefined }));
+      const r = await p.classify("hello");
+      assert.equal(r.kind, "unclassified");
+    });
+  });
+
+  describe("cache", () => {
+    it("returns classified from cache hit", async () => {
+      const p = createPipeline(
+        deps({ cacheLookup: () => "economical" }),
+      );
+      const r = await p.classify("hello");
+      assert.equal(r.kind, "classified");
+      if (r.kind === "classified") {
+        assert.equal(r.tier, "economical");
+        assert.equal(r.source, "cache");
+      }
+    });
+
+    it("skips cache when result is not a known tier", async () => {
+      const p = createPipeline(
+        deps({ cacheLookup: () => "unknown" }),
+      );
+      const r = await p.classify("hello");
+      // Falls through to default
+      assert.notEqual(r.kind, "classified");
+    });
+  });
+
+  describe("classifier", () => {
+    it("uses first successful classifier model", async () => {
+      let calls = 0;
+      const p = createPipeline(
+        deps({
+          classifierModels: [makeModel("a", "m1"), makeModel("b", "m2")],
+          classifyWithLLM: async (model) => {
+            calls++;
+            if (model.kind === "registry" && model.model.id === "m1") return "frontier";
+            return undefined;
+          },
+        }),
+      );
+      const r = await p.classify("debug this");
+      assert.equal(calls, 1); // second model never tried
+      assert.equal(r.kind, "classified");
+      if (r.kind === "classified") {
+        assert.equal(r.tier, "frontier");
+        assert.equal(r.source, "classifier");
+      }
+    });
+
+    it("tries second model when first fails", async () => {
+      let calls: string[] = [];
+      const p = createPipeline(
+        deps({
+          classifierModels: [makeModel("a", "m1"), makeModel("b", "m2")],
+          classifyWithLLM: async (model) => {
+            calls.push(model.kind === "registry" ? model.model.id : model.id);
+            if (model.kind === "registry" && model.model.id === "m2") return "economical";
+            return undefined;
+          },
+        }),
+      );
+      const r = await p.classify("hello");
+      assert.deepEqual(calls, ["m1", "m2"]);
+      assert.equal(r.kind, "classified");
+      if (r.kind === "classified") {
+        assert.equal(r.source, "classifier");
+      }
+    });
+
+    it("validates classifier result against known tiers", async () => {
+      const p = createPipeline(
+        deps({
+          classifierModels: [makeModel("a", "m1")],
+          classifyWithLLM: async () => "unknown",
+          defaultTier: "economical",
+        }),
+      );
+      const r = await p.classify("hello");
+      // Unknown tier → falls through to default
+      assert.equal(r.kind, "fallback");
+      if (r.kind === "fallback") {
+        assert.equal(r.tier, "economical");
+      }
+    });
+
+    it("skips classifier when classifierModels is empty", async () => {
+      let called = false;
+      const p = createPipeline(
+        deps({
+          classifierModels: [],
+          classifyWithLLM: async () => { called = true; return "frontier"; },
+          regexRules: [{ pattern: "hello", model: "frontier" }],
+        }),
+      );
+      await p.classify("hello");
+      assert.equal(called, false);
+    });
+  });
+
+  describe("regex", () => {
+    it("matches regex rule", async () => {
+      const p = createPipeline(
+        deps({
+          regexRules: [{ pattern: "\\bdebug\\b", model: "frontier" }],
+        }),
+      );
+      const r = await p.classify("debug the thing");
+      assert.equal(r.kind, "classified");
+      if (r.kind === "classified") {
+        assert.equal(r.tier, "frontier");
+        assert.equal(r.source, "regex");
+      }
+    });
+
+    it("falls through to default when no rule matches", async () => {
+      const p = createPipeline(
+        deps({
+          regexRules: [{ pattern: "\\bdebug\\b", model: "frontier" }],
+          defaultTier: "economical",
+        }),
+      );
+      const r = await p.classify("hello world");
+      assert.equal(r.kind, "fallback");
+      if (r.kind === "fallback") {
+        assert.equal(r.tier, "economical");
+      }
+    });
+  });
+
+  describe("priority order", () => {
+    it("cache beats classifier", async () => {
+      let classifierCalled = false;
+      const p = createPipeline(
+        deps({
+          cacheLookup: () => "frontier",
+          classifierModels: [makeModel("a", "m1")],
+          classifyWithLLM: async () => { classifierCalled = true; return "economical"; },
+        }),
+      );
+      const r = await p.classify("test");
+      assert.equal(classifierCalled, false);
+      assert.equal(r.kind, "classified");
+      if (r.kind === "classified") {
+        assert.equal(r.source, "cache");
+      }
+    });
+
+    it("classifier beats regex", async () => {
+      const p = createPipeline(
+        deps({
+          classifierModels: [makeModel("a", "m1")],
+          classifyWithLLM: async () => "economical",
+          regexRules: [{ pattern: ".*", model: "frontier" }],
+        }),
+      );
+      const r = await p.classify("test");
+      assert.equal(r.kind, "classified");
+      if (r.kind === "classified") {
+        assert.equal(r.source, "classifier");
+        assert.equal(r.tier, "economical");
+      }
+    });
+
+    it("regex beats default", async () => {
+      const p = createPipeline(
+        deps({
+          regexRules: [{ pattern: ".*", model: "frontier" }],
+          defaultTier: "economical",
+        }),
+      );
+      const r = await p.classify("test");
+      assert.equal(r.kind, "classified");
+      if (r.kind === "classified") {
+        assert.equal(r.source, "regex");
+      }
+    });
+  });
+
+  describe("fallback", () => {
+    it("returns fallback when only default matches", async () => {
+      const p = createPipeline(
+        deps({ defaultTier: "economical" }),
+      );
+      const r = await p.classify("hello");
+      assert.equal(r.kind, "fallback");
+      if (r.kind === "fallback") {
+        assert.equal(r.tier, "economical");
+      }
+    });
+  });
+
+  describe("classifier error resilience", () => {
+    it("catches classifier throw and falls through to regex", async () => {
+      const p = createPipeline(
+        deps({
+          classifierModels: [makeModel("a", "m1")],
+          classifyWithLLM: async () => { throw new Error("boom"); },
+          regexRules: [{ pattern: ".*", model: "frontier" }],
+        }),
+      );
+      const r = await p.classify("hello");
+      assert.equal(r.kind, "classified");
+      if (r.kind === "classified") {
+        assert.equal(r.source, "regex");
+      }
+    });
+
+    it("catches classifier throw and falls through to default", async () => {
+      const p = createPipeline(
+        deps({
+          classifierModels: [makeModel("a", "m1")],
+          classifyWithLLM: async () => { throw new Error("boom"); },
+          defaultTier: "economical",
+        }),
+      );
+      const r = await p.classify("hello");
+      assert.equal(r.kind, "fallback");
+      if (r.kind === "fallback") {
+        assert.equal(r.tier, "economical");
+      }
+    });
+  });
+});

package/tests/routing.test.ts

@@ -0,0 +1,224 @@
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import type { Api, Model } from "@earendil-works/pi-ai";
+import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
+import {
+  findOneModel,
+  findCandidates,
+  selectModel,
+  resolveModel,
+  modelKey,
+  modelCost,
+  getStrategy,
+  classify,
+} from "../routing.ts";
+
+function makeModel(
+  provider: string,
+  id: string,
+  inputCost = 0,
+  outputCost = 0,
+  contextWindow = 128000,
+): Model<Api> {
+  return {
+    provider,
+    id,
+    name: id,
+    api: "openai-completions" as Api,
+    baseUrl: "http://localhost:1234/v1",
+    reasoning: false,
+    input: ["text"],
+    cost: { input: inputCost, output: outputCost, cacheRead: 0, cacheWrite: 0 },
+    contextWindow,
+    maxTokens: 4096,
+  } as unknown as Model<Api>;
+}
+
+function makeRegistry(models: Model<Api>[]) {
+  return {
+    find: (provider: string, id: string) =>
+      models.find((m) => m.provider === provider && m.id === id),
+    getAvailable: () => models,
+  };
+}
+
+function makeCtx(models: Model<Api>[]): ExtensionContext {
+  return {
+    modelRegistry: makeRegistry(models),
+  } as unknown as ExtensionContext;
+}
+
+describe("routing", () => {
+  describe("modelKey", () => {
+    it("returns provider/id", () => {
+      const m = makeModel("anthropic", "claude-opus", 15);
+      assert.equal(modelKey(m), "anthropic/claude-opus");
+    });
+
+    it("returns none for undefined", () => {
+      assert.equal(modelKey(undefined), "none");
+    });
+  });
+
+  describe("modelCost", () => {
+    it("sums input and output cost", () => {
+      const m = makeModel("x", "y", 3);
+      m.cost.output = 7;
+      assert.equal(modelCost(m), 10);
+    });
+  });
+
+  describe("findOneModel", () => {
+    it("finds exact provider/id", () => {
+      const ctx = makeCtx([makeModel("anthropic", "claude-opus", 15)]);
+      const m = findOneModel(ctx, "anthropic/claude-opus");
+      assert.ok(m);
+      assert.equal(modelKey(m), "anthropic/claude-opus");
+    });
+
+    it("finds ids containing slashes", () => {
+      const ctx = makeCtx([makeModel("lmstudio", "qwen/qwen3-vl-8b", 0)]);
+      const m = findOneModel(ctx, "lmstudio/qwen/qwen3-vl-8b");
+      assert.ok(m);
+      assert.equal(modelKey(m), "lmstudio/qwen/qwen3-vl-8b");
+    });
+
+    it("finds by substring", () => {
+      const ctx = makeCtx([
+        makeModel("anthropic", "claude-sonnet", 3),
+        makeModel("anthropic", "claude-opus", 15),
+      ]);
+      const m = findOneModel(ctx, "opus");
+      assert.ok(m);
+      assert.equal(modelKey(m), "anthropic/claude-opus");
+    });
+
+    it("returns undefined when not found", () => {
+      const ctx = makeCtx([]);
+      assert.equal(findOneModel(ctx, "anthropic/missing"), undefined);
+    });
+  });
+
+  describe("findCandidates", () => {
+    it("returns multiple models for an array", () => {
+      const ctx = makeCtx([
+        makeModel("anthropic", "claude-opus", 15),
+        makeModel("anthropic", "claude-sonnet", 3),
+      ]);
+      const candidates = findCandidates(ctx, [
+        "anthropic/claude-opus",
+        "anthropic/claude-sonnet",
+      ]);
+      assert.equal(candidates.length, 2);
+    });
+
+    it("deduplicates models", () => {
+      const ctx = makeCtx([makeModel("anthropic", "claude-opus", 15)]);
+      const candidates = findCandidates(ctx, [
+        "anthropic/claude-opus",
+        "anthropic/claude-opus",
+      ]);
+      assert.equal(candidates.length, 1);
+    });
+
+    it("matches substring and exact together", () => {
+      const ctx = makeCtx([
+        makeModel("anthropic", "claude-opus", 15),
+        makeModel("lmstudio", "qwen/qwen3-vl-8b", 0),
+      ]);
+      const candidates = findCandidates(ctx, ["anthropic/claude-opus", "lmstudio"]);
+      assert.equal(candidates.length, 2);
+    });
+  });
+
+  describe("selectModel", () => {
+    it("returns first candidate for first strategy", () => {
+      const a = makeModel("a", "a", 5, 10, 32000);
+      const b = makeModel("b", "b", 1, 2, 128000);
+      const m = selectModel([a, b], "first");
+      assert.equal(modelKey(m), "a/a");
+    });
+
+    it("returns cheapest (input+output)", () => {
+      const a = makeModel("a", "a", 5, 0);
+      const b = makeModel("b", "b", 1, 0);
+      const c = makeModel("c", "c", 3, 2);
+      const m = selectModel([a, b, c], "cheapest");
+      assert.equal(modelKey(m), "b/b");
+    });
+
+    it("returns cheapest input cost", () => {
+      const a = makeModel("a", "a", 5, 0);
+      const b = makeModel("b", "b", 1, 10);
+      const m = selectModel([a, b], "cheapest_input");
+      assert.equal(modelKey(m), "b/b");
+    });
+
+    it("returns cheapest output cost", () => {
+      const a = makeModel("a", "a", 0, 5);
+      const b = makeModel("b", "b", 5, 1);
+      const m = selectModel([a, b], "cheapest_output");
+      assert.equal(modelKey(m), "b/b");
+    });
+
+    it("returns largest context window", () => {
+      const a = makeModel("a", "a", 0, 0, 32000);
+      const b = makeModel("b", "b", 0, 0, 256000);
+      const m = selectModel([a, b], "largest_context");
+      assert.equal(modelKey(m), "b/b");
+    });
+
+    it("returns a random candidate", () => {
+      const a = makeModel("a", "a", 0, 0);
+      const b = makeModel("b", "b", 0, 0);
+      const results = new Set();
+      for (let i = 0; i < 20; i++) results.add(selectModel([a, b], "random")!.id);
+      assert.ok(results.has("a"));
+      assert.ok(results.has("b"));
+    });
+
+    it("returns undefined for empty candidates", () => {
+      assert.equal(selectModel([], "first"), undefined);
+    });
+  });
+
+  describe("resolveModel", () => {
+    it("resolves a single pattern", () => {
+      const ctx = makeCtx([makeModel("anthropic", "claude-opus", 15)]);
+      const m = resolveModel(ctx, "anthropic/claude-opus", "first");
+      assert.equal(modelKey(m), "anthropic/claude-opus");
+    });
+
+    it("resolves array to first available", () => {
+      const ctx = makeCtx([
+        makeModel("anthropic", "claude-opus", 15),
+        makeModel("anthropic", "claude-sonnet", 3),
+      ]);
+      const m = resolveModel(ctx, ["anthropic/missing", "anthropic/claude-sonnet"], "first");
+      assert.equal(modelKey(m), "anthropic/claude-sonnet");
+    });
+  });
+
+  describe("getStrategy", () => {
+    it("returns category strategy if set", () => {
+      const categoryStrategies = { economical: "cheapest" as const };
+      assert.equal(getStrategy(categoryStrategies, "first", "economical"), "cheapest");
+      assert.equal(getStrategy(categoryStrategies, "first", "frontier"), "first");
+    });
+
+    it("returns global strategy as fallback", () => {
+      assert.equal(getStrategy(undefined, "first", "economical"), "first");
+    });
+
+    it("defaults to first", () => {
+      assert.equal(getStrategy(undefined, undefined, "economical"), "first");
+    });
+  });
+
+  describe("classify (regex)", () => {
+    it("returns undefined for invalid regex pattern", () => {
+      const result = classify("hello", [{ pattern: "***invalid[", model: "frontier" }]);
+      assert.equal(result, undefined);
+    });
+  });
+});

package/tsconfig.json

@@ -0,0 +1,19 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "types": ["node"],
+    "strict": true,
+    "noUnusedLocals": true,
+    "noUnusedParameters": true,
+    "noFallthroughCasesInSwitch": true,
+    "noUncheckedSideEffectImports": true,
+    "allowImportingTsExtensions": true,
+    "noEmit": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true
+  },
+  "include": ["*.ts", "tests/*.ts"],
+  "exclude": ["node_modules"]
+}