pi-system-one 1.3.1 → 1.3.2

package/tests/debug.test.ts

@@ -0,0 +1,345 @@
+import assert from "node:assert/strict";
+import { it } from "node:test";
+import { visibleWidth } from "@earendil-works/pi-tui";
+import { createSessionConfig } from "../src/config.ts";
+import { DebugHistory, observedProvider } from "../src/debug.ts";
+import { DebugOverlay, debugSummary } from "../src/debug-ui.ts";
+
+it("shows page position, distinct question counts, target, and readable controls", () => {
+  const history = new DebugHistory();
+  for (let i = 0; i < 17; i++) {
+    const entry = history.start("custom", {
+      q: { type: "score", instructions: "hidden", criteria: ["low", "high"] },
+    });
+    entry.provider = "configured";
+    entry.outcome = "failure";
+    entry.httpStatus = 405;
+    entry.failure = "HTTP error";
+    entry.requestTarget = "POST http://localhost:8008/v1/systemone";
+  }
+  const colors: string[] = [];
+  const overlay = new DebugOverlay(
+    history.entries,
+    {
+      fg: (color: string, text: string) => {
+        colors.push(`${color}:${text}`);
+        return text;
+      },
+    } as never,
+    () => {},
+    () => {},
+  );
+  const initial = overlay.render(68).join("\n");
+  assert.match(initial, /1-7 of 17/);
+  assert.match(initial, /#1/);
+  assert.match(initial, /1 score/);
+  assert.match(initial, /POST http:\/\/localhost:8008\/v1\/systemone/);
+  assert.ok(
+    colors.some(
+      (line) => line.startsWith("text:") && line.includes("recent call"),
+    ),
+  );
+  assert.ok(
+    colors.some((line) => line.startsWith("text:") && line.includes("browse")),
+  );
+  for (let i = 0; i < 7; i++) overlay.handleInput("\u001b[B");
+  assert.match(overlay.render(68).join("\n"), /8-14 of 17/);
+});
+
+it("renders bounded debug overlay and lets keyboard move focus and close", () => {
+  const history = new DebugHistory();
+  const entry = history.start("auto", {
+    q: { type: "noul", instructions: "secret instruction" },
+  });
+  entry.provider = "pi-native";
+  entry.reason = "compatible request";
+  entry.outcome = "success";
+  entry.durationMs = 7;
+  history.start("custom", {
+    q: {
+      type: "score",
+      instructions: "private rubric",
+      criteria: ["low", "high"],
+    },
+  });
+  let renders = 0;
+  let closed = false;
+  const overlay = new DebugOverlay(
+    history.entries,
+    { fg: (_color: string, text: string) => text } as never,
+    () => {
+      closed = true;
+    },
+    () => {
+      renders++;
+    },
+  );
+  const first = overlay.render(44);
+  assert.ok(first.every((line) => visibleWidth(line) <= 44));
+  assert.doesNotMatch(first.join("\n"), /secret instruction|private rubric/);
+  assert.match(first.join("\n"), /Provider: not selected/);
+  overlay.handleInput("\u001b[B");
+  assert.ok(renders > 0);
+  assert.match(overlay.render(44).join("\n"), /Provider: pi-native/);
+  overlay.handleInput("\u001b");
+  assert.ok(closed);
+  assert.match(debugSummary([]), /No system_one calls yet/);
+  history.clear();
+  assert.equal(history.entries.length, 0);
+});
+
+it("shows POST target shape for custom 405 without retaining URL secrets", async () => {
+  const originalFetch = globalThis.fetch;
+  globalThis.fetch = (async () =>
+    new Response("not allowed", { status: 405 })) as typeof fetch;
+  try {
+    const history = new DebugHistory();
+    const request = {
+      state: "private",
+      questions: { q: { type: "noul" as const, instructions: "Yes?" } },
+    };
+    const duplicate = observedProvider(
+      createSessionConfig({
+        SYSTEM_ONE_BASE_URL: "http://localhost:8008/v1/systemone",
+      }),
+      {} as never,
+      history,
+    );
+    await assert.rejects(duplicate.evaluate(request), /provider error 405/);
+    assert.match(
+      history.entries[0]?.requestTarget ?? "",
+      /POST http:\/\/localhost:8008\/v1\/systemone\/v1\/systemone/,
+    );
+    const secret = observedProvider(
+      createSessionConfig({
+        SYSTEM_ONE_BASE_URL:
+          "https://user:password@private.example/api/token123?key=secret#fragment",
+      }),
+      {} as never,
+      history,
+    );
+    await assert.rejects(secret.evaluate(request), /provider error 405/);
+    const record = JSON.stringify(history.entries);
+    assert.match(
+      history.entries[0]?.requestTarget ?? "",
+      /POST https:\/\/private.example\/api\/\[redacted\]/,
+    );
+    assert.match(history.entries[0]?.requestTarget ?? "", /query\/fragment/);
+    assert.doesNotMatch(
+      record,
+      /user:password|token123|key=secret|#fragment|private"/,
+    );
+  } finally {
+    globalThis.fetch = originalFetch;
+  }
+});
+
+it("records credential failure as selected but not dispatched, without error text", async () => {
+  const history = new DebugHistory();
+  const provider = observedProvider(
+    createSessionConfig({}, { mode: "typesafe" }),
+    {} as never,
+    history,
+  );
+  await assert.rejects(
+    provider.evaluate({
+      state: "secret-state",
+      questions: { q: { type: "noul", instructions: "secret-question" } },
+    }),
+    /TypeSafe key missing/,
+  );
+  assert.equal(history.entries[0]?.provider, "typesafe");
+  assert.equal(history.entries[0]?.outcome, "failure");
+  assert.equal(history.entries[0]?.attempted, false);
+  assert.doesNotMatch(
+    JSON.stringify(history.entries),
+    /secret-state|secret-question|TypeSafe key missing/,
+  );
+});
+
+it("reports native Score rejection as not dispatched and gives a fixed reason", async () => {
+  const history = new DebugHistory();
+  const provider = observedProvider(
+    createSessionConfig({}, { mode: "native" }),
+    {
+      modelRegistry: {
+        findOfType: () => ({ id: "jev-latest" }),
+        classify: () => {
+          throw Error("must not classify unsupported Score");
+        },
+      },
+    } as never,
+    history,
+  );
+  await assert.rejects(
+    provider.evaluate({
+      state: "x",
+      questions: {
+        q: { type: "score", instructions: "Rate", criteria: ["low", "high"] },
+      },
+    }),
+    /omits Score probabilities/,
+  );
+  assert.equal(history.entries[0]?.failure, "native Score unsupported");
+  assert.equal(history.entries[0]?.attempted, false);
+});
+
+it("distinguishes native provider error from malformed success without logging content", async () => {
+  const cases = [
+    {
+      result: {
+        stopReason: "error",
+        errorMessage: "Bearer secret-provider-body",
+      },
+      category: "native classifier returned error",
+    },
+    {
+      result: {
+        stopReason: "stop",
+        answers: { q: { type: "bool", probability: 2 } },
+      },
+      category: "invalid native response",
+    },
+  ];
+  for (const { result, category } of cases) {
+    const history = new DebugHistory();
+    const provider = observedProvider(
+      createSessionConfig({}, { mode: "native" }),
+      {
+        modelRegistry: {
+          findOfType: () => ({ id: "jev-latest" }),
+          classify: async () => result,
+        },
+      } as never,
+      history,
+    );
+    await assert.rejects(
+      provider.evaluate({
+        state: "private state",
+        questions: { q: { type: "noul", instructions: "Yes?" } },
+      }),
+      /Pi native classifier failed/,
+    );
+    assert.equal(history.entries[0]?.failure, category);
+    assert.equal(history.entries[0]?.attempted, true);
+    assert.doesNotMatch(
+      JSON.stringify(history.entries),
+      /secret-provider-body|private state/,
+    );
+  }
+});
+
+it("records failed native attempt without retaining provider-controlled error text", async () => {
+  const history = new DebugHistory();
+  const provider = observedProvider(
+    createSessionConfig({}, { mode: "auto" }),
+    {
+      modelRegistry: {
+        findOfType: () => ({ id: "jev-latest" }),
+        classify: async () => {
+          throw Error("private response body");
+        },
+      },
+    } as never,
+    history,
+  );
+  await assert.rejects(
+    provider.evaluate({
+      state: "x",
+      questions: { q: { type: "noul", instructions: "Yes?" } },
+    }),
+    /Pi native classifier failed/,
+  );
+  assert.equal(history.entries[0]?.provider, "pi-native");
+  assert.equal(history.entries[0]?.outcome, "failure");
+  assert.equal(history.entries[0]?.attempted, true);
+  assert.equal(history.entries[0]?.failure, "native classify threw");
+  assert.doesNotMatch(JSON.stringify(history.entries), /private response body/);
+});
+
+it("records HTTP fallback for auto Score and keeps only bounded history", async () => {
+  const originalFetch = globalThis.fetch;
+  const history = new DebugHistory();
+  globalThis.fetch = (async () =>
+    new Response(
+      JSON.stringify({
+        answers: {
+          q: {
+            type: "score",
+            score: 0,
+            confidence: 1,
+            probabilities: { "0": 1, "1": 0 },
+            legend: { "0": "low", "1": "high" },
+          },
+        },
+      }),
+      { status: 200 },
+    )) as typeof fetch;
+  try {
+    const provider = observedProvider(
+      createSessionConfig({}, { mode: "auto" }),
+      {
+        modelRegistry: {
+          findOfType: () => ({ id: "jev-latest" }),
+          classify: () => {
+            throw Error("must not classify Score");
+          },
+          getApiKeyForProvider: async () => "secret-pi-key",
+        },
+      } as never,
+      history,
+    );
+    const request = {
+      state: "private-state",
+      questions: {
+        q: {
+          type: "score" as const,
+          instructions: "Score private item",
+          criteria: ["low", "high"],
+        },
+      },
+    };
+    for (let i = 0; i < 21; i++) await provider.evaluate(request);
+    assert.equal(history.entries.length, 20);
+    assert.equal(history.entries[0]?.provider, "typesafe");
+    assert.equal(history.entries[0]?.reason, "contains Score");
+    assert.equal(history.entries[0]?.outcome, "success");
+    assert.doesNotMatch(
+      JSON.stringify(history.entries),
+      /private-state|private item|secret-pi-key/,
+    );
+  } finally {
+    globalThis.fetch = originalFetch;
+  }
+});
+
+it("records actual auto route without retaining request state or credentials", async () => {
+  const history = new DebugHistory();
+  const ctx = {
+    modelRegistry: {
+      findOfType: () => ({ id: "jev-latest" }),
+      classify: async () => ({
+        stopReason: "stop",
+        answers: { q: { type: "bool", probability: 0.8 } },
+      }),
+    },
+  } as never;
+  const provider = observedProvider(
+    createSessionConfig(
+      { SYSTEM_ONE_API_KEY: "private-api-key" },
+      { mode: "auto" },
+    ),
+    ctx,
+    history,
+  );
+  await provider.evaluate({
+    state: "private-user-state",
+    questions: { q: { type: "noul", instructions: "private-instruction" } },
+  });
+  assert.equal(history.entries.length, 1);
+  assert.match(JSON.stringify(history.entries), /pi-native|native/);
+  assert.doesNotMatch(
+    JSON.stringify(history.entries),
+    /private-user-state|private-instruction|private-api-key/,
+  );
+});

package/tests/extension.test.ts

@@ -27,25 +27,29 @@ describe("extension resources", () => {
         join(dir, "pi-system-one.json"),
         JSON.stringify({ "Bearer secret-token": "ignored" }),
       );
-      const handlers: Record<string, (event: never, ctx: never) => void> = {};
+      const handlers: Record<string, ((event: never, ctx: never) => void)[]> =
+        {};
       piSystemOneExtension({
         registerTool: () => {},
         registerCommand: () => {},
         on: (name: string, handler: (event: never, ctx: never) => void) => {
-          handlers[name] = handler;
+          handlers[name] ??= [];
+          handlers[name].push(handler);
         },
       } as never);
       let warning = "";
-      handlers.session_start(
-        { reason: "startup" } as never,
-        {
-          ui: {
-            notify: (message: string) => {
-              warning = message;
+      for (const handler of handlers.session_start) {
+        handler(
+          { reason: "startup" } as never,
+          {
+            ui: {
+              notify: (message: string) => {
+                warning = message;
+              },
             },
-          },
-        } as never,
-      );
+          } as never,
+        );
+      }
       assert.match(warning, /Ignoring saved settings/);
       assert.doesNotMatch(warning, /Bearer|secret-token/);
     } finally {
@@ -116,6 +120,70 @@ describe("extension resources", () => {
       rmSync(dir, { recursive: true, force: true });
     }
   });
+  it("loads saved native mode instead of an inherited TypeSafe endpoint on restart", async () => {
+    const dir = mkdtempSync(join(tmpdir(), "so-saved-native-"));
+    const oldDir = process.env.PI_CODING_AGENT_DIR;
+    const oldUrl = process.env.SYSTEM_ONE_BASE_URL;
+    const originalFetch = globalThis.fetch;
+    try {
+      process.env.PI_CODING_AGENT_DIR = dir;
+      process.env.SYSTEM_ONE_BASE_URL = "https://api.typesafe.ai";
+      writeFileSync(join(dir, "pi-system-one.json"), '{"mode":"native"}');
+      let tool: ReturnType<typeof buildSystemOneTool> | undefined;
+      let command: ((args: string, ctx: never) => Promise<void>) | undefined;
+      piSystemOneExtension({
+        registerTool: (value: ReturnType<typeof buildSystemOneTool>) => {
+          tool = value;
+        },
+        registerCommand: (
+          _name: string,
+          options: { handler: typeof command },
+        ) => {
+          command = options.handler;
+        },
+        on: () => {},
+      } as never);
+      let status = "";
+      await command?.("status", {
+        ui: {
+          notify: (message: string) => {
+            status = message;
+          },
+        },
+      } as never);
+      assert.match(status, /mode: native \(saved\)/);
+      globalThis.fetch = (async () => {
+        throw Error("saved native mode must not make HTTP calls");
+      }) as typeof fetch;
+      assert.ok(tool);
+      const result = await tool.execute(
+        "saved-native",
+        {
+          state: "x",
+          questions: { q: { type: "noul", instructions: "Yes?" } },
+        },
+        undefined,
+        undefined,
+        {
+          modelRegistry: {
+            findOfType: () => ({ id: "jev-latest" }),
+            classify: async () => ({
+              stopReason: "stop",
+              answers: { q: { type: "bool", probability: 0.8 } },
+            }),
+          },
+        } as never,
+      );
+      assert.deepEqual(result.details.answers.q, { type: "noul", noul: 0.8 });
+    } finally {
+      globalThis.fetch = originalFetch;
+      if (oldDir === undefined) delete process.env.PI_CODING_AGENT_DIR;
+      else process.env.PI_CODING_AGENT_DIR = oldDir;
+      if (oldUrl === undefined) delete process.env.SYSTEM_ONE_BASE_URL;
+      else process.env.SYSTEM_ONE_BASE_URL = oldUrl;
+      rmSync(dir, { recursive: true, force: true });
+    }
+  });
   it("reloads saved custom config across extension instances", async () => {
     const dir = mkdtempSync(join(tmpdir(), "so-extension-"));
     const prev = {

package/tests/persistence.test.ts

@@ -27,6 +27,21 @@ it("round-trips persistent settings without writing a key", () => {
   }
 });
 
+it("persists auto routing policy without a detected provider or credentials", () => {
+  const dir = mkdtempSync(join(tmpdir(), "so-auto-"));
+  try {
+    const path = join(dir, "pi-system-one.json");
+    saveStoredConfig(path, { mode: "auto", timeoutMs: 5000 });
+    assert.deepEqual(loadStoredConfig(path), { mode: "auto", timeoutMs: 5000 });
+    assert.doesNotMatch(
+      readFileSync(path, "utf8"),
+      /baseUrl|apiKey|jev-latest/,
+    );
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+});
+
 it("rejects credential-bearing endpoint URLs without persisting them", () => {
   const dir = mkdtempSync(join(tmpdir(), "so-config-"));
   try {

package/tests/provider.test.ts

@@ -4,6 +4,215 @@ import { createSessionConfig } from "../src/config.ts";
 import { resolveSessionProvider } from "../src/provider.ts";
 import { buildSystemOneTool } from "../src/tool.ts";
 
+it("auto sends compatible Noul to native and Score to canonical TypeSafe HTTP", async () => {
+  const originalFetch = globalThis.fetch;
+  const urls: string[] = [];
+  let nativeCalls = 0;
+  const session = createSessionConfig(
+    {
+      SYSTEM_ONE_BASE_URL: "https://custom.example",
+      SYSTEM_ONE_API_KEY: "custom-secret",
+    },
+    { mode: "auto" as never },
+  );
+  const ctx = {
+    modelRegistry: {
+      findOfType: () => ({ id: "jev-latest" }),
+      classify: async () => {
+        nativeCalls++;
+        return {
+          stopReason: "stop",
+          model: "jev-latest",
+          answers: { q: { type: "bool", probability: 0.8 } },
+        };
+      },
+      getApiKeyForProvider: async () => "pi-key",
+    },
+  } as never;
+  globalThis.fetch = (async (url, options) => {
+    urls.push(String(url));
+    assert.equal(
+      new Headers(options?.headers).get("authorization"),
+      "Bearer pi-key",
+    );
+    return new Response(
+      JSON.stringify({
+        model: "jev-latest",
+        answers: {
+          score: {
+            type: "score",
+            score: 0,
+            confidence: 1,
+            probabilities: { "0": 1, "1": 0 },
+            legend: { "0": "low", "1": "high" },
+          },
+        },
+      }),
+      { status: 200 },
+    );
+  }) as typeof fetch;
+  try {
+    const provider = await resolveSessionProvider(session, ctx);
+    const native = await provider.evaluate({
+      state: "x",
+      questions: { q: { type: "noul", instructions: "Is it?" } },
+    });
+    assert.deepEqual(native.answers.q, { type: "noul", noul: 0.8 });
+    const score = await provider.evaluate({
+      state: "x",
+      questions: {
+        score: {
+          type: "score",
+          instructions: "Rate",
+          criteria: ["low", "high"],
+        },
+      },
+    });
+    assert.equal(score.answers.score.type, "score");
+    assert.equal(nativeCalls, 1);
+    assert.deepEqual(urls, ["https://api.typesafe.ai/v1/systemone"]);
+  } finally {
+    globalThis.fetch = originalFetch;
+  }
+});
+
+it("auto routes mixed and structured requests as one TypeSafe HTTP call", async () => {
+  const originalFetch = globalThis.fetch;
+  let calls = 0;
+  const session = createSessionConfig({}, { mode: "auto" });
+  const ctx = {
+    modelRegistry: {
+      findOfType: () => ({ id: "jev-latest" }),
+      classify: () => {
+        throw new Error("native must not be called");
+      },
+      getApiKeyForProvider: async () => "pi-key",
+    },
+  } as never;
+  globalThis.fetch = (async (url, options) => {
+    calls++;
+    assert.equal(String(url), "https://api.typesafe.ai/v1/systemone");
+    const request = JSON.parse(String(options?.body));
+    assert.ok(request.questions.noul && request.questions.score);
+    return new Response(
+      JSON.stringify({
+        answers: {
+          noul: { type: "noul", noul: 0.8 },
+          score: {
+            type: "score",
+            score: 0,
+            confidence: 1,
+            probabilities: { "0": 1, "1": 0 },
+            legend: { "0": "low", "1": "high" },
+          },
+        },
+      }),
+      { status: 200 },
+    );
+  }) as typeof fetch;
+  try {
+    const provider = await resolveSessionProvider(session, ctx);
+    await provider.evaluate({
+      state: "x",
+      questions: {
+        noul: { type: "noul", instructions: "Is it?" },
+        score: {
+          type: "score",
+          instructions: "Rate",
+          criteria: ["low", "high"],
+        },
+      },
+    });
+    assert.equal(calls, 1);
+  } finally {
+    globalThis.fetch = originalFetch;
+  }
+});
+
+it("auto uses TypeSafe HTTP when classifier is missing or input is incompatible", async () => {
+  const originalFetch = globalThis.fetch;
+  let calls = 0;
+  let modelAvailable = false;
+  const session = createSessionConfig({}, { mode: "auto" });
+  const ctx = {
+    modelRegistry: {
+      findOfType: () => (modelAvailable ? { id: "jev-latest" } : undefined),
+      classify: () => {
+        throw new Error("native must not be called");
+      },
+      getApiKeyForProvider: async () => "pi-key",
+    },
+  } as never;
+  globalThis.fetch = (async (url) => {
+    calls++;
+    assert.equal(String(url), "https://api.typesafe.ai/v1/systemone");
+    return new Response(
+      JSON.stringify({ answers: { q: { type: "noul", noul: 0.7 } } }),
+      { status: 200 },
+    );
+  }) as typeof fetch;
+  try {
+    const provider = await resolveSessionProvider(session, ctx);
+    await provider.evaluate({
+      state: "x",
+      questions: { q: { type: "noul", instructions: "Yes?" } },
+    });
+    modelAvailable = true;
+    await provider.evaluate({
+      state: "x",
+      questions: {
+        q: { type: "noul", instructions: { text: "Yes?" } },
+      },
+    });
+    await provider.evaluate({
+      state: "x",
+      questions: {
+        q: {
+          type: "noul",
+          instructions: "Yes?",
+          criteria: { true: "yes", false: "no", maybe: "unsure" },
+        },
+      },
+    });
+    assert.equal(calls, 3);
+  } finally {
+    globalThis.fetch = originalFetch;
+  }
+});
+
+it("auto does not retry native failures through HTTP", async () => {
+  const originalFetch = globalThis.fetch;
+  const session = createSessionConfig({}, { mode: "auto" });
+  const ctx = {
+    modelRegistry: {
+      findOfType: () => ({ id: "jev-latest" }),
+      classify: async () => ({
+        stopReason: "error",
+        errorMessage: "secret provider error",
+      }),
+    },
+  } as never;
+  globalThis.fetch = (async () => {
+    throw Error("HTTP fallback forbidden");
+  }) as typeof fetch;
+  try {
+    const provider = await resolveSessionProvider(session, ctx);
+    await assert.rejects(
+      provider.evaluate({
+        state: "x",
+        questions: { q: { type: "noul", instructions: "Is it?" } },
+      }),
+      (error: Error) => {
+        assert.match(error.message, /Pi native classifier failed/);
+        assert.doesNotMatch(error.message, /secret provider error/);
+        return true;
+      },
+    );
+  } finally {
+    globalThis.fetch = originalFetch;
+  }
+});
+
 it("rejects malformed successful native probability before tool output", async () => {
   const session = createSessionConfig({});
   session.mode = "native";