specpi 0.26.0 → 0.27.0

package/extensions/jev-advisor/client.mjs

@@ -0,0 +1,182 @@
+// One POST, no SDK. AGENTS.md forbids adding executable dependencies without need, and a single
+// JSON request does not need one.
+//
+// Every failure mode returns `unavailable` rather than throwing: a timeout, an HTTP error, a
+// missing key, a malformed body or an unparseable answer all mean "no advice", and the caller runs
+// the path it would have run before this extension existed. That is fail-silent, not fail-closed —
+// nothing here is ever the reason a tool is blocked.
+
+export const DEFAULT_MODEL = "jev-1.13.0";
+export const OPENROUTER_MODEL = "typesafe/jev-1.13";
+// Measured round trip is ~250-400ms through OpenRouter. 800ms left no headroom for a slow call,
+// and a timeout costs the advice without saving the latency already spent, so the budget is set
+// above the observed spread rather than at it.
+export const DEFAULT_TIMEOUT_MS = 1500;
+const MAX_TIMEOUT_MS = 5000;
+
+/**
+ * Jev is reached through OpenRouter by default: that is where it is published, it is what
+ * specpi-jev-guard already uses, and an OpenRouter key (`sk-or-...`) is rejected by the direct
+ * TypeSafe API with a bare 401. `JEV_BACKEND=typesafe` selects the direct API for a TypeSafe key.
+ */
+export function backend() {
+    return process.env.JEV_BACKEND === "typesafe" ? "typesafe" : "openrouter";
+}
+
+/**
+ * The key variable follows the backend, matching specpi-jev-guard's own `keyEnvName`, so one key
+ * serves the whole layer. TYPESAFE_API_KEY is still accepted on the OpenRouter path so an existing
+ * env file keeps working.
+ */
+export function apiKey() {
+    const name = backend() === "openrouter" ? "OPENROUTER_API_KEY" : "TYPESAFE_API_KEY";
+    const direct = process.env[name];
+    if (typeof direct === "string" && direct.trim().length > 0) {
+        return direct.trim();
+    }
+
+    const legacy = backend() === "openrouter" ? process.env.TYPESAFE_API_KEY : undefined;
+
+    return typeof legacy === "string" && legacy.trim().length > 0 ? legacy.trim() : undefined;
+}
+
+/** Overridable so tests never reach the network and the eval proxy can price the traffic. */
+export function baseUrl() {
+    const configured = process.env.TYPESAFE_BASE_URL;
+    if (configured && configured.trim().length > 0) {
+        return configured.trim().replace(/\/+$/u, "");
+    }
+
+    return backend() === "openrouter" ? "https://openrouter.ai" : "https://api.typesafe.ai";
+}
+
+// The two services expose the same state/questions body under different paths. Keeping both
+// suffixes distinct is also what lets the eval proxy tell the traffic apart and forward it on.
+export function endpoint() {
+    return `${baseUrl()}${backend() === "openrouter" ? "/api/alpha/decisions" : "/v1/systemone"}`;
+}
+
+export function defaultModel() {
+    return backend() === "openrouter" ? OPENROUTER_MODEL : DEFAULT_MODEL;
+}
+
+/** Choose one option from a set. Up to 255 options; output is free, so rich enums cost nothing. */
+export function choice(instructions, criteria) {
+    return { type: "choice", instructions, criteria };
+}
+
+/** Rate against ordered levels. Two to ten; the returned score may be fractional. */
+export function score(instructions, criteria) {
+    return { type: "score", instructions, criteria };
+}
+
+/** Yes or no as a probability. Returns a bare number with no confidence field. */
+export function noul(instructions) {
+    return { type: "noul", instructions };
+}
+
+function unavailable(reason) {
+    return { ok: false, reason, answers: {} };
+}
+
+/**
+ * Answers are normalized to a single shape so gates never branch on which primitive produced them.
+ * A Noul has no confidence, and inventing one would let a caller gate on a number the model never
+ * reported, so it stays undefined.
+ */
+function normalizeAnswer(raw) {
+    if (!raw || typeof raw !== "object") {
+        return undefined;
+    }
+
+    if (typeof raw.noul === "number") {
+        return { kind: "noul", value: raw.noul, probabilities: undefined, confidence: undefined };
+    }
+
+    if (typeof raw.choice === "string") {
+        return {
+            kind: "choice",
+            value: raw.choice,
+            probabilities: raw.probabilities && typeof raw.probabilities === "object" ? raw.probabilities : undefined,
+            confidence: typeof raw.confidence === "number" ? raw.confidence : undefined,
+        };
+    }
+
+    if (typeof raw.score === "number") {
+        return {
+            kind: "score",
+            value: raw.score,
+            probabilities: Array.isArray(raw.probabilities) ? raw.probabilities : undefined,
+            confidence: typeof raw.confidence === "number" ? raw.confidence : undefined,
+        };
+    }
+
+    return undefined;
+}
+
+/**
+ * Ask one batch. Questions are evaluated in parallel against one state, so callers should send
+ * every question that state can answer rather than paying for the state again.
+ */
+export async function ask(state, questions, options = {}) {
+    const key = apiKey();
+    if (!key) {
+        return unavailable("no-key");
+    }
+
+    if (!questions || Object.keys(questions).length === 0) {
+        return unavailable("no-questions");
+    }
+
+    const timeoutMs = Math.min(Math.max(options.timeoutMs ?? DEFAULT_TIMEOUT_MS, 50), MAX_TIMEOUT_MS);
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeoutMs);
+    const startedAt = Date.now();
+    const payload = { model: options.model ?? defaultModel(), state, questions };
+    try {
+        const response = await fetch(endpoint(), {
+            method: "POST",
+            headers: {
+                "content-type": "application/json",
+                authorization: `Bearer ${key}`,
+                // OpenRouter attributes traffic by these; they are ignored by the direct API.
+                "HTTP-Referer": "https://pi.dev",
+                "X-Title": "specpi-jev-advisor",
+            },
+            body: JSON.stringify(payload),
+            signal: options.signal ? AbortSignal.any([controller.signal, options.signal]) : controller.signal,
+        });
+        if (!response.ok) {
+            return { ...unavailable(`http-${response.status}`), latencyMs: Date.now() - startedAt };
+        }
+
+        const body = await response.json();
+        const answers = {};
+        for (const [name, raw] of Object.entries(body?.answers ?? {})) {
+            const normalized = normalizeAnswer(raw);
+            if (normalized) {
+                answers[name] = normalized;
+            }
+        }
+
+        if (Object.keys(answers).length === 0) {
+            return { ...unavailable("empty-answers"), latencyMs: Date.now() - startedAt };
+        }
+
+        return {
+            ok: true,
+            answers,
+            model: typeof body?.model === "string" ? body.model : undefined,
+            // Reported by OpenRouter, absent on the direct API. Preferred over an estimate wherever
+            // it exists, so the eval cost column rests on logged usage rather than a guess.
+            usage: body?.usage && typeof body.usage === "object" ? body.usage : undefined,
+            latencyMs: Date.now() - startedAt,
+        };
+    } catch (error) {
+        const reason = error?.name === "AbortError" ? "timeout" : "network";
+
+        return { ...unavailable(reason), latencyMs: Date.now() - startedAt };
+    } finally {
+        clearTimeout(timer);
+    }
+}

package/extensions/jev-advisor/config.mjs

@@ -0,0 +1,254 @@
+// Jev is the first thing in SpecPi that talks to a third party, so its switch is the first thing
+// every other module in this directory consults. Master off means no key read, no consent read, no
+// network call and no prompt injection: the harness behaves exactly as it did before the extension
+// existed.
+//
+// The file is SpecPi's own, hardened the same way as web-access and capability-policy: atomic
+// write, mode 0600, symlinks refused, and a missing or unreadable file read as off.
+
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { randomUUID } from "node:crypto";
+
+/** Systems that may run inside a session. Offline scripts are not gated here. */
+export const SYSTEM_NAMES = Object.freeze([
+    "retention",
+    "compaction",
+    "gap",
+    "sources",
+    "progress",
+    "untrusted",
+    "capability",
+]);
+
+/**
+ * What a confident stuck verdict is allowed to do. `notify` tells the person and cannot be wrong in
+ * a way that costs anything; `message` appends a fixed line the model reads before its next
+ * request, which changes behaviour.
+ *
+ * It ships on `notify`. Not because the gate cannot tell the cases apart -- it demonstrably can: on
+ * the recorded fixtures a session repeating one failing call scores 0.89 for stuck with the mode at
+ * 0.99 confidence, and a session working steadily scores 0.30 and reports "unknown" below the gate.
+ * The missing number is the false-positive rate on real sessions, and the two things that bear on it
+ * point the other way: running the same taxonomy over the 24 recorded failures left 14 of them
+ * ungated, and a wrong nudge costs a turn, which is the exact quantity this system exists to save.
+ *
+ * So the condition for changing this default is a measurement, not an opinion, and the eval suite
+ * is where it comes from: `--harness=specpi-jev` sets `message` and discloses it, because a
+ * notification in a headless run reaches nobody and would measure the cost of the system with none
+ * of its effect.
+ */
+export const NUDGE_MODES = Object.freeze(["notify", "message"]);
+
+const MAX_SETTINGS_BYTES = 4096;
+const MAX_CALL_BUDGET = 256;
+const MAX_TOTAL_BUDGET = 512;
+
+/**
+ * One shared budget could not survive a turn-level system. A system that fires once per turn would
+ * reach a shared ceiling of 8 inside the first few turns and starve retention for the rest of the
+ * session, and which one won would be decided by event ordering rather than by anyone's policy.
+ *
+ * So the ceiling is two-level: each system gets its own, and the total is a real constraint because
+ * it is deliberately less than their sum. Running out of one system's budget stops that system and
+ * nothing else.
+ *
+ * The per-system numbers follow how often each one can fire: retention on every large read-only
+ * result, compaction once or twice in a long session, gap per report, sources per delegation batch.
+ */
+export const DEFAULT_BUDGETS = Object.freeze({
+    // A backstop, not a working limit, and the number says which. Measured, a full tier-3 task -- a
+    // 120-step repair chain over about 25 model requests -- spends 4 to 7 calls, and the busiest
+    // attempt ever recorded spent 12. A session would have to run for days before 512 bound
+    // anything a person was actually doing, which is the point: the ceiling should only ever be hit
+    // by a loop, and hitting it should therefore be information rather than an inconvenience.
+    //
+    // The earlier 120 was sized against eval attempts, which is the wrong reference. An attempt
+    // runs for two minutes; an interactive session runs for a day, and a turn-level system at one
+    // call every four turns reaches 120 somewhere in the afternoon and then goes quiet without
+    // having found anything wrong. A ceiling that a normal long session reaches is not protecting
+    // anyone, it is just failing later than it looks.
+    //
+    // Cost is not what these are for. A call is about $0.00003, so the whole total is about a cent
+    // and a half. They bound two things that do not get cheaper with scale: how much digest leaves
+    // the machine for a third party, at up to 1 KB a call, and how much awaited latency a runaway
+    // loop can add before something stops it. Half a megabyte of digest and an announced stop is
+    // the shape of the trade.
+    total: 512,
+    retention: 208,
+    compaction: 12,
+    gap: 48,
+    sources: 32,
+    // Turn-level, but gated behind local signals and a four-turn cooldown, so it only spends on
+    // sessions that already look wrong. The ceiling is what stops a genuinely thrashing session
+    // from spending the total on being told it is thrashing.
+    progress: 176,
+    // Usually free: when retention is on, system 7's question rides the call retention was already
+    // making against the same state. This ceiling only binds when retention is off, or when the
+    // fetched result is too small for retention to be interested in it.
+    untrusted: 104,
+    // Once per session by construction, and only when local signals already suggest it. Two rather
+    // than one so a retried first turn is not silently un-served.
+    capability: 2,
+});
+
+/** 0 is a real budget meaning no calls. Switching a system off is what `systems[name] = false` is for. */
+export const MAX_BUDGETS = Object.freeze({ system: MAX_CALL_BUDGET, total: MAX_TOTAL_BUDGET });
+
+export function agentDirectory() {
+    const configured = process.env.PI_CODING_AGENT_DIR;
+
+    return path.resolve(configured && configured.length > 0 ? configured : path.join(os.homedir(), ".pi", "agent"));
+}
+
+export function jevDirectory() {
+    return path.join(agentDirectory(), "specpi", "jev");
+}
+
+function settingsFile() {
+    return path.join(jevDirectory(), "settings.json");
+}
+
+/** Refuses links and irregular files so the preference cannot redirect a write. */
+export function regularFile(file, label) {
+    const stat = fs.lstatSync(file, { throwIfNoEntry: false });
+    if (!stat) {
+        return false;
+    }
+
+    if (!stat.isFile() || stat.isSymbolicLink() || stat.nlink !== 1 || stat.size > MAX_SETTINGS_BYTES) {
+        throw new Error(`Unsupported ${label} file`);
+    }
+
+    return true;
+}
+
+/** Every unknown shape collapses to the same all-off default rather than a partial enable. */
+export function defaultSettings() {
+    return {
+        schema: 2,
+        master: false,
+        startup: false,
+        systems: Object.fromEntries(SYSTEM_NAMES.map((name) => [name, false])),
+        budgets: { ...DEFAULT_BUDGETS },
+        progressNudge: "notify",
+        // The guard is a separate package with its own gate, so it carries its own switch rather
+        // than riding the advisor's master. Both ship off: nothing in the Jev layer is active on a
+        // fresh install, and `startup` is how a user chooses to default one on.
+        guard: { enabled: false, startup: false },
+    };
+}
+
+function clamp(value, fallback, ceiling) {
+    return Number.isInteger(value) ? Math.min(Math.max(value, 0), ceiling) : fallback;
+}
+
+function normalizeBudgets(raw) {
+    const budgets = { total: clamp(raw?.total, DEFAULT_BUDGETS.total, MAX_TOTAL_BUDGET) };
+    for (const name of SYSTEM_NAMES) {
+        budgets[name] = clamp(raw?.[name], DEFAULT_BUDGETS[name] ?? 0, MAX_CALL_BUDGET);
+    }
+
+    return budgets;
+}
+
+/**
+ * Schema 1 carried one `callBudgetPerSession`. Reading it as an unknown shape would switch the
+ * layer off for anyone who had turned it on, which is a reset the user never asked for -- "unknown
+ * shapes collapse to all-off" is a rule for corrupt input, not for our own previous version. The
+ * one number becomes the total, and each system gets the smaller of its default and that total, so
+ * a user who set a deliberately tight ceiling keeps it.
+ */
+function migrate(raw) {
+    // Schema 1 read 0 as "no ceiling"; schema 2 reads it as "no calls", because a per-system 0 that
+    // silently meant unlimited is the wrong way for a budget to fail. Carrying the old meaning
+    // forward here is what stops the bump from inverting a user's intent.
+    const stored = raw?.callBudgetPerSession === 0 ? MAX_TOTAL_BUDGET : raw?.callBudgetPerSession;
+    const total = clamp(stored, DEFAULT_BUDGETS.total, MAX_TOTAL_BUDGET);
+    const budgets = { total };
+    for (const name of SYSTEM_NAMES) {
+        budgets[name] = Math.min(DEFAULT_BUDGETS[name] ?? 0, total);
+    }
+
+    return { ...raw, schema: 2, budgets, callBudgetPerSession: undefined };
+}
+
+function normalize(raw) {
+    const source = raw?.schema === 1 ? migrate(raw) : raw;
+    if (source?.schema !== 2) {
+        return defaultSettings();
+    }
+
+    const systems = Object.fromEntries(SYSTEM_NAMES.map((name) => [name, source.systems?.[name] === true]));
+
+    return {
+        schema: 2,
+        master: source.master === true,
+        startup: source.startup === true,
+        systems,
+        budgets: normalizeBudgets(source.budgets),
+        progressNudge: NUDGE_MODES.includes(source.progressNudge) ? source.progressNudge : "notify",
+        guard: { enabled: source.guard?.enabled === true, startup: source.guard?.startup === true },
+    };
+}
+
+export function loadSettings() {
+    try {
+        const file = settingsFile();
+        if (!regularFile(file, "Jev settings")) {
+            return defaultSettings();
+        }
+
+        return normalize(JSON.parse(fs.readFileSync(file, "utf8")));
+    } catch {
+        return defaultSettings();
+    }
+}
+
+/** Shared atomic write for every file this extension owns. */
+export function writeFileAtomic(file, contents) {
+    const directory = path.dirname(file);
+    fs.mkdirSync(directory, { recursive: true, mode: 0o700 });
+    const temporary = path.join(directory, `.${path.basename(file)}.${randomUUID()}.tmp`);
+    try {
+        fs.writeFileSync(temporary, contents, { mode: 0o600, flag: "wx" });
+        fs.renameSync(temporary, file);
+    } finally {
+        fs.rmSync(temporary, { force: true });
+    }
+}
+
+export function saveSettings(settings) {
+    // A caller handing back a schema-1 shape is migrated rather than reset, so a round trip through
+    // an old reader cannot quietly disable the layer.
+    const next = normalize(settings?.schema === 1 ? settings : { ...settings, schema: 2 });
+    const file = settingsFile();
+    if (fs.existsSync(file)) {
+        regularFile(file, "Jev settings");
+    }
+
+    writeFileAtomic(file, `${JSON.stringify(next, null, 4)}\n`);
+
+    return next;
+}
+
+export function settingsPath() {
+    return settingsFile();
+}
+
+/**
+ * The key is never read into any structure that gets logged or serialized. Callers only ever ask
+ * whether one is present; the client reads it directly at call time.
+ */
+export function keyPresent() {
+    // OPENROUTER_API_KEY on the default path, TYPESAFE_API_KEY on the direct one; see client.mjs.
+    for (const name of ["OPENROUTER_API_KEY", "TYPESAFE_API_KEY"]) {
+        const value = process.env[name];
+        if (typeof value === "string" && value.trim().length > 0) {
+            return true;
+        }
+    }
+
+    return false;
+}

package/extensions/jev-advisor/consent.mjs

@@ -0,0 +1,133 @@
+// A config flag records an intention. It does not record that a human was told what the intention
+// costs. This file holds the separate, explicit grant: the first time any system would put session
+// state on the wire, a dialog names the endpoint, the shape of the data and the byte budget, and
+// nothing is sent until someone says yes.
+//
+// Mirrors capability-policy: SpecPi's own file, atomic, mode 0600, symlinks refused, missing or
+// unreadable read as "ask". No interactive UI means no send, ever.
+
+import fs from "node:fs";
+import path from "node:path";
+import { jevDirectory, regularFile, writeFileAtomic } from "./config.mjs";
+import { baseUrl } from "./client.mjs";
+import { MAX_STATE_BYTES } from "./sanitize.mjs";
+
+/**
+ * The host the bytes will actually reach, not the service they are nominally for. This was a fixed
+ * "api.typesafe.ai" until the default backend became OpenRouter, at which point the dialog named a
+ * host the data no longer went to, which is the one thing a consent dialog may never do. It is
+ * derived now, so each of the three selectable destinations (OpenRouter, the direct TypeSafe API,
+ * and a TYPESAFE_BASE_URL override) names itself, and so the rule below is true rather than
+ * aspirational: a grant is keyed on this string, so repointing the client really does ask again.
+ */
+export function endpointLabel() {
+    const base = baseUrl();
+    try {
+        return new URL(base).host;
+    } catch {
+        // Unparseable means the fetch will fail anyway. Returning the raw value keeps the dialog
+        // honest and cannot collide with a host a real grant was given for.
+        return base;
+    }
+}
+
+function consentFile() {
+    return path.join(jevDirectory(), "consent.json");
+}
+
+export function loadConsent() {
+    try {
+        const file = consentFile();
+        if (!regularFile(file, "Jev consent")) {
+            return undefined;
+        }
+
+        const stored = JSON.parse(fs.readFileSync(file, "utf8"));
+        if (stored?.schema !== 1 || stored.granted !== true || typeof stored.endpoint !== "string") {
+            return undefined;
+        }
+
+        // A grant is for the endpoint it was given for. Repointing the client asks again.
+        return stored.endpoint === endpointLabel() ? stored : undefined;
+    } catch {
+        return undefined;
+    }
+}
+
+export function granted() {
+    return loadConsent() !== undefined;
+}
+
+export function saveConsent() {
+    const file = consentFile();
+    if (fs.existsSync(file)) {
+        regularFile(file, "Jev consent");
+    }
+
+    const stored = {
+        schema: 1,
+        granted: true,
+        endpoint: endpointLabel(),
+        maxStateBytes: MAX_STATE_BYTES,
+        grantedAt: new Date().toISOString(),
+    };
+    writeFileAtomic(file, `${JSON.stringify(stored, null, 4)}\n`);
+
+    return stored;
+}
+
+export function revokeConsent() {
+    try {
+        fs.rmSync(consentFile(), { force: true });
+    } catch {
+        // A consent file that cannot be removed still reads as granted; the master switch is the
+        // reliable stop, and /jev status reports both.
+    }
+}
+
+export function consentPath() {
+    return consentFile();
+}
+
+export const CONSENT_TITLE = "Allow SpecPi to send task summaries to Jev?";
+
+export function consentBody(systemLabel) {
+    return [
+        `${systemLabel} wants to ask TypeSafe's Jev classifier a question about this session.`,
+        "",
+        `What is sent: a summary object of at most ${MAX_STATE_BYTES} bytes to ${endpointLabel()}, over HTTPS.`,
+        "It carries tool names, byte counts, relative paths and short descriptions.",
+        "It never carries file contents, command output, credentials, URLs or session history.",
+        "",
+        "Every call is recorded locally in transmissions.jsonl with a hash of exactly what was sent,",
+        "which you can read with /jev ledger. Turn this off at any time with /jev off.",
+    ].join("\n");
+}
+
+/**
+ * Resolve consent for a system, prompting once. Returns false without prompting when there is no
+ * interactive human: an advisor must never be the reason a headless run sends data.
+ */
+export async function ensureConsent(ctx, systemLabel) {
+    if (granted()) {
+        return true;
+    }
+
+    if (!ctx?.hasUI || typeof ctx.ui?.confirm !== "function") {
+        return false;
+    }
+
+    const accepted = await ctx.ui.confirm(CONSENT_TITLE, consentBody(systemLabel));
+    if (!accepted) {
+        return false;
+    }
+
+    try {
+        saveConsent();
+    } catch {
+        // An unwritable grant means asking again next time, which is the safe direction.
+        return true;
+    }
+
+    return true;
+}