specpi 0.26.0 → 0.27.0

package/extensions/jev-advisor/ledger.mjs

@@ -0,0 +1,138 @@
+// "We only send digests" is a promise until someone can check it. This ledger is the check: one
+// line per call recording what was sent, how big it was and what came back, with a SHA-256 of the
+// exact payload and never the payload itself.
+//
+// It doubles as the calibration corpus. Thresholds in this layer are meant to be read off a curve
+// rather than guessed, and the curve has to come from somewhere.
+
+import fs from "node:fs";
+import path from "node:path";
+import { createHash } from "node:crypto";
+import { jevDirectory, regularFile } from "./config.mjs";
+
+const MAX_LINES = 5000;
+const MAX_LEDGER_BYTES = 8 * 1024 * 1024;
+
+export function ledgerPath() {
+    return path.join(jevDirectory(), "transmissions.jsonl");
+}
+
+export function payloadDigest(payload) {
+    return createHash("sha256").update(JSON.stringify(payload)).digest("hex");
+}
+
+/**
+ * Append one record. Ledger failure never fails the call that produced it: an advisor that cannot
+ * write its audit line still must not break the session, so the error is swallowed and the caller
+ * proceeds. The absence of a line is itself visible in `/jev ledger`.
+ */
+export function record(entry) {
+    try {
+        const file = ledgerPath();
+        fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 });
+        if (fs.existsSync(file)) {
+            const stat = fs.lstatSync(file, { throwIfNoEntry: false });
+            if (!stat || !stat.isFile() || stat.isSymbolicLink() || stat.nlink !== 1) {
+                return false;
+            }
+
+            if (stat.size > MAX_LEDGER_BYTES) {
+                rotate(file);
+            }
+        }
+
+        fs.appendFileSync(file, `${JSON.stringify({ at: new Date().toISOString(), ...entry })}\n`, {
+            mode: 0o600,
+        });
+
+        return true;
+    } catch {
+        return false;
+    }
+}
+
+/** Keep the newest half. Unbounded growth is a worse failure than losing old audit lines. */
+function rotate(file) {
+    const lines = fs.readFileSync(file, "utf8").split("\n").filter(Boolean);
+    const kept = lines.slice(-Math.floor(MAX_LINES / 2));
+    fs.writeFileSync(file, kept.length > 0 ? `${kept.join("\n")}\n` : "", { mode: 0o600 });
+}
+
+/**
+ * Roll the ledger up into the few numbers a report wants. Kept here rather than in the eval suite
+ * because the shape of a line is this module's business, and a reader that has to know it is a
+ * second copy of the schema waiting to drift.
+ */
+export function summarize(entries) {
+    const lines = Array.isArray(entries) ? entries : [];
+    const bySystem = {};
+    for (const entry of lines) {
+        const name = typeof entry?.system === "string" ? entry.system : "unknown";
+        const bucket = (bySystem[name] ??= {
+            calls: 0,
+            failed: 0,
+            applied: 0,
+            savedBytes: 0,
+            stateBytes: 0,
+            outcomes: {},
+        });
+        bucket.calls += 1;
+        bucket.failed += entry?.ok === true ? 0 : 1;
+        bucket.applied += entry?.applied === true ? 1 : 0;
+        bucket.savedBytes += Number.isFinite(entry?.savedBytes) ? entry.savedBytes : 0;
+        bucket.stateBytes += Number.isFinite(entry?.stateBytes) ? entry.stateBytes : 0;
+        // "Asked 3 times, applied 0" is a number. "Asked 3 times, applied 0, all three because the
+        // result might hold the answer" is a finding.
+        const outcome =
+            typeof entry?.outcome === "string" ? entry.outcome : entry?.ok === true ? "unrecorded" : "failed";
+        bucket.outcomes[outcome] = (bucket.outcomes[outcome] ?? 0) + 1;
+    }
+
+    const totals = Object.values(bySystem);
+
+    return {
+        calls: lines.length,
+        failed: totals.reduce((sum, item) => sum + item.failed, 0),
+        applied: totals.reduce((sum, item) => sum + item.applied, 0),
+        savedBytes: totals.reduce((sum, item) => sum + item.savedBytes, 0),
+        stateBytes: totals.reduce((sum, item) => sum + item.stateBytes, 0),
+        // Named for what retention does, because it is the only system that shortens anything and
+        // the number is meaningless averaged with systems that cannot.
+        elisions: bySystem.retention?.applied ?? 0,
+        bytesDropped: bySystem.retention?.savedBytes ?? 0,
+        bySystem,
+    };
+}
+
+/** Every line, for a caller that wants to summarize rather than display. `read` is for display. */
+export function readAll() {
+    return read(Number.MAX_SAFE_INTEGER);
+}
+
+export function read(limit = 20) {
+    try {
+        const file = ledgerPath();
+        if (!regularFile(file, "Jev ledger")) {
+            return [];
+        }
+    } catch {
+        // An oversize ledger is still readable; only a link or irregular file is refused.
+    }
+
+    try {
+        const lines = fs.readFileSync(ledgerPath(), "utf8").split("\n").filter(Boolean);
+
+        return lines
+            .slice(-Math.max(1, limit))
+            .map((line) => {
+                try {
+                    return JSON.parse(line);
+                } catch {
+                    return undefined;
+                }
+            })
+            .filter(Boolean);
+    } catch {
+        return [];
+    }
+}

package/extensions/jev-advisor/questions/capabilities.mjs

@@ -0,0 +1,124 @@
+// System 6: decide, once, before the first provider request, whether this session is going to need
+// a withdrawn tool group -- and if so, offer it then rather than in the middle.
+//
+// This is permitted by the standing rule because it is a once-per-session decision made before the
+// first request, which is one of the three cache-safe shapes. It is also distinct from the tool
+// router this project rejected: that rejection was about six tools whose usefulness local state can
+// already determine from wishlist and goal state. "Will this task need a browser" is not in local
+// state, and no amount of inspecting the repository answers it.
+//
+// It exists because Phase 7 measured what the rejection was always assumed to be worth. Flipping
+// Browser QA on at turn 6 of `t3-cascade-ledger` collapsed cached tokens to 3,200 at the next
+// request in three attempts out of three, while the prompt kept climbing, and the re-warm cost 20%
+// of the attempt on average. Arming the same group from the first request cost 16% more than never
+// arming it, against 47% for flipping mid-session. Paying up front is about three times cheaper
+// than paying when the need appears -- so the whole value of this system is moving the decision
+// earlier, and that is all it does.
+//
+// It holds no authority. It activates nothing on its own: it pre-fills the same confirmation the
+// human would have seen from `request_capability`, at turn 0 instead of turn 6. A decline is
+// remembered for the session, and with no interactive human it proposes nothing at all.
+
+import { choice, noul } from "../client.mjs";
+import { choiceValue, nounTrue } from "../gate.mjs";
+import { compact } from "../sanitize.mjs";
+
+export const TASK_KINDS = Object.freeze({
+    code: "Reading or changing source code in this repository",
+    web: "Looking something up online, or reading external pages",
+    ui: "Checking how a page renders or behaves in a browser",
+    ops: "Builds, releases, configuration or tooling",
+    docs: "Writing or editing prose",
+    other: "Anything else",
+});
+
+/**
+ * Cheap, local, and checked before anything is sent. The rule is ask local state first, and while
+ * local state cannot answer "will this need a browser", it answers "is that question worth asking":
+ * a task with no web signal in its wording and a repository with no web assets is not one.
+ */
+const PROMPT_SIGNALS =
+    /\b(https?:\/\/|www\.|url|link|page|site|website|web|browser|render|screenshot|accessibility|a11y|css|dom|localhost|search online|look .{0,12}up)\b/iu;
+
+/** Names that mean this repository has something a browser could open. */
+const WEB_ASSETS = /^(index\.html|.*\.html?|vite\.config\..*|next\.config\..*|svelte\.config\..*|astro\.config\..*)$/iu;
+
+export function promptSignal(prompt) {
+    return PROMPT_SIGNALS.test(String(prompt ?? ""));
+}
+
+/** One bounded readdir of the working directory. Never recursive: this is a hint, not a survey. */
+export function repositorySignal(entries) {
+    return (entries ?? []).slice(0, 200).some((name) => WEB_ASSETS.test(String(name)));
+}
+
+export function localSignals({ prompt, entries }) {
+    const reasons = [];
+    if (promptSignal(prompt)) {
+        reasons.push("prompt-mentions-web");
+    }
+
+    if (repositorySignal(entries)) {
+        reasons.push("repository-has-web-assets");
+    }
+
+    return { ask: reasons.length > 0, reasons };
+}
+
+/**
+ * The prompt is the one place in this layer where the user's own words are sent rather than a
+ * digest of them, and it is bounded hard for that reason. It is the only thing that can answer the
+ * question, and it is one message rather than a transcript.
+ */
+export function buildInput({ prompt, reasons, available, cwdEntries = [] }) {
+    return {
+        request: compact(prompt ?? "", 400),
+        reasons,
+        withdrawnGroups: available,
+        workspaceFiles: cwdEntries.slice(0, 12).map((name) => compact(name, 40)),
+    };
+}
+
+export function questions({ available = [] } = {}) {
+    const asked = { task_kind: choice("What kind of work is this request asking for?", TASK_KINDS) };
+    if (available.includes("web")) {
+        asked.needs_web = noul("Completing this request will require searching the web or fetching an external page");
+    }
+
+    if (available.includes("browser")) {
+        asked.needs_browser = noul(
+            "Completing this request will require opening a page in a browser to check how it renders or behaves",
+        );
+    }
+
+    // Asked whenever the session is capable of it, because the answer is a suggestion to the human
+    // rather than an activation: delegation binds a model and a host and needs its own command.
+    asked.needs_delegation = noul(
+        "This request would be better answered by reading a large set of files than by reasoning about a few",
+    );
+
+    return asked;
+}
+
+/**
+ * Deliberately asymmetric. A false positive costs the group's schema on every request for the rest
+ * of the session and a confirmation the human did not need; a false negative costs nothing at all,
+ * because it leaves today's behaviour exactly as it is and `request_capability` is still there. So
+ * this proposes only on a high bar, and the `capability` thresholds are the strictest in the layer.
+ */
+export function decide(answers, available = []) {
+    const propose = [];
+    if (available.includes("web") && nounTrue(answers?.needs_web, "capability")) {
+        propose.push("web");
+    }
+
+    if (available.includes("browser") && nounTrue(answers?.needs_browser, "capability")) {
+        propose.push("browser");
+    }
+
+    return {
+        propose,
+        taskKind: choiceValue(answers?.task_kind, "capability"),
+        suggestDelegation: nounTrue(answers?.needs_delegation, "capability"),
+    };
+}

package/extensions/jev-advisor/questions/compaction.mjs

@@ -0,0 +1,153 @@
+// System 1b: guide compaction, which is the one moment the prompt cache is discarded anyway.
+//
+// Pi's `findCutPoint` is documented as "walk backwards from newest, accumulating estimated message
+// sizes, stop when we've accumulated >= keepRecentTokens". It is a token ruler: it cannot tell the
+// load-bearing finding from six dead-end greps, and it discards whichever falls on the wrong side
+// of the line. Since the prefix is being rebuilt regardless, improving that choice costs nothing.
+//
+// This system never sets the cut itself. It supplies `customInstructions` — an existing documented
+// parameter on the compaction path — so the summariser is told what this session was actually
+// about. The token budget still bounds the result, so bad advice can shape a summary, never blow
+// the budget or drop an entry the preparation meant to keep.
+
+import { choice, noul } from "../client.mjs";
+import { choiceValue, nounTrue } from "../gate.mjs";
+import { compact } from "../sanitize.mjs";
+
+export const WORK_KINDS = Object.freeze({
+    debugging: "Tracking down why something fails",
+    building: "Adding or changing a feature",
+    refactoring: "Restructuring code without changing behaviour",
+    research: "Reading and answering questions about a codebase",
+    testing: "Writing or repairing tests",
+    ops: "Builds, releases, configuration or tooling",
+    review: "Reading a diff and judging it",
+});
+
+/**
+ * A digest of what compaction is about to discard: entry kinds and scale, never their text. The
+ * summariser still sees the real conversation; this only steers what it keeps.
+ */
+export function buildInput({ preparation, objective }) {
+    const messages = preparation?.messagesToSummarize ?? [];
+    const kinds = {};
+    for (const message of messages) {
+        const role = typeof message?.role === "string" ? message.role : "unknown";
+        kinds[role] = (kinds[role] ?? 0) + 1;
+    }
+
+    const files = preparation?.fileOps ?? {};
+
+    return {
+        objective: compact(objective ?? "", 180),
+        discarding: messages.length,
+        roles: kinds,
+        tokensBefore: preparation?.tokensBefore ?? 0,
+        splitTurn: preparation?.isSplitTurn === true,
+        filesRead: [...(files.read ?? [])].slice(0, 12).map((item) => compact(item, 60)),
+        filesWritten: [...(files.written ?? []), ...(files.edited ?? [])].slice(0, 12).map((item) => compact(item, 60)),
+        hadPreviousSummary: typeof preparation?.previousSummary === "string",
+    };
+}
+
+/**
+ * The same digest for a branch being left behind. `/tree` hands a different preparation shape --
+ * session entries rather than messages, and no token count, because nothing is being cut to fit a
+ * budget -- so it gets its own builder rather than a compaction input with three fields quietly
+ * reading undefined.
+ */
+export function buildBranchInput({ preparation, objective }) {
+    const entries = preparation?.entriesToSummarize ?? [];
+    const kinds = {};
+    for (const entry of entries) {
+        const kind = typeof entry?.type === "string" ? entry.type : "unknown";
+        kinds[kind] = (kinds[kind] ?? 0) + 1;
+    }
+
+    return {
+        objective: compact(objective ?? "", 180),
+        abandoning: entries.length,
+        kinds,
+        wantsSummary: preparation?.userWantsSummary === true,
+        // Navigating to an ancestor is backing out of a line of work; navigating elsewhere is
+        // moving between siblings. The distinction is most of what a label has to capture.
+        toAncestor: preparation?.targetId === preparation?.commonAncestorId,
+    };
+}
+
+/**
+ * Short, navigational, and a fixed enum so no model-written text reaches the session file. `/tree`
+ * can filter to labelled entries, so a branch that says what it was is the difference between a
+ * navigable tree and a list of timestamps.
+ */
+export const BRANCH_LABELS = Object.freeze({
+    "dead end": "The branch was abandoned because the approach did not work",
+    "alternative tried": "A different approach to the same goal, set aside for another",
+    "work completed": "The branch finished what it set out to do",
+    research: "The branch was reading and answering questions, not changing anything",
+    reverted: "The branch's changes were undone",
+    interrupted: "The branch stopped part-way for an unrelated reason",
+});
+
+export function questions({ branch = false } = {}) {
+    return {
+        ...(branch ? { branch_label: choice("What was this abandoned branch?", BRANCH_LABELS) } : {}),
+        work_kind: choice("What kind of work has this session mostly been doing?", WORK_KINDS),
+        unresolved_thread: noul("There is an unfinished investigation whose findings must survive compaction"),
+        discarded_span_was_dead_ends: noul(
+            "The work being discarded was mostly abandoned attempts that led nowhere useful",
+        ),
+    };
+}
+
+const FOCUS = Object.freeze({
+    debugging: "the symptom, what has been ruled out, and the current hypothesis",
+    building: "what has been implemented so far and what remains",
+    refactoring: "the invariants being preserved and which call sites have been updated",
+    research: "the questions answered so far, with the files each answer came from",
+    testing: "which tests exist, which fail, and why",
+    ops: "the commands run, their outcomes, and the current configuration state",
+    review: "the findings raised so far and their severity",
+});
+
+/**
+ * Build `customInstructions` from gated answers only. With nothing gated this returns undefined and
+ * Pi's own default prompt is used unchanged.
+ */
+/**
+ * The label for a branch summary entry, or undefined when the answer is ungated. Separate from
+ * `decide` because the compaction hook has no label to set and would carry a dead field.
+ */
+export function label(answers) {
+    const value = choiceValue(answers?.branch_label, "compaction");
+
+    return value && Object.hasOwn(BRANCH_LABELS, value) ? value : undefined;
+}
+
+export function decide(answers) {
+    const kind = choiceValue(answers?.work_kind, "compaction");
+    const unresolved = nounTrue(answers?.unresolved_thread, "compaction");
+    const deadEnds = nounTrue(answers?.discarded_span_was_dead_ends, "compaction");
+    const parts = [];
+    if (kind && FOCUS[kind]) {
+        parts.push(`This session has mainly been ${kind}. Prioritise ${FOCUS[kind]}.`);
+    }
+
+    if (unresolved) {
+        parts.push(
+            "An investigation is still open. Preserve its findings and the current hypothesis in full, even at the cost of earlier detail.",
+        );
+    }
+
+    if (deadEnds) {
+        parts.push(
+            "Most of the discarded work was abandoned attempts. Record what was ruled out in one line each rather than recounting them, so the same paths are not retried.",
+        );
+    }
+
+    if (parts.length === 0) {
+        return { customInstructions: undefined, deadEnds, unresolved };
+    }
+
+    return { customInstructions: parts.join(" "), deadEnds, unresolved };
+}

package/extensions/jev-advisor/questions/gap.mjs

@@ -0,0 +1,140 @@
+// System 2: triage a capability-gap report before it is written.
+//
+// This fixes two real defects in the improvement loop rather than saving tokens.
+//
+// Fragmentation: `aggregateEvents` groups by exact `canonicalKey` string, with manual alias
+// decisions as the only correction, and qualification needs `priority >= blocked || occurrences
+// >= 2`. A genuinely recurring problem that the model names differently each time fragments into
+// singletons and can stay permanently unqualified — it never reaches the human at
+// /harness-improvement. Lexical matching cannot fix that; semantic clustering can.
+//
+// Self-reported severity: `priority` sums IMPACT_WEIGHT[event.impact], and `impact` is reported by
+// the model about its own gap. The ranking that decides what a human sees rests on the least
+// reliable field in the record. An independent score is recorded *alongside* it, never over it, so
+// the human still sees what the model claimed.
+//
+// Authority is unchanged. Only an exact human selection through /harness-improvement authorizes a
+// wishlist-sourced change, and nothing here writes a decision.
+
+import { choice, noul, score } from "../client.mjs";
+import { choiceValue, nounTrue, scoreLevel } from "../gate.mjs";
+import { compact } from "../sanitize.mjs";
+
+export const IMPACT_LEVELS = Object.freeze([
+    "Minor: a small inconvenience with an easy workaround",
+    "Moderate: real friction that cost time or forced a detour",
+    "Blocked: the task could not be completed as asked",
+]);
+
+export const FIX_KINDS = Object.freeze({
+    tool: "A new or changed tool would solve it",
+    skill: "A skill or documented procedure would solve it",
+    prompt: "Different instructions would solve it",
+    config: "A configuration or setting change would solve it",
+    bug: "Something is broken and should be repaired",
+    unknown: "Not clear from what was observed",
+});
+
+export const NEW_CLUSTER = "__new__";
+
+/** The Choice ceiling is 255; above the cap, rank by similarity first and choose among the top N. */
+export const MAX_CLUSTER_OPTIONS = 200;
+
+function similarity(a, b) {
+    const left = new Set(
+        String(a)
+            .toLowerCase()
+            .split(/[^a-z0-9]+/u)
+            .filter(Boolean),
+    );
+    const right = new Set(
+        String(b)
+            .toLowerCase()
+            .split(/[^a-z0-9]+/u)
+            .filter(Boolean),
+    );
+    if (left.size === 0 || right.size === 0) {
+        return 0;
+    }
+
+    let shared = 0;
+    for (const token of left) {
+        if (right.has(token)) {
+            shared += 1;
+        }
+    }
+
+    return shared / Math.max(left.size, right.size);
+}
+
+/**
+ * TypeSafe's own two-stage pattern for high cardinality: score candidates cheaply, then make one
+ * explicit choice among the survivors. Below the cap every key is offered.
+ */
+export function clusterOptions(existing, gap) {
+    const keys = [...new Set(existing.map((item) => item.canonicalKey).filter(Boolean))];
+    const probe = `${gap?.capability ?? ""} ${gap?.scenario ?? ""}`;
+    const ranked =
+        keys.length <= MAX_CLUSTER_OPTIONS
+            ? keys
+            : keys
+                  .map((key) => ({ key, rank: similarity(key, probe) }))
+                  .sort((a, b) => b.rank - a.rank)
+                  .slice(0, MAX_CLUSTER_OPTIONS)
+                  .map((item) => item.key);
+
+    const criteria = { [NEW_CLUSTER]: "This is a distinct problem not already on the list" };
+    for (const key of ranked) {
+        const match = existing.find((item) => item.canonicalKey === key);
+        criteria[key] = compact(match?.title ?? key, 90);
+    }
+
+    return criteria;
+}
+
+export function buildInput({ gap, existing = [] }) {
+    return {
+        capability: compact(gap?.capability ?? "", 120),
+        scenario: compact(gap?.scenario ?? "", 200),
+        limitation: compact(gap?.limitation ?? "", 160),
+        workaround: compact(gap?.workaround ?? "", 160),
+        claimedImpact: compact(gap?.impact ?? "", 20),
+        knownProblems: existing.slice(0, 8).map((item) => compact(item.title ?? item.canonicalKey, 60)),
+    };
+}
+
+export function questions({ gap, existing = [] }) {
+    return {
+        cluster: choice(
+            "Which known problem is this the same underlying problem as, if any?",
+            clusterOptions(existing, gap),
+        ),
+        independent_impact: score("How badly did this actually obstruct the task?", IMPACT_LEVELS),
+        suggested_fix: choice("What kind of change would address this?", FIX_KINDS),
+        contains_secret_or_path: noul(
+            "This report contains a credential, an absolute filesystem path, or other machine-specific detail",
+        ),
+        is_transient_or_user_error: noul(
+            "This was a one-off failure or a mistake in how the task was asked, not a reusable gap in the harness",
+        ),
+    };
+}
+
+const IMPACT_NAMES = Object.freeze(["minor", "moderate", "blocked"]);
+
+/**
+ * Advice, in the shape the caller applies it. `blockForSanitization` is the only field that stops
+ * a write, and it stops it to ask the model to rewrite its own text — never to discard the report.
+ */
+export function decide(answers) {
+    const cluster = choiceValue(answers?.cluster, "gap");
+    const level = scoreLevel(answers?.independent_impact, "gap");
+
+    return {
+        canonicalKey: cluster && cluster !== NEW_CLUSTER ? cluster : undefined,
+        independentImpact: level === undefined ? undefined : IMPACT_NAMES[level],
+        suggestedFix: choiceValue(answers?.suggested_fix, "gap"),
+        blockForSanitization: nounTrue(answers?.contains_secret_or_path, "gap"),
+        transient: nounTrue(answers?.is_transient_or_user_error, "gap"),
+    };
+}