useathena 0.1.0

package/dist/config.js

@@ -0,0 +1,26 @@
+import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { dirname, join } from "node:path";
+/**
+ * athena is a personal layer: one store per user by default, overridable for
+ * tests and project-scoped setups via ATHENA_DB.
+ */
+export function dbPath() {
+    return process.env.ATHENA_DB ?? join(homedir(), ".athena", "athena.db");
+}
+export function configPath() {
+    return process.env.ATHENA_CONFIG ?? join(homedir(), ".athena", "config.json");
+}
+export function loadConfig() {
+    try {
+        return JSON.parse(readFileSync(configPath(), "utf8"));
+    }
+    catch {
+        return {};
+    }
+}
+export function saveConfig(config) {
+    const path = configPath();
+    mkdirSync(dirname(path), { recursive: true });
+    writeFileSync(path, `${JSON.stringify(config, null, 2)}\n`, { mode: 0o600 });
+}

package/dist/core/fixtures.js

@@ -0,0 +1,65 @@
+import { newId } from "./ids.js";
+/**
+ * Fixture builders for tests and the eval harness. Defaults are valid records;
+ * override only what the scenario needs.
+ */
+export const TEST_ACTOR = "act_0000000000TESTACTOR0000XX";
+export function makeInstance(overrides = {}) {
+    const base = {
+        id: newId("ins"),
+        kind: "correction",
+        observedAt: new Date(0).toISOString(),
+        situation: {
+            summary: "drafting cold outreach to enterprise CTO",
+            domain: "email.outreach",
+            cues: [],
+            objectIds: [],
+        },
+        before: {
+            mediaType: "text/plain",
+            content: "Hi Priya, hope you're doing great!",
+            contentHash: "hash-before",
+        },
+        after: {
+            mediaType: "text/plain",
+            content: "Dear Ms. Shah, I hope this finds you well.",
+            contentHash: "hash-after",
+        },
+        diff: {
+            summary: "changed greeting from casual first-name to formal title",
+            hunks: [{ before: "Hi Priya", after: "Dear Ms. Shah" }],
+            magnitude: "minor",
+        },
+        probeAnswers: [],
+        sensorId: "sen_test",
+        actorId: TEST_ACTOR,
+        sourceRefs: [],
+        objectIds: [],
+        visibility: "user_private_raw",
+        canPromote: false,
+        canUseForAgents: false,
+    };
+    return { ...base, ...overrides };
+}
+export function makeHypothesis(overrides = {}) {
+    const base = {
+        id: newId("hyp"),
+        status: "candidate",
+        rule: "When drafting outreach to enterprise contacts, use formal titles in greetings.",
+        cues: ["recipient is senior at an enterprise account", "first written contact"],
+        expectancies: ["recipient replies without tone friction"],
+        domain: "email.outreach",
+        appliesWhen: ["first or early-stage outreach", "recipient seniority is director+"],
+        doesNotApplyWhen: ["established casual rapport with the recipient"],
+        supportingInstanceIds: [newId("ins")],
+        counterexampleInstanceIds: [],
+        confidence: 0.5,
+        validity: { fires: 0, upheld: 0, overridden: 0 },
+        replay: { tested: 0, reproduced: 0 },
+        createdAt: new Date(0).toISOString(),
+        staleAfter: new Date(90 * 24 * 3600 * 1000).toISOString(),
+        visibility: "user_private",
+        review: { state: "unreviewed" },
+    };
+    return { ...base, ...overrides };
+}

package/dist/core/ids.js

@@ -0,0 +1,34 @@
+import { randomBytes } from "node:crypto";
+/**
+ * Prefixed ULIDs: time-sortable, collision-safe, and self-describing
+ * (`ins_01J...` is an instance wherever it appears).
+ */
+const ENCODING = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"; // Crockford base32
+const TIME_LEN = 10;
+const RANDOM_LEN = 16;
+function encodeTime(now) {
+    let value = now;
+    let out = "";
+    for (let i = 0; i < TIME_LEN; i++) {
+        out = ENCODING[value % 32] + out;
+        value = Math.floor(value / 32);
+    }
+    return out;
+}
+function encodeRandom() {
+    const bytes = randomBytes(RANDOM_LEN);
+    let out = "";
+    for (let i = 0; i < RANDOM_LEN; i++) {
+        out += ENCODING[bytes[i] % 32];
+    }
+    return out;
+}
+export function ulid(now = Date.now()) {
+    return encodeTime(now) + encodeRandom();
+}
+export function newId(prefix, now) {
+    return `${prefix}_${ulid(now)}`;
+}
+export function hasPrefix(id, prefix) {
+    return id.startsWith(`${prefix}_`) && id.length === prefix.length + 1 + TIME_LEN + RANDOM_LEN;
+}

package/dist/core/refs.js

@@ -0,0 +1,25 @@
+const KIND_BY_PREFIX = {
+    ins: "instance",
+    hyp: "hypothesis",
+    obj: "object",
+    src: "source",
+    out: "outcome",
+    brf: "brief",
+};
+export function refTo(id, fragment) {
+    const prefix = id.split("_")[0] ?? "";
+    const kind = KIND_BY_PREFIX[prefix];
+    if (!kind)
+        throw new Error(`cannot build ref for unknown id prefix: ${id}`);
+    return `athena://${kind}/${id}${fragment ? `#${fragment}` : ""}`;
+}
+export function parseRef(ref) {
+    const match = /^athena:\/\/([a-z]+)\/([a-z]{3}_[0-9A-HJKMNP-TV-Z]+)(?:#(.+))?$/.exec(ref);
+    if (!match)
+        throw new Error(`invalid athena ref: ${ref}`);
+    const [, kindRaw, id, fragment] = match;
+    const kind = Object.values(KIND_BY_PREFIX).find((k) => k === kindRaw);
+    if (!kind)
+        throw new Error(`invalid athena ref kind: ${ref}`);
+    return fragment === undefined ? { kind, id: id } : { kind, id: id, fragment };
+}

package/dist/core/types.js

@@ -0,0 +1,10 @@
+/**
+ * athena core domain types. Contract: docs/schema.md.
+ *
+ * Design rules enforced here:
+ * - Instances are immutable evidence; hypotheses are revisable views over evidence.
+ * - Everything cites: hypotheses carry instance ids, brief facts carry refs.
+ * - Privacy-zone fields are mandatory on capture-derived records.
+ * - Enums start minimal; widen only when real data forces it.
+ */
+export {};

package/dist/engine/engine.js

@@ -0,0 +1,136 @@
+import { newId } from "../core/ids.js";
+import { INFERENCE_SYSTEM, REPLAY_SYSTEM, inferencePrompt, replayPrompt } from "./prompts.js";
+import { parseInferenceResponse, parseReplayResponse } from "./parse.js";
+/**
+ * The hypothesis engine: instances in, evidence-cited hypotheses out.
+ *
+ * Per domain cluster:
+ *   1. hold out the most recent correction (the engine never sees its "after"),
+ *   2. one contrastive inference pass over the rest,
+ *   3. replay every candidate against the holdout — the validation gate.
+ *
+ * Candidates that reproduce the held-out edit become "validated"; candidates
+ * that fail keep "candidate" status with docked confidence. There is no
+ * deterministic fallback: no model, no inference.
+ */
+const MIN_CLUSTER_SIZE = 2;
+const MIN_CORRECTIONS_FOR_HOLDOUT = 3;
+const STALE_AFTER_DAYS = 90;
+const FAILED_REPLAY_CONFIDENCE_CAP = 0.35;
+const DEFAULT_REPLAY_SAMPLES = 3;
+export class LlmHypothesisEngine {
+    model;
+    now;
+    replaySamples;
+    constructor(model, options = {}) {
+        this.model = model;
+        this.now = options.now ?? (() => new Date());
+        this.replaySamples = Math.max(1, options.replaySamples ?? DEFAULT_REPLAY_SAMPLES);
+    }
+    async infer(instances) {
+        const hypotheses = [];
+        for (const [domain, cluster] of clusterByDomain(instances)) {
+            if (cluster.length < MIN_CLUSTER_SIZE)
+                continue;
+            hypotheses.push(...(await this.inferCluster(domain, cluster)));
+        }
+        return hypotheses;
+    }
+    async inferCluster(domain, cluster) {
+        const { train, holdout } = splitHoldout(cluster);
+        const raw = await this.model.generateJson({
+            system: INFERENCE_SYSTEM,
+            prompt: inferencePrompt(domain, train),
+        });
+        const candidates = parseInferenceResponse(raw, train.length);
+        const results = [];
+        for (const candidate of candidates) {
+            const supportingInstanceIds = candidate.supporting.map((index) => train[index - 1].id);
+            if (supportingInstanceIds.length === 0)
+                continue; // everything cites — uncited rules are dropped
+            const hypothesis = this.toHypothesis(domain, candidate, supportingInstanceIds, train);
+            for (const heldOut of holdout) {
+                const verdict = await this.replayVerdict(hypothesis, heldOut);
+                hypothesis.replay.tested += 1;
+                if (verdict)
+                    hypothesis.replay.reproduced += 1;
+            }
+            hypothesis.replay.lastRunAt = this.now().toISOString();
+            if (hypothesis.replay.tested > 0) {
+                if (hypothesis.replay.reproduced === hypothesis.replay.tested) {
+                    hypothesis.status = "validated";
+                }
+                else if (hypothesis.replay.reproduced === 0) {
+                    hypothesis.confidence = Math.min(hypothesis.confidence, FAILED_REPLAY_CONFIDENCE_CAP);
+                }
+            }
+            results.push(hypothesis);
+        }
+        return results;
+    }
+    /** Majority vote across samples; stops early once a side is unreachable. */
+    async replayVerdict(hypothesis, heldOut) {
+        const needed = Math.floor(this.replaySamples / 2) + 1;
+        let matches = 0;
+        let misses = 0;
+        while (matches < needed && misses < needed) {
+            const verdict = parseReplayResponse(await this.model.generateJson({ system: REPLAY_SYSTEM, prompt: replayPrompt(hypothesis, heldOut) }));
+            if (verdict)
+                matches += 1;
+            else
+                misses += 1;
+        }
+        return matches >= needed;
+    }
+    toHypothesis(domain, candidate, supportingInstanceIds, train) {
+        const now = this.now();
+        return {
+            id: newId("hyp", now.getTime()),
+            status: "candidate",
+            rule: candidate.rule,
+            cues: candidate.cues,
+            expectancies: candidate.expectancies,
+            ...(candidate.goal !== undefined ? { goal: candidate.goal } : {}),
+            domain,
+            appliesWhen: candidate.appliesWhen,
+            doesNotApplyWhen: candidate.doesNotApplyWhen,
+            supportingInstanceIds,
+            counterexampleInstanceIds: candidate.counterexamples.map((index) => train[index - 1].id),
+            inferredRationale: candidate.rationale,
+            confidence: candidate.confidence,
+            validity: { fires: 0, upheld: 0, overridden: 0 },
+            replay: { tested: 0, reproduced: 0 },
+            createdAt: now.toISOString(),
+            staleAfter: new Date(now.getTime() + STALE_AFTER_DAYS * 24 * 3600 * 1000).toISOString(),
+            visibility: "user_private",
+            review: { state: "unreviewed" },
+        };
+    }
+}
+function clusterByDomain(instances) {
+    const clusters = new Map();
+    for (const instance of instances) {
+        const cluster = clusters.get(instance.situation.domain) ?? [];
+        cluster.push(instance);
+        clusters.set(instance.situation.domain, cluster);
+    }
+    for (const cluster of clusters.values()) {
+        cluster.sort((a, b) => a.observedAt.localeCompare(b.observedAt));
+    }
+    return clusters;
+}
+/**
+ * Hold out the most recent correction so replay tests generalization, not recall.
+ * Approvals always stay in train — they are boundary evidence, not edits to predict.
+ */
+export function splitHoldout(cluster) {
+    const corrections = cluster.filter((i) => i.kind !== "approval" && i.before && i.after);
+    if (corrections.length < MIN_CORRECTIONS_FOR_HOLDOUT) {
+        return { train: cluster, holdout: [] };
+    }
+    const heldOut = corrections[corrections.length - 1];
+    return {
+        train: cluster.filter((i) => i.id !== heldOut.id),
+        holdout: [heldOut],
+    };
+}

package/dist/engine/parse.js

@@ -0,0 +1,76 @@
+/**
+ * Strict parsing of the inference response. The model is untrusted input;
+ * everything is validated, indices are checked against the instance count,
+ * and a malformed response throws with a message naming what was wrong.
+ */
+export function parseInferenceResponse(raw, instanceCount) {
+    if (typeof raw !== "object" || raw === null || !Array.isArray(raw.hypotheses)) {
+        throw new Error(`inference response missing "hypotheses" array: ${preview(raw)}`);
+    }
+    return (raw.hypotheses).map((entry, i) => parseHypothesis(entry, i, instanceCount));
+}
+function parseHypothesis(entry, position, instanceCount) {
+    if (typeof entry !== "object" || entry === null) {
+        throw new Error(`hypothesis ${position} is not an object`);
+    }
+    const record = entry;
+    const rule = requireString(record, "rule", position);
+    const confidence = record.confidence;
+    if (typeof confidence !== "number" || confidence < 0 || confidence > 1) {
+        throw new Error(`hypothesis ${position} ("${rule.slice(0, 40)}") has invalid confidence`);
+    }
+    const supporting = requireIndexArray(record, "supporting", position, instanceCount);
+    return {
+        rule,
+        cues: stringArray(record.cues),
+        expectancies: stringArray(record.expectancies),
+        ...(typeof record.goal === "string" && record.goal.length > 0 ? { goal: record.goal } : {}),
+        appliesWhen: stringArray(record.appliesWhen),
+        doesNotApplyWhen: stringArray(record.doesNotApplyWhen),
+        supporting,
+        counterexamples: requireIndexArray(record, "counterexamples", position, instanceCount),
+        rationale: typeof record.rationale === "string" ? record.rationale : "",
+        confidence: clamp(confidence, 0.05, 0.95),
+    };
+}
+export function parseReplayResponse(raw) {
+    if (typeof raw === "object" && raw !== null && typeof raw.match === "boolean") {
+        return raw.match;
+    }
+    throw new Error(`replay response missing boolean "match": ${preview(raw)}`);
+}
+function requireString(record, key, position) {
+    const value = record[key];
+    if (typeof value !== "string" || value.trim().length === 0) {
+        throw new Error(`hypothesis ${position} missing string "${key}"`);
+    }
+    return value;
+}
+function requireIndexArray(record, key, position, instanceCount) {
+    const value = record[key];
+    if (value === undefined)
+        return [];
+    if (!Array.isArray(value) || !value.every((n) => Number.isInteger(n))) {
+        throw new Error(`hypothesis ${position} has invalid "${key}" (must be instance numbers)`);
+    }
+    const indexes = value;
+    for (const index of indexes) {
+        if (index < 1 || index > instanceCount) {
+            throw new Error(`hypothesis ${position} cites instance ${index}, but only 1..${instanceCount} exist`);
+        }
+    }
+    return indexes;
+}
+function stringArray(value) {
+    if (value === undefined)
+        return [];
+    if (!Array.isArray(value) || !value.every((s) => typeof s === "string"))
+        return [];
+    return value;
+}
+function clamp(value, low, high) {
+    return Math.min(high, Math.max(low, value));
+}
+function preview(raw) {
+    return JSON.stringify(raw)?.slice(0, 200) ?? String(raw);
+}

package/dist/engine/prompts.js

@@ -0,0 +1,64 @@
+/**
+ * Prompts for the hypothesis engine. The design encodes the capture doctrine
+ * (docs/research): behavior over self-report, bands over one-way rules,
+ * approvals as boundary evidence, no rules beyond evidence.
+ */
+export const INFERENCE_SYSTEM = `You extract tacit judgment rules from observed corrections of agent-drafted work.
+
+You receive numbered instances from ONE domain. Corrections show a draft and the human's version. Approvals show a draft the human deliberately kept unchanged.
+
+Extract the transferable judgment rules behind these edits. Requirements:
+
+- A rule is a transferable judgment ("in situations like S, do X because Y"), never a description of a single edit.
+- If corrections move in OPPOSITE directions (some make text warmer, some less salesy; some add an ask, some soften an ask), that is ONE calibration-band rule with both limits stated — not two rules, and never a one-directional rule.
+- Approvals are deliberate keep-decisions and they OUTRANK your generalizations. If an approval keeps something your rule would change (a long reply kept, formality kept, a pitch kept), work out from that approval's situation what licensed the exception, and state that trigger explicitly in doesNotApplyWhen (e.g. "the thread explicitly asked for a detailed write-up"). A rule that would re-edit an approved draft in its own training set is wrong as stated.
+- Only propose a rule supported by at least 2 instances, unless a probe answer explicitly states the human said it is a general rule.
+- Do not invent rules about anything that was never corrected. Fewer, better-supported rules beat many speculative ones.
+- cues = what signals that a situation is this kind of situation. expectancies = what you would observe when the rule is working, and what signals it is misfiring.
+- Cite supporting instances by their number. Cite counterexamples (instances that cut against the rule) by number too.
+- confidence is 0..1: your honest estimate that this rule reflects a stable preference rather than coincidence.
+
+Respond with STRICT JSON only, no prose, matching:
+{"hypotheses": [{"rule": string, "cues": string[], "expectancies": string[], "goal": string?, "appliesWhen": string[], "doesNotApplyWhen": string[], "supporting": number[], "counterexamples": number[], "rationale": string, "confidence": number}]}`;
+export function inferencePrompt(domain, instances) {
+    const blocks = instances.map((instance, i) => renderInstance(instance, i + 1));
+    return `Domain: ${domain}\n\n${blocks.join("\n\n")}`;
+}
+function renderInstance(instance, index) {
+    const lines = [];
+    const kindNote = instance.kind === "approval" ? "approval — the human kept this draft unchanged" : instance.kind;
+    lines.push(`Instance ${index} (${kindNote}):`);
+    lines.push(`Situation: ${instance.situation.summary}`);
+    if (instance.situation.task)
+        lines.push(`Task: ${instance.situation.task}`);
+    if (instance.before)
+        lines.push(`Draft before:\n"""\n${instance.before.content}\n"""`);
+    if (instance.after && instance.kind !== "approval") {
+        lines.push(`Human's version:\n"""\n${instance.after.content}\n"""`);
+    }
+    if (instance.diff)
+        lines.push(`Edit summary: ${instance.diff.summary}`);
+    for (const probe of instance.probeAnswers) {
+        if (!probe.dismissed)
+            lines.push(`Probe (${probe.probe}) "${probe.question}" → "${probe.answer}"`);
+    }
+    return lines.join("\n");
+}
+export const REPLAY_SYSTEM = `You judge whether a stored judgment rule would have prevented a correction.
+
+You see a rule, a situation, the agent's draft, and the human's actual version. Question: would an agent that knew and followed this rule have produced the human's version (or avoided the mistake) on its own? The rule must genuinely prescribe the change that was made — topical overlap is not enough.
+
+Respond with STRICT JSON only: {"match": true} or {"match": false}.`;
+export function replayPrompt(hypothesis, instance) {
+    return [
+        `Rule: ${hypothesis.rule}`,
+        hypothesis.appliesWhen.length ? `Applies when: ${hypothesis.appliesWhen.join("; ")}` : "",
+        hypothesis.doesNotApplyWhen.length ? `Does not apply when: ${hypothesis.doesNotApplyWhen.join("; ")}` : "",
+        "",
+        `Situation: ${instance.situation.summary}`,
+        `Agent's draft:\n"""\n${instance.before?.content ?? "(none)"}\n"""`,
+        `Human's version:\n"""\n${instance.after?.content ?? "(unchanged)"}\n"""`,
+    ]
+        .filter((line) => line !== "")
+        .join("\n");
+}

package/dist/eval/harness.js

@@ -0,0 +1,123 @@
+export async function runScenario(engine, scenario, judge) {
+    const hypotheses = await engine.infer(scenario.train);
+    // Match every hypothesis against every planted rule once; everything else reads this map.
+    const matches = new Map();
+    const matchedHypothesisIds = new Set();
+    for (const planted of scenario.planted) {
+        const matched = [];
+        for (const hypothesis of hypotheses) {
+            if (await judge.ruleMatches(hypothesis, planted)) {
+                matched.push(hypothesis);
+                matchedHypothesisIds.add(hypothesis.id);
+            }
+        }
+        matches.set(planted.key, matched);
+    }
+    const found = scenario.planted.filter((p) => (matches.get(p.key) ?? []).length > 0).map((p) => p.key);
+    const missed = scenario.planted.map((p) => p.key).filter((key) => !found.includes(key));
+    // Rules matching an alsoAcceptable pattern are real discoveries, not hallucinations —
+    // they don't count toward recall, but they aren't false either.
+    const falseHypothesisIds = [];
+    for (const hypothesis of hypotheses) {
+        if (matchedHypothesisIds.has(hypothesis.id))
+            continue;
+        let acceptable = false;
+        for (const extra of scenario.alsoAcceptable ?? []) {
+            if (await judge.ruleMatches(hypothesis, extra)) {
+                acceptable = true;
+                break;
+            }
+        }
+        if (!acceptable)
+            falseHypothesisIds.push(hypothesis.id);
+    }
+    let boundariesCovered = 0;
+    let boundariesTotal = 0;
+    for (const planted of scenario.planted) {
+        for (const boundary of planted.boundaries ?? []) {
+            boundariesTotal += 1;
+            const candidates = matches.get(planted.key) ?? [];
+            const verdicts = await Promise.all(candidates.map((h) => judge.boundaryCovered(h, boundary.mustMention)));
+            if (verdicts.some(Boolean))
+                boundariesCovered += 1;
+        }
+    }
+    let applyReproduced = 0;
+    let applyTotal = 0;
+    let boundaryRespected = 0;
+    let boundaryTotal = 0;
+    for (const heldOut of scenario.heldOut) {
+        const candidates = matches.get(heldOut.ruleKey) ?? [];
+        const verdicts = await Promise.all(candidates.map((h) => judge.predictsEdit(h, heldOut)));
+        const hit = verdicts.some(Boolean);
+        if (heldOut.kind === "apply") {
+            applyTotal += 1;
+            if (hit)
+                applyReproduced += 1;
+        }
+        else {
+            boundaryTotal += 1;
+            if (hit)
+                boundaryRespected += 1;
+        }
+    }
+    return {
+        scenarioId: scenario.id,
+        hypothesisCount: hypotheses.length,
+        recall: { found, missed },
+        falseHypothesisIds,
+        boundaries: { covered: boundariesCovered, total: boundariesTotal },
+        replay: { applyReproduced, applyTotal, boundaryRespected, boundaryTotal },
+    };
+}
+export async function runEval(engine, scenarios, judge) {
+    const reports = [];
+    for (const scenario of scenarios) {
+        reports.push(await runScenario(engine, scenario, judge));
+    }
+    const plantedTotal = reports.reduce((n, r) => n + r.recall.found.length + r.recall.missed.length, 0);
+    const foundTotal = reports.reduce((n, r) => n + r.recall.found.length, 0);
+    const hypothesisTotal = reports.reduce((n, r) => n + r.hypothesisCount, 0);
+    const falseTotal = reports.reduce((n, r) => n + r.falseHypothesisIds.length, 0);
+    const boundaryTotal = reports.reduce((n, r) => n + r.boundaries.total, 0);
+    const boundaryCovered = reports.reduce((n, r) => n + r.boundaries.covered, 0);
+    const applyTotal = reports.reduce((n, r) => n + r.replay.applyTotal, 0);
+    const applyReproduced = reports.reduce((n, r) => n + r.replay.applyReproduced, 0);
+    const respectTotal = reports.reduce((n, r) => n + r.replay.boundaryTotal, 0);
+    const respected = reports.reduce((n, r) => n + r.replay.boundaryRespected, 0);
+    return {
+        scenarios: reports,
+        totals: {
+            ruleRecall: ratio(foundTotal, plantedTotal),
+            falseRuleRate: ratio(falseTotal, hypothesisTotal),
+            boundaryCoverage: ratio(boundaryCovered, boundaryTotal),
+            replayReproduction: ratio(applyReproduced, applyTotal),
+            boundaryRespect: ratio(respected, respectTotal),
+        },
+    };
+}
+export function formatReport(report) {
+    const lines = [];
+    for (const s of report.scenarios) {
+        const flags = [];
+        if (s.recall.missed.length)
+            flags.push(`missed: ${s.recall.missed.join(", ")}`);
+        if (s.falseHypothesisIds.length)
+            flags.push(`${s.falseHypothesisIds.length} false`);
+        lines.push(`${s.scenarioId.padEnd(20)} rules ${s.recall.found.length}/${s.recall.found.length + s.recall.missed.length}` +
+            `  boundaries ${s.boundaries.covered}/${s.boundaries.total}` +
+            `  replay ${s.replay.applyReproduced}/${s.replay.applyTotal}` +
+            `  respect ${s.replay.boundaryRespected}/${s.replay.boundaryTotal}` +
+            (flags.length ? `  [${flags.join("; ")}]` : ""));
+    }
+    const t = report.totals;
+    lines.push(`TOTALS  recall ${pct(t.ruleRecall)}  false-rule ${pct(t.falseRuleRate)}  ` +
+        `boundary ${pct(t.boundaryCoverage)}  replay ${pct(t.replayReproduction)}  respect ${pct(t.boundaryRespect)}`);
+    return lines.join("\n");
+}
+function ratio(numerator, denominator) {
+    return denominator === 0 ? 1 : numerator / denominator;
+}
+function pct(value) {
+    return `${Math.round(value * 100)}%`;
+}

package/dist/eval/judge.js

@@ -0,0 +1,75 @@
+export function hypothesisText(hypothesis) {
+    return [
+        hypothesis.rule,
+        hypothesis.goal ?? "",
+        hypothesis.inferredRationale ?? "",
+        ...hypothesis.cues,
+        ...hypothesis.expectancies,
+        ...hypothesis.appliesWhen,
+        ...hypothesis.doesNotApplyWhen,
+    ]
+        .join("\n")
+        .toLowerCase();
+}
+export function boundaryText(hypothesis) {
+    return [...hypothesis.appliesWhen, ...hypothesis.doesNotApplyWhen].join("\n").toLowerCase();
+}
+/** Every group must hit; a group hits when any alternative appears. */
+export function mentionsAll(text, mustMention) {
+    return mustMention.every((group) => group.some((alt) => text.includes(alt.toLowerCase())));
+}
+export class RubricJudge {
+    ruleMatches(hypothesis, planted) {
+        return Promise.resolve(hypothesis.domain === planted.domain && mentionsAll(hypothesisText(hypothesis), planted.mustMention));
+    }
+    boundaryCovered(hypothesis, mustMention) {
+        return Promise.resolve(mentionsAll(boundaryText(hypothesis), mustMention));
+    }
+    predictsEdit(hypothesis, heldOut) {
+        const text = heldOut.kind === "boundary" ? boundaryText(hypothesis) : hypothesisText(hypothesis);
+        return Promise.resolve(mentionsAll(text, heldOut.expectMention));
+    }
+}
+export class LlmJudge {
+    model;
+    constructor(model) {
+        this.model = model;
+    }
+    async ruleMatches(hypothesis, planted) {
+        return this.verdict("You judge whether an inferred judgment rule expresses the same latent rule as a reference rule. " +
+            "Same behavioral prescription and same scope counts as a match; stylistic differences do not matter.", `Reference rule: ${planted.statement}\n\nInferred rule: ${renderHypothesis(hypothesis)}`);
+    }
+    async boundaryCovered(hypothesis, mustMention) {
+        return this.verdict("You judge whether an inferred judgment rule carries a given exception/boundary condition.", `Boundary concepts (any phrasing): ${mustMention.map((g) => g.join(" / ")).join("; ")}\n\n` +
+            `Inferred rule: ${renderHypothesis(hypothesis)}`);
+    }
+    async predictsEdit(hypothesis, heldOut) {
+        const { instance } = heldOut;
+        const question = heldOut.kind === "boundary"
+            ? "Would an agent following this rule correctly leave the draft alone (the rule's boundaries exclude this case)?"
+            : "Would an agent following this rule have produced the human's version (or avoided the mistake) on its own?";
+        return this.verdict(`You judge whether a stored judgment rule would have prevented a correction. ${question}`, `Situation: ${instance.situation.summary} (domain: ${instance.situation.domain})\n\n` +
+            `Draft before: ${instance.before?.content ?? "(none)"}\n\n` +
+            `Human's version: ${instance.after?.content ?? "(unchanged)"}\n\n` +
+            `Rule: ${renderHypothesis(hypothesis)}`);
+    }
+    async verdict(system, prompt) {
+        const raw = await this.model.generateJson({
+            system: `${system} Respond with JSON: {"match": true|false}.`,
+            prompt,
+        });
+        if (typeof raw === "object" && raw !== null && typeof raw.match === "boolean") {
+            return raw.match;
+        }
+        throw new Error(`LlmJudge: invalid verdict from ${this.model.id}: ${JSON.stringify(raw)}`);
+    }
+}
+function renderHypothesis(hypothesis) {
+    return [
+        hypothesis.rule,
+        hypothesis.appliesWhen.length ? `Applies when: ${hypothesis.appliesWhen.join("; ")}` : "",
+        hypothesis.doesNotApplyWhen.length ? `Does not apply when: ${hypothesis.doesNotApplyWhen.join("; ")}` : "",
+    ]
+        .filter(Boolean)
+        .join("\n");
+}