productfoo 0.0.0-stage → 1.0.1

package/dist-public/src/evalRecord.js

@@ -0,0 +1,165 @@
+import { readFileSync, readdirSync, existsSync } from "node:fs";
+import path from "node:path";
+import { parse as parseYaml } from "./vendorYaml.js";
+// The eval contract (foo-eval-system-v0 R-001). An eval is a typed foo record
+// that states its criterion in plain language; how a criterion gets scored is
+// the runner's business, not the record's. The lint below keeps runner and
+// template vocabulary out of records, which is what lets the judge transport
+// change without rewriting a corpus.
+export const EVAL_RECORD_SCHEMA_VERSION = "0.1";
+/**
+ * Where golden sets, owner labels, and seeded artifacts live (R-008).
+ *
+ * The harness is open source; the corpus is not, and it never enters git
+ * history at all — committed records reference it by pointer and content hash
+ * only. This path sits under the already-ignored `.foo/`, and two pinned tests
+ * hold the line: nothing under it may be tracked, and the published payload may
+ * never contain it. Path exclusion cannot rest on the secret scanner, which
+ * waves ordinary prose through by design.
+ */
+export const EVAL_CORPUS_HOME = ".foo/eval-corpus";
+// Runner vocabulary that must never appear in a record (V-006). A record
+// states a criterion; the moment it carries a runner's config format, changing
+// how evals are scored becomes a corpus migration. Token boundaries follow the
+// humanSurfaceEval lesson: a substring is not evidence the phrase is present.
+const ENGINE_VOCABULARY = [
+    { pattern: /\{\{\s*[\w.]+\s*\}\}/, label: "template placeholder syntax" },
+    { pattern: /(?<![\w-])llm-rubric(?![\w-])/i, label: "third-party assertion type" },
+    { pattern: /(?<![\w-])(?:model-graded-closedqa|answer-relevance|context-faithfulness)(?![\w-])/i, label: "third-party assertion type" },
+    { pattern: /(?<![\w-])assert(?:ions)?\s*:/i, label: "runner config key" },
+    { pattern: /(?<![\w-])(?:providers|defaultTest|testCases)\s*:/, label: "runner config key" },
+    { pattern: /(?<![\w-])promptfoo(?![\w-])/i, label: "third-party engine name" },
+];
+const RECORD_ID = /^E-\d{3,4}$/;
+function lintEngineVocabulary(record, findings) {
+    const surfaces = [
+        ["pass", record.pass],
+        ["target", record.target],
+        ["judge.prompt", record.judge?.prompt],
+    ];
+    for (const [field, text] of surfaces) {
+        if (!text)
+            continue;
+        for (const { pattern, label } of ENGINE_VOCABULARY) {
+            const match = text.match(pattern);
+            if (match) {
+                findings.push({
+                    rule: "ER-002",
+                    record: record.id ?? "(missing id)",
+                    field,
+                    phrase: match[0],
+                    message: `Engine vocabulary (${label}) belongs only in compiler output; state the check in productfoo's contract terms.`,
+                });
+            }
+        }
+    }
+}
+function validateRecord(raw, findings) {
+    if (typeof raw !== "object" || raw === null) {
+        findings.push({
+            rule: "ER-001",
+            record: "(unparsed)",
+            field: "(root)",
+            phrase: String(raw),
+            message: "An eval record must be a mapping.",
+        });
+        return undefined;
+    }
+    const record = raw;
+    const problems = [];
+    if (record.schema_version !== EVAL_RECORD_SCHEMA_VERSION) {
+        problems.push(["schema_version", `expected ${EVAL_RECORD_SCHEMA_VERSION}`]);
+    }
+    if (typeof record.id !== "string" || !RECORD_ID.test(record.id)) {
+        problems.push(["id", "expected an E-001 style id"]);
+    }
+    if (typeof record.suite !== "string" || record.suite.length === 0) {
+        problems.push(["suite", "required"]);
+    }
+    if (typeof record.target !== "string" || record.target.length === 0) {
+        problems.push(["target", "required"]);
+    }
+    if (record.kind !== "deterministic" && record.kind !== "judge") {
+        problems.push(["kind", "expected deterministic or judge"]);
+    }
+    if (typeof record.pass !== "string" || record.pass.length === 0) {
+        problems.push(["pass", "a binary pass definition is required"]);
+    }
+    if (record.kind === "deterministic" && typeof record.check?.id !== "string") {
+        problems.push(["check", "a deterministic record names its registered check"]);
+    }
+    if (record.kind === "judge") {
+        if (typeof record.judge?.prompt !== "string" || record.judge.prompt.length === 0) {
+            problems.push(["judge.prompt", "required"]);
+        }
+        if (typeof record.judge?.model !== "string" || record.judge.model.length === 0) {
+            problems.push(["judge.model", "a judge pins its model"]);
+        }
+    }
+    const provenance = record.provenance;
+    const provenanceOk = (provenance?.source === "roast-run" && typeof provenance.run === "string") ||
+        (provenance?.source === "observed-failure" && typeof provenance.ref === "string");
+    if (!provenanceOk) {
+        problems.push(["provenance", "every eval names the roast run or observed failure that produced it"]);
+    }
+    if (typeof record.approved !== "boolean") {
+        problems.push(["approved", "required — unapproved records never run"]);
+    }
+    for (const [field, message] of problems) {
+        findings.push({
+            rule: "ER-001",
+            record: typeof record.id === "string" ? record.id : "(missing id)",
+            field,
+            phrase: "",
+            message,
+        });
+    }
+    if (problems.length > 0)
+        return undefined;
+    lintEngineVocabulary(record, findings);
+    return record;
+}
+/** Validate a set of parsed records: shape, unique ids, engine-boundary lint. */
+export function validateEvalRecords(rawRecords) {
+    const findings = [];
+    const records = [];
+    const seen = new Set();
+    for (const raw of rawRecords) {
+        const record = validateRecord(raw, findings);
+        if (!record)
+            continue;
+        if (seen.has(record.id)) {
+            findings.push({
+                rule: "ER-003",
+                record: record.id,
+                field: "id",
+                phrase: record.id,
+                message: "Duplicate eval id — ids are minted once and never reused.",
+            });
+            continue;
+        }
+        seen.add(record.id);
+        records.push(record);
+    }
+    return { ok: findings.length === 0, records, findings };
+}
+/**
+ * Load a packet's eval suite from `specs/<packet>/evals/*.yaml` (R-001: records
+ * live in the target's packet). Missing directory is an empty suite, not an error.
+ */
+export function loadEvalSuite(packetDir) {
+    const home = path.join(packetDir, "evals");
+    if (!existsSync(home))
+        return { ok: true, records: [], findings: [] };
+    const rawRecords = [];
+    for (const entry of readdirSync(home).sort()) {
+        if (!entry.endsWith(".yaml") && !entry.endsWith(".yml"))
+            continue;
+        const parsed = parseYaml(readFileSync(path.join(home, entry), "utf8"));
+        if (Array.isArray(parsed))
+            rawRecords.push(...parsed);
+        else if (parsed != null)
+            rawRecords.push(parsed);
+    }
+    return validateEvalRecords(rawRecords);
+}

package/dist-public/src/evalRoast.d.ts

@@ -0,0 +1,88 @@
+import { type EvalCheckKind, type EvalRecord } from "./evalRecord.js";
+/** A roast run, durable enough that a compiled eval can cite it years later. */
+export type RoastRunRecord = {
+    schema_version: string;
+    /** `roast-YYYY-MM-DD-<slug>`; minted by `foo roast --deep`, never by hand. */
+    run: string;
+    /** Artifact path or named behavior the roast examined. */
+    target: string;
+    at: string;
+    candidates: RoastCandidate[];
+};
+/**
+ * One candidate check as the roast named it. Deliberately three fields: the
+ * roast decides WHAT to judge, never how it is scored (R-001).
+ */
+export type RoastCandidate = {
+    kind: EvalCheckKind;
+    /** Binary pass definition, plain language. This is what the owner approves. */
+    pass: string;
+    /** Optional narrower target; defaults to the run's target. */
+    target?: string;
+    /** Deterministic only: which registered check, and its params. */
+    check?: {
+        id: string;
+        params?: Record<string, unknown>;
+    };
+    /** Judge only: the criterion and the pinned model. */
+    judge?: {
+        prompt: string;
+        model: string;
+    };
+};
+export type RoastCompileFinding = {
+    rule: "RC-001" | "RC-002" | "RC-003" | "RC-004";
+    candidate: number;
+    message: string;
+};
+export type RoastCompileResult = {
+    ok: boolean;
+    run: string;
+    /** Compiled records, always unapproved: approval is a separate human act. */
+    records: EvalRecord[];
+    findings: RoastCompileFinding[];
+};
+/**
+ * Mint a run id from the target and the day. Content-derived rather than
+ * random so the same roast of the same artifact on the same day is the same
+ * run — a re-roast that finds the same check should not fork provenance.
+ */
+export declare function mintRoastRunId(target: string, at?: Date): string;
+/**
+ * Read a roast run record. The candidate block is data the roast emitted, so it
+ * is parsed defensively: a malformed block is a finding, never a throw that
+ * loses the rest of the run.
+ */
+export declare function parseRoastRun(raw: unknown): {
+    run: RoastRunRecord | null;
+    findings: RoastCompileFinding[];
+};
+/**
+ * Compile a roast run's candidates into eval records.
+ *
+ * Every record comes out `approved: false`. That is the whole point of the
+ * separation: compiling is mechanical, approving is judgment, and a suite that
+ * grows without anyone saying yes is the sprawl R-004 is guarding against.
+ */
+export declare function compileRoastRun(run: RoastRunRecord, suite: string, options?: {
+    startAt?: number;
+}): RoastCompileResult;
+/**
+ * The approval surface: each check in the roast's own words, with where it came
+ * from. Rendered for a human to read and answer, never for a machine to parse.
+ */
+export declare function renderApprovalRequest(result: RoastCompileResult, glyphsOn: boolean): string;
+/** Where a packet keeps its eval records. Matches loadEvalSuite's home. */
+export declare function evalHome(packetDir: string): string;
+/**
+ * Write compiled records into the packet, merging by id so a re-compile of the
+ * same roast never silently drops an approval the owner already gave.
+ */
+export declare function writeCompiledSuite(packetDir: string, records: EvalRecord[]): {
+    written: number;
+    kept: number;
+};
+/** Flip one record's approval. Returns null when the id is not in the suite. */
+export declare function setApproval(packetDir: string, id: string, approved: boolean): {
+    record: EvalRecord;
+} | null;

package/dist-public/src/evalRoast.js

@@ -0,0 +1,249 @@
+// foo-eval-system-v0 T-005 (R-004): roast is the authoring front door.
+//
+// A roast already ends by naming what the next generation should be judged
+// against. That naming was prose, so it evaporated: nothing carried a candidate
+// check from the roast that found it to the suite that would run it, and the
+// only way to get an eval was to hand-author one — which R-001 forbids reaching
+// engine configuration, and which nobody does twice.
+//
+// So `foo roast --deep` mints a run id, the roast emits a machine-readable
+// candidate block beside its prose, and this module compiles that block into
+// eval records whose provenance points back at the run. The prose stays the
+// approval and provenance surface: a candidate arrives unapproved, is shown to
+// the owner in the roast's own words with its origin, and only an explicit
+// per-check approval lets it run.
+import { createHash } from "node:crypto";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import path from "node:path";
+import { parse as parseYaml } from "./vendorYaml.js";
+import { EVAL_RECORD_SCHEMA_VERSION, } from "./evalRecord.js";
+import { DETERMINISTIC_CHECK_IDS } from "./evalChecks.js";
+const RUN_ID = /^roast-\d{4}-\d{2}-\d{2}-[a-z0-9][a-z0-9-]*$/;
+/**
+ * Mint a run id from the target and the day. Content-derived rather than
+ * random so the same roast of the same artifact on the same day is the same
+ * run — a re-roast that finds the same check should not fork provenance.
+ */
+export function mintRoastRunId(target, at = new Date()) {
+    const day = at.toISOString().slice(0, 10);
+    const slugSource = path.basename(target).toLowerCase();
+    const slug = slugSource
+        .replace(/\.[a-z0-9]+$/, "")
+        .replace(/[^a-z0-9]+/g, "-")
+        .replace(/^-+|-+$/g, "")
+        .slice(0, 40);
+    const stable = slug || createHash("sha256").update(target).digest("hex").slice(0, 8);
+    return `roast-${day}-${stable}`;
+}
+function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+/**
+ * Read a roast run record. The candidate block is data the roast emitted, so it
+ * is parsed defensively: a malformed block is a finding, never a throw that
+ * loses the rest of the run.
+ */
+export function parseRoastRun(raw) {
+    const findings = [];
+    if (!isRecord(raw)) {
+        findings.push({ rule: "RC-001", candidate: -1, message: "roast run record is not an object" });
+        return { run: null, findings };
+    }
+    const run = typeof raw.run === "string" ? raw.run : "";
+    if (!RUN_ID.test(run)) {
+        findings.push({
+            rule: "RC-001",
+            candidate: -1,
+            message: `roast run id "${run}" is not a minted id (expected roast-YYYY-MM-DD-<slug>; run \`foo roast --deep\` to mint one)`,
+        });
+        return { run: null, findings };
+    }
+    if (typeof raw.target !== "string" || raw.target.trim() === "") {
+        findings.push({ rule: "RC-001", candidate: -1, message: "roast run record names no target" });
+        return { run: null, findings };
+    }
+    const candidates = Array.isArray(raw.candidates) ? raw.candidates : [];
+    return {
+        run: {
+            schema_version: typeof raw.schema_version === "string" ? raw.schema_version : EVAL_RECORD_SCHEMA_VERSION,
+            run,
+            target: raw.target,
+            at: typeof raw.at === "string" ? raw.at : new Date().toISOString(),
+            candidates: candidates,
+        },
+        findings,
+    };
+}
+/**
+ * Compile a roast run's candidates into eval records.
+ *
+ * Every record comes out `approved: false`. That is the whole point of the
+ * separation: compiling is mechanical, approving is judgment, and a suite that
+ * grows without anyone saying yes is the sprawl R-004 is guarding against.
+ */
+export function compileRoastRun(run, suite, options = {}) {
+    const findings = [];
+    const records = [];
+    let next = options.startAt ?? 1;
+    run.candidates.forEach((candidate, index) => {
+        if (!isRecord(candidate)) {
+            findings.push({ rule: "RC-002", candidate: index, message: "candidate is not an object" });
+            return;
+        }
+        if (candidate.kind !== "deterministic" && candidate.kind !== "judge") {
+            findings.push({ rule: "RC-002", candidate: index, message: `candidate kind must be deterministic or judge, got "${String(candidate.kind)}"` });
+            return;
+        }
+        if (typeof candidate.pass !== "string" || candidate.pass.trim() === "") {
+            findings.push({ rule: "RC-002", candidate: index, message: "candidate has no binary pass definition — the roast must say what passing means" });
+            return;
+        }
+        if (candidate.kind === "deterministic") {
+            const id = candidate.check?.id;
+            if (!id) {
+                findings.push({ rule: "RC-003", candidate: index, message: "deterministic candidate names no registered check id" });
+                return;
+            }
+            if (!DETERMINISTIC_CHECK_IDS.includes(id)) {
+                findings.push({
+                    rule: "RC-003",
+                    candidate: index,
+                    message: `"${id}" is not a registered deterministic check (registered: ${DETERMINISTIC_CHECK_IDS.join(", ")})`,
+                });
+                return;
+            }
+        }
+        else {
+            if (!candidate.judge?.prompt || !candidate.judge.model) {
+                findings.push({ rule: "RC-004", candidate: index, message: "judge candidate needs both a criterion and a pinned model" });
+                return;
+            }
+        }
+        records.push({
+            schema_version: EVAL_RECORD_SCHEMA_VERSION,
+            id: `E-${String(next).padStart(3, "0")}`,
+            suite,
+            target: candidate.target ?? run.target,
+            kind: candidate.kind,
+            pass: candidate.pass,
+            ...(candidate.kind === "deterministic" ? { check: candidate.check } : {}),
+            // A compiled judge is uncalibrated by construction: no calibration
+            // reference is minted here, so R-005 keeps its verdicts advisory-only
+            // until a real blind pass certifies it.
+            ...(candidate.kind === "judge" ? { judge: { prompt: candidate.judge.prompt, model: candidate.judge.model } } : {}),
+            provenance: { source: "roast-run", run: run.run },
+            approved: false,
+        });
+        next += 1;
+    });
+    return { ok: findings.length === 0, run: run.run, records, findings };
+}
+/**
+ * The approval surface: each check in the roast's own words, with where it came
+ * from. Rendered for a human to read and answer, never for a machine to parse.
+ */
+export function renderApprovalRequest(result, glyphsOn) {
+    const lines = [];
+    const mark = glyphsOn ? "✋ needs your input" : "needs your input";
+    lines.push(`${mark} — ${result.records.length} check(s) compiled from ${result.run}. Nothing runs until you approve it.`);
+    lines.push("");
+    for (const record of result.records) {
+        lines.push(`  ${record.id}  [${record.kind}]  ${record.target}`);
+        lines.push(`      passes when: ${record.pass}`);
+        lines.push(`      from your roast: ${result.run}`);
+        lines.push("");
+    }
+    if (result.findings.length > 0) {
+        lines.push("Not compiled:");
+        for (const finding of result.findings) {
+            const which = finding.candidate >= 0 ? `candidate ${finding.candidate + 1}` : "the run record";
+            lines.push(`  ${which}: ${finding.message}`);
+        }
+        lines.push("");
+    }
+    lines.push("Approve one:  foo eval approve <packet-dir> --check E-001 --by <who>");
+    lines.push("Reject one:   foo eval reject <packet-dir> --check E-001 --by <who> --reason <why>");
+    return lines.join("\n");
+}
+const SUITE_FILE = "compiled.yaml";
+/** Where a packet keeps its eval records. Matches loadEvalSuite's home. */
+export function evalHome(packetDir) {
+    return path.join(packetDir, "evals");
+}
+/**
+ * Write compiled records into the packet, merging by id so a re-compile of the
+ * same roast never silently drops an approval the owner already gave.
+ */
+export function writeCompiledSuite(packetDir, records) {
+    const home = evalHome(packetDir);
+    mkdirSync(home, { recursive: true });
+    const file = path.join(home, SUITE_FILE);
+    const existing = existsSync(file)
+        ? (parseYaml(readFileSync(file, "utf8")) ?? [])
+        : [];
+    const byId = new Map(existing.map((record) => [record.id, record]));
+    let written = 0;
+    let kept = 0;
+    for (const record of records) {
+        const prior = byId.get(record.id);
+        if (prior) {
+            // An approval is a human act; re-compiling must never revoke or re-grant it.
+            byId.set(record.id, { ...record, approved: prior.approved });
+            kept += 1;
+            continue;
+        }
+        byId.set(record.id, record);
+        written += 1;
+    }
+    const merged = [...byId.values()].sort((left, right) => left.id.localeCompare(right.id));
+    writeFileSync(file, `${merged.map((record) => yamlRecord(record)).join("\n")}\n`);
+    return { written, kept };
+}
+/** Minimal, stable YAML emission — no dependency on a writer's formatting. */
+function yamlRecord(record) {
+    const lines = [
+        `- schema_version: "${record.schema_version}"`,
+        `  id: "${record.id}"`,
+        `  suite: "${record.suite}"`,
+        `  target: ${JSON.stringify(record.target)}`,
+        `  kind: "${record.kind}"`,
+        `  pass: ${JSON.stringify(record.pass)}`,
+    ];
+    if (record.check) {
+        lines.push(`  check:`);
+        lines.push(`    id: "${record.check.id}"`);
+        if (record.check.params)
+            lines.push(`    params: ${JSON.stringify(record.check.params)}`);
+    }
+    if (record.judge) {
+        lines.push(`  judge:`);
+        lines.push(`    prompt: ${JSON.stringify(record.judge.prompt)}`);
+        lines.push(`    model: "${record.judge.model}"`);
+        if (record.judge.calibration)
+            lines.push(`    calibration: "${record.judge.calibration}"`);
+    }
+    lines.push(`  provenance:`);
+    if (record.provenance.source === "roast-run") {
+        lines.push(`    source: "roast-run"`);
+        lines.push(`    run: "${record.provenance.run}"`);
+    }
+    else {
+        lines.push(`    source: "observed-failure"`);
+        lines.push(`    ref: ${JSON.stringify(record.provenance.ref)}`);
+    }
+    lines.push(`  approved: ${record.approved}`);
+    return lines.join("\n");
+}
+/** Flip one record's approval. Returns null when the id is not in the suite. */
+export function setApproval(packetDir, id, approved) {
+    const file = path.join(evalHome(packetDir), SUITE_FILE);
+    if (!existsSync(file))
+        return null;
+    const records = (parseYaml(readFileSync(file, "utf8")) ?? []);
+    const target = records.find((record) => record.id === id);
+    if (!target)
+        return null;
+    target.approved = approved;
+    writeFileSync(file, `${records.map((record) => yamlRecord(record)).join("\n")}\n`);
+    return { record: target };
+}

package/dist-public/src/evidenceIntake.d.ts

@@ -0,0 +1,66 @@
+import { type SourceId } from "./sourceIngress.js";
+export interface SweepSource {
+    source_id: SourceId;
+    source_type: "github_pr" | "linear_issue";
+    url: string;
+    title?: string;
+    /** review comments / issue thread bodies, in order */
+    comments: string[];
+}
+export type SuggestedRoute = "guidance" | "lint" | "example" | "eval" | "no-change";
+export interface Candidate {
+    id: string;
+    summary: string;
+    source_url: string;
+    source_type: SweepSource["source_type"];
+    suggested_route: SuggestedRoute;
+    rationale: string;
+}
+export declare function suggestRoute(text: string): {
+    route: SuggestedRoute;
+    rationale: string;
+};
+export declare function sweepCandidates(governingPacketDir: string, sources: SweepSource[]): {
+    candidates: Candidate[];
+    refused: Array<{
+        source_id: SourceId;
+        reason: string;
+    }>;
+};
+export declare function emitPendingPacket(root: string, name: string, candidates: Candidate[], at: string): string;
+export type TriageBucket = "now" | "next" | "later";
+export declare function suggestTriage(text: string): {
+    bucket: TriageBucket;
+    rationale: string;
+};
+export interface ReplayReport {
+    schema_version: "0.1";
+    packet: string;
+    since: string;
+    events: number;
+    by_type: Record<string, number>;
+    decisions: Array<{
+        at: string;
+        by: string;
+        summary: string;
+    }>;
+    proofs: Array<{
+        at: string;
+        task: string;
+    }>;
+}
+export declare function replayWindow(packetDir: string, since: string): ReplayReport;
+export type Disposition = "accept" | "override" | "reject";
+export declare function recordDisposition(packetDir: string, candidateId: string, opts: {
+    action: Disposition;
+    route?: SuggestedRoute;
+    by: string;
+    at?: string;
+}): {
+    action: Disposition;
+    route?: SuggestedRoute;
+};
+export declare function runEvidenceCommand(root: string, args: string[]): {
+    output: string;
+    exitCode: number;
+};