stilltrue 0.1.0 → 0.3.0

package/README.md

@@ -11,7 +11,7 @@ no server, no dashboard, no SaaS.
 |---|---|---|
 | `npx stilltrue drift` | Curated facts vs their live authoritative sources — fails the build on rot | **v0.1 focus** |
 | `npx stilltrue golden` | Deterministic regression evals for answers | later (may wrap promptfoo) |
-| `verify` (library) | Judge answers against their sources in the request path | later (P9-shaped staged pipeline) |
+| `verify` (library) | Staged acceptance pipeline for AI output — judge answers against sources in the request path | **shipped 0.3.0** |
 
 **Why drift first:** "drift detection" in the market means statistical ML
 feature drift; website-change monitors alert humans, not CI. Nobody ships
@@ -69,10 +69,37 @@ jobs:
 deterministic either way). `expect` is your curated file. Rot fails the run
 with one message per vanished fact; unreachable sources only warn.
 
+## Verify — staged acceptance for AI output
+
+An ordered, short-circuiting pipeline: each stage is a pluggable judge — a
+Zod parse, a deterministic validator, or an LLM. Rejection is structured
+(`{ stage, messages[] }`, one issue per entry) and designed to be fed back to
+the generator for a retry. Stages that *throw* are infrastructure failures,
+not verdicts: `failPolicy: 'open'` (default) warns and continues — a judge
+outage must never block your app.
+
+```ts
+import { runVerify, zodStage, generateVerified, formatRejection } from 'stilltrue';
+
+const stages = [
+  zodStage('parse_schema', AnswerSchema),
+  { name: 'grounding', timeoutMs: 10_000, check: llmJudge },     // may return a revision
+  { name: 'chase-sim', tier: 'ci', check: expensiveSimulation }, // skipped inline
+];
+
+// Request path:
+const result = await runVerify(stages, draft);
+publish(result.value); // draft, parsed draft, or the judge's rewrite
+
+// Generator loop (one automatic retry, then fall back — don't loop forever):
+const gen = await generateVerified({ generate: (feedback) => ask(model, prompt, feedback && formatRejection(feedback)), stages });
+```
+
 ## Status
 
-v0.1 — `drift` is real and running in production (AskGwinnett). `golden` and
-`verify` are designed but not yet shipped; see the brief.
+v0.3 — `drift` and `report` running in production CI (AskGwinnett); `verify`
+shipped as a library. `golden` remains open: wrap promptfoo or defer to it
+(see the brief's competitive research).
 
 ## License
 

package/dist/cli.js

@@ -6,9 +6,12 @@
  * outage must not block CI), 1 = rot detected or usage error.
  */
 import { writeFile } from 'node:fs/promises';
+import { resolve } from 'node:path';
 import { parseArgs } from 'node:util';
 import { findConfigPath, loadConfig } from "./loadConfig.js";
 import { runDrift } from "./runner.js";
+import { DEFAULT_HISTORY_PATH, loadHistory, recordRun } from "./history.js";
+import { renderReport } from "./report.js";
 const GH = !!process.env.GITHUB_ACTIONS;
 function printResult(r) {
     const icon = { pass: '✓', rot: '✗', error: '⚠' }[r.outcome];
@@ -31,6 +34,8 @@ async function drift(argv) {
             config: { type: 'string' },
             only: { type: 'string' },
             json: { type: 'string' },
+            record: { type: 'boolean' },
+            history: { type: 'string' },
         },
     });
     const configPath = findConfigPath(process.cwd(), values.config);
@@ -53,21 +58,49 @@ async function drift(argv) {
         await writeFile(values.json, JSON.stringify(results, null, 2));
         console.log(`results written to ${values.json}`);
     }
+    if (values.record) {
+        const historyPath = resolve(values.history ?? DEFAULT_HISTORY_PATH);
+        await recordRun(results, historyPath);
+        console.log(`run recorded to ${historyPath}`);
+    }
     return rotted > 0 ? 1 : 0;
 }
+async function report(argv) {
+    const { values } = parseArgs({
+        args: argv,
+        options: {
+            history: { type: 'string' },
+            out: { type: 'string' },
+            title: { type: 'string' },
+        },
+    });
+    const historyPath = resolve(values.history ?? DEFAULT_HISTORY_PATH);
+    const history = await loadHistory(historyPath);
+    if (!history.length) {
+        console.error(`no recorded runs at ${historyPath} — run \`stilltrue drift --record\` first (use --history to point elsewhere)`);
+        return 1;
+    }
+    const out = resolve(values.out ?? 'stilltrue-report.html');
+    await writeFile(out, renderReport(history, values.title));
+    console.log(`report written to ${out} (${history.length} runs)`);
+    return 0;
+}
 const [command, ...rest] = process.argv.slice(2);
 try {
     switch (command) {
         case 'drift':
             process.exit(await drift(rest));
             break;
-        case 'golden':
         case 'report':
-            console.error(`stilltrue ${command}: not implemented yet — drift shipped first (docs/BRIEF.md)`);
+            process.exit(await report(rest));
+            break;
+        case 'golden':
+            console.error('stilltrue golden: not implemented yet — drift shipped first (docs/BRIEF.md)');
             process.exit(1);
             break;
         default:
-            console.log('usage: stilltrue drift [--config <path>] [--only <names>] [--json <path>]');
+            console.log('usage: stilltrue drift [--config <path>] [--only <names>] [--json <path>] [--record] [--history <path>]\n' +
+                '       stilltrue report [--history <path>] [--out <path>] [--title <text>]');
             process.exit(command ? 1 : 0);
     }
 }

package/dist/config.d.ts

@@ -56,9 +56,10 @@ export interface DriftCheck<E = unknown, A = unknown> {
 }
 export interface StilltrueConfig {
     drift?: DriftCheck<any, any>[];
-    /** golden and verify are later phases — see docs/BRIEF.md build plan. */
+    /** Named verify pipelines (see src/verify.ts) — usable from library code. */
+    verify?: Record<string, import('./verify.ts').VerifyStage<any>[]>;
+    /** golden is a later phase — see docs/BRIEF.md build plan. */
     golden?: unknown;
-    verify?: unknown;
 }
 /** Identity function for typed configs: `export default defineStilltrue({...})`. */
 export declare function defineStilltrue(config: StilltrueConfig): StilltrueConfig;

package/dist/history.d.ts

@@ -0,0 +1,11 @@
+import type { DriftResult } from './config.ts';
+/** One recorded `stilltrue drift` run. */
+export interface RunRecord {
+    at: string;
+    results: DriftResult[];
+}
+export declare const DEFAULT_HISTORY_PATH = ".stilltrue/history.jsonl";
+/** Append a run to the JSONL history file (created on first record). */
+export declare function recordRun(results: DriftResult[], historyPath: string): Promise<RunRecord>;
+/** Load history oldest-first. Missing file → []. Corrupt lines are skipped. */
+export declare function loadHistory(historyPath: string): Promise<RunRecord[]>;

package/dist/history.js

@@ -0,0 +1,37 @@
+import { mkdir, readFile, appendFile } from 'node:fs/promises';
+import { dirname } from 'node:path';
+export const DEFAULT_HISTORY_PATH = '.stilltrue/history.jsonl';
+/** Append a run to the JSONL history file (created on first record). */
+export async function recordRun(results, historyPath) {
+    const record = { at: new Date().toISOString(), results };
+    await mkdir(dirname(historyPath), { recursive: true });
+    await appendFile(historyPath, JSON.stringify(record) + '\n');
+    return record;
+}
+/** Load history oldest-first. Missing file → []. Corrupt lines are skipped. */
+export async function loadHistory(historyPath) {
+    let raw;
+    try {
+        raw = await readFile(historyPath, 'utf8');
+    }
+    catch (err) {
+        if (err.code === 'ENOENT')
+            return [];
+        throw err;
+    }
+    const records = [];
+    for (const line of raw.split('\n')) {
+        if (!line.trim())
+            continue;
+        try {
+            const parsed = JSON.parse(line);
+            if (parsed && typeof parsed.at === 'string' && Array.isArray(parsed.results)) {
+                records.push(parsed);
+            }
+        }
+        catch {
+            // A truncated line (interrupted CI write) shouldn't sink the report.
+        }
+    }
+    return records;
+}

package/dist/index.d.ts

@@ -2,3 +2,4 @@ export { defineStilltrue, type StilltrueConfig, type DriftCheck, type DriftResul
 export { corpus, json, surname, htmlToText, type CorpusOptions } from './helpers.ts';
 export { runDrift, runDriftCheck, type RunDriftOptions } from './runner.ts';
 export { resolveCompare } from './compare.ts';
+export { runVerify, zodStage, formatRejection, generateVerified, type VerifyStage, type VerifyResult, type StageOutcome, type StageContext, type RunVerifyOptions, type GenerateVerifiedOptions, } from './verify.ts';

package/dist/index.js

@@ -2,3 +2,4 @@ export { defineStilltrue, } from "./config.js";
 export { corpus, json, surname, htmlToText } from "./helpers.js";
 export { runDrift, runDriftCheck } from "./runner.js";
 export { resolveCompare } from "./compare.js";
+export { runVerify, zodStage, formatRejection, generateVerified, } from "./verify.js";

package/dist/report.d.ts

@@ -0,0 +1,2 @@
+import type { RunRecord } from './history.ts';
+export declare function renderReport(history: RunRecord[], title?: string): string;

package/dist/report.js

@@ -0,0 +1,120 @@
+const STATUS = {
+    pass: { color: '#0ca30c', glyph: '✓', ink: '#ffffff', label: 'pass' },
+    rot: { color: '#d03b3b', glyph: '✗', ink: '#ffffff', label: 'rot' },
+    error: { color: '#fab219', glyph: '⚠', ink: '#1a1a19', label: 'error' },
+};
+const MAX_COLUMNS = 60;
+const MAX_LOG = 50;
+function esc(s) {
+    return s.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/"/g, '&quot;');
+}
+function day(iso) {
+    return iso.slice(5, 10); // MM-DD
+}
+export function renderReport(history, title = 'stilltrue drift report') {
+    const runs = history.slice(-MAX_COLUMNS);
+    const latest = runs[runs.length - 1];
+    // Stable row order: latest run's order first, then any check seen earlier.
+    const names = [];
+    for (const record of [...runs].reverse()) {
+        for (const r of record.results)
+            if (!names.includes(r.name))
+                names.push(r.name);
+    }
+    const counts = { pass: 0, rot: 0, error: 0 };
+    for (const r of latest?.results ?? [])
+        counts[r.outcome]++;
+    const tiles = ['pass', 'rot', 'error']
+        .map((o) => `<div class="tile">
+        <div class="tile-num" style="color:${counts[o] && o !== 'pass' ? STATUS[o].color : 'var(--ink)'}">${counts[o]}</div>
+        <div class="tile-label">${STATUS[o].glyph} ${STATUS[o].label}</div>
+      </div>`)
+        .join('');
+    const headerCells = runs
+        .map((rec, i) => {
+        const sparse = runs.length <= 10 || i % Math.ceil(runs.length / 10) === 0 || i === runs.length - 1;
+        return `<th title="${esc(rec.at)}">${sparse ? day(rec.at) : ''}</th>`;
+    })
+        .join('');
+    const rows = names
+        .map((name) => {
+        const cells = runs
+            .map((rec) => {
+            const r = rec.results.find((x) => x.name === name);
+            if (!r)
+                return '<td><span class="cell none" title="not run"></span></td>';
+            const s = STATUS[r.outcome];
+            const detail = [r.outcome, ...r.messages, ...r.warnings.map((w) => `warning: ${w}`)].join('\n');
+            return `<td><span class="cell" style="background:${s.color};color:${s.ink}" title="${esc(rec.at)}\n${esc(detail)}">${s.glyph}</span></td>`;
+        })
+            .join('');
+        return `<tr><th class="check-name">${esc(name)}</th>${cells}</tr>`;
+    })
+        .join('');
+    const events = [...runs]
+        .reverse()
+        .flatMap((rec) => rec.results
+        .filter((r) => r.outcome !== 'pass' || r.warnings.length)
+        .map((r) => ({ at: rec.at, ...r })))
+        .slice(0, MAX_LOG);
+    const log = events.length
+        ? events
+            .map((e) => {
+            const s = STATUS[e.outcome];
+            const items = [...e.messages, ...e.warnings.map((w) => `warning: ${w}`)]
+                .map((m) => `<li>${esc(m)}</li>`)
+                .join('');
+            return `<details ${e.outcome === 'rot' ? 'open' : ''}>
+            <summary><span class="badge" style="background:${s.color};color:${s.ink}">${s.glyph} ${s.label}</span>
+            <strong>${esc(e.name)}</strong> <span class="muted">${esc(e.at)}</span></summary>
+            <ul>${items}</ul>
+          </details>`;
+        })
+            .join('')
+        : '<p class="muted">Nothing has rotted, errored, or warned in recorded history.</p>';
+    return `<!doctype html>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>${esc(title)}</title>
+<style>
+  :root { --bg:#ffffff; --ink:#1a1a19; --ink-2:#5f5e5a; --line:#e4e2dd; --surface:#f6f5f2; }
+  @media (prefers-color-scheme: dark) {
+    :root { --bg:#1a1a19; --ink:#f2f1ee; --ink-2:#a5a39d; --line:#3a3936; --surface:#242422; }
+  }
+  * { box-sizing: border-box; }
+  body { margin:0; padding:2rem 1.5rem; background:var(--bg); color:var(--ink);
+         font:15px/1.5 ui-sans-serif, system-ui, sans-serif; max-width:64rem; margin-inline:auto; }
+  h1 { font-size:1.3rem; margin:0 0 .25rem; }
+  h2 { font-size:1rem; margin:2rem 0 .75rem; }
+  .muted { color:var(--ink-2); font-size:.85rem; }
+  .tiles { display:flex; gap:.75rem; margin:1.25rem 0; flex-wrap:wrap; }
+  .tile { background:var(--surface); border:1px solid var(--line); border-radius:8px;
+          padding:.75rem 1.25rem; min-width:6.5rem; }
+  .tile-num { font-size:1.6rem; font-weight:600; font-variant-numeric:tabular-nums; }
+  .tile-label { color:var(--ink-2); font-size:.85rem; }
+  .grid-wrap { overflow-x:auto; border:1px solid var(--line); border-radius:8px; padding:.75rem; }
+  table { border-collapse:separate; border-spacing:2px; }
+  th { font-weight:500; color:var(--ink-2); font-size:.7rem; text-align:left; white-space:nowrap; }
+  thead th { vertical-align:bottom; padding-bottom:.25rem; }
+  .check-name { font-size:.85rem; color:var(--ink); padding-right:.75rem; position:sticky; left:0; background:var(--bg); }
+  .cell { display:inline-flex; width:20px; height:20px; border-radius:4px;
+          align-items:center; justify-content:center; font-size:12px; }
+  .cell.none { background:var(--surface); border:1px dashed var(--line); }
+  .badge { display:inline-block; border-radius:4px; padding:0 .45rem; font-size:.8rem; margin-right:.5rem; }
+  details { border-left:2px solid var(--line); padding:.35rem .75rem; margin:.4rem 0; }
+  summary { cursor:pointer; }
+  details ul { margin:.4rem 0 .2rem; padding-left:1.2rem; }
+  details li { font-size:.9rem; }
+</style>
+<h1>${esc(title)}</h1>
+<p class="muted">${runs.length} recorded run${runs.length === 1 ? '' : 's'}${latest ? `, latest ${esc(latest.at)}` : ''} — pass / rot (facts changed) / error (source unreachable)</p>
+<div class="tiles">${tiles}</div>
+<h2>Timeline</h2>
+<div class="grid-wrap"><table>
+  <thead><tr><th></th>${headerCells}</tr></thead>
+  <tbody>${rows}</tbody>
+</table></div>
+<h2>Rot, errors &amp; warnings</h2>
+${log}
+`;
+}

package/dist/verify.d.ts

@@ -0,0 +1,100 @@
+/**
+ * `verify` — an ordered, short-circuiting acceptance pipeline for AI (or any
+ * generator) output. Modeled on the P9 level-rejection contract, generalized
+ * with the review.js production doctrine:
+ *
+ * - Stages run in a fixed order and stop at the first rejection; later
+ *   stages may assume earlier ones passed. (P9)
+ * - Rejection is structured and machine-readable — `{ stage, messages[] }`,
+ *   one issue per entry — designed to be fed back to the generator. (P9)
+ * - The judge is pluggable: a Zod parse, a deterministic validator, or an
+ *   LLM. The pipeline neither knows nor cares. (P9 + review.js)
+ * - A stage that THROWS is an infrastructure failure, not a verdict.
+ *   `failPolicy: 'open'` (default) warns and continues — a judge outage must
+ *   never block the app; `'closed'` treats it as rejection. (review.js)
+ * - A rejecting stage may supply a `revision` — a corrected value ready to
+ *   use (review.js's judge rewrite). The pipeline stops there; the result
+ *   carries the revision and is marked `revised`.
+ * - Tiers: `inline` stages are cheap and run everywhere; `ci` stages are
+ *   expensive (simulations, live fetches) and run only when asked. (P9)
+ */
+import type { ZodType } from 'zod';
+export type StageOutcome<T> = 
+/** Pass. `value` optionally replaces the input for later stages (e.g. a parse). */
+{
+    ok: true;
+    value?: T;
+}
+/** Reject: one issue per message. `revision` is an optional corrected value. */
+ | {
+    ok: false;
+    messages: string[];
+    revision?: T;
+};
+export interface StageContext {
+    warn: (message: string) => void;
+    /** Which tier this run was invoked with. */
+    tier: 'inline' | 'ci';
+}
+export interface VerifyStage<T = unknown> {
+    name: string;
+    check: (input: T, ctx: StageContext) => StageOutcome<T> | Promise<StageOutcome<T>>;
+    /**
+     * What a THROWN stage means. 'open' (default): infrastructure failure —
+     * warn and continue with the input unchanged. 'closed': reject.
+     */
+    failPolicy?: 'open' | 'closed';
+    /** Abort the stage after this long; the timeout follows failPolicy. */
+    timeoutMs?: number;
+    /** 'inline' (default) runs always; 'ci' runs only with `{ tier: 'ci' }`. */
+    tier?: 'inline' | 'ci';
+}
+export interface VerifyResult<T> {
+    ok: boolean;
+    /** First failing stage; absent on success. */
+    stage?: string;
+    /** The failing stage's messages only — not cumulative (P9 §2). */
+    messages: string[];
+    /** Final value: the (possibly transformed) input, or the revision. */
+    value: T;
+    /** True when a rejecting stage supplied a corrected value. */
+    revised: boolean;
+    /** Fail-open infrastructure failures and stage warnings. */
+    warnings: string[];
+    stagesRun: string[];
+}
+export interface RunVerifyOptions {
+    /** 'inline' (default) skips `tier: 'ci'` stages; 'ci' runs everything. */
+    tier?: 'inline' | 'ci';
+}
+export declare function runVerify<T>(stages: VerifyStage<T>[], input: T, options?: RunVerifyOptions): Promise<VerifyResult<T>>;
+/**
+ * Stage helper: a P9-style `parse_schema` gate from a Zod schema. On success
+ * the parsed (typed, defaulted, stripped) value replaces the input for later
+ * stages; on failure, one message per Zod issue with its path.
+ */
+export declare function zodStage<T>(name: string, schema: ZodType<T>): VerifyStage<any>;
+/**
+ * Format a rejection as generator feedback — stage id + messages verbatim as
+ * bullets (P9 §3: preserve strings so distances and ids stay fixable).
+ */
+export declare function formatRejection(result: VerifyResult<any>): string;
+export interface GenerateVerifiedOptions<T> {
+    /** Produce a candidate. On retries, `feedback` is the previous rejection. */
+    generate: (feedback?: VerifyResult<T>) => T | Promise<T>;
+    stages: VerifyStage<T>[];
+    /**
+     * Total attempts, default 2 (P9 §3: one automatic retry; more wastes
+     * tokens — after the cap, fall back or ask a human, don't loop).
+     */
+    attempts?: number;
+    tier?: 'inline' | 'ci';
+}
+/**
+ * The generate → verify → feed-back-and-retry loop as a first-class helper.
+ * Returns the first passing result, or the LAST rejection after the cap —
+ * never silently discards errors (P9 §3.4).
+ */
+export declare function generateVerified<T>(options: GenerateVerifiedOptions<T>): Promise<VerifyResult<T> & {
+    attempts: number;
+}>;

package/dist/verify.js

@@ -0,0 +1,96 @@
+async function runStage(stage, input, ctx) {
+    const run = Promise.resolve(stage.check(input, ctx));
+    if (!stage.timeoutMs)
+        return run;
+    let timer;
+    const timeout = new Promise((_, reject) => {
+        timer = setTimeout(() => reject(new Error(`stage timeout after ${stage.timeoutMs}ms`)), stage.timeoutMs);
+    });
+    try {
+        return await Promise.race([run, timeout]);
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
+export async function runVerify(stages, input, options = {}) {
+    const tier = options.tier ?? 'inline';
+    const warnings = [];
+    const stagesRun = [];
+    const ctx = { warn: (m) => warnings.push(m), tier };
+    let value = input;
+    for (const stage of stages) {
+        if ((stage.tier ?? 'inline') === 'ci' && tier !== 'ci')
+            continue;
+        stagesRun.push(stage.name);
+        let outcome;
+        try {
+            outcome = await runStage(stage, value, ctx);
+        }
+        catch (err) {
+            const message = `${stage.name}: ${err instanceof Error ? err.message : String(err)}`;
+            if ((stage.failPolicy ?? 'open') === 'open') {
+                warnings.push(`fail-open: ${message}`);
+                continue;
+            }
+            return { ok: false, stage: stage.name, messages: [message], value, revised: false, warnings, stagesRun };
+        }
+        if (!outcome.ok) {
+            const revised = outcome.revision !== undefined;
+            return {
+                ok: false,
+                stage: stage.name,
+                messages: outcome.messages,
+                value: revised ? outcome.revision : value,
+                revised,
+                warnings,
+                stagesRun,
+            };
+        }
+        if (outcome.value !== undefined)
+            value = outcome.value;
+    }
+    return { ok: true, messages: [], value, revised: false, warnings, stagesRun };
+}
+/**
+ * Stage helper: a P9-style `parse_schema` gate from a Zod schema. On success
+ * the parsed (typed, defaulted, stripped) value replaces the input for later
+ * stages; on failure, one message per Zod issue with its path.
+ */
+export function zodStage(name, schema) {
+    return {
+        name,
+        check: (input) => {
+            const parsed = schema.safeParse(input);
+            if (parsed.success)
+                return { ok: true, value: parsed.data };
+            return {
+                ok: false,
+                messages: parsed.error.issues.map((i) => `${i.path.length ? i.path.join('.') : '(root)'}: ${i.message}`),
+            };
+        },
+    };
+}
+/**
+ * Format a rejection as generator feedback — stage id + messages verbatim as
+ * bullets (P9 §3: preserve strings so distances and ids stay fixable).
+ */
+export function formatRejection(result) {
+    return `Rejected at stage "${result.stage}":\n${result.messages.map((m) => `- ${m}`).join('\n')}`;
+}
+/**
+ * The generate → verify → feed-back-and-retry loop as a first-class helper.
+ * Returns the first passing result, or the LAST rejection after the cap —
+ * never silently discards errors (P9 §3.4).
+ */
+export async function generateVerified(options) {
+    const attempts = Math.max(1, options.attempts ?? 2);
+    let last;
+    for (let i = 0; i < attempts; i++) {
+        const candidate = await options.generate(last);
+        last = await runVerify(options.stages, candidate, { tier: options.tier });
+        if (last.ok)
+            return { ...last, attempts: i + 1 };
+    }
+    return { ...last, attempts };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "stilltrue",
-  "version": "0.1.0",
+  "version": "0.3.0",
   "description": "Your tests check your code. stilltrue checks your facts. CI checks for AI-in-the-loop apps: drift (curated facts vs live sources), golden (regression evals), verify (judge answers against sources).",
   "keywords": [
     "drift",