stilltrue 0.2.0 → 0.4.0

package/README.md

@@ -9,9 +9,14 @@ no server, no dashboard, no SaaS.
 
 | Command | Watchdog | Status |
 |---|---|---|
-| `npx stilltrue drift` | Curated facts vs their live authoritative sources — fails the build on rot | **v0.1 focus** |
-| `npx stilltrue golden` | Deterministic regression evals for answers | later (may wrap promptfoo) |
-| `verify` (library) | Judge answers against their sources in the request path | later (P9-shaped staged pipeline) |
+| `npx stilltrue drift` | Curated facts vs their live authoritative sources — fails the build on rot | shipped |
+| `npx stilltrue report` | Self-contained HTML drift timeline from recorded runs | shipped |
+| `verify` (library) | Staged acceptance pipeline for AI output — judge answers against sources in the request path | shipped |
+
+For golden-style regression evals (did a code change break an answer?), use
+[promptfoo](https://promptfoo.dev) — **promptfoo tests your prompts;
+stilltrue tests your facts.** Our eval doctrine and how the tools pair:
+[docs/GOLDEN.md](docs/GOLDEN.md).
 
 **Why drift first:** "drift detection" in the market means statistical ML
 feature drift; website-change monitors alert humans, not CI. Nobody ships
@@ -69,10 +74,38 @@ jobs:
 deterministic either way). `expect` is your curated file. Rot fails the run
 with one message per vanished fact; unreachable sources only warn.
 
+## Verify — staged acceptance for AI output
+
+An ordered, short-circuiting pipeline: each stage is a pluggable judge — a
+Zod parse, a deterministic validator, or an LLM. Rejection is structured
+(`{ stage, messages[] }`, one issue per entry) and designed to be fed back to
+the generator for a retry. Stages that *throw* are infrastructure failures,
+not verdicts: `failPolicy: 'open'` (default) warns and continues — a judge
+outage must never block your app.
+
+```ts
+import { runVerify, zodStage, generateVerified, formatRejection } from 'stilltrue';
+
+const stages = [
+  zodStage('parse_schema', AnswerSchema),
+  { name: 'grounding', timeoutMs: 10_000, check: llmJudge },     // may return a revision
+  { name: 'chase-sim', tier: 'ci', check: expensiveSimulation }, // skipped inline
+];
+
+// Request path:
+const result = await runVerify(stages, draft);
+publish(result.value); // draft, parsed draft, or the judge's rewrite
+
+// Generator loop (one automatic retry, then fall back — don't loop forever):
+const gen = await generateVerified({ generate: (feedback) => ask(model, prompt, feedback && formatRejection(feedback)), stages });
+```
+
 ## Status
 
-v0.1 — `drift` is real and running in production (AskGwinnett). `golden` and
-`verify` are designed but not yet shipped; see the brief.
+v0.4 — feature-complete for the 1.0 shape: `drift` and `report` run in
+production CI (AskGwinnett), `verify` runs in the production request path
+(AskGwinnett's pre-publish check sweep). Golden is deliberately out of
+scope — settled in [docs/GOLDEN.md](docs/GOLDEN.md).
 
 ## License
 

package/dist/cli.js

@@ -95,7 +95,8 @@ try {
             process.exit(await report(rest));
             break;
         case 'golden':
-            console.error('stilltrue golden: not implemented yet — drift shipped first (docs/BRIEF.md)');
+            console.error('stilltrue deliberately has no golden runner — promptfoo tests your prompts; stilltrue tests your facts.\n' +
+                'Use https://promptfoo.dev for prompt/answer regression evals. Doctrine + pairing guide: docs/GOLDEN.md');
             process.exit(1);
             break;
         default:

package/dist/config.d.ts

@@ -56,9 +56,8 @@ export interface DriftCheck<E = unknown, A = unknown> {
 }
 export interface StilltrueConfig {
     drift?: DriftCheck<any, any>[];
-    /** golden and verify are later phases — see docs/BRIEF.md build plan. */
-    golden?: unknown;
-    verify?: unknown;
+    /** Named verify pipelines (see src/verify.ts) — usable from library code. */
+    verify?: Record<string, import('./verify.ts').VerifyStage<any>[]>;
 }
 /** Identity function for typed configs: `export default defineStilltrue({...})`. */
 export declare function defineStilltrue(config: StilltrueConfig): StilltrueConfig;

package/dist/index.d.ts

@@ -2,3 +2,4 @@ export { defineStilltrue, type StilltrueConfig, type DriftCheck, type DriftResul
 export { corpus, json, surname, htmlToText, type CorpusOptions } from './helpers.ts';
 export { runDrift, runDriftCheck, type RunDriftOptions } from './runner.ts';
 export { resolveCompare } from './compare.ts';
+export { runVerify, zodStage, formatRejection, generateVerified, type VerifyStage, type VerifyResult, type StageOutcome, type StageContext, type RunVerifyOptions, type GenerateVerifiedOptions, } from './verify.ts';

package/dist/index.js

@@ -2,3 +2,4 @@ export { defineStilltrue, } from "./config.js";
 export { corpus, json, surname, htmlToText } from "./helpers.js";
 export { runDrift, runDriftCheck } from "./runner.js";
 export { resolveCompare } from "./compare.js";
+export { runVerify, zodStage, formatRejection, generateVerified, } from "./verify.js";

package/dist/verify.d.ts

@@ -0,0 +1,100 @@
+/**
+ * `verify` — an ordered, short-circuiting acceptance pipeline for AI (or any
+ * generator) output. Modeled on the P9 level-rejection contract, generalized
+ * with the review.js production doctrine:
+ *
+ * - Stages run in a fixed order and stop at the first rejection; later
+ *   stages may assume earlier ones passed. (P9)
+ * - Rejection is structured and machine-readable — `{ stage, messages[] }`,
+ *   one issue per entry — designed to be fed back to the generator. (P9)
+ * - The judge is pluggable: a Zod parse, a deterministic validator, or an
+ *   LLM. The pipeline neither knows nor cares. (P9 + review.js)
+ * - A stage that THROWS is an infrastructure failure, not a verdict.
+ *   `failPolicy: 'open'` (default) warns and continues — a judge outage must
+ *   never block the app; `'closed'` treats it as rejection. (review.js)
+ * - A rejecting stage may supply a `revision` — a corrected value ready to
+ *   use (review.js's judge rewrite). The pipeline stops there; the result
+ *   carries the revision and is marked `revised`.
+ * - Tiers: `inline` stages are cheap and run everywhere; `ci` stages are
+ *   expensive (simulations, live fetches) and run only when asked. (P9)
+ */
+import type { ZodType } from 'zod';
+export type StageOutcome<T> = 
+/** Pass. `value` optionally replaces the input for later stages (e.g. a parse). */
+{
+    ok: true;
+    value?: T;
+}
+/** Reject: one issue per message. `revision` is an optional corrected value. */
+ | {
+    ok: false;
+    messages: string[];
+    revision?: T;
+};
+export interface StageContext {
+    warn: (message: string) => void;
+    /** Which tier this run was invoked with. */
+    tier: 'inline' | 'ci';
+}
+export interface VerifyStage<T = unknown> {
+    name: string;
+    check: (input: T, ctx: StageContext) => StageOutcome<T> | Promise<StageOutcome<T>>;
+    /**
+     * What a THROWN stage means. 'open' (default): infrastructure failure —
+     * warn and continue with the input unchanged. 'closed': reject.
+     */
+    failPolicy?: 'open' | 'closed';
+    /** Abort the stage after this long; the timeout follows failPolicy. */
+    timeoutMs?: number;
+    /** 'inline' (default) runs always; 'ci' runs only with `{ tier: 'ci' }`. */
+    tier?: 'inline' | 'ci';
+}
+export interface VerifyResult<T> {
+    ok: boolean;
+    /** First failing stage; absent on success. */
+    stage?: string;
+    /** The failing stage's messages only — not cumulative (P9 §2). */
+    messages: string[];
+    /** Final value: the (possibly transformed) input, or the revision. */
+    value: T;
+    /** True when a rejecting stage supplied a corrected value. */
+    revised: boolean;
+    /** Fail-open infrastructure failures and stage warnings. */
+    warnings: string[];
+    stagesRun: string[];
+}
+export interface RunVerifyOptions {
+    /** 'inline' (default) skips `tier: 'ci'` stages; 'ci' runs everything. */
+    tier?: 'inline' | 'ci';
+}
+export declare function runVerify<T>(stages: VerifyStage<T>[], input: T, options?: RunVerifyOptions): Promise<VerifyResult<T>>;
+/**
+ * Stage helper: a P9-style `parse_schema` gate from a Zod schema. On success
+ * the parsed (typed, defaulted, stripped) value replaces the input for later
+ * stages; on failure, one message per Zod issue with its path.
+ */
+export declare function zodStage<T>(name: string, schema: ZodType<T>): VerifyStage<any>;
+/**
+ * Format a rejection as generator feedback — stage id + messages verbatim as
+ * bullets (P9 §3: preserve strings so distances and ids stay fixable).
+ */
+export declare function formatRejection(result: VerifyResult<any>): string;
+export interface GenerateVerifiedOptions<T> {
+    /** Produce a candidate. On retries, `feedback` is the previous rejection. */
+    generate: (feedback?: VerifyResult<T>) => T | Promise<T>;
+    stages: VerifyStage<T>[];
+    /**
+     * Total attempts, default 2 (P9 §3: one automatic retry; more wastes
+     * tokens — after the cap, fall back or ask a human, don't loop).
+     */
+    attempts?: number;
+    tier?: 'inline' | 'ci';
+}
+/**
+ * The generate → verify → feed-back-and-retry loop as a first-class helper.
+ * Returns the first passing result, or the LAST rejection after the cap —
+ * never silently discards errors (P9 §3.4).
+ */
+export declare function generateVerified<T>(options: GenerateVerifiedOptions<T>): Promise<VerifyResult<T> & {
+    attempts: number;
+}>;

package/dist/verify.js

@@ -0,0 +1,96 @@
+async function runStage(stage, input, ctx) {
+    const run = Promise.resolve(stage.check(input, ctx));
+    if (!stage.timeoutMs)
+        return run;
+    let timer;
+    const timeout = new Promise((_, reject) => {
+        timer = setTimeout(() => reject(new Error(`stage timeout after ${stage.timeoutMs}ms`)), stage.timeoutMs);
+    });
+    try {
+        return await Promise.race([run, timeout]);
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
+export async function runVerify(stages, input, options = {}) {
+    const tier = options.tier ?? 'inline';
+    const warnings = [];
+    const stagesRun = [];
+    const ctx = { warn: (m) => warnings.push(m), tier };
+    let value = input;
+    for (const stage of stages) {
+        if ((stage.tier ?? 'inline') === 'ci' && tier !== 'ci')
+            continue;
+        stagesRun.push(stage.name);
+        let outcome;
+        try {
+            outcome = await runStage(stage, value, ctx);
+        }
+        catch (err) {
+            const message = `${stage.name}: ${err instanceof Error ? err.message : String(err)}`;
+            if ((stage.failPolicy ?? 'open') === 'open') {
+                warnings.push(`fail-open: ${message}`);
+                continue;
+            }
+            return { ok: false, stage: stage.name, messages: [message], value, revised: false, warnings, stagesRun };
+        }
+        if (!outcome.ok) {
+            const revised = outcome.revision !== undefined;
+            return {
+                ok: false,
+                stage: stage.name,
+                messages: outcome.messages,
+                value: revised ? outcome.revision : value,
+                revised,
+                warnings,
+                stagesRun,
+            };
+        }
+        if (outcome.value !== undefined)
+            value = outcome.value;
+    }
+    return { ok: true, messages: [], value, revised: false, warnings, stagesRun };
+}
+/**
+ * Stage helper: a P9-style `parse_schema` gate from a Zod schema. On success
+ * the parsed (typed, defaulted, stripped) value replaces the input for later
+ * stages; on failure, one message per Zod issue with its path.
+ */
+export function zodStage(name, schema) {
+    return {
+        name,
+        check: (input) => {
+            const parsed = schema.safeParse(input);
+            if (parsed.success)
+                return { ok: true, value: parsed.data };
+            return {
+                ok: false,
+                messages: parsed.error.issues.map((i) => `${i.path.length ? i.path.join('.') : '(root)'}: ${i.message}`),
+            };
+        },
+    };
+}
+/**
+ * Format a rejection as generator feedback — stage id + messages verbatim as
+ * bullets (P9 §3: preserve strings so distances and ids stay fixable).
+ */
+export function formatRejection(result) {
+    return `Rejected at stage "${result.stage}":\n${result.messages.map((m) => `- ${m}`).join('\n')}`;
+}
+/**
+ * The generate → verify → feed-back-and-retry loop as a first-class helper.
+ * Returns the first passing result, or the LAST rejection after the cap —
+ * never silently discards errors (P9 §3.4).
+ */
+export async function generateVerified(options) {
+    const attempts = Math.max(1, options.attempts ?? 2);
+    let last;
+    for (let i = 0; i < attempts; i++) {
+        const candidate = await options.generate(last);
+        last = await runVerify(options.stages, candidate, { tier: options.tier });
+        if (last.ok)
+            return { ...last, attempts: i + 1 };
+    }
+    return { ...last, attempts };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "stilltrue",
-  "version": "0.2.0",
+  "version": "0.4.0",
   "description": "Your tests check your code. stilltrue checks your facts. CI checks for AI-in-the-loop apps: drift (curated facts vs live sources), golden (regression evals), verify (judge answers against sources).",
   "keywords": [
     "drift",