productfoo 0.0.0-stage → 1.0.1

package/dist-public/src/errorTaxonomy.js

@@ -0,0 +1,189 @@
+// error-instrumentation-v0 T-001 (R-001, R-002, D-001): what kind of failure was
+// that, actually?
+//
+// Before this module, `main()` bucketed EVERY nonzero exit as "command_error", so
+// `foo swarm` typed without an argument — which prints a correct usage line and
+// exits 1 — was recorded identically to an unhandled crash. Across 14,492 events
+// the field only ever held "none" or "command_error", and not one genuine crash
+// was ever distinguishable. The dashboard cried wolf; the owner learned to ignore it.
+//
+// Three outcomes were collapsed into one. They are not the same thing:
+//
+//   usage_error  the human mistyped. foo printed usage. Nothing is broken.
+//   gate_block   a boundary (B1/B2/B4) refused by design. The product WORKED.
+//   command_error  something genuinely, unexpectedly failed. THIS is worth waking someone for.
+//
+// Only the last one should ever page a human. That is the whole point of the file.
+//
+// D-001: the code is a BOUNDED SYMBOL, never free text. It crosses the wire, where
+// productfoo's disclosure promises metadata only — so it must never be able to carry a path,
+// an argument, a filename, or user content. Adding a code means adding it to ErrorCode
+// below, on purpose, where it can be read and reviewed. Rich detail (message, stack)
+// stays in the local event log, which never leaves the machine.
+import { createHash } from "node:crypto";
+import { AsyncLocalStorage } from "node:async_hooks";
+/** Does a failure of this class deserve a human's attention at 3am? */
+export function pages(errorClass) {
+    return errorClass === "command_error" || errorClass === "io_error";
+}
+const failureScopes = new AsyncLocalStorage();
+let unscopedPending;
+/** Give one top-level invocation its own mark across every awaited dispatch path. */
+export function runWithFailureScope(operation) {
+    return failureScopes.run({}, operation);
+}
+/** Record what kind of failure just happened. Called at the failure site. */
+export function markFailure(mark) {
+    const scope = failureScopes.getStore();
+    if (scope)
+        scope.pending = mark;
+    else
+        unscopedPending = mark;
+}
+/** Take the mark and clear it. Cleared on read so it can never bleed into the next run. */
+export function consumeFailure() {
+    const scope = failureScopes.getStore();
+    const mark = scope ? scope.pending : unscopedPending;
+    if (scope)
+        scope.pending = undefined;
+    else
+        unscopedPending = undefined;
+    return mark;
+}
+/** Drop any mark. Called at the start of each run so a prior command can't contaminate this one. */
+export function resetFailure() {
+    const scope = failureScopes.getStore();
+    if (scope)
+        scope.pending = undefined;
+    else
+        unscopedPending = undefined;
+}
+/**
+ * "You invoked this wrong" — thrown rather than returned. Same meaning as failWith: a typo.
+ * Throw this instead of a bare Error and the failure classifies itself correctly.
+ */
+export class UsageError extends Error {
+    errorCode;
+    constructor(message, errorCode = "E_BAD_FLAG") {
+        super(message);
+        this.name = "UsageError";
+        this.errorCode = errorCode;
+    }
+}
+/**
+ * A boundary refused by design — B1 safety, B2 spec identity, B4 autonomy, a default-deny
+ * policy. THE PRODUCT IS WORKING. It exits nonzero because it stopped you on purpose, and it
+ * must never, ever page anyone.
+ */
+export class GateBlock extends Error {
+    errorCode;
+    constructor(message, errorCode = "E_GATE_BLOCKED") {
+        super(message);
+        this.name = "GateBlock";
+        this.errorCode = errorCode;
+    }
+}
+/**
+ * The thing you asked for does not exist — a packet dir, a loop id, a file. Nonzero, but not
+ * a defect: foo worked, the target just isn't there. Classifies as `not_found`, which never
+ * pages. Thrown rather than returned so the caller doesn't thread a special return value out.
+ */
+export class NotFoundError extends Error {
+    errorCode;
+    constructor(message, errorCode = "E_PACKET_NOT_FOUND") {
+        super(message);
+        this.name = "NotFoundError";
+        this.errorCode = errorCode;
+    }
+}
+// productfoo throws from 255 places. Most are genuine faults, but two families are not, and both
+// were being reported as crashes until a dogfood run caught them:
+//
+//   "work missing — usage: foo spec <work>"        — a typo. The command printed its usage.
+//   "loop(s) without an agreed budget: L-001"      — B4 refusing. The product WORKED.
+//
+// Converting all 255 call sites is not realistic, so new and converted code throws UsageError
+// or GateBlock (checked first, below) and these patterns bridge the rest. They are deliberately
+// narrow: a usage line and a named boundary are unmistakable, and anything we are not sure
+// about stays `command_error` and pages. A false "typo" hides a real bug; a false "bug" only
+// pings someone. When in doubt, ping.
+const LOOKS_LIKE_USAGE = /\busage: foo\b|\bunknown foo command\b/i;
+const LOOKS_LIKE_GATE = /without an agreed budget|default-deny|\bB[124]\b (?:safety|boundary|blocks)/i;
+/**
+ * Classify a thrown error that nothing marked. An `E*` errno (ENOENT, EACCES) is the
+ * filesystem or network failing under us; a typo and a gate refusal are not failures at all;
+ * anything else is a genuine unexpected fault, and gets someone's attention.
+ */
+export function classifyThrown(error) {
+    const message = error instanceof Error ? error.message : String(error);
+    const stack = error instanceof Error ? error.stack : undefined;
+    // The command said what it was. Believe it.
+    if (error instanceof UsageError)
+        return { errorClass: "usage_error", errorCode: error.errorCode, message, stack };
+    if (error instanceof GateBlock)
+        return { errorClass: "gate_block", errorCode: error.errorCode, message, stack };
+    if (error instanceof NotFoundError)
+        return { errorClass: "not_found", errorCode: error.errorCode, message, stack };
+    const code = error && typeof error === "object" && "code" in error ? error.code : undefined;
+    if (typeof code === "string" && /^E[A-Z]+$/.test(code)) {
+        return { errorClass: "io_error", errorCode: "E_IO", message, stack };
+    }
+    if (LOOKS_LIKE_GATE.test(message))
+        return { errorClass: "gate_block", errorCode: "E_GATE_BLOCKED", message, stack };
+    if (LOOKS_LIKE_USAGE.test(message))
+        return { errorClass: "usage_error", errorCode: "E_BAD_FLAG", message, stack };
+    return { errorClass: "command_error", errorCode: "E_UNKNOWN", message, stack };
+}
+// The one frame that says WHERE a failure came from, reduced to something safe to fingerprint
+// AND stable across the people who hit the same bug. Three reductions, each load-bearing:
+//
+//   - the absolute path is dropped to a basename — a stack names `/Users/taylor/...`, and the
+//     whole point of the disclosure promise is that a tester's home directory is not ours;
+//   - :line:col is dropped — the same bug drifts a few lines between releases, and a
+//     fingerprint that changes every version cannot say "this is the same bug, still";
+//   - only the first real frame is kept — deeper frames are shared plumbing that would make
+//     unrelated failures collide.
+//
+// What remains is `functionName (basename.js)`: enough to tell two bugs apart, free of any
+// path, argument, or user content — the same content-free bar the bounded error code meets.
+function fingerprintFrame(stack) {
+    if (!stack)
+        return null;
+    const m = stack.split(/\r?\n/).map(line => /^\s*at\s+(\S+)\s+\(([^)]+)\)/.exec(line)).find(Boolean);
+    if (!m)
+        return null;
+    const file = m[2].split(/[\\/]/).pop()?.replace(/:\d+:\d+$/, "") ?? m[2];
+    return `${m[1]} (${file})`;
+}
+/**
+ * A stable, privacy-safe grouping key for a failure: the bounded code plus the normalized
+ * origin frame. Two failures with the same code AND the same frame are the same bug — which
+ * is what lets the digest tell "one bug hit nine people" apart from "nine unrelated
+ * E_UNKNOWNs in the same command".
+ *
+ * It is a one-way hash of content-free inputs (a bounded code and a `fn (basename)` string —
+ * no path, no line, no argument, no message), so it is safe on the metadata-only wire.
+ *
+ * NO STACK MEANS NO FINGERPRINT. This used to hash a `<no-stack>` sentinel instead, and the
+ * comment defended it as degrading "deterministically ... rather than collapsing at random".
+ * It had it exactly backwards: hashing a constant does not spread stackless failures out, it
+ * funnels EVERY one of them that shares a code into a single identity. In production that
+ * produced `3b07a3cdfb33` = sha256("E_UNKNOWN\n<no-stack>") on eight unrelated commands at
+ * once — asp reconcile, self check, self gate, research gate, asp validate, research
+ * validate, commands validate, reconcile — which the console then drew as ONE bug, named
+ * after whichever row happened to sort first, with the blast radius of all eight.
+ *
+ * It also silently broke the join to error reports: a report fingerprints WITH a stack and
+ * these rows fingerprinted WITHOUT one, so the two could never match. 0 of 140 did. Every
+ * stack the console displayed came from a fallback that guessed by error code, which is how
+ * `foo redact` came to show a "work-item missing" crash from specCmd.js.
+ *
+ * Returning undefined is the honest answer: callers already group by command + code when
+ * there is no fingerprint, which keeps unrelated commands apart instead of fusing them.
+ */
+export function errorFingerprint(errorCode, stack) {
+    const frame = fingerprintFrame(stack);
+    if (!frame)
+        return undefined;
+    return createHash("sha256").update(`${errorCode}\n${frame}`).digest("hex").slice(0, 12);
+}

package/dist-public/src/evalCalibration.d.ts

@@ -0,0 +1,82 @@
+import type { EvalJudgeSpec } from "./evalRecord.js";
+export declare const CALIBRATION_LABEL_COUNT = 30;
+export declare const CALIBRATION_MIN_RATE = 0.9;
+export declare const CALIBRATION_MIN_FAIL_LABELS = 10;
+export type OwnerLabel = {
+    /** Content hash of the labelled artifact — the corpus itself stays private. */
+    artifact: string;
+    verdict: "pass" | "fail";
+    /** True only for labels given blind (no critique context, anonymised). */
+    blind: boolean;
+    labeled_at: string;
+    /** Free-text capture note: actionable description, no root-cause analysis. */
+    note?: string;
+    /** How the label was given; a dispute relabel supersedes by later append. */
+    origin?: "capture" | "dispute";
+    /** For a relabel: what surfaced the dispute. */
+    provenance?: string;
+};
+/**
+ * A blind dispute relabel that disagreed with the judge (or refined an earlier
+ * label): a refinement signal for the record's criterion or the judge prompt,
+ * never a gate input — the relabel already replaced the scored label.
+ */
+export type DisputeSignal = {
+    /** Content hash of the disputed artifact (private corpus pointer). */
+    artifact: string;
+    /** The owner's blind relabel — supersedes the original by later append. */
+    owner_verdict: "pass" | "fail";
+    /** The judge sample's verdict on the same artifact, when one exists. */
+    judge_verdict?: "pass" | "fail";
+    /** The owner's note, when one accompanied the relabel. */
+    note?: string;
+};
+export type JudgeSample = {
+    artifact: string;
+    verdict: "pass" | "fail";
+    /** Verdict from the order-swapped second call, when taken. */
+    swapped_verdict?: "pass" | "fail";
+};
+export type CalibrationReport = {
+    schema_version: "0.1";
+    /** hash(effective prompt protocol, judge prompt, judge model) — the validity key. */
+    judge_key: string;
+    labels: {
+        total: number;
+        pass: number;
+        fail: number;
+        non_blind_excluded: number;
+    };
+    tpr: number | null;
+    tnr: number | null;
+    /** Share of double-called samples where the order swap flipped the verdict. */
+    flip_rate: number | null;
+    /** The gate: min(TPR, TNR) ≥ bar AND label floors met, on blind labels only. */
+    certified: boolean;
+    reasons: string[];
+    /** Dispute relabels — refinement signals, reported but never gating. */
+    disputes: DisputeSignal[];
+    /**
+     * Cohen's kappa over the blind labels that have a judge sample —
+     * chance-corrected owner/judge agreement. Diagnostic only: `certified`
+     * never consults it, and the gate is untouched by it.
+     */
+    kappa: number | null;
+};
+export declare function judgeKey(judge: Pick<EvalJudgeSpec, "prompt" | "model">): string;
+/**
+ * Score a judge's samples against the owner's labels. Non-blind labels are
+ * excluded from the measurement, never silently: the report counts them.
+ * Loop-round rulings arrive with blind:false and therefore can never certify
+ * a judge (D-007).
+ */
+export declare function computeCalibration(judge: Pick<EvalJudgeSpec, "prompt" | "model">, labels: OwnerLabel[], samples: JudgeSample[]): CalibrationReport;
+/**
+ * May this judge gate a run? Only with a certified report whose key matches
+ * the judge's current effective prompt+model — an edited judge is an uncalibrated judge
+ * until re-validated (against the same labels; no new labeling needed).
+ */
+export declare function judgeMayGate(judge: Pick<EvalJudgeSpec, "prompt" | "model">, report: CalibrationReport | undefined): {
+    allowed: boolean;
+    reason: string;
+};

package/dist-public/src/evalCalibration.js

@@ -0,0 +1,169 @@
+import { createHash } from "node:crypto";
+import { JUDGE_PROMPT_PROTOCOL } from "./evalJudge.js";
+// Judge calibration (foo-eval-system-v0 R-003). A judge's verdicts count only
+// after a blind, binary, owner-labelled validation clears the bar — and the
+// bar is per-class: min(TPR, TNR), with a floor on real-failure labels, so an
+// always-pass judge cannot clear it on a good-artifact corpus. Calibration is
+// keyed to hash(effective prompt protocol, judge prompt, judge model): any drift voids it.
+export const CALIBRATION_LABEL_COUNT = 30;
+export const CALIBRATION_MIN_RATE = 0.9;
+export const CALIBRATION_MIN_FAIL_LABELS = 10;
+export function judgeKey(judge) {
+    return createHash("sha256").update(JUDGE_PROMPT_PROTOCOL).update("\u0000").update(judge.prompt).update("\u0000").update(judge.model).digest("hex");
+}
+const round = (value) => Math.round(value * 1000) / 1000;
+/**
+ * Cohen's kappa over the blind∩sampled agreement set. po is the observed
+ * agreement; pe is what two independent raters would agree on by chance,
+ * from BOTH marginals of the same set — the owner-label distribution and
+ * the judge-verdict distribution. Null when the set is empty or pe === 1
+ * (a single class: nothing left for chance to explain away). Diagnostic
+ * only — the gate never reads it.
+ */
+function cohenKappa(counts) {
+    const { agree, total, ownerPass, judgePass } = counts;
+    if (total === 0)
+        return null;
+    const po = agree / total;
+    const ownerFail = total - ownerPass;
+    const judgeFail = total - judgePass;
+    const pe = (ownerPass * judgePass + ownerFail * judgeFail) / (total * total);
+    if (pe === 1)
+        return null;
+    return round((po - pe) / (1 - pe));
+}
+/**
+ * Score a judge's samples against the owner's labels. Non-blind labels are
+ * excluded from the measurement, never silently: the report counts them.
+ * Loop-round rulings arrive with blind:false and therefore can never certify
+ * a judge (D-007).
+ */
+export function computeCalibration(judge, labels, samples) {
+    const blind = labels.filter((label) => label.blind);
+    const nonBlindExcluded = labels.length - blind.length;
+    const byArtifact = new Map(samples.map((sample) => [sample.artifact, sample]));
+    let truePass = 0;
+    let falseFail = 0;
+    let trueFail = 0;
+    let falsePass = 0;
+    let doubleCalled = 0;
+    let flipped = 0;
+    let unsampled = 0;
+    let kappaAgree = 0;
+    let kappaTotal = 0;
+    let ownerPassCount = 0;
+    let judgePassCount = 0;
+    for (const label of blind) {
+        const sample = byArtifact.get(label.artifact);
+        if (!sample) {
+            unsampled += 1;
+            continue;
+        }
+        kappaTotal += 1;
+        if (label.verdict === "pass")
+            ownerPassCount += 1;
+        if (sample.verdict === "pass")
+            judgePassCount += 1;
+        if (label.verdict === sample.verdict)
+            kappaAgree += 1;
+        if (sample.swapped_verdict !== undefined) {
+            doubleCalled += 1;
+            if (sample.swapped_verdict !== sample.verdict)
+                flipped += 1;
+        }
+        if (label.verdict === "pass") {
+            if (sample.verdict === "pass")
+                truePass += 1;
+            else
+                falseFail += 1;
+        }
+        else {
+            if (sample.verdict === "fail")
+                trueFail += 1;
+            else
+                falsePass += 1;
+        }
+    }
+    const passLabels = truePass + falseFail;
+    const failLabels = trueFail + falsePass;
+    const tpr = passLabels > 0 ? round(truePass / passLabels) : null;
+    const tnr = failLabels > 0 ? round(trueFail / failLabels) : null;
+    const flipRate = doubleCalled > 0 ? round(flipped / doubleCalled) : null;
+    const reasons = [];
+    const measured = passLabels + failLabels;
+    if (measured < CALIBRATION_LABEL_COUNT) {
+        reasons.push(`Only ${measured} blind labels measured; certification needs ${CALIBRATION_LABEL_COUNT}.`);
+    }
+    if (failLabels < CALIBRATION_MIN_FAIL_LABELS) {
+        reasons.push(`Only ${failLabels} fail-class labels; the floor is ${CALIBRATION_MIN_FAIL_LABELS} — without real failures the false-pass rate is unmeasured.`);
+    }
+    if (tpr !== null && tpr < CALIBRATION_MIN_RATE) {
+        reasons.push(`TPR ${tpr} is below the ${CALIBRATION_MIN_RATE} bar.`);
+    }
+    if (tnr !== null && tnr < CALIBRATION_MIN_RATE) {
+        reasons.push(`TNR ${tnr} is below the ${CALIBRATION_MIN_RATE} bar.`);
+    }
+    if (unsampled > 0) {
+        reasons.push(`${unsampled} blind label(s) have no judge sample yet.`);
+    }
+    if (nonBlindExcluded > 0) {
+        reasons.push(`${nonBlindExcluded} non-blind label(s) excluded from measurement (loop rulings are provisional signal, never calibration).`);
+    }
+    const certified = measured >= CALIBRATION_LABEL_COUNT &&
+        failLabels >= CALIBRATION_MIN_FAIL_LABELS &&
+        tpr !== null &&
+        tnr !== null &&
+        Math.min(tpr, tnr) >= CALIBRATION_MIN_RATE &&
+        unsampled === 0;
+    const disputes = blind
+        .filter((label) => label.origin === "dispute")
+        .map((label) => {
+        const sample = byArtifact.get(label.artifact);
+        return {
+            artifact: label.artifact,
+            owner_verdict: label.verdict,
+            ...(sample ? { judge_verdict: sample.verdict } : {}),
+            ...(label.note ? { note: label.note } : {}),
+        };
+    });
+    return {
+        schema_version: "0.1",
+        judge_key: judgeKey(judge),
+        labels: { total: blind.length, pass: passLabels, fail: failLabels, non_blind_excluded: nonBlindExcluded },
+        tpr,
+        tnr,
+        flip_rate: flipRate,
+        certified,
+        reasons,
+        disputes,
+        kappa: cohenKappa({
+            agree: kappaAgree,
+            total: kappaTotal,
+            ownerPass: ownerPassCount,
+            judgePass: judgePassCount,
+        }),
+    };
+}
+/**
+ * May this judge gate a run? Only with a certified report whose key matches
+ * the judge's current effective prompt+model — an edited judge is an uncalibrated judge
+ * until re-validated (against the same labels; no new labeling needed).
+ */
+export function judgeMayGate(judge, report) {
+    if (!report) {
+        return { allowed: false, reason: "No calibration report — the judge runs advisory-only." };
+    }
+    if (report.judge_key !== judgeKey(judge)) {
+        return {
+            allowed: false,
+            reason: "The judge's prompt, request protocol, or model changed since calibration — re-run the blind validation with fresh judge samples against the existing labels before it gates.",
+        };
+    }
+    if (!report.certified) {
+        return {
+            allowed: false,
+            reason: `Calibration did not certify: ${report.reasons.join(" ")}`,
+        };
+    }
+    return { allowed: true, reason: "Certified against the owner's blind labels." };
+}

package/dist-public/src/evalChecks.d.ts

@@ -0,0 +1,21 @@
+export type EvalCheckSeverity = "fail" | "warn";
+export type EvalCheckFinding = {
+    check: string;
+    line: number;
+    phrase: string;
+    message: string;
+    severity: EvalCheckSeverity;
+};
+export type EvalChecksResult = {
+    ok: boolean;
+    findings: EvalCheckFinding[];
+};
+export type EvalCheckRequest = {
+    id: string;
+    params?: Record<string, unknown>;
+};
+export declare const DETERMINISTIC_CHECK_IDS: readonly string[];
+/** Run the requested deterministic checks over an artifact's markdown. */
+export declare function runDeterministicChecks(artifactText: string, checks: EvalCheckRequest[], options?: {
+    strict?: boolean;
+}): EvalChecksResult;