tickmarkr 1.69.0 → 1.71.0

package/dist/gates/acceptance.js

@@ -7,13 +7,14 @@ import { checkDiffCap, fetchTaskDiff } from "./review.js";
 import { COMPLETION_FAKING_CHECKLIST, extractVerdictJson, generateVerdictNonce, runLlm, verdictNonceLine } from "./llm.js";
 // Fable F4: acceptance judge shares review's 900s timeout — 300s default killed frontier judges on cap-sized diffs.
 const JUDGE_TIMEOUT_MS = 900_000;
+const CitationSchema = z.object({ path: z.string(), line: z.number().int() });
 const JudgeVerdictRowSchema = z.object({
     criterion: z.string(),
     met: z.boolean(),
     reason: z.string(),
-    // v1.64: a verbatim quote from the judged diff grounding the ruling — required; a verdict
-    // omitting it is malformed and fails closed like any other shape violation.
-    evidence: z.string(),
+    // Required, either form: a row omitting evidence is malformed and fails closed like any other shape
+    // violation. Object → structured citation; string → legacy quote.
+    evidence: z.union([CitationSchema, z.string()]),
 });
 const JudgeVerdictSchema = z.object({
     pass: z.boolean(),
@@ -103,6 +104,64 @@ function tail(out, n = 8) {
         return "";
     return "\n" + t.split("\n").slice(-n).join("\n");
 }
+// v1.70: the new-file line numbers each hunk actually ADDS, per changed path — parsed from the
+// unified-diff hunk headers so a citation is validated against real changed locations, not a substring
+// of the whole diff text (which also matches unchanged context lines and the diff's own +++/@@ headers).
+// Only `+` lines are changed locations; context lines advance the new-file counter but are not changes.
+// ponytail: assumes git's default a/ b/ prefixes (fetchTaskDiff uses plain `git diff`); revisit if a
+// caller passes a --no-prefix diff.
+function changedLinesByFile(diff) {
+    const byFile = new Map();
+    let path = null;
+    let newLine = 0;
+    let inHunk = false;
+    for (const raw of diff.split("\n")) {
+        if (raw.startsWith("diff --git")) {
+            inHunk = false;
+            path = null;
+            continue;
+        }
+        if (!inHunk && raw.startsWith("--- "))
+            continue;
+        if (!inHunk && raw.startsWith("+++ ")) {
+            const p = raw.slice(4).trim();
+            path = p === "/dev/null" ? null : p.replace(/^[ab]\//, "");
+            continue;
+        }
+        const hunk = /^@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@/.exec(raw);
+        if (hunk) {
+            newLine = Number(hunk[1]);
+            inHunk = true;
+            continue;
+        }
+        if (!inHunk || path === null)
+            continue;
+        const c = raw[0];
+        if (c === "+") {
+            let set = byFile.get(path);
+            if (!set) {
+                set = new Set();
+                byFile.set(path, set);
+            }
+            set.add(newLine);
+            newLine++;
+        }
+        else if (c === " ") {
+            newLine++;
+        }
+        else if (c !== "-") {
+            inHunk = false; // "\ No newline", a trailing blank, or the next section — hunk body ended
+        }
+    }
+    return byFile;
+}
+// A citation is valid evidence iff the diff adds the cited line of the cited file. A legacy free-text
+// quote (string) keeps v1.64's substring check — non-empty and present somewhere in the diff.
+function citesChangedLocation(evidence, changed, diff) {
+    if (typeof evidence === "string")
+        return evidence.trim().length > 0 && diff.includes(evidence);
+    return changed.get(evidence.path)?.has(evidence.line) ?? false;
+}
 export async function acceptanceGate(task, worktree, baseRef, judge, via, opts = {}) {
     // 1. deterministic oracles — exit code decides, fail-closed, zero LLM calls (spec §2, T2).
     //    A failure returns here, before any runLlm() call: a judge can never override it.
@@ -176,9 +235,9 @@ ${diff}
 ${verdictNonceLine(nonce)}
 
 Respond with ONLY this JSON (no prose before or after):
-{"nonce": "${nonce}", "pass": true|false, "criteria": [{"criterion": "c1", "met": true|false, "reason": "...", "evidence": "..."}]}
+{"nonce": "${nonce}", "pass": true|false, "criteria": [{"criterion": "c1", "met": true|false, "reason": "...", "evidence": {"path": "path/to/file", "line": 42}}]}
 Each criteria[].criterion MUST be the stable id from the rubric (c1, c2, ...) exactly once.
-Each criteria[].evidence MUST be a short verbatim quote copied from the diff above that grounds the ruling; a quote not found in the diff voids the whole verdict.
+Each criteria[].evidence MUST be a structured citation {"path", "line"} pointing at a line the diff above actually adds or changes (the new-file line number); a citation to an unchanged or nonexistent location voids the whole verdict.
 `;
     const raw = await runLlm(judge.adapter, judge.model, prompt, worktree, via, JUDGE_TIMEOUT_MS);
     const extracted = extractVerdictJson(raw, nonce);
@@ -191,13 +250,16 @@ Each criteria[].evidence MUST be a short verbatim quote copied from the diff abo
             meta: { unparseable: true, judge: channelKey({ adapter: judge.adapter.id, model: judge.model }) } };
     }
     const { verdict: v, inconsistencies } = checkJudgeVerdict(extracted, expectedIds);
-    // v1.64: quoted evidence must appear verbatim in `diff` — the exact string embedded in the prompt
-    // above, never the worktree or any other artifact. A quote the diff doesn't contain is a
-    // hallucinated verdict: treated as unparseable so GATE-09 retries the judge on a failover channel.
-    const fabricated = v.criteria.filter((row) => !row.evidence.trim() || !diff.includes(row.evidence));
+    // v1.70: each citation must point at a line the diff actually changed — validated against the hunks
+    // of `diff` (the exact string embedded in the prompt above), never the worktree or any other artifact.
+    // A citation to an untouched file or a line outside every changed hunk is a hallucinated verdict:
+    // treated as unparseable so GATE-09 retries the judge on a failover channel. (A legacy free-text quote
+    // still validates by substring, for the fake seam and pre-v1.70 fixtures.)
+    const changed = changedLinesByFile(diff);
+    const fabricated = v.criteria.filter((row) => !citesChangedLocation(row.evidence, changed, diff));
     if (fabricated.length) {
         return { gate: "acceptance", pass: false,
-            details: warn + detBlock + `judge verdict quotes evidence absent from the judged diff (${fabricated.map((row) => row.criterion).join(", ")}) — treating as unparseable, failing closed`,
+            details: warn + detBlock + `judge verdict cites evidence absent from the judged diff (${fabricated.map((row) => row.criterion).join(", ")}) — treating as unparseable, failing closed`,
             meta: { unparseable: true, judge: channelKey({ adapter: judge.adapter.id, model: judge.model }) } };
     }
     const pass = v.pass === true && inconsistencies.length === 0 && v.criteria.every((row) => row.met);

package/dist/gates/review.d.ts

@@ -3,9 +3,17 @@ import { type TickmarkrConfig } from "../config/config.js";
 import { type Task } from "../graph/schema.js";
 import { type GateVia } from "./llm.js";
 import type { GateResult } from "./types.js";
+export type ReviewSeverity = "material" | "minor";
+export interface ReviewFinding {
+    note: string;
+    severity: ReviewSeverity;
+    defer?: boolean;
+    rationale?: string;
+}
 export interface ReviewVerdict {
-    approve: boolean;
-    issues: string[];
+    approve?: boolean;
+    issues?: string[];
+    findings?: ReviewFinding[];
 }
 export declare function fetchTaskDiff(worktree: string, baseRef: string): Promise<{
     full: string;

package/dist/gates/review.js

@@ -5,6 +5,63 @@ import { getAdapter } from "../adapters/registry.js";
 import { shOk } from "../run/git.js";
 import { marginalCostRank } from "../route/router.js";
 import { COMPLETION_FAKING_CHECKLIST, extractVerdictJson, generateVerdictNonce, runLlm, verdictNonceLine } from "./llm.js";
+// legacy flat `issues` shape — every issue blocks; the approve flag must agree with the list.
+function classifyReviewIssues(approve, issues) {
+    const inconsistencies = [];
+    issues.forEach((issue, i) => {
+        if (typeof issue !== "string")
+            inconsistencies.push(`review verdict inconsistent: issues[${i}] must be a string`);
+    });
+    if (approve && issues.length) {
+        inconsistencies.push("review verdict inconsistent: approve=true requires issues to be empty");
+    }
+    else if (!approve && !issues.length) {
+        inconsistencies.push("review verdict inconsistent: approve=false requires at least one issue");
+    }
+    const pass = approve === true && inconsistencies.length === 0;
+    const lines = issues.map((issue) => `- ${typeof issue === "string" ? issue : JSON.stringify(issue)}`);
+    lines.push(...inconsistencies);
+    return { pass, headline: pass ? "approved" : approve ? "approval rejected" : "requested changes", lines };
+}
+// v1.70 T5: classified findings — only material (non-deferred) findings block approval. Deferred
+// findings carry their rationale into the details (never dropped). Malformed rows fail closed like any
+// other shape violation, so a garbage "findings" array can never fake an approval.
+function classifyReviewFindings(findings) {
+    const inconsistencies = [];
+    const lines = [];
+    let material = 0;
+    let deferred = 0;
+    findings.forEach((f, i) => {
+        if (!f || typeof f !== "object") {
+            inconsistencies.push(`review finding inconsistent: findings[${i}] must be an object`);
+            return;
+        }
+        const { note, severity, defer, rationale } = f;
+        if (typeof note !== "string")
+            inconsistencies.push(`review finding inconsistent: findings[${i}].note must be a string`);
+        if (severity !== "material" && severity !== "minor")
+            inconsistencies.push(`review finding inconsistent: findings[${i}].severity must be "material" or "minor"`);
+        if (defer !== undefined && typeof defer !== "boolean")
+            inconsistencies.push(`review finding inconsistent: findings[${i}].defer must be a boolean`);
+        const isDeferred = defer === true;
+        if (isDeferred && (typeof rationale !== "string" || !rationale.trim())) {
+            inconsistencies.push(`review finding inconsistent: deferred findings[${i}] requires a rationale`);
+        }
+        if (severity === "material" && !isDeferred)
+            material++;
+        if (isDeferred)
+            deferred++;
+        const label = isDeferred ? `deferred/${severity ?? "?"}` : String(severity ?? "?");
+        const why = isDeferred && typeof rationale === "string" ? ` — rationale: ${rationale}` : "";
+        lines.push(`- [${label}] ${typeof note === "string" ? note : JSON.stringify(note)}${why}`);
+    });
+    const pass = material === 0 && inconsistencies.length === 0;
+    lines.push(...inconsistencies);
+    const headline = pass
+        ? deferred ? `approved (${deferred} deferred)` : "approved"
+        : `requested changes (${material} material)`;
+    return { pass, headline, lines };
+}
 // OBS-48: cap on zero-context diff bytes (git diff -U0), not context-padded full diff — scattered
 // one-line hunks no longer trip at ~370 diff-bytes per changed line. Full diff still goes to the judge.
 const DIFF_CAP_REMEDY = "split the task, or raise gates.diffCap";
@@ -95,8 +152,14 @@ ${diff}
 
 ${verdictNonceLine(nonce)}
 
+Classify every concern as "material" (a correctness, security, or acceptance-criteria defect that must
+block the merge) or "minor" (style, naming, or preference that should not block). ONLY material findings
+block approval. For a minor concern you have decided not to block on, set "defer": true and give a
+one-line "rationale" — it is recorded in the review, never dropped.
+
 Respond with ONLY this JSON:
-{"nonce": "${nonce}", "approve": true|false, "issues": ["..."]}
+{"nonce": "${nonce}", "approve": true|false, "findings": [{"note": "...", "severity": "material"|"minor", "defer": false, "rationale": ""}]}
+Approve iff no material finding remains; an empty findings list is a clean approval.
 `;
     const raw = await runLlm(getAdapter(reviewer.adapter, adapters), reviewer.model, prompt, worktree, via ? { driver: via.driver, keep: via.keep, onSlot: via.onSlot, name: via.nameFor("review", reviewer.adapter), label: via.labelFor("review") } : undefined, 
     // frontier reviewers routinely need >5min on a configured-cap-sized diff, and `claude -p` buffers all
@@ -106,7 +169,9 @@ Respond with ONLY this JSON:
     // second knob-turner appears.
     900_000);
     const v = extractVerdictJson(raw, nonce);
-    if (!v || typeof v.approve !== "boolean" || !Array.isArray(v.issues)) {
+    const findings = v && Array.isArray(v.findings) ? v.findings : null;
+    // findings decides the verdict on its own; the legacy path still needs approve + issues to parse.
+    if (!v || (findings === null && (typeof v.approve !== "boolean" || !Array.isArray(v.issues)))) {
         return {
             gate: "review",
             pass: false,
@@ -114,25 +179,13 @@ Respond with ONLY this JSON:
             meta: { reviewer: channelKey(reviewer) },
         };
     }
-    const issues = v.issues;
-    const inconsistencies = [];
-    issues.forEach((issue, i) => {
-        if (typeof issue !== "string")
-            inconsistencies.push(`review verdict inconsistent: issues[${i}] must be a string`);
-    });
-    if (v.approve && issues.length) {
-        inconsistencies.push("review verdict inconsistent: approve=true requires issues to be empty");
-    }
-    else if (!v.approve && !issues.length) {
-        inconsistencies.push("review verdict inconsistent: approve=false requires at least one issue");
-    }
-    const pass = v.approve === true && inconsistencies.length === 0;
-    const lines = issues.map((issue) => `- ${typeof issue === "string" ? issue : JSON.stringify(issue)}`);
-    lines.push(...inconsistencies);
+    const decided = findings !== null
+        ? classifyReviewFindings(findings)
+        : classifyReviewIssues(v.approve, v.issues);
     return {
         gate: "review",
-        pass,
-        details: `reviewer ${reviewer.adapter}:${reviewer.model} (${reviewer.vendor}): ${pass ? "approved" : v.approve ? "approval rejected" : "requested changes"}${lines.length ? "\n" + lines.join("\n") : ""}`,
+        pass: decided.pass,
+        details: `reviewer ${reviewer.adapter}:${reviewer.model} (${reviewer.vendor}): ${decided.headline}${decided.lines.length ? "\n" + decided.lines.join("\n") : ""}`,
         meta: { reviewer: channelKey(reviewer) },
     };
 }

package/dist/report/bundle.d.ts

@@ -0,0 +1,45 @@
+import type { EvidenceCitation } from "../gates/acceptance.js";
+import type { RunEnvironment } from "../run/environment.js";
+import type { JournalEvent } from "../run/journal.js";
+export declare const BUNDLE_SCHEMA_VERSION = 1;
+export declare const KNOWN_LIMITS: readonly string[];
+export type BundleEvidence = string | EvidenceCitation;
+export interface BundleJudgeCriterion {
+    criterion: string;
+    met: boolean;
+    reason: string;
+    evidence: BundleEvidence;
+}
+export interface BundleGateResult {
+    gate: string;
+    pass: boolean;
+    details: string;
+}
+export interface BundleTask {
+    taskId: string;
+    outcome: "done" | "failed" | "human" | "not-recorded";
+    gates: BundleGateResult[];
+    /** Judge criteria with evidence citations, when the journal recorded them structured. */
+    judgeCriteria: BundleJudgeCriterion[];
+}
+export interface BundleContentHashes {
+    /** sha256 of the canonical JSON of the source journal events used to build this packet. */
+    journal: string;
+    /** graphDefinitionHash from run-start when recorded. */
+    graphDefinitionHash?: string;
+}
+export interface ProofBundle {
+    /** Schema version a future reader must check before parsing the rest. */
+    schemaVersion: number;
+    runId: string;
+    environment?: RunEnvironment;
+    contentHashes: BundleContentHashes;
+    tasks: BundleTask[];
+    /** Plain-language known limits — this is not an unconditional proof of correctness. */
+    knownLimits: string[];
+}
+/**
+ * Build a schema-versioned proof packet from a run's journal events.
+ * Pure: no filesystem, no network. Caller writes the result.
+ */
+export declare function buildProofBundle(runId: string, events: JournalEvent[]): ProofBundle;

package/dist/report/bundle.js

@@ -0,0 +1,158 @@
+// v1.70 T4: pure proof-packet builder for `tickmarkr report --bundle <path>`.
+// One portable, schema-versioned snapshot of a run — task outcomes, judge evidence
+// citations, environment identity, content hashes — secrets redacted via the shared
+// redactSecrets seam. No I/O and no network: the caller (report CLI) owns the write.
+import { createHash } from "node:crypto";
+import { redactSecrets } from "../run/redact.js";
+import { recordedEnvironment } from "./compare.js";
+// Bump when the packet shape changes in a way a future reader must branch on before parsing.
+export const BUNDLE_SCHEMA_VERSION = 1;
+// Plain-language known limits — the packet is a journal snapshot, not an unconditional proof.
+export const KNOWN_LIMITS = [
+    "This packet is a portable snapshot of journaled run facts, not an independent re-verification of the work or its gates.",
+    "Judge evidence citations are copied from the journal as the judge recorded them; this packet does not re-validate citations against the judged diff.",
+    "Content hashes bind this packet to the journal bytes used to build it; they do not prove the underlying gates, merges, or tip verification were correct.",
+    "Secret-shaped strings are redacted by local pattern matching only; redaction is not a formal security audit.",
+    "Producing this packet never contacts a network and never uploads anything.",
+];
+function contentHash(events) {
+    return createHash("sha256").update(JSON.stringify(events)).digest("hex");
+}
+function graphHash(events) {
+    for (const e of events) {
+        if (e.event !== "run-start")
+            continue;
+        const h = e.data.graphDefinitionHash;
+        return typeof h === "string" ? h : undefined;
+    }
+    return undefined;
+}
+function taskIds(events) {
+    const seen = new Set();
+    const out = [];
+    for (const e of events) {
+        if (!e.taskId || seen.has(e.taskId))
+            continue;
+        seen.add(e.taskId);
+        out.push(e.taskId);
+    }
+    return out;
+}
+function outcomeFor(events, taskId, runEnd) {
+    for (let i = events.length - 1; i >= 0; i--) {
+        const e = events[i];
+        if (e.taskId !== taskId)
+            continue;
+        if (e.event === "task-done")
+            return "done";
+        if (e.event === "task-failed")
+            return "failed";
+        if (e.event === "task-human")
+            return "human";
+    }
+    if (runEnd) {
+        const d = runEnd.data;
+        if (Array.isArray(d.done) && d.done.includes(taskId))
+            return "done";
+        if (Array.isArray(d.failed) && d.failed.includes(taskId))
+            return "failed";
+        if (Array.isArray(d.human) && d.human.includes(taskId))
+            return "human";
+    }
+    return "not-recorded";
+}
+// Parse a structured evidence citation without rewriting path/line — unaltered copy.
+function parseEvidence(raw) {
+    if (typeof raw === "string")
+        return raw;
+    if (!raw || typeof raw !== "object" || Array.isArray(raw))
+        return undefined;
+    const o = raw;
+    if (typeof o.path === "string" && typeof o.line === "number" && Number.isInteger(o.line)) {
+        return { path: o.path, line: o.line };
+    }
+    return undefined;
+}
+// Structured judge criteria on an acceptance gate-result (when journaled). Evidence is copied
+// unaltered so a future reader can match citations byte-for-byte to what the judge recorded.
+function parseJudgeCriteria(data) {
+    const raw = data.criteria;
+    if (!Array.isArray(raw))
+        return [];
+    const out = [];
+    for (const row of raw) {
+        if (!row || typeof row !== "object" || Array.isArray(row))
+            continue;
+        const r = row;
+        if (typeof r.criterion !== "string" || typeof r.met !== "boolean" || typeof r.reason !== "string")
+            continue;
+        const evidence = parseEvidence(r.evidence);
+        if (evidence === undefined)
+            continue;
+        out.push({ criterion: r.criterion, met: r.met, reason: r.reason, evidence });
+    }
+    return out;
+}
+// Deep-walk string leaves through redactSecrets. Numbers/booleans/null stay as-is.
+// Structured evidence citations keep path/line unaltered (redact only free-text string evidence).
+function redactValue(v) {
+    if (typeof v === "string")
+        return redactSecrets(v);
+    if (v === null || typeof v !== "object")
+        return v;
+    if (Array.isArray(v))
+        return v.map(redactValue);
+    const o = v;
+    // Preserve structured {path, line} citations unaltered (criterion "unaltered").
+    if (typeof o.path === "string" && typeof o.line === "number" && Object.keys(o).length === 2) {
+        return { path: o.path, line: o.line };
+    }
+    const out = {};
+    for (const [k, val] of Object.entries(o))
+        out[k] = redactValue(val);
+    return out;
+}
+/**
+ * Build a schema-versioned proof packet from a run's journal events.
+ * Pure: no filesystem, no network. Caller writes the result.
+ */
+export function buildProofBundle(runId, events) {
+    const runEnd = [...events].reverse().find((e) => e.event === "run-end");
+    const environment = recordedEnvironment(events);
+    const journalHash = contentHash(events);
+    const gHash = graphHash(events);
+    const tasks = taskIds(events).map((taskId) => {
+        const gateEvents = events.filter((e) => e.taskId === taskId && e.event === "gate-result");
+        const gates = [];
+        const judgeCriteria = [];
+        for (const g of gateEvents) {
+            const gate = typeof g.data.gate === "string" ? g.data.gate : "unknown";
+            const pass = g.data.pass === true;
+            const details = typeof g.data.details === "string" ? g.data.details : "";
+            gates.push({ gate, pass, details });
+            if (gate === "acceptance") {
+                for (const c of parseJudgeCriteria(g.data))
+                    judgeCriteria.push(c);
+            }
+        }
+        return {
+            taskId,
+            outcome: outcomeFor(events, taskId, runEnd),
+            gates,
+            judgeCriteria,
+        };
+    });
+    const packet = {
+        schemaVersion: BUNDLE_SCHEMA_VERSION,
+        runId,
+        ...(environment ? { environment } : {}),
+        contentHashes: {
+            journal: journalHash,
+            ...(gHash ? { graphDefinitionHash: gHash } : {}),
+        },
+        tasks,
+        knownLimits: [...KNOWN_LIMITS],
+    };
+    // Redact secret-shaped strings anywhere in the packet. Structured {path,line} citations stay intact.
+    return redactValue(packet);
+}

package/dist/report/compare.d.ts

@@ -0,0 +1,64 @@
+import type { RunEnvironment } from "../run/environment.js";
+import type { JournalEvent, TelemetryRow } from "../run/journal.js";
+import { type CostConfig } from "./cost.js";
+export type EnvironmentCompare = {
+    comparable: true;
+    recorded: string;
+} | {
+    comparable: false;
+    reason: "mismatch";
+    recorded: string;
+} | {
+    comparable: false;
+    reason: "unbound";
+};
+export declare function hasRunStart(events: JournalEvent[]): boolean;
+export declare function recordedEnvironment(events: JournalEvent[]): RunEnvironment | undefined;
+export declare function environmentComparable(baselineEnv: RunEnvironment | undefined, currentEnv: RunEnvironment | undefined): EnvironmentCompare;
+export interface RunMetrics {
+    durationMs: number | undefined;
+    gatePass: number;
+    gateFail: number;
+    gateTotal: number;
+    costUsd: number | undefined;
+    tokensTotal: number | undefined;
+}
+export declare function runMetrics(events: JournalEvent[], rows?: TelemetryRow[], cost?: CostConfig): RunMetrics;
+export interface RunDelta {
+    durationMs: number | undefined;
+    gateFail: number;
+    costUsd: number | undefined;
+    tokensTotal: number | undefined;
+}
+export type CompareOutcome = {
+    ok: false;
+    reason: string;
+} | {
+    ok: true;
+    runId: string;
+    baselineRunId: string;
+    comparability: EnvironmentCompare;
+    current: RunMetrics;
+    baseline: RunMetrics;
+    delta: RunDelta;
+    text: string;
+};
+export declare function renderComparison(opts: {
+    runId: string;
+    baselineRunId: string;
+    comparability: EnvironmentCompare;
+    baselineEnv: RunEnvironment | undefined;
+    currentEnv: RunEnvironment | undefined;
+    current: RunMetrics;
+    baseline: RunMetrics;
+    delta: RunDelta;
+}): string;
+export declare function compareRuns(opts: {
+    runId: string;
+    baselineRunId: string;
+    events: JournalEvent[];
+    baselineEvents: JournalEvent[];
+    rows?: TelemetryRow[];
+    baselineRows?: TelemetryRow[];
+    cost?: CostConfig;
+}): CompareOutcome;