tickmarkr 1.68.0 → 1.70.0

package/dist/eval/dispatch.js

@@ -0,0 +1,207 @@
+import { randomBytes } from "node:crypto";
+import { existsSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { channelKey, modelAuthed } from "../adapters/types.js";
+import { buildTaskPrompt } from "../adapters/prompt.js";
+import { compileSource } from "../compile/index.js";
+import { detectGateCommands } from "../gates/baseline.js";
+import { testFiltered } from "../gates/acceptance.js";
+import { renderAcceptanceItem } from "../graph/schema.js";
+import { sh } from "../run/git.js";
+import { seedFixture } from "./fixtures.js";
+function isDeterministic(item) {
+    return typeof item === "object" && (item.oracle === "command" || item.oracle === "test");
+}
+// Mirrors the vitest/jest summary parser in src/gates/acceptance.ts so the selfcheck uses the same
+// fail-closed rule for named-test oracles: exit 0 is vacuous if the name filter matched zero tests.
+function testsRan(output) {
+    const lines = output.replace(/\x1b\[[\d;#]*[A-Za-z]/g, "").split("\n");
+    for (let i = lines.length - 1; i >= 0; i--) {
+        const line = lines[i].trim();
+        const m = line.match(/^Tests\s+(.+?)\s*\(\d+\)\s*$/);
+        if (!m)
+            continue;
+        let ran = 0;
+        for (const chunk of m[1].split("|").map((s) => s.trim())) {
+            const n = chunk.match(/^(\d+)\s+(passed|failed)\b/);
+            if (n)
+                ran += Number(n[1]);
+        }
+        return ran;
+    }
+    return null;
+}
+function tail(out, n = 8) {
+    const t = out.trim();
+    if (!t)
+        return "";
+    return "\n" + t.split("\n").slice(-n).join("\n");
+}
+async function runDeterministicAcceptance(task, cwd, cfg) {
+    const testCmd = detectGateCommands(cwd, cfg).test;
+    const passed = [];
+    for (const item of task.acceptance) {
+        if (!isDeterministic(item))
+            continue;
+        if (item.oracle === "command") {
+            const r = await sh(item.command, cwd);
+            if (r.code !== 0) {
+                return {
+                    pass: false,
+                    details: `oracle failed: ${renderAcceptanceItem(item)} (exit ${r.code})${tail(r.stderr || r.stdout)}`,
+                };
+            }
+            passed.push(`✓ ${renderAcceptanceItem(item)} (exit 0)`);
+        }
+        else {
+            if (!testCmd) {
+                return {
+                    pass: false,
+                    details: `oracle failed: ${renderAcceptanceItem(item)} — no test command detected (failing closed)`,
+                };
+            }
+            const r = await sh(testFiltered(testCmd, item.test), cwd);
+            const out = (r.stderr || "") + "\n" + (r.stdout || "");
+            if (r.code !== 0) {
+                return {
+                    pass: false,
+                    details: `oracle failed: ${renderAcceptanceItem(item)} (exit ${r.code})${tail(r.stderr || r.stdout)}`,
+                };
+            }
+            const ran = testsRan(out);
+            if (ran === null || ran < 1) {
+                return {
+                    pass: false,
+                    details: `oracle failed: ${renderAcceptanceItem(item)} — name filter matched zero tests (filter: ${item.test})${tail(out)}`,
+                };
+            }
+            passed.push(`✓ ${renderAcceptanceItem(item)} (exit 0)`);
+        }
+    }
+    return {
+        pass: true,
+        details: passed.length ? passed.join("\n") : "no deterministic acceptance oracles",
+    };
+}
+function findSpec(fixtureDir) {
+    const direct = join(fixtureDir, "spec.md");
+    if (existsSync(direct))
+        return direct;
+    for (const ent of readdirSyncSafe(fixtureDir)) {
+        if (!ent.isFile())
+            continue;
+        const { name } = ent;
+        if (name.endsWith(".native.md") ||
+            name.endsWith(".prd.md") ||
+            name.endsWith(".spec.md") ||
+            name === "tasks.md") {
+            return join(fixtureDir, name);
+        }
+    }
+    return undefined;
+}
+function readdirSyncSafe(dir) {
+    try {
+        return readdirSync(dir, { withFileTypes: true });
+    }
+    catch {
+        return [];
+    }
+}
+function compileFixtureTask(fixture) {
+    const specPath = findSpec(fixture.path);
+    if (!specPath)
+        return { task: undefined, error: "fixture has no spec" };
+    try {
+        const graph = compileSource(specPath);
+        if (graph.tasks.length !== 1) {
+            return {
+                task: undefined,
+                error: `fixture spec must contain exactly one task (found ${graph.tasks.length})`,
+            };
+        }
+        const task = graph.tasks[0];
+        const runnable = task.acceptance.some(isDeterministic);
+        if (!runnable) {
+            return { task: undefined, error: "fixture has no deterministic acceptance oracle" };
+        }
+        return { task };
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        return { task: undefined, error: `fixture spec failed to compile: ${message}` };
+    }
+}
+function skipReason(channel, adapters, health) {
+    const adapter = adapters.find((a) => a.id === channel.adapter);
+    if (!adapter)
+        return `unknown adapter ${channel.adapter}`;
+    const h = health[channel.adapter];
+    if (!h?.installed)
+        return `adapter ${channel.adapter} is not installed`;
+    if (!h?.authed)
+        return `adapter ${channel.adapter} is not authenticated`;
+    if (!modelAuthed(h, channel.model)) {
+        const v = h.modelAuth?.[channel.model];
+        if (v?.authed === false)
+            return `${channel.adapter}:${channel.model} is not authenticated (${v.reason ?? "probe failed"})`;
+        return `${channel.adapter}:${channel.model} has no auth verdict — run tickmarkr doctor`;
+    }
+    return undefined;
+}
+async function dispatchChannel(task, channel, adapter, promptFile, nonce, fixture, cfg) {
+    const seeded = await seedFixture(fixture);
+    const assignment = {
+        adapter: channel.adapter,
+        model: channel.model,
+        channel: channel.channel,
+        tier: channel.tier,
+    };
+    const invocation = adapter.invoke(task, seeded.repo, assignment, { promptFile });
+    const run = await sh(invocation.command, seeded.repo, cfg.taskTimeoutMinutes * 60_000);
+    const output = `${run.stdout}\n${run.stderr}`.trim();
+    const worker = adapter.parse(output, nonce);
+    const acceptance = await runDeterministicAcceptance(task, seeded.repo, cfg);
+    return { repo: seeded.repo, worker, acceptance, seeded };
+}
+/**
+ * Render one prompt for a fixture and dispatch that identical, unmodified prompt to every channel
+ * under test. Each channel runs inside its own isolated seeded repository, and the fixture's own
+ * deterministic acceptance oracles are evaluated against each channel's resulting diff independently.
+ * Channels that fail install or model authentication are skipped with a recorded reason rather than
+ * dispatched. The dispatch path reuses the existing adapter `invoke` and `parse` contract.
+ */
+export async function dispatchFixture(opts) {
+    const { task, error } = compileFixtureTask(opts.fixture);
+    if (error || !task) {
+        throw new Error(error ?? "fixture task could not be compiled");
+    }
+    const nonce = randomBytes(8).toString("hex");
+    const promptDir = mkdtempSync(join(tmpdir(), "tickmarkr-eval-prompt-"));
+    const promptFile = join(promptDir, "prompt.md");
+    writeFileSync(promptFile, buildTaskPrompt(task, "", nonce));
+    const results = [];
+    try {
+        for (const channel of opts.channels) {
+            const key = channelKey(channel);
+            const reason = skipReason(channel, opts.adapters, opts.health);
+            if (reason) {
+                results.push({ channel, channelKey: key, skipped: true, skipReason: reason });
+                continue;
+            }
+            const adapter = opts.adapters.find((a) => a.id === channel.adapter);
+            const { seeded, ...rest } = await dispatchChannel(task, channel, adapter, promptFile, nonce, opts.fixture, opts.cfg);
+            try {
+                results.push({ channel, channelKey: key, skipped: false, repo: rest.repo, worker: rest.worker, acceptance: rest.acceptance });
+            }
+            finally {
+                await seeded.cleanup();
+            }
+        }
+    }
+    finally {
+        rmSync(promptDir, { recursive: true, force: true });
+    }
+    return results;
+}

package/dist/eval/fixtures.d.ts

@@ -0,0 +1,22 @@
+export interface Fixture {
+    id: string;
+    path: string;
+    startDir: string;
+    solutionDir: string;
+}
+export interface InvalidFixture {
+    id: string;
+    path: string;
+    reason: string;
+}
+export interface SeededFixture {
+    repo: string;
+    cleanup: () => Promise<void>;
+}
+export declare const FIXTURE_REQUIRED_PARTS: readonly ["start", "solution"];
+export declare function discoverFixtures(root: string): {
+    valid: Fixture[];
+    invalid: InvalidFixture[];
+};
+export declare function seedFixture(fixture: Fixture): Promise<SeededFixture>;
+export declare function resolveFixturesRoot(arg: string | undefined, cwd: string): string;

package/dist/eval/fixtures.js

@@ -0,0 +1,85 @@
+import { existsSync, lstatSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync, } from "node:fs";
+import { tmpdir } from "node:os";
+import { isAbsolute, join, relative } from "node:path";
+import { shGitOk } from "../run/git.js";
+export const FIXTURE_REQUIRED_PARTS = ["start", "solution"];
+function isFixtureDir(dir) {
+    for (const part of FIXTURE_REQUIRED_PARTS) {
+        const p = join(dir, part);
+        if (!existsSync(p) || !lstatSync(p).isDirectory())
+            return false;
+    }
+    return true;
+}
+function missingPart(dir) {
+    for (const part of FIXTURE_REQUIRED_PARTS) {
+        const p = join(dir, part);
+        if (!existsSync(p) || !lstatSync(p).isDirectory())
+            return part;
+    }
+    return undefined;
+}
+function copyDirContents(src, dst) {
+    for (const ent of readdirSync(src, { withFileTypes: true })) {
+        const from = join(src, ent.name);
+        const to = join(dst, ent.name);
+        if (ent.isDirectory()) {
+            mkdirSync(to, { recursive: true });
+            copyDirContents(from, to);
+        }
+        else {
+            writeFileSync(to, readFileSync(from));
+        }
+    }
+}
+export function discoverFixtures(root) {
+    const valid = [];
+    const invalid = [];
+    if (!existsSync(root) || !lstatSync(root).isDirectory()) {
+        return { valid, invalid };
+    }
+    for (const ent of readdirSync(root, { withFileTypes: true })) {
+        if (!ent.isDirectory())
+            continue;
+        const dir = join(root, ent.name);
+        if (isFixtureDir(dir)) {
+            valid.push({
+                id: relative(root, dir),
+                path: dir,
+                startDir: join(dir, "start"),
+                solutionDir: join(dir, "solution"),
+            });
+        }
+        else {
+            const part = missingPart(dir);
+            invalid.push({
+                id: relative(root, dir),
+                path: dir,
+                reason: part ? `missing required part: ${part}` : "missing required part",
+            });
+        }
+    }
+    valid.sort((a, b) => a.id.localeCompare(b.id));
+    invalid.sort((a, b) => a.id.localeCompare(b.id));
+    return { valid, invalid };
+}
+export async function seedFixture(fixture) {
+    const repo = mkdtempSync(join(tmpdir(), "tickmarkr-eval-"));
+    await shGitOk("git init -b main", repo);
+    await shGitOk("git config user.email tickmarkr@eval.local", repo);
+    await shGitOk("git config user.name tickmarkr-eval", repo);
+    // Copy only the starting files; the reference solution must never be seeded.
+    copyDirContents(fixture.startDir, repo);
+    await shGitOk("git add -A", repo);
+    await shGitOk("git commit -m init --no-gpg-sign", repo);
+    return {
+        repo,
+        cleanup: async () => {
+            rmSync(repo, { recursive: true, force: true });
+        },
+    };
+}
+export function resolveFixturesRoot(arg, cwd) {
+    const root = arg ?? "fixtures/eval";
+    return isAbsolute(root) ? root : join(cwd, root);
+}

package/dist/eval/report.d.ts

@@ -0,0 +1,35 @@
+import type { BillingChannel, WorkerResult } from "../adapters/types.js";
+import type { AcceptanceRunResult, ChannelResult } from "./dispatch.js";
+import type { Fixture } from "./fixtures.js";
+export interface ReportedResult {
+    runAt: string;
+    fixture: {
+        id: string;
+        revision: string;
+    };
+    channel: BillingChannel & {
+        channelKey: string;
+    };
+    skipped: boolean;
+    skipReason?: string;
+    worker?: WorkerResult;
+    acceptance?: AcceptanceRunResult;
+}
+/**
+ * Deterministic content identity for a fixture revision. Hashes every file under
+ * the fixture root (start + solution) so any change to the fixture's contents
+ * changes the revision digest.
+ */
+export declare function fixtureRevisionHash(fixture: Fixture): string;
+/**
+ * Append a single fixture-channel result to the incremental JSON output file.
+ * The write is synchronous so the bytes are on disk before the caller moves on
+ * to the next channel.
+ */
+export declare function appendChannelResult(path: string, fixture: Fixture, result: ChannelResult, runAt?: string): void;
+/**
+ * Read every parseable line from an incremental JSON report. Torn or corrupt
+ * trailing lines are ignored, so an interrupted run still yields every result
+ * that was fully written.
+ */
+export declare function readReport(path: string): ReportedResult[];

package/dist/eval/report.js

@@ -0,0 +1,82 @@
+import { createHash } from "node:crypto";
+import { appendFileSync, existsSync, mkdirSync, readdirSync, readFileSync } from "node:fs";
+import { dirname, relative } from "node:path";
+const EXCLUDED_NAMES = new Set([".git", "node_modules", ".DS_Store"]);
+/**
+ * Deterministic content identity for a fixture revision. Hashes every file under
+ * the fixture root (start + solution) so any change to the fixture's contents
+ * changes the revision digest.
+ */
+export function fixtureRevisionHash(fixture) {
+    const hash = createHash("sha256");
+    function walk(dir) {
+        const entries = readdirSync(dir, { withFileTypes: true });
+        entries.sort((a, b) => a.name.localeCompare(b.name));
+        for (const ent of entries) {
+            if (EXCLUDED_NAMES.has(ent.name))
+                continue;
+            const full = `${dir}/${ent.name}`;
+            const rel = relative(fixture.path, full);
+            if (ent.isDirectory()) {
+                walk(full);
+            }
+            else if (ent.isFile()) {
+                hash.update(`${rel}\0`);
+                hash.update(readFileSync(full));
+                hash.update("\0");
+            }
+        }
+    }
+    if (existsSync(fixture.path)) {
+        walk(fixture.path);
+    }
+    return hash.digest("hex");
+}
+/**
+ * Append a single fixture-channel result to the incremental JSON output file.
+ * The write is synchronous so the bytes are on disk before the caller moves on
+ * to the next channel.
+ */
+export function appendChannelResult(path, fixture, result, runAt) {
+    const dir = dirname(path);
+    if (!existsSync(dir)) {
+        mkdirSync(dir, { recursive: true });
+    }
+    const row = {
+        runAt: runAt ?? new Date().toISOString(),
+        fixture: {
+            id: fixture.id,
+            revision: fixtureRevisionHash(fixture),
+        },
+        channel: {
+            ...result.channel,
+            channelKey: result.channelKey,
+        },
+        skipped: result.skipped,
+        ...(result.skipReason !== undefined ? { skipReason: result.skipReason } : {}),
+        ...(result.worker !== undefined ? { worker: result.worker } : {}),
+        ...(result.acceptance !== undefined ? { acceptance: result.acceptance } : {}),
+    };
+    appendFileSync(path, JSON.stringify(row) + "\n");
+}
+/**
+ * Read every parseable line from an incremental JSON report. Torn or corrupt
+ * trailing lines are ignored, so an interrupted run still yields every result
+ * that was fully written.
+ */
+export function readReport(path) {
+    if (!existsSync(path))
+        return [];
+    const out = [];
+    for (const line of readFileSync(path, "utf8").split("\n")) {
+        if (!line.trim())
+            continue;
+        try {
+            out.push(JSON.parse(line));
+        }
+        catch {
+            // torn trailing write after an interrupt — ignore; earlier lines are intact
+        }
+    }
+    return out;
+}

package/dist/eval/selfcheck.d.ts

@@ -0,0 +1,22 @@
+import type { Fixture } from "./fixtures.js";
+export interface SelfcheckRunResult {
+    pass: boolean;
+    details: string;
+}
+export interface SelfcheckResult {
+    fixtureId: string;
+    specPath: string;
+    start: SelfcheckRunResult;
+    solution: SelfcheckRunResult;
+    valid: boolean;
+    invalidReason?: string;
+}
+/**
+ * Run the fixture selfcheck: compile the fixture's own spec and run its deterministic acceptance
+ * oracles (command/test) against both the starting tree and the reference solution tree.
+ *
+ * The fixture is valid only when the starting tree fails the acceptance check and the reference
+ * solution tree passes the identical check — mechanizing the starting-state-must-fail principle
+ * already applied to shell-runnable oracles.
+ */
+export declare function runSelfcheck(fixture: Fixture): Promise<SelfcheckResult>;

package/dist/eval/selfcheck.js

@@ -0,0 +1,177 @@
+import { existsSync, readdirSync } from "node:fs";
+import { join } from "node:path";
+import { DEFAULT_CONFIG } from "../config/config.js";
+import { compileSource } from "../compile/index.js";
+import { detectGateCommands } from "../gates/baseline.js";
+import { testFiltered } from "../gates/acceptance.js";
+import { renderAcceptanceItem } from "../graph/schema.js";
+import { sh } from "../run/git.js";
+function findSpec(fixtureDir) {
+    const direct = join(fixtureDir, "spec.md");
+    if (existsSync(direct))
+        return direct;
+    for (const ent of readdirSync(fixtureDir, { withFileTypes: true })) {
+        if (!ent.isFile())
+            continue;
+        const { name } = ent;
+        if (name.endsWith(".native.md") ||
+            name.endsWith(".prd.md") ||
+            name.endsWith(".spec.md") ||
+            name === "tasks.md") {
+            return join(fixtureDir, name);
+        }
+    }
+    return undefined;
+}
+function isDeterministic(item) {
+    return typeof item === "object" && (item.oracle === "command" || item.oracle === "test");
+}
+// Mirrors the vitest/jest summary parser in src/gates/acceptance.ts so the selfcheck uses the same
+// fail-closed rule for named-test oracles: exit 0 is vacuous if the name filter matched zero tests.
+function testsRan(output) {
+    const lines = output.replace(/\x1b\[[\d;#]*[A-Za-z]/g, "").split("\n");
+    for (let i = lines.length - 1; i >= 0; i--) {
+        const line = lines[i].trim();
+        const m = line.match(/^Tests\s+(.+?)\s*\(\d+\)\s*$/);
+        if (!m)
+            continue;
+        let ran = 0;
+        for (const chunk of m[1].split("|").map((s) => s.trim())) {
+            const n = chunk.match(/^(\d+)\s+(passed|failed)\b/);
+            if (n)
+                ran += Number(n[1]);
+        }
+        return ran;
+    }
+    return null;
+}
+function tail(out, n = 8) {
+    const t = out.trim();
+    if (!t)
+        return "";
+    return "\n" + t.split("\n").slice(-n).join("\n");
+}
+async function runDeterministicAcceptance(task, cwd) {
+    const testCmd = detectGateCommands(cwd, DEFAULT_CONFIG).test;
+    const passed = [];
+    for (const item of task.acceptance) {
+        if (!isDeterministic(item))
+            continue;
+        if (item.oracle === "command") {
+            const r = await sh(item.command, cwd);
+            if (r.code !== 0) {
+                return {
+                    pass: false,
+                    details: `oracle failed: ${renderAcceptanceItem(item)} (exit ${r.code})${tail(r.stderr || r.stdout)}`,
+                };
+            }
+            passed.push(`✓ ${renderAcceptanceItem(item)} (exit 0)`);
+        }
+        else {
+            if (!testCmd) {
+                return {
+                    pass: false,
+                    details: `oracle failed: ${renderAcceptanceItem(item)} — no test command detected (failing closed)`,
+                };
+            }
+            const r = await sh(testFiltered(testCmd, item.test), cwd);
+            const out = (r.stderr || "") + "\n" + (r.stdout || "");
+            if (r.code !== 0) {
+                return {
+                    pass: false,
+                    details: `oracle failed: ${renderAcceptanceItem(item)} (exit ${r.code})${tail(r.stderr || r.stdout)}`,
+                };
+            }
+            const ran = testsRan(out);
+            if (ran === null || ran < 1) {
+                return {
+                    pass: false,
+                    details: `oracle failed: ${renderAcceptanceItem(item)} — name filter matched zero tests (filter: ${item.test})${tail(out)}`,
+                };
+            }
+            passed.push(`✓ ${renderAcceptanceItem(item)} (exit 0)`);
+        }
+    }
+    return {
+        pass: true,
+        details: passed.length ? passed.join("\n") : "no deterministic acceptance oracles",
+    };
+}
+/**
+ * Run the fixture selfcheck: compile the fixture's own spec and run its deterministic acceptance
+ * oracles (command/test) against both the starting tree and the reference solution tree.
+ *
+ * The fixture is valid only when the starting tree fails the acceptance check and the reference
+ * solution tree passes the identical check — mechanizing the starting-state-must-fail principle
+ * already applied to shell-runnable oracles.
+ */
+export async function runSelfcheck(fixture) {
+    const specPath = findSpec(fixture.path);
+    if (!specPath) {
+        return {
+            fixtureId: fixture.id,
+            specPath: "",
+            start: { pass: false, details: "no spec found" },
+            solution: { pass: false, details: "no spec found" },
+            valid: false,
+            invalidReason: "fixture has no spec (expected spec.md or a recognized spec file)",
+        };
+    }
+    let task;
+    try {
+        const graph = compileSource(specPath);
+        if (graph.tasks.length !== 1) {
+            return {
+                fixtureId: fixture.id,
+                specPath,
+                start: { pass: false, details: "spec has more than one task" },
+                solution: { pass: false, details: "spec has more than one task" },
+                valid: false,
+                invalidReason: `fixture spec must contain exactly one task (found ${graph.tasks.length})`,
+            };
+        }
+        task = graph.tasks[0];
+        const runnable = task.acceptance.some(isDeterministic);
+        if (!runnable) {
+            return {
+                fixtureId: fixture.id,
+                specPath,
+                start: { pass: false, details: "no deterministic oracles" },
+                solution: { pass: false, details: "no deterministic oracles" },
+                valid: false,
+                invalidReason: "fixture has no shell-runnable acceptance oracle (command: or test:)",
+            };
+        }
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        return {
+            fixtureId: fixture.id,
+            specPath,
+            start: { pass: false, details: message },
+            solution: { pass: false, details: message },
+            valid: false,
+            invalidReason: `fixture spec failed to compile: ${message}`,
+        };
+    }
+    const start = await runDeterministicAcceptance(task, fixture.startDir);
+    const solution = await runDeterministicAcceptance(task, fixture.solutionDir);
+    let valid = true;
+    let invalidReason;
+    if (start.pass) {
+        valid = false;
+        invalidReason = "starting tree already passes the acceptance check";
+    }
+    else if (!solution.pass) {
+        valid = false;
+        invalidReason = "reference tree still fails the acceptance check";
+    }
+    return {
+        fixtureId: fixture.id,
+        specPath,
+        start,
+        solution,
+        valid,
+        invalidReason,
+    };
+}

package/dist/gates/acceptance.d.ts

@@ -2,13 +2,17 @@ import { type WorkerAdapter } from "../adapters/types.js";
 import { type Task } from "../graph/schema.js";
 import { type LlmVia } from "./llm.js";
 import type { GateResult } from "./types.js";
+export interface EvidenceCitation {
+    path: string;
+    line: number;
+}
 export interface JudgeVerdict {
     pass: boolean;
     criteria: Array<{
         criterion: string;
         met: boolean;
         reason: string;
-        evidence: string;
+        evidence: string | EvidenceCitation;
     }>;
 }
 export declare function judgeCriterionId(index: number): string;