scriptonia 0.7.0 → 0.9.0-rc.1

package/bin/verify.mjs

@@ -0,0 +1,266 @@
+import fs from "node:fs";
+import path from "node:path";
+import { createHash } from "node:crypto";
+import { spawnSync } from "node:child_process";
+
+const RANK = { PASS: 0, WARN: 1, BLOCK: 2 };
+const CHECK_SCRIPTS = ["test", "build", "lint", "typecheck", "type-check"];
+
+export async function runVerification(args, { cfg, cwd = process.cwd(), fetchImpl = fetch } = {}) {
+  if (!cfg?.url || !cfg?.token) throw new Error("run `npx scriptonia login` first");
+  const flags = parseFlags(args);
+  const planPath = path.resolve(cwd, flags.plan ?? "PLAN.md");
+  if (!fs.existsSync(planPath)) throw new Error(`PLAN.md not found at ${planPath} (use --plan <file>)`);
+
+  const plan = fs.readFileSync(planPath, "utf8");
+  const meta = parsePlanMetadata(plan, planPath);
+  const { diff, files, source } = collectDiff(cwd, flags);
+  if (!diff.trim()) throw new Error("no git diff to verify (use --base <ref> or --diff <file>)");
+
+  const diffForJudge = diff.length > 290_000
+    ? `${diff.slice(0, 290_000)}\n\n[diff truncated by CLI; verdicts requiring omitted evidence must WARN]`
+    : diff;
+  const semanticPromise = fetchImpl(`${cfg.url}/api/verify`, {
+    method: "POST",
+    headers: { authorization: `Bearer ${cfg.token}`, "content-type": "application/json" },
+    body: JSON.stringify({ plan, diff: diffForJudge }),
+  });
+  const repositoryChecks = flags["skip-checks"] ? skippedRepositoryGate() : runRepositoryChecks(cwd, files);
+  const unresolvedConstraints = buildUnresolvedGate(plan);
+  const testCoverage = buildTestCoverageGate(files);
+
+  const response = await semanticPromise;
+  const semantic = await response.json().catch(() => ({}));
+  if (!response.ok) throw new Error(`verification service failed (${response.status}): ${JSON.stringify(semantic).slice(0, 240)}`);
+  for (const name of ["acceptance_criteria", "non_goals", "product_decisions"]) {
+    const result = semantic[name];
+    if (!result || !(result.verdict in RANK) || !Array.isArray(result.checks)) {
+      throw new Error(`verification service returned an invalid ${name} gate`);
+    }
+  }
+
+  const gates = {
+    acceptance_criteria: semantic.acceptance_criteria,
+    non_goals: semantic.non_goals,
+    product_decisions: semantic.product_decisions,
+    unresolved_constraints: unresolvedConstraints,
+    repository_checks: repositoryChecks,
+    test_coverage: testCoverage,
+  };
+  const verdict = worstVerdict(Object.values(gates).map((gate) => gate.verdict));
+  const evalPath = path.resolve(cwd, flags.out ?? path.join(".scriptonia", "evals", `${meta.slug}.yaml`));
+  fs.mkdirSync(path.dirname(evalPath), { recursive: true });
+  fs.writeFileSync(evalPath, renderEvalYaml({
+    id: meta.slug,
+    plan: path.relative(cwd, planPath) || path.basename(planPath),
+    planVersion: meta.version,
+    diffSource: source,
+    diffSha256: createHash("sha256").update(diff).digest("hex"),
+    verdict,
+    gates,
+  }));
+
+  printResult({ verdict, gates, evalPath, cwd });
+  return { verdict, exitCode: verdict === "BLOCK" ? 1 : 0, evalPath, gates };
+}
+
+export function parsePlanMetadata(plan, planPath = "PLAN.md") {
+  const frontmatter = plan.match(/^---\s*\n([\s\S]*?)\n---/m)?.[1] ?? "";
+  const value = (key) => frontmatter.match(new RegExp(`^${key}:\\s*(.+)$`, "m"))?.[1]?.trim().replace(/^['"]|['"]$/g, "");
+  const slug = value("plan") || slugify(plan.match(/^#\s+(.+)$/m)?.[1] || path.basename(planPath, path.extname(planPath)));
+  return { slug: slugify(slug), version: Number(value("version") || 1) };
+}
+
+export function buildUnresolvedGate(plan) {
+  const matches = plan.split("\n").filter((line) => /(?:⚠\s*)?UNRESOLVED/i.test(line));
+  const blocked = matches.length > 0;
+  return gate(blocked ? "BLOCK" : "PASS", {
+    evidence: blocked ? matches.slice(0, 6) : ["No UNRESOLVED marker appears in PLAN.md."],
+    plan_criterion: "Every unresolved constraint requires human approval before implementation.",
+    citation: decisionCitation(matches.join(" ")),
+    reason: blocked ? "PLAN.md still contains an unresolved approval gate." : "No unresolved approval gate remains in the approved plan.",
+  });
+}
+
+export function buildTestCoverageGate(files) {
+  const tests = files.filter(isTestFile);
+  const behavior = files.filter((file) => isSourceFile(file) && !isTestFile(file));
+  if (tests.length) return gate("PASS", {
+    evidence: tests,
+    plan_criterion: "Add or update tests for changed product behavior.",
+    citation: "PLAN.md test plan",
+    reason: "The diff includes test changes that a reviewer can inspect.",
+  });
+  if (behavior.length) return gate("BLOCK", {
+    evidence: behavior.slice(0, 8),
+    plan_criterion: "Add or update tests for changed product behavior.",
+    citation: "PLAN.md test plan",
+    reason: "Product code changed without any test file added or updated.",
+  });
+  return gate("PASS", {
+    evidence: files.length ? files.slice(0, 8) : ["No changed source files."],
+    plan_criterion: "Add or update tests for changed product behavior.",
+    citation: "PLAN.md test plan",
+    reason: "The diff contains no product-code change requiring test coverage.",
+  });
+}
+
+function runRepositoryChecks(cwd, files) {
+  const checks = [];
+  for (const root of relevantPackageRoots(cwd, files)) {
+    let pkg;
+    try { pkg = JSON.parse(fs.readFileSync(path.join(root, "package.json"), "utf8")); } catch { continue; }
+    const manager = packageManager(root);
+    const seen = new Set();
+    for (const name of CHECK_SCRIPTS) {
+      if (!pkg.scripts?.[name] || seen.has(pkg.scripts[name])) continue;
+      seen.add(pkg.scripts[name]);
+      const command = manager === "yarn" ? ["yarn", name] : manager === "pnpm" ? ["pnpm", "run", name] : manager === "bun" ? ["bun", "run", name] : ["npm", "run", name];
+      const result = spawnSync(command[0], command.slice(1), { cwd: root, encoding: "utf8", timeout: 300_000, maxBuffer: 2_000_000, env: { ...process.env, CI: "1" } });
+      const output = `${result.stdout ?? ""}\n${result.stderr ?? ""}`.trim();
+      checks.push({ name, command: command.join(" "), root: path.relative(cwd, root) || ".", ok: result.status === 0, output: output.slice(-1200) || `(exit ${result.status ?? "unknown"})` });
+    }
+  }
+  if (!checks.length) return gate("WARN", {
+    evidence: ["No test, build, lint, or type-check script was found for the changed packages."],
+    plan_criterion: "Run the repository's existing quality checks.",
+    citation: "repository package scripts",
+    reason: "There were no recognized repository checks to run automatically.",
+  });
+  const failed = checks.filter((check) => !check.ok);
+  return {
+    verdict: failed.length ? "BLOCK" : "PASS",
+    checks: checks.map((check) => ({
+      verdict: check.ok ? "PASS" : "BLOCK",
+      evidence: [`${check.root}: ${check.command}`, check.output],
+      plan_criterion: "Run the repository's existing quality checks.",
+      citation: "repository package scripts",
+      reason: check.ok ? `${check.name} completed successfully.` : `${check.name} failed and must be fixed before merge.`,
+    })),
+  };
+}
+
+function skippedRepositoryGate() {
+  return gate("WARN", {
+    evidence: ["Repository checks skipped with --skip-checks."],
+    plan_criterion: "Run the repository's existing quality checks.",
+    citation: "CLI flag",
+    reason: "CI should run the repository checks separately before merge.",
+  });
+}
+
+function relevantPackageRoots(cwd, files) {
+  const candidates = [];
+  if (fs.existsSync(path.join(cwd, "package.json"))) candidates.push(cwd);
+  for (const entry of fs.readdirSync(cwd, { withFileTypes: true })) {
+    if (entry.isDirectory() && !entry.name.startsWith(".") && fs.existsSync(path.join(cwd, entry.name, "package.json"))) candidates.push(path.join(cwd, entry.name));
+  }
+  const relevant = candidates.filter((root) => {
+    const prefix = path.relative(cwd, root);
+    return !prefix || files.some((file) => file === prefix || file.startsWith(prefix + "/"));
+  });
+  return relevant.length ? relevant : candidates.length === 1 ? candidates : [];
+}
+
+function packageManager(root) {
+  if (fs.existsSync(path.join(root, "pnpm-lock.yaml"))) return "pnpm";
+  if (fs.existsSync(path.join(root, "yarn.lock"))) return "yarn";
+  if (fs.existsSync(path.join(root, "bun.lockb")) || fs.existsSync(path.join(root, "bun.lock"))) return "bun";
+  return "npm";
+}
+
+function collectDiff(cwd, flags) {
+  if (flags.diff) {
+    const file = path.resolve(cwd, flags.diff);
+    const diff = fs.readFileSync(file, "utf8");
+    return { diff, files: diffFiles(diff), source: `file:${path.relative(cwd, file)}` };
+  }
+  const base = flags.base || (process.env.GITHUB_BASE_REF ? `origin/${process.env.GITHUB_BASE_REF}` : null);
+  const range = base ? `${base}...HEAD` : "HEAD";
+  const result = spawnSync("git", ["diff", range, "--no-ext-diff", "--unified=40"], { cwd, encoding: "utf8", maxBuffer: 20_000_000 });
+  if (result.status !== 0) throw new Error(`could not read git diff for ${range}: ${(result.stderr || "").trim()}`);
+  let diff = result.stdout || "";
+  if (!base) diff += untrackedDiff(cwd);
+  return { diff, files: diffFiles(diff), source: `git:${range}` };
+}
+
+function untrackedDiff(cwd) {
+  const status = spawnSync("git", ["status", "--porcelain", "--untracked-files=all"], { cwd, encoding: "utf8" });
+  if (status.status !== 0) return "";
+  const files = status.stdout.split("\n").filter((line) => line.startsWith("?? ")).map((line) => line.slice(3));
+  const chunks = [];
+  for (const file of files) {
+    const full = path.join(cwd, file);
+    try {
+      const stat = fs.statSync(full);
+      if (!stat.isFile() || stat.size > 500_000) continue;
+      const body = fs.readFileSync(full, "utf8");
+      chunks.push(`\ndiff --git a/${file} b/${file}\nnew file mode 100644\n--- /dev/null\n+++ b/${file}\n@@ -0,0 +1,${body.split("\n").length} @@\n${body.split("\n").map((line) => `+${line}`).join("\n")}\n`);
+    } catch { /* ignore unreadable/binary untracked files */ }
+  }
+  return chunks.join("");
+}
+
+function diffFiles(diff) {
+  return [...new Set([...diff.matchAll(/^diff --git a\/(.+?) b\/(.+)$/gm)].map((match) => match[2]))];
+}
+
+function isTestFile(file) {
+  return /(^|\/)(?:__tests__|tests?|spec)(?:\/|$)|\.(?:test|spec)\.[^.]+$/i.test(file);
+}
+
+function isSourceFile(file) {
+  return /\.(?:[cm]?[jt]sx?|py|go|rs|java|rb|php|swift|kt)$/i.test(file) && !/(?:^|\/)(?:scripts?|migrations?|drizzle)(?:\/|$)/i.test(file);
+}
+
+function gate(verdict, check) {
+  return { verdict, checks: [{ verdict, ...check }] };
+}
+
+function worstVerdict(verdicts) {
+  return verdicts.reduce((worst, verdict) => (RANK[verdict] > RANK[worst] ? verdict : worst), "PASS");
+}
+
+function decisionCitation(text) {
+  return text.match(/(?:decision|dec)[-_:/ ]?[a-z0-9_-]+/i)?.[0] || "PLAN.md constraint";
+}
+
+function renderEvalYaml(data) {
+  const q = (value) => JSON.stringify(String(value ?? ""));
+  const lines = ["version: 1", `id: ${q(data.id)}`, `plan: ${q(data.plan)}`, `plan_version: ${data.planVersion}`, `diff_source: ${q(data.diffSource)}`, `diff_sha256: ${q(data.diffSha256)}`, `generated_at: ${q(new Date().toISOString())}`, `verdict: ${data.verdict}`, "gates:"];
+  for (const [name, gateResult] of Object.entries(data.gates)) {
+    lines.push(`  ${name}:`, `    verdict: ${gateResult.verdict}`, "    checks:");
+    for (const check of gateResult.checks ?? []) {
+      lines.push(`      - verdict: ${check.verdict}`, `        plan_criterion: ${q(check.plan_criterion)}`, `        citation: ${q(check.citation)}`, `        reason: ${q(check.reason)}`, "        evidence:");
+      for (const evidence of check.evidence ?? []) lines.push(`          - ${q(String(evidence).slice(0, 1600))}`);
+    }
+  }
+  return lines.join("\n") + "\n";
+}
+
+function printResult({ verdict, gates, evalPath, cwd }) {
+  const icon = verdict === "PASS" ? "✓" : verdict === "WARN" ? "!" : "✕";
+  console.log(`\n  ${icon} SCRIPTONIA VERIFY · ${verdict}\n`);
+  for (const [name, result] of Object.entries(gates)) {
+    console.log(`  ${result.verdict.padEnd(5)}  ${name.replaceAll("_", " ")}`);
+    for (const check of (result.checks ?? []).filter((item) => item.verdict !== "PASS").slice(0, 3)) console.log(`         ${check.reason}`);
+  }
+  console.log(`\n  case   ${path.relative(cwd, evalPath)}`);
+  console.log(`  exit   ${verdict === "BLOCK" ? 1 : 0}\n`);
+}
+
+function parseFlags(args) {
+  const out = {};
+  for (let i = 0; i < args.length; i++) {
+    if (!args[i].startsWith("--")) continue;
+    const [name, inline] = args[i].slice(2).split(/=(.*)/s);
+    if (inline !== undefined) out[name] = inline;
+    else if (args[i + 1] && !args[i + 1].startsWith("--")) out[name] = args[++i];
+    else out[name] = true;
+  }
+  return out;
+}
+
+function slugify(value) {
+  return String(value).toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/(^-|-$)/g, "").slice(0, 80) || "plan";
+}