replume-cli 0.2.0 → 0.2.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "replume-cli",
-  "version": "0.2.0",
+  "version": "0.2.3",
   "type": "module",
   "bin": {
     "replume": "dist/cli.js"
@@ -10,7 +10,8 @@
     "test": "vitest run",
     "typecheck": "tsc --noEmit",
     "check-context": "node scripts/check-context.mjs",
-    "prepublishOnly": "npm run build && npm run typecheck && npm run check-context && npm test"
+    "prepublishOnly": "npm run build && npm run typecheck && npm run check-context && npm test",
+    "eval": "node eval/run.mjs"
   },
   "dependencies": {
     "commander": "^12.1.0"
@@ -20,7 +21,7 @@
     "tsup": "^8.3.0",
     "tsx": "^4.23.12",
     "typescript": "^5.6.0",
-    "vitest": "^2.1.0"
+    "vitest": "^4.1.12"
   },
   "description": "Verification and supervision for AI coding agents",
   "keywords": [
@@ -34,9 +35,14 @@
   },
   "files": [
     "dist/cli.js",
+    "templates/pr-check.yml",
+    "templates/pr-check-publish.mjs",
     "README.md",
     "LICENSE",
     "THIRD-PARTY-NOTICES.md"
   ],
-  "license": "SEE LICENSE IN LICENSE"
+  "license": "SEE LICENSE IN LICENSE",
+  "overrides": {
+    "esbuild": "^0.28.2"
+  }
 }

package/templates/pr-check-publish.mjs

@@ -0,0 +1,125 @@
+// Copy beside the workflow as .github/replume-pr-check-publish.mjs on the trusted base branch.
+// The publisher reads the verify artifact; --check-policy reads committed policy from the PR base.
+import { execFileSync } from "node:child_process";
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+
+const marker = "<!-- replume-pr-check -->";
+const text = (value) => String(value ?? "").replace(/[\r\n\t]+/g, " ").replace(/[<>`]/g, "").slice(0, 350);
+const items = (entries) => entries.length ? entries.map((entry) => `- ${entry}`).join("\n") : "- None recorded.";
+const contractPath = ".replume/contract.yaml";
+
+// Replume's contract writer emits a flow or indented list. Fail closed if the base has no non-empty protect list.
+function hasProtectedTests(source) {
+  const lines = source.split(/\r?\n/);
+  const index = lines.findIndex((line) => /^protect:(?:\s|$)/.test(line));
+  if (index < 0) return false;
+  const value = lines[index].slice("protect:".length).replace(/\s+#.*$/, "").trim();
+  if (value) return /^\[.*\]$/.test(value) && value.slice(1, -1).trim().length > 0;
+  for (const line of lines.slice(index + 1)) {
+    const content = line.replace(/\s+#.*$/, "").trimEnd();
+    if (!content.trim()) continue;
+    if (!/^\s/.test(content)) break;
+    if (/^\s+-\s+\S/.test(content)) return true;
+  }
+  return false;
+}
+
+export function checkBasePolicy(baseSha, repo = process.cwd()) {
+  if (!/^[0-9a-f]{40}$/i.test(baseSha ?? "")) throw new Error("PR base SHA must be a full commit hash");
+  let source;
+  try {
+    source = execFileSync("git", ["show", `${baseSha}:${contractPath}`], { cwd: repo, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] });
+  } catch {
+    return `Commit ${contractPath} with a confirmed task and non-empty protect test globs to the PR base branch.`;
+  }
+  return hasProtectedTests(source) ? null : `Commit non-empty protect test globs in ${contractPath} on the PR base branch.`;
+}
+
+function result(reportPath, exitPath, policyPath = join(dirname(reportPath), "policy.txt")) {
+  let policyNote = "";
+  try { if (existsSync(policyPath)) policyNote = readFileSync(policyPath, "utf8").trim().split(/\r?\n/)[0]; } catch { /* incomplete below */ }
+  if (policyNote) return { report: null, verdict: "INCOMPLETE", reason: policyNote };
+  let report = null;
+  try { if (existsSync(reportPath)) report = JSON.parse(readFileSync(reportPath, "utf8")); } catch { /* incomplete below */ }
+  let exit = "";
+  try { if (existsSync(exitPath)) exit = readFileSync(exitPath, "utf8").trim(); } catch { /* incomplete below */ }
+  const expected = report?.verdict === "pass" ? "0" : report?.verdict === "fail" ? "1" : report?.verdict === "incomplete" ? "2" : "";
+  const verdict = expected && exit === expected ? report.verdict.toUpperCase() : "INCOMPLETE";
+  return { report: report && typeof report === "object" ? report : null, verdict,
+    reason: verdict === "INCOMPLETE" && expected !== exit ? "Verification did not produce a usable, matching result." : "" };
+}
+
+export function commentBody(reportPath, exitPath, policyPath) {
+  const { report, verdict, reason } = result(reportPath, exitPath, policyPath);
+  const done = Array.isArray(report?.done) ? report.done : [];
+  const failed = Array.isArray(report?.failed) ? report.failed : [];
+  const extra = Array.isArray(report?.extra) ? report.extra : [];
+  const tampering = Array.isArray(report?.tampering) ? report.tampering : [];
+  const notChecked = Array.isArray(report?.notChecked) ? report.notChecked : [];
+  const verified = done.map((entry) => `${text(entry.id)}: ${text(entry.evidence)}`);
+  const failures = failed.map((entry) => `${text(entry.id)}: ${text(entry.evidence)}`);
+  const beyond = [
+    ...extra.map((entry) => `${text(entry.path)}: ${text(entry.reason)}${entry.informational ? " (information only)" : ""}`),
+    ...tampering.map((entry) => `${text(entry.path)}: ${text(entry.detail)}`),
+  ];
+  const unchecked = [
+    ...notChecked.map((entry) => `${text(entry.path)}: ${text(entry.reason)}`),
+    "CI did not observe agent steps; this result covers the commit diff and checks run here.",
+  ];
+  if (reason) unchecked.unshift(reason);
+  return [marker, `**Replume: ${verdict}${reason ? ` — ${text(reason)}` : ""}**`, "",
+    "## Verified", items(verified), "",
+    ...(failures.length ? ["## Failed", items(failures), ""] : []),
+    "## Changed beyond the task", items(beyond), "",
+    "## Not checked", items(unchecked), "",
+    "_Tests ran in an isolated working copy. This comment was made from the saved CI result; it did not rerun verification._",
+  ].join("\n");
+}
+
+async function api(path, method, token, body) {
+  const base = (process.env.GITHUB_API_URL || "https://api.github.com").replace(/\/$/, "");
+  const response = await fetch(`${base}${path}`, {
+    method, signal: AbortSignal.timeout(15_000),
+    headers: { authorization: `Bearer ${token}`, accept: "application/vnd.github+json", "content-type": "application/json" },
+    ...(body === undefined ? {} : { body: JSON.stringify(body) }),
+  });
+  if (!response.ok) throw new Error(`GitHub ${method} ${path} returned HTTP ${response.status}`);
+  return response.json();
+}
+
+export async function publish(env = process.env) {
+  const repo = env.GITHUB_REPOSITORY, pr = Number(env.PR_NUMBER), sha = env.HEAD_SHA, token = env.GITHUB_TOKEN;
+  if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repo ?? "") || !Number.isSafeInteger(pr) || pr < 1 || !/^[a-fA-F0-9]{40}$/.test(sha ?? "") || !token)
+    throw new Error("Missing or invalid GitHub repository, PR, head SHA, or token");
+  const reportPath = env.REPLUME_REPORT || "replume-ci/report.json";
+  const exitPath = env.REPLUME_VERIFY_EXIT_FILE || "replume-ci/exit.txt";
+  const policyPath = env.REPLUME_POLICY_NOTE || join(dirname(reportPath), "policy.txt");
+  const body = commentBody(reportPath, exitPath, policyPath);
+  const verdict = result(reportPath, exitPath, policyPath).verdict;
+  let existing = null;
+  for (let page = 1; page <= 20; page++) {
+    const comments = await api(`/repos/${repo}/issues/${pr}/comments?per_page=100&page=${page}`, "GET", token);
+    if (!Array.isArray(comments)) throw new Error("GitHub comments response was not a list");
+    existing = comments.find((entry) => entry?.body?.startsWith(marker) && entry?.user?.login === "github-actions[bot]") ?? null;
+    if (existing || comments.length < 100) break;
+  }
+  if (existing) await api(`/repos/${repo}/issues/comments/${existing.id}`, "PATCH", token, { body });
+  else await api(`/repos/${repo}/issues/${pr}/comments`, "POST", token, { body });
+  const conclusion = verdict === "PASS" ? "success" : verdict === "FAIL" ? "failure" : "neutral";
+  await api(`/repos/${repo}/check-runs`, "POST", token, {
+    name: "Replume / task", head_sha: sha, status: "completed", conclusion,
+    output: { title: `Replume: ${verdict}`, summary: body.slice(0, 60000) },
+  });
+  process.stdout.write(`Replume: ${verdict}; ${existing ? "updated" : "posted"} one PR comment; check ${conclusion}.\n`);
+}
+
+if (process.argv[1] && new URL(import.meta.url).pathname === process.argv[1]) {
+  if (process.argv[2] === "--check-policy") {
+    try {
+      const reason = checkBasePolicy(process.argv[3]);
+      writeFileSync(process.argv[4], reason ? `${reason}\n` : "");
+      if (reason) { process.stdout.write(`Replume: INCOMPLETE — ${reason}\n`); process.exitCode = 2; }
+    } catch (error) { process.stderr.write(`replume-pr-check: ${error.message}\n`); process.exitCode = 2; }
+  } else publish().catch((error) => { process.stderr.write(`replume-pr-check: ${error.message}\n`); process.exitCode = 1; });
+}

package/templates/pr-check.yml

@@ -0,0 +1,94 @@
+name: Replume PR check
+on: pull_request
+
+concurrency:
+  group: replume-pr-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+jobs:
+  verify:
+    name: Replume / verify
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+      - run: mkdir -p "$RUNNER_TEMP/replume-ci"
+      - run: npm install -g replume-cli@0.2.1
+      - name: Install project dependencies
+        if: ${{ hashFiles('package-lock.json') != '' }}
+        run: npm ci
+      - name: Verify the PR merge commit against base policy
+        shell: bash
+        env:
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+        run: |
+          set +e
+          if ! git show "$BASE_SHA:.github/replume-pr-check-publish.mjs" > "$RUNNER_TEMP/replume-base-publisher.mjs"; then
+            printf '%s\n' 'Commit .github/replume-pr-check-publish.mjs to the PR base branch.' > "$RUNNER_TEMP/replume-ci/policy.txt"
+            printf '2\n' > "$RUNNER_TEMP/replume-ci/exit.txt"
+            exit 0
+          fi
+          node "$RUNNER_TEMP/replume-base-publisher.mjs" --check-policy "$BASE_SHA" "$RUNNER_TEMP/replume-ci/policy.txt"
+          if [ "$?" -ne 0 ]; then
+            printf '2\n' > "$RUNNER_TEMP/replume-ci/exit.txt"
+            exit 0
+          fi
+          replume verify --commit HEAD --trusted-base "$BASE_SHA" --isolation worktree --json > "$RUNNER_TEMP/replume-ci/report.json" 2> "$RUNNER_TEMP/replume-ci/verify.log"
+          code=$?
+          printf '%s\n' "$code" > "$RUNNER_TEMP/replume-ci/exit.txt"
+          cat "$RUNNER_TEMP/replume-ci/verify.log"
+          exit 0
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: replume-pr-result
+          path: ${{ runner.temp }}/replume-ci/
+          if-no-files-found: error
+      - name: Require Replume PASS
+        if: always()
+        shell: bash
+        run: |
+          if [ ! -f "$RUNNER_TEMP/replume-ci/exit.txt" ]; then exit 2; fi
+          exit "$(cat "$RUNNER_TEMP/replume-ci/exit.txt")"
+
+  publish:
+    name: Replume / PR result
+    if: ${{ always() && github.event.pull_request.head.repo.full_name == github.repository }}
+    needs: verify
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      contents: read
+      pull-requests: write
+      checks: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.base.sha }}
+          persist-credentials: false
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+      - uses: actions/download-artifact@v4
+        continue-on-error: true
+        with:
+          name: replume-pr-result
+          path: replume-ci/
+      - name: Update one PR comment and the Replume check
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_REPOSITORY: ${{ github.repository }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+          REPLUME_REPORT: replume-ci/report.json
+          REPLUME_VERIFY_EXIT_FILE: replume-ci/exit.txt
+          REPLUME_POLICY_NOTE: replume-ci/policy.txt
+        run: node .github/replume-pr-check-publish.mjs