premanmcp 0.10.5 → 0.10.7

package/bin/tests.js

@@ -0,0 +1,341 @@
+/**
+ * `preman tests generate|review|setup|enrich` — Collections workbench test generation.
+ *
+ * Thin HTTP wrappers over the same `/workbench/generate-tests` routes the
+ * dashboard uses. Distinct from `preman test <id>`, which generates + runs
+ * scenarios for one endpoint via `POST /mcp/call-tool`.
+ */
+
+import { callBackendJson, cliInvocation, makeArgs, resolveApiKey } from "./shared.js";
+
+export const TESTS_HELP = `
+Collections tests:
+  preman tests generate [--workspace id] [--integration-id id] [--json-out]
+      Scan the connected GitHub repo and write heuristic suites on saved requests.
+      No LLM. Writes and ambiguous routes are flagged for review.
+  preman tests review [--workspace id] [--json-out]
+      List suites still flagged (review_status=required) with the reason.
+  preman tests review --approve <request_id> [--workspace id] [--json-out]
+      Approve one flagged suite so it can be scheduled. Does not raise
+      unattended_policy. DELETE stays skipped unless allow_destructive.
+  preman tests review --all-required [--workspace id] [--json-out]
+      Approve every currently flagged suite. Same gates as --approve.
+  preman tests setup [--list] [--workspace id] [--json-out]
+      Harvest path-param fixture IDs from list GETs / schema examples.
+      Stores them on the workspace and prints a .env snippet.
+      Does not invent UUIDs, create secrets, or approve suites.
+      --list shows missing fixtures without calling the API.
+  preman tests enrich [--list] [--workspace id] [--json-out]
+      LLM-enrich remaining review-cleared heuristic suites (cap 40).
+      --list shows the remaining queue without calling the model.
+      Skips destructive/billing unless standing policy already allows that risk.
+      Does not enable writes and does not publish a Guard baseline.
+`;
+
+class CliError extends Error {
+  constructor(message, exitCode = 1) {
+    super(message);
+    this.exitCode = exitCode;
+  }
+}
+
+function requireKey(args) {
+  const token = resolveApiKey(args);
+  if (!token) {
+    throw new CliError(
+      `No PreMan API key. Run \`${cliInvocation()} login\` or pass --api-key pm_live_...`,
+      2,
+    );
+  }
+  return token;
+}
+
+function workspaceHeaders(args) {
+  return { "x-workspace-id": args.value("--workspace", "") };
+}
+
+function wantsJson(args) {
+  return args.has("--json-out") || args.has("--json");
+}
+
+function printJson(value) {
+  process.stdout.write(`${JSON.stringify(value, null, 2)}\n`);
+}
+
+function errorDetail(result) {
+  const detail = result.detail;
+  if (typeof detail === "string") return detail;
+  if (Array.isArray(detail)) {
+    return detail.map((item) => item.msg || JSON.stringify(item)).join("; ");
+  }
+  if (detail && typeof detail === "object") {
+    return detail.message || detail.error || JSON.stringify(detail);
+  }
+  return result.message || result.raw || "backend error";
+}
+
+async function workbench(args, method, routePath, { json } = {}) {
+  const token = requireKey(args);
+  const result = await callBackendJson(args, method, routePath, {
+    token,
+    json,
+    headers: workspaceHeaders(args),
+  });
+  if (!result.ok) {
+    throw new CliError(
+      `${method} ${routePath} failed: ${result.status_code} ${errorDetail(result)}`,
+    );
+  }
+  const { status_code, ...payload } = result;
+  return payload;
+}
+
+function flagLine(item) {
+  const method = item.method || "";
+  const url = item.url || item.name || "";
+  const reason = item.reason_label || item.reason || "Needs review";
+  const id = item.request_id || "";
+  return `  ${id}  ${method} ${url}  ${reason}`;
+}
+
+export function printGenerateSummary(payload) {
+  const generated = payload.suites_generated || 0;
+  const repo = payload.repo_url || "the connected repository";
+  process.stdout.write(`Generated ${generated} suite(s) from ${repo}\n`);
+  process.stdout.write(
+    `  scanned ${payload.scanned || 0}  matched ${payload.matched || 0}  unmatched ${payload.unmatched || 0}\n`,
+  );
+  process.stdout.write(
+    `  auto-approved ${payload.auto_approved || 0}  flagged ${payload.flagged_for_review || 0}` +
+      `  waiting for base URL ${payload.waiting_for_base_url || 0}\n`,
+  );
+  if (payload.bodies_filled_from_schema) {
+    process.stdout.write(`  bodies filled from schema ${payload.bodies_filled_from_schema}\n`);
+  }
+  const flagged = payload.flagged || [];
+  if (flagged.length) {
+    process.stdout.write(`\n${flagged.length} flagged for review:\n`);
+    for (const item of flagged) process.stdout.write(`${flagLine(item)}\n`);
+    process.stdout.write(
+      `\nNext: \`${cliInvocation()} tests review --approve <request_id>\` enables a read-only schedule only.\n`,
+    );
+  } else if ((payload.flagged_for_review || 0) === 0 && generated) {
+    process.stdout.write(
+      `\nNo suites flagged. Optional: \`${cliInvocation()} tests enrich\` to add LLM cases to remaining heuristic suites.\n`,
+    );
+  }
+  const samples = payload.unmatched_samples || [];
+  if (samples.length) {
+    process.stdout.write(`\nUnmatched repo routes (not imported):\n`);
+    for (const sample of samples.slice(0, 10)) process.stdout.write(`  ${sample}\n`);
+  }
+}
+
+export function printReviewList(payload) {
+  const items = payload.items || [];
+  const total = payload.total ?? items.length;
+  process.stdout.write(`${total} suite(s) flagged for review\n`);
+  for (const item of items) process.stdout.write(`${flagLine(item)}\n`);
+  if (total) {
+    process.stdout.write(
+      `\nApprove one: \`${cliInvocation()} tests review --approve <request_id>\`\n` +
+        `Approve enables a read-only schedule only. DELETE stays skipped unless allow_destructive.\n`,
+    );
+  }
+}
+
+export function printApproveResult(payload) {
+  const id = payload.request_id || "";
+  const status = payload.review_status || "approved";
+  const enabled = payload.enabled ? "enabled" : "disabled";
+  process.stdout.write(`Approved ${id}  ${status}  ${enabled}\n`);
+  process.stdout.write(
+    "Schedule is enabled. unattended_policy is unchanged; DELETE still skips unless allow_destructive.\n",
+  );
+}
+
+export function printApproveAll(results) {
+  process.stdout.write(`Approved ${results.length} flagged suite(s)\n`);
+  for (const payload of results) {
+    const enabled = payload.enabled ? "enabled" : "disabled";
+    process.stdout.write(`  ${payload.request_id || ""}  ${payload.review_status || "approved"}  ${enabled}\n`);
+  }
+  if (results.length) {
+    process.stdout.write(
+      "Schedule is enabled. unattended_policy is unchanged; DELETE still skips unless allow_destructive.\n",
+    );
+  }
+}
+
+export function printEnrichList(payload) {
+  const items = payload.items || [];
+  const total = payload.total ?? items.length;
+  process.stdout.write(`${total} suite(s) remaining for agent enrichment\n`);
+  for (const item of items) process.stdout.write(`${flagLine(item)}\n`);
+  if (total) {
+    process.stdout.write(
+      `\nRun \`${cliInvocation()} tests enrich\` to append LLM cases (cap 40). Does not enable writes.\n`,
+    );
+  }
+}
+
+export function printSetupList(payload) {
+  const items = (payload.items || []).filter((item) => !item.value_set);
+  const total = payload.total ?? items.length;
+  process.stdout.write(`${total} path-param fixture(s) still missing\n`);
+  for (const item of payload.items || []) {
+    const state = item.value_set ? "set" : "missing";
+    process.stdout.write(
+      `  ${item.name || item.param || ""}  ${item.method || ""} ${item.url || ""}  ${state}\n`,
+    );
+  }
+  if (payload.dotenv) {
+    process.stdout.write(`\nCurrent .env:\n${payload.dotenv}`);
+  }
+  if (total) {
+    process.stdout.write(
+      `\nRun \`${cliInvocation()} tests setup\` to harvest ids from list GETs.\n`,
+    );
+  }
+}
+
+export function printSetupSummary(payload) {
+  process.stdout.write(
+    `Stored ${payload.stored || 0} fixture(s). Missing ${payload.missing || 0}.\n`,
+  );
+  process.stdout.write(
+    `  harvested ${payload.harvested || 0}` +
+      `  list GETs ${payload.list_gets_called || 0}` +
+      `  from lists ${payload.from_list_get || 0}` +
+      `  from schema ${payload.from_schema || 0}\n`,
+  );
+  if (payload.dotenv) {
+    process.stdout.write(`\n.env\n${payload.dotenv}`);
+  }
+  const missing = payload.missing_items || [];
+  if (missing.length) {
+    process.stdout.write(`\nStill missing:\n`);
+    for (const item of missing) {
+      process.stdout.write(`  ${item.name || item.param || ""}  ${item.method || ""} ${item.url || ""}\n`);
+    }
+  }
+}
+
+export function printEnrichSummary(payload) {
+  process.stdout.write(
+    `Enriched ${payload.enriched || 0} suite(s). Remaining ${payload.remaining || 0}.\n`,
+  );
+  process.stdout.write(
+    `  skipped sensitive ${payload.skipped_sensitive || 0}` +
+      `  pending review ${payload.skipped_pending_review || 0}` +
+      `  already enriched ${payload.skipped_already_enriched || 0}` +
+      `  no suite ${payload.skipped_no_suite || 0}` +
+      `  failed ${payload.failed || 0}\n`,
+  );
+  const remaining = payload.remaining_items || [];
+  if (remaining.length) {
+    process.stdout.write(`\nStill remaining:\n`);
+    for (const item of remaining) process.stdout.write(`${flagLine(item)}\n`);
+  }
+}
+
+function requestIdFlag(args, flag) {
+  const value = args.value(flag, "");
+  if (!value || value.startsWith("-")) {
+    throw new CliError(`pass a request id after ${flag}${TESTS_HELP}`, 2);
+  }
+  return value;
+}
+
+async function generateCommand(args) {
+  const integrationId = args.value("--integration-id", "");
+  const json = {};
+  if (integrationId && !integrationId.startsWith("-")) json.integration_id = integrationId;
+  const payload = await workbench(args, "POST", "/workbench/generate-tests", { json });
+  if (wantsJson(args)) return printJson(payload);
+  printGenerateSummary(payload);
+  return payload;
+}
+
+async function reviewCommand(args) {
+  if (args.has("--approve") && args.has("--all-required")) {
+    throw new CliError("use either --approve <request_id> or --all-required, not both", 2);
+  }
+
+  if (args.has("--approve")) {
+    const requestId = encodeURIComponent(requestIdFlag(args, "--approve"));
+    const payload = await workbench(
+      args,
+      "POST",
+      `/workbench/generate-tests/review/${requestId}/approve`,
+    );
+    if (wantsJson(args)) return printJson(payload);
+    printApproveResult(payload);
+    return payload;
+  }
+
+  const listed = await workbench(args, "GET", "/workbench/generate-tests/review");
+  if (args.has("--all-required")) {
+    const items = listed.items || [];
+    const results = [];
+    for (const item of items) {
+      const requestId = encodeURIComponent(item.request_id || "");
+      if (!requestId) continue;
+      results.push(
+        await workbench(args, "POST", `/workbench/generate-tests/review/${requestId}/approve`),
+      );
+    }
+    if (wantsJson(args)) return printJson({ approved: results, listed });
+    if (!results.length) {
+      process.stdout.write("0 suite(s) flagged for review\n");
+      return { approved: results, listed };
+    }
+    printApproveAll(results);
+    return { approved: results, listed };
+  }
+
+  if (wantsJson(args)) return printJson(listed);
+  printReviewList(listed);
+  return listed;
+}
+
+async function setupCommand(args) {
+  if (args.has("--list")) {
+    const listed = await workbench(args, "GET", "/workbench/generate-tests/fixtures");
+    if (wantsJson(args)) return printJson(listed);
+    printSetupList(listed);
+    return listed;
+  }
+  const payload = await workbench(args, "POST", "/workbench/generate-tests/fixtures");
+  if (wantsJson(args)) return printJson(payload);
+  printSetupSummary(payload);
+  return payload;
+}
+
+async function enrichCommand(args) {
+  if (args.has("--list")) {
+    const listed = await workbench(args, "GET", "/workbench/generate-tests/enrich");
+    if (wantsJson(args)) return printJson(listed);
+    printEnrichList(listed);
+    return listed;
+  }
+  const payload = await workbench(args, "POST", "/workbench/generate-tests/enrich");
+  if (wantsJson(args)) return printJson(payload);
+  printEnrichSummary(payload);
+  return payload;
+}
+
+export async function testsCommand(commandArgs = []) {
+  const sub = commandArgs[0] && !commandArgs[0].startsWith("-") ? commandArgs[0] : "";
+  const rest = sub ? commandArgs.slice(1) : commandArgs;
+  const args = makeArgs(rest);
+
+  if (!sub || sub === "help") {
+    throw new CliError(`pass generate, review, setup, or enrich${TESTS_HELP}`, 2);
+  }
+  if (sub === "generate") return generateCommand(args);
+  if (sub === "review") return reviewCommand(args);
+  if (sub === "setup") return setupCommand(args);
+  if (sub === "enrich") return enrichCommand(args);
+  throw new CliError(`unknown tests subcommand: ${sub}${TESTS_HELP}`, 2);
+}

package/bin/verify.js

@@ -18,6 +18,7 @@
  * unless the repository has explicitly opted into blocking.
  */
 
+import { randomUUID } from "node:crypto";
 import { readFileSync } from "node:fs";
 import os from "node:os";
 import path from "node:path";
@@ -25,7 +26,7 @@ import path from "node:path";
 import { changedFilesForPush, readChangedFiles, readPushRefsFromStdin, repoRoot } from "./changed.js";
 import { resolveLocalTarget } from "./detect.js";
 import { createReporter } from "./progress.js";
-import { backendUrl, callBackendJson, cliInvocation, makeArgs, resolveApiKey } from "./shared.js";
+import { backendUrl, callBackendJson, cliInvocation, makeArgs, nowMs, resolveApiKey } from "./shared.js";
 
 export const VERIFY_HELP = `
 Verify options:
@@ -42,6 +43,10 @@ Verify options:
 const DEFAULT_TIMEOUT_SECONDS = 120;
 const REQUEST_TIMEOUT_MS = 10_000;
 const CONCURRENCY = 6;
+const INVENTORY_PAGE_SIZE = 200;
+const INVENTORY_MAX_ENDPOINTS = 2000;
+// POST /projects/{id}/api-runs rejects more than this (schemas.pulse.MAX_UPLOAD_RUNS).
+const UPLOAD_BATCH_SIZE = 200;
 const READ_METHODS = new Set(["GET", "HEAD", "OPTIONS"]);
 // Never fired automatically even against loopback: a destructive verb that
 // happens to point at a real datastore is not made safe by the port being local.
@@ -55,10 +60,6 @@ const REPO_CONFIG = ".preman.json";
 // push — the exact failure the advisory guarantee exists to prevent.
 export const BLOCK_EXIT_CODE = 97;
 
-function nowMs() {
-  return Number(process.hrtime.bigint() / 1000000n);
-}
-
 function originLabel() {
   const raw = `${os.hostname() || "local"}`.toLowerCase();
   const cleaned = raw.replace(/[^a-z0-9._-]/g, "-").replace(/^[^a-z0-9]+/, "");
@@ -103,7 +104,7 @@ async function runCase(baseUrl, endpoint, { batchId }) {
       client_run_id: `${batchId}:${method}:${endpoint.path_template || endpoint.path}`,
       method,
       url: target,
-      status: "errored",
+      status: "error",
       http_status: null,
       latency_ms: nowMs() - t0,
       error_message: aborted ? `timed out after ${REQUEST_TIMEOUT_MS}ms` : String(error?.message || error),
@@ -201,7 +202,7 @@ async function runGeneratedCase(testCase, { batchId, endpoint }) {
       endpoint: `${endpoint.method} ${endpoint.path}`,
       method,
       url: testCase.url,
-      status: "errored",
+      status: "error",
       http_status: null,
       latency_ms: nowMs() - t0,
       error_message: aborted ? `timed out after ${REQUEST_TIMEOUT_MS}ms` : String(error?.message || error),
@@ -239,19 +240,66 @@ async function safeCall(args, method, routePath, options) {
 }
 
 async function fetchInventory(args, token) {
-  const result = await safeCall(args, "POST", "/mcp/call-tool", {
-    token,
-    json: { tool: "get_endpoints", arguments: { format: "json" } },
-  });
-  if (!result.ok) return { endpoints: [], error: result.detail || `status ${result.status_code}` };
-
-  const payload = result.result ?? result.data ?? result;
-  const endpoints =
-    payload?.endpoints ||
-    payload?.result?.endpoints ||
-    (Array.isArray(payload) ? payload : []) ||
-    [];
-  return { endpoints: Array.isArray(endpoints) ? endpoints : [] };
+  const endpoints = [];
+  for (let offset = 0; offset < INVENTORY_MAX_ENDPOINTS; offset += INVENTORY_PAGE_SIZE) {
+    const result = await safeCall(args, "POST", "/mcp/call-tool", {
+      token,
+      json: {
+        tool: "get_endpoints",
+        arguments: { format: "json", limit: INVENTORY_PAGE_SIZE, offset },
+      },
+    });
+    if (!result.ok) {
+      return {
+        endpoints,
+        error: result.detail || `status ${result.status_code}`,
+      };
+    }
+
+    const payload = result.result ?? result.data ?? result;
+    const page =
+      payload?.endpoints ||
+      payload?.result?.endpoints ||
+      (Array.isArray(payload) ? payload : []) ||
+      [];
+    const rows = Array.isArray(page) ? page : [];
+    endpoints.push(...rows);
+    const pagination = payload?.pagination || payload?.result?.pagination;
+    if (!pagination?.has_more || rows.length === 0) break;
+    if (offset + INVENTORY_PAGE_SIZE >= INVENTORY_MAX_ENDPOINTS) {
+      return {
+        endpoints,
+        error: `endpoint inventory exceeds the ${INVENTORY_MAX_ENDPOINTS}-endpoint verification limit`,
+        incomplete: true,
+      };
+    }
+  }
+  return { endpoints };
+}
+
+async function uploadRuns(args, { token, projectId, batchId, runs }) {
+  if (!projectId || !runs.length) return { inserted: 0, error: null };
+  let inserted = 0;
+  for (let offset = 0; offset < runs.length; offset += UPLOAD_BATCH_SIZE) {
+    const chunk = runs.slice(offset, offset + UPLOAD_BATCH_SIZE);
+    const upload = await safeCall(args, "POST", `/projects/${projectId}/api-runs`, {
+      token,
+      json: {
+        origin: "local",
+        origin_label: originLabel(),
+        source: "ci",
+        batch_id: batchId,
+        batch_label: "pre-push verify",
+        runs: chunk,
+      },
+    });
+    if (!upload.ok) {
+      const detail = upload.detail || (upload.status_code ? `HTTP ${upload.status_code}` : "request failed");
+      return { inserted, error: `verification results were not uploaded: ${detail}` };
+    }
+    inserted += Number(upload.inserted || 0);
+  }
+  return { inserted, error: null };
 }
 
 function selectMethods(args) {
@@ -390,7 +438,7 @@ async function runPlan(
     stream: asJson ? process.stderr : process.stdout,
     live: asJson ? false : undefined,
   });
-  const batchId = `prepush-${Date.now().toString(36)}`;
+  const batchId = randomUUID();
   const changes = plan.changes || [];
 
   if (!asJson) {
@@ -449,21 +497,8 @@ async function runPlan(
   );
   const removed = changes.filter((c) => c.change_type === "removed");
 
-  let uploaded = 0;
-  if (projectId && completed.length) {
-    const upload = await safeCall(args, "POST", `/projects/${projectId}/api-runs`, {
-      token,
-      json: {
-        origin: "local",
-        origin_label: originLabel(),
-        source: "ci",
-        batch_id: batchId,
-        batch_label: "pre-push verify",
-        runs: completed,
-      },
-    });
-    if (upload.ok) uploaded = Number(upload.inserted || 0);
-  }
+  const upload = await uploadRuns(args, { token, projectId, batchId, runs: completed });
+  const uploaded = upload.inserted;
 
   const gate = blockOnBreaking(args, root);
   const shouldBlock = gate.blocking && (breaking.length > 0 || removed.length > 0);
@@ -479,6 +514,7 @@ async function runPlan(
     }
     if (withheld) reporter.line(`[preman] ${withheld} mutating case${withheld === 1 ? "" : "s"} withheld`);
     if (uploaded) reporter.line(`[preman] uploaded ${uploaded} run${uploaded === 1 ? "" : "s"}`);
+    if (upload.error) reporter.line(`[preman] ${upload.error}`);
     if (shouldBlock) {
       reporter.line(`[preman] blocking this push (${gate.from}). Set PREMAN_SKIP_HOOK=1 to override.`);
     } else if (breaking.length || removed.length) {
@@ -518,6 +554,7 @@ async function runPlan(
       error: r.error_message || null,
     })),
     uploaded,
+    upload_error: upload.error,
     // Advisory unless the repository opted in. `advisory` only flattens an
     // unexpected throw; a deliberate block has to survive it.
     exitCode: shouldBlock ? BLOCK_EXIT_CODE : 0,
@@ -589,7 +626,8 @@ async function runVerify(args) {
   }
 
   const extraPorts = args.value("--port", "") ? [args.value("--port", "")] : [];
-  const resolved = await resolveLocalTarget([...planEndpoints, ...endpoints], {
+  const confirmationEndpoints = planEndpoints.length ? planEndpoints : endpoints;
+  const resolved = await resolveLocalTarget(confirmationEndpoints, {
     extraPorts,
     timeoutMs: 2000,
   });
@@ -605,6 +643,7 @@ async function runVerify(args) {
 
   const target = resolved.target;
   const policy = selectMethods(args);
+  const targetInventory = resolved.inventory || endpoints;
 
   if (planned.plan?.selected?.length) {
     return finish(
@@ -623,7 +662,7 @@ async function runVerify(args) {
     );
   }
 
-  const selected = endpoints.filter((ep) => {
+  const selected = targetInventory.filter((ep) => {
     const method = String(ep.method || "GET").toUpperCase();
     if (READ_METHODS.has(method)) return true;
     if (DESTRUCTIVE_METHODS.has(method)) return false;
@@ -638,7 +677,7 @@ async function runVerify(args) {
     );
   }
 
-  const batchId = `prepush-${Date.now().toString(36)}`;
+  const batchId = randomUUID();
   const runs = await runPool(
     selected,
     (endpoint) =>
@@ -650,21 +689,8 @@ async function runVerify(args) {
   const completed = runs.filter(Boolean);
   const failed = completed.filter((r) => r.status !== "passed");
 
-  let uploaded = 0;
-  if (projectId && completed.length) {
-    const upload = await safeCall(args, "POST", `/projects/${projectId}/api-runs`, {
-      token,
-      json: {
-        origin: "local",
-        origin_label: originLabel(),
-        source: "ci",
-        batch_id: batchId,
-        batch_label: "pre-push verify",
-        runs: completed,
-      },
-    });
-    if (upload.ok) uploaded = Number(upload.inserted || 0);
-  }
+  const upload = await uploadRuns(args, { token, projectId, batchId, runs: completed });
+  const uploaded = upload.inserted;
 
   const outcome = {
     state: failed.length ? "failed" : "passed",
@@ -678,6 +704,7 @@ async function runVerify(args) {
       error: r.error_message || null,
     })),
     uploaded,
+    upload_error: upload.error,
     // Advisory by construction: a real failure still reports zero so the push
     // proceeds. The report is the product, not the gate.
     exitCode: 0,
@@ -695,6 +722,7 @@ async function runVerify(args) {
       process.stdout.write(`[preman] all ${completed.length} endpoints healthy\n`);
     }
     if (uploaded) process.stdout.write(`[preman] uploaded ${uploaded} run${uploaded === 1 ? "" : "s"}\n`);
+    if (upload.error) process.stdout.write(`[preman] ${upload.error}\n`);
   }
 
   return finish({ ...outcome, message: null });