ravensight-playtest 0.1.0

package/src/run/report.js

@@ -0,0 +1,222 @@
+import { access, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import Ajv2020 from 'ajv/dist/2020.js';
+import addFormats from 'ajv-formats';
+import { RUN_FILES } from './paths.js';
+import { assertNoSecrets } from './secretScan.js';
+
+/**
+ * `report.json` and `report.md`: validated here before they are written,
+ * because the server validates them again at
+ * `POST /jobs/:jobId/runs/:runId/complete` and answers 422 `invalid_report`
+ * with the same ajv details. Catching it locally turns a wasted round trip
+ * into one cheap repair turn, which is what the dogfood gate asks for
+ * ("every report.json validates at complete on the first or the repair
+ * attempt").
+ *
+ * The schema is the one from the content pack, not a copy: `packs.get()`
+ * ships `schemas[]` with the server's own
+ * `playtest-report.schema.json` in it, so there is one definition of the
+ * shape and the CLI cannot drift from it between releases.
+ *
+ * Four fields are overwritten rather than trusted: `run_id`, `job_id`,
+ * `actions_taken` and `usage`. The model does not get to decide what run it
+ * was, how many actions it spent or what it cost. Everything else, including
+ * the narrative and the findings, is the model's.
+ */
+
+/** Written into the report so a finding can be attributed to a prompt revision. */
+export const REPORT_SCHEMA_VERSION = '1.0';
+
+/**
+ * The four sections `report.md` has to carry.
+ *
+ * This list is the server's, verbatim from `reportValidator.js`'s
+ * `REQUIRED_MARKDOWN_SECTIONS`, and the server's check is a plain substring
+ * test. Missing one is a lint finding rather than a validation failure there,
+ * which is exactly why it is worth checking locally: the developer never sees
+ * `report_lint` on a run that already finished, whereas a repair turn here
+ * costs one metered question and fixes it.
+ */
+export const REQUIRED_MARKDOWN_SECTIONS = Object.freeze([
+  'What worked',
+  'Top issues',
+  "What this run didn't cover",
+  'Suggested next step'
+]);
+
+/**
+ * Which required sections this markdown is missing, by the same substring test
+ * the server uses.
+ *
+ * @param {string} markdown
+ * @returns {Array<string>}
+ */
+export function lintMarkdownSections(markdown) {
+  const text = typeof markdown === 'string' ? markdown : '';
+  return REQUIRED_MARKDOWN_SECTIONS.filter(section => !text.includes(section));
+}
+
+export class ReportInvalidError extends Error {
+  /**
+   * @param {Array<{instancePath: string, message: string}>} details
+   */
+  constructor(details) {
+    super(`report.json does not match the playtest report schema (${details.length} problem(s)).`);
+    this.name = 'ReportInvalidError';
+    this.details = details;
+  }
+}
+
+/**
+ * @param {Array<{name: string, content: Object}>} schemas the pack's schemas
+ * @returns {{validate: (report: any) => {ok: boolean, details: Array}}}
+ */
+export function createReportValidator(schemas) {
+  const entry = (schemas || []).find(schema => schema.name === 'playtest-report');
+  if (!entry) throw new Error('the content pack has no playtest-report schema; run a pack sync');
+  const ajv = new Ajv2020({ allErrors: true, strict: false });
+  addFormats(ajv);
+  const compiled = ajv.compile(entry.content);
+  return {
+    /**
+     * @param {any} report
+     */
+    validate(report) {
+      const ok = compiled(report);
+      const details = (compiled.errors || []).map(error => ({
+        // ajv always sets one of the two: every error but "required" carries
+        // a non-empty instancePath, a "required" error carries
+        // missingProperty, and an "additionalProperties" error at the root
+        // carries additionalProperty (which used to print as "/undefined").
+        instancePath: error.instancePath || `/${(error.params && (error.params.missingProperty || error.params.additionalProperty)) || ''}`,
+        message: error.message
+      }));
+      return { ok: Boolean(ok), details };
+    }
+  };
+}
+
+/**
+ * Stamp the fields the runner owns onto whatever the model wrote.
+ *
+ * @param {Object} report parsed report.json
+ * @param {Object} facts
+ * @returns {Object} a new report object
+ */
+export function stampReport(report, facts) {
+  const stamped = { ...(report && typeof report === 'object' ? report : {}) };
+  stamped.schema_version = REPORT_SCHEMA_VERSION;
+  stamped.run_id = facts.runId;
+  stamped.job_id = facts.jobId;
+  stamped.persona = facts.persona;
+  stamped.driver = facts.driver;
+  stamped.target = facts.target;
+  stamped.started_at = facts.startedAt;
+  stamped.ended_at = facts.endedAt;
+  stamped.actions_taken = facts.actionsTaken;
+  stamped.usage = facts.usage;
+  if (facts.promptVersion && !stamped.prompt_version) stamped.prompt_version = facts.promptVersion;
+  // A run the proxy cut off for money did not quit for the reason the model
+  // thinks it did. `cost_cap` is in the schema's quit_reason enum precisely
+  // for this.
+  if (facts.quitReason) stamped.quit_reason = facts.quitReason;
+  // Why it stopped, in the runner's words rather than the model's guess: the
+  // action budget ran out, the screen stopped changing, the proxy cut it off.
+  if (facts.quitDetail) stamped.quit_detail = facts.quitDetail;
+  return stamped;
+}
+
+/**
+ * Check every `screenshot` evidence ref against the files actually on disk.
+ *
+ * The server does this against the upload manifest and quarantines what does
+ * not match, so a finding citing a screenshot that was never taken is a 422
+ * waiting to happen. Reported rather than rewritten: an invented evidence
+ * ref is the model's mistake to fix in its repair turn, and silently
+ * deleting it would leave a finding with no evidence, which the server then
+ * drops anyway.
+ *
+ * @param {Object} report
+ * @param {string} runDirectory
+ * @returns {Promise<Array<string>>} human readable problems, empty when clean
+ */
+export async function checkEvidence(report, runDirectory) {
+  const problems = [];
+  const findings = Array.isArray(report && report.findings) ? report.findings : [];
+  for (const finding of findings) {
+    const evidence = Array.isArray(finding && finding.evidence) ? finding.evidence : [];
+    if (evidence.length === 0) {
+      problems.push(`${finding && finding.id}: no evidence, so the server will drop this finding`);
+      continue;
+    }
+    for (const item of evidence) {
+      if (!item || item.type !== 'screenshot') continue;
+      const ref = String(item.ref || '');
+      const candidate = path.resolve(runDirectory, ref);
+      // Refuse a ref that climbs out of the run directory before touching
+      // the filesystem with it.
+      if (!candidate.startsWith(path.resolve(runDirectory) + path.sep)) {
+        problems.push(`${finding.id}: evidence ref ${ref} points outside the run directory`);
+        continue;
+      }
+      try {
+        await access(candidate);
+      } catch {
+        problems.push(`${finding.id}: evidence ref ${ref} is not a file in this run directory`);
+      }
+    }
+  }
+  return problems;
+}
+
+/**
+ * Write both report files, refusing rather than redacting if either one
+ * carries something secret shaped.
+ *
+ * @param {Object} args
+ * @param {string} args.runDirectory
+ * @param {string} args.markdown
+ * @param {Object} args.report
+ * @returns {Promise<{markdownPath: string, reportPath: string}>}
+ */
+export async function writeReportFiles({ runDirectory, markdown, report }) {
+  const json = `${JSON.stringify(report, null, 2)}\n`;
+  assertNoSecrets('report.md', markdown);
+  assertNoSecrets('report.json', json);
+  const markdownPath = path.join(runDirectory, RUN_FILES.reportMd);
+  const reportPath = path.join(runDirectory, RUN_FILES.reportJson);
+  await writeFile(markdownPath, markdown, 'utf8');
+  await writeFile(reportPath, json, 'utf8');
+  return { markdownPath, reportPath };
+}
+
+/**
+ * The repair prompt: exactly what was wrong, nothing else. One turn, one
+ * chance, because each turn is a metered question.
+ *
+ * @param {Array<{instancePath: string, message: string}>} details
+ * @param {Array<string>} evidenceProblems
+ * @returns {string}
+ */
+export function repairInstruction(details, evidenceProblems, missingSections = []) {
+  const lines = ['The report you wrote was refused. Fix exactly these problems and call write_file again for the file that changed.'];
+  for (const detail of details) lines.push(`report.json ${detail.instancePath || '/'}: ${detail.message}`);
+  for (const problem of evidenceProblems) lines.push(`evidence: ${problem}`);
+  for (const section of missingSections) {
+    lines.push(`report.md is missing the required section heading "${section}". Add it with real content under it.`);
+  }
+  lines.push('Change nothing else. Do not add findings, and do not remove a finding you can still evidence.');
+  return lines.join('\n');
+}
+
+export default {
+  createReportValidator,
+  stampReport,
+  checkEvidence,
+  writeReportFiles,
+  repairInstruction,
+  lintMarkdownSections,
+  REQUIRED_MARKDOWN_SECTIONS,
+  ReportInvalidError
+};

package/src/run/resume.js

@@ -0,0 +1,272 @@
+import { optionsFrom } from './args.js';
+import { getDeps } from './deps.js';
+import EXIT from './exit.js';
+import { runDir as runDirFor } from './paths.js';
+import { buildPartialReport, readTranscript, shouldFinalize } from './regenerate.js';
+import { createReportValidator, writeReportFiles } from './report.js';
+import * as stateFile from './state.js';
+import { playJob, resolvePersonas } from '../commands/run.js';
+
+/**
+ * `resumeJob`, the hand-off cli-core's `resume` command makes once it has
+ * decided there is something to carry on with.
+ *
+ * Three things make a resume safe, and all three are about not charging twice:
+ *
+ * 1. **The register is replayed, not remade.** `POST /jobs` with the
+ *    `idempotency_key` out of `state.json` answers the stored job and freshly
+ *    minted per-run tokens, without pricing or charging anything. That is also
+ *    the only way to get tokens back: they are never written to disk (see
+ *    ./state.js), because a JWT under the developer's repo is a secret on disk.
+ * 2. **A finished run keeps its result.** The journal, not the model, decides
+ *    what has already been played, and a run marked `completed` is skipped.
+ * 3. **An unfinished persona run starts over, unless it got far enough to be
+ *    worth finalising.** Live game state cannot be restored, so half a
+ *    playthrough is normally worth less than a clean one (spec 17 section 1.8).
+ *    But a run that recorded a real session and then lost its heartbeat has
+ *    observations, notes and screenshots on disk, and replaying it spends the
+ *    action budget again for a different playthrough. Past
+ *    `PARTIAL_CHECKPOINT_FLOOR` checkpoints the run is finalised from its own
+ *    transcript instead, with no model turn spent;
+ *    `--restart-interrupted` forces the replay.
+ *
+ * Spend carries forward because the server is the one counting it: the run
+ * budgets and the job ceiling are what they were, and the proxy enforces them
+ * against usage already recorded.
+ *
+ * @param {Object} args cli-core's hand-off
+ * @param {Object} args.api the API client
+ * @param {string} args.gameId
+ * @param {string} args.jobId
+ * @param {Object} args.job the job row
+ * @param {Array<Object>} args.runs the server's run rows
+ * @param {Object|null} args.state the journal cli-core read
+ * @param {string} args.repoRoot
+ * @param {Object} args.flags
+ * @param {Object} [injected] test seam
+ * @returns {Promise<number>} an exit code
+ */
+export async function resumeJob({ api, gameId, jobId, job, runs, state, repoRoot, flags = {} }, injected = {}) {
+  const deps = injected.deps || (await getDeps());
+  const log = injected.log || (deps.ui && deps.ui.info ? message => deps.ui.info(message) : message => console.log(message));
+  const { options } = optionsFrom(flags);
+
+  const journal = state || (await stateFile.load(repoRoot, jobId));
+  if (!journal) {
+    log(`There is no journal for ${jobId} in this repository, so there is nothing to resume from. "ravensight-playtest upload ${jobId}" can still drain any pending uploads.`);
+    return EXIT.ENVIRONMENT;
+  }
+  if (!journal.idempotency_key) {
+    log('That journal has no idempotency key, so the per-run tokens cannot be minted again safely. Start a new job instead.');
+    return EXIT.ENVIRONMENT;
+  }
+
+  // Reconcile the journal with what the server actually thinks: a run the sweep
+  // marked interrupted, or one that finished after the journal was last
+  // written, is the server's word against a stale file, and the server wins.
+  for (const row of runs || []) {
+    if (!journal.runs[row.run_id]) {
+      journal.runs[row.run_id] = stateFile.newRunEntry({ runId: row.run_id, module: row.module, persona: row.persona });
+    }
+    const entry = journal.runs[row.run_id];
+    entry.state = row.state;
+    if (stateFile.RUN_DONE_STATES.includes(row.state)) entry.completed = true;
+    if (Number.isFinite(Number(row.actions_taken))) entry.actions_taken = Number(row.actions_taken);
+  }
+  await stateFile.save(repoRoot, journal);
+
+  if ((runs || []).every(row => stateFile.isRunDone(journal.runs[row.run_id]))) {
+    log('Every run of that job has finished. Nothing to replay.');
+    return EXIT.OK;
+  }
+
+  const driverName = journal.driver || options.driver || 'playwright_web';
+  const target = journal.target || options.buildUrl || options.godotProject;
+  if (!target) {
+    log('The journal does not say what this job was pointed at. Pass --build-url or --godot-project.');
+    return EXIT.ENVIRONMENT;
+  }
+
+  const modules = journal.modules && journal.modules.length > 0 ? journal.modules : ['persona_playtest'];
+  const pack = await deps.packs.get({ api, gameId, modules });
+  // The brief the job was run against. A resumed run has to be told the same
+  // thing the first attempt was, or it reports against different rules.
+  const brief = await deps.brief.get({ api, gameId });
+
+  // Runs that got far enough to be worth finalising are dealt with first: each
+  // costs a `complete` and no model turn, and doing them before the replay pass
+  // means `outstanding` below already reflects them.
+  const finalized = [];
+  for (const row of (runs || []).filter(entry => entry.module === 'persona_playtest')) {
+    const entry = journal.runs[row.run_id];
+    if (stateFile.isRunDone(entry) || !shouldFinalize(entry, options)) continue;
+    if (await finalizeFromTranscript({ deps, api, gameId, jobId, repoRoot, row, entry, journal, options, pack, log })) {
+      finalized.push(row.run_id);
+    }
+  }
+
+  const outstanding = (runs || []).filter(row => !stateFile.isRunDone(journal.runs[row.run_id]));
+  if (outstanding.length === 0) {
+    log(`${finalized.length} interrupted run(s) were finalised from their own transcripts. Nothing left to replay.`);
+    if (!isTerminalJob(job)) {
+      const settled = await api.jobs.finish(gameId, jobId, { reason: 'resumed and finalised' });
+      journal.finished = true;
+      await stateFile.save(repoRoot, journal);
+      log(`job ${jobId} ${settled.job ? settled.job.state : 'finished'}: refunded ${formatCents(settled.refunded_cents)}.`);
+    }
+    return EXIT.OK;
+  }
+
+  const personaSlugs = [...new Set((runs || []).filter(row => row.persona).map(row => row.persona))];
+  const slugs = personaSlugs.length > 0 ? personaSlugs : resolvePersonas(options, null, pack);
+  const personas = slugs
+    .map(slug => (pack.personas || []).find(persona => persona.slug === slug))
+    .filter(Boolean);
+  if (personas.length === 0) {
+    log('None of that job\'s personas are in the current content pack. Run a pack sync and try again.');
+    return EXIT.ENVIRONMENT;
+  }
+
+  // The replay. Same key, same body: the server matches on `{gameId,
+  // idempotencyKey}` and answers the stored job, so nothing is priced and
+  // nothing is charged. The tokens that come back are new ones.
+  const replayed = await api.jobs.register(gameId, {
+    modules,
+    personas: slugs,
+    driver: driverName,
+    cli_version: deps.cliVersion || api.cliVersion,
+    confirm_price_cents: Number.isInteger(journal.price_cents) ? journal.price_cents : 0,
+    build: { kind: driverName === 'playwright_web' ? 'url' : 'godot_project', ref: target },
+    repo: { commit: '', dirty: false },
+    max_actions: (journal.options && journal.options.maxActions) ?? null,
+    pack_versions: { pack: pack.pack_version }
+  }, { idempotencyKey: journal.idempotency_key });
+
+  if (!replayed.replayed) {
+    // A register that was NOT a replay means the key did not match a stored
+    // job, which means money just moved. Say so rather than letting it look
+    // like a free resume.
+    log('That idempotency key did not match a stored job, so a new job was registered and charged.');
+  }
+
+  log(`resuming ${jobId}: ${outstanding.length} run(s) to play again of ${(runs || []).length}.`);
+
+  const resumeOptions = {
+    ...options,
+    maxActions: (journal.options && journal.options.maxActions) ?? options.maxActions,
+    concurrency: options.concurrency ?? (journal.options && journal.options.concurrency),
+    uploadVideo: options.uploadVideo ?? (journal.options && journal.options.uploadVideo),
+    uploadTranscript: options.uploadTranscript ?? (journal.options && journal.options.uploadTranscript)
+  };
+
+  return playJob({
+    deps,
+    api,
+    apiUrl: api.apiUrl,
+    gameId,
+    jobId,
+    repoRoot,
+    runs: replayed.runs || [],
+    pack,
+    personas,
+    driverName,
+    target,
+    options: resumeOptions,
+    journal,
+    config: null,
+    brief,
+    injected,
+    log,
+    // A job that is already terminal must not be settled twice: `finish` is
+    // the one writer of a terminal state and the one issuer of a refund.
+    finish: !isTerminalJob(job)
+  });
+}
+
+/**
+ * Finalise one interrupted run from its transcript: no game, no model turn,
+ * just what it already wrote down.
+ *
+ * @returns {Promise<boolean>} whether the server took it
+ */
+async function finalizeFromTranscript({ deps, api, gameId, jobId, repoRoot, row, entry, journal, options, pack, log }) {
+  const directory = runDirFor(repoRoot, jobId, row.run_id);
+  const transcript = await readTranscript(directory);
+  if (transcript.observations === 0) return false;
+
+  const { report, markdown } = buildPartialReport({
+    runId: row.run_id,
+    jobId,
+    persona: row.persona || entry.persona || 'unknown',
+    driver: journal.driver || 'playwright_web',
+    target: journal.target || '',
+    transcript,
+    usage: { by_model: [], total_usd: 0 }
+  });
+
+  // Validated against the same pack schema a live report is, because the server
+  // validates it again and a partial report that 422s is worse than none.
+  if (pack && Array.isArray(pack.schemas)) {
+    const validation = createReportValidator(pack.schemas).validate(report);
+    if (!validation.ok) {
+      log(`run ${row.run_id}: the rebuilt report does not validate (${validation.details.map(detail => `${detail.instancePath}: ${detail.message}`).join('; ')}); replaying it instead.`);
+      return false;
+    }
+  }
+
+  await writeReportFiles({ runDirectory: directory, markdown, report });
+
+  try {
+    // `interrupted` can go to `reporting`, which is where this run actually is:
+    // it has a report now. Then `uploading`, then the delivery.
+    await api.runs.transition(gameId, jobId, row.run_id, 'reporting', { reason: 'rebuilt from transcript' });
+    await deps.finalizeRun(directory, {
+      api,
+      gameId,
+      jobId,
+      runId: row.run_id,
+      includeVideo: Boolean(options.uploadVideo),
+      includeTranscript: Boolean(options.uploadTranscript),
+      body: {
+        state: 'failed',
+        reason: 'interrupted; finalised from its transcript',
+        actions_taken: report.actions_taken,
+        quit_reason: 'error',
+        report,
+        report_md: markdown
+      }
+    });
+  } catch (error) {
+    log(`run ${row.run_id}: could not be finalised (${String(error && error.message)}); it will be replayed.`);
+    return false;
+  }
+
+  await stateFile.patchRun(repoRoot, journal, row.run_id, {
+    state: 'failed',
+    completed: true,
+    uploaded: true,
+    report_written: true,
+    quit_reason: 'error',
+    actions_taken: report.actions_taken,
+    finalized_from_transcript: true
+  });
+  log(`run ${row.run_id}: finalised from its transcript at ${report.actions_taken} action(s), no replay and no model turn spent.`);
+  return true;
+}
+
+function formatCents(cents) {
+  if (!Number.isFinite(Number(cents))) return 'unknown';
+  return `$${(Number(cents) / 100).toFixed(2)}`;
+}
+
+const TERMINAL_JOB_STATES = ['succeeded', 'failed', 'canceled', 'budget_exceeded', 'stalled'];
+
+/**
+ * @param {Object|null|undefined} job
+ * @returns {boolean}
+ */
+export function isTerminalJob(job) {
+  return Boolean(job) && TERMINAL_JOB_STATES.includes(job.state);
+}
+
+export default resumeJob;

package/src/run/secretScan.js

@@ -0,0 +1,171 @@
+/**
+ * Local secret scan, run before this CLI writes a report or hands a
+ * directory to the upload queue.
+ *
+ * This is a port of the Ravensight API's `src/playtest/secretScan.js`, and
+ * it is a port on purpose: the server refuses `POST .../complete` with 422
+ * `secrets_detected` using those patterns, so a local scan built on
+ * different patterns would either pass something the server then refuses
+ * (a wasted run) or refuse something the server would have taken. Keep the
+ * two lists in step.
+ *
+ * Two rules carried over verbatim:
+ *
+ * 1. Nothing is ever redacted in place. A hit means the artifact is
+ *    refused, because the secret has to be rotated and the only way the
+ *    developer learns that is by being told which shape was found.
+ * 2. The matched text never appears in the return value, a log line or an
+ *    error message. A function whose whole job is finding secrets is the
+ *    last place that should leak one, so callers only ever see
+ *    `{ type, count }`.
+ *
+ * Every quantifier below is bounded. The JWT shape is deliberately not a
+ * backtracking regex: `JWT_CANDIDATE` grabs one bounded run of
+ * JWT-charset-or-dot characters in a single linear pass, and
+ * `candidateHasJwtShape` then checks segment lengths with plain string
+ * operations. See the server file's header for the measurements behind
+ * that.
+ */
+
+/** Hard cap. Refused outright rather than scanned. */
+export const MAX_SCAN_BYTES = 25 * 1024 * 1024;
+
+/** How much new text each pass looks at. */
+export const SLICE_SIZE = 65536;
+
+/**
+ * How far each window reaches past its own core, so a match that starts in
+ * the core is always fully visible and a word-boundary check at the core's
+ * first character sees the real character before it. Must exceed the
+ * longest possible single match (the JWT candidate cap, 1600).
+ */
+export const SLICE_OVERLAP = 2048;
+
+const JWT_SEGMENT_MIN = 10;
+const JWT_SEGMENT_MAX = 512;
+const JWT_CANDIDATE = /[A-Za-z0-9_.-]{22,1600}/g;
+
+/**
+ * Whether a candidate run holds three consecutive dot-separated segments
+ * each of JWT segment length. Every window of three is checked, not just
+ * the first, because the candidate run can pick up a leading `v1.` style
+ * token that pushes the real JWT past position three.
+ *
+ * @param {string} candidate
+ * @returns {boolean}
+ */
+export function candidateHasJwtShape(candidate) {
+  const parts = candidate.split('.');
+  for (let i = 0; i + 2 < parts.length; i += 1) {
+    const ok = [parts[i], parts[i + 1], parts[i + 2]].every(
+      part => part.length >= JWT_SEGMENT_MIN && part.length <= JWT_SEGMENT_MAX
+    );
+    if (ok) return true;
+  }
+  return false;
+}
+
+/**
+ * The patterns, one per shape. A string can trip more than one (a
+ * `TOKEN=gt_live_...` line trips both `key_value_secret` and
+ * `ravensight_token`) and that is intended: two reasons to refuse, not a
+ * competition over which pattern owns the match.
+ */
+export const PATTERNS = [
+  { type: 'anthropic_api_key', regex: /\bsk-ant-[A-Za-z0-9_-]{10,256}\b/g },
+  { type: 'aws_access_key', regex: /\bAKIA[0-9A-Z]{16}\b/g },
+  { type: 'github_token', regex: /\b(?:ghp_[A-Za-z0-9]{20,255}|github_pat_[A-Za-z0-9_]{20,255})\b/g },
+  { type: 'stripe_live_key', regex: /\b(?:sk|rk)_live_[A-Za-z0-9]{10,256}\b/g },
+  { type: 'private_key_block', regex: /-----BEGIN [A-Z0-9 ]{0,40}PRIVATE KEY-----/g },
+  { type: 'ravensight_token', regex: /\bgt_(?:live|mcp|cli)_[A-Za-z0-9]{10,256}\b/g },
+  { type: 'jwt', regex: JWT_CANDIDATE, accept: candidateHasJwtShape },
+  { type: 'key_value_secret', regex: /\b(?:KEY|SECRET|TOKEN|PASSWORD)\s{0,20}=\s{0,20}\S{12,512}/gi }
+];
+
+export class ScanInputTooLargeError extends Error {
+  constructor(byteLength) {
+    super(`secretScan: input is ${byteLength} bytes, over the ${MAX_SCAN_BYTES} byte cap`);
+    this.name = 'ScanInputTooLargeError';
+  }
+}
+
+export class SecretsFoundError extends Error {
+  /**
+   * @param {string} what human name of the thing that was refused
+   * @param {Array<{type: string, count: number}>} secrets
+   */
+  constructor(what, secrets) {
+    const shapes = secrets.map(s => `${s.type} (${s.count})`).join(', ');
+    super(`${what} contains something shaped like a secret: ${shapes}. Rotate it, take it out and try again.`);
+    this.name = 'SecretsFoundError';
+    this.secrets = secrets;
+  }
+}
+
+/**
+ * Count matches of one pattern whose start falls inside this slice's own
+ * region, which is what makes slicing exactly-once rather than either
+ * double counting a match seen in two windows or missing one at a boundary.
+ *
+ * @param {{regex: RegExp, accept?: (text: string) => boolean}} pattern
+ * @param {string} window
+ * @param {number} windowOffset
+ * @param {number} coreStart
+ * @param {number} coreEnd
+ * @returns {number}
+ */
+function countCoreMatches({ regex, accept }, window, windowOffset, coreStart, coreEnd) {
+  regex.lastIndex = 0;
+  let count = 0;
+  let match = regex.exec(window);
+  while (match !== null) {
+    if (!accept || accept(match[0])) {
+      const absoluteStart = windowOffset + match.index;
+      if (absoluteStart >= coreStart && absoluteStart < coreEnd) count += 1;
+    }
+    // Never true for the patterns above, but it keeps the loop provably
+    // finite if a future pattern can match zero characters.
+    if (match[0].length === 0) regex.lastIndex += 1;
+    match = regex.exec(window);
+  }
+  return count;
+}
+
+/**
+ * @param {string} text
+ * @returns {Array<{type: string, count: number}>} only the shapes that
+ *   matched, never the matched text
+ * @throws {ScanInputTooLargeError}
+ */
+export function scanForSecrets(text) {
+  if (typeof text !== 'string' || text.length === 0) return [];
+
+  const byteLength = Buffer.byteLength(text, 'utf8');
+  if (byteLength > MAX_SCAN_BYTES) throw new ScanInputTooLargeError(byteLength);
+
+  const counts = new Map();
+  for (let coreStart = 0; coreStart < text.length; coreStart += SLICE_SIZE) {
+    const coreEnd = Math.min(coreStart + SLICE_SIZE, text.length);
+    const windowStart = Math.max(0, coreStart - SLICE_OVERLAP);
+    const windowEnd = Math.min(coreEnd + SLICE_OVERLAP, text.length);
+    const window = text.slice(windowStart, windowEnd);
+    for (const pattern of PATTERNS) {
+      const matched = countCoreMatches(pattern, window, windowStart, coreStart, coreEnd);
+      if (matched > 0) counts.set(pattern.type, (counts.get(pattern.type) || 0) + matched);
+    }
+  }
+  return [...counts.entries()].map(([type, count]) => ({ type, count }));
+}
+
+/**
+ * Refuse rather than write. Every file this CLI produces from model output
+ * goes through here first.
+ *
+ * @param {string} what human name used in the error
+ * @param {string} text
+ * @throws {SecretsFoundError}
+ */
+export function assertNoSecrets(what, text) {
+  const secrets = scanForSecrets(text);
+  if (secrets.length > 0) throw new SecretsFoundError(what, secrets);
+}