ravensight-playtest 0.2.7 → 0.2.9

package/src/run/placeholders.js

@@ -0,0 +1,186 @@
+/**
+ * Template placeholders a model copied into a report instead of filling.
+ *
+ * The persona skill's report template reads `**Date:** <date> ...`, and the
+ * report prompt says never to write a calendar date, so a model that obeys
+ * both writes the placeholder itself (`<run date>`) and the developer reads
+ * it as the report's date.
+ *
+ * Deliberately narrow, because angle brackets in a report are usually real:
+ * a key (`<Enter>`, `<Ctrl+A>`), an element (`<form>`, `<title>`), a type
+ * (`Array<string>`), a link, or the game's own text. So only two things are
+ * treated as a slot:
+ *
+ * 1. A slot the runner knows (date, driver, target, persona, actions, quit
+ *    reason, game, version), filled from its own facts, or "not recorded"
+ *    when it has no value for it.
+ * 2. Lowercase words in brackets on a `**Field:**` line, the shape of the
+ *    template's header, which become "not recorded".
+ *
+ * Never touched: a `<` straight after a word character, a key name, anything
+ * quoted, HTML, autolinks, code spans and fenced blocks.
+ */
+
+/** What a slot with no value becomes. */
+export const NOT_RECORDED = 'not recorded';
+
+const HTML_TAGS = new Set([
+  'a', 'abbr', 'article', 'aside', 'b', 'blockquote', 'body', 'br', 'button', 'canvas', 'caption', 'code', 'dd', 'del',
+  'details', 'dialog', 'div', 'dl', 'dt', 'em', 'figcaption', 'figure', 'footer', 'form', 'h1', 'h2', 'h3', 'h4', 'h5',
+  'h6', 'head', 'header', 'hr', 'html', 'i', 'iframe', 'img', 'input', 'ins', 'kbd', 'label', 'li', 'link', 'main', 'mark',
+  'meta', 'nav', 'ol', 'option', 'p', 'picture', 'pre', 'q', 's', 'samp', 'script', 'section', 'select', 'small',
+  'source', 'span', 'strong', 'style', 'sub', 'summary', 'sup', 'table', 'tbody', 'td', 'template', 'textarea',
+  'tfoot', 'th', 'thead', 'title', 'tr', 'u', 'ul', 'video'
+]);
+
+const KEY_NAMES = new Set([
+  'enter', 'return', 'tab', 'space', 'spacebar', 'esc', 'escape', 'backspace', 'delete', 'del', 'insert', 'home', 'end',
+  'pageup', 'pagedown', 'up', 'down', 'left', 'right', 'arrowup', 'arrowdown', 'arrowleft', 'arrowright', 'shift',
+  'ctrl', 'control', 'alt', 'option', 'cmd', 'command', 'meta', 'super', 'win', 'fn', 'capslock'
+]);
+
+/** `<...>` with no nested bracket and no line break, short enough to be a template slot. */
+const ANGLE = /<([^<>\n]{1,80})>/g;
+
+/** A line in the template's header shape: `**Date:** ...`. */
+const FIELD_LINE = /\*\*[^*\n]+:\*\*/;
+
+const QUOTES = { '"': '"', "'": "'", '“': '”', '‘': '’', '`': '`' };
+
+function normalise(inner) {
+  return String(inner).trim().toLowerCase().replace(/[_-]+/g, ' ').replace(/\s+/g, ' ');
+}
+
+/**
+ * Whether the bracket is a slot the runner knows, and its value if it has one.
+ *
+ * @param {string} inner
+ * @param {Object} facts
+ * @returns {{known: boolean, value: string|null}}
+ */
+function knownSlot(inner, facts) {
+  const key = normalise(inner);
+  const date = facts.generatedAt ? String(facts.generatedAt).slice(0, 10) : null;
+  const actions = facts.actionsTaken === null || facts.actionsTaken === undefined || !Number.isFinite(Number(facts.actionsTaken))
+    ? null
+    : Number(facts.actionsTaken);
+  const slot = value => ({ known: true, value: value === null || value === undefined || value === '' ? null : String(value) });
+  if (/^(run |report |session )?(date|timestamp|date and time)$/.test(key) || key === 'generated at') return slot(date);
+  if (/^driver( name)?$/.test(key)) return slot(facts.driver);
+  if (/^(target|target url|build url|game url)$/.test(key)) return slot(facts.target);
+  if (/^persona( name| slug)?$/.test(key)) return slot(facts.persona);
+  if (/^(n|number of) actions$/.test(key)) return slot(actions === null ? null : `${actions} actions`);
+  if (key === 'actions taken') return slot(actions);
+  if (/^(quit|stop) reason$/.test(key)) return slot(facts.quitReason);
+  if (key === 'game' || key === 'game name') return slot(facts.game);
+  if (key === 'version' || key === 'game version') return slot(facts.version);
+  return { known: false, value: null };
+}
+
+/**
+ * The replacement for one bracket match, or null to leave it as written.
+ *
+ * @param {string} text the prose segment the match is in
+ * @param {string} whole the match, brackets included
+ * @param {string} inner
+ * @param {number} offset
+ * @param {string} line the whole line, for the field check
+ * @param {Object} facts
+ * @returns {string|null}
+ */
+function replacementFor(text, whole, inner, offset, line, facts) {
+  const before = text[offset - 1];
+  const after = text[offset + whole.length];
+  // `Array<string>`, `Map<K, V>`: a type, not a slot.
+  if (before !== undefined && /\w/.test(before)) return null;
+  // The game's own words, quoted as evidence.
+  if (before !== undefined && QUOTES[before] !== undefined && QUOTES[before] === after) return null;
+  const value = String(inner).trim();
+  if (!/^[A-Za-z]/.test(value)) return null;
+  // A key or a chord: <Enter>, <Ctrl+A>, <Shift + Tab>.
+  if (value.includes('+') || KEY_NAMES.has(value.toLowerCase())) return null;
+  const known = knownSlot(value, facts);
+  if (known.known) return known.value || NOT_RECORDED;
+  if (FIELD_LINE.test(line) && /^[a-z]+(?: [a-z]+)*$/.test(value) && !HTML_TAGS.has(value)) return NOT_RECORDED;
+  return null;
+}
+
+/**
+ * Walk the prose of a markdown document (not fenced blocks, not code spans),
+ * handing each segment and its whole line to `fn`.
+ *
+ * @param {string} markdown
+ * @param {(text: string, line: string) => string} fn
+ * @returns {string}
+ */
+function mapProse(markdown, fn) {
+  let fenced = false;
+  return String(markdown == null ? '' : markdown).split('\n').map(line => {
+    if (/^\s*(```|~~~)/.test(line)) {
+      fenced = !fenced;
+      return line;
+    }
+    if (fenced) return line;
+    return line.split(/(`[^`]*`)/).map(part => (part.length > 1 && part.startsWith('`') && part.endsWith('`') ? part : fn(part, line))).join('');
+  }).join('\n');
+}
+
+/**
+ * Fill the slots the runner knows, and the header fields it does not, from
+ * its own facts. Everything else in angle brackets is left as written.
+ *
+ * @param {string} markdown
+ * @param {{generatedAt?: string, driver?: string, target?: string, persona?: string,
+ *   actionsTaken?: number, quitReason?: string, game?: string, version?: string}} [facts]
+ * @returns {string}
+ */
+export function fillPlaceholders(markdown, facts = {}) {
+  return mapProse(markdown, (text, line) => text.replace(ANGLE, (whole, inner, offset) => {
+    const replacement = replacementFor(text, whole, inner, offset, line, facts);
+    return replacement === null ? whole : replacement;
+  }));
+}
+
+/**
+ * Every slot `fillPlaceholders` would still replace, for tests and a last check.
+ *
+ * @param {string} markdown
+ * @returns {Array<string>}
+ */
+export function findPlaceholders(markdown) {
+  const found = [];
+  mapProse(markdown, (text, line) => {
+    for (const match of text.matchAll(ANGLE)) {
+      if (replacementFor(text, match[0], match[1], match.index, line, {}) !== null) found.push(match[0]);
+    }
+    return text;
+  });
+  return found;
+}
+
+/**
+ * The same filling over the prose fields of report.json, which the dashboard
+ * shows: titles, descriptions, repro steps and quotes.
+ *
+ * @param {Object} report
+ * @param {Object} [facts]
+ * @returns {Object}
+ */
+export function fillReportPlaceholders(report, facts = {}) {
+  if (!report || !Array.isArray(report.findings)) return report;
+  const fill = value => (typeof value === 'string' ? fillPlaceholders(value, facts) : value);
+  return {
+    ...report,
+    findings: report.findings.map(finding => {
+      if (!finding || typeof finding !== 'object') return finding;
+      const out = { ...finding };
+      for (const field of ['title', 'description', 'persona_quote']) {
+        if (field in out) out[field] = fill(out[field]);
+      }
+      if (Array.isArray(out.repro_steps)) out.repro_steps = out.repro_steps.map(fill);
+      return out;
+    })
+  };
+}
+
+export default { fillPlaceholders, findPlaceholders, fillReportPlaceholders, NOT_RECORDED };

package/src/run/report.js

@@ -127,6 +127,86 @@ export function stampReport(report, facts) {
   return stamped;
 }
 
+/** Keys a model has been seen to put an evidence path under instead of `ref`. */
+const REF_ALIASES = ['path', 'file', 'screenshot', 'src', 'href', 'location', 'value'];
+
+/** The four evidence types the schema allows. */
+const EVIDENCE_TYPES = new Set(['screenshot', 'video_ts', 'transcript_step', 'file_line']);
+
+/**
+ * One evidence entry in the schema's shape, `{type, ref}` and nothing else,
+ * when the model's own words say unambiguously what it meant.
+ *
+ * The 0.2.8 report that failed upload wrote
+ * `{"type": "screenshot", "path": "screenshots/04-04-school-selected.png"}`
+ * five times: a real file, under the wrong key. The schema refuses the extra
+ * key and the missing `ref`, so the whole report was refused and the run
+ * failed over a key name. This moves the value, never invents one: a path
+ * still has to name a file in the run directory (`checkEvidence`), and the
+ * server still checks it against what was uploaded.
+ *
+ * Also: a bare string becomes a screenshot or a transcript step by its look;
+ * a missing type on a `screenshots/...` ref becomes `screenshot`; a ref
+ * written `./screenshots/x.png` or as an absolute path inside the run
+ * directory becomes the run relative path the server's manifest uses.
+ *
+ * @param {any} item
+ * @param {string|null} runDirectory
+ * @returns {any} the normalised entry, or the input unchanged when it is not an object it can read
+ */
+export function normaliseEvidenceItem(item, runDirectory = null) {
+  if (typeof item === 'string') {
+    const text = item.trim();
+    if (/^(?:\.\/)?screenshots\//.test(text)) return normaliseEvidenceItem({ type: 'screenshot', ref: text }, runDirectory);
+    if (/^(?:observation|note|step|action)\s*#?\s*\d+$/i.test(text)) return { type: 'transcript_step', ref: text };
+    return item;
+  }
+  if (!item || typeof item !== 'object' || Array.isArray(item)) return item;
+  let ref = typeof item.ref === 'string' && item.ref.trim() ? item.ref : null;
+  if (ref === null) {
+    const alias = REF_ALIASES.find(key => typeof item[key] === 'string' && item[key].trim());
+    if (alias) ref = item[alias];
+  }
+  let type = typeof item.type === 'string' ? item.type : null;
+  if (!type && ref && /^(?:\.\/)?screenshots\//.test(ref.trim())) type = 'screenshot';
+  if (type === null || ref === null || !EVIDENCE_TYPES.has(type)) return item;
+  ref = ref.trim();
+  if (type === 'screenshot') ref = runRelative(ref, runDirectory);
+  return { type, ref };
+}
+
+function runRelative(ref, runDirectory) {
+  let value = ref.replace(/\\/g, '/');
+  if (runDirectory && path.isAbsolute(ref)) {
+    const relative = path.relative(path.resolve(runDirectory), path.resolve(ref));
+    if (relative && !relative.startsWith('..') && !path.isAbsolute(relative)) value = relative.split(path.sep).join('/');
+  }
+  while (value.startsWith('./')) value = value.slice(2);
+  return value;
+}
+
+/**
+ * Every finding's evidence through {@link normaliseEvidenceItem}.
+ *
+ * @param {Object} report
+ * @param {string|null} [runDirectory]
+ * @returns {{report: Object, changed: number}} a new report, and how many entries were rewritten
+ */
+export function normaliseEvidence(report, runDirectory = null) {
+  if (!report || typeof report !== 'object' || !Array.isArray(report.findings)) return { report, changed: 0 };
+  let changed = 0;
+  const findings = report.findings.map(finding => {
+    if (!finding || typeof finding !== 'object' || !Array.isArray(finding.evidence)) return finding;
+    const evidence = finding.evidence.map(item => {
+      const out = normaliseEvidenceItem(item, runDirectory);
+      if (JSON.stringify(out) !== JSON.stringify(item)) changed += 1;
+      return out;
+    });
+    return { ...finding, evidence };
+  });
+  return { report: { ...report, findings }, changed };
+}
+
 /**
  * Check every `screenshot` evidence ref against the files actually on disk.
  *
@@ -151,8 +231,17 @@ export async function checkEvidence(report, runDirectory) {
       continue;
     }
     for (const item of evidence) {
-      if (!item || item.type !== 'screenshot') continue;
-      const ref = String(item.ref || '');
+      if (!item || typeof item !== 'object') {
+        problems.push(`${finding.id}: an evidence entry is not an object; write {"type": ..., "ref": ...}`);
+        continue;
+      }
+      if (typeof item.ref !== 'string' || !item.ref.trim()) {
+        const keys = Object.keys(item).filter(key => key !== 'type');
+        problems.push(`${finding.id}: a ${item.type || 'typeless'} evidence entry has no "ref"${keys.length > 0 ? ` (it has ${keys.map(key => `"${key}"`).join(', ')})` : ''}; every entry is exactly {"type": ..., "ref": ...}`);
+        continue;
+      }
+      if (item.type !== 'screenshot') continue;
+      const ref = String(item.ref);
       const candidate = path.resolve(runDirectory, ref);
       // Refuse a ref that climbs out of the run directory before touching
       // the filesystem with it.
@@ -214,6 +303,8 @@ export default {
   createReportValidator,
   stampReport,
   checkEvidence,
+  normaliseEvidence,
+  normaliseEvidenceItem,
   writeReportFiles,
   repairInstruction,
   lintMarkdownSections,

package/src/run/resume.js

@@ -3,6 +3,7 @@ import { getDeps } from './deps.js';
 import EXIT from './exit.js';
 import { runDir as runDirFor } from './paths.js';
 import { buildPartialReport, readTranscript, shouldFinalize } from './regenerate.js';
+import { reconcileDroppedFindings } from './dropped.js';
 import { createReportValidator, writeReportFiles } from './report.js';
 import * as stateFile from './state.js';
 import { playJob, resolvePersonas } from '../commands/run.js';
@@ -61,6 +62,15 @@ export async function resumeJob({ api, gameId, jobId, job, runs, state, repoRoot
     return EXIT.ENVIRONMENT;
   }
 
+  // A run whose report the server refused is done here, whatever the server
+  // says its state is: it is never finalised from its transcript (which would
+  // rewrite the kept report.json and report.md) and never replayed.
+  for (const [runId, entry] of Object.entries(journal.runs || {})) {
+    if (entry && entry.report_refused === true) {
+      log(`run ${runId}: the server refused its report, so it is not replayed or rebuilt. Its report.json, report.md and screenshots are in ${runDirFor(repoRoot, jobId, runId)}.`);
+    }
+  }
+
   // Reconcile the journal with what the server actually thinks: a run the sweep
   // marked interrupted, or one that finished after the journal was last
   // written, is the server's word against a stale file, and the server wins.
@@ -221,7 +231,7 @@ async function finalizeFromTranscript({ deps, api, gameId, jobId, repoRoot, row,
     // `interrupted` can go to `reporting`, which is where this run actually is:
     // it has a report now. Then `uploading`, then the delivery.
     await api.runs.transition(gameId, jobId, row.run_id, 'reporting', { reason: 'rebuilt from transcript' });
-    await deps.finalizeRun(directory, {
+    const answer = await deps.finalizeRun(directory, {
       api,
       gameId,
       jobId,
@@ -237,6 +247,20 @@ async function finalizeFromTranscript({ deps, api, gameId, jobId, repoRoot, row,
         report_md: markdown
       }
     });
+    if (answer && answer.dropped_findings > 0) {
+      log(`run ${row.run_id}: the server dropped ${answer.dropped_findings} finding(s) with no usable evidence.`);
+      try {
+        await reconcileDroppedFindings({
+          directory,
+          droppedCount: answer.dropped_findings,
+          droppedIds: answer.dropped_finding_ids,
+          options,
+          journal: { jobId, runId: row.run_id, repoRoot }
+        });
+      } catch (markError) {
+        log(`run ${row.run_id}: could not mark the dropped finding(s) in the local report (${String(markError && markError.message)}).`);
+      }
+    }
   } catch (error) {
     log(`run ${row.run_id}: could not be finalised (${String(error && error.message)}); it will be replayed.`);
     return false;

package/src/run/state.js

@@ -192,7 +192,10 @@ export async function patchRun(repoRoot, state, runId, patch) {
  * @returns {boolean}
  */
 export function isRunDone(entry) {
-  return Boolean(entry) && (entry.completed === true || RUN_DONE_STATES.includes(entry.state));
+  // A run whose report the server refused was played to the end: replaying
+  // it would spend a unit again, and rebuilding its report from the
+  // transcript would overwrite the one the developer is meant to read.
+  return Boolean(entry) && (entry.completed === true || entry.report_refused === true || RUN_DONE_STATES.includes(entry.state));
 }
 
 export default { SCHEMA_VERSION, newState, newRunEntry, save, load, patchRun, isRunDone, stateFile };

package/src/run/synthetic.js

@@ -193,6 +193,8 @@ export async function emitRunEvents({ apiUrl, ingestKey, playtestToken, pageSdk
 
     session.track('playtest_run_started', { driver });
     for (const finding of Array.isArray(report && report.findings) ? report.findings : []) {
+      // The server kept no record of a dropped finding, so neither does the game's telemetry.
+      if (finding && finding.dropped === true) continue;
       session.track('playtest_finding', {
         finding_id: finding.id,
         severity: finding.severity,

package/src/run/tools.js

@@ -86,7 +86,7 @@ export const PLAY_TOOLS = Object.freeze([
   },
   {
     name: 'type',
-    description: 'Type text. Spends an action. Set submit to press Enter afterwards.',
+    description: 'Type text. Spends an action. Set submit to press Enter afterwards. To correct or clear a text field, set replace: the field is emptied first and holds only this text afterwards (an empty text clears it). Do that rather than a select-all key chord, which differs between platforms.',
     input_schema: {
       type: 'object',
       additionalProperties: false,
@@ -94,6 +94,8 @@ export const PLAY_TOOLS = Object.freeze([
       properties: {
         text: { type: 'string' },
         submit: { type: 'boolean' },
+        replace: { type: 'boolean', description: 'Empty the field first, so it holds only this text. playwright_web only; other drivers refuse it.' },
+        target: { type: 'string', description: 'playwright_web only, other drivers refuse it: the field to type into, named like a click target, such as textbox "Email". Without it, the focused field.' },
         thought_in_character: thought,
         note
       }
@@ -101,7 +103,7 @@ export const PLAY_TOOLS = Object.freeze([
   },
   {
     name: 'key',
-    description: 'Press one key. Spends an action. Names are the usual ones: Enter, Space, ArrowUp, Escape, a, 1.',
+    description: 'Press one key. Spends an action. Names are the usual ones: Enter, Space, ArrowUp, Escape, a, 1. Hold a modifier with +, such as Shift+Tab; editing shortcuts such as Control+A are sent with this platform\'s own modifier. To clear a text field, use type with replace instead.',
     input_schema: {
       type: 'object',
       additionalProperties: false,
@@ -187,8 +189,12 @@ export function actionFor(toolUse) {
       return Number.isFinite(Number(input.x)) && Number.isFinite(Number(input.y))
         ? { kind: 'click', target: { x: Number(input.x), y: Number(input.y) } }
         : { kind: 'click', target: String(input.target || '') };
-    case 'type':
-      return { kind: 'type', text: String(input.text || ''), submit: Boolean(input.submit) };
+    case 'type': {
+      const action = { kind: 'type', text: String(input.text || ''), submit: Boolean(input.submit) };
+      if (input.replace === true) action.replace = true;
+      if (typeof input.target === 'string' && input.target.trim()) action.target = input.target;
+      return action;
+    }
     case 'key':
       return { kind: 'key', key: String(input.key || '') };
     case 'navigate':

package/src/upload/index.js

@@ -274,7 +274,7 @@ export async function uploadRunDir(runDir, options) {
   const {
     api, gameId, jobId, runId = null,
     includeVideo = false, includeTranscript = false,
-    dryRun = false, force = false, only = null, allowedPaths = null,
+    dryRun = false, force = false, only = null, allowedPaths = null, exclude = null, excludeReason = 'excluded',
     onFile, repoRoot, fetch: fetchImpl, sleep
   } = options;
 
@@ -295,6 +295,14 @@ export async function uploadRunDir(runDir, options) {
     files = files.filter(file => wanted.has(file.path));
   }
 
+  if (Array.isArray(exclude) && exclude.length > 0) {
+    const unwanted = new Set(exclude);
+    for (const file of files) {
+      if (unwanted.has(file.path)) skipped.push({ path: file.path, reason: excludeReason });
+    }
+    files = files.filter(file => !unwanted.has(file.path));
+  }
+
   if (!force) {
     const done = await uploadedKeys(jobId, journalOptions);
     const remaining = [];

package/src/upload/queue.js

@@ -14,7 +14,10 @@ import { paths } from '../paths.js';
  * was.
  *
  * Each line is `{ at, kind, run_id, path, sha256, size, status, detail }`.
- * `status` is one of `queued`, `uploaded`, `failed`, `skipped`.
+ * `status` is one of `queued`, `uploaded`, `failed`, `skipped`, `local_edit`.
+ * A `local_edit` is a file the CLI changed on disk after it was delivered
+ * (dropped findings marked in a run's report): the delivered copy is the
+ * record, so the new hash counts as done and is never sent again.
  *
  * Idempotency is by sha256: a path recorded `uploaded` with the same hash is
  * not sent again, which is what makes `resume` and the manual `upload <job>`
@@ -25,9 +28,13 @@ export const STATUS = Object.freeze({
   QUEUED: 'queued',
   UPLOADED: 'uploaded',
   FAILED: 'failed',
-  SKIPPED: 'skipped'
+  SKIPPED: 'skipped',
+  LOCAL_EDIT: 'local_edit'
 });
 
+/** Statuses that mean "nothing left to send at this hash". */
+const DONE = new Set([STATUS.UPLOADED, STATUS.LOCAL_EDIT]);
+
 /**
  * Append one entry.
  * @param {string} jobId
@@ -70,7 +77,8 @@ export async function readQueue(jobId, options = {}) {
 }
 
 /**
- * The set of `<runId>|<path>|<sha256>` triples already recorded as uploaded.
+ * The set of `<runId>|<path>|<sha256>` triples already recorded as uploaded,
+ * or as a local edit of a file that was.
  * The sha256 is part of the key on purpose: a corrected file at the same path
  * has a different hash and must be sent again.
  * @param {string} jobId
@@ -80,7 +88,7 @@ export async function readQueue(jobId, options = {}) {
 export async function uploadedKeys(jobId, options = {}) {
   const done = new Set();
   for (const entry of await readQueue(jobId, options)) {
-    if (entry.status === STATUS.UPLOADED) {
+    if (DONE.has(entry.status)) {
       done.add(keyFor(entry.run_id || null, entry.path, entry.sha256));
     }
   }
@@ -105,7 +113,7 @@ export async function pending(jobId, options = {}) {
   for (const entry of entries) {
     if (!entry.path) continue;
     const key = keyFor(entry.run_id || null, entry.path, entry.sha256);
-    if (entry.status === STATUS.UPLOADED) done.add(key);
+    if (DONE.has(entry.status)) done.add(key);
     latest.set(key, entry);
   }
   return [...latest.entries()]