ravensight-playtest 0.2.6 → 0.2.8

package/src/run/drivers/web.js

@@ -35,6 +35,12 @@ export const MAX_WAIT_MS = 10000;
 /** How long a single Playwright call may take before it is an error, not a hang. */
 export const ACTION_TIMEOUT_MS = 15000;
 
+/**
+ * How long a replace waits for a focused field. Short, because nothing
+ * focused is an ordinary answer (a canvas game), not a page still loading.
+ */
+export const FOCUSED_FILL_TIMEOUT_MS = 2000;
+
 /**
  * The web observation ceiling. An aria snapshot is re-sent on every turn the
  * observation stays in context, and the run's model budget is fixed, so a busy
@@ -116,6 +122,74 @@ export function unquoteKey(body) {
   return body;
 }
 
+/**
+ * The modifier a platform's editing shortcuts use: Command on macOS,
+ * Control everywhere else. Chromium follows the host's conventions even
+ * headless, so on a Mac `Control+A` in a text field moves the caret to the
+ * start of the line instead of selecting the text, and a persona that
+ * "selects all and retypes" ends up with two values joined together.
+ *
+ * @param {string} [platform] a `process.platform` value
+ * @returns {'Meta'|'Control'}
+ */
+export function platformModifier(platform = process.platform) {
+  return platform === 'darwin' ? 'Meta' : 'Control';
+}
+
+/** Editing shortcuts (select all, copy, cut, paste, undo, redo) that follow the platform modifier. */
+const EDITING_KEYS = new Set(['a', 'c', 'v', 'x', 'z', 'y']);
+
+/** Spellings a model writes for a modifier, mapped to Playwright's names. */
+const MODIFIER_ALIASES = Object.freeze({
+  ctrl: 'Control',
+  control: 'Control',
+  cmd: 'Meta',
+  command: 'Meta',
+  meta: 'Meta',
+  super: 'Meta',
+  win: 'Meta',
+  mod: 'ControlOrMeta',
+  controlormeta: 'ControlOrMeta',
+  alt: 'Alt',
+  option: 'Alt',
+  opt: 'Alt',
+  shift: 'Shift'
+});
+
+/**
+ * Rewrite a key chord the persona asked for into one that does the same
+ * thing on this machine.
+ *
+ * Modifier spellings are normalised (`Ctrl`, `Cmd`, `Option`), and an
+ * editing shortcut held with Control or Command (`Control+A`, `Cmd+V`,
+ * `Ctrl+Shift+Z`) uses the platform modifier, because the persona means
+ * "select all", not "the Control key". Any other chord is left as written:
+ * a game that binds Control+S to something of its own wants Control on
+ * every platform.
+ *
+ * @param {string} key
+ * @param {'Meta'|'Control'} modifier the platform modifier
+ * @returns {string}
+ */
+export function normalizeChord(key, modifier) {
+  const value = String(key == null ? '' : key);
+  // A lone "+" is the plus key, and "Shift++" is Shift and plus.
+  const parts = value.length > 1 ? value.split(/\+(?=.)/) : [value];
+  if (parts.length < 2) return value;
+  const mapped = parts.map((part, index) => {
+    if (index === parts.length - 1) return part;
+    return MODIFIER_ALIASES[part.trim().toLowerCase()] || part;
+  });
+  const last = mapped[mapped.length - 1];
+  const editing = EDITING_KEYS.has(String(last).toLowerCase());
+  return mapped.map((part, index) => {
+    if (index === mapped.length - 1) return part;
+    if (part === 'ControlOrMeta') return modifier;
+    if (editing && (part === 'Control' || part === 'Meta')) return modifier;
+    return part;
+  }).join('+');
+}
+
 const KNOWN_SELECTOR_ENGINES = /^(role|text|css|xpath|id|data-testid|internal:)=/;
 
 /**
@@ -231,9 +305,14 @@ function injectPlaytestIdentity(identity) {
  * @param {{token?: string, runId?: string, jobId?: string, persona?: string}} [opts.playtest]
  *   the run's playtest identity, handed to the page so the game's own SDK
  *   session is marked synthetic
+ * @param {string} [opts.platform] the host platform, `process.platform` by
+ *   default; decides the editing modifier
+ * @param {'Meta'|'Control'} [opts.modifier] the editing modifier, overriding
+ *   the one the platform implies
  * @returns {Object} a driver
  */
-export function createWebDriver({ importer, headless = true, playtest = null } = {}) {
+export function createWebDriver({ importer, headless = true, playtest = null, platform = process.platform, modifier = null } = {}) {
+  const editModifier = modifier === 'Meta' || modifier === 'Control' ? modifier : platformModifier(platform);
   let browser = null;
   let context = null;
   let page = null;
@@ -297,8 +376,42 @@ export function createWebDriver({ importer, headless = true, playtest = null } =
     return truncateObservation(snap.aria ? compactAriaSnapshot(snap.text) : snap.text, WEB_OBSERVATION_MAX_CHARS);
   }
 
+  /**
+   * Type, or with `replace` put this text in place of what the field holds.
+   *
+   * Replacing goes through Playwright's `fill`, which clears the field
+   * natively on every platform, rather than a select-all chord: the chord is
+   * Command+A on a Mac and Control+A elsewhere, and getting it wrong joins the
+   * old value to the new one. A target names the field; without one the
+   * focused element is filled. Only an element `fill` refuses (a canvas, a
+   * custom widget) falls back to the platform select-all, then typing.
+   *
+   * @param {any} current the page
+   * @param {{text: string, target?: string, replace?: boolean}} action
+   */
+  async function typeText(current, action) {
+    const text = String(action.text == null ? '' : action.text);
+    const field = action.target ? resolveLocator(current, action.target) : null;
+    if (action.replace) {
+      try {
+        await (field || current.locator('*:focus')).fill(text, { timeout: field ? ACTION_TIMEOUT_MS : FOCUSED_FILL_TIMEOUT_MS });
+        return;
+      } catch {
+        // Not something fill can write to. Focus it if it was named, then
+        // clear it the way a player on this machine would.
+        if (field) await field.click({ timeout: ACTION_TIMEOUT_MS });
+        await current.keyboard.press(`${editModifier}+a`);
+        await current.keyboard.press('Backspace');
+      }
+    } else if (field) {
+      await field.click({ timeout: ACTION_TIMEOUT_MS });
+    }
+    if (text) await current.keyboard.type(text, { delay: 20 });
+  }
+
   return {
     name: 'playwright_web',
+    editModifier,
     keepObservations: WEB_KEEP_OBSERVATIONS,
 
     /**
@@ -361,6 +474,16 @@ export function createWebDriver({ importer, headless = true, playtest = null } =
         url: current.url(),
         elapsedMs: Date.now() - started
       };
+      // The page title is what a note's "after" state is checked against in
+      // the report step. A page that cannot answer has no title, not an error.
+      try {
+        if (typeof current.title === 'function') {
+          const title = await current.title();
+          if (title) observation.title = String(title);
+        }
+      } catch {
+        // No title is still an observation.
+      }
       // Canvas builds (a Godot or Unity web export is one <canvas> and
       // nothing else) have no accessibility tree worth reading, so vision
       // is the only way to play them. Spend an image only then.
@@ -397,10 +520,10 @@ export function createWebDriver({ importer, headless = true, playtest = null } =
       try {
         switch (action.kind) {
           case 'key':
-            await current.keyboard.press(String(action.key), { timeout: ACTION_TIMEOUT_MS });
+            await current.keyboard.press(normalizeChord(action.key, editModifier), { timeout: ACTION_TIMEOUT_MS });
             break;
           case 'type':
-            await current.keyboard.type(String(action.text), { delay: 20 });
+            await typeText(current, action);
             if (action.submit) await current.keyboard.press('Enter');
             break;
           case 'click': {

package/src/run/drivers/webdriver.js

@@ -51,6 +51,7 @@ export function createWebDriverAdapter({name='desktop_driver',fetch:fetchImpl=gl
     }else if(action.kind==='swipe'){
      const from=point(action.from),to=point(action.to);await actions('pointer',[{type:'pointerMove',duration:0,origin:'viewport',...from},{type:'pointerDown',button:0},{type:'pointerMove',duration:Math.min(2000,Math.max(100,Number(action.durationMs)||500)),origin:'viewport',...to},{type:'pointerUp',button:0}],touch);
     }else if(action.kind==='key'||action.kind==='type'){
+     if(action.kind==='type'&&(action.replace||action.target))throw new DriverError('replace (empty a field, then type) and a typing target are only available on playwright_web; this driver can only type more text into what has focus');
      const chars=action.kind==='type'?Array.from(String(action.text||'').slice(0,2000)):[KEYS[action.key]||action.key];
      if(chars.some(key=>typeof key!=='string'||Array.from(key).length!==1))throw new DriverError('Unsupported device key.');
      if(action.submit)chars.push(KEYS.Enter);

package/src/run/dropped.js

@@ -0,0 +1,294 @@
+import { readFile, stat } from 'node:fs/promises';
+import path from 'node:path';
+import { sha256File } from '../fsutil.js';
+import { planRunDir } from '../upload/index.js';
+import { record, STATUS } from '../upload/queue.js';
+import { TRANSCRIPT_PATH, VIDEO_PATHS } from '../upload/allowlist.js';
+import { RUN_FILES } from './paths.js';
+import { fillPlaceholders } from './placeholders.js';
+import { writeReportFiles } from './report.js';
+
+/**
+ * Findings the server dropped, marked in the local report so it agrees with
+ * the dashboard.
+ *
+ * At `complete` the server keeps a finding only when at least one of its
+ * evidence refs resolves to a file the run actually uploaded, and answers
+ * with how many it dropped, not which. Without this step the CLI said "the
+ * server dropped 1 finding" while report.md and report.json still listed it,
+ * major severity and all, and the dashboard showed one fewer.
+ *
+ * The CLI uploaded the files itself, so it can apply the server's rule to
+ * the same list and name them. When its count matches the server's, each
+ * one is marked `dropped: true` in report.json and moved in report.md to a
+ * section of its own: kept for the developer to read, out of the top issues,
+ * the aggregate and the synthetic telemetry. When the counts disagree it
+ * does not guess which; it says how many the server dropped and points at
+ * the dashboard.
+ */
+
+/** The section dropped findings are moved to. */
+export const DROPPED_HEADING = '## Dropped by the server (no usable evidence)';
+
+/**
+ * The uploaded path, or candidate paths, an evidence entry needs. The same
+ * mapping as the server's `manifestPathForEvidence`.
+ *
+ * @param {{type?: string, ref?: string}} evidence
+ * @returns {string|Array<string>|null}
+ */
+export function evidencePath(evidence) {
+  if (!evidence || typeof evidence.type !== 'string') return null;
+  const ref = typeof evidence.ref === 'string' ? evidence.ref : '';
+  switch (evidence.type) {
+    case 'screenshot':
+      return ref || null;
+    case 'video_ts':
+      return [...VIDEO_PATHS];
+    case 'transcript_step':
+      return TRANSCRIPT_PATH;
+    case 'file_line':
+      return ref ? ref.split(':')[0] : null;
+    default:
+      return null;
+  }
+}
+
+/**
+ * The findings none of whose evidence resolves to an uploaded file.
+ *
+ * @param {Object} report
+ * @param {Iterable<string>} uploaded the run relative paths that were uploaded
+ * @returns {Array<Object>}
+ */
+export function unbackedFindings(report, uploaded) {
+  const paths = new Set(uploaded);
+  const findings = Array.isArray(report && report.findings) ? report.findings : [];
+  return findings.filter(finding => {
+    if (!finding || finding.dropped === true) return false;
+    const evidence = Array.isArray(finding.evidence) ? finding.evidence : [];
+    return !evidence.some(entry => {
+      const wanted = evidencePath(entry);
+      if (wanted === null) return false;
+      return (Array.isArray(wanted) ? wanted : [wanted]).some(candidate => paths.has(candidate));
+    });
+  });
+}
+
+/**
+ * report.json with the named findings marked.
+ *
+ * @param {Object} report
+ * @param {Set<string>} ids
+ * @returns {Object}
+ */
+export function markDroppedFindings(report, ids) {
+  return {
+    ...report,
+    findings: (report.findings || []).map(finding => (finding && ids.has(finding.id) ? { ...finding, dropped: true } : finding))
+  };
+}
+
+const SECTION = /^## /;
+const LIST_ITEM = /^\s*(?:[-*+]|\d+[.)])\s+/;
+
+/**
+ * report.md with the dropped findings out of the summary and in a section of
+ * their own.
+ *
+ * A list item that names a dropped finding id and no kept one is removed
+ * wherever it is, which is how "Top issues" names them (`F003 ...`). A
+ * section left with nothing under it says so. Then the dropped findings are
+ * listed under {@link DROPPED_HEADING}, replacing one written before, so a
+ * second reconcile does not stack them.
+ *
+ * @param {string} markdown
+ * @param {Array<Object>} dropped the dropped findings, from report.json
+ * @param {Array<Object>} kept the findings the server kept
+ * @param {{unidentified?: number}} [opts] a count the CLI could not attribute
+ * @returns {string}
+ */
+export function rewriteMarkdown(markdown, dropped, kept, { unidentified = 0 } = {}) {
+  const lines = withoutDroppedSection(String(markdown || '').replace(/\s+$/, '')).split('\n');
+  const droppedIds = dropped.map(finding => String(finding.id)).filter(Boolean);
+  const keptIds = kept.map(finding => String(finding.id)).filter(Boolean);
+  const names = (line, id) => new RegExp(`(^|[^A-Za-z0-9])${escapeRegExp(id)}(?![0-9])`).test(line);
+
+  const out = [];
+  let section = null;
+  let sectionRemoved = 0;
+  let sectionKept = 0;
+  const closeSection = () => {
+    if (section !== null && sectionRemoved > 0 && sectionKept === 0) {
+      // Trailing blank lines stay after the note, as they were after the list.
+      let blanks = 0;
+      while (out.length > 0 && out[out.length - 1].trim() === '') {
+        out.pop();
+        blanks += 1;
+      }
+      out.push('- Nothing here that the server kept. See the dropped findings below.');
+      for (let i = 0; i < blanks; i += 1) out.push('');
+    }
+  };
+  for (const line of lines) {
+    if (SECTION.test(line)) {
+      closeSection();
+      section = line;
+      sectionRemoved = 0;
+      sectionKept = 0;
+      out.push(line);
+      continue;
+    }
+    if (LIST_ITEM.test(line) && droppedIds.some(id => names(line, id)) && !keptIds.some(id => names(line, id))) {
+      sectionRemoved += 1;
+      continue;
+    }
+    if (LIST_ITEM.test(line)) sectionKept += 1;
+    out.push(line);
+  }
+  closeSection();
+
+  const tail = ['', DROPPED_HEADING];
+  if (dropped.length > 0) {
+    tail.push(
+      'The server keeps a finding only when at least one of its evidence references points at a file this run uploaded. These did not, so the dashboard does not show them and they are not counted. They are kept here so you can judge them yourself.',
+      ''
+    );
+    for (const finding of dropped) {
+      const evidence = (Array.isArray(finding.evidence) ? finding.evidence : [])
+        .map(entry => `${entry && entry.type} ${entry && entry.ref}`)
+        .join(', ') || 'none';
+      // The title is the model's prose, copied here; fill any slot it carries first.
+      tail.push(`- ${finding.id} (${finding.severity}, ${finding.category}): ${fillPlaceholders(String(finding.title || ''))}. Evidence cited: ${evidence}.`);
+    }
+  }
+  if (unidentified > 0) {
+    tail.push(
+      `The server dropped ${unidentified} finding(s) with no usable evidence, and this report could not tell which, so every finding above is still listed and counted here. The dashboard shows the findings the server kept.`
+    );
+  }
+  return `${out.join('\n')}\n${tail.join('\n')}\n`;
+}
+
+/**
+ * Remove a dropped section written by an earlier reconcile.
+ *
+ * @param {string} markdown
+ * @returns {string}
+ */
+function withoutDroppedSection(markdown) {
+  const lines = markdown.split('\n');
+  const start = lines.findIndex(line => line.trim() === DROPPED_HEADING);
+  if (start === -1) return markdown;
+  let end = lines.findIndex((line, index) => index > start && SECTION.test(line));
+  if (end === -1) end = lines.length;
+  return [...lines.slice(0, start), ...lines.slice(end)].join('\n').replace(/\s+$/, '');
+}
+
+function escapeRegExp(value) {
+  return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+/**
+ * Mark the findings the server dropped in a run directory's report files.
+ *
+ * The server's own list (`dropped_finding_ids`) is used when it sends one.
+ * Without it the CLI applies the server's evidence rule to its upload plan,
+ * and marks only when that count matches the server's. Findings marked by an
+ * earlier reconcile count towards the server's number, so running it twice
+ * changes nothing.
+ *
+ * Rewriting the two files changes their sha256, which would make `upload
+ * <job>` send them again over the validated artifacts. So each new hash is
+ * recorded in the job's upload journal as a local edit, which the journal
+ * treats as already delivered.
+ *
+ * @param {Object} args
+ * @param {string} args.directory the run directory
+ * @param {number} args.droppedCount `dropped_findings` from the complete answer
+ * @param {Array<string>} [args.droppedIds] `dropped_finding_ids` from the complete answer, when the server sends it
+ * @param {{uploadVideo?: boolean, uploadTranscript?: boolean}} [args.options] what was uploaded
+ * @param {{jobId: string, runId: string, repoRoot?: string}} [args.journal] where to record the local edit
+ * @param {(dir: string, opts: Object) => Promise<{files: Array<{path: string}>}>} [args.plan] test seam
+ * @returns {Promise<{report: Object, markdown: string, identified: boolean, dropped: Array<string>, source: string}|null>}
+ *   null when there was nothing to mark or no report on disk
+ */
+export async function reconcileDroppedFindings({ directory, droppedCount, droppedIds = null, options = {}, journal = null, plan = planRunDir }) {
+  const serverIds = Array.isArray(droppedIds) ? droppedIds.map(String).filter(Boolean) : null;
+  const count = serverIds && serverIds.length > 0 ? serverIds.length : Number(droppedCount) || 0;
+  if (count <= 0) return null;
+  let report;
+  let markdown;
+  try {
+    report = JSON.parse(await readFile(path.join(directory, RUN_FILES.reportJson), 'utf8'));
+    markdown = await readFile(path.join(directory, RUN_FILES.reportMd), 'utf8');
+  } catch {
+    return null;
+  }
+  if (!report || !Array.isArray(report.findings)) return null;
+
+  const already = report.findings.filter(finding => finding && finding.dropped === true).map(finding => finding.id);
+  let ids;
+  let identified;
+  let source;
+  if (serverIds && serverIds.length > 0) {
+    const present = new Set(report.findings.filter(Boolean).map(finding => finding.id));
+    ids = new Set(serverIds.filter(id => present.has(id)));
+    identified = ids.size === serverIds.length;
+    source = 'server';
+  } else {
+    const planned = await plan(directory, { video: Boolean(options.uploadVideo), transcript: Boolean(options.uploadTranscript) });
+    const unbacked = unbackedFindings(report, planned.files.map(file => file.path)).map(finding => finding.id);
+    identified = already.length + unbacked.length === count;
+    ids = new Set(identified ? [...already, ...unbacked] : already);
+    source = 'plan';
+  }
+  const marked = markDroppedFindings(report, ids);
+  const dropped = marked.findings.filter(finding => finding && finding.dropped === true);
+  const kept = marked.findings.filter(finding => !(finding && finding.dropped === true));
+  const rewritten = rewriteMarkdown(markdown, dropped, kept, { unidentified: identified ? 0 : count - dropped.length });
+  await writeReportFiles({ runDirectory: directory, markdown: rewritten, report: marked });
+  await recordLocalEdits(directory, journal);
+  return { report: marked, markdown: rewritten, identified, dropped: [...ids], source };
+}
+
+/**
+ * Record the rewritten report files' hashes as local edits, so the upload
+ * journal treats them as delivered rather than owed.
+ *
+ * @param {string} directory
+ * @param {{jobId: string, runId: string, repoRoot?: string}|null} journal
+ */
+async function recordLocalEdits(directory, journal) {
+  const where = journal && journal.jobId && journal.repoRoot ? journal : journalFromDirectory(directory) || (journal && journal.jobId ? journal : null);
+  if (!where) return;
+  for (const name of [RUN_FILES.reportMd, RUN_FILES.reportJson]) {
+    const file = path.join(directory, name);
+    const info = await stat(file);
+    await record(where.jobId, {
+      kind: 'run',
+      run_id: where.runId,
+      path: name,
+      sha256: await sha256File(file),
+      size: info.size,
+      status: STATUS.LOCAL_EDIT,
+      detail: 'dropped findings marked after complete; the delivered copy stays the record'
+    }, where.repoRoot ? { repoRoot: where.repoRoot } : {});
+  }
+}
+
+/**
+ * The job, run and repository a run directory belongs to, from its own path
+ * (`<repo>/.ravensight/jobs/<jobId>/runs/<runId>`), or null for any other.
+ *
+ * @param {string} directory
+ * @returns {{jobId: string, runId: string, repoRoot: string}|null}
+ */
+export function journalFromDirectory(directory) {
+  const parts = path.resolve(directory).split(path.sep);
+  const n = parts.length;
+  if (n < 6 || parts[n - 2] !== 'runs' || parts[n - 4] !== 'jobs' || parts[n - 5] !== '.ravensight') return null;
+  return { jobId: parts[n - 3], runId: parts[n - 1], repoRoot: parts.slice(0, n - 5).join(path.sep) || path.sep };
+}
+
+export default { reconcileDroppedFindings, unbackedFindings, markDroppedFindings, rewriteMarkdown, evidencePath, DROPPED_HEADING };