ravensight-playtest 0.2.8 → 0.3.0

package/src/commands/run.js

@@ -12,6 +12,7 @@ import { createModelClient, ModelUnavailableError, ProxyError } from '../run/mod
 import { jobDir, runDir as runDirFor, JOB_FILES, RUN_FILES } from '../run/paths.js';
 import { readBridgeConfig } from "../run/drivers/stdio.js";
 import { WEBDRIVER_NAMES, readDriverConfig } from '../run/drivers/webdriver.js';
+import { automationChecks } from './check.js';
 import { runPersona } from '../run/personaLoop.js';
 import { reconcileDroppedFindings } from '../run/dropped.js';
 import * as stateFile from '../run/state.js';
@@ -106,6 +107,31 @@ async function run(flags, injected, deps, log) {
   // dedup and success-criteria passes.
   if (WEBDRIVER_NAMES.includes(driverName)) await readDriverConfig(target);
   if (driverName === "cli_stdio") await readBridgeConfig(target);
+
+  // The free capability probe. A native run depends on the developer's own
+  // automation server, which no amount of SDK or token setup proves, so the
+  // automation rows of `check` run first and a failure stops the run before
+  // the estimate, the confirmation and the register: no job, no reservation.
+  if (WEBDRIVER_NAMES.includes(driverName) && !options.skipPreflight) {
+    // Each row is printed as it completes, and the slow step says so before
+    // it starts, so a first session on a fresh simulator does not look hung.
+    const preflight = await (injected.preflight || automationChecks)(
+      { driverName, target, repoRoot: process.cwd() },
+      {
+        ...(injected.preflightDeps || {}),
+        onProgress: line => log(line),
+        onRow: entry => log(`preflight ${entry.status} ${entry.name}: ${entry.detail}`)
+      }
+    );
+    const failed = preflight.rows.filter(entry => entry.status === 'fail');
+    if (failed.length > 0) {
+      for (const entry of failed) {
+        log(`Preflight failed at "${entry.name}": ${entry.detail}${entry.fix ? `. Fix: ${entry.fix}` : ''}`);
+      }
+      log('No job was registered and nothing was reserved. Run "ravensight-playtest check" for every row, or pass --skip-preflight to register anyway.');
+      return EXIT.FAILED;
+    }
+  }
   const modules = ['persona_playtest', 'aggregate_report'];
   const pack = await deps.packs.get({ api, gameId, modules });
   if (options.controller === 'jev' && !pack.routing?.action_controllers?.includes('jev')) { log('This server does not currently support the requested action controller. No job was registered.'); return EXIT.ENVIRONMENT; }
@@ -445,7 +471,10 @@ export async function playJob({
   const reportBudgetRefused = results.some(result => result.reportStepBudget === 'refused');
   const reportBudgetDelivered = results.some(result => result.reportStepBudget === 'delivered' || result.reportStepBudget === 'dry_upload');
   const canceled = results.some(result => result.state === 'canceled');
-  const anySucceeded = results.some(result => result.state === 'succeeded');
+  // A run whose report the server refused played, but the server never
+  // completed it: it is not a success, and CI should hear about it.
+  const anySucceeded = results.some(result => result.state === 'succeeded' && !result.reportRefused);
+  const reportRefused = results.some(result => result.reportRefused);
 
   if (finish) {
     const finished = await api.jobs.finish(gameId, jobId, {
@@ -459,7 +488,7 @@ export async function playJob({
   log(`Reports: ${directory}`);
 
   if (canceled) return EXIT.CANCELED;
-  if (reportBudgetRefused) return EXIT.FAILED;
+  if (reportBudgetRefused || reportRefused) return EXIT.FAILED;
   if (playBudgetHit) return EXIT.BUDGET;
   return anySucceeded || reportBudgetDelivered ? EXIT.OK : EXIT.FAILED;
 }
@@ -558,13 +587,31 @@ export async function playPersonaRun({
       heartbeat,
       ledger,
       recordVideo: Boolean(options.uploadVideo),
+      // Written the moment the warning arrives, not after the run: a run that
+      // then fails still says in state.json that it was warned.
+      onBudgetWarning: async () => {
+        try {
+          await stateFile.patchRun(repoRoot, journal, runId, { budget_warned: true });
+        } catch {
+          // The journal write must not kill a paid run.
+        }
+      },
       log: message => log(`run ${runId}: ${message}`)
     });
   } catch (error) {
     heartbeat.stop();
     if (inFlight) inFlight.delete(runId);
     const failed = error.outcome || { state: 'failed', quitReason: 'error', actionsTaken: 0, report: null, reportLint: [], reachedState: 'playing' };
-    await stateFile.patchRun(repoRoot, journal, runId, { state: failed.state, error: String(error && error.message), quit_reason: failed.quitReason });
+    await stateFile.patchRun(repoRoot, journal, runId, {
+      state: failed.state,
+      error: String(error && error.message),
+      quit_reason: failed.quitReason,
+      // The failure path records these too. In the Texas run state.json said
+      // budget_warned: false for a run whose log had printed the warning,
+      // because only the success path wrote it.
+      ...(failed.budgetWarned ? { budget_warned: true } : {}),
+      report_written: Boolean(failed.report && !failed.report.partial)
+    });
     const delivery = {};
     const answer = await deliver({ deps, api, gameId, jobId, runId, directory, outcome: failed, options, log, repoRoot, delivery });
     if (isReportStepBudget(failed)) {
@@ -597,7 +644,8 @@ export async function playPersonaRun({
           state: 'failed',
           reason: `ran out of budget while writing the report; the server refused budget_exceeded (${delivery.message || 'no reason given'})`,
           actions_taken: failed.actionsTaken,
-          quit_reason: failed.quitReason
+          quit_reason: failed.quitReason,
+          ...(failed.duration ? { duration: failed.duration } : {})
         });
         await stateFile.patchRun(repoRoot, journal, runId, { completed: true, state: 'failed' });
         log(`run ${runId} (${persona.slug}): completed as failed instead.`);
@@ -620,7 +668,15 @@ export async function playPersonaRun({
     budget_warned: outcome.budgetWarned
   });
 
-  await deliver({ deps, api, gameId, jobId, runId, directory, outcome, options, log, journal, repoRoot });
+  const delivered = {};
+  await deliver({ deps, api, gameId, jobId, runId, directory, outcome, options, log, journal, repoRoot, delivery: delivered });
+
+  if (delivered.reportInvalid) {
+    // No synthetic telemetry for findings the server does not hold, and no
+    // "succeeded" line for a run the server never completed.
+    log(`run ${runId} (${persona.slug}) played ${outcome.actionsTaken} actions (${outcome.quitReason}), but the server refused its report, so the run is not complete.`);
+    return { runId, persona: persona.slug, state: outcome.state, quitReason: outcome.quitReason, reportRefused: true };
+  }
 
   if (outcome.report) {
     const emitted = await emitRunEvents({
@@ -665,6 +721,14 @@ export async function deliver({ deps, api, gameId, jobId, runId, directory, outc
     actions_taken: outcome.actionsTaken,
     quit_reason: outcome.quitReason
   };
+  // The run card's duration. The server lane accepts `duration`; an older
+  // server ignores a key it does not know.
+  if (outcome.duration) {
+    body.duration = {
+      wall_seconds: Number.isFinite(Number(outcome.duration.wall_seconds)) ? Number(outcome.duration.wall_seconds) : null,
+      model_seconds: Number.isFinite(Number(outcome.duration.model_seconds)) ? Number(outcome.duration.model_seconds) : null
+    };
+  }
   if (outcome.report) body.report = outcome.report;
   // `report_md` is what the server lints for its four required sections. Not
   // sending it means `report_lint` is silently empty on every run, which reads
@@ -713,7 +777,7 @@ export async function deliver({ deps, api, gameId, jobId, runId, directory, outc
       const { report: ignored, report_md: alsoIgnored, ...withoutReport } = body;
       answer = await api.runs.complete(gameId, jobId, runId, withoutReport);
     }
-    if (journal) await stateFile.patchRun(repoRoot, journal, runId, { uploaded: true, completed: true });
+    if (journal) await stateFile.patchRun(repoRoot, journal, runId, { uploaded: true, completed: true, report_refused: false });
     if (answer && answer.dropped_findings > 0) {
       log(`run ${runId}: the server dropped ${answer.dropped_findings} finding(s) with no usable evidence.`);
       await markDropped({ directory, jobId, runId, repoRoot, outcome, droppedCount: answer.dropped_findings, droppedIds: answer.dropped_finding_ids, options, log });
@@ -728,11 +792,36 @@ export async function deliver({ deps, api, gameId, jobId, runId, directory, outc
       delivery.status = error && error.status;
       delivery.code = error && error.code;
     }
+    if (error && error.status === 422 && error.code === 'invalid_report') {
+      // Every validation message first, one per line, before anything says
+      // the run did not complete: the summary line alone ("does not match
+      // the playtest report schema") is not something anyone can act on.
+      const details = refusalDetails(error);
+      log(`run ${runId}: the server refused report.json (${details.length} problem(s)):\n${details.length > 0 ? details.map(line => `  - ${line}`).join('\n') : '  - (the server named no problem)'}`);
+      if (delivery) delivery.reportInvalid = true;
+      if (journal) await stateFile.patchRun(repoRoot, journal, runId, { report_refused: true });
+      log(`run ${runId}: not completed, so the server keeps no findings from it; whether its unit is refunded is decided when the job settles. report.json and the screenshots stay in ${directory}.`);
+      return null;
+    }
     log(`run ${runId}: upload or complete was refused (${String(error && error.message)}). The artifacts are in the run directory and "ravensight-playtest resume" can retry it.`);
     return null;
   }
 }
 
+/**
+ * The validation messages a 422 carried, as `pointer: message` lines. Read
+ * from `details` (the API client's error) or `body.details` (the envelope).
+ *
+ * @param {any} error
+ * @returns {Array<string>}
+ */
+export function refusalDetails(error) {
+  const list = Array.isArray(error && error.details) && error.details.length > 0
+    ? error.details
+    : (error && error.body && Array.isArray(error.body.details) ? error.body.details : []);
+  return list.map(detail => `${(detail && detail.instancePath) || '/'}: ${(detail && detail.message) || 'is invalid'}`);
+}
+
 /**
  * Bring the local report into line with what the server kept, so report.md,
  * report.json, the aggregate and the telemetry count what the dashboard
@@ -843,7 +932,7 @@ async function aggregatePhase({
 
   // The aggregate reads reports off disk, so a resumed job whose earlier runs
   // were played by a previous process is covered too.
-  const reports = await loadRunReports(repoRoot, jobId, personaRuns);
+  const reports = await loadRunReports(repoRoot, jobId, personaRuns, journal);
   let aggregate;
   try {
     aggregate = await buildAggregate({
@@ -1107,9 +1196,16 @@ export function resolveKeyModifier(value, env = process.env) {
  * from the notes when the money ran out) is loaded as a `budget_exceeded`
  * run: its findings count, and the aggregate says the job is partial.
  */
-export async function loadRunReports(repoRoot, jobId, personaRuns) {
+export async function loadRunReports(repoRoot, jobId, personaRuns, journal = null) {
   const out = [];
   for (const entry of personaRuns) {
+    // A report the server refused is on disk but not in the record: the
+    // summary must not count findings the dashboard does not have.
+    const recorded = journal && journal.runs ? journal.runs[entry.run_id] : null;
+    if (recorded && recorded.report_refused === true) {
+      out.push({ runId: entry.run_id, persona: entry.persona, state: 'failed', report: null, reportRefused: true });
+      continue;
+    }
     const file = path.join(runDirFor(repoRoot, jobId, entry.run_id), RUN_FILES.reportJson);
     try {
       const report = JSON.parse(await readFile(file, 'utf8'));

package/src/run/aggregate.js

@@ -90,43 +90,100 @@ export function issueId(signature) {
 }
 
 /**
- * Candidate clusters: same signature, or a shared evidence ref.
+ * A title with case, punctuation and spacing taken out, for the exact
+ * duplicate check: "Roster never loads!" and "roster never loads" are one
+ * finding written twice.
+ *
+ * @param {string} title
+ * @returns {string}
+ */
+export function normalisedTitle(title) {
+  return String(title || '').toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim();
+}
+
+/**
+ * Candidate clusters: same signature, a shared screenshot or file_line ref,
+ * the same normalised title, or the same observation of the same run.
+ *
+ * A cluster is `certain` when its members share one normalised title and one
+ * category, and come from one run or share a screenshot or observation of
+ * one run: it is one finding written twice, merged before any model is asked
+ * anything. Every other cluster, including one joined only by a shared
+ * observation, goes to the dedup judge, which may still split it.
  *
  * @param {Array<{runId: string, persona: string, findings: Array}>} runs
- * @returns {Array<{signature: string, members: Array}>}
+ * @returns {Array<{signature: string, members: Array, certain: boolean}>}
  */
 export function clusterFindings(runs) {
-  /** @type {Map<string, Array>} */
-  const bySignature = new Map();
-  /** @type {Map<string, string>} */
-  const byEvidence = new Map();
-
+  const members = [];
   for (const run of runs) {
     for (const finding of run.findings || []) {
-      const member = {
-        ref: `${run.runId}:${finding.id}`,
-        runId: run.runId,
-        persona: run.persona,
-        finding
-      };
-      let signature = signatureFor(finding);
-      // A shared evidence ref is strong enough to pull two differently
-      // worded findings into one candidate cluster.
-      for (const item of finding.evidence || []) {
-        if (!item || (item.type !== 'screenshot' && item.type !== 'file_line')) continue;
-        const key = `${item.type}:${item.ref}`;
-        const existing = byEvidence.get(key);
-        if (existing) signature = existing;
-        else byEvidence.set(key, signature);
-      }
-      const bucket = bySignature.get(signature) || [];
-      bucket.push(member);
-      bySignature.set(signature, bucket);
+      members.push({ ref: `${run.runId}:${finding.id}`, runId: run.runId, persona: run.persona, finding });
     }
   }
 
-  return [...bySignature.entries()]
-    .map(([signature, members]) => ({ signature, members }))
+  // Union-find over member indexes, joined through any key two members share.
+  const parent = members.map((_, index) => index);
+  const find = index => (parent[index] === index ? index : (parent[index] = find(parent[index])));
+  const union = (a, b) => {
+    const left = find(a);
+    const right = find(b);
+    if (left !== right) parent[Math.max(left, right)] = Math.min(left, right);
+  };
+  const firstByKey = new Map();
+  const join = (key, index) => {
+    if (firstByKey.has(key)) union(firstByKey.get(key), index);
+    else firstByKey.set(key, index);
+  };
+  const observationKeys = member => (member.finding.evidence || [])
+    .filter(item => item && item.type === 'transcript_step' && /^\s*observation\s*#?\s*\d+\s*$/i.test(String(item.ref || '')))
+    .map(item => `obs:${member.runId}:${String(item.ref).replace(/\D+/g, '')}`);
+
+  members.forEach((member, index) => {
+    join(`sig:${signatureFor(member.finding)}`, index);
+    const title = normalisedTitle(member.finding.title);
+    if (title) join(`title:${title}`, index);
+    for (const item of member.finding.evidence || []) {
+      if (item && (item.type === 'screenshot' || item.type === 'file_line')) join(`${item.type}:${item.ref}`, index);
+    }
+    for (const key of observationKeys(member)) join(key, index);
+  });
+
+  const groups = new Map();
+  members.forEach((member, index) => {
+    const root = find(index);
+    if (!groups.has(root)) groups.set(root, []);
+    groups.get(root).push(member);
+  });
+
+  // Evidence a certain merge can rest on is scoped to one run: two runs'
+  // "screenshots/01-title.png" are different files.
+  const runScopedKeys = member => new Set([
+    ...observationKeys(member),
+    ...(member.finding.evidence || [])
+      .filter(item => item && item.type === 'screenshot' && item.ref)
+      .map(item => `shot:${member.runId}:${item.ref}`)
+  ]);
+
+  return [...groups.values()]
+    .map(group => {
+      const titles = new Set(group.map(member => normalisedTitle(member.finding.title)));
+      const categories = new Set(group.map(member => member.finding.category || ''));
+      const sameRun = new Set(group.map(member => member.runId)).size === 1;
+      const keys = group.map(runScopedKeys);
+      const sharedEvidence = keys.length > 0 && [...keys[0]].some(key => keys.every(set => set.has(key)));
+      return {
+        // The first member's signature names the cluster, so a cluster's id is
+        // as stable as it was when only signatures merged.
+        signature: group.map(member => signatureFor(member.finding)).sort()[0],
+        members: group,
+        // One finding written twice: the same normalised title AND the same
+        // category AND (the same run, or a shared screenshot or observation).
+        // Anything less, a shared observation alone included, goes to the
+        // dedup judge, which may split it.
+        certain: group.length > 1 && titles.size === 1 && !titles.has('') && categories.size === 1 && (sameRun || sharedEvidence)
+      };
+    })
     .sort((a, b) => a.signature.localeCompare(b.signature));
 }
 
@@ -141,7 +198,8 @@ export function clusterFindings(runs) {
  */
 export async function judgeClusters({ client, jobId, runId, clusters, log = () => {} }) {
   const verdicts = new Map();
-  const ambiguous = clusters.filter(cluster => cluster.members.length > 1);
+  // A certain cluster is one finding written twice; nothing to ask.
+  const ambiguous = clusters.filter(cluster => cluster.members.length > 1 && !cluster.certain);
   if (ambiguous.length === 0 || !runId) return verdicts;
 
   const prompt = [
@@ -215,7 +273,7 @@ export function applyQualityGate(runs) {
  *   said they are different
  */
 export function toIssues(cluster, verdict) {
-  const split = cluster.members.length > 1 && verdict && verdict.same === false;
+  const split = cluster.members.length > 1 && !cluster.certain && verdict && verdict.same === false;
   if (split) {
     return cluster.members.map(member => buildIssue(`${cluster.signature}#${member.ref}`, [member], null));
   }
@@ -452,6 +510,7 @@ export async function judgeAggregate({ client, jobId, runId, ranked, inputs, bri
     ' "tldr":["up to five short bullets: the most important fixes and the strongest positive signal"]}',
     '',
     'Never write calendar dates or times in the tldr or anywhere else; the runner stamps generated_at on the report.',
+    'Never write persona counts such as "2/3 personas" in the tldr; the runner adds counts from the run list.',
     'Rules. A criterion no run touched is "not testable"; never invent evidence for one. An issue is "known" only when the brief already describes it. An issue is excluded by intent only when the persona experience is CONSISTENT with the stated intent; when it CONTRADICTS the intent, leave it in the list, because that is the valuable case.',
     '',
     `success criteria:\n${criteria.length > 0 ? criteria.map(row => `  ${row.criterion_id}: ${row.text}`).join('\n') : '  (none stated)'}`,
@@ -564,8 +623,13 @@ export function criteriaFrom(brief) {
  */
 export function renderAggregateMarkdown(json) {
   const lines = [];
+  // Counts come from the run list, never from a model and never from how
+  // many findings a cluster merged: one persona filing the same issue twice
+  // is one persona, so "2/1 personas" cannot happen.
+  const personasRun = new Set((json.runs || []).map(run => run.persona).filter(Boolean)).size;
+  const affected = issue => Math.min(personasRun, new Set(issue.personas_affected || []).size);
   lines.push(`# Playtest Report: ${json.game_id}`);
-  lines.push(`**Job:** ${json.job_id} - **Date:** ${json.generated_at} - **Personas:** ${json.runs.length}`);
+  lines.push(`**Job:** ${json.job_id} - **Date:** ${json.generated_at} - **Personas:** ${personasRun}`);
   if (json.partial) {
     lines.push('');
     lines.push('> Partial: at least one run did not finish, so this report does not cover the whole pack.');
@@ -577,7 +641,7 @@ export function renderAggregateMarkdown(json) {
     lines.push(json.tldr.map(line => `- ${line}`).join('\n'));
   } else {
     lines.push(top.length > 0
-      ? top.map(issue => `- ${issue.label} ${issue.title} (${issue.severity}, ${issue.occurrences} of ${json.runs.length} personas)`).join('\n')
+      ? top.map(issue => `- ${issue.label} ${issue.title} (${issue.severity}, ${affected(issue)} of ${personasRun} personas)`).join('\n')
       : '- No ranked issues: no run produced an evidenced finding.');
   }
   lines.push('');
@@ -598,7 +662,7 @@ export function renderAggregateMarkdown(json) {
   lines.push('## Top issues (ranked)');
   if (json.issues.length === 0) lines.push('None.');
   for (const issue of json.issues) {
-    lines.push(`### ${issue.label}: ${issue.title}  \`${issue.severity}\` - ${issue.category} - ${issue.occurrences}/${json.runs.length} personas`);
+    lines.push(`### ${issue.label}: ${issue.title}  \`${issue.severity}\` - ${issue.category} - ${affected(issue)}/${personasRun} personas`);
     lines.push(`**Why it matters:** ${issue.description || ''}`);
     if (issue.repro_steps.length > 0) lines.push(`**Repro:** ${issue.repro_steps.map((step, i) => `${i + 1}. ${step}`).join(' ')}`);
     lines.push(`**Evidence:** ${issue.evidence.map(item => `${item.type} ${item.ref}`).join(', ') || 'none'}`);

package/src/run/args.js

@@ -35,6 +35,8 @@ const BOOLEAN = new Map([
   ['--headed', 'headed'],
   ['--headless', 'headless'],
   ['--verbose', 'verbose'],
+  // Native drivers only: register without the free automation probe first.
+  ['--skip-preflight', 'skipPreflight'],
   // Resume only: replay an interrupted run from the start instead of writing a
   // partial report from the transcript it already has.
   ['--restart-interrupted', 'restartInterrupted']

package/src/run/drivers/driver.js

@@ -93,6 +93,13 @@ export const DRIVER_METHODS = Object.freeze(['launch', 'observe', 'act', 'screen
 /** Action kinds a driver may be asked for. A driver may refuse a kind it cannot do. */
 export const ACTION_KINDS = Object.freeze(['key', 'type', 'click', 'press_action', 'navigate', 'wait']);
 
+/**
+ * What a ref read off a screen that is gone is answered with: a new document
+ * since the observation it came from, or an earlier action in the same turn
+ * that changed the screen.
+ */
+export const STALE_REF_MESSAGE = 'ref from an older screen, observe again';
+
 export class DriverError extends Error {
   /**
    * @param {string} message