ravensight-playtest 0.2.9 → 0.3.1

package/src/commands/run.js

@@ -12,6 +12,7 @@ import { createModelClient, ModelUnavailableError, ProxyError } from '../run/mod
 import { jobDir, runDir as runDirFor, JOB_FILES, RUN_FILES } from '../run/paths.js';
 import { readBridgeConfig } from "../run/drivers/stdio.js";
 import { WEBDRIVER_NAMES, readDriverConfig } from '../run/drivers/webdriver.js';
+import { automationChecks } from './check.js';
 import { runPersona } from '../run/personaLoop.js';
 import { reconcileDroppedFindings } from '../run/dropped.js';
 import * as stateFile from '../run/state.js';
@@ -106,6 +107,31 @@ async function run(flags, injected, deps, log) {
   // dedup and success-criteria passes.
   if (WEBDRIVER_NAMES.includes(driverName)) await readDriverConfig(target);
   if (driverName === "cli_stdio") await readBridgeConfig(target);
+
+  // The free capability probe. A native run depends on the developer's own
+  // automation server, which no amount of SDK or token setup proves, so the
+  // automation rows of `check` run first and a failure stops the run before
+  // the estimate, the confirmation and the register: no job, no reservation.
+  if (WEBDRIVER_NAMES.includes(driverName) && !options.skipPreflight) {
+    // Each row is printed as it completes, and the slow step says so before
+    // it starts, so a first session on a fresh simulator does not look hung.
+    const preflight = await (injected.preflight || automationChecks)(
+      { driverName, target, repoRoot: process.cwd() },
+      {
+        ...(injected.preflightDeps || {}),
+        onProgress: line => log(line),
+        onRow: entry => log(`preflight ${entry.status} ${entry.name}: ${entry.detail}`)
+      }
+    );
+    const failed = preflight.rows.filter(entry => entry.status === 'fail');
+    if (failed.length > 0) {
+      for (const entry of failed) {
+        log(`Preflight failed at "${entry.name}": ${entry.detail}${entry.fix ? `. Fix: ${entry.fix}` : ''}`);
+      }
+      log('No job was registered and nothing was reserved. Run "ravensight-playtest check" for every row, or pass --skip-preflight to register anyway.');
+      return EXIT.FAILED;
+    }
+  }
   const modules = ['persona_playtest', 'aggregate_report'];
   const pack = await deps.packs.get({ api, gameId, modules });
   if (options.controller === 'jev' && !pack.routing?.action_controllers?.includes('jev')) { log('This server does not currently support the requested action controller. No job was registered.'); return EXIT.ENVIRONMENT; }
@@ -561,13 +587,31 @@ export async function playPersonaRun({
       heartbeat,
       ledger,
       recordVideo: Boolean(options.uploadVideo),
+      // Written the moment the warning arrives, not after the run: a run that
+      // then fails still says in state.json that it was warned.
+      onBudgetWarning: async () => {
+        try {
+          await stateFile.patchRun(repoRoot, journal, runId, { budget_warned: true });
+        } catch {
+          // The journal write must not kill a paid run.
+        }
+      },
       log: message => log(`run ${runId}: ${message}`)
     });
   } catch (error) {
     heartbeat.stop();
     if (inFlight) inFlight.delete(runId);
     const failed = error.outcome || { state: 'failed', quitReason: 'error', actionsTaken: 0, report: null, reportLint: [], reachedState: 'playing' };
-    await stateFile.patchRun(repoRoot, journal, runId, { state: failed.state, error: String(error && error.message), quit_reason: failed.quitReason });
+    await stateFile.patchRun(repoRoot, journal, runId, {
+      state: failed.state,
+      error: String(error && error.message),
+      quit_reason: failed.quitReason,
+      // The failure path records these too. In the Texas run state.json said
+      // budget_warned: false for a run whose log had printed the warning,
+      // because only the success path wrote it.
+      ...(failed.budgetWarned ? { budget_warned: true } : {}),
+      report_written: Boolean(failed.report && !failed.report.partial)
+    });
     const delivery = {};
     const answer = await deliver({ deps, api, gameId, jobId, runId, directory, outcome: failed, options, log, repoRoot, delivery });
     if (isReportStepBudget(failed)) {
@@ -600,7 +644,8 @@ export async function playPersonaRun({
           state: 'failed',
           reason: `ran out of budget while writing the report; the server refused budget_exceeded (${delivery.message || 'no reason given'})`,
           actions_taken: failed.actionsTaken,
-          quit_reason: failed.quitReason
+          quit_reason: failed.quitReason,
+          ...(failed.duration ? { duration: failed.duration } : {})
         });
         await stateFile.patchRun(repoRoot, journal, runId, { completed: true, state: 'failed' });
         log(`run ${runId} (${persona.slug}): completed as failed instead.`);
@@ -653,6 +698,18 @@ export async function playPersonaRun({
   return { runId, persona: persona.slug, state: outcome.state, quitReason: outcome.quitReason };
 }
 
+/**
+ * A run's duration as `complete` sends it: `{ wall_seconds, model_seconds }`,
+ * each a number or null.
+ *
+ * @param {{wall_seconds?: number, model_seconds?: number}} duration
+ * @returns {{wall_seconds: number|null, model_seconds: number|null}}
+ */
+export function durationBody(duration) {
+  const value = key => (duration && Number.isFinite(Number(duration[key])) && duration[key] !== null ? Number(duration[key]) : null);
+  return { wall_seconds: value('wall_seconds'), model_seconds: value('model_seconds') };
+}
+
 /**
  * Upload the run directory and complete the run, through cli-core's
  * `finalizeRun` so the 422 `upload_unverified` repair pass lives in one place.
@@ -676,6 +733,9 @@ export async function deliver({ deps, api, gameId, jobId, runId, directory, outc
     actions_taken: outcome.actionsTaken,
     quit_reason: outcome.quitReason
   };
+  // The run card's duration. The server lane accepts `duration`; an older
+  // server ignores a key it does not know.
+  if (outcome.duration) body.duration = durationBody(outcome.duration);
   if (outcome.report) body.report = outcome.report;
   // `report_md` is what the server lints for its four required sections. Not
   // sending it means `report_lint` is silently empty on every run, which reads
@@ -857,6 +917,8 @@ async function aggregatePhase({
   log
 }) {
   const runId = aggregateRun ? aggregateRun.run_id : null;
+  // Created here, so its wall clock is measured from the start of the
+  // aggregate phase: what the summary step's card shows as its duration.
   const ledger = createUsageLedger();
   const heartbeat = runId
     ? createHeartbeat({ api, gameId, jobId, runId, onUsage: usage => ledger.recordServerUsage(usage), onError: () => {} })
@@ -913,7 +975,7 @@ async function aggregatePhase({
     log('The aggregate was built locally: this job has no aggregate_report run to spend a model turn from, so the dedup and judgement passes were skipped.');
   }
 
-  const published = await publishAggregate({ deps, api, gameId, jobId, repoRoot, directory, options, log });
+  const published = await publishAggregate({ deps, api, gameId, jobId, repoRoot, directory, options, log, duration: durationBody(ledger.duration()) });
   if (published === 'finished') reached = 'succeeded';
   else if (published) await advance('uploading');
 
@@ -923,7 +985,8 @@ async function aggregatePhase({
         state: reached === 'uploading' ? 'succeeded' : 'failed',
         actions_taken: 0,
         quit_reason: reached === 'uploading' ? 'goal_reached' : 'error',
-        reason: reached === 'uploading' ? '' : 'the aggregate was not published'
+        reason: reached === 'uploading' ? '' : 'the aggregate was not published',
+        duration: durationBody(ledger.duration())
       });
     } catch (error) {
       log(`aggregate run ${runId}: complete was refused (${String(error && error.message)}).`);
@@ -943,7 +1006,7 @@ async function aggregatePhase({
  *   sent, 'published' when the server has it, 'finished' when it also closed
  *   the aggregate run
  */
-async function publishAggregate({ deps, api, gameId, jobId, repoRoot, directory, options, log }) {
+async function publishAggregate({ deps, api, gameId, jobId, repoRoot, directory, options, log, duration = null }) {
   if (options.dryUpload) {
     log('The aggregate was not published: --dry-upload.');
     return false;
@@ -970,7 +1033,11 @@ async function publishAggregate({ deps, api, gameId, jobId, repoRoot, directory,
       md_size: markdown.size,
       md_sha256: markdown.sha256,
       json_size: json.size,
-      json_sha256: json.sha256
+      json_sha256: json.sha256,
+      // The server closes the aggregate run on this call, so the duration
+      // rides here as well as on the run's own complete; an older server
+      // ignores a key it does not know.
+      ...(duration ? { duration } : {})
     });
     // The server finishes the aggregate run itself when it accepts the
     // report, and says so in `run`. A caller that then tries to move the

package/src/run/aggregate.js

@@ -90,43 +90,100 @@ export function issueId(signature) {
 }
 
 /**
- * Candidate clusters: same signature, or a shared evidence ref.
+ * A title with case, punctuation and spacing taken out, for the exact
+ * duplicate check: "Roster never loads!" and "roster never loads" are one
+ * finding written twice.
+ *
+ * @param {string} title
+ * @returns {string}
+ */
+export function normalisedTitle(title) {
+  return String(title || '').toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim();
+}
+
+/**
+ * Candidate clusters: same signature, a shared screenshot or file_line ref,
+ * the same normalised title, or the same observation of the same run.
+ *
+ * A cluster is `certain` when its members share one normalised title and one
+ * category, and come from one run or share a screenshot or observation of
+ * one run: it is one finding written twice, merged before any model is asked
+ * anything. Every other cluster, including one joined only by a shared
+ * observation, goes to the dedup judge, which may still split it.
  *
  * @param {Array<{runId: string, persona: string, findings: Array}>} runs
- * @returns {Array<{signature: string, members: Array}>}
+ * @returns {Array<{signature: string, members: Array, certain: boolean}>}
  */
 export function clusterFindings(runs) {
-  /** @type {Map<string, Array>} */
-  const bySignature = new Map();
-  /** @type {Map<string, string>} */
-  const byEvidence = new Map();
-
+  const members = [];
   for (const run of runs) {
     for (const finding of run.findings || []) {
-      const member = {
-        ref: `${run.runId}:${finding.id}`,
-        runId: run.runId,
-        persona: run.persona,
-        finding
-      };
-      let signature = signatureFor(finding);
-      // A shared evidence ref is strong enough to pull two differently
-      // worded findings into one candidate cluster.
-      for (const item of finding.evidence || []) {
-        if (!item || (item.type !== 'screenshot' && item.type !== 'file_line')) continue;
-        const key = `${item.type}:${item.ref}`;
-        const existing = byEvidence.get(key);
-        if (existing) signature = existing;
-        else byEvidence.set(key, signature);
-      }
-      const bucket = bySignature.get(signature) || [];
-      bucket.push(member);
-      bySignature.set(signature, bucket);
+      members.push({ ref: `${run.runId}:${finding.id}`, runId: run.runId, persona: run.persona, finding });
     }
   }
 
-  return [...bySignature.entries()]
-    .map(([signature, members]) => ({ signature, members }))
+  // Union-find over member indexes, joined through any key two members share.
+  const parent = members.map((_, index) => index);
+  const find = index => (parent[index] === index ? index : (parent[index] = find(parent[index])));
+  const union = (a, b) => {
+    const left = find(a);
+    const right = find(b);
+    if (left !== right) parent[Math.max(left, right)] = Math.min(left, right);
+  };
+  const firstByKey = new Map();
+  const join = (key, index) => {
+    if (firstByKey.has(key)) union(firstByKey.get(key), index);
+    else firstByKey.set(key, index);
+  };
+  const observationKeys = member => (member.finding.evidence || [])
+    .filter(item => item && item.type === 'transcript_step' && /^\s*observation\s*#?\s*\d+\s*$/i.test(String(item.ref || '')))
+    .map(item => `obs:${member.runId}:${String(item.ref).replace(/\D+/g, '')}`);
+
+  members.forEach((member, index) => {
+    join(`sig:${signatureFor(member.finding)}`, index);
+    const title = normalisedTitle(member.finding.title);
+    if (title) join(`title:${title}`, index);
+    for (const item of member.finding.evidence || []) {
+      if (item && (item.type === 'screenshot' || item.type === 'file_line')) join(`${item.type}:${item.ref}`, index);
+    }
+    for (const key of observationKeys(member)) join(key, index);
+  });
+
+  const groups = new Map();
+  members.forEach((member, index) => {
+    const root = find(index);
+    if (!groups.has(root)) groups.set(root, []);
+    groups.get(root).push(member);
+  });
+
+  // Evidence a certain merge can rest on is scoped to one run: two runs'
+  // "screenshots/01-title.png" are different files.
+  const runScopedKeys = member => new Set([
+    ...observationKeys(member),
+    ...(member.finding.evidence || [])
+      .filter(item => item && item.type === 'screenshot' && item.ref)
+      .map(item => `shot:${member.runId}:${item.ref}`)
+  ]);
+
+  return [...groups.values()]
+    .map(group => {
+      const titles = new Set(group.map(member => normalisedTitle(member.finding.title)));
+      const categories = new Set(group.map(member => member.finding.category || ''));
+      const sameRun = new Set(group.map(member => member.runId)).size === 1;
+      const keys = group.map(runScopedKeys);
+      const sharedEvidence = keys.length > 0 && [...keys[0]].some(key => keys.every(set => set.has(key)));
+      return {
+        // The first member's signature names the cluster, so a cluster's id is
+        // as stable as it was when only signatures merged.
+        signature: group.map(member => signatureFor(member.finding)).sort()[0],
+        members: group,
+        // One finding written twice: the same normalised title AND the same
+        // category AND (the same run, or a shared screenshot or observation).
+        // Anything less, a shared observation alone included, goes to the
+        // dedup judge, which may split it.
+        certain: group.length > 1 && titles.size === 1 && !titles.has('') && categories.size === 1 && (sameRun || sharedEvidence)
+      };
+    })
     .sort((a, b) => a.signature.localeCompare(b.signature));
 }
 
@@ -139,9 +196,10 @@ export function clusterFindings(runs) {
  *
  * @returns {Promise<Map<string, {same: boolean, title: string|null}>>}
  */
-export async function judgeClusters({ client, jobId, runId, clusters, log = () => {} }) {
+export async function judgeClusters({ client, jobId, runId, clusters, ledger = null, log = () => {} }) {
   const verdicts = new Map();
-  const ambiguous = clusters.filter(cluster => cluster.members.length > 1);
+  // A certain cluster is one finding written twice; nothing to ask.
+  const ambiguous = clusters.filter(cluster => cluster.members.length > 1 && !cluster.certain);
   if (ambiguous.length === 0 || !runId) return verdicts;
 
   const prompt = [
@@ -156,6 +214,7 @@ export async function judgeClusters({ client, jobId, runId, clusters, log = () =
   ].join('\n');
 
   try {
+    const askedAt = Date.now();
     const answer = await callStepWithRetry({
       client,
       jobId,
@@ -164,6 +223,7 @@ export async function judgeClusters({ client, jobId, runId, clusters, log = () =
       messages: [{ role: 'user', content: [{ type: 'text', text: prompt }] }],
       maxTokens: 512
     });
+    if (ledger) ledger.record({ model: answer.model, usage: answer.usage, ms: Date.now() - askedAt });
     for (const entry of parseJsonArray(answer.text)) {
       const cluster = ambiguous[Number(entry.cluster)];
       if (!cluster) continue;
@@ -215,7 +275,7 @@ export function applyQualityGate(runs) {
  *   said they are different
  */
 export function toIssues(cluster, verdict) {
-  const split = cluster.members.length > 1 && verdict && verdict.same === false;
+  const split = cluster.members.length > 1 && !cluster.certain && verdict && verdict.same === false;
   if (split) {
     return cluster.members.map(member => buildIssue(`${cluster.signature}#${member.ref}`, [member], null));
   }
@@ -342,7 +402,7 @@ export async function buildAggregate({
   const { kept, observations } = applyQualityGate(inputs);
   const clusters = clusterFindings(kept);
   const verdicts = aggregateRunId
-    ? await judgeClusters({ client, jobId, runId: aggregateRunId, clusters, log })
+    ? await judgeClusters({ client, jobId, runId: aggregateRunId, clusters, ledger, log })
     : new Map();
 
   const issues = clusters.flatMap(cluster => toIssues(cluster, verdicts.get(cluster.signature)));
@@ -452,6 +512,7 @@ export async function judgeAggregate({ client, jobId, runId, ranked, inputs, bri
     ' "tldr":["up to five short bullets: the most important fixes and the strongest positive signal"]}',
     '',
     'Never write calendar dates or times in the tldr or anywhere else; the runner stamps generated_at on the report.',
+    'Never write persona counts such as "2/3 personas" in the tldr; the runner adds counts from the run list.',
     'Rules. A criterion no run touched is "not testable"; never invent evidence for one. An issue is "known" only when the brief already describes it. An issue is excluded by intent only when the persona experience is CONSISTENT with the stated intent; when it CONTRADICTS the intent, leave it in the list, because that is the valuable case.',
     '',
     `success criteria:\n${criteria.length > 0 ? criteria.map(row => `  ${row.criterion_id}: ${row.text}`).join('\n') : '  (none stated)'}`,
@@ -464,6 +525,7 @@ export async function judgeAggregate({ client, jobId, runId, ranked, inputs, bri
   ].join('\n');
 
   try {
+    const askedAt = Date.now();
     const answer = await callStepWithRetry({
       client,
       jobId,
@@ -472,7 +534,7 @@ export async function judgeAggregate({ client, jobId, runId, ranked, inputs, bri
       messages: [{ role: 'user', content: [{ type: 'text', text: prompt }] }],
       maxTokens: 8192
     });
-    if (ledger) ledger.record({ model: answer.model, usage: answer.usage });
+    if (ledger) ledger.record({ model: answer.model, usage: answer.usage, ms: Date.now() - askedAt });
     return parseJsonObject(answer.text);
   } catch (error) {
     // A budget or cancel refusal decides the caller's exit code and has to
@@ -564,8 +626,13 @@ export function criteriaFrom(brief) {
  */
 export function renderAggregateMarkdown(json) {
   const lines = [];
+  // Counts come from the run list, never from a model and never from how
+  // many findings a cluster merged: one persona filing the same issue twice
+  // is one persona, so "2/1 personas" cannot happen.
+  const personasRun = new Set((json.runs || []).map(run => run.persona).filter(Boolean)).size;
+  const affected = issue => Math.min(personasRun, new Set(issue.personas_affected || []).size);
   lines.push(`# Playtest Report: ${json.game_id}`);
-  lines.push(`**Job:** ${json.job_id} - **Date:** ${json.generated_at} - **Personas:** ${json.runs.length}`);
+  lines.push(`**Job:** ${json.job_id} - **Date:** ${json.generated_at} - **Personas:** ${personasRun}`);
   if (json.partial) {
     lines.push('');
     lines.push('> Partial: at least one run did not finish, so this report does not cover the whole pack.');
@@ -577,7 +644,7 @@ export function renderAggregateMarkdown(json) {
     lines.push(json.tldr.map(line => `- ${line}`).join('\n'));
   } else {
     lines.push(top.length > 0
-      ? top.map(issue => `- ${issue.label} ${issue.title} (${issue.severity}, ${issue.occurrences} of ${json.runs.length} personas)`).join('\n')
+      ? top.map(issue => `- ${issue.label} ${issue.title} (${issue.severity}, ${affected(issue)} of ${personasRun} personas)`).join('\n')
       : '- No ranked issues: no run produced an evidenced finding.');
   }
   lines.push('');
@@ -598,7 +665,7 @@ export function renderAggregateMarkdown(json) {
   lines.push('## Top issues (ranked)');
   if (json.issues.length === 0) lines.push('None.');
   for (const issue of json.issues) {
-    lines.push(`### ${issue.label}: ${issue.title}  \`${issue.severity}\` - ${issue.category} - ${issue.occurrences}/${json.runs.length} personas`);
+    lines.push(`### ${issue.label}: ${issue.title}  \`${issue.severity}\` - ${issue.category} - ${affected(issue)}/${personasRun} personas`);
     lines.push(`**Why it matters:** ${issue.description || ''}`);
     if (issue.repro_steps.length > 0) lines.push(`**Repro:** ${issue.repro_steps.map((step, i) => `${i + 1}. ${step}`).join(' ')}`);
     lines.push(`**Evidence:** ${issue.evidence.map(item => `${item.type} ${item.ref}`).join(', ') || 'none'}`);

package/src/run/args.js

@@ -35,6 +35,8 @@ const BOOLEAN = new Map([
   ['--headed', 'headed'],
   ['--headless', 'headless'],
   ['--verbose', 'verbose'],
+  // Native drivers only: register without the free automation probe first.
+  ['--skip-preflight', 'skipPreflight'],
   // Resume only: replay an interrupted run from the start instead of writing a
   // partial report from the transcript it already has.
   ['--restart-interrupted', 'restartInterrupted']

package/src/run/drivers/driver.js

@@ -93,6 +93,13 @@ export const DRIVER_METHODS = Object.freeze(['launch', 'observe', 'act', 'screen
 /** Action kinds a driver may be asked for. A driver may refuse a kind it cannot do. */
 export const ACTION_KINDS = Object.freeze(['key', 'type', 'click', 'press_action', 'navigate', 'wait']);
 
+/**
+ * What a ref read off a screen that is gone is answered with: a new document
+ * since the observation it came from, or an earlier action in the same turn
+ * that changed the screen.
+ */
+export const STALE_REF_MESSAGE = 'ref from an older screen, observe again';
+
 export class DriverError extends Error {
   /**
    * @param {string} message