ravensight-playtest 0.4.0 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,8 +1,10 @@
1
+ import { recordingForCompletion, recordingCompletionBody } from '../upload/recording.js';
1
2
  import { randomUUID } from 'node:crypto';
2
3
  import { mkdir, readFile, stat } from 'node:fs/promises';
3
4
  import path from 'node:path';
4
5
  import { buildAggregate, writeAggregate } from '../run/aggregate.js';
5
6
  import { optionsFrom, unknownOptionNotes } from '../run/args.js';
7
+ import { controllerPlan, quoteMatchesPlan, runsMatchPlan } from '../run/controllers.js';
6
8
  import { getDeps } from '../run/deps.js';
7
9
  import { createDriver } from '../run/drivers/index.js';
8
10
  import EXIT from '../run/exit.js';
@@ -94,10 +96,13 @@ async function run(flags, injected, deps, log) {
94
96
  if (setup.code !== undefined) return setup.code;
95
97
  const { context, gameId, repoRoot, apiUrl, cliVersion, api } = setup;
96
98
 
97
- if (options.controller && !['standard', 'jev'].includes(options.controller)) { log('Choose --controller standard or --controller jev.'); return EXIT.ENVIRONMENT; }
99
+ let controllers;
100
+ try { controllers = controllerPlan(options); } catch (error) { log(error.message); return EXIT.ENVIRONMENT; }
101
+ options.controllers = controllers;
98
102
  const keyModifier = resolveKeyModifier(options.keyModifier);
99
103
  if (keyModifier.error) { log(keyModifier.error); return EXIT.ENVIRONMENT; }
100
104
  const driverName = options.driver || (context.config && context.config.driver) || 'playwright_web';
105
+ if (controllers?.length > 1 && !['playwright_web', 'godot_driver'].includes(driverName)) { log('Comparisons currently require a web or Godot build. Nothing was charged.'); return EXIT.ENVIRONMENT; }
101
106
  const target = resolveTarget(options, context.config, driverName);
102
107
  if (!target) {
103
108
  log('Point the run at a build: --build-url for a web build, --godot-project for Godot, or --driver-config for a native or text bridge.');
@@ -147,7 +152,7 @@ async function run(flags, injected, deps, log) {
147
152
  }
148
153
  const modules = [...RUN_MODULES];
149
154
  const pack = await deps.packs.get({ api, gameId, modules });
150
- if (options.controller === 'jev' && !pack.routing?.action_controllers?.includes('jev')) { log('This server does not currently support the requested action controller. No job was registered.'); return EXIT.ENVIRONMENT; }
155
+ if (controllers && (!pack.routing?.controller_plans || controllers.some(controller => !pack.routing?.action_controllers?.includes(controller)))) { log('This server does not currently support the requested controller plan. No job was registered or charged.'); return EXIT.ENVIRONMENT; }
151
156
  const personaSlugs = resolvePersonas(options, context.config, pack);
152
157
  const unknownSlugs = unknownPersonas(personaSlugs, pack.personas || []);
153
158
  if (unknownSlugs.length > 0) {
@@ -161,7 +166,7 @@ async function run(flags, injected, deps, log) {
161
166
  // fetch, nothing charged. `pack.routing` is the clientRouting subset, and an
162
167
  // older cached pack without `max_personas` skips the check rather than
163
168
  // refusing on a field that does not exist yet.
164
- const personaRefusal = personaCountRefusal(personaSlugs.length, pack.routing && pack.routing.max_personas);
169
+ const personaRefusal = personaCountRefusal(personaSlugs.length * (controllers?.length || 1), pack.routing && pack.routing.max_personas);
165
170
  if (personaRefusal) {
166
171
  log(personaRefusal);
167
172
  return EXIT.ENVIRONMENT;
@@ -192,7 +197,7 @@ async function run(flags, injected, deps, log) {
192
197
  const runLength = options.runLength === 'long' ? 'long' : 'standard';
193
198
  let estimate;
194
199
  try {
195
- estimate = await api.jobs.estimate(gameId, { modules, personas: personaSlugs, runLength });
200
+ estimate = await api.jobs.estimate(gameId, { modules, personas: personaSlugs, runLength, ...(controllers ? { controllers, driver: driverName } : {}) });
196
201
  } catch (error) {
197
202
  if (error && error.code === 'invalid_estimate') {
198
203
  log(`The server refused to price this job: ${String(error.message || 'invalid estimate')}${runLength === 'long' ? ' This server may not price long runs yet; run without --long.' : ''} Nothing was registered or charged.`);
@@ -200,6 +205,9 @@ async function run(flags, injected, deps, log) {
200
205
  }
201
206
  throw error;
202
207
  }
208
+ if (!quoteMatchesPlan(estimate, controllers, personaSlugs)) { log('The server did not quote every requested controller run. Nothing was registered or charged.'); return EXIT.ENVIRONMENT; }
209
+ if (controllers) log(`Controllers: ${controllers.join(' + ')}. ${personas.length * controllers.length} independent playthrough(s); each run is included in the total quote.`);
210
+ if (controllers?.length > 1) log('Each controller gets a fresh game process or browser session. Server-side game progress and randomness may differ; comparison is evidence, not proof of identical starting state.');
203
211
  // A long run is its own unit on the server. An older server that ignores
204
212
  // run_length prices a standard run, and confirming that would buy a run
205
213
  // the CLI then plays as long.
@@ -245,6 +253,7 @@ async function run(flags, injected, deps, log) {
245
253
  registered = await api.jobs.register(gameId, {
246
254
  modules,
247
255
  personas: personaSlugs,
256
+ ...(controllers ? { controllers } : {}),
248
257
  driver: driverName,
249
258
  cli_version: cliVersion,
250
259
  confirm_price_cents: estimate.estimate_cents,
@@ -280,6 +289,8 @@ async function run(flags, injected, deps, log) {
280
289
  runLength,
281
290
  ...(runLength === 'long' ? { longRunFactor: options.longRunFactor } : {}),
282
291
  controller: options.controller || 'standard',
292
+ ...(controllers ? { controllers } : {}),
293
+ recordVideo: Boolean(options.recordVideo || options.uploadVideo),
283
294
  concurrency: options.concurrency ?? DEFAULT_CONCURRENCY,
284
295
  uploadVideo: Boolean(options.uploadVideo),
285
296
  uploadTranscript: Boolean(options.uploadTranscript)
@@ -291,6 +302,10 @@ async function run(flags, injected, deps, log) {
291
302
  }
292
303
  }
293
304
  await stateFile.save(repoRoot, journal);
305
+ if (!runsMatchPlan(registered.runs || [], controllers, personaSlugs)) {
306
+ log(`The registered controller plan does not match the quote. Job ${jobId} is saved for reconciliation; no playthrough or model call was started.`);
307
+ return EXIT.FAILED;
308
+ }
294
309
  log(`job ${jobId}${registered.replayed ? ' (replayed)' : ''}: ${formatCents(registered.price && registered.price.price_cents)} charged. ${directory}`);
295
310
 
296
311
  return playJob({
@@ -451,7 +466,7 @@ export async function playJob({
451
466
 
452
467
  const results = [];
453
468
  const queue = personaRuns.filter(entry => !stateFile.isRunDone(journal.runs[entry.run_id]));
454
- const concurrency = WEBDRIVER_NAMES.includes(driverName) ? 1 : Math.max(1, Math.min(Number(options.concurrency) || DEFAULT_CONCURRENCY, queue.length || 1));
469
+ const concurrency = (options.controllers?.length > 1 || WEBDRIVER_NAMES.includes(driverName)) ? 1 : Math.max(1, Math.min(Number(options.concurrency) || DEFAULT_CONCURRENCY, queue.length || 1));
455
470
 
456
471
  /**
457
472
  * Runs the process is in the middle of. A Ctrl-C has to move these to
@@ -462,7 +477,8 @@ export async function playJob({
462
477
  * back out of, which is exactly what it is for.
463
478
  */
464
479
  const inFlight = new Map();
465
- const interrupt = createInterruptHandler({ api, gameId, jobId, repoRoot, journal, inFlight, log, injected });
480
+ const abort = new AbortController();
481
+ const interrupt = createInterruptHandler({ api, gameId, jobId, repoRoot, journal, inFlight, log, injected, onInterrupt: () => abort.abort() });
466
482
 
467
483
  try {
468
484
  await Promise.all(Array.from({ length: concurrency }, async () => {
@@ -483,7 +499,7 @@ export async function playJob({
483
499
  pack,
484
500
  driverName,
485
501
  target,
486
- options,
502
+ options: { ...options, signal: abort.signal },
487
503
  journal,
488
504
  inFlight,
489
505
  makeDriver,
@@ -633,7 +649,8 @@ export async function playPersonaRun({
633
649
  try {
634
650
  if (inFlight) inFlight.set(runId, { state: 'launching' });
635
651
  outcome = await runPersona({
636
- controller: options.controller || 'standard',
652
+ controller: entry.controller || options.controller || 'standard',
653
+ signal: options.signal,
637
654
  client,
638
655
  api,
639
656
  gameId,
@@ -654,7 +671,7 @@ export async function playPersonaRun({
654
671
  maxActions,
655
672
  heartbeat,
656
673
  ledger,
657
- recordVideo: Boolean(options.uploadVideo),
674
+ recordVideo: Boolean(options.recordVideo || options.uploadVideo),
658
675
  // Written the moment the warning arrives, not after the run: a run that
659
676
  // then fails still says in state.json that it was warned.
660
677
  onBudgetWarning: async () => {
@@ -668,6 +685,10 @@ export async function playPersonaRun({
668
685
  });
669
686
  } catch (error) {
670
687
  heartbeat.stop();
688
+ if (options.signal?.aborted) {
689
+ await stateFile.patchRun(repoRoot, journal, runId, { state: 'interrupted', completed: false, actions_taken: error.outcome?.actionsTaken || 0 });
690
+ return { runId, persona: persona.slug, state: 'interrupted', error };
691
+ }
671
692
  if (inFlight) inFlight.delete(runId);
672
693
  const failed = error.outcome || { state: 'failed', quitReason: 'error', actionsTaken: 0, report: null, reportLint: [], reachedState: 'playing' };
673
694
  await stateFile.patchRun(repoRoot, journal, runId, {
@@ -726,6 +747,10 @@ export async function playPersonaRun({
726
747
  return { runId, persona: persona.slug, state: failed.state, quitReason: failed.quitReason, error };
727
748
  }
728
749
  heartbeat.stop();
750
+ if (options.signal?.aborted) {
751
+ await stateFile.patchRun(repoRoot, journal, runId, { state: 'interrupted', completed: false, actions_taken: outcome.actionsTaken, report_written: Boolean(outcome.report) });
752
+ return { runId, persona: persona.slug, state: 'interrupted' };
753
+ }
729
754
  if (inFlight) inFlight.delete(runId);
730
755
 
731
756
  await stateFile.patchRun(repoRoot, journal, runId, {
@@ -804,6 +829,9 @@ export async function deliver({ deps, api, gameId, jobId, runId, directory, outc
804
829
  // The run card's duration. The server lane accepts `duration`; an older
805
830
  // server ignores a key it does not know.
806
831
  if (outcome.duration) body.duration = durationBody(outcome.duration);
832
+ // Keep the same bounded opt-in status in the older-server partial-report fallback.
833
+ const recording = options.uploadVideo === true ? recordingForCompletion(outcome.recording) : null;
834
+ Object.assign(body, await recordingCompletionBody(directory, recording ? { recording } : {}, options.uploadVideo === true));
807
835
  if (outcome.report) body.report = outcome.report;
808
836
  // `report_md` is what the server lints for its four required sections. Not
809
837
  // sending it means `report_lint` is silently empty on every run, which reads
@@ -842,11 +870,11 @@ export async function deliver({ deps, api, gameId, jobId, runId, directory, outc
842
870
  body
843
871
  });
844
872
  } catch (error) {
845
- // A server older than partial reports refuses one as `invalid_report`.
846
- // The uploads are already done by then, so the run is completed
847
- // without it rather than left open: the notes stay on disk in
848
- // report.json either way.
849
- if (!(body.report && body.report.partial && error && error.status === 422 && error.code === 'invalid_report')) throw error;
873
+ // A server older than partial reports refuses one as `invalid_report`,
874
+ // and any server refuses one with a secret shaped value in it. The
875
+ // uploads are already done by then, so the run is completed without it
876
+ // rather than left open: the notes stay on disk in report.json either way.
877
+ if (!(body.report && body.report.partial && error && error.status === 422 && ['invalid_report', 'secrets_detected'].includes(error.code))) throw error;
850
878
  log(`run ${runId}: the server refused the partial report (${String(error.message)}), so the run is completed without it. The notes are in ${directory}.`);
851
879
  if (delivery) delivery.reportRefused = true;
852
880
  const { report: ignored, report_md: alsoIgnored, ...withoutReport } = body;
@@ -1009,7 +1037,10 @@ async function aggregatePhase({
1009
1037
 
1010
1038
  // The aggregate reads reports off disk, so a resumed job whose earlier runs
1011
1039
  // were played by a previous process is covered too.
1012
- const reports = await loadRunReports(repoRoot, jobId, personaRuns, journal);
1040
+ // A failed run's report explains the failure and its unit is refunded. It is
1041
+ // not a result to summarise, and the server refuses a summary of only those.
1042
+ const reports = (await loadRunReports(repoRoot, jobId, personaRuns, journal))
1043
+ .map(run => (run.state === 'failed' ? { ...run, report: null } : run));
1013
1044
  let aggregate;
1014
1045
  try {
1015
1046
  aggregate = await buildAggregate({
@@ -1135,7 +1166,7 @@ async function publishAggregate({ deps, api, gameId, jobId, repoRoot, directory,
1135
1166
  *
1136
1167
  * @returns {{signalled: boolean, settle: () => Promise<void>, release: () => void}}
1137
1168
  */
1138
- export function createInterruptHandler({ api, gameId, jobId, repoRoot, journal, inFlight, log, message = null, injected = {} }) {
1169
+ export function createInterruptHandler({ api, gameId, jobId, repoRoot, journal, inFlight, log, message = null, injected = {}, onInterrupt = () => {} }) {
1139
1170
  const signals = injected.signals || ['SIGINT', 'SIGTERM'];
1140
1171
  const target = injected.process || process;
1141
1172
  const state = { signalled: false, signal: null };
@@ -1144,6 +1175,7 @@ export function createInterruptHandler({ api, gameId, jobId, repoRoot, journal,
1144
1175
  if (state.signalled) return;
1145
1176
  state.signalled = true;
1146
1177
  state.signal = signal;
1178
+ onInterrupt(signal);
1147
1179
  log(`\n${signal}: ${message || `finishing the turn in flight, then stopping. Nothing is lost; "ravensight-playtest resume ${jobId}" carries on.`}`);
1148
1180
  // Hand the signal back to the default behaviour, so a second press kills
1149
1181
  // the process rather than being swallowed by a handler that is now done.
@@ -1285,15 +1317,15 @@ export async function loadRunReports(repoRoot, jobId, personaRuns, journal = nul
1285
1317
  // summary must not count findings the dashboard does not have.
1286
1318
  const recorded = journal && journal.runs ? journal.runs[entry.run_id] : null;
1287
1319
  if (recorded && recorded.report_refused === true) {
1288
- out.push({ runId: entry.run_id, persona: entry.persona, state: 'failed', report: null, reportRefused: true });
1320
+ out.push({ runId: entry.run_id, persona: entry.persona, ...(entry.controller ? { controller: entry.controller } : {}), state: 'failed', report: null, reportRefused: true });
1289
1321
  continue;
1290
1322
  }
1291
1323
  const file = path.join(runDirFor(repoRoot, jobId, entry.run_id), RUN_FILES.reportJson);
1292
1324
  try {
1293
1325
  const report = JSON.parse(await readFile(file, 'utf8'));
1294
- out.push({ runId: entry.run_id, persona: entry.persona, state: report && report.partial === true ? 'budget_exceeded' : 'succeeded', report });
1326
+ out.push({ runId: entry.run_id, persona: entry.persona, ...(entry.controller ? { controller: entry.controller } : {}), state: recorded?.state || entry.state || (report && report.partial === true ? 'budget_exceeded' : 'succeeded'), report });
1295
1327
  } catch {
1296
- out.push({ runId: entry.run_id, persona: entry.persona, state: 'failed', report: null });
1328
+ out.push({ runId: entry.run_id, persona: entry.persona, ...(entry.controller ? { controller: entry.controller } : {}), state: 'failed', report: null });
1297
1329
  }
1298
1330
  }
1299
1331
  return out;
@@ -1,5 +1,5 @@
1
1
  import { ProxyError, STEPS } from './model.js';
2
- export const CONTROLLERS = ['standard', 'jev'];
2
+ export { CONTROLLERS } from './controllers.js';
3
3
  export const JEV_CONTROLLER_PROMPT = '\nRavensight Playtester action-selection mode: describe the currently visible screen briefly in text, including each proposed target and its screenshot coordinates, then propose one to three ALTERNATIVE next actions as tool calls. Only one will be executed after a separate selector chooses. These are alternatives, never a sequential plan. Use only observed controls. For a canvas game without accessible DOM controls, include a target label describing the visible control AND screenshot x/y coordinates; the driver uses those coordinates. Never invent selectors such as textbox or button names. When the brief identifies a field as a public player label, use Ravensight Playtester. Credentials or private access codes require human review. When the game is loading or transitioning, propose a bounded wait tool call. Do not place secrets in proposals. After their tool results, observe the resulting state before proposing again.';
4
4
  export function describeAction(tool) {
5
5
  const input = tool.input || {};
@@ -14,7 +14,7 @@ export function describeAction(tool) {
14
14
  };
15
15
  return String(descriptions[tool.name] || `${tool.name}: ${JSON.stringify(input)}`).slice(0, 300);
16
16
  }
17
- export async function selectAction({ api, jobId, runId, persona, observation, toolUses, recentActions = [], intentNotes = [] }) {
17
+ export async function selectAction({ api, jobId, runId, persona, observation, toolUses, controller = 'jev', recentActions = [], intentNotes = [] }) {
18
18
  const proposals = toolUses.slice(0, 8);
19
19
  // Observation does not input anything into the game. Keep it in the existing
20
20
  // bounded driver loop; Jev selects game inputs, not whether a frame may load.
@@ -22,6 +22,7 @@ export async function selectAction({ api, jobId, runId, persona, observation, to
22
22
  return { status: 'selected', toolId: proposals[0].id, source: 'observation' };
23
23
  }
24
24
  const body = {
25
+ controller,
25
26
  objective: ('Play the game according to the supplied persona, exploring visible controls and gathering evidence. Developer scope: ' + intentNotes.map(note => note.text || '').join(' ')).slice(0, 1000),
26
27
  // The observer already applies the full persona. The selector needs its identity,
27
28
  // current visible state and proposed inputs, not a second personality prompt.
@@ -1,4 +1,5 @@
1
1
  import { createHash } from 'node:crypto';
2
+ import { compareControllers, comparisonMarkdown } from './comparison.js';
2
3
  import { fillPlaceholders } from './placeholders.js';
3
4
  import { writeFile } from 'node:fs/promises';
4
5
  import path from 'node:path';
@@ -458,7 +459,9 @@ export async function buildAggregate({
458
459
  usage: ledger ? aggregateUsage(ledger) : { usd: 0, by_model: {} }
459
460
  };
460
461
 
461
- return { json, markdown: renderAggregateMarkdown(json) };
462
+ const comparison = compareControllers(runs);
463
+ if (comparison) json.controller_comparison = comparison;
464
+ return { json, markdown: renderAggregateMarkdown(json) + comparisonMarkdown(comparison) };
462
465
  }
463
466
 
464
467
  /**
package/src/run/args.js CHANGED
@@ -16,6 +16,7 @@ const VALUED = new Map([
16
16
  ['--driver', 'driver'],
17
17
  ['--driver-config', 'driverConfig'],
18
18
  ['--controller', 'controller'],
19
+ ['--compare-with', 'compareWith'],
19
20
  ['--build-url', 'buildUrl'],
20
21
  ['--build-dir', 'buildDir'],
21
22
  ['--godot-project', 'godotProject'],
@@ -29,6 +30,7 @@ const VALUED = new Map([
29
30
  /** Options that are just present or absent. */
30
31
  const BOOLEAN = new Map([
31
32
  ['--upload-video', 'uploadVideo'],
33
+ ['--record-video', 'recordVideo'],
32
34
  ['--upload-transcript', 'uploadTranscript'],
33
35
  ['--dry-upload', 'dryUpload'],
34
36
  ['--offline-upload', 'offlineUpload'],
@@ -0,0 +1,32 @@
1
+ // Deterministic comparison of the reports already paid for. No new model call.
2
+ export function compareControllers(runs) {
3
+ const planned = runs.filter(run => run.controller);
4
+ const controllers = [...new Set(planned.map(run => run.controller))];
5
+ if (controllers.length < 2) return null;
6
+ const rows = planned.map(run => ({
7
+ run_id: run.runId, persona: run.persona, controller: run.controller,
8
+ state: run.state, report_available: Boolean(run.report),
9
+ partial: !run.report || run.report.partial === true || run.state !== 'succeeded',
10
+ actions_taken: run.report?.actions_taken ?? null,
11
+ }));
12
+ const groups = new Map();
13
+ for (const run of planned) for (const finding of run.report?.findings || []) {
14
+ if (finding.dropped || !Array.isArray(finding.evidence) || !finding.evidence.length) continue;
15
+ const key = JSON.stringify([run.persona, finding.category || '', String(finding.title || '').trim().toLowerCase().replace(/\s+/g, ' ')]);
16
+ if (!groups.has(key)) groups.set(key, { title: finding.title, persona: run.persona, category: finding.category, occurrences: [] });
17
+ groups.get(key).occurrences.push({ run_id: run.runId, controller: run.controller, evidence: finding.evidence });
18
+ }
19
+ const findings = [...groups.values()].map(group => ({ ...group,
20
+ match: new Set(group.occurrences.map(row => row.controller)).size > 1 ? 'reported_by_multiple_controllers' : 'reported_by_one_controller' }));
21
+ return { schema_version: '1.0', controllers, status: rows.some(row => row.partial) ? 'inconclusive' : 'reports_complete',
22
+ limits: ['Controllers use separate game sessions. Server saves and random state are not proven identical.', 'Matches use the same persona, category and exact normalized title. Agreement does not verify a bug; absence does not prove a pass.'], runs: rows, findings };
23
+ }
24
+ export function comparisonMarkdown(comparison) {
25
+ if (!comparison) return '';
26
+ const safe = value => String(value ?? '').replace(/[\r\n|]/g, ' ');
27
+ return '\n\n## Controller comparison\n\n' + comparison.limits.join(' ') + '\n\n' +
28
+ (comparison.status === 'inconclusive' ? 'At least one playthrough is missing or partial. This comparison is inconclusive.\n\n' : '') +
29
+ '| Controller | Persona | Run | Report |\n| --- | --- | --- | --- |\n' +
30
+ comparison.runs.map(row => `| ${safe(row.controller)} | ${safe(row.persona)} | ${safe(row.run_id)} | ${row.partial ? 'Incomplete' : 'Complete'} |`).join('\n') +
31
+ '\n\n' + comparison.findings.map(group => `- ${safe(group.title)}: ${[...new Set(group.occurrences.map(row => row.controller))].join(', ')} (${group.occurrences.map(row => safe(row.run_id)).join(', ')}).`).join('\n');
32
+ }
@@ -0,0 +1,17 @@
1
+ export const CONTROLLERS = ['standard', 'jev', 'rune'];
2
+ export function controllerPlan(options = {}) {
3
+ if (options.compareWith && options.controller) throw new Error('Use --controller for one playthrough or --compare-with for a comparison.');
4
+ if (!options.compareWith && !options.controller) return null;
5
+ const plan = options.compareWith ? ['standard', ...String(options.compareWith).split(',')] : [options.controller];
6
+ if (plan.length > 3 || plan.some(x => !CONTROLLERS.includes(x)) || new Set(plan).size !== plan.length) throw new Error('Choose standard, jev or rune once each. --compare-with jev runs standard and Jev separately.');
7
+ return plan;
8
+ }
9
+ export function quoteMatchesPlan(quote, plan, personas) {
10
+ return !plan || (Array.isArray(quote.controllers) && JSON.stringify(quote.controllers) === JSON.stringify(plan) && quote.persona_run_count === plan.length * personas.length);
11
+ }
12
+ export function runsMatchPlan(runs, plan, personas) {
13
+ if (!plan) return true;
14
+ const actual = runs.filter(run => run.module === 'persona_playtest').map(run => `${run.persona}/${run.controller}`).sort();
15
+ const expected = personas.flatMap(persona => plan.map(controller => `${persona}/${controller}`)).sort();
16
+ return JSON.stringify(actual) === JSON.stringify(expected);
17
+ }
@@ -18,7 +18,7 @@ import { createHash } from 'node:crypto';
18
18
  * observe() -> Observation
19
19
  * act(action) -> ActResult
20
20
  * screenshot(name, opts) -> { path, relativePath }
21
- * stop() -> { videoPath?: string|null, framesDir?: string }
21
+ * stop({ reason }) -> { videoPath?: string|null, recording?: Object }
22
22
  * ```
23
23
  *
24
24
  * Optional, used when present:
@@ -37,10 +37,10 @@ import { createHash } from 'node:crypto';
37
37
  * description in ../tools.js says so, because the model is the one
38
38
  * choosing.
39
39
  * - **`stop()` may return no video.** `playwright_web` hands back a
40
- * `session.webm` from Playwright's own recorder. `godot_driver` returns
41
- * `videoPath: null` and leaves a `frames/` sequence, which the CLI
42
- * stitches only when ffmpeg is present. That is what keeps ffmpeg
43
- * optional, and it means "no video" is a normal outcome, not a failure.
40
+ * `session.webm` from Playwright's own recorder. `godot_driver` encodes
41
+ * timestamped viewport frames to `session.mp4` with optional local ffmpeg.
42
+ * `recording.status` distinguishes complete, partial, unsupported and failed
43
+ * capture; absence of an encoder or rendered viewport never masquerades as video.
44
44
  *
45
45
  * `Observation.scene`, `.scenes` and `.scripts` are how a finding from a
46
46
  * running Godot game can cite a `res://` path; the web driver leaves them
@@ -9,8 +9,8 @@
9
9
  //
10
10
  // Both of the differences ./driver.js names fall on this driver's side: a
11
11
  // string `click` target is a scene tree node path, not an accessibility ref,
12
- // and `stop()` answers `videoPath: null` with a `frames/` sequence beside it,
13
- // because stitching is ffmpeg's job and ffmpeg stays optional.
12
+ // and recording uses the selected viewport, with optional local ffmpeg encoding.
13
+ // Without an encoder or a rendered viewport it reports unsupported, never a video.
14
14
  //
15
15
  // What happens at launch, in order:
16
16
  // 1. find the developer's own Godot binary (spec 17 detection order),
@@ -34,6 +34,8 @@ import {
34
34
  prepareProjectCopy,
35
35
  } from './godot-project.js';
36
36
  import { DriverRpcError, GodotRpcClient } from './godot-rpc.js';
37
+ import { detectFfmpeg } from '../../detect.js';
38
+ import { recordingLimits, recordingViewport, recordingDirectory, recordingMetadata, saveRecording, encodeFrames } from './recording.js';
37
39
 
38
40
  export const DRIVER_NAME = 'godot_driver';
39
41
  export const BOOT_TIMEOUT_MS = 60000;
@@ -124,6 +126,9 @@ export class GodotDriver {
124
126
  prepare = prepareProjectCopy,
125
127
  choosePort = findFreePort,
126
128
  now = () => Date.now(),
129
+ recording = {},
130
+ detectEncoder = detectFfmpeg,
131
+ encode = encodeFrames,
127
132
  } = {}) {
128
133
  this.name = DRIVER_NAME;
129
134
  this.projectDir = projectDir;
@@ -148,6 +153,11 @@ export class GodotDriver {
148
153
  this._prepare = prepare;
149
154
  this._choosePort = choosePort;
150
155
  this._now = now;
156
+ this._recordingLimits = recordingLimits(recording);
157
+ this._detectEncoder = detectEncoder;
158
+ this._encode = encode;
159
+ this._frames = [];
160
+ this._frameBytes = 0;
151
161
 
152
162
  this.rpc = null;
153
163
  this.child = null;
@@ -178,8 +188,13 @@ export class GodotDriver {
178
188
  return !this._stopped && this.exitCode !== null && this.exitCode !== 0;
179
189
  }
180
190
 
181
- async launch({ recordVideo = false, viewport } = {}) {
191
+ async launch({ recordVideo = false, viewport, signal } = {}) {
192
+ if (signal?.aborted) throw new Error('run canceled before game launch');
182
193
  if (viewport) this.viewport = viewport;
194
+ if (recordVideo) {
195
+ const size = recordingViewport(this.viewport, this._recordingLimits);
196
+ this.viewport = { w: size.width, h: size.height };
197
+ }
183
198
  if (!this.projectDir) throw new Error('godot driver needs a projectDir');
184
199
  this.godot = this.godotPath
185
200
  ? await this._detect({ explicitPath: this.godotPath, env: this.env, projectDir: this.projectDir })
@@ -198,6 +213,7 @@ export class GodotDriver {
198
213
  this.logPath = this.runDir ? path.join(this.runDir, 'godot.log') : null;
199
214
 
200
215
  const args = ['--path', this.projectCopy];
216
+ if (recordVideo) args.push('--audio-driver', 'Dummy');
201
217
  if (this.headless) args.push('--headless');
202
218
  else args.push('--resolution', `${this.viewport.w}x${this.viewport.h}`);
203
219
  args.push('--', '--ravensight-driver', `--ravensight-driver-port=${this.port}`);
@@ -226,6 +242,10 @@ export class GodotDriver {
226
242
  }
227
243
 
228
244
  if (recordVideo) await this._startFrameCapture();
245
+ this._signal = signal;
246
+ this._onAbort = () => { void this.stop({ reason: 'canceled' }).catch(() => {}); };
247
+ signal?.addEventListener('abort', this._onAbort, { once: true });
248
+ if (signal?.aborted) { await this.stop({ reason: 'canceled' }); throw new Error('run canceled during game launch'); }
229
249
  return {};
230
250
  } catch (error) {
231
251
  // Everything past this point has already spawned a game and copied a
@@ -289,6 +309,10 @@ export class GodotDriver {
289
309
  this.exitCode = code;
290
310
  this.exitSignal = signal;
291
311
  if (log) log.end();
312
+ if (!this._stopped && this._recordingStartedAt != null) {
313
+ this._endCapture(code === 0 ? 'game_exited' : 'game_crashed');
314
+ void this._finalizeRecording().catch(() => {});
315
+ }
292
316
  });
293
317
  this.child.on('error', (error) => {
294
318
  this._consoleQueue.push({ kind: 'driver', text: `failed to launch Godot: ${error.message}` });
@@ -501,43 +525,114 @@ export class GodotDriver {
501
525
  this.chapters.push({ title, at: this._startedAt === null ? 0 : this._now() - this._startedAt });
502
526
  }
503
527
 
504
- /**
505
- * Video on a local Godot run is a sequence of stills, not a container: the
506
- * frames land in the run dir and whoever wants a webm stitches them with
507
- * ffmpeg, which stays optional for the CLI.
508
- */
509
- async _startFrameCapture({ fps = 4 } = {}) {
510
- if (this.headless || !this.runDir) return;
511
- this._framesDir = path.join(this.runDir, 'frames');
512
- await fs.mkdir(this._framesDir, { recursive: true });
513
- this._videoTimer = setInterval(() => {
514
- void this._captureFrame();
515
- }, Math.round(1000 / fps));
516
- if (typeof this._videoTimer.unref === 'function') this._videoTimer.unref();
528
+ // Capture only this project's viewport. No screen/audio device is opened.
529
+ async _startFrameCapture() {
530
+ this._recordingStartedAt = this._now();
531
+ if (this.runDir) this._framesDir = await recordingDirectory(this.runDir);
532
+ if (this.headless || !this.runDir) {
533
+ this._recordingUnsupported = this.headless ? 'headless_no_viewport' : 'run_directory_required';
534
+ return;
535
+ }
536
+ this._encoder = await this._detectEncoder();
537
+ if (!this._encoder?.path) {
538
+ this._recordingUnsupported = 'encoder_unavailable';
539
+ this._consoleQueue.push({ kind: 'driver', text: 'Video unavailable: install optional ffmpeg or set RAVENSIGHT_FFMPEG. No video was recorded.' });
540
+ return;
541
+ }
542
+ this._recordingStartedAt = this._now();
543
+ await this._captureFrame();
544
+ if (this._captureEndedAt != null) return;
545
+ this._videoTimer = setInterval(() => { void this._captureFrame(); }, Math.round(1000 / this._recordingLimits.fps));
546
+ this._videoTimer.unref?.();
547
+ this._videoDeadline = setTimeout(() => this._endCapture('duration_limit'), this._recordingLimits.durationMs);
548
+ this._videoDeadline.unref?.();
549
+ }
550
+
551
+ _endCapture(reason) {
552
+ if (this._captureEndedAt != null) return;
553
+ this._captureEndedAt = Math.min(this._now(), this._recordingStartedAt + this._recordingLimits.durationMs);
554
+ this._captureReason = reason;
555
+ clearInterval(this._videoTimer);
556
+ clearTimeout(this._videoDeadline);
557
+ this._videoTimer = null;
517
558
  }
518
559
 
519
560
  async _captureFrame() {
520
- if (this._capturing || this._stopped) return;
521
- this._capturing = true;
522
- try {
523
- const shot = await this._rawScreenshot();
524
- if (!shot) return;
525
- this._frameIndex += 1;
526
- const file = path.join(this._framesDir, `${String(this._frameIndex).padStart(6, '0')}.png`);
527
- await fs.writeFile(file, Buffer.from(shot.data, 'base64'));
528
- } catch {
529
- // A dropped frame is never worth interrupting a run for.
530
- } finally {
531
- this._capturing = false;
561
+ if (this._capturePromise || this._stopped || this._captureEndedAt != null) return;
562
+ if (this._now() - this._recordingStartedAt >= this._recordingLimits.durationMs) {
563
+ this._endCapture('duration_limit'); return;
532
564
  }
565
+ const at = this._now();
566
+ this._capturePromise = (async () => {
567
+ try {
568
+ const shot = await this._rawScreenshot();
569
+ if (!shot) return;
570
+ const max = this._recordingLimits.singleFrameBytes;
571
+ if (typeof shot.data !== 'string' || shot.data.length > Math.ceil(max / 3) * 4) {
572
+ this._endCapture('frame_size_limit'); return;
573
+ }
574
+ const bytes = Buffer.from(shot.data, 'base64');
575
+ if (bytes.length > max || this._frameBytes + bytes.length > this._recordingLimits.frameBytes) {
576
+ this._endCapture('frame_bytes_limit'); return;
577
+ }
578
+ const name = `${String(this._frameIndex + 1).padStart(6, '0')}.png`;
579
+ await fs.writeFile(path.join(this._framesDir, name), bytes, { flag: 'wx', mode: 0o600 });
580
+ this._frames.push({ name, at });
581
+ this._frameIndex += 1;
582
+ this._frameBytes += bytes.length;
583
+ } catch {
584
+ // A disconnected/crashed game can still leave a useful partial recording.
585
+ this._endCapture('capture_failed');
586
+ }
587
+ })();
588
+ try { await this._capturePromise; } finally { this._capturePromise = null; }
533
589
  }
534
590
 
535
- async stop({ removeTemp = !this.keepTemp } = {}) {
591
+ _finalizeRecording() {
592
+ if (this._finalizePromise) return this._finalizePromise;
593
+ this._finalizePromise = (async () => {
594
+ if (this._recordingStartedAt == null) return {};
595
+ await this._capturePromise;
596
+ const endedAt = this._captureEndedAt ?? this._now();
597
+ let metadata = recordingMetadata({ driver: this.name, startedAt: this._recordingStartedAt, endedAt,
598
+ reason: this._recordingUnsupported ?? this._captureReason ?? 'stopped',
599
+ status: this._recordingUnsupported ? 'unsupported' : 'failed', frameCount: this._frameIndex,
600
+ captureFpsLimit: this._recordingLimits.fps, timeOrigin: 'first-viewport-frame',
601
+ });
602
+ let videoPath = null;
603
+ if (!this._recordingUnsupported && this._framesDir) {
604
+ try {
605
+ const encoded = await this._encode({ directory: this._framesDir, frames: this._frames, endedAt,
606
+ encoder: this._encoder, limits: this._recordingLimits });
607
+ videoPath = path.join(this.runDir, 'session.mp4');
608
+ await fs.rename(encoded.file, videoPath);
609
+ metadata = { ...metadata, status: ['stopped', 'completed', 'game_exited'].includes(metadata.reason) ? 'complete' : 'partial',
610
+ relativePath: 'session.mp4', mimeType: encoded.mimeType, bytes: encoded.bytes,
611
+ durationMs: encoded.durationMs, durationSource: encoded.durationSource ?? 'frame-timestamps',
612
+ firstFrameOffsetMs: this._frames[0].at - this._recordingStartedAt };
613
+ if (encoded.truncated) { metadata.status = 'partial'; metadata.reason = 'size_limit'; }
614
+ } catch (error) {
615
+ metadata.reason = ['no_frames', 'encoder_unavailable', 'encoding_timeout', 'encoding_failed'].includes(error.message) ? error.message : 'encoding_failed';
616
+ }
617
+ }
618
+ // Captured game frames are local temporary data, never upload artifacts.
619
+ if (this._framesDir) await fs.rm(this._framesDir, { recursive: true, force: true });
620
+ if (this.runDir) await saveRecording(this.runDir, metadata);
621
+ return { videoPath, recording: metadata };
622
+ })();
623
+ return this._finalizePromise;
624
+ }
625
+
626
+ stop(options = {}) {
627
+ if (!this._stopPromise) this._stopPromise = Promise.resolve().then(() => this._stop(options));
628
+ return this._stopPromise;
629
+ }
630
+
631
+ async _stop({ removeTemp = !this.keepTemp, reason = 'stopped' } = {}) {
536
632
  this._stopped = true;
537
- if (this._videoTimer) {
538
- clearInterval(this._videoTimer);
539
- this._videoTimer = null;
540
- }
633
+ this._signal?.removeEventListener('abort', this._onAbort);
634
+ if (this._recordingStartedAt != null) this._endCapture(reason);
635
+ await this._capturePromise;
541
636
  if (this.rpc && !this.rpc.closed) {
542
637
  await this.rpc.call('quit', { code: 0 }, { timeoutMs: 2000 }).catch(() => {});
543
638
  }
@@ -549,12 +644,9 @@ export class GodotDriver {
549
644
  this.projectCopy = null;
550
645
  }
551
646
  return {
552
- videoPath: null,
553
- framesDir: this._framesDir,
554
- frameCount: this._frameIndex,
555
- chapters: this.chapters,
556
- logPath: this.logPath,
557
- exitCode: this.exitCode,
647
+ videoPath: null, ...await this._finalizeRecording(),
648
+ frameCount: this._frameIndex, chapters: this.chapters,
649
+ logPath: this.logPath, exitCode: this.exitCode,
558
650
  };
559
651
  }
560
652