ravensight-playtest 0.4.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +60 -4
- package/docs/SIMULATION.md +1 -1
- package/docs/godot-driver.md +310 -0
- package/package.json +2 -1
- package/src/api/client.js +2 -2
- package/src/cli.js +3 -1
- package/src/commands/run.js +51 -19
- package/src/run/actionSelector.js +3 -2
- package/src/run/aggregate.js +4 -1
- package/src/run/args.js +2 -0
- package/src/run/comparison.js +32 -0
- package/src/run/controllers.js +17 -0
- package/src/run/drivers/driver.js +5 -5
- package/src/run/drivers/godot.js +131 -39
- package/src/run/drivers/recording.js +148 -0
- package/src/run/drivers/stdio.js +24 -8
- package/src/run/drivers/web.js +131 -67
- package/src/run/drivers/webdriver.js +1 -1
- package/src/run/personaLoop.js +140 -49
- package/src/run/resume.js +16 -5
- package/src/upload/index.js +4 -2
- package/src/upload/recording.js +56 -0
package/src/commands/run.js
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
|
+
import { recordingForCompletion, recordingCompletionBody } from '../upload/recording.js';
|
|
1
2
|
import { randomUUID } from 'node:crypto';
|
|
2
3
|
import { mkdir, readFile, stat } from 'node:fs/promises';
|
|
3
4
|
import path from 'node:path';
|
|
4
5
|
import { buildAggregate, writeAggregate } from '../run/aggregate.js';
|
|
5
6
|
import { optionsFrom, unknownOptionNotes } from '../run/args.js';
|
|
7
|
+
import { controllerPlan, quoteMatchesPlan, runsMatchPlan } from '../run/controllers.js';
|
|
6
8
|
import { getDeps } from '../run/deps.js';
|
|
7
9
|
import { createDriver } from '../run/drivers/index.js';
|
|
8
10
|
import EXIT from '../run/exit.js';
|
|
@@ -94,10 +96,13 @@ async function run(flags, injected, deps, log) {
|
|
|
94
96
|
if (setup.code !== undefined) return setup.code;
|
|
95
97
|
const { context, gameId, repoRoot, apiUrl, cliVersion, api } = setup;
|
|
96
98
|
|
|
97
|
-
|
|
99
|
+
let controllers;
|
|
100
|
+
try { controllers = controllerPlan(options); } catch (error) { log(error.message); return EXIT.ENVIRONMENT; }
|
|
101
|
+
options.controllers = controllers;
|
|
98
102
|
const keyModifier = resolveKeyModifier(options.keyModifier);
|
|
99
103
|
if (keyModifier.error) { log(keyModifier.error); return EXIT.ENVIRONMENT; }
|
|
100
104
|
const driverName = options.driver || (context.config && context.config.driver) || 'playwright_web';
|
|
105
|
+
if (controllers?.length > 1 && !['playwright_web', 'godot_driver'].includes(driverName)) { log('Comparisons currently require a web or Godot build. Nothing was charged.'); return EXIT.ENVIRONMENT; }
|
|
101
106
|
const target = resolveTarget(options, context.config, driverName);
|
|
102
107
|
if (!target) {
|
|
103
108
|
log('Point the run at a build: --build-url for a web build, --godot-project for Godot, or --driver-config for a native or text bridge.');
|
|
@@ -147,7 +152,7 @@ async function run(flags, injected, deps, log) {
|
|
|
147
152
|
}
|
|
148
153
|
const modules = [...RUN_MODULES];
|
|
149
154
|
const pack = await deps.packs.get({ api, gameId, modules });
|
|
150
|
-
if (
|
|
155
|
+
if (controllers && (!pack.routing?.controller_plans || controllers.some(controller => !pack.routing?.action_controllers?.includes(controller)))) { log('This server does not currently support the requested controller plan. No job was registered or charged.'); return EXIT.ENVIRONMENT; }
|
|
151
156
|
const personaSlugs = resolvePersonas(options, context.config, pack);
|
|
152
157
|
const unknownSlugs = unknownPersonas(personaSlugs, pack.personas || []);
|
|
153
158
|
if (unknownSlugs.length > 0) {
|
|
@@ -161,7 +166,7 @@ async function run(flags, injected, deps, log) {
|
|
|
161
166
|
// fetch, nothing charged. `pack.routing` is the clientRouting subset, and an
|
|
162
167
|
// older cached pack without `max_personas` skips the check rather than
|
|
163
168
|
// refusing on a field that does not exist yet.
|
|
164
|
-
const personaRefusal = personaCountRefusal(personaSlugs.length, pack.routing && pack.routing.max_personas);
|
|
169
|
+
const personaRefusal = personaCountRefusal(personaSlugs.length * (controllers?.length || 1), pack.routing && pack.routing.max_personas);
|
|
165
170
|
if (personaRefusal) {
|
|
166
171
|
log(personaRefusal);
|
|
167
172
|
return EXIT.ENVIRONMENT;
|
|
@@ -192,7 +197,7 @@ async function run(flags, injected, deps, log) {
|
|
|
192
197
|
const runLength = options.runLength === 'long' ? 'long' : 'standard';
|
|
193
198
|
let estimate;
|
|
194
199
|
try {
|
|
195
|
-
estimate = await api.jobs.estimate(gameId, { modules, personas: personaSlugs, runLength });
|
|
200
|
+
estimate = await api.jobs.estimate(gameId, { modules, personas: personaSlugs, runLength, ...(controllers ? { controllers, driver: driverName } : {}) });
|
|
196
201
|
} catch (error) {
|
|
197
202
|
if (error && error.code === 'invalid_estimate') {
|
|
198
203
|
log(`The server refused to price this job: ${String(error.message || 'invalid estimate')}${runLength === 'long' ? ' This server may not price long runs yet; run without --long.' : ''} Nothing was registered or charged.`);
|
|
@@ -200,6 +205,9 @@ async function run(flags, injected, deps, log) {
|
|
|
200
205
|
}
|
|
201
206
|
throw error;
|
|
202
207
|
}
|
|
208
|
+
if (!quoteMatchesPlan(estimate, controllers, personaSlugs)) { log('The server did not quote every requested controller run. Nothing was registered or charged.'); return EXIT.ENVIRONMENT; }
|
|
209
|
+
if (controllers) log(`Controllers: ${controllers.join(' + ')}. ${personas.length * controllers.length} independent playthrough(s); each run is included in the total quote.`);
|
|
210
|
+
if (controllers?.length > 1) log('Each controller gets a fresh game process or browser session. Server-side game progress and randomness may differ; comparison is evidence, not proof of identical starting state.');
|
|
203
211
|
// A long run is its own unit on the server. An older server that ignores
|
|
204
212
|
// run_length prices a standard run, and confirming that would buy a run
|
|
205
213
|
// the CLI then plays as long.
|
|
@@ -245,6 +253,7 @@ async function run(flags, injected, deps, log) {
|
|
|
245
253
|
registered = await api.jobs.register(gameId, {
|
|
246
254
|
modules,
|
|
247
255
|
personas: personaSlugs,
|
|
256
|
+
...(controllers ? { controllers } : {}),
|
|
248
257
|
driver: driverName,
|
|
249
258
|
cli_version: cliVersion,
|
|
250
259
|
confirm_price_cents: estimate.estimate_cents,
|
|
@@ -280,6 +289,8 @@ async function run(flags, injected, deps, log) {
|
|
|
280
289
|
runLength,
|
|
281
290
|
...(runLength === 'long' ? { longRunFactor: options.longRunFactor } : {}),
|
|
282
291
|
controller: options.controller || 'standard',
|
|
292
|
+
...(controllers ? { controllers } : {}),
|
|
293
|
+
recordVideo: Boolean(options.recordVideo || options.uploadVideo),
|
|
283
294
|
concurrency: options.concurrency ?? DEFAULT_CONCURRENCY,
|
|
284
295
|
uploadVideo: Boolean(options.uploadVideo),
|
|
285
296
|
uploadTranscript: Boolean(options.uploadTranscript)
|
|
@@ -291,6 +302,10 @@ async function run(flags, injected, deps, log) {
|
|
|
291
302
|
}
|
|
292
303
|
}
|
|
293
304
|
await stateFile.save(repoRoot, journal);
|
|
305
|
+
if (!runsMatchPlan(registered.runs || [], controllers, personaSlugs)) {
|
|
306
|
+
log(`The registered controller plan does not match the quote. Job ${jobId} is saved for reconciliation; no playthrough or model call was started.`);
|
|
307
|
+
return EXIT.FAILED;
|
|
308
|
+
}
|
|
294
309
|
log(`job ${jobId}${registered.replayed ? ' (replayed)' : ''}: ${formatCents(registered.price && registered.price.price_cents)} charged. ${directory}`);
|
|
295
310
|
|
|
296
311
|
return playJob({
|
|
@@ -451,7 +466,7 @@ export async function playJob({
|
|
|
451
466
|
|
|
452
467
|
const results = [];
|
|
453
468
|
const queue = personaRuns.filter(entry => !stateFile.isRunDone(journal.runs[entry.run_id]));
|
|
454
|
-
const concurrency = WEBDRIVER_NAMES.includes(driverName) ? 1 : Math.max(1, Math.min(Number(options.concurrency) || DEFAULT_CONCURRENCY, queue.length || 1));
|
|
469
|
+
const concurrency = (options.controllers?.length > 1 || WEBDRIVER_NAMES.includes(driverName)) ? 1 : Math.max(1, Math.min(Number(options.concurrency) || DEFAULT_CONCURRENCY, queue.length || 1));
|
|
455
470
|
|
|
456
471
|
/**
|
|
457
472
|
* Runs the process is in the middle of. A Ctrl-C has to move these to
|
|
@@ -462,7 +477,8 @@ export async function playJob({
|
|
|
462
477
|
* back out of, which is exactly what it is for.
|
|
463
478
|
*/
|
|
464
479
|
const inFlight = new Map();
|
|
465
|
-
const
|
|
480
|
+
const abort = new AbortController();
|
|
481
|
+
const interrupt = createInterruptHandler({ api, gameId, jobId, repoRoot, journal, inFlight, log, injected, onInterrupt: () => abort.abort() });
|
|
466
482
|
|
|
467
483
|
try {
|
|
468
484
|
await Promise.all(Array.from({ length: concurrency }, async () => {
|
|
@@ -483,7 +499,7 @@ export async function playJob({
|
|
|
483
499
|
pack,
|
|
484
500
|
driverName,
|
|
485
501
|
target,
|
|
486
|
-
options,
|
|
502
|
+
options: { ...options, signal: abort.signal },
|
|
487
503
|
journal,
|
|
488
504
|
inFlight,
|
|
489
505
|
makeDriver,
|
|
@@ -633,7 +649,8 @@ export async function playPersonaRun({
|
|
|
633
649
|
try {
|
|
634
650
|
if (inFlight) inFlight.set(runId, { state: 'launching' });
|
|
635
651
|
outcome = await runPersona({
|
|
636
|
-
controller: options.controller || 'standard',
|
|
652
|
+
controller: entry.controller || options.controller || 'standard',
|
|
653
|
+
signal: options.signal,
|
|
637
654
|
client,
|
|
638
655
|
api,
|
|
639
656
|
gameId,
|
|
@@ -654,7 +671,7 @@ export async function playPersonaRun({
|
|
|
654
671
|
maxActions,
|
|
655
672
|
heartbeat,
|
|
656
673
|
ledger,
|
|
657
|
-
recordVideo: Boolean(options.uploadVideo),
|
|
674
|
+
recordVideo: Boolean(options.recordVideo || options.uploadVideo),
|
|
658
675
|
// Written the moment the warning arrives, not after the run: a run that
|
|
659
676
|
// then fails still says in state.json that it was warned.
|
|
660
677
|
onBudgetWarning: async () => {
|
|
@@ -668,6 +685,10 @@ export async function playPersonaRun({
|
|
|
668
685
|
});
|
|
669
686
|
} catch (error) {
|
|
670
687
|
heartbeat.stop();
|
|
688
|
+
if (options.signal?.aborted) {
|
|
689
|
+
await stateFile.patchRun(repoRoot, journal, runId, { state: 'interrupted', completed: false, actions_taken: error.outcome?.actionsTaken || 0 });
|
|
690
|
+
return { runId, persona: persona.slug, state: 'interrupted', error };
|
|
691
|
+
}
|
|
671
692
|
if (inFlight) inFlight.delete(runId);
|
|
672
693
|
const failed = error.outcome || { state: 'failed', quitReason: 'error', actionsTaken: 0, report: null, reportLint: [], reachedState: 'playing' };
|
|
673
694
|
await stateFile.patchRun(repoRoot, journal, runId, {
|
|
@@ -726,6 +747,10 @@ export async function playPersonaRun({
|
|
|
726
747
|
return { runId, persona: persona.slug, state: failed.state, quitReason: failed.quitReason, error };
|
|
727
748
|
}
|
|
728
749
|
heartbeat.stop();
|
|
750
|
+
if (options.signal?.aborted) {
|
|
751
|
+
await stateFile.patchRun(repoRoot, journal, runId, { state: 'interrupted', completed: false, actions_taken: outcome.actionsTaken, report_written: Boolean(outcome.report) });
|
|
752
|
+
return { runId, persona: persona.slug, state: 'interrupted' };
|
|
753
|
+
}
|
|
729
754
|
if (inFlight) inFlight.delete(runId);
|
|
730
755
|
|
|
731
756
|
await stateFile.patchRun(repoRoot, journal, runId, {
|
|
@@ -804,6 +829,9 @@ export async function deliver({ deps, api, gameId, jobId, runId, directory, outc
|
|
|
804
829
|
// The run card's duration. The server lane accepts `duration`; an older
|
|
805
830
|
// server ignores a key it does not know.
|
|
806
831
|
if (outcome.duration) body.duration = durationBody(outcome.duration);
|
|
832
|
+
// Keep the same bounded opt-in status in the older-server partial-report fallback.
|
|
833
|
+
const recording = options.uploadVideo === true ? recordingForCompletion(outcome.recording) : null;
|
|
834
|
+
Object.assign(body, await recordingCompletionBody(directory, recording ? { recording } : {}, options.uploadVideo === true));
|
|
807
835
|
if (outcome.report) body.report = outcome.report;
|
|
808
836
|
// `report_md` is what the server lints for its four required sections. Not
|
|
809
837
|
// sending it means `report_lint` is silently empty on every run, which reads
|
|
@@ -842,11 +870,11 @@ export async function deliver({ deps, api, gameId, jobId, runId, directory, outc
|
|
|
842
870
|
body
|
|
843
871
|
});
|
|
844
872
|
} catch (error) {
|
|
845
|
-
// A server older than partial reports refuses one as `invalid_report
|
|
846
|
-
//
|
|
847
|
-
//
|
|
848
|
-
// report.json either way.
|
|
849
|
-
if (!(body.report && body.report.partial && error && error.status === 422 && error.code
|
|
873
|
+
// A server older than partial reports refuses one as `invalid_report`,
|
|
874
|
+
// and any server refuses one with a secret shaped value in it. The
|
|
875
|
+
// uploads are already done by then, so the run is completed without it
|
|
876
|
+
// rather than left open: the notes stay on disk in report.json either way.
|
|
877
|
+
if (!(body.report && body.report.partial && error && error.status === 422 && ['invalid_report', 'secrets_detected'].includes(error.code))) throw error;
|
|
850
878
|
log(`run ${runId}: the server refused the partial report (${String(error.message)}), so the run is completed without it. The notes are in ${directory}.`);
|
|
851
879
|
if (delivery) delivery.reportRefused = true;
|
|
852
880
|
const { report: ignored, report_md: alsoIgnored, ...withoutReport } = body;
|
|
@@ -1009,7 +1037,10 @@ async function aggregatePhase({
|
|
|
1009
1037
|
|
|
1010
1038
|
// The aggregate reads reports off disk, so a resumed job whose earlier runs
|
|
1011
1039
|
// were played by a previous process is covered too.
|
|
1012
|
-
|
|
1040
|
+
// A failed run's report explains the failure and its unit is refunded. It is
|
|
1041
|
+
// not a result to summarise, and the server refuses a summary of only those.
|
|
1042
|
+
const reports = (await loadRunReports(repoRoot, jobId, personaRuns, journal))
|
|
1043
|
+
.map(run => (run.state === 'failed' ? { ...run, report: null } : run));
|
|
1013
1044
|
let aggregate;
|
|
1014
1045
|
try {
|
|
1015
1046
|
aggregate = await buildAggregate({
|
|
@@ -1135,7 +1166,7 @@ async function publishAggregate({ deps, api, gameId, jobId, repoRoot, directory,
|
|
|
1135
1166
|
*
|
|
1136
1167
|
* @returns {{signalled: boolean, settle: () => Promise<void>, release: () => void}}
|
|
1137
1168
|
*/
|
|
1138
|
-
export function createInterruptHandler({ api, gameId, jobId, repoRoot, journal, inFlight, log, message = null, injected = {} }) {
|
|
1169
|
+
export function createInterruptHandler({ api, gameId, jobId, repoRoot, journal, inFlight, log, message = null, injected = {}, onInterrupt = () => {} }) {
|
|
1139
1170
|
const signals = injected.signals || ['SIGINT', 'SIGTERM'];
|
|
1140
1171
|
const target = injected.process || process;
|
|
1141
1172
|
const state = { signalled: false, signal: null };
|
|
@@ -1144,6 +1175,7 @@ export function createInterruptHandler({ api, gameId, jobId, repoRoot, journal,
|
|
|
1144
1175
|
if (state.signalled) return;
|
|
1145
1176
|
state.signalled = true;
|
|
1146
1177
|
state.signal = signal;
|
|
1178
|
+
onInterrupt(signal);
|
|
1147
1179
|
log(`\n${signal}: ${message || `finishing the turn in flight, then stopping. Nothing is lost; "ravensight-playtest resume ${jobId}" carries on.`}`);
|
|
1148
1180
|
// Hand the signal back to the default behaviour, so a second press kills
|
|
1149
1181
|
// the process rather than being swallowed by a handler that is now done.
|
|
@@ -1285,15 +1317,15 @@ export async function loadRunReports(repoRoot, jobId, personaRuns, journal = nul
|
|
|
1285
1317
|
// summary must not count findings the dashboard does not have.
|
|
1286
1318
|
const recorded = journal && journal.runs ? journal.runs[entry.run_id] : null;
|
|
1287
1319
|
if (recorded && recorded.report_refused === true) {
|
|
1288
|
-
out.push({ runId: entry.run_id, persona: entry.persona, state: 'failed', report: null, reportRefused: true });
|
|
1320
|
+
out.push({ runId: entry.run_id, persona: entry.persona, ...(entry.controller ? { controller: entry.controller } : {}), state: 'failed', report: null, reportRefused: true });
|
|
1289
1321
|
continue;
|
|
1290
1322
|
}
|
|
1291
1323
|
const file = path.join(runDirFor(repoRoot, jobId, entry.run_id), RUN_FILES.reportJson);
|
|
1292
1324
|
try {
|
|
1293
1325
|
const report = JSON.parse(await readFile(file, 'utf8'));
|
|
1294
|
-
out.push({ runId: entry.run_id, persona: entry.persona, state: report && report.partial === true ? 'budget_exceeded' : 'succeeded', report });
|
|
1326
|
+
out.push({ runId: entry.run_id, persona: entry.persona, ...(entry.controller ? { controller: entry.controller } : {}), state: recorded?.state || entry.state || (report && report.partial === true ? 'budget_exceeded' : 'succeeded'), report });
|
|
1295
1327
|
} catch {
|
|
1296
|
-
out.push({ runId: entry.run_id, persona: entry.persona, state: 'failed', report: null });
|
|
1328
|
+
out.push({ runId: entry.run_id, persona: entry.persona, ...(entry.controller ? { controller: entry.controller } : {}), state: 'failed', report: null });
|
|
1297
1329
|
}
|
|
1298
1330
|
}
|
|
1299
1331
|
return out;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { ProxyError, STEPS } from './model.js';
|
|
2
|
-
export
|
|
2
|
+
export { CONTROLLERS } from './controllers.js';
|
|
3
3
|
export const JEV_CONTROLLER_PROMPT = '\nRavensight Playtester action-selection mode: describe the currently visible screen briefly in text, including each proposed target and its screenshot coordinates, then propose one to three ALTERNATIVE next actions as tool calls. Only one will be executed after a separate selector chooses. These are alternatives, never a sequential plan. Use only observed controls. For a canvas game without accessible DOM controls, include a target label describing the visible control AND screenshot x/y coordinates; the driver uses those coordinates. Never invent selectors such as textbox or button names. When the brief identifies a field as a public player label, use Ravensight Playtester. Credentials or private access codes require human review. When the game is loading or transitioning, propose a bounded wait tool call. Do not place secrets in proposals. After their tool results, observe the resulting state before proposing again.';
|
|
4
4
|
export function describeAction(tool) {
|
|
5
5
|
const input = tool.input || {};
|
|
@@ -14,7 +14,7 @@ export function describeAction(tool) {
|
|
|
14
14
|
};
|
|
15
15
|
return String(descriptions[tool.name] || `${tool.name}: ${JSON.stringify(input)}`).slice(0, 300);
|
|
16
16
|
}
|
|
17
|
-
export async function selectAction({ api, jobId, runId, persona, observation, toolUses, recentActions = [], intentNotes = [] }) {
|
|
17
|
+
export async function selectAction({ api, jobId, runId, persona, observation, toolUses, controller = 'jev', recentActions = [], intentNotes = [] }) {
|
|
18
18
|
const proposals = toolUses.slice(0, 8);
|
|
19
19
|
// Observation does not input anything into the game. Keep it in the existing
|
|
20
20
|
// bounded driver loop; Jev selects game inputs, not whether a frame may load.
|
|
@@ -22,6 +22,7 @@ export async function selectAction({ api, jobId, runId, persona, observation, to
|
|
|
22
22
|
return { status: 'selected', toolId: proposals[0].id, source: 'observation' };
|
|
23
23
|
}
|
|
24
24
|
const body = {
|
|
25
|
+
controller,
|
|
25
26
|
objective: ('Play the game according to the supplied persona, exploring visible controls and gathering evidence. Developer scope: ' + intentNotes.map(note => note.text || '').join(' ')).slice(0, 1000),
|
|
26
27
|
// The observer already applies the full persona. The selector needs its identity,
|
|
27
28
|
// current visible state and proposed inputs, not a second personality prompt.
|
package/src/run/aggregate.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { createHash } from 'node:crypto';
|
|
2
|
+
import { compareControllers, comparisonMarkdown } from './comparison.js';
|
|
2
3
|
import { fillPlaceholders } from './placeholders.js';
|
|
3
4
|
import { writeFile } from 'node:fs/promises';
|
|
4
5
|
import path from 'node:path';
|
|
@@ -458,7 +459,9 @@ export async function buildAggregate({
|
|
|
458
459
|
usage: ledger ? aggregateUsage(ledger) : { usd: 0, by_model: {} }
|
|
459
460
|
};
|
|
460
461
|
|
|
461
|
-
|
|
462
|
+
const comparison = compareControllers(runs);
|
|
463
|
+
if (comparison) json.controller_comparison = comparison;
|
|
464
|
+
return { json, markdown: renderAggregateMarkdown(json) + comparisonMarkdown(comparison) };
|
|
462
465
|
}
|
|
463
466
|
|
|
464
467
|
/**
|
package/src/run/args.js
CHANGED
|
@@ -16,6 +16,7 @@ const VALUED = new Map([
|
|
|
16
16
|
['--driver', 'driver'],
|
|
17
17
|
['--driver-config', 'driverConfig'],
|
|
18
18
|
['--controller', 'controller'],
|
|
19
|
+
['--compare-with', 'compareWith'],
|
|
19
20
|
['--build-url', 'buildUrl'],
|
|
20
21
|
['--build-dir', 'buildDir'],
|
|
21
22
|
['--godot-project', 'godotProject'],
|
|
@@ -29,6 +30,7 @@ const VALUED = new Map([
|
|
|
29
30
|
/** Options that are just present or absent. */
|
|
30
31
|
const BOOLEAN = new Map([
|
|
31
32
|
['--upload-video', 'uploadVideo'],
|
|
33
|
+
['--record-video', 'recordVideo'],
|
|
32
34
|
['--upload-transcript', 'uploadTranscript'],
|
|
33
35
|
['--dry-upload', 'dryUpload'],
|
|
34
36
|
['--offline-upload', 'offlineUpload'],
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
// Deterministic comparison of the reports already paid for. No new model call.
|
|
2
|
+
export function compareControllers(runs) {
|
|
3
|
+
const planned = runs.filter(run => run.controller);
|
|
4
|
+
const controllers = [...new Set(planned.map(run => run.controller))];
|
|
5
|
+
if (controllers.length < 2) return null;
|
|
6
|
+
const rows = planned.map(run => ({
|
|
7
|
+
run_id: run.runId, persona: run.persona, controller: run.controller,
|
|
8
|
+
state: run.state, report_available: Boolean(run.report),
|
|
9
|
+
partial: !run.report || run.report.partial === true || run.state !== 'succeeded',
|
|
10
|
+
actions_taken: run.report?.actions_taken ?? null,
|
|
11
|
+
}));
|
|
12
|
+
const groups = new Map();
|
|
13
|
+
for (const run of planned) for (const finding of run.report?.findings || []) {
|
|
14
|
+
if (finding.dropped || !Array.isArray(finding.evidence) || !finding.evidence.length) continue;
|
|
15
|
+
const key = JSON.stringify([run.persona, finding.category || '', String(finding.title || '').trim().toLowerCase().replace(/\s+/g, ' ')]);
|
|
16
|
+
if (!groups.has(key)) groups.set(key, { title: finding.title, persona: run.persona, category: finding.category, occurrences: [] });
|
|
17
|
+
groups.get(key).occurrences.push({ run_id: run.runId, controller: run.controller, evidence: finding.evidence });
|
|
18
|
+
}
|
|
19
|
+
const findings = [...groups.values()].map(group => ({ ...group,
|
|
20
|
+
match: new Set(group.occurrences.map(row => row.controller)).size > 1 ? 'reported_by_multiple_controllers' : 'reported_by_one_controller' }));
|
|
21
|
+
return { schema_version: '1.0', controllers, status: rows.some(row => row.partial) ? 'inconclusive' : 'reports_complete',
|
|
22
|
+
limits: ['Controllers use separate game sessions. Server saves and random state are not proven identical.', 'Matches use the same persona, category and exact normalized title. Agreement does not verify a bug; absence does not prove a pass.'], runs: rows, findings };
|
|
23
|
+
}
|
|
24
|
+
export function comparisonMarkdown(comparison) {
|
|
25
|
+
if (!comparison) return '';
|
|
26
|
+
const safe = value => String(value ?? '').replace(/[\r\n|]/g, ' ');
|
|
27
|
+
return '\n\n## Controller comparison\n\n' + comparison.limits.join(' ') + '\n\n' +
|
|
28
|
+
(comparison.status === 'inconclusive' ? 'At least one playthrough is missing or partial. This comparison is inconclusive.\n\n' : '') +
|
|
29
|
+
'| Controller | Persona | Run | Report |\n| --- | --- | --- | --- |\n' +
|
|
30
|
+
comparison.runs.map(row => `| ${safe(row.controller)} | ${safe(row.persona)} | ${safe(row.run_id)} | ${row.partial ? 'Incomplete' : 'Complete'} |`).join('\n') +
|
|
31
|
+
'\n\n' + comparison.findings.map(group => `- ${safe(group.title)}: ${[...new Set(group.occurrences.map(row => row.controller))].join(', ')} (${group.occurrences.map(row => safe(row.run_id)).join(', ')}).`).join('\n');
|
|
32
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
export const CONTROLLERS = ['standard', 'jev', 'rune'];
|
|
2
|
+
export function controllerPlan(options = {}) {
|
|
3
|
+
if (options.compareWith && options.controller) throw new Error('Use --controller for one playthrough or --compare-with for a comparison.');
|
|
4
|
+
if (!options.compareWith && !options.controller) return null;
|
|
5
|
+
const plan = options.compareWith ? ['standard', ...String(options.compareWith).split(',')] : [options.controller];
|
|
6
|
+
if (plan.length > 3 || plan.some(x => !CONTROLLERS.includes(x)) || new Set(plan).size !== plan.length) throw new Error('Choose standard, jev or rune once each. --compare-with jev runs standard and Jev separately.');
|
|
7
|
+
return plan;
|
|
8
|
+
}
|
|
9
|
+
export function quoteMatchesPlan(quote, plan, personas) {
|
|
10
|
+
return !plan || (Array.isArray(quote.controllers) && JSON.stringify(quote.controllers) === JSON.stringify(plan) && quote.persona_run_count === plan.length * personas.length);
|
|
11
|
+
}
|
|
12
|
+
export function runsMatchPlan(runs, plan, personas) {
|
|
13
|
+
if (!plan) return true;
|
|
14
|
+
const actual = runs.filter(run => run.module === 'persona_playtest').map(run => `${run.persona}/${run.controller}`).sort();
|
|
15
|
+
const expected = personas.flatMap(persona => plan.map(controller => `${persona}/${controller}`)).sort();
|
|
16
|
+
return JSON.stringify(actual) === JSON.stringify(expected);
|
|
17
|
+
}
|
|
@@ -18,7 +18,7 @@ import { createHash } from 'node:crypto';
|
|
|
18
18
|
* observe() -> Observation
|
|
19
19
|
* act(action) -> ActResult
|
|
20
20
|
* screenshot(name, opts) -> { path, relativePath }
|
|
21
|
-
* stop()
|
|
21
|
+
* stop({ reason }) -> { videoPath?: string|null, recording?: Object }
|
|
22
22
|
* ```
|
|
23
23
|
*
|
|
24
24
|
* Optional, used when present:
|
|
@@ -37,10 +37,10 @@ import { createHash } from 'node:crypto';
|
|
|
37
37
|
* description in ../tools.js says so, because the model is the one
|
|
38
38
|
* choosing.
|
|
39
39
|
* - **`stop()` may return no video.** `playwright_web` hands back a
|
|
40
|
-
* `session.webm` from Playwright's own recorder. `godot_driver`
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
*
|
|
40
|
+
* `session.webm` from Playwright's own recorder. `godot_driver` encodes
|
|
41
|
+
* timestamped viewport frames to `session.mp4` with optional local ffmpeg.
|
|
42
|
+
* `recording.status` distinguishes complete, partial, unsupported and failed
|
|
43
|
+
* capture; absence of an encoder or rendered viewport never masquerades as video.
|
|
44
44
|
*
|
|
45
45
|
* `Observation.scene`, `.scenes` and `.scripts` are how a finding from a
|
|
46
46
|
* running Godot game can cite a `res://` path; the web driver leaves them
|
package/src/run/drivers/godot.js
CHANGED
|
@@ -9,8 +9,8 @@
|
|
|
9
9
|
//
|
|
10
10
|
// Both of the differences ./driver.js names fall on this driver's side: a
|
|
11
11
|
// string `click` target is a scene tree node path, not an accessibility ref,
|
|
12
|
-
// and
|
|
13
|
-
//
|
|
12
|
+
// and recording uses the selected viewport, with optional local ffmpeg encoding.
|
|
13
|
+
// Without an encoder or a rendered viewport it reports unsupported, never a video.
|
|
14
14
|
//
|
|
15
15
|
// What happens at launch, in order:
|
|
16
16
|
// 1. find the developer's own Godot binary (spec 17 detection order),
|
|
@@ -34,6 +34,8 @@ import {
|
|
|
34
34
|
prepareProjectCopy,
|
|
35
35
|
} from './godot-project.js';
|
|
36
36
|
import { DriverRpcError, GodotRpcClient } from './godot-rpc.js';
|
|
37
|
+
import { detectFfmpeg } from '../../detect.js';
|
|
38
|
+
import { recordingLimits, recordingViewport, recordingDirectory, recordingMetadata, saveRecording, encodeFrames } from './recording.js';
|
|
37
39
|
|
|
38
40
|
export const DRIVER_NAME = 'godot_driver';
|
|
39
41
|
export const BOOT_TIMEOUT_MS = 60000;
|
|
@@ -124,6 +126,9 @@ export class GodotDriver {
|
|
|
124
126
|
prepare = prepareProjectCopy,
|
|
125
127
|
choosePort = findFreePort,
|
|
126
128
|
now = () => Date.now(),
|
|
129
|
+
recording = {},
|
|
130
|
+
detectEncoder = detectFfmpeg,
|
|
131
|
+
encode = encodeFrames,
|
|
127
132
|
} = {}) {
|
|
128
133
|
this.name = DRIVER_NAME;
|
|
129
134
|
this.projectDir = projectDir;
|
|
@@ -148,6 +153,11 @@ export class GodotDriver {
|
|
|
148
153
|
this._prepare = prepare;
|
|
149
154
|
this._choosePort = choosePort;
|
|
150
155
|
this._now = now;
|
|
156
|
+
this._recordingLimits = recordingLimits(recording);
|
|
157
|
+
this._detectEncoder = detectEncoder;
|
|
158
|
+
this._encode = encode;
|
|
159
|
+
this._frames = [];
|
|
160
|
+
this._frameBytes = 0;
|
|
151
161
|
|
|
152
162
|
this.rpc = null;
|
|
153
163
|
this.child = null;
|
|
@@ -178,8 +188,13 @@ export class GodotDriver {
|
|
|
178
188
|
return !this._stopped && this.exitCode !== null && this.exitCode !== 0;
|
|
179
189
|
}
|
|
180
190
|
|
|
181
|
-
async launch({ recordVideo = false, viewport } = {}) {
|
|
191
|
+
async launch({ recordVideo = false, viewport, signal } = {}) {
|
|
192
|
+
if (signal?.aborted) throw new Error('run canceled before game launch');
|
|
182
193
|
if (viewport) this.viewport = viewport;
|
|
194
|
+
if (recordVideo) {
|
|
195
|
+
const size = recordingViewport(this.viewport, this._recordingLimits);
|
|
196
|
+
this.viewport = { w: size.width, h: size.height };
|
|
197
|
+
}
|
|
183
198
|
if (!this.projectDir) throw new Error('godot driver needs a projectDir');
|
|
184
199
|
this.godot = this.godotPath
|
|
185
200
|
? await this._detect({ explicitPath: this.godotPath, env: this.env, projectDir: this.projectDir })
|
|
@@ -198,6 +213,7 @@ export class GodotDriver {
|
|
|
198
213
|
this.logPath = this.runDir ? path.join(this.runDir, 'godot.log') : null;
|
|
199
214
|
|
|
200
215
|
const args = ['--path', this.projectCopy];
|
|
216
|
+
if (recordVideo) args.push('--audio-driver', 'Dummy');
|
|
201
217
|
if (this.headless) args.push('--headless');
|
|
202
218
|
else args.push('--resolution', `${this.viewport.w}x${this.viewport.h}`);
|
|
203
219
|
args.push('--', '--ravensight-driver', `--ravensight-driver-port=${this.port}`);
|
|
@@ -226,6 +242,10 @@ export class GodotDriver {
|
|
|
226
242
|
}
|
|
227
243
|
|
|
228
244
|
if (recordVideo) await this._startFrameCapture();
|
|
245
|
+
this._signal = signal;
|
|
246
|
+
this._onAbort = () => { void this.stop({ reason: 'canceled' }).catch(() => {}); };
|
|
247
|
+
signal?.addEventListener('abort', this._onAbort, { once: true });
|
|
248
|
+
if (signal?.aborted) { await this.stop({ reason: 'canceled' }); throw new Error('run canceled during game launch'); }
|
|
229
249
|
return {};
|
|
230
250
|
} catch (error) {
|
|
231
251
|
// Everything past this point has already spawned a game and copied a
|
|
@@ -289,6 +309,10 @@ export class GodotDriver {
|
|
|
289
309
|
this.exitCode = code;
|
|
290
310
|
this.exitSignal = signal;
|
|
291
311
|
if (log) log.end();
|
|
312
|
+
if (!this._stopped && this._recordingStartedAt != null) {
|
|
313
|
+
this._endCapture(code === 0 ? 'game_exited' : 'game_crashed');
|
|
314
|
+
void this._finalizeRecording().catch(() => {});
|
|
315
|
+
}
|
|
292
316
|
});
|
|
293
317
|
this.child.on('error', (error) => {
|
|
294
318
|
this._consoleQueue.push({ kind: 'driver', text: `failed to launch Godot: ${error.message}` });
|
|
@@ -501,43 +525,114 @@ export class GodotDriver {
|
|
|
501
525
|
this.chapters.push({ title, at: this._startedAt === null ? 0 : this._now() - this._startedAt });
|
|
502
526
|
}
|
|
503
527
|
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
this.
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
528
|
+
// Capture only this project's viewport. No screen/audio device is opened.
|
|
529
|
+
async _startFrameCapture() {
|
|
530
|
+
this._recordingStartedAt = this._now();
|
|
531
|
+
if (this.runDir) this._framesDir = await recordingDirectory(this.runDir);
|
|
532
|
+
if (this.headless || !this.runDir) {
|
|
533
|
+
this._recordingUnsupported = this.headless ? 'headless_no_viewport' : 'run_directory_required';
|
|
534
|
+
return;
|
|
535
|
+
}
|
|
536
|
+
this._encoder = await this._detectEncoder();
|
|
537
|
+
if (!this._encoder?.path) {
|
|
538
|
+
this._recordingUnsupported = 'encoder_unavailable';
|
|
539
|
+
this._consoleQueue.push({ kind: 'driver', text: 'Video unavailable: install optional ffmpeg or set RAVENSIGHT_FFMPEG. No video was recorded.' });
|
|
540
|
+
return;
|
|
541
|
+
}
|
|
542
|
+
this._recordingStartedAt = this._now();
|
|
543
|
+
await this._captureFrame();
|
|
544
|
+
if (this._captureEndedAt != null) return;
|
|
545
|
+
this._videoTimer = setInterval(() => { void this._captureFrame(); }, Math.round(1000 / this._recordingLimits.fps));
|
|
546
|
+
this._videoTimer.unref?.();
|
|
547
|
+
this._videoDeadline = setTimeout(() => this._endCapture('duration_limit'), this._recordingLimits.durationMs);
|
|
548
|
+
this._videoDeadline.unref?.();
|
|
549
|
+
}
|
|
550
|
+
|
|
551
|
+
_endCapture(reason) {
|
|
552
|
+
if (this._captureEndedAt != null) return;
|
|
553
|
+
this._captureEndedAt = Math.min(this._now(), this._recordingStartedAt + this._recordingLimits.durationMs);
|
|
554
|
+
this._captureReason = reason;
|
|
555
|
+
clearInterval(this._videoTimer);
|
|
556
|
+
clearTimeout(this._videoDeadline);
|
|
557
|
+
this._videoTimer = null;
|
|
517
558
|
}
|
|
518
559
|
|
|
519
560
|
async _captureFrame() {
|
|
520
|
-
if (this.
|
|
521
|
-
this.
|
|
522
|
-
|
|
523
|
-
const shot = await this._rawScreenshot();
|
|
524
|
-
if (!shot) return;
|
|
525
|
-
this._frameIndex += 1;
|
|
526
|
-
const file = path.join(this._framesDir, `${String(this._frameIndex).padStart(6, '0')}.png`);
|
|
527
|
-
await fs.writeFile(file, Buffer.from(shot.data, 'base64'));
|
|
528
|
-
} catch {
|
|
529
|
-
// A dropped frame is never worth interrupting a run for.
|
|
530
|
-
} finally {
|
|
531
|
-
this._capturing = false;
|
|
561
|
+
if (this._capturePromise || this._stopped || this._captureEndedAt != null) return;
|
|
562
|
+
if (this._now() - this._recordingStartedAt >= this._recordingLimits.durationMs) {
|
|
563
|
+
this._endCapture('duration_limit'); return;
|
|
532
564
|
}
|
|
565
|
+
const at = this._now();
|
|
566
|
+
this._capturePromise = (async () => {
|
|
567
|
+
try {
|
|
568
|
+
const shot = await this._rawScreenshot();
|
|
569
|
+
if (!shot) return;
|
|
570
|
+
const max = this._recordingLimits.singleFrameBytes;
|
|
571
|
+
if (typeof shot.data !== 'string' || shot.data.length > Math.ceil(max / 3) * 4) {
|
|
572
|
+
this._endCapture('frame_size_limit'); return;
|
|
573
|
+
}
|
|
574
|
+
const bytes = Buffer.from(shot.data, 'base64');
|
|
575
|
+
if (bytes.length > max || this._frameBytes + bytes.length > this._recordingLimits.frameBytes) {
|
|
576
|
+
this._endCapture('frame_bytes_limit'); return;
|
|
577
|
+
}
|
|
578
|
+
const name = `${String(this._frameIndex + 1).padStart(6, '0')}.png`;
|
|
579
|
+
await fs.writeFile(path.join(this._framesDir, name), bytes, { flag: 'wx', mode: 0o600 });
|
|
580
|
+
this._frames.push({ name, at });
|
|
581
|
+
this._frameIndex += 1;
|
|
582
|
+
this._frameBytes += bytes.length;
|
|
583
|
+
} catch {
|
|
584
|
+
// A disconnected/crashed game can still leave a useful partial recording.
|
|
585
|
+
this._endCapture('capture_failed');
|
|
586
|
+
}
|
|
587
|
+
})();
|
|
588
|
+
try { await this._capturePromise; } finally { this._capturePromise = null; }
|
|
533
589
|
}
|
|
534
590
|
|
|
535
|
-
|
|
591
|
+
_finalizeRecording() {
|
|
592
|
+
if (this._finalizePromise) return this._finalizePromise;
|
|
593
|
+
this._finalizePromise = (async () => {
|
|
594
|
+
if (this._recordingStartedAt == null) return {};
|
|
595
|
+
await this._capturePromise;
|
|
596
|
+
const endedAt = this._captureEndedAt ?? this._now();
|
|
597
|
+
let metadata = recordingMetadata({ driver: this.name, startedAt: this._recordingStartedAt, endedAt,
|
|
598
|
+
reason: this._recordingUnsupported ?? this._captureReason ?? 'stopped',
|
|
599
|
+
status: this._recordingUnsupported ? 'unsupported' : 'failed', frameCount: this._frameIndex,
|
|
600
|
+
captureFpsLimit: this._recordingLimits.fps, timeOrigin: 'first-viewport-frame',
|
|
601
|
+
});
|
|
602
|
+
let videoPath = null;
|
|
603
|
+
if (!this._recordingUnsupported && this._framesDir) {
|
|
604
|
+
try {
|
|
605
|
+
const encoded = await this._encode({ directory: this._framesDir, frames: this._frames, endedAt,
|
|
606
|
+
encoder: this._encoder, limits: this._recordingLimits });
|
|
607
|
+
videoPath = path.join(this.runDir, 'session.mp4');
|
|
608
|
+
await fs.rename(encoded.file, videoPath);
|
|
609
|
+
metadata = { ...metadata, status: ['stopped', 'completed', 'game_exited'].includes(metadata.reason) ? 'complete' : 'partial',
|
|
610
|
+
relativePath: 'session.mp4', mimeType: encoded.mimeType, bytes: encoded.bytes,
|
|
611
|
+
durationMs: encoded.durationMs, durationSource: encoded.durationSource ?? 'frame-timestamps',
|
|
612
|
+
firstFrameOffsetMs: this._frames[0].at - this._recordingStartedAt };
|
|
613
|
+
if (encoded.truncated) { metadata.status = 'partial'; metadata.reason = 'size_limit'; }
|
|
614
|
+
} catch (error) {
|
|
615
|
+
metadata.reason = ['no_frames', 'encoder_unavailable', 'encoding_timeout', 'encoding_failed'].includes(error.message) ? error.message : 'encoding_failed';
|
|
616
|
+
}
|
|
617
|
+
}
|
|
618
|
+
// Captured game frames are local temporary data, never upload artifacts.
|
|
619
|
+
if (this._framesDir) await fs.rm(this._framesDir, { recursive: true, force: true });
|
|
620
|
+
if (this.runDir) await saveRecording(this.runDir, metadata);
|
|
621
|
+
return { videoPath, recording: metadata };
|
|
622
|
+
})();
|
|
623
|
+
return this._finalizePromise;
|
|
624
|
+
}
|
|
625
|
+
|
|
626
|
+
stop(options = {}) {
|
|
627
|
+
if (!this._stopPromise) this._stopPromise = Promise.resolve().then(() => this._stop(options));
|
|
628
|
+
return this._stopPromise;
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
async _stop({ removeTemp = !this.keepTemp, reason = 'stopped' } = {}) {
|
|
536
632
|
this._stopped = true;
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
}
|
|
633
|
+
this._signal?.removeEventListener('abort', this._onAbort);
|
|
634
|
+
if (this._recordingStartedAt != null) this._endCapture(reason);
|
|
635
|
+
await this._capturePromise;
|
|
541
636
|
if (this.rpc && !this.rpc.closed) {
|
|
542
637
|
await this.rpc.call('quit', { code: 0 }, { timeoutMs: 2000 }).catch(() => {});
|
|
543
638
|
}
|
|
@@ -549,12 +644,9 @@ export class GodotDriver {
|
|
|
549
644
|
this.projectCopy = null;
|
|
550
645
|
}
|
|
551
646
|
return {
|
|
552
|
-
videoPath: null,
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
chapters: this.chapters,
|
|
556
|
-
logPath: this.logPath,
|
|
557
|
-
exitCode: this.exitCode,
|
|
647
|
+
videoPath: null, ...await this._finalizeRecording(),
|
|
648
|
+
frameCount: this._frameIndex, chapters: this.chapters,
|
|
649
|
+
logPath: this.logPath, exitCode: this.exitCode,
|
|
558
650
|
};
|
|
559
651
|
}
|
|
560
652
|
|