ravensight-playtest 0.2.3 → 0.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +89 -6
- package/docs/ADAPTERS.md +77 -0
- package/package.json +2 -1
- package/src/cli.js +12 -6
- package/src/commands/check.js +81 -0
- package/src/commands/fix.js +8 -4
- package/src/commands/init.js +29 -4
- package/src/commands/profile.js +2 -1
- package/src/commands/run.js +81 -26
- package/src/run/actionSelector.js +47 -0
- package/src/run/aggregate.js +10 -5
- package/src/run/args.js +19 -3
- package/src/run/context.js +12 -1
- package/src/run/drivers/driver.js +6 -3
- package/src/run/drivers/index.js +6 -6
- package/src/run/drivers/stdio.js +111 -0
- package/src/run/drivers/web.js +153 -7
- package/src/run/drivers/webdriver.js +67 -0
- package/src/run/fixLoop.js +52 -3
- package/src/run/personaLoop.js +204 -16
- package/src/run/resume.js +1 -0
- package/src/run/tools.js +19 -3
- package/src/states.js +4 -1
- package/src/ui/index.js +28 -0
package/src/commands/run.js
CHANGED
|
@@ -2,7 +2,7 @@ import { randomUUID } from 'node:crypto';
|
|
|
2
2
|
import { mkdir, readFile, stat } from 'node:fs/promises';
|
|
3
3
|
import path from 'node:path';
|
|
4
4
|
import { buildAggregate, writeAggregate } from '../run/aggregate.js';
|
|
5
|
-
import { optionsFrom } from '../run/args.js';
|
|
5
|
+
import { optionsFrom, unknownOptionNotes } from '../run/args.js';
|
|
6
6
|
import { getDeps } from '../run/deps.js';
|
|
7
7
|
import { createDriver } from '../run/drivers/index.js';
|
|
8
8
|
import EXIT from '../run/exit.js';
|
|
@@ -10,6 +10,8 @@ import { createHeartbeat } from '../run/heartbeat.js';
|
|
|
10
10
|
import { createUsageLedger } from '../run/usage.js';
|
|
11
11
|
import { createModelClient, ModelUnavailableError, ProxyError } from '../run/model.js';
|
|
12
12
|
import { jobDir, runDir as runDirFor, JOB_FILES, RUN_FILES } from '../run/paths.js';
|
|
13
|
+
import { readBridgeConfig } from "../run/drivers/stdio.js";
|
|
14
|
+
import { WEBDRIVER_NAMES, readDriverConfig } from '../run/drivers/webdriver.js';
|
|
13
15
|
import { runPersona } from '../run/personaLoop.js';
|
|
14
16
|
import * as stateFile from '../run/state.js';
|
|
15
17
|
import { emitRunEvents } from '../run/synthetic.js';
|
|
@@ -78,6 +80,7 @@ async function run(flags, injected, deps, log) {
|
|
|
78
80
|
const { options, unknown } = optionsFrom(flags);
|
|
79
81
|
if (unknown.length > 0) {
|
|
80
82
|
log(`I do not know these options: ${unknown.join(', ')}. Run "ravensight-playtest run --help".`);
|
|
83
|
+
for (const line of unknownOptionNotes(unknown)) log(line);
|
|
81
84
|
return EXIT.ENVIRONMENT;
|
|
82
85
|
}
|
|
83
86
|
for (const line of unusedFlagWarnings(options)) log(line);
|
|
@@ -86,10 +89,11 @@ async function run(flags, injected, deps, log) {
|
|
|
86
89
|
if (setup.code !== undefined) return setup.code;
|
|
87
90
|
const { context, gameId, repoRoot, apiUrl, cliVersion, api } = setup;
|
|
88
91
|
|
|
92
|
+
if (options.controller && !['standard', 'jev'].includes(options.controller)) { log('Choose --controller standard or --controller jev.'); return EXIT.ENVIRONMENT; }
|
|
89
93
|
const driverName = options.driver || (context.config && context.config.driver) || 'playwright_web';
|
|
90
94
|
const target = resolveTarget(options, context.config, driverName);
|
|
91
95
|
if (!target) {
|
|
92
|
-
log('Point the run at a build: --build-url for a web build,
|
|
96
|
+
log('Point the run at a build: --build-url for a web build, --godot-project for Godot, or --driver-config for a native or text bridge.');
|
|
93
97
|
return EXIT.ENVIRONMENT;
|
|
94
98
|
}
|
|
95
99
|
|
|
@@ -97,8 +101,11 @@ async function run(flags, injected, deps, log) {
|
|
|
97
101
|
// `aggregate.*` steps are spendable from, and it is free (the price list has
|
|
98
102
|
// no unit for it), so asking costs nothing and not asking costs the judged
|
|
99
103
|
// dedup and success-criteria passes.
|
|
104
|
+
if (WEBDRIVER_NAMES.includes(driverName)) await readDriverConfig(target);
|
|
105
|
+
if (driverName === "cli_stdio") await readBridgeConfig(target);
|
|
100
106
|
const modules = ['persona_playtest', 'aggregate_report'];
|
|
101
107
|
const pack = await deps.packs.get({ api, gameId, modules });
|
|
108
|
+
if (options.controller === 'jev' && !pack.routing?.action_controllers?.includes('jev')) { log('This server does not currently support the requested action controller. No job was registered.'); return EXIT.ENVIRONMENT; }
|
|
102
109
|
const personaSlugs = resolvePersonas(options, context.config, pack);
|
|
103
110
|
const personas = personaSlugs.map(slug => {
|
|
104
111
|
const found = (pack.personas || []).find(persona => persona.slug === slug);
|
|
@@ -164,7 +171,7 @@ async function run(flags, injected, deps, log) {
|
|
|
164
171
|
cli_version: cliVersion,
|
|
165
172
|
confirm_price_cents: estimate.estimate_cents,
|
|
166
173
|
brief_version: brief.version ?? null,
|
|
167
|
-
build: { kind: driverName === 'playwright_web' ? 'url' : 'godot_project', ref: target },
|
|
174
|
+
build: { kind: driverName === 'playwright_web' ? 'url' : (WEBDRIVER_NAMES.includes(driverName) || driverName === 'cli_stdio') ? 'driver_config' : 'godot_project', ref: target },
|
|
168
175
|
repo: { commit: (context.config && context.config.commit) || '', dirty: Boolean(context.config && context.config.dirty) },
|
|
169
176
|
max_actions: Number.isInteger(options.maxActions) ? options.maxActions : null,
|
|
170
177
|
pack_versions: { pack: pack.pack_version }
|
|
@@ -191,6 +198,7 @@ async function run(flags, injected, deps, log) {
|
|
|
191
198
|
priceCents: registered.price && registered.price.price_cents,
|
|
192
199
|
options: {
|
|
193
200
|
maxActions: options.maxActions ?? null,
|
|
201
|
+
controller: options.controller || 'standard',
|
|
194
202
|
concurrency: options.concurrency ?? DEFAULT_CONCURRENCY,
|
|
195
203
|
uploadVideo: Boolean(options.uploadVideo),
|
|
196
204
|
uploadTranscript: Boolean(options.uploadTranscript)
|
|
@@ -287,17 +295,17 @@ export function personaCountRefusal(count, maxPersonas) {
|
|
|
287
295
|
/**
|
|
288
296
|
* Flags this release parses and then does not act on.
|
|
289
297
|
*
|
|
290
|
-
* Saying so is the point: a developer who passes
|
|
291
|
-
*
|
|
298
|
+
* Saying so is the point: a developer who passes a flag and is not told it
|
|
299
|
+
* did nothing will believe it did something. `--budget-usd` is not here
|
|
300
|
+
* because it is not parsed at all any more: the model budget is the server's,
|
|
301
|
+
* derived from what the job was charged and enforced per call at the proxy,
|
|
302
|
+
* so a spending flag the CLI cannot honour is refused rather than accepted.
|
|
292
303
|
*
|
|
293
304
|
* @param {Object} options
|
|
294
305
|
* @returns {Array<string>}
|
|
295
306
|
*/
|
|
296
307
|
export function unusedFlagWarnings(options) {
|
|
297
308
|
const warnings = [];
|
|
298
|
-
if (options.budgetUsd !== undefined) {
|
|
299
|
-
warnings.push('--budget-usd is not applied by this release: the model budget is the server\'s, derived from what the job was charged, and it is enforced per call at the proxy.');
|
|
300
|
-
}
|
|
301
309
|
if (options.quality !== undefined) {
|
|
302
310
|
warnings.push('--quality is not applied by this release: the model tier is the server\'s decision, and the CLI is not allowed to name one.');
|
|
303
311
|
}
|
|
@@ -349,7 +357,7 @@ export async function playJob({
|
|
|
349
357
|
|
|
350
358
|
const results = [];
|
|
351
359
|
const queue = personaRuns.filter(entry => !stateFile.isRunDone(journal.runs[entry.run_id]));
|
|
352
|
-
const concurrency = Math.max(1, Math.min(Number(options.concurrency) || DEFAULT_CONCURRENCY, queue.length || 1));
|
|
360
|
+
const concurrency = WEBDRIVER_NAMES.includes(driverName) ? 1 : Math.max(1, Math.min(Number(options.concurrency) || DEFAULT_CONCURRENCY, queue.length || 1));
|
|
353
361
|
|
|
354
362
|
/**
|
|
355
363
|
* Runs the process is in the middle of. A Ctrl-C has to move these to
|
|
@@ -385,6 +393,7 @@ export async function playJob({
|
|
|
385
393
|
journal,
|
|
386
394
|
inFlight,
|
|
387
395
|
makeDriver,
|
|
396
|
+
brief,
|
|
388
397
|
log
|
|
389
398
|
}));
|
|
390
399
|
}
|
|
@@ -467,6 +476,7 @@ export async function playPersonaRun({
|
|
|
467
476
|
journal,
|
|
468
477
|
inFlight = null,
|
|
469
478
|
makeDriver,
|
|
479
|
+
brief = null,
|
|
470
480
|
log
|
|
471
481
|
}) {
|
|
472
482
|
const runId = entry.run_id;
|
|
@@ -479,7 +489,7 @@ export async function playPersonaRun({
|
|
|
479
489
|
// Headless is right for a browser and wrong for Godot: a headless Godot
|
|
480
490
|
// routes no GUI input, so it is a smoke-test mode rather than a way to play.
|
|
481
491
|
const headless = driverName === 'godot_driver' ? options.headless === true : options.headed !== true;
|
|
482
|
-
const driver = await makeDriver(driverName, { headless });
|
|
492
|
+
const driver = await makeDriver(driverName, driverOptions({ headless, driverName, target, directory, entry, jobId }));
|
|
483
493
|
// The ledger is created here rather than inside the loop so the heartbeat
|
|
484
494
|
// can feed it the server's own figure for what this run has spent. Prices
|
|
485
495
|
// are server side only, so that answer is the only real one.
|
|
@@ -516,6 +526,7 @@ export async function playPersonaRun({
|
|
|
516
526
|
try {
|
|
517
527
|
if (inFlight) inFlight.set(runId, { state: 'launching' });
|
|
518
528
|
outcome = await runPersona({
|
|
529
|
+
controller: options.controller || 'standard',
|
|
519
530
|
client,
|
|
520
531
|
api,
|
|
521
532
|
gameId,
|
|
@@ -527,7 +538,9 @@ export async function playPersonaRun({
|
|
|
527
538
|
target,
|
|
528
539
|
runDirectory: directory,
|
|
529
540
|
skill,
|
|
530
|
-
|
|
541
|
+
// The brief this command fetched and checked is complete, not the
|
|
542
|
+
// pack's copy, which a pack bundle does not always carry.
|
|
543
|
+
brief: brief || pack.brief || null,
|
|
531
544
|
intentNotes: pack.intent_notes || [],
|
|
532
545
|
schemas: pack.schemas || [],
|
|
533
546
|
maxActions,
|
|
@@ -579,10 +592,10 @@ export async function playPersonaRun({
|
|
|
579
592
|
* Upload the run directory and complete the run, through cli-core's
|
|
580
593
|
* `finalizeRun` so the 422 `upload_unverified` repair pass lives in one place.
|
|
581
594
|
*
|
|
582
|
-
* A
|
|
583
|
-
*
|
|
584
|
-
*
|
|
585
|
-
* the
|
|
595
|
+
* A cancelled run sends no `report`. A budget-capped run sends the partial
|
|
596
|
+
* report built from its notes (`partial: true`, placeholder sentiment the
|
|
597
|
+
* report itself marks as no verdict), because the notes are what the run
|
|
598
|
+
* bought before the money ran out.
|
|
586
599
|
*
|
|
587
600
|
* A refused delivery does not take the job down. `finish` still settles it, the
|
|
588
601
|
* artifacts are on disk, and `resume` can retry.
|
|
@@ -624,15 +637,27 @@ export async function deliver({ deps, api, gameId, jobId, runId, directory, outc
|
|
|
624
637
|
}
|
|
625
638
|
|
|
626
639
|
try {
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
640
|
+
let answer;
|
|
641
|
+
try {
|
|
642
|
+
answer = await deps.finalizeRun(directory, {
|
|
643
|
+
api,
|
|
644
|
+
gameId,
|
|
645
|
+
jobId,
|
|
646
|
+
runId,
|
|
647
|
+
includeVideo: Boolean(options.uploadVideo),
|
|
648
|
+
includeTranscript: Boolean(options.uploadTranscript),
|
|
649
|
+
body
|
|
650
|
+
});
|
|
651
|
+
} catch (error) {
|
|
652
|
+
// A server older than partial reports refuses one as `invalid_report`.
|
|
653
|
+
// The uploads are already done by then, so the run is completed
|
|
654
|
+
// without it rather than left open: the notes stay on disk in
|
|
655
|
+
// report.json either way.
|
|
656
|
+
if (!(body.report && body.report.partial && error && error.status === 422 && error.code === 'invalid_report')) throw error;
|
|
657
|
+
log(`run ${runId}: the server refused the partial report (${String(error.message)}), so the run is completed without it. The notes are in ${directory}.`);
|
|
658
|
+
const { report: ignored, report_md: alsoIgnored, ...withoutReport } = body;
|
|
659
|
+
answer = await api.runs.complete(gameId, jobId, runId, withoutReport);
|
|
660
|
+
}
|
|
636
661
|
if (journal) await stateFile.patchRun(repoRoot, journal, runId, { uploaded: true, completed: true });
|
|
637
662
|
if (answer && answer.dropped_findings > 0) {
|
|
638
663
|
log(`run ${runId}: the server dropped ${answer.dropped_findings} finding(s) with no usable evidence.`);
|
|
@@ -911,12 +936,41 @@ export async function resolveSetup(deps, options, log) {
|
|
|
911
936
|
};
|
|
912
937
|
}
|
|
913
938
|
|
|
914
|
-
|
|
939
|
+
/**
|
|
940
|
+
* What a driver is created with. The playtest identity is the run's own: the
|
|
941
|
+
* Godot driver passes it into the game's environment and the web driver into
|
|
942
|
+
* the page, and the game's SDK sends the token on `POST /session`, which is
|
|
943
|
+
* what makes the server force the device id and mark the session synthetic.
|
|
944
|
+
* Without it the game's own events from a playtest count as a real player.
|
|
945
|
+
*
|
|
946
|
+
* @returns {Object}
|
|
947
|
+
*/
|
|
948
|
+
export function driverOptions({ headless, driverName, target, directory, entry, jobId }) {
|
|
949
|
+
return {
|
|
950
|
+
headless,
|
|
951
|
+
playtest: {
|
|
952
|
+
token: entry.playtest_token || null,
|
|
953
|
+
runId: entry.run_id,
|
|
954
|
+
jobId,
|
|
955
|
+
persona: entry.persona || null
|
|
956
|
+
},
|
|
957
|
+
projectDir: driverName === 'godot_driver' ? target : undefined,
|
|
958
|
+
runDir: directory
|
|
959
|
+
};
|
|
960
|
+
}
|
|
961
|
+
|
|
962
|
+
/**
|
|
963
|
+
* The persona reports on disk, for the aggregate. A partial report (written
|
|
964
|
+
* from the notes when the money ran out) is loaded as a `budget_exceeded`
|
|
965
|
+
* run: its findings count, and the aggregate says the job is partial.
|
|
966
|
+
*/
|
|
967
|
+
export async function loadRunReports(repoRoot, jobId, personaRuns) {
|
|
915
968
|
const out = [];
|
|
916
969
|
for (const entry of personaRuns) {
|
|
917
970
|
const file = path.join(runDirFor(repoRoot, jobId, entry.run_id), RUN_FILES.reportJson);
|
|
918
971
|
try {
|
|
919
|
-
|
|
972
|
+
const report = JSON.parse(await readFile(file, 'utf8'));
|
|
973
|
+
out.push({ runId: entry.run_id, persona: entry.persona, state: report && report.partial === true ? 'budget_exceeded' : 'succeeded', report });
|
|
920
974
|
} catch {
|
|
921
975
|
out.push({ runId: entry.run_id, persona: entry.persona, state: 'failed', report: null });
|
|
922
976
|
}
|
|
@@ -985,6 +1039,7 @@ export function resolveMaxActions(options, persona, pack) {
|
|
|
985
1039
|
}
|
|
986
1040
|
|
|
987
1041
|
export function resolveTarget(options, config, driverName) {
|
|
1042
|
+
if (WEBDRIVER_NAMES.includes(driverName) || driverName === 'cli_stdio') return options.driverConfig || (config?.build?.kind === 'driver_config' ? config.build.ref : null);
|
|
988
1043
|
const build = (config && config.build) || {};
|
|
989
1044
|
if (driverName === 'playwright_web') return options.buildUrl || (build.kind === 'url' ? build.ref : null) || null;
|
|
990
1045
|
if (driverName === 'godot_driver') return options.godotProject || (build.kind === 'godot_project' ? build.ref : null) || null;
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { ProxyError, STEPS } from './model.js';
|
|
2
|
+
export const CONTROLLERS = ['standard', 'jev'];
|
|
3
|
+
export const JEV_CONTROLLER_PROMPT = '\nRavensight Playtester action-selection mode: describe the currently visible screen briefly in text, including each proposed target and its screenshot coordinates, then propose one to three ALTERNATIVE next actions as tool calls. Only one will be executed after a separate selector chooses. These are alternatives, never a sequential plan. Use only observed controls. For a canvas game without accessible DOM controls, include a target label describing the visible control AND screenshot x/y coordinates; the driver uses those coordinates. Never invent selectors such as textbox or button names. When the brief identifies a field as a public player label, use Ravensight Playtester. Credentials or private access codes require human review. When the game is loading or transitioning, propose a bounded wait tool call. Do not place secrets in proposals. After their tool results, observe the resulting state before proposing again.';
|
|
4
|
+
export function describeAction(tool) {
|
|
5
|
+
const input = tool.input || {};
|
|
6
|
+
const descriptions = {
|
|
7
|
+
click: input.target ? `Click the visible ${input.target}${Number.isFinite(input.x) && Number.isFinite(input.y) ? ` at (${input.x}, ${input.y})` : ''}.` : `Click the observed screenshot position (${input.x}, ${input.y}).`,
|
|
8
|
+
type: `Type the prepared text into the focused field${input.submit ? ' and press Enter' : ''}.`,
|
|
9
|
+
key: `Press the ${input.key} key.`,
|
|
10
|
+
wait: `Wait ${input.ms} milliseconds for the screen to finish transitioning.`,
|
|
11
|
+
snapshot: 'Observe the current screen again.',
|
|
12
|
+
screenshot: 'Save a screenshot of the current screen for the report.',
|
|
13
|
+
navigate: `Navigate to the supplied build URL ${input.url}.`
|
|
14
|
+
};
|
|
15
|
+
return String(descriptions[tool.name] || `${tool.name}: ${JSON.stringify(input)}`).slice(0, 300);
|
|
16
|
+
}
|
|
17
|
+
export async function selectAction({ api, jobId, runId, persona, observation, toolUses, recentActions = [], intentNotes = [] }) {
|
|
18
|
+
const proposals = toolUses.slice(0, 8);
|
|
19
|
+
// Observation does not input anything into the game. Keep it in the existing
|
|
20
|
+
// bounded driver loop; Jev selects game inputs, not whether a frame may load.
|
|
21
|
+
if (proposals.length && proposals.every(tool => ['wait', 'snapshot', 'screenshot'].includes(tool.name))) {
|
|
22
|
+
return { status: 'selected', toolId: proposals[0].id, source: 'observation' };
|
|
23
|
+
}
|
|
24
|
+
const body = {
|
|
25
|
+
objective: ('Play the game according to the supplied persona, exploring visible controls and gathering evidence. Developer scope: ' + intentNotes.map(note => note.text || '').join(' ')).slice(0, 1000),
|
|
26
|
+
// The observer already applies the full persona. The selector needs its identity,
|
|
27
|
+
// current visible state and proposed inputs, not a second personality prompt.
|
|
28
|
+
persona: String(persona.name || persona.slug || 'Playtester').slice(0, 200),
|
|
29
|
+
observation: String(observation).slice(0, 6000),
|
|
30
|
+
actions: proposals.map((tool, index) => ({ id: `action_${index}`, description: describeAction(tool) })),
|
|
31
|
+
recentActions: recentActions.slice(-10).map(value => String(value).slice(0, 100))
|
|
32
|
+
};
|
|
33
|
+
let result;
|
|
34
|
+
try {
|
|
35
|
+
result = (await api.request('POST', '/api/v1/playtest/model/v1/decisions', {
|
|
36
|
+
body, idempotent: false,
|
|
37
|
+
headers: { 'X-Playtest-Job': jobId, 'X-Playtest-Run': runId, 'X-Playtest-Step': STEPS.personaAct }
|
|
38
|
+
})).body;
|
|
39
|
+
} catch (error) {
|
|
40
|
+
if (error.status === 402 || error.status === 409) throw new ProxyError({ status: error.status, code: error.code || 'decision_refused', message: 'The action selector stopped because this run is no longer spendable.' });
|
|
41
|
+
return { status: 'review', reason: 'decision_unavailable' };
|
|
42
|
+
}
|
|
43
|
+
if (result.status !== 'suggested') return result;
|
|
44
|
+
const index = body.actions.findIndex(action => action.id === result.actionId);
|
|
45
|
+
if (index < 0 || !Number.isFinite(result.confidence) || result.confidence < 0.75) return { status: 'review', reason: result.actionId === 'stop' ? 'selector_stopped' : 'uncertain_action' };
|
|
46
|
+
return { status: 'selected', toolId: proposals[index].id, confidence: result.confidence, budget: result.budget };
|
|
47
|
+
}
|
package/src/run/aggregate.js
CHANGED
|
@@ -344,7 +344,9 @@ export async function buildAggregate({
|
|
|
344
344
|
const issues = clusters.flatMap(cluster => toIssues(cluster, verdicts.get(cluster.signature)));
|
|
345
345
|
const ranked = rankIssues(issues, personasRun);
|
|
346
346
|
|
|
347
|
-
|
|
347
|
+
// A run whose report is partial (the CLI wrote it from the notes when the
|
|
348
|
+
// money ran out) counts toward the issues, and makes the job partial.
|
|
349
|
+
const partial = runs.some(run => !run.report || run.report.partial === true || ['failed', 'canceled', 'budget_exceeded'].includes(run.state));
|
|
348
350
|
const judgement = aggregateRunId
|
|
349
351
|
? await judgeAggregate({ client, jobId, runId: aggregateRunId, ranked, inputs, brief, intentNotes, ledger, log })
|
|
350
352
|
: null;
|
|
@@ -374,8 +376,11 @@ export async function buildAggregate({
|
|
|
374
376
|
state: run.state,
|
|
375
377
|
actions_taken: run.report.actions_taken,
|
|
376
378
|
quit_reason: run.report.quit_reason,
|
|
377
|
-
|
|
378
|
-
|
|
379
|
+
// A partial report's sentiment is the schema's placeholder, not a
|
|
380
|
+
// persona verdict, so it is not shown as one.
|
|
381
|
+
sentiment: run.report.partial ? null : run.report.sentiment,
|
|
382
|
+
would_continue: run.report.partial ? null : run.report.would_continue,
|
|
383
|
+
...(run.report.partial ? { partial: true } : {})
|
|
379
384
|
})),
|
|
380
385
|
issues: applied.issues,
|
|
381
386
|
praise: ranked.praise,
|
|
@@ -450,7 +455,7 @@ export async function judgeAggregate({ client, jobId, runId, ranked, inputs, bri
|
|
|
450
455
|
'',
|
|
451
456
|
`issues:\n${ranked.issues.map(issue => ` ${issue.issue_id} [${issue.severity}/${issue.category}] ${issue.title}: ${String(issue.description || '').slice(0, 300)} (personas: ${issue.personas_affected.join(', ') || 'none'})`).join('\n') || ' (none)'}`,
|
|
452
457
|
`praise:\n${ranked.praise.map(issue => ` ${issue.issue_id} ${issue.title}`).join('\n') || ' (none)'}`,
|
|
453
|
-
`runs:\n${inputs.map(run => ` ${run.persona}: ${run.report.actions_taken} actions, ${run.report.quit_reason}, would_continue=${run.report.would_continue}`).join('\n') || ' (none)'}`
|
|
458
|
+
`runs:\n${inputs.map(run => ` ${run.persona}: ${run.report.actions_taken} actions, ${run.report.quit_reason}, ${run.report.partial ? 'partial (out of budget, no verdict)' : `would_continue=${run.report.would_continue}`}`).join('\n') || ' (none)'}`
|
|
454
459
|
].join('\n');
|
|
455
460
|
|
|
456
461
|
try {
|
|
@@ -582,7 +587,7 @@ export function renderAggregateMarkdown(json) {
|
|
|
582
587
|
lines.push('|---|---|---|---|---|---|---|');
|
|
583
588
|
for (const run of json.runs) {
|
|
584
589
|
const sentiment = run.sentiment || {};
|
|
585
|
-
lines.push(`| ${run.persona} | ${run.actions_taken} | ${run.quit_reason} | ${sentiment.fun ?? '-'} | ${sentiment.clarity ?? '-'} | ${sentiment.frustration ?? '-'} | ${run.would_continue ? 'yes' : 'no'} |`);
|
|
590
|
+
lines.push(`| ${run.persona} | ${run.actions_taken} | ${run.quit_reason} | ${sentiment.fun ?? '-'} | ${sentiment.clarity ?? '-'} | ${sentiment.frustration ?? '-'} | ${run.would_continue === null || run.would_continue === undefined ? '-' : run.would_continue ? 'yes' : 'no'} |`);
|
|
586
591
|
}
|
|
587
592
|
lines.push('');
|
|
588
593
|
lines.push('## Top issues (ranked)');
|
package/src/run/args.js
CHANGED
|
@@ -14,13 +14,14 @@
|
|
|
14
14
|
const VALUED = new Map([
|
|
15
15
|
['--personas', 'personas'],
|
|
16
16
|
['--driver', 'driver'],
|
|
17
|
+
['--driver-config', 'driverConfig'],
|
|
18
|
+
['--controller', 'controller'],
|
|
17
19
|
['--build-url', 'buildUrl'],
|
|
18
20
|
['--build-dir', 'buildDir'],
|
|
19
21
|
['--godot-project', 'godotProject'],
|
|
20
22
|
['--cli', 'cli'],
|
|
21
23
|
['--max-actions', 'maxActions'],
|
|
22
24
|
['--concurrency', 'concurrency'],
|
|
23
|
-
['--budget-usd', 'budgetUsd'],
|
|
24
25
|
['--quality', 'quality']
|
|
25
26
|
]);
|
|
26
27
|
|
|
@@ -39,7 +40,7 @@ const BOOLEAN = new Map([
|
|
|
39
40
|
]);
|
|
40
41
|
|
|
41
42
|
/** Options whose value is a number. */
|
|
42
|
-
const NUMERIC = new Set(['maxActions', 'concurrency'
|
|
43
|
+
const NUMERIC = new Set(['maxActions', 'concurrency']);
|
|
43
44
|
|
|
44
45
|
/**
|
|
45
46
|
* @param {string[]} argv the leftovers commander handed over
|
|
@@ -75,6 +76,21 @@ export function parseRunArgv(argv = []) {
|
|
|
75
76
|
return { options, unknown };
|
|
76
77
|
}
|
|
77
78
|
|
|
79
|
+
/**
|
|
80
|
+
* Extra lines for unknown options that used to exist, so a removed flag is
|
|
81
|
+
* explained rather than just refused.
|
|
82
|
+
*
|
|
83
|
+
* @param {string[]} unknown
|
|
84
|
+
* @returns {string[]}
|
|
85
|
+
*/
|
|
86
|
+
export function unknownOptionNotes(unknown = []) {
|
|
87
|
+
const notes = [];
|
|
88
|
+
if (unknown.includes('--budget-usd')) {
|
|
89
|
+
notes.push('--budget-usd was removed: the model budget is set by the server for each run, from what the job was charged, and enforced at the proxy.');
|
|
90
|
+
}
|
|
91
|
+
return notes;
|
|
92
|
+
}
|
|
93
|
+
|
|
78
94
|
/**
|
|
79
95
|
* Merge the flags cli-core parsed with the ones this file parsed, flags
|
|
80
96
|
* winning, and normalise the two names for the repository root.
|
|
@@ -113,4 +129,4 @@ function stripUndefined(value) {
|
|
|
113
129
|
return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== undefined));
|
|
114
130
|
}
|
|
115
131
|
|
|
116
|
-
export default { parseRunArgv, optionsFrom };
|
|
132
|
+
export default { parseRunArgv, optionsFrom, unknownOptionNotes };
|
package/src/run/context.js
CHANGED
|
@@ -31,6 +31,13 @@ export const MAX_IMAGES_IN_CONTEXT = 1;
|
|
|
31
31
|
/** How many full observations stay in context (spec 03's K). */
|
|
32
32
|
export const KEEP_OBSERVATIONS = 6;
|
|
33
33
|
|
|
34
|
+
/**
|
|
35
|
+
* What an observation identical to the one before it is sent as. The driver
|
|
36
|
+
* marks it (`observation.unchanged`) and ./tools.js writes this in place of
|
|
37
|
+
* the screen, keeping the `step N` line so it is still an observation.
|
|
38
|
+
*/
|
|
39
|
+
export const UNCHANGED_OBSERVATION_TEXT = '(unchanged)';
|
|
40
|
+
|
|
34
41
|
const DROPPED_IMAGE_TEXT = '[screenshot dropped from context; only the newest screen is kept]';
|
|
35
42
|
const DROPPED_OBSERVATION_TEXT = '[older observation dropped from context; ask for a snapshot if you need to look again]';
|
|
36
43
|
|
|
@@ -138,7 +145,11 @@ export function pruneObservations(messages, keep = KEEP_OBSERVATIONS) {
|
|
|
138
145
|
message.content.forEach((block, j) => {
|
|
139
146
|
if (block && block.type === 'tool_result' && Array.isArray(block.content)) {
|
|
140
147
|
const first = block.content.find(child => child && child.type === 'text');
|
|
141
|
-
|
|
148
|
+
// An unchanged marker is not counted: it is a few bytes, and counting
|
|
149
|
+
// it would let a run of identical screens push the last REAL screen
|
|
150
|
+
// out of context, leaving the persona nothing to act on.
|
|
151
|
+
const text = String(first && first.text);
|
|
152
|
+
if (first && /^(?:[^\n]*\n)?step \d+/.test(text) && !text.endsWith(`\n${UNCHANGED_OBSERVATION_TEXT}`)) indexes.push([i, j]);
|
|
142
153
|
}
|
|
143
154
|
});
|
|
144
155
|
});
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { createHash } from 'node:crypto';
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
|
-
* The driver interface. One shape
|
|
4
|
+
* The driver interface. One shared shape across implementations (spec 03's
|
|
5
5
|
* "Driver interface (all drivers)"): `playwright_web` here in ./web.js,
|
|
6
|
-
* `godot_driver` in ./godot.js, and
|
|
6
|
+
* `godot_driver` in ./godot.js, plus native and custom bridge adapters.
|
|
7
7
|
*
|
|
8
8
|
* A driver is the ONLY thing in this CLI that touches the game. The model
|
|
9
9
|
* never gets a shell, a file write outside the run directory, or a network
|
|
@@ -138,7 +138,10 @@ export function assertDriver(driver, name) {
|
|
|
138
138
|
export function observationHash(observation) {
|
|
139
139
|
const text = typeof observation === 'string' ? observation : String((observation && observation.text) || '');
|
|
140
140
|
const normalized = text.replace(/\s+/g, ' ').trim();
|
|
141
|
-
|
|
141
|
+
const hash = createHash('sha256').update(normalized, 'utf8');
|
|
142
|
+
// Canvas games keep the same DOM text while their visible state changes.
|
|
143
|
+
if (observation && observation.screenshotBase64) hash.update(observation.screenshotBase64, 'base64');
|
|
144
|
+
return hash.digest('hex').slice(0, 32);
|
|
142
145
|
}
|
|
143
146
|
|
|
144
147
|
/**
|
package/src/run/drivers/index.js
CHANGED
|
@@ -1,16 +1,18 @@
|
|
|
1
1
|
import { assertDriver, DriverError } from './driver.js';
|
|
2
2
|
import { createWebDriver } from './web.js';
|
|
3
|
+
import { createWebDriverAdapter, WEBDRIVER_NAMES } from './webdriver.js';
|
|
4
|
+
import { createStdioDriver } from './stdio.js';
|
|
3
5
|
|
|
4
6
|
/**
|
|
5
7
|
* The driver registry. `playwright_web` is here; `godot_driver` lives in
|
|
6
8
|
* ./godot.js and is imported lazily so this package still loads on a
|
|
7
|
-
* machine with no Godot and no addon, and
|
|
9
|
+
* machine with no Godot and no addon, and native/custom adapters use local developer-owned connections.
|
|
8
10
|
*
|
|
9
11
|
* The names are the server's `PlaytestRun.driver` enum and the report
|
|
10
12
|
* schema's `driver` enum, not ours to invent.
|
|
11
13
|
*/
|
|
12
14
|
|
|
13
|
-
export const DRIVERS = Object.freeze(['playwright_web', 'godot_driver', 'cli_stdio']);
|
|
15
|
+
export const DRIVERS = Object.freeze(['playwright_web', 'godot_driver', 'cli_stdio', ...WEBDRIVER_NAMES]);
|
|
14
16
|
|
|
15
17
|
/**
|
|
16
18
|
* @param {string} name
|
|
@@ -19,6 +21,7 @@ export const DRIVERS = Object.freeze(['playwright_web', 'godot_driver', 'cli_std
|
|
|
19
21
|
* @returns {Promise<Object>} a driver satisfying ./driver.js
|
|
20
22
|
*/
|
|
21
23
|
export async function createDriver(name, options = {}) {
|
|
24
|
+
if (WEBDRIVER_NAMES.includes(name)) return assertDriver(createWebDriverAdapter({...options,name}),name);
|
|
22
25
|
const load = options.importer || (specifier => import(specifier));
|
|
23
26
|
switch (name) {
|
|
24
27
|
case 'playwright_web':
|
|
@@ -40,10 +43,7 @@ export async function createDriver(name, options = {}) {
|
|
|
40
43
|
return assertDriver(await factory(options), name);
|
|
41
44
|
}
|
|
42
45
|
case 'cli_stdio':
|
|
43
|
-
|
|
44
|
-
'The cli_stdio driver is not in this release. In v1 a text build is played through its web front end with --driver playwright_web.',
|
|
45
|
-
{ fatal: true }
|
|
46
|
-
);
|
|
46
|
+
return assertDriver(createStdioDriver(options), name);
|
|
47
47
|
default:
|
|
48
48
|
throw new DriverError(`unknown driver "${name}"; one of: ${DRIVERS.join(', ')}`, { fatal: true });
|
|
49
49
|
}
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
import fs from 'node:fs/promises';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { spawn } from 'node:child_process';
|
|
4
|
+
import { DriverError, observationHash, screenshotName, truncateObservation } from './driver.js';
|
|
5
|
+
|
|
6
|
+
// A developer-owned bridge implements newline-delimited JSON-RPC. The model
|
|
7
|
+
// receives game controls only, never command arguments or process access.
|
|
8
|
+
export async function readBridgeConfig(file) {
|
|
9
|
+
if ((await fs.stat(file)).size > 64000) throw new DriverError('Bridge configuration exceeds 64 KB.');
|
|
10
|
+
const config = JSON.parse(await fs.readFile(file, 'utf8'));
|
|
11
|
+
if (typeof config.command !== 'string' || !path.isAbsolute(config.command) ||
|
|
12
|
+
(config.args !== undefined && (!Array.isArray(config.args) || config.args.some(v => typeof v !== 'string'))) ||
|
|
13
|
+
(config.cwd !== undefined && !path.isAbsolute(config.cwd))) {
|
|
14
|
+
throw new DriverError('Bridge config needs an absolute command, string arguments and an optional absolute cwd.');
|
|
15
|
+
}
|
|
16
|
+
await fs.access(config.command, fs.constants.X_OK);
|
|
17
|
+
return config;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export function createStdioDriver({ spawn: spawnImpl = spawn, timeoutMs = 30000 } = {}) {
|
|
21
|
+
let child, buffer = '', nextId = 0, runDirectory, step = 0, shots = 0, lastHash;
|
|
22
|
+
const pending = new Map();
|
|
23
|
+
const fail = message => {
|
|
24
|
+
for (const entry of pending.values()) { clearTimeout(entry.timer); entry.reject(new DriverError(message, { fatal: true })); }
|
|
25
|
+
pending.clear();
|
|
26
|
+
};
|
|
27
|
+
function request(method, params = {}) {
|
|
28
|
+
if (!child || child.exitCode !== null || child.killed) return Promise.reject(new DriverError('Game bridge is not running.', { fatal: true }));
|
|
29
|
+
return new Promise((resolve, reject) => {
|
|
30
|
+
const id = ++nextId;
|
|
31
|
+
const timer = setTimeout(() => { pending.delete(id); reject(new DriverError('Game bridge timed out.', { fatal: true })); child.kill(); }, timeoutMs);
|
|
32
|
+
pending.set(id, { resolve, reject, timer });
|
|
33
|
+
child.stdin.write(JSON.stringify({ jsonrpc: '2.0', id, method, params }) + '\n', error => {
|
|
34
|
+
if (error) { clearTimeout(timer); pending.delete(id); reject(new DriverError('Game bridge input failed.')); }
|
|
35
|
+
});
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
async function observe() {
|
|
39
|
+
const value = await request('observe');
|
|
40
|
+
if (!value || typeof value.text !== 'string') throw new DriverError('Bridge observation needs text.');
|
|
41
|
+
const observation = { step: ++step, text: truncateObservation(value.text), elapsedMs: 0 };
|
|
42
|
+
if (typeof value.screenshotBase64 === 'string' && value.screenshotBase64.startsWith('iVBORw0KGgo')) {
|
|
43
|
+
observation.screenshotBase64 = value.screenshotBase64;
|
|
44
|
+
observation.screenshotMediaType = 'image/png';
|
|
45
|
+
}
|
|
46
|
+
lastHash = observationHash(observation);
|
|
47
|
+
return observation;
|
|
48
|
+
}
|
|
49
|
+
return {
|
|
50
|
+
name: 'cli_stdio', supportedActions: ['type', 'key', 'click', 'wait'],
|
|
51
|
+
async launch({ target, runDir }) {
|
|
52
|
+
const config = await readBridgeConfig(target);
|
|
53
|
+
runDirectory = runDir;
|
|
54
|
+
child = spawnImpl(config.command, config.args || [], { cwd: config.cwd || path.dirname(path.resolve(target)), shell: false, stdio: ['pipe', 'pipe', 'pipe'] });
|
|
55
|
+
child.stdout.setEncoding('utf8');
|
|
56
|
+
child.stdout.on('data', chunk => {
|
|
57
|
+
buffer += chunk;
|
|
58
|
+
if (Buffer.byteLength(buffer) > 8 * 1024 * 1024) { fail('Game bridge response exceeded 8 MB.'); child.kill(); return; }
|
|
59
|
+
let end;
|
|
60
|
+
while ((end = buffer.indexOf('\n')) !== -1) {
|
|
61
|
+
const line = buffer.slice(0, end); buffer = buffer.slice(end + 1);
|
|
62
|
+
let value;
|
|
63
|
+
try { value = JSON.parse(line); } catch { fail('Game bridge returned invalid JSON.'); child.kill(); return; }
|
|
64
|
+
const entry = pending.get(value?.id);
|
|
65
|
+
if (!entry) continue;
|
|
66
|
+
pending.delete(value.id); clearTimeout(entry.timer);
|
|
67
|
+
if (value.jsonrpc !== '2.0' || value.error) entry.reject(new DriverError('Game bridge command failed.'));
|
|
68
|
+
else entry.resolve(value.result);
|
|
69
|
+
}
|
|
70
|
+
});
|
|
71
|
+
// Drain diagnostics without copying arbitrary game output into reports.
|
|
72
|
+
child.stderr.resume();
|
|
73
|
+
child.stdin.on('error', () => fail('Game bridge input failed.'));
|
|
74
|
+
child.on('error', () => fail('Game bridge could not start.'));
|
|
75
|
+
child.on('exit', () => fail('Game bridge exited.'));
|
|
76
|
+
await fs.mkdir(path.join(runDir, 'screenshots'), { recursive: true });
|
|
77
|
+
await request('launch', { protocolVersion: 1 });
|
|
78
|
+
},
|
|
79
|
+
observe,
|
|
80
|
+
async act(action) {
|
|
81
|
+
if (!['type', 'key', 'click', 'wait'].includes(action.kind)) return { ok: false, changed: false, error: 'Unsupported bridge action.' };
|
|
82
|
+
const before = lastHash;
|
|
83
|
+
const safe = action.kind === 'type' ? { kind: 'type', text: String(action.text || '').slice(0, 2000), submit: Boolean(action.submit) }
|
|
84
|
+
: action.kind === 'key' ? { kind: 'key', key: String(action.key || '').slice(0, 64) }
|
|
85
|
+
: action.kind === 'wait' ? { kind: 'wait', ms: Math.min(5000, Math.max(0, Number(action.ms) || 0)) }
|
|
86
|
+
: { kind: 'click', target: typeof action.target === 'string' ? action.target.slice(0, 500) : action.target };
|
|
87
|
+
const answer = await request('act', safe);
|
|
88
|
+
if (answer?.ok !== true) return { ok: false, changed: false, error: 'Game bridge refused this action.' };
|
|
89
|
+
return { ok: true, changed: observationHash(await observe()) !== before };
|
|
90
|
+
},
|
|
91
|
+
async screenshot(reason) {
|
|
92
|
+
const image = await request('screenshot');
|
|
93
|
+
if (typeof image?.pngBase64 !== 'string' || !image.pngBase64.startsWith('iVBORw0KGgo')) throw new DriverError('This bridge did not return a PNG screenshot.');
|
|
94
|
+
const relativePath = path.join('screenshots', screenshotName(++shots, reason));
|
|
95
|
+
const file = path.join(runDirectory, relativePath);
|
|
96
|
+
await fs.writeFile(file, Buffer.from(image.pngBase64, 'base64'));
|
|
97
|
+
return { path: file, relativePath };
|
|
98
|
+
},
|
|
99
|
+
async consoleErrors() { return []; },
|
|
100
|
+
async stop() {
|
|
101
|
+
if (child && child.exitCode === null && !child.killed) {
|
|
102
|
+
child.kill('SIGTERM');
|
|
103
|
+
const process = child;
|
|
104
|
+
const timer = setTimeout(() => { if (process.exitCode === null) process.kill('SIGKILL'); }, 1000);
|
|
105
|
+
timer.unref();
|
|
106
|
+
}
|
|
107
|
+
fail('Game bridge stopped.');
|
|
108
|
+
return { videoPath: null };
|
|
109
|
+
}
|
|
110
|
+
};
|
|
111
|
+
}
|