ravensight-playtest 0.2.3 → 0.2.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,7 +2,7 @@ import { randomUUID } from 'node:crypto';
2
2
  import { mkdir, readFile, stat } from 'node:fs/promises';
3
3
  import path from 'node:path';
4
4
  import { buildAggregate, writeAggregate } from '../run/aggregate.js';
5
- import { optionsFrom } from '../run/args.js';
5
+ import { optionsFrom, unknownOptionNotes } from '../run/args.js';
6
6
  import { getDeps } from '../run/deps.js';
7
7
  import { createDriver } from '../run/drivers/index.js';
8
8
  import EXIT from '../run/exit.js';
@@ -10,6 +10,8 @@ import { createHeartbeat } from '../run/heartbeat.js';
10
10
  import { createUsageLedger } from '../run/usage.js';
11
11
  import { createModelClient, ModelUnavailableError, ProxyError } from '../run/model.js';
12
12
  import { jobDir, runDir as runDirFor, JOB_FILES, RUN_FILES } from '../run/paths.js';
13
+ import { readBridgeConfig } from "../run/drivers/stdio.js";
14
+ import { WEBDRIVER_NAMES, readDriverConfig } from '../run/drivers/webdriver.js';
13
15
  import { runPersona } from '../run/personaLoop.js';
14
16
  import * as stateFile from '../run/state.js';
15
17
  import { emitRunEvents } from '../run/synthetic.js';
@@ -78,6 +80,7 @@ async function run(flags, injected, deps, log) {
78
80
  const { options, unknown } = optionsFrom(flags);
79
81
  if (unknown.length > 0) {
80
82
  log(`I do not know these options: ${unknown.join(', ')}. Run "ravensight-playtest run --help".`);
83
+ for (const line of unknownOptionNotes(unknown)) log(line);
81
84
  return EXIT.ENVIRONMENT;
82
85
  }
83
86
  for (const line of unusedFlagWarnings(options)) log(line);
@@ -86,10 +89,11 @@ async function run(flags, injected, deps, log) {
86
89
  if (setup.code !== undefined) return setup.code;
87
90
  const { context, gameId, repoRoot, apiUrl, cliVersion, api } = setup;
88
91
 
92
+ if (options.controller && !['standard', 'jev'].includes(options.controller)) { log('Choose --controller standard or --controller jev.'); return EXIT.ENVIRONMENT; }
89
93
  const driverName = options.driver || (context.config && context.config.driver) || 'playwright_web';
90
94
  const target = resolveTarget(options, context.config, driverName);
91
95
  if (!target) {
92
- log('Point the run at a build: --build-url for a web build, or --godot-project for a Godot project.');
96
+ log('Point the run at a build: --build-url for a web build, --godot-project for Godot, or --driver-config for a native or text bridge.');
93
97
  return EXIT.ENVIRONMENT;
94
98
  }
95
99
 
@@ -97,8 +101,11 @@ async function run(flags, injected, deps, log) {
97
101
  // `aggregate.*` steps are spendable from, and it is free (the price list has
98
102
  // no unit for it), so asking costs nothing and not asking costs the judged
99
103
  // dedup and success-criteria passes.
104
+ if (WEBDRIVER_NAMES.includes(driverName)) await readDriverConfig(target);
105
+ if (driverName === "cli_stdio") await readBridgeConfig(target);
100
106
  const modules = ['persona_playtest', 'aggregate_report'];
101
107
  const pack = await deps.packs.get({ api, gameId, modules });
108
+ if (options.controller === 'jev' && !pack.routing?.action_controllers?.includes('jev')) { log('This server does not currently support the requested action controller. No job was registered.'); return EXIT.ENVIRONMENT; }
102
109
  const personaSlugs = resolvePersonas(options, context.config, pack);
103
110
  const personas = personaSlugs.map(slug => {
104
111
  const found = (pack.personas || []).find(persona => persona.slug === slug);
@@ -164,7 +171,7 @@ async function run(flags, injected, deps, log) {
164
171
  cli_version: cliVersion,
165
172
  confirm_price_cents: estimate.estimate_cents,
166
173
  brief_version: brief.version ?? null,
167
- build: { kind: driverName === 'playwright_web' ? 'url' : 'godot_project', ref: target },
174
+ build: { kind: driverName === 'playwright_web' ? 'url' : (WEBDRIVER_NAMES.includes(driverName) || driverName === 'cli_stdio') ? 'driver_config' : 'godot_project', ref: target },
168
175
  repo: { commit: (context.config && context.config.commit) || '', dirty: Boolean(context.config && context.config.dirty) },
169
176
  max_actions: Number.isInteger(options.maxActions) ? options.maxActions : null,
170
177
  pack_versions: { pack: pack.pack_version }
@@ -191,6 +198,7 @@ async function run(flags, injected, deps, log) {
191
198
  priceCents: registered.price && registered.price.price_cents,
192
199
  options: {
193
200
  maxActions: options.maxActions ?? null,
201
+ controller: options.controller || 'standard',
194
202
  concurrency: options.concurrency ?? DEFAULT_CONCURRENCY,
195
203
  uploadVideo: Boolean(options.uploadVideo),
196
204
  uploadTranscript: Boolean(options.uploadTranscript)
@@ -287,17 +295,17 @@ export function personaCountRefusal(count, maxPersonas) {
287
295
  /**
288
296
  * Flags this release parses and then does not act on.
289
297
  *
290
- * Saying so is the point: a developer who passes `--budget-usd 1` and is not
291
- * told it did nothing will believe they capped the spend.
298
+ * Saying so is the point: a developer who passes a flag and is not told it
299
+ * did nothing will believe it did something. `--budget-usd` is not here
300
+ * because it is not parsed at all any more: the model budget is the server's,
301
+ * derived from what the job was charged and enforced per call at the proxy,
302
+ * so a spending flag the CLI cannot honour is refused rather than accepted.
292
303
  *
293
304
  * @param {Object} options
294
305
  * @returns {Array<string>}
295
306
  */
296
307
  export function unusedFlagWarnings(options) {
297
308
  const warnings = [];
298
- if (options.budgetUsd !== undefined) {
299
- warnings.push('--budget-usd is not applied by this release: the model budget is the server\'s, derived from what the job was charged, and it is enforced per call at the proxy.');
300
- }
301
309
  if (options.quality !== undefined) {
302
310
  warnings.push('--quality is not applied by this release: the model tier is the server\'s decision, and the CLI is not allowed to name one.');
303
311
  }
@@ -349,7 +357,7 @@ export async function playJob({
349
357
 
350
358
  const results = [];
351
359
  const queue = personaRuns.filter(entry => !stateFile.isRunDone(journal.runs[entry.run_id]));
352
- const concurrency = Math.max(1, Math.min(Number(options.concurrency) || DEFAULT_CONCURRENCY, queue.length || 1));
360
+ const concurrency = WEBDRIVER_NAMES.includes(driverName) ? 1 : Math.max(1, Math.min(Number(options.concurrency) || DEFAULT_CONCURRENCY, queue.length || 1));
353
361
 
354
362
  /**
355
363
  * Runs the process is in the middle of. A Ctrl-C has to move these to
@@ -385,6 +393,7 @@ export async function playJob({
385
393
  journal,
386
394
  inFlight,
387
395
  makeDriver,
396
+ brief,
388
397
  log
389
398
  }));
390
399
  }
@@ -467,6 +476,7 @@ export async function playPersonaRun({
467
476
  journal,
468
477
  inFlight = null,
469
478
  makeDriver,
479
+ brief = null,
470
480
  log
471
481
  }) {
472
482
  const runId = entry.run_id;
@@ -479,7 +489,7 @@ export async function playPersonaRun({
479
489
  // Headless is right for a browser and wrong for Godot: a headless Godot
480
490
  // routes no GUI input, so it is a smoke-test mode rather than a way to play.
481
491
  const headless = driverName === 'godot_driver' ? options.headless === true : options.headed !== true;
482
- const driver = await makeDriver(driverName, { headless });
492
+ const driver = await makeDriver(driverName, driverOptions({ headless, driverName, target, directory, entry, jobId }));
483
493
  // The ledger is created here rather than inside the loop so the heartbeat
484
494
  // can feed it the server's own figure for what this run has spent. Prices
485
495
  // are server side only, so that answer is the only real one.
@@ -516,6 +526,7 @@ export async function playPersonaRun({
516
526
  try {
517
527
  if (inFlight) inFlight.set(runId, { state: 'launching' });
518
528
  outcome = await runPersona({
529
+ controller: options.controller || 'standard',
519
530
  client,
520
531
  api,
521
532
  gameId,
@@ -527,7 +538,9 @@ export async function playPersonaRun({
527
538
  target,
528
539
  runDirectory: directory,
529
540
  skill,
530
- brief: pack.brief || null,
541
+ // The brief this command fetched and checked is complete, not the
542
+ // pack's copy, which a pack bundle does not always carry.
543
+ brief: brief || pack.brief || null,
531
544
  intentNotes: pack.intent_notes || [],
532
545
  schemas: pack.schemas || [],
533
546
  maxActions,
@@ -579,10 +592,10 @@ export async function playPersonaRun({
579
592
  * Upload the run directory and complete the run, through cli-core's
580
593
  * `finalizeRun` so the 422 `upload_unverified` repair pass lives in one place.
581
594
  *
582
- * A budget-capped or cancelled run sends no `report`: the route accepts that,
583
- * and it is the honest answer. Inventing a sentiment score nobody gave so a
584
- * placeholder report would validate would put a fabricated verdict in front of
585
- * the developer.
595
+ * A cancelled run sends no `report`. A budget-capped run sends the partial
596
+ * report built from its notes (`partial: true`, placeholder sentiment the
597
+ * report itself marks as no verdict), because the notes are what the run
598
+ * bought before the money ran out.
586
599
  *
587
600
  * A refused delivery does not take the job down. `finish` still settles it, the
588
601
  * artifacts are on disk, and `resume` can retry.
@@ -624,15 +637,27 @@ export async function deliver({ deps, api, gameId, jobId, runId, directory, outc
624
637
  }
625
638
 
626
639
  try {
627
- const answer = await deps.finalizeRun(directory, {
628
- api,
629
- gameId,
630
- jobId,
631
- runId,
632
- includeVideo: Boolean(options.uploadVideo),
633
- includeTranscript: Boolean(options.uploadTranscript),
634
- body
635
- });
640
+ let answer;
641
+ try {
642
+ answer = await deps.finalizeRun(directory, {
643
+ api,
644
+ gameId,
645
+ jobId,
646
+ runId,
647
+ includeVideo: Boolean(options.uploadVideo),
648
+ includeTranscript: Boolean(options.uploadTranscript),
649
+ body
650
+ });
651
+ } catch (error) {
652
+ // A server older than partial reports refuses one as `invalid_report`.
653
+ // The uploads are already done by then, so the run is completed
654
+ // without it rather than left open: the notes stay on disk in
655
+ // report.json either way.
656
+ if (!(body.report && body.report.partial && error && error.status === 422 && error.code === 'invalid_report')) throw error;
657
+ log(`run ${runId}: the server refused the partial report (${String(error.message)}), so the run is completed without it. The notes are in ${directory}.`);
658
+ const { report: ignored, report_md: alsoIgnored, ...withoutReport } = body;
659
+ answer = await api.runs.complete(gameId, jobId, runId, withoutReport);
660
+ }
636
661
  if (journal) await stateFile.patchRun(repoRoot, journal, runId, { uploaded: true, completed: true });
637
662
  if (answer && answer.dropped_findings > 0) {
638
663
  log(`run ${runId}: the server dropped ${answer.dropped_findings} finding(s) with no usable evidence.`);
@@ -911,12 +936,41 @@ export async function resolveSetup(deps, options, log) {
911
936
  };
912
937
  }
913
938
 
914
- async function loadRunReports(repoRoot, jobId, personaRuns) {
939
+ /**
940
+ * What a driver is created with. The playtest identity is the run's own: the
941
+ * Godot driver passes it into the game's environment and the web driver into
942
+ * the page, and the game's SDK sends the token on `POST /session`, which is
943
+ * what makes the server force the device id and mark the session synthetic.
944
+ * Without it the game's own events from a playtest count as a real player.
945
+ *
946
+ * @returns {Object}
947
+ */
948
+ export function driverOptions({ headless, driverName, target, directory, entry, jobId }) {
949
+ return {
950
+ headless,
951
+ playtest: {
952
+ token: entry.playtest_token || null,
953
+ runId: entry.run_id,
954
+ jobId,
955
+ persona: entry.persona || null
956
+ },
957
+ projectDir: driverName === 'godot_driver' ? target : undefined,
958
+ runDir: directory
959
+ };
960
+ }
961
+
962
+ /**
963
+ * The persona reports on disk, for the aggregate. A partial report (written
964
+ * from the notes when the money ran out) is loaded as a `budget_exceeded`
965
+ * run: its findings count, and the aggregate says the job is partial.
966
+ */
967
+ export async function loadRunReports(repoRoot, jobId, personaRuns) {
915
968
  const out = [];
916
969
  for (const entry of personaRuns) {
917
970
  const file = path.join(runDirFor(repoRoot, jobId, entry.run_id), RUN_FILES.reportJson);
918
971
  try {
919
- out.push({ runId: entry.run_id, persona: entry.persona, state: 'succeeded', report: JSON.parse(await readFile(file, 'utf8')) });
972
+ const report = JSON.parse(await readFile(file, 'utf8'));
973
+ out.push({ runId: entry.run_id, persona: entry.persona, state: report && report.partial === true ? 'budget_exceeded' : 'succeeded', report });
920
974
  } catch {
921
975
  out.push({ runId: entry.run_id, persona: entry.persona, state: 'failed', report: null });
922
976
  }
@@ -985,6 +1039,7 @@ export function resolveMaxActions(options, persona, pack) {
985
1039
  }
986
1040
 
987
1041
  export function resolveTarget(options, config, driverName) {
1042
+ if (WEBDRIVER_NAMES.includes(driverName) || driverName === 'cli_stdio') return options.driverConfig || (config?.build?.kind === 'driver_config' ? config.build.ref : null);
988
1043
  const build = (config && config.build) || {};
989
1044
  if (driverName === 'playwright_web') return options.buildUrl || (build.kind === 'url' ? build.ref : null) || null;
990
1045
  if (driverName === 'godot_driver') return options.godotProject || (build.kind === 'godot_project' ? build.ref : null) || null;
@@ -0,0 +1,47 @@
1
+ import { ProxyError, STEPS } from './model.js';
2
+ export const CONTROLLERS = ['standard', 'jev'];
3
+ export const JEV_CONTROLLER_PROMPT = '\nRavensight Playtester action-selection mode: describe the currently visible screen briefly in text, including each proposed target and its screenshot coordinates, then propose one to three ALTERNATIVE next actions as tool calls. Only one will be executed after a separate selector chooses. These are alternatives, never a sequential plan. Use only observed controls. For a canvas game without accessible DOM controls, include a target label describing the visible control AND screenshot x/y coordinates; the driver uses those coordinates. Never invent selectors such as textbox or button names. When the brief identifies a field as a public player label, use Ravensight Playtester. Credentials or private access codes require human review. When the game is loading or transitioning, propose a bounded wait tool call. Do not place secrets in proposals. After their tool results, observe the resulting state before proposing again.';
4
+ export function describeAction(tool) {
5
+ const input = tool.input || {};
6
+ const descriptions = {
7
+ click: input.target ? `Click the visible ${input.target}${Number.isFinite(input.x) && Number.isFinite(input.y) ? ` at (${input.x}, ${input.y})` : ''}.` : `Click the observed screenshot position (${input.x}, ${input.y}).`,
8
+ type: `Type the prepared text into the focused field${input.submit ? ' and press Enter' : ''}.`,
9
+ key: `Press the ${input.key} key.`,
10
+ wait: `Wait ${input.ms} milliseconds for the screen to finish transitioning.`,
11
+ snapshot: 'Observe the current screen again.',
12
+ screenshot: 'Save a screenshot of the current screen for the report.',
13
+ navigate: `Navigate to the supplied build URL ${input.url}.`
14
+ };
15
+ return String(descriptions[tool.name] || `${tool.name}: ${JSON.stringify(input)}`).slice(0, 300);
16
+ }
17
+ export async function selectAction({ api, jobId, runId, persona, observation, toolUses, recentActions = [], intentNotes = [] }) {
18
+ const proposals = toolUses.slice(0, 8);
19
+ // Observation does not input anything into the game. Keep it in the existing
20
+ // bounded driver loop; Jev selects game inputs, not whether a frame may load.
21
+ if (proposals.length && proposals.every(tool => ['wait', 'snapshot', 'screenshot'].includes(tool.name))) {
22
+ return { status: 'selected', toolId: proposals[0].id, source: 'observation' };
23
+ }
24
+ const body = {
25
+ objective: ('Play the game according to the supplied persona, exploring visible controls and gathering evidence. Developer scope: ' + intentNotes.map(note => note.text || '').join(' ')).slice(0, 1000),
26
+ // The observer already applies the full persona. The selector needs its identity,
27
+ // current visible state and proposed inputs, not a second personality prompt.
28
+ persona: String(persona.name || persona.slug || 'Playtester').slice(0, 200),
29
+ observation: String(observation).slice(0, 6000),
30
+ actions: proposals.map((tool, index) => ({ id: `action_${index}`, description: describeAction(tool) })),
31
+ recentActions: recentActions.slice(-10).map(value => String(value).slice(0, 100))
32
+ };
33
+ let result;
34
+ try {
35
+ result = (await api.request('POST', '/api/v1/playtest/model/v1/decisions', {
36
+ body, idempotent: false,
37
+ headers: { 'X-Playtest-Job': jobId, 'X-Playtest-Run': runId, 'X-Playtest-Step': STEPS.personaAct }
38
+ })).body;
39
+ } catch (error) {
40
+ if (error.status === 402 || error.status === 409) throw new ProxyError({ status: error.status, code: error.code || 'decision_refused', message: 'The action selector stopped because this run is no longer spendable.' });
41
+ return { status: 'review', reason: 'decision_unavailable' };
42
+ }
43
+ if (result.status !== 'suggested') return result;
44
+ const index = body.actions.findIndex(action => action.id === result.actionId);
45
+ if (index < 0 || !Number.isFinite(result.confidence) || result.confidence < 0.75) return { status: 'review', reason: result.actionId === 'stop' ? 'selector_stopped' : 'uncertain_action' };
46
+ return { status: 'selected', toolId: proposals[index].id, confidence: result.confidence, budget: result.budget };
47
+ }
@@ -344,7 +344,9 @@ export async function buildAggregate({
344
344
  const issues = clusters.flatMap(cluster => toIssues(cluster, verdicts.get(cluster.signature)));
345
345
  const ranked = rankIssues(issues, personasRun);
346
346
 
347
- const partial = runs.some(run => !run.report || ['failed', 'canceled', 'budget_exceeded'].includes(run.state));
347
+ // A run whose report is partial (the CLI wrote it from the notes when the
348
+ // money ran out) counts toward the issues, and makes the job partial.
349
+ const partial = runs.some(run => !run.report || run.report.partial === true || ['failed', 'canceled', 'budget_exceeded'].includes(run.state));
348
350
  const judgement = aggregateRunId
349
351
  ? await judgeAggregate({ client, jobId, runId: aggregateRunId, ranked, inputs, brief, intentNotes, ledger, log })
350
352
  : null;
@@ -374,8 +376,11 @@ export async function buildAggregate({
374
376
  state: run.state,
375
377
  actions_taken: run.report.actions_taken,
376
378
  quit_reason: run.report.quit_reason,
377
- sentiment: run.report.sentiment,
378
- would_continue: run.report.would_continue
379
+ // A partial report's sentiment is the schema's placeholder, not a
380
+ // persona verdict, so it is not shown as one.
381
+ sentiment: run.report.partial ? null : run.report.sentiment,
382
+ would_continue: run.report.partial ? null : run.report.would_continue,
383
+ ...(run.report.partial ? { partial: true } : {})
379
384
  })),
380
385
  issues: applied.issues,
381
386
  praise: ranked.praise,
@@ -450,7 +455,7 @@ export async function judgeAggregate({ client, jobId, runId, ranked, inputs, bri
450
455
  '',
451
456
  `issues:\n${ranked.issues.map(issue => ` ${issue.issue_id} [${issue.severity}/${issue.category}] ${issue.title}: ${String(issue.description || '').slice(0, 300)} (personas: ${issue.personas_affected.join(', ') || 'none'})`).join('\n') || ' (none)'}`,
452
457
  `praise:\n${ranked.praise.map(issue => ` ${issue.issue_id} ${issue.title}`).join('\n') || ' (none)'}`,
453
- `runs:\n${inputs.map(run => ` ${run.persona}: ${run.report.actions_taken} actions, ${run.report.quit_reason}, would_continue=${run.report.would_continue}`).join('\n') || ' (none)'}`
458
+ `runs:\n${inputs.map(run => ` ${run.persona}: ${run.report.actions_taken} actions, ${run.report.quit_reason}, ${run.report.partial ? 'partial (out of budget, no verdict)' : `would_continue=${run.report.would_continue}`}`).join('\n') || ' (none)'}`
454
459
  ].join('\n');
455
460
 
456
461
  try {
@@ -582,7 +587,7 @@ export function renderAggregateMarkdown(json) {
582
587
  lines.push('|---|---|---|---|---|---|---|');
583
588
  for (const run of json.runs) {
584
589
  const sentiment = run.sentiment || {};
585
- lines.push(`| ${run.persona} | ${run.actions_taken} | ${run.quit_reason} | ${sentiment.fun ?? '-'} | ${sentiment.clarity ?? '-'} | ${sentiment.frustration ?? '-'} | ${run.would_continue ? 'yes' : 'no'} |`);
590
+ lines.push(`| ${run.persona} | ${run.actions_taken} | ${run.quit_reason} | ${sentiment.fun ?? '-'} | ${sentiment.clarity ?? '-'} | ${sentiment.frustration ?? '-'} | ${run.would_continue === null || run.would_continue === undefined ? '-' : run.would_continue ? 'yes' : 'no'} |`);
586
591
  }
587
592
  lines.push('');
588
593
  lines.push('## Top issues (ranked)');
package/src/run/args.js CHANGED
@@ -14,13 +14,14 @@
14
14
  const VALUED = new Map([
15
15
  ['--personas', 'personas'],
16
16
  ['--driver', 'driver'],
17
+ ['--driver-config', 'driverConfig'],
18
+ ['--controller', 'controller'],
17
19
  ['--build-url', 'buildUrl'],
18
20
  ['--build-dir', 'buildDir'],
19
21
  ['--godot-project', 'godotProject'],
20
22
  ['--cli', 'cli'],
21
23
  ['--max-actions', 'maxActions'],
22
24
  ['--concurrency', 'concurrency'],
23
- ['--budget-usd', 'budgetUsd'],
24
25
  ['--quality', 'quality']
25
26
  ]);
26
27
 
@@ -39,7 +40,7 @@ const BOOLEAN = new Map([
39
40
  ]);
40
41
 
41
42
  /** Options whose value is a number. */
42
- const NUMERIC = new Set(['maxActions', 'concurrency', 'budgetUsd']);
43
+ const NUMERIC = new Set(['maxActions', 'concurrency']);
43
44
 
44
45
  /**
45
46
  * @param {string[]} argv the leftovers commander handed over
@@ -75,6 +76,21 @@ export function parseRunArgv(argv = []) {
75
76
  return { options, unknown };
76
77
  }
77
78
 
79
+ /**
80
+ * Extra lines for unknown options that used to exist, so a removed flag is
81
+ * explained rather than just refused.
82
+ *
83
+ * @param {string[]} unknown
84
+ * @returns {string[]}
85
+ */
86
+ export function unknownOptionNotes(unknown = []) {
87
+ const notes = [];
88
+ if (unknown.includes('--budget-usd')) {
89
+ notes.push('--budget-usd was removed: the model budget is set by the server for each run, from what the job was charged, and enforced at the proxy.');
90
+ }
91
+ return notes;
92
+ }
93
+
78
94
  /**
79
95
  * Merge the flags cli-core parsed with the ones this file parsed, flags
80
96
  * winning, and normalise the two names for the repository root.
@@ -113,4 +129,4 @@ function stripUndefined(value) {
113
129
  return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== undefined));
114
130
  }
115
131
 
116
- export default { parseRunArgv, optionsFrom };
132
+ export default { parseRunArgv, optionsFrom, unknownOptionNotes };
@@ -31,6 +31,13 @@ export const MAX_IMAGES_IN_CONTEXT = 1;
31
31
  /** How many full observations stay in context (spec 03's K). */
32
32
  export const KEEP_OBSERVATIONS = 6;
33
33
 
34
+ /**
35
+ * What an observation identical to the one before it is sent as. The driver
36
+ * marks it (`observation.unchanged`) and ./tools.js writes this in place of
37
+ * the screen, keeping the `step N` line so it is still an observation.
38
+ */
39
+ export const UNCHANGED_OBSERVATION_TEXT = '(unchanged)';
40
+
34
41
  const DROPPED_IMAGE_TEXT = '[screenshot dropped from context; only the newest screen is kept]';
35
42
  const DROPPED_OBSERVATION_TEXT = '[older observation dropped from context; ask for a snapshot if you need to look again]';
36
43
 
@@ -138,7 +145,11 @@ export function pruneObservations(messages, keep = KEEP_OBSERVATIONS) {
138
145
  message.content.forEach((block, j) => {
139
146
  if (block && block.type === 'tool_result' && Array.isArray(block.content)) {
140
147
  const first = block.content.find(child => child && child.type === 'text');
141
- if (first && /^(?:[^\n]*\n)?step \d+/.test(String(first.text))) indexes.push([i, j]);
148
+ // An unchanged marker is not counted: it is a few bytes, and counting
149
+ // it would let a run of identical screens push the last REAL screen
150
+ // out of context, leaving the persona nothing to act on.
151
+ const text = String(first && first.text);
152
+ if (first && /^(?:[^\n]*\n)?step \d+/.test(text) && !text.endsWith(`\n${UNCHANGED_OBSERVATION_TEXT}`)) indexes.push([i, j]);
142
153
  }
143
154
  });
144
155
  });
@@ -1,9 +1,9 @@
1
1
  import { createHash } from 'node:crypto';
2
2
 
3
3
  /**
4
- * The driver interface. One shape, three implementations (spec 03's
4
+ * The driver interface. One shared shape across implementations (spec 03's
5
5
  * "Driver interface (all drivers)"): `playwright_web` here in ./web.js,
6
- * `godot_driver` in ./godot.js, and `cli_stdio` later.
6
+ * `godot_driver` in ./godot.js, plus native and custom bridge adapters.
7
7
  *
8
8
  * A driver is the ONLY thing in this CLI that touches the game. The model
9
9
  * never gets a shell, a file write outside the run directory, or a network
@@ -138,7 +138,10 @@ export function assertDriver(driver, name) {
138
138
  export function observationHash(observation) {
139
139
  const text = typeof observation === 'string' ? observation : String((observation && observation.text) || '');
140
140
  const normalized = text.replace(/\s+/g, ' ').trim();
141
- return createHash('sha256').update(normalized, 'utf8').digest('hex').slice(0, 32);
141
+ const hash = createHash('sha256').update(normalized, 'utf8');
142
+ // Canvas games keep the same DOM text while their visible state changes.
143
+ if (observation && observation.screenshotBase64) hash.update(observation.screenshotBase64, 'base64');
144
+ return hash.digest('hex').slice(0, 32);
142
145
  }
143
146
 
144
147
  /**
@@ -1,16 +1,18 @@
1
1
  import { assertDriver, DriverError } from './driver.js';
2
2
  import { createWebDriver } from './web.js';
3
+ import { createWebDriverAdapter, WEBDRIVER_NAMES } from './webdriver.js';
4
+ import { createStdioDriver } from './stdio.js';
3
5
 
4
6
  /**
5
7
  * The driver registry. `playwright_web` is here; `godot_driver` lives in
6
8
  * ./godot.js and is imported lazily so this package still loads on a
7
- * machine with no Godot and no addon, and `cli_stdio` is not in v1.
9
+ * machine with no Godot and no addon, and native/custom adapters use local developer-owned connections.
8
10
  *
9
11
  * The names are the server's `PlaytestRun.driver` enum and the report
10
12
  * schema's `driver` enum, not ours to invent.
11
13
  */
12
14
 
13
- export const DRIVERS = Object.freeze(['playwright_web', 'godot_driver', 'cli_stdio']);
15
+ export const DRIVERS = Object.freeze(['playwright_web', 'godot_driver', 'cli_stdio', ...WEBDRIVER_NAMES]);
14
16
 
15
17
  /**
16
18
  * @param {string} name
@@ -19,6 +21,7 @@ export const DRIVERS = Object.freeze(['playwright_web', 'godot_driver', 'cli_std
19
21
  * @returns {Promise<Object>} a driver satisfying ./driver.js
20
22
  */
21
23
  export async function createDriver(name, options = {}) {
24
+ if (WEBDRIVER_NAMES.includes(name)) return assertDriver(createWebDriverAdapter({...options,name}),name);
22
25
  const load = options.importer || (specifier => import(specifier));
23
26
  switch (name) {
24
27
  case 'playwright_web':
@@ -40,10 +43,7 @@ export async function createDriver(name, options = {}) {
40
43
  return assertDriver(await factory(options), name);
41
44
  }
42
45
  case 'cli_stdio':
43
- throw new DriverError(
44
- 'The cli_stdio driver is not in this release. In v1 a text build is played through its web front end with --driver playwright_web.',
45
- { fatal: true }
46
- );
46
+ return assertDriver(createStdioDriver(options), name);
47
47
  default:
48
48
  throw new DriverError(`unknown driver "${name}"; one of: ${DRIVERS.join(', ')}`, { fatal: true });
49
49
  }
@@ -0,0 +1,111 @@
1
+ import fs from 'node:fs/promises';
2
+ import path from 'node:path';
3
+ import { spawn } from 'node:child_process';
4
+ import { DriverError, observationHash, screenshotName, truncateObservation } from './driver.js';
5
+
6
+ // A developer-owned bridge implements newline-delimited JSON-RPC. The model
7
+ // receives game controls only, never command arguments or process access.
8
+ export async function readBridgeConfig(file) {
9
+ if ((await fs.stat(file)).size > 64000) throw new DriverError('Bridge configuration exceeds 64 KB.');
10
+ const config = JSON.parse(await fs.readFile(file, 'utf8'));
11
+ if (typeof config.command !== 'string' || !path.isAbsolute(config.command) ||
12
+ (config.args !== undefined && (!Array.isArray(config.args) || config.args.some(v => typeof v !== 'string'))) ||
13
+ (config.cwd !== undefined && !path.isAbsolute(config.cwd))) {
14
+ throw new DriverError('Bridge config needs an absolute command, string arguments and an optional absolute cwd.');
15
+ }
16
+ await fs.access(config.command, fs.constants.X_OK);
17
+ return config;
18
+ }
19
+
20
+ export function createStdioDriver({ spawn: spawnImpl = spawn, timeoutMs = 30000 } = {}) {
21
+ let child, buffer = '', nextId = 0, runDirectory, step = 0, shots = 0, lastHash;
22
+ const pending = new Map();
23
+ const fail = message => {
24
+ for (const entry of pending.values()) { clearTimeout(entry.timer); entry.reject(new DriverError(message, { fatal: true })); }
25
+ pending.clear();
26
+ };
27
+ function request(method, params = {}) {
28
+ if (!child || child.exitCode !== null || child.killed) return Promise.reject(new DriverError('Game bridge is not running.', { fatal: true }));
29
+ return new Promise((resolve, reject) => {
30
+ const id = ++nextId;
31
+ const timer = setTimeout(() => { pending.delete(id); reject(new DriverError('Game bridge timed out.', { fatal: true })); child.kill(); }, timeoutMs);
32
+ pending.set(id, { resolve, reject, timer });
33
+ child.stdin.write(JSON.stringify({ jsonrpc: '2.0', id, method, params }) + '\n', error => {
34
+ if (error) { clearTimeout(timer); pending.delete(id); reject(new DriverError('Game bridge input failed.')); }
35
+ });
36
+ });
37
+ }
38
+ async function observe() {
39
+ const value = await request('observe');
40
+ if (!value || typeof value.text !== 'string') throw new DriverError('Bridge observation needs text.');
41
+ const observation = { step: ++step, text: truncateObservation(value.text), elapsedMs: 0 };
42
+ if (typeof value.screenshotBase64 === 'string' && value.screenshotBase64.startsWith('iVBORw0KGgo')) {
43
+ observation.screenshotBase64 = value.screenshotBase64;
44
+ observation.screenshotMediaType = 'image/png';
45
+ }
46
+ lastHash = observationHash(observation);
47
+ return observation;
48
+ }
49
+ return {
50
+ name: 'cli_stdio', supportedActions: ['type', 'key', 'click', 'wait'],
51
+ async launch({ target, runDir }) {
52
+ const config = await readBridgeConfig(target);
53
+ runDirectory = runDir;
54
+ child = spawnImpl(config.command, config.args || [], { cwd: config.cwd || path.dirname(path.resolve(target)), shell: false, stdio: ['pipe', 'pipe', 'pipe'] });
55
+ child.stdout.setEncoding('utf8');
56
+ child.stdout.on('data', chunk => {
57
+ buffer += chunk;
58
+ if (Buffer.byteLength(buffer) > 8 * 1024 * 1024) { fail('Game bridge response exceeded 8 MB.'); child.kill(); return; }
59
+ let end;
60
+ while ((end = buffer.indexOf('\n')) !== -1) {
61
+ const line = buffer.slice(0, end); buffer = buffer.slice(end + 1);
62
+ let value;
63
+ try { value = JSON.parse(line); } catch { fail('Game bridge returned invalid JSON.'); child.kill(); return; }
64
+ const entry = pending.get(value?.id);
65
+ if (!entry) continue;
66
+ pending.delete(value.id); clearTimeout(entry.timer);
67
+ if (value.jsonrpc !== '2.0' || value.error) entry.reject(new DriverError('Game bridge command failed.'));
68
+ else entry.resolve(value.result);
69
+ }
70
+ });
71
+ // Drain diagnostics without copying arbitrary game output into reports.
72
+ child.stderr.resume();
73
+ child.stdin.on('error', () => fail('Game bridge input failed.'));
74
+ child.on('error', () => fail('Game bridge could not start.'));
75
+ child.on('exit', () => fail('Game bridge exited.'));
76
+ await fs.mkdir(path.join(runDir, 'screenshots'), { recursive: true });
77
+ await request('launch', { protocolVersion: 1 });
78
+ },
79
+ observe,
80
+ async act(action) {
81
+ if (!['type', 'key', 'click', 'wait'].includes(action.kind)) return { ok: false, changed: false, error: 'Unsupported bridge action.' };
82
+ const before = lastHash;
83
+ const safe = action.kind === 'type' ? { kind: 'type', text: String(action.text || '').slice(0, 2000), submit: Boolean(action.submit) }
84
+ : action.kind === 'key' ? { kind: 'key', key: String(action.key || '').slice(0, 64) }
85
+ : action.kind === 'wait' ? { kind: 'wait', ms: Math.min(5000, Math.max(0, Number(action.ms) || 0)) }
86
+ : { kind: 'click', target: typeof action.target === 'string' ? action.target.slice(0, 500) : action.target };
87
+ const answer = await request('act', safe);
88
+ if (answer?.ok !== true) return { ok: false, changed: false, error: 'Game bridge refused this action.' };
89
+ return { ok: true, changed: observationHash(await observe()) !== before };
90
+ },
91
+ async screenshot(reason) {
92
+ const image = await request('screenshot');
93
+ if (typeof image?.pngBase64 !== 'string' || !image.pngBase64.startsWith('iVBORw0KGgo')) throw new DriverError('This bridge did not return a PNG screenshot.');
94
+ const relativePath = path.join('screenshots', screenshotName(++shots, reason));
95
+ const file = path.join(runDirectory, relativePath);
96
+ await fs.writeFile(file, Buffer.from(image.pngBase64, 'base64'));
97
+ return { path: file, relativePath };
98
+ },
99
+ async consoleErrors() { return []; },
100
+ async stop() {
101
+ if (child && child.exitCode === null && !child.killed) {
102
+ child.kill('SIGTERM');
103
+ const process = child;
104
+ const timer = setTimeout(() => { if (process.exitCode === null) process.kill('SIGKILL'); }, 1000);
105
+ timer.unref();
106
+ }
107
+ fail('Game bridge stopped.');
108
+ return { videoPath: null };
109
+ }
110
+ };
111
+ }