ravensight-playtest 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +380 -0
- package/addons/ravensight_driver/driver.gd +836 -0
- package/addons/ravensight_driver/export_plugin.gd +51 -0
- package/addons/ravensight_driver/plugin.cfg +7 -0
- package/addons/ravensight_driver/plugin.gd +36 -0
- package/bin/ravensight-playtest.js +31 -0
- package/package.json +45 -0
- package/src/api/README.md +500 -0
- package/src/api/client.js +340 -0
- package/src/api/errors.js +115 -0
- package/src/api/http.js +194 -0
- package/src/api/index.js +107 -0
- package/src/auth/deviceCode.js +79 -0
- package/src/auth/keychain.js +159 -0
- package/src/auth/session.js +128 -0
- package/src/cli.js +335 -0
- package/src/commands/brief.js +303 -0
- package/src/commands/check.js +318 -0
- package/src/commands/fakeCore.js +379 -0
- package/src/commands/init.js +120 -0
- package/src/commands/login.js +90 -0
- package/src/commands/logout.js +70 -0
- package/src/commands/open.js +125 -0
- package/src/commands/profile.js +262 -0
- package/src/commands/resume.js +156 -0
- package/src/commands/run.js +1015 -0
- package/src/commands/upload.js +137 -0
- package/src/config.js +100 -0
- package/src/dashboard.js +97 -0
- package/src/detect.js +77 -0
- package/src/errors.js +44 -0
- package/src/fsutil.js +77 -0
- package/src/godot.js +85 -0
- package/src/packs/index.js +191 -0
- package/src/paths.js +129 -0
- package/src/run/aggregate.js +658 -0
- package/src/run/args.js +111 -0
- package/src/run/context.js +181 -0
- package/src/run/deps.js +184 -0
- package/src/run/drivers/driver.js +183 -0
- package/src/run/drivers/godot-observation.js +138 -0
- package/src/run/drivers/godot-project.js +475 -0
- package/src/run/drivers/godot-rpc.js +225 -0
- package/src/run/drivers/godot.js +587 -0
- package/src/run/drivers/index.js +52 -0
- package/src/run/drivers/web.js +385 -0
- package/src/run/exit.js +21 -0
- package/src/run/heartbeat.js +131 -0
- package/src/run/index.js +31 -0
- package/src/run/json.js +56 -0
- package/src/run/model.js +384 -0
- package/src/run/paths.js +88 -0
- package/src/run/personaLoop.js +871 -0
- package/src/run/profile.js +214 -0
- package/src/run/regenerate.js +149 -0
- package/src/run/repoTools.js +286 -0
- package/src/run/report.js +222 -0
- package/src/run/resume.js +272 -0
- package/src/run/secretScan.js +171 -0
- package/src/run/state.js +198 -0
- package/src/run/synthetic.js +206 -0
- package/src/run/tools.js +344 -0
- package/src/run/transcript.js +93 -0
- package/src/run/usage.js +115 -0
- package/src/state/index.js +105 -0
- package/src/states.js +104 -0
- package/src/ui/index.js +195 -0
- package/src/upload/allowlist.js +116 -0
- package/src/upload/index.js +467 -0
- package/src/upload/queue.js +114 -0
- package/src/version.js +63 -0
package/src/run/args.js
ADDED
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `run` and `profile` are declared in cli-core's commander program with
|
|
3
|
+
* `allowUnknownOption()`, and everything commander did not recognise arrives
|
|
4
|
+
* as `flags.argv`. So the runner parses its own options, which is the right
|
|
5
|
+
* place for them: they are about driving a game, and cli-core has no business
|
|
6
|
+
* knowing what `--godot-project` means.
|
|
7
|
+
*
|
|
8
|
+
* Only the flags spec 17 section 1.2 lists are understood. An unknown one is
|
|
9
|
+
* reported rather than ignored, because a silently dropped `--max-actions 5`
|
|
10
|
+
* spends the developer's money on 25 actions.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
/** Options that take a value. */
|
|
14
|
+
const VALUED = new Map([
|
|
15
|
+
['--personas', 'personas'],
|
|
16
|
+
['--driver', 'driver'],
|
|
17
|
+
['--build-url', 'buildUrl'],
|
|
18
|
+
['--build-dir', 'buildDir'],
|
|
19
|
+
['--godot-project', 'godotProject'],
|
|
20
|
+
['--cli', 'cli'],
|
|
21
|
+
['--max-actions', 'maxActions'],
|
|
22
|
+
['--concurrency', 'concurrency'],
|
|
23
|
+
['--budget-usd', 'budgetUsd'],
|
|
24
|
+
['--quality', 'quality']
|
|
25
|
+
]);
|
|
26
|
+
|
|
27
|
+
/** Options that are just present or absent. */
|
|
28
|
+
const BOOLEAN = new Map([
|
|
29
|
+
['--upload-video', 'uploadVideo'],
|
|
30
|
+
['--upload-transcript', 'uploadTranscript'],
|
|
31
|
+
['--dry-upload', 'dryUpload'],
|
|
32
|
+
['--offline-upload', 'offlineUpload'],
|
|
33
|
+
['--headed', 'headed'],
|
|
34
|
+
['--headless', 'headless'],
|
|
35
|
+
['--verbose', 'verbose'],
|
|
36
|
+
// Resume only: replay an interrupted run from the start instead of writing a
|
|
37
|
+
// partial report from the transcript it already has.
|
|
38
|
+
['--restart-interrupted', 'restartInterrupted']
|
|
39
|
+
]);
|
|
40
|
+
|
|
41
|
+
/** Options whose value is a number. */
|
|
42
|
+
const NUMERIC = new Set(['maxActions', 'concurrency', 'budgetUsd']);
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* @param {string[]} argv the leftovers commander handed over
|
|
46
|
+
* @returns {{options: Object, unknown: string[]}}
|
|
47
|
+
*/
|
|
48
|
+
export function parseRunArgv(argv = []) {
|
|
49
|
+
const options = {};
|
|
50
|
+
const unknown = [];
|
|
51
|
+
for (let i = 0; i < argv.length; i += 1) {
|
|
52
|
+
const token = String(argv[i]);
|
|
53
|
+
// `--flag=value` and `--flag value` are both what a developer will type.
|
|
54
|
+
const split = token.indexOf('=');
|
|
55
|
+
const name = split === -1 ? token : token.slice(0, split);
|
|
56
|
+
const inline = split === -1 ? null : token.slice(split + 1);
|
|
57
|
+
|
|
58
|
+
if (BOOLEAN.has(name)) {
|
|
59
|
+
options[BOOLEAN.get(name)] = inline === null ? true : inline !== 'false';
|
|
60
|
+
continue;
|
|
61
|
+
}
|
|
62
|
+
if (VALUED.has(name)) {
|
|
63
|
+
const key = VALUED.get(name);
|
|
64
|
+
const raw = inline === null ? argv[i + 1] : inline;
|
|
65
|
+
if (inline === null) i += 1;
|
|
66
|
+
if (raw === undefined) {
|
|
67
|
+
unknown.push(`${name} needs a value`);
|
|
68
|
+
continue;
|
|
69
|
+
}
|
|
70
|
+
options[key] = NUMERIC.has(key) ? Number(raw) : String(raw);
|
|
71
|
+
continue;
|
|
72
|
+
}
|
|
73
|
+
if (name.startsWith('-')) unknown.push(name);
|
|
74
|
+
}
|
|
75
|
+
return { options, unknown };
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Merge the flags cli-core parsed with the ones this file parsed, flags
|
|
80
|
+
* winning, and normalise the two names for the repository root.
|
|
81
|
+
*
|
|
82
|
+
* @param {Object} flags
|
|
83
|
+
* @returns {{options: Object, unknown: string[]}}
|
|
84
|
+
*/
|
|
85
|
+
export function optionsFrom(flags = {}) {
|
|
86
|
+
const { options, unknown } = parseRunArgv(flags.argv);
|
|
87
|
+
return {
|
|
88
|
+
options: {
|
|
89
|
+
...options,
|
|
90
|
+
...stripUndefined({
|
|
91
|
+
repo: flags.repoRoot || flags.repo,
|
|
92
|
+
apiUrl: flags.apiUrl,
|
|
93
|
+
game: flags.game,
|
|
94
|
+
yes: flags.yes,
|
|
95
|
+
json: flags.json,
|
|
96
|
+
token: flags.token,
|
|
97
|
+
idempotencyKey: flags.idempotencyKey,
|
|
98
|
+
// `resume` declares this one in commander rather than leaving it to the
|
|
99
|
+
// argv leftovers, so it arrives as a flag and has to be read here too.
|
|
100
|
+
restartInterrupted: flags.restartInterrupted
|
|
101
|
+
})
|
|
102
|
+
},
|
|
103
|
+
unknown
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function stripUndefined(value) {
|
|
108
|
+
return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== undefined));
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
export default { parseRunArgv, optionsFrom };
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Keeping the conversation small enough to be affordable and legal.
|
|
3
|
+
*
|
|
4
|
+
* Two independent ceilings:
|
|
5
|
+
*
|
|
6
|
+
* 1. **Images.** `routing.json`'s `persona_playtest.act` sets
|
|
7
|
+
* `max_images: 1`, and the proxy counts image blocks across the WHOLE
|
|
8
|
+
* `messages` array and answers 400 `too_many_images` past that. So the
|
|
9
|
+
* ceiling is on the context, not on the newest turn: exactly one image
|
|
10
|
+
* may be in flight, and it has to be the newest one, because that is the
|
|
11
|
+
* screen the persona is looking at. Older images become a one line
|
|
12
|
+
* placeholder rather than being deleted, so the turn structure and the
|
|
13
|
+
* tool_use / tool_result pairing stay intact.
|
|
14
|
+
*
|
|
15
|
+
* The brief asked for at most two images per call. One is what the
|
|
16
|
+
* server actually allows on this step, and a body with two comes back
|
|
17
|
+
* 400 with nothing played, so this file follows routing.json. It is a
|
|
18
|
+
* constant rather than a literal so that a routing change is a one line
|
|
19
|
+
* change here, and `shrinkImages` exists so a caller that meets
|
|
20
|
+
* `too_many_images` anyway can retry smaller instead of failing the run.
|
|
21
|
+
*
|
|
22
|
+
* 2. **Observations.** Spec 03 keeps the last K=6 observations and lets
|
|
23
|
+
* older ones fall back to a stub. A snapshot of a busy screen is several
|
|
24
|
+
* thousand characters, so without this a 25 action run pays for its first
|
|
25
|
+
* screen twenty five times.
|
|
26
|
+
*/
|
|
27
|
+
|
|
28
|
+
/** What `persona_playtest.act` allows in one request. */
|
|
29
|
+
export const MAX_IMAGES_IN_CONTEXT = 1;
|
|
30
|
+
|
|
31
|
+
/** How many full observations stay in context (spec 03's K). */
|
|
32
|
+
export const KEEP_OBSERVATIONS = 6;
|
|
33
|
+
|
|
34
|
+
const DROPPED_IMAGE_TEXT = '[screenshot dropped from context; only the newest screen is kept]';
|
|
35
|
+
const DROPPED_OBSERVATION_TEXT = '[older observation dropped from context; ask for a snapshot if you need to look again]';
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* @param {Array} content
|
|
39
|
+
* @returns {boolean}
|
|
40
|
+
*/
|
|
41
|
+
function hasImage(content) {
|
|
42
|
+
return Array.isArray(content) && content.some(block => block && block.type === 'image');
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Keep the newest `max` image blocks and replace the rest with a
|
|
47
|
+
* placeholder. Walks backwards, because "newest" is what matters.
|
|
48
|
+
*
|
|
49
|
+
* @param {Array} messages
|
|
50
|
+
* @param {number} [max]
|
|
51
|
+
* @returns {Array} a new array; the input is not mutated
|
|
52
|
+
*/
|
|
53
|
+
export function pruneImages(messages, max = MAX_IMAGES_IN_CONTEXT) {
|
|
54
|
+
let budget = Math.max(0, max);
|
|
55
|
+
const out = messages.map(message => message);
|
|
56
|
+
for (let i = out.length - 1; i >= 0; i -= 1) {
|
|
57
|
+
const message = out[i];
|
|
58
|
+
if (!message || !Array.isArray(message.content)) continue;
|
|
59
|
+
const content = [];
|
|
60
|
+
// Within one message the blocks are also newest-last, so the same
|
|
61
|
+
// backwards walk keeps the right image when a single tool_result
|
|
62
|
+
// somehow carries two.
|
|
63
|
+
const blocks = [...message.content];
|
|
64
|
+
for (let j = blocks.length - 1; j >= 0; j -= 1) {
|
|
65
|
+
const block = blocks[j];
|
|
66
|
+
if (block && block.type === 'image') {
|
|
67
|
+
if (budget > 0) {
|
|
68
|
+
budget -= 1;
|
|
69
|
+
content.unshift(block);
|
|
70
|
+
} else {
|
|
71
|
+
content.unshift({ type: 'text', text: DROPPED_IMAGE_TEXT });
|
|
72
|
+
}
|
|
73
|
+
continue;
|
|
74
|
+
}
|
|
75
|
+
if (block && block.type === 'tool_result' && hasImage(block.content)) {
|
|
76
|
+
const inner = [];
|
|
77
|
+
const innerBlocks = [...block.content];
|
|
78
|
+
for (let k = innerBlocks.length - 1; k >= 0; k -= 1) {
|
|
79
|
+
const child = innerBlocks[k];
|
|
80
|
+
if (child && child.type === 'image') {
|
|
81
|
+
if (budget > 0) {
|
|
82
|
+
budget -= 1;
|
|
83
|
+
inner.unshift(child);
|
|
84
|
+
} else {
|
|
85
|
+
inner.unshift({ type: 'text', text: DROPPED_IMAGE_TEXT });
|
|
86
|
+
}
|
|
87
|
+
continue;
|
|
88
|
+
}
|
|
89
|
+
inner.unshift(child);
|
|
90
|
+
}
|
|
91
|
+
content.unshift({ ...block, content: inner });
|
|
92
|
+
continue;
|
|
93
|
+
}
|
|
94
|
+
content.unshift(block);
|
|
95
|
+
}
|
|
96
|
+
out[i] = { ...message, content };
|
|
97
|
+
}
|
|
98
|
+
return out;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Count image blocks the way the proxy does, so a caller can assert before
|
|
103
|
+
* it spends a call.
|
|
104
|
+
*
|
|
105
|
+
* @param {Array} messages
|
|
106
|
+
* @returns {number}
|
|
107
|
+
*/
|
|
108
|
+
export function countImages(messages) {
|
|
109
|
+
let images = 0;
|
|
110
|
+
for (const message of messages) {
|
|
111
|
+
if (!message || !Array.isArray(message.content)) continue;
|
|
112
|
+
for (const block of message.content) {
|
|
113
|
+
if (!block) continue;
|
|
114
|
+
if (block.type === 'image') images += 1;
|
|
115
|
+
else if (block.type === 'tool_result' && Array.isArray(block.content)) {
|
|
116
|
+
for (const child of block.content) {
|
|
117
|
+
if (child && child.type === 'image') images += 1;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
return images;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* Replace all but the newest `keep` observation texts with a stub. An
|
|
127
|
+
* observation is a `tool_result` whose text begins `step N`, which is the
|
|
128
|
+
* shape ./tools.js writes.
|
|
129
|
+
*
|
|
130
|
+
* @param {Array} messages
|
|
131
|
+
* @param {number} [keep]
|
|
132
|
+
* @returns {Array}
|
|
133
|
+
*/
|
|
134
|
+
export function pruneObservations(messages, keep = KEEP_OBSERVATIONS) {
|
|
135
|
+
const indexes = [];
|
|
136
|
+
messages.forEach((message, i) => {
|
|
137
|
+
if (!message || !Array.isArray(message.content)) return;
|
|
138
|
+
message.content.forEach((block, j) => {
|
|
139
|
+
if (block && block.type === 'tool_result' && Array.isArray(block.content)) {
|
|
140
|
+
const first = block.content.find(child => child && child.type === 'text');
|
|
141
|
+
if (first && /^(?:[^\n]*\n)?step \d+/.test(String(first.text))) indexes.push([i, j]);
|
|
142
|
+
}
|
|
143
|
+
});
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
const stale = indexes.slice(0, Math.max(0, indexes.length - keep));
|
|
147
|
+
if (stale.length === 0) return messages;
|
|
148
|
+
|
|
149
|
+
const out = messages.map(message => (message && Array.isArray(message.content) ? { ...message, content: [...message.content] } : message));
|
|
150
|
+
for (const [i, j] of stale) {
|
|
151
|
+
const block = out[i].content[j];
|
|
152
|
+
out[i].content[j] = { ...block, content: [{ type: 'text', text: DROPPED_OBSERVATION_TEXT }] };
|
|
153
|
+
}
|
|
154
|
+
return out;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* Both ceilings, in the order that matters: drop stale observations first
|
|
159
|
+
* (which removes their images too), then enforce the image budget on what
|
|
160
|
+
* is left.
|
|
161
|
+
*
|
|
162
|
+
* @param {Array} messages
|
|
163
|
+
* @param {{maxImages?: number, keepObservations?: number}} [opts]
|
|
164
|
+
* @returns {Array}
|
|
165
|
+
*/
|
|
166
|
+
export function prepareMessages(messages, { maxImages = MAX_IMAGES_IN_CONTEXT, keepObservations = KEEP_OBSERVATIONS } = {}) {
|
|
167
|
+
return pruneImages(pruneObservations(messages, keepObservations), maxImages);
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* The retry after a `too_many_images`: the step allows fewer than we
|
|
172
|
+
* thought, so halve and try again rather than losing the run.
|
|
173
|
+
*
|
|
174
|
+
* @param {number} current
|
|
175
|
+
* @returns {number}
|
|
176
|
+
*/
|
|
177
|
+
export function shrinkImages(current) {
|
|
178
|
+
return Math.max(0, Math.floor(current / 2));
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
export default { prepareMessages, pruneImages, pruneObservations, countImages, shrinkImages, MAX_IMAGES_IN_CONTEXT, KEEP_OBSERVATIONS };
|
package/src/run/deps.js
ADDED
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The seam between the runner (this directory) and cli-core.
|
|
3
|
+
*
|
|
4
|
+
* Nothing under `src/run/**` imports cli-core except through `getDeps()`, so
|
|
5
|
+
* the whole integration is this one file, and every test in this directory
|
|
6
|
+
* injects fakes with `setDeps()` rather than standing up an HTTP server it
|
|
7
|
+
* does not need.
|
|
8
|
+
*
|
|
9
|
+
* ## What cli-core provides
|
|
10
|
+
*
|
|
11
|
+
* `src/api/index.js` is the only importable module (its own README says so).
|
|
12
|
+
* The names used here:
|
|
13
|
+
*
|
|
14
|
+
* | Name | Shape |
|
|
15
|
+
* |---|---|
|
|
16
|
+
* | `createContext({ apiUrl, repoRoot, requireAuth })` | `{ api, apiUrl, token, config, repoRoot }` in one call |
|
|
17
|
+
* | `api.jobs.estimate(gameId, { modules, personas })` | `{ estimate_cents, breakdown, balance_cents, ok, topup_url }` |
|
|
18
|
+
* | `api.jobs.register(gameId, body, { idempotencyKey })` | `{ job_id, runs: [{ run_id, module, persona, playtest_token, budget_usd }], price, links, replayed }` |
|
|
19
|
+
* | `api.jobs.finish(gameId, jobId, { state, reason })` | `{ job, refunded_cents, already_settled }` |
|
|
20
|
+
* | `api.jobs.capabilityReport(gameId, jobId, body)` | `{ capability_report }` |
|
|
21
|
+
* | `api.jobs.aggregateComplete(gameId, jobId, body)` | `{ aggregate }` |
|
|
22
|
+
* | `api.jobs.uploads(gameId, jobId, { files })` | job-level presign, used for the aggregate |
|
|
23
|
+
* | `api.brief.get(gameId)` | `{ brief, rendered }` or `{ brief: null }`; the frozen expectations brief every persona is told to respect |
|
|
24
|
+
* | `api.runs.heartbeat(gameId, jobId, runId, opts)` | `{ run, cancel_requested }`; `opts` is camelCase (`actionsTaken`, `checkpointStep`) |
|
|
25
|
+
* | `api.runs.transition(gameId, jobId, runId, state, opts)` | the same PATCH with a state |
|
|
26
|
+
* | `api.runs.complete(gameId, jobId, runId, body)` | reached through `finalizeRun` |
|
|
27
|
+
* | `packs.get({ api, gameId, modules })` | `{ bundle, etag, fromCache, stale }`; the runner wants `bundle` |
|
|
28
|
+
* | `uploadRunDir(runDir, opts)` / `finalizeRun(runDir, opts)` | presign, PUT, `complete`, with the 422 `upload_unverified` repair pass |
|
|
29
|
+
* | `putFile`, `contentTypeFor` | the one PUT the runner makes itself, for the job-level aggregate |
|
|
30
|
+
* | `ui`, `paths`, `loadConfig`, `resolveGameId`, `CLI_VERSION`, `ApiError` | output, layout, config |
|
|
31
|
+
*
|
|
32
|
+
* Two deliberate differences from what cli-core hands over:
|
|
33
|
+
*
|
|
34
|
+
* 1. **`packs.get` is unwrapped here.** Everything above this file wants the
|
|
35
|
+
* bundle, not the cache metadata, and a `bundle.bundle` would be a trap.
|
|
36
|
+
* 2. **The job-level upload is `uploadRunDir` with no `runId`.** The aggregate
|
|
37
|
+
* lives at job level rather than in a run directory, and cli-core's uploader
|
|
38
|
+
* already covers that: its allowlist has `aggregate-report.md` and
|
|
39
|
+
* `aggregate-report.json` in it, and its presign helper falls back to
|
|
40
|
+
* `api.jobs.uploads` when there is no run. So `publishJobFiles` delegates,
|
|
41
|
+
* which is what keeps the upload journal (`upload-queue.jsonl`, so a crash
|
|
42
|
+
* can be drained later) and the 403 re-presign in the one module that owns
|
|
43
|
+
* them. All it adds is the size and digest of the markdown, which
|
|
44
|
+
* `aggregate/complete` has to be told and the uploader does not return.
|
|
45
|
+
*/
|
|
46
|
+
|
|
47
|
+
import { createHash } from 'node:crypto';
|
|
48
|
+
import { readFile } from 'node:fs/promises';
|
|
49
|
+
import path from 'node:path';
|
|
50
|
+
|
|
51
|
+
/** @type {Object|null} */
|
|
52
|
+
let injected = null;
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Replace every dependency. Tests use this; so does anything that wants to
|
|
56
|
+
* drive the loop against a fake.
|
|
57
|
+
*
|
|
58
|
+
* @param {Object|null} deps
|
|
59
|
+
*/
|
|
60
|
+
export function setDeps(deps) {
|
|
61
|
+
injected = deps;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** Undo `setDeps`. */
|
|
65
|
+
export function resetDeps() {
|
|
66
|
+
injected = null;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* The path prefix every playtest route hangs off. Tenancy is structural on the
|
|
71
|
+
* server (`/api/v1/games/:gameId/playtest/...` behind `requireGame`), so the
|
|
72
|
+
* game id is in the path rather than in a body field, and it is built once.
|
|
73
|
+
*
|
|
74
|
+
* @param {string} gameId
|
|
75
|
+
* @returns {string}
|
|
76
|
+
*/
|
|
77
|
+
export function playtestBase(gameId) {
|
|
78
|
+
if (!gameId) throw new Error('a game id is required; run "ravensight-playtest init --game <gameId>" first');
|
|
79
|
+
return `/api/v1/games/${encodeURIComponent(gameId)}/playtest`;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Wrap cli-core's barrel into the surface `src/run/**` uses.
|
|
84
|
+
*
|
|
85
|
+
* @param {Object} core the `src/api/index.js` module
|
|
86
|
+
* @returns {Object}
|
|
87
|
+
*/
|
|
88
|
+
export function normalise(core) {
|
|
89
|
+
if (!core || typeof core.createContext !== 'function') {
|
|
90
|
+
throw new Error('src/api/index.js did not export createContext; see src/run/deps.js for the interface the runner needs');
|
|
91
|
+
}
|
|
92
|
+
return {
|
|
93
|
+
core,
|
|
94
|
+
createContext: core.createContext,
|
|
95
|
+
loadConfig: core.loadConfig,
|
|
96
|
+
resolveGameId: core.resolveGameId,
|
|
97
|
+
cliVersion: core.CLI_VERSION,
|
|
98
|
+
ui: core.ui,
|
|
99
|
+
paths: core.paths,
|
|
100
|
+
ApiError: core.ApiError,
|
|
101
|
+
uploadRunDir: core.uploadRunDir,
|
|
102
|
+
finalizeRun: core.finalizeRun,
|
|
103
|
+
brief: {
|
|
104
|
+
/**
|
|
105
|
+
* The expectations brief, or null when this game has none.
|
|
106
|
+
*
|
|
107
|
+
* Unwrapped from `{ brief, rendered }` because everything above this
|
|
108
|
+
* file wants the brief itself, and because a null brief is a normal
|
|
109
|
+
* answer: a `game_profile` job legitimately runs without one, and the
|
|
110
|
+
* persona skill has a "when expectations_brief is not null" branch for
|
|
111
|
+
* exactly this. A failure to fetch is NOT swallowed: a persona playing
|
|
112
|
+
* without the brief reports findings nobody asked for and re-reports
|
|
113
|
+
* known issues, which is worse than not running.
|
|
114
|
+
*
|
|
115
|
+
* @param {{api: Object, gameId: string}} options
|
|
116
|
+
* @returns {Promise<Object|null>}
|
|
117
|
+
*/
|
|
118
|
+
async get({ api, gameId }) {
|
|
119
|
+
const answer = await api.brief.get(gameId);
|
|
120
|
+
return (answer && answer.brief) || null;
|
|
121
|
+
}
|
|
122
|
+
},
|
|
123
|
+
packs: {
|
|
124
|
+
/**
|
|
125
|
+
* @param {{api: Object, gameId: string, modules: string[], offline?: boolean}} options
|
|
126
|
+
* @returns {Promise<Object>} the bundle itself
|
|
127
|
+
*/
|
|
128
|
+
async get(options) {
|
|
129
|
+
const answer = await core.packs.get(options);
|
|
130
|
+
return answer && answer.bundle ? answer.bundle : answer;
|
|
131
|
+
}
|
|
132
|
+
},
|
|
133
|
+
publishJobFiles: buildPublishJobFiles(core)
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* Upload job-level files (the aggregate) and answer their sizes and digests,
|
|
139
|
+
* which is what `aggregate/complete` then has to be told.
|
|
140
|
+
*
|
|
141
|
+
* @param {Object} core
|
|
142
|
+
* @returns {(args: Object) => Promise<Object>}
|
|
143
|
+
*/
|
|
144
|
+
export function buildPublishJobFiles(core) {
|
|
145
|
+
return async function publishJobFiles({ api, gameId, jobId, dir, files, repoRoot }) {
|
|
146
|
+
// The digests first, because `aggregate/complete` verifies the markdown
|
|
147
|
+
// against a size and a sha256 and the uploader does not hand those back.
|
|
148
|
+
// Same bytes either way: both read the file that is on disk now.
|
|
149
|
+
const digests = {};
|
|
150
|
+
for (const name of files) {
|
|
151
|
+
const body = await readFile(path.join(dir, name));
|
|
152
|
+
digests[name] = { size: body.length, sha256: createHash('sha256').update(body).digest('hex') };
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
// `runId: null` is what makes this the job-level presign, and `allowedPaths`
|
|
156
|
+
// confines the walk to the two aggregate files so nothing else in the job
|
|
157
|
+
// directory can be swept up.
|
|
158
|
+
const result = await core.uploadRunDir(dir, {
|
|
159
|
+
api,
|
|
160
|
+
gameId,
|
|
161
|
+
jobId,
|
|
162
|
+
runId: null,
|
|
163
|
+
allowedPaths: files,
|
|
164
|
+
repoRoot
|
|
165
|
+
});
|
|
166
|
+
|
|
167
|
+
const failed = (result && result.skipped ? result.skipped : [])
|
|
168
|
+
.filter(entry => files.includes(entry.path) && entry.reason !== 'already uploaded');
|
|
169
|
+
if (failed.length > 0) {
|
|
170
|
+
throw new Error(`the aggregate was not uploaded: ${failed.map(entry => `${entry.path} (${entry.reason})`).join(', ')}`);
|
|
171
|
+
}
|
|
172
|
+
return digests;
|
|
173
|
+
};
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* @returns {Promise<Object>}
|
|
178
|
+
*/
|
|
179
|
+
export async function getDeps() {
|
|
180
|
+
if (injected) return injected;
|
|
181
|
+
return normalise(await import('../api/index.js'));
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
export default { getDeps, setDeps, resetDeps, normalise, playtestBase, buildPublishJobFiles };
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* The driver interface. One shape, three implementations (spec 03's
|
|
5
|
+
* "Driver interface (all drivers)"): `playwright_web` here in ./web.js,
|
|
6
|
+
* `godot_driver` in ./godot.js, and `cli_stdio` later.
|
|
7
|
+
*
|
|
8
|
+
* A driver is the ONLY thing in this CLI that touches the game. The model
|
|
9
|
+
* never gets a shell, a file write outside the run directory, or a network
|
|
10
|
+
* tool: the whole tool surface it sees is this interface, wrapped by
|
|
11
|
+
* ../tools.js. That is what keeps prompt injection from game text
|
|
12
|
+
* (a level name, an NPC line, a leaderboard entry) to a small blast
|
|
13
|
+
* radius, and it is why `act` takes a closed union of actions rather than
|
|
14
|
+
* anything the model would like to run.
|
|
15
|
+
*
|
|
16
|
+
* ```
|
|
17
|
+
* launch({ target, runDir, recordVideo, viewport, env, signal }) -> void
|
|
18
|
+
* observe() -> Observation
|
|
19
|
+
* act(action) -> ActResult
|
|
20
|
+
* screenshot(name, opts) -> { path, relativePath }
|
|
21
|
+
* stop() -> { videoPath?: string|null, framesDir?: string }
|
|
22
|
+
* ```
|
|
23
|
+
*
|
|
24
|
+
* Optional, used when present:
|
|
25
|
+
*
|
|
26
|
+
* ```
|
|
27
|
+
* chapter(title) -> void video chapter marker
|
|
28
|
+
* consoleErrors() -> ConsoleEntry[] errors since the previous call
|
|
29
|
+
* ```
|
|
30
|
+
*
|
|
31
|
+
* Two places where the drivers legitimately differ, and callers have to
|
|
32
|
+
* cope with both rather than assume the web one:
|
|
33
|
+
*
|
|
34
|
+
* - **A string `click` target is driver-specific.** On `playwright_web` it
|
|
35
|
+
* is an accessibility node, a text fragment or a CSS selector; on
|
|
36
|
+
* `godot_driver` it is a node path out of the scene tree. The tool
|
|
37
|
+
* description in ../tools.js says so, because the model is the one
|
|
38
|
+
* choosing.
|
|
39
|
+
* - **`stop()` may return no video.** `playwright_web` hands back a
|
|
40
|
+
* `session.webm` from Playwright's own recorder. `godot_driver` returns
|
|
41
|
+
* `videoPath: null` and leaves a `frames/` sequence, which the CLI
|
|
42
|
+
* stitches only when ffmpeg is present. That is what keeps ffmpeg
|
|
43
|
+
* optional, and it means "no video" is a normal outcome, not a failure.
|
|
44
|
+
*
|
|
45
|
+
* `Observation.scene`, `.scenes` and `.scripts` are how a finding from a
|
|
46
|
+
* running Godot game can cite a `res://` path; the web driver leaves them
|
|
47
|
+
* unset.
|
|
48
|
+
*
|
|
49
|
+
* @typedef {Object} Observation
|
|
50
|
+
* @property {number} step monotonic, one per observe()
|
|
51
|
+
* @property {string} text accessibility snapshot, scene tree summary or
|
|
52
|
+
* stdout delta, truncated to OBSERVATION_MAX_CHARS
|
|
53
|
+
* @property {unknown} [structured] the raw payload, never sent to the model
|
|
54
|
+
* unless it is small
|
|
55
|
+
* @property {string} [screenshotBase64] a PNG, only when the text
|
|
56
|
+
* observation is too thin to play from (canvas games)
|
|
57
|
+
* @property {string} [screenshotMediaType]
|
|
58
|
+
* @property {string} [url]
|
|
59
|
+
* @property {string} [scene] the current scene path, godot_driver only
|
|
60
|
+
* @property {string[]} [scenes] scene paths behind this screen
|
|
61
|
+
* @property {string[]} [scripts] script paths behind this screen
|
|
62
|
+
* @property {number} elapsedMs
|
|
63
|
+
*
|
|
64
|
+
* @typedef {{kind: 'key', key: string}
|
|
65
|
+
* | {kind: 'type', text: string, submit?: boolean}
|
|
66
|
+
* | {kind: 'click', target: string | {x: number, y: number}}
|
|
67
|
+
* | {kind: 'press_action', action: string, durationMs?: number}
|
|
68
|
+
* | {kind: 'navigate', url: string}
|
|
69
|
+
* | {kind: 'wait', ms: number}} Action
|
|
70
|
+
*
|
|
71
|
+
* @typedef {Object} ActResult
|
|
72
|
+
* @property {boolean} ok
|
|
73
|
+
* @property {string} [error]
|
|
74
|
+
* @property {boolean} changed whether the observation hash moved
|
|
75
|
+
*
|
|
76
|
+
* @typedef {{type: string, text: string}} ConsoleEntry
|
|
77
|
+
*/
|
|
78
|
+
|
|
79
|
+
/** Observation text is truncated to this many characters (spec 03). */
|
|
80
|
+
export const OBSERVATION_MAX_CHARS = 12000;
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Under this much snapshot text the accessibility tree is not worth
|
|
84
|
+
* playing from, so the driver attaches a screenshot instead. Typical of a
|
|
85
|
+
* Godot or Unity web export, which is one canvas element and nothing else
|
|
86
|
+
* (spec 03, `playwright_web`).
|
|
87
|
+
*/
|
|
88
|
+
export const THIN_OBSERVATION_CHARS = 200;
|
|
89
|
+
|
|
90
|
+
/** The five methods every driver has to have. */
|
|
91
|
+
export const DRIVER_METHODS = Object.freeze(['launch', 'observe', 'act', 'screenshot', 'stop']);
|
|
92
|
+
|
|
93
|
+
/** Action kinds a driver may be asked for. A driver may refuse a kind it cannot do. */
|
|
94
|
+
export const ACTION_KINDS = Object.freeze(['key', 'type', 'click', 'press_action', 'navigate', 'wait']);
|
|
95
|
+
|
|
96
|
+
export class DriverError extends Error {
|
|
97
|
+
/**
|
|
98
|
+
* @param {string} message
|
|
99
|
+
* @param {{fatal?: boolean}} [opts] fatal means the driver itself is
|
|
100
|
+
* unusable, which ends the run as `error` rather than as a finding. A
|
|
101
|
+
* crashed *game* is a finding (spec 03); a crashed *driver* is not.
|
|
102
|
+
*/
|
|
103
|
+
constructor(message, { fatal = false } = {}) {
|
|
104
|
+
super(message);
|
|
105
|
+
this.name = 'DriverError';
|
|
106
|
+
this.fatal = fatal;
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* Refuse a driver that does not implement the interface at load time,
|
|
112
|
+
* rather than discovering it three actions into a paid run.
|
|
113
|
+
*
|
|
114
|
+
* @param {any} driver
|
|
115
|
+
* @param {string} name
|
|
116
|
+
* @returns {any} the same driver
|
|
117
|
+
*/
|
|
118
|
+
export function assertDriver(driver, name) {
|
|
119
|
+
if (!driver || typeof driver !== 'object') {
|
|
120
|
+
throw new DriverError(`driver ${name} did not return an object`, { fatal: true });
|
|
121
|
+
}
|
|
122
|
+
const missing = DRIVER_METHODS.filter(method => typeof driver[method] !== 'function');
|
|
123
|
+
if (missing.length > 0) {
|
|
124
|
+
throw new DriverError(`driver ${name} is missing: ${missing.join(', ')}`, { fatal: true });
|
|
125
|
+
}
|
|
126
|
+
return driver;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* The hash that decides `changed` and feeds the stuck detector. Whitespace
|
|
131
|
+
* is collapsed first, because a repainted but identical screen often
|
|
132
|
+
* differs only in trailing spaces and a stuck persona should still read as
|
|
133
|
+
* stuck.
|
|
134
|
+
*
|
|
135
|
+
* @param {Observation|string|null|undefined} observation
|
|
136
|
+
* @returns {string}
|
|
137
|
+
*/
|
|
138
|
+
export function observationHash(observation) {
|
|
139
|
+
const text = typeof observation === 'string' ? observation : String((observation && observation.text) || '');
|
|
140
|
+
const normalized = text.replace(/\s+/g, ' ').trim();
|
|
141
|
+
return createHash('sha256').update(normalized, 'utf8').digest('hex').slice(0, 32);
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* @param {string} text
|
|
146
|
+
* @param {number} [max]
|
|
147
|
+
* @returns {string}
|
|
148
|
+
*/
|
|
149
|
+
export function truncateObservation(text, max = OBSERVATION_MAX_CHARS) {
|
|
150
|
+
const value = String(text == null ? '' : text);
|
|
151
|
+
if (value.length <= max) return value;
|
|
152
|
+
return `${value.slice(0, max)}\n... [truncated, ${value.length - max} more characters]`;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* `screenshots/NN-slug.png`, the name the report schema's `screenshot`
|
|
157
|
+
* evidence refs point at. The number is the driver's own screenshot
|
|
158
|
+
* counter, so the files sort in the order they were taken.
|
|
159
|
+
*
|
|
160
|
+
* @param {number} index 1-based
|
|
161
|
+
* @param {string} name
|
|
162
|
+
* @returns {string}
|
|
163
|
+
*/
|
|
164
|
+
export function screenshotName(index, name) {
|
|
165
|
+
const slug = String(name || 'shot')
|
|
166
|
+
.toLowerCase()
|
|
167
|
+
.replace(/[^a-z0-9]+/g, '-')
|
|
168
|
+
.replace(/^-+|-+$/g, '')
|
|
169
|
+
.slice(0, 48) || 'shot';
|
|
170
|
+
return `${String(index).padStart(2, '0')}-${slug}.png`;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
export default {
|
|
174
|
+
OBSERVATION_MAX_CHARS,
|
|
175
|
+
THIN_OBSERVATION_CHARS,
|
|
176
|
+
DRIVER_METHODS,
|
|
177
|
+
ACTION_KINDS,
|
|
178
|
+
DriverError,
|
|
179
|
+
assertDriver,
|
|
180
|
+
observationHash,
|
|
181
|
+
truncateObservation,
|
|
182
|
+
screenshotName
|
|
183
|
+
};
|