ravensight-playtest 0.5.2 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +319 -14
- package/addons/ravensight_driver/driver.gd +406 -8
- package/addons/ravensight_simulate/simulate.gd +360 -0
- package/docs/godot-driver.md +70 -8
- package/package.json +4 -3
- package/src/api/README.md +4 -1
- package/src/api/client.js +120 -3
- package/src/api/errors.js +17 -1
- package/src/api/http.js +43 -17
- package/src/assets/formats.js +267 -0
- package/src/assets/index.js +260 -0
- package/src/assets/integration.js +429 -0
- package/src/assets/inventory.js +337 -0
- package/src/assets/measure.js +392 -0
- package/src/cli.js +233 -0
- package/src/commands/assets.js +62 -0
- package/src/commands/check.js +4 -1
- package/src/commands/create.js +734 -0
- package/src/commands/fakeCore.js +16 -1
- package/src/commands/feature.js +845 -0
- package/src/commands/fix.js +647 -70
- package/src/commands/init.js +19 -3
- package/src/commands/lists.js +206 -0
- package/src/commands/profile.js +25 -6
- package/src/commands/punch.js +116 -0
- package/src/commands/replay.js +204 -0
- package/src/commands/resume.js +7 -0
- package/src/commands/run.js +200 -24
- package/src/commands/tape.js +74 -0
- package/src/commands/template.js +70 -0
- package/src/commands/validate.js +414 -0
- package/src/commands/validateHosted.js +292 -0
- package/src/create/scaffold.js +192 -0
- package/src/run/acceptance.js +212 -0
- package/src/run/args.js +18 -1
- package/src/run/checks.js +705 -0
- package/src/run/dimension.js +95 -0
- package/src/run/drivers/godot-observation.js +59 -11
- package/src/run/drivers/godot-project.js +201 -31
- package/src/run/drivers/godot.js +532 -45
- package/src/run/drivers/web.js +18 -0
- package/src/run/evidence.js +170 -0
- package/src/run/featureLoop.js +288 -0
- package/src/run/fixLoop.js +82 -12
- package/src/run/git.js +110 -20
- package/src/run/json.js +65 -9
- package/src/run/labsOutcome.js +136 -0
- package/src/run/liveWork.js +103 -0
- package/src/run/model.js +105 -5
- package/src/run/openPr.js +107 -0
- package/src/run/paths.js +4 -1
- package/src/run/pendingRegister.js +81 -0
- package/src/run/personaLoop.js +111 -4
- package/src/run/profile.js +68 -7
- package/src/run/replay.js +236 -0
- package/src/run/repoTools.js +227 -27
- package/src/run/report.js +7 -1
- package/src/run/resume.js +19 -1
- package/src/run/secretScan.js +35 -0
- package/src/run/settle.js +41 -0
- package/src/run/tools.js +33 -4
- package/src/run/trace.js +124 -0
- package/src/simulation/godotPack.js +112 -0
- package/src/ui/index.js +61 -9
- package/src/upload/allowlist.js +6 -2
- package/src/upload/index.js +5 -4
- package/templates/godot-2d/README.md +36 -0
- package/templates/godot-2d/addons/ravensight/LICENSE +21 -0
- package/templates/godot-2d/addons/ravensight/Ravensight.gd +1823 -0
- package/templates/godot-2d/export_presets.cfg +27 -0
- package/templates/godot-2d/gitignore +10 -0
- package/templates/godot-2d/project.godot +58 -0
- package/templates/godot-2d/scenes/main.tscn +67 -0
- package/templates/godot-2d/scripts/autoload/game_config.gd +83 -0
- package/templates/godot-2d/scripts/autoload/game_state.gd +44 -0
- package/templates/godot-2d/scripts/autoload/telemetry.gd +37 -0
- package/templates/godot-2d/scripts/main.gd +32 -0
- package/templates/godot-2d/scripts/player.gd +10 -0
- package/templates/godot-2d/scripts/state_hook.gd +16 -0
- package/templates/godot-2d/template.json +14 -0
- package/templates/godot-2d/tools/inject_release_key.sh +19 -0
- package/templates/godot-2d/tools/selftest.gd +70 -0
- package/templates/godot-3d/README.md +49 -0
- package/templates/godot-3d/addons/ravensight/LICENSE +21 -0
- package/templates/godot-3d/addons/ravensight/Ravensight.gd +1823 -0
- package/templates/godot-3d/export_presets.cfg +27 -0
- package/templates/godot-3d/gitignore +10 -0
- package/templates/godot-3d/project.godot +56 -0
- package/templates/godot-3d/scenes/main.tscn +154 -0
- package/templates/godot-3d/scripts/autoload/game_config.gd +83 -0
- package/templates/godot-3d/scripts/autoload/game_state.gd +44 -0
- package/templates/godot-3d/scripts/autoload/telemetry.gd +37 -0
- package/templates/godot-3d/scripts/beacon.gd +20 -0
- package/templates/godot-3d/scripts/main.gd +49 -0
- package/templates/godot-3d/scripts/player.gd +19 -0
- package/templates/godot-3d/scripts/state_hook.gd +25 -0
- package/templates/godot-3d/template.json +16 -0
- package/templates/godot-3d/tools/inject_release_key.sh +19 -0
- package/templates/godot-3d/tools/selftest.gd +77 -0
package/src/run/drivers/web.js
CHANGED
|
@@ -1429,6 +1429,8 @@ export function createWebDriver({ importer, headless = true, playtest = null, pl
|
|
|
1429
1429
|
return {
|
|
1430
1430
|
name: 'playwright_web',
|
|
1431
1431
|
supportsRefs: true,
|
|
1432
|
+
// A key action may hold its keys (hold_ms, with_keys).
|
|
1433
|
+
supportsHold: true,
|
|
1432
1434
|
editModifier,
|
|
1433
1435
|
keepObservations: WEB_KEEP_OBSERVATIONS,
|
|
1434
1436
|
|
|
@@ -1613,6 +1615,22 @@ export function createWebDriver({ importer, headless = true, playtest = null, pl
|
|
|
1613
1615
|
try {
|
|
1614
1616
|
switch (action.kind) {
|
|
1615
1617
|
case 'key':
|
|
1618
|
+
if (action.holdMs > 0 || (Array.isArray(action.keys) && action.keys.length > 0)) {
|
|
1619
|
+
// Held keys go down together and come up in reverse, even when
|
|
1620
|
+
// one is refused, so nothing stays held for the rest of the run.
|
|
1621
|
+
const keys = [action.key, ...(action.keys || [])].map(key => normalizeChord(key, editModifier));
|
|
1622
|
+
const down = [];
|
|
1623
|
+
try {
|
|
1624
|
+
for (const key of keys) {
|
|
1625
|
+
await current.keyboard.down(key);
|
|
1626
|
+
down.push(key);
|
|
1627
|
+
}
|
|
1628
|
+
await current.waitForTimeout(Math.min(Math.max(Number(action.holdMs) || 0, 0), MAX_WAIT_MS));
|
|
1629
|
+
} finally {
|
|
1630
|
+
for (const key of down.reverse()) await current.keyboard.up(key).catch(() => {});
|
|
1631
|
+
}
|
|
1632
|
+
break;
|
|
1633
|
+
}
|
|
1616
1634
|
await current.keyboard.press(normalizeChord(action.key, editModifier), { timeout: ACTION_TIMEOUT_MS });
|
|
1617
1635
|
break;
|
|
1618
1636
|
case 'type':
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Fix v2's evidence report: the pull request body, followed by what was
|
|
3
|
+
* measured and what was judged, kept apart.
|
|
4
|
+
*
|
|
5
|
+
* "Verified facts" are things the CLI or git observed: which checks ran and
|
|
6
|
+
* what they printed, which files changed. "Model judgement" is everything a
|
|
7
|
+
* model wrote or scored: the hypothesis, the description, and the verify
|
|
8
|
+
* score, which is a judge's reading of a persona's report rather than a
|
|
9
|
+
* measurement. A reviewer should be able to tell the two apart at a glance.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
/** How a judged `reproduced` score reads; the same thresholds `fix` uses. */
|
|
13
|
+
function scoreWords(verify, { fixedBelow, reproducedAbove }) {
|
|
14
|
+
if (!verify) return 'not run';
|
|
15
|
+
if (verify.failed === 'game_boot_failed') return 'not verified, the game did not boot';
|
|
16
|
+
if (verify.failed) return 'not verified, the verify run failed';
|
|
17
|
+
const score = verify.reproduced;
|
|
18
|
+
if (typeof score !== 'number' || !Number.isFinite(score)) return 'no verdict';
|
|
19
|
+
if (score < fixedBelow) return `fixed (reproduced ${score})`;
|
|
20
|
+
if (score > reproducedAbove) return `still reproduced (${score})`;
|
|
21
|
+
return `unknown (reproduced ${score})`;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function cell(value) {
|
|
25
|
+
return String(value ?? '').replace(/\|/g, '\\|').replace(/\r?\n/g, ' ');
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* The risks a reviewer should know about, computed from what happened rather
|
|
30
|
+
* than written by a model.
|
|
31
|
+
*
|
|
32
|
+
* @returns {Array<string>}
|
|
33
|
+
*/
|
|
34
|
+
export function remainingRisks({ checks = [], verifies = [], wantsVerify = true, budgetTouched = false, repairs = 0, thresholds }) {
|
|
35
|
+
const risks = [];
|
|
36
|
+
const latest = new Map();
|
|
37
|
+
for (const result of checks) latest.set(result.name, result);
|
|
38
|
+
for (const result of latest.values()) {
|
|
39
|
+
if (result.status === 'unavailable') risks.push(`${result.name} could not run: ${result.detail}.`);
|
|
40
|
+
if (result.status === 'failed') risks.push(`${result.name} still fails on the final change: ${result.detail}.`);
|
|
41
|
+
if (result.status === 'check_refused') risks.push(`${result.name} was refused on the final change: ${result.detail}.`);
|
|
42
|
+
}
|
|
43
|
+
if (!latest.has('godot_import')) risks.push('The project import check did not run on the final change.');
|
|
44
|
+
const last = verifies.at(-1) || null;
|
|
45
|
+
if (!wantsVerify) risks.push('The fix was not verified (--no-verify).');
|
|
46
|
+
else if (!last) risks.push('No verify run played the final change.');
|
|
47
|
+
else if (last.failed) risks.push('The last verify run did not play the change, so the fix is unverified.');
|
|
48
|
+
else {
|
|
49
|
+
const words = scoreWords(last, thresholds);
|
|
50
|
+
if (!words.startsWith('fixed')) risks.push(`The last verify reads ${words}.`);
|
|
51
|
+
}
|
|
52
|
+
if (verifies.length > 0) risks.push('Verify is a persona replaying the repro steps and a judge scoring its report: a probability, not a deterministic replay.');
|
|
53
|
+
if (repairs > 0) risks.push(`The change needed ${repairs} repair turn set(s) before this result; read the whole diff, not just the first edit.`);
|
|
54
|
+
if (budgetTouched) risks.push('The fix stopped early on its model budget, so the change may be incomplete.');
|
|
55
|
+
return risks;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* The checks still failing on the final change, by name. A check that failed and
|
|
60
|
+
* then passed after a repair is not one of them.
|
|
61
|
+
*
|
|
62
|
+
* @param {Array<Object>} checks
|
|
63
|
+
* @returns {Array<string>}
|
|
64
|
+
*/
|
|
65
|
+
export function checksStillFailing(checks = []) {
|
|
66
|
+
const latest = new Map();
|
|
67
|
+
for (const result of checks) latest.set(result.name, result);
|
|
68
|
+
return [...latest.values()].filter(result => result.status === 'failed' || result.status === 'check_refused').map(result => result.name);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* @param {Object} args
|
|
73
|
+
* @returns {string} markdown
|
|
74
|
+
*/
|
|
75
|
+
export function evidenceReport({
|
|
76
|
+
prBody,
|
|
77
|
+
summarySource = 'model',
|
|
78
|
+
acceptance,
|
|
79
|
+
finding,
|
|
80
|
+
sourceRunId = null,
|
|
81
|
+
plan = null,
|
|
82
|
+
files = [],
|
|
83
|
+
checks = [],
|
|
84
|
+
verifies = [],
|
|
85
|
+
repairs = 0,
|
|
86
|
+
maxRepairs = 2,
|
|
87
|
+
wantsVerify = true,
|
|
88
|
+
budgetTouched = false,
|
|
89
|
+
thresholds
|
|
90
|
+
}) {
|
|
91
|
+
const label = finding.label || finding.finding_id;
|
|
92
|
+
// First, above anything a model wrote, so no summary and no clamp on the body
|
|
93
|
+
// can leave a reviewer thinking this change passed.
|
|
94
|
+
const stillFailing = checksStillFailing(checks);
|
|
95
|
+
const lines = [
|
|
96
|
+
...(stillFailing.length > 0 ? [`**Checks failed on this change: ${stillFailing.join(', ')}.** Details are under Evidence below.`, ''] : []),
|
|
97
|
+
String(prBody || '').trimEnd(), '', '## Evidence', ''
|
|
98
|
+
];
|
|
99
|
+
|
|
100
|
+
lines.push('### Acceptance contract', '', 'Frozen on the job before the first edit. The fix was judged against this alone.', '');
|
|
101
|
+
lines.push(`- Expected: ${acceptance.expected}`);
|
|
102
|
+
lines.push(`- Actual: ${acceptance.actual}`);
|
|
103
|
+
lines.push(`- Repro steps: ${acceptance.reproSteps.length > 0 ? acceptance.reproSteps.map((step, index) => `${index + 1}. ${step}`).join(' ') : 'none recorded'}`);
|
|
104
|
+
lines.push(`- Verify and regression personas: ${acceptance.regressionPersonas.join(', ') || 'none'}`, '');
|
|
105
|
+
|
|
106
|
+
lines.push('### Verified facts', '');
|
|
107
|
+
lines.push('Measured by this CLI and git, not judged by a model.', '');
|
|
108
|
+
if (checks.length > 0) {
|
|
109
|
+
lines.push('Checks, in the order they ran (each on a temporary copy of the project):', '');
|
|
110
|
+
lines.push('| Check | Result | Time | Detail |', '| --- | --- | --- | --- |');
|
|
111
|
+
for (const result of checks) {
|
|
112
|
+
lines.push(`| ${cell(result.name)} | ${cell(result.status)} | ${cell(`${Math.round((result.durationMs || 0) / 100) / 10} s`)} | ${cell(result.detail)} |`);
|
|
113
|
+
}
|
|
114
|
+
const failedErrors = checks.filter(result => result.status === 'failed' && result.errors && result.errors.length > 0).at(-1);
|
|
115
|
+
if (failedErrors) {
|
|
116
|
+
lines.push('', `Errors from the last failing check (${failedErrors.name}):`, '');
|
|
117
|
+
for (const error of failedErrors.errors.slice(0, 10)) lines.push(`- \`${error.replace(/`/g, '\'')}\``);
|
|
118
|
+
}
|
|
119
|
+
} else {
|
|
120
|
+
lines.push('No checks ran.');
|
|
121
|
+
}
|
|
122
|
+
lines.push('', 'Files changed:', '');
|
|
123
|
+
for (const file of files) lines.push(`- \`${file.path}\`: +${file.additions} -${file.deletions}`);
|
|
124
|
+
if (files.length === 0) lines.push('- none');
|
|
125
|
+
lines.push('', `Repair turn sets used: ${repairs} of ${maxRepairs}.`, '');
|
|
126
|
+
|
|
127
|
+
lines.push('### Before and after', '');
|
|
128
|
+
lines.push(`- Before: ${label} was filed${finding.persona ? ` by ${finding.persona}` : ''}${sourceRunId ? ` in run ${sourceRunId}` : ''}, which saw the actual behaviour above.`);
|
|
129
|
+
if (verifies.length === 0) lines.push(`- After: ${wantsVerify ? 'no verify run played the change' : 'not verified (--no-verify)'}.`);
|
|
130
|
+
verifies.forEach((verify, index) => {
|
|
131
|
+
lines.push(`- After, verify ${index + 1}${verify && verify.run_id ? ` (run ${verify.run_id})` : ''}: ${scoreWords(verify, thresholds)}.`);
|
|
132
|
+
});
|
|
133
|
+
lines.push('');
|
|
134
|
+
|
|
135
|
+
lines.push('### Model judgement', '');
|
|
136
|
+
lines.push('Written or scored by a model. Read the diff before trusting any of it.', '');
|
|
137
|
+
if (plan && plan.hypothesis) lines.push(`- Hypothesis: ${plan.hypothesis}`);
|
|
138
|
+
lines.push(`- The description at the top was written ${summarySource === 'model' ? 'by the model from the diff' : 'locally, because the model could not write it'}.`);
|
|
139
|
+
if (verifies.some(verify => verify && !verify.failed)) lines.push('- Each verify score is a judge model\'s probability, from 0 to 1, that the finding is still present, from reading the verify persona\'s report against the acceptance contract. It is not a percentage of actions or attempts.');
|
|
140
|
+
lines.push('');
|
|
141
|
+
|
|
142
|
+
lines.push('### Remaining risks', '');
|
|
143
|
+
const risks = remainingRisks({ checks, verifies, wantsVerify, budgetTouched, repairs, thresholds });
|
|
144
|
+
for (const risk of risks) lines.push(`- ${risk}`);
|
|
145
|
+
if (risks.length === 0) lines.push('- None found by the checks above.');
|
|
146
|
+
lines.push('');
|
|
147
|
+
return lines.join('\n');
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* The report with the model's text fitted around the evidence rather than the other
|
|
152
|
+
* way round. Evidence comes after the model's description and the whole is cut at the
|
|
153
|
+
* changeset's limit, so a long enough description would cut every measured fact off
|
|
154
|
+
* the end, and could put its own "Verified facts" where they had been. So the
|
|
155
|
+
* measured part is sized first, and the headings only it may use are taken back.
|
|
156
|
+
*
|
|
157
|
+
* @param {(prBody: string) => string} render the report around a given description
|
|
158
|
+
* @param {string} prBody the model's text
|
|
159
|
+
* @param {number} [limit]
|
|
160
|
+
* @returns {string}
|
|
161
|
+
*/
|
|
162
|
+
export function withEvidenceIntact(render, prBody, limit = 20000) {
|
|
163
|
+
const text = String(prBody || '').replace(/^([ \t]{0,3}#{1,6}[ \t]*)(evidence|verified facts)\b/gim, '$1From the model: $2');
|
|
164
|
+
const note = '\n\n(the description was cut to keep the evidence below it whole)';
|
|
165
|
+
const room = Math.max(0, limit - render('').length - note.length);
|
|
166
|
+
const whole = render(text.length <= room ? text : `${text.slice(0, room)}${note}`);
|
|
167
|
+
return whole.length <= limit ? whole : `${whole.slice(0, limit - 40)}\n\n(cut at the changeset's size limit)\n`;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
export default { evidenceReport, remainingRisks, checksStillFailing, withEvidenceIntact };
|
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
import { readFile } from 'node:fs/promises';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
|
|
4
|
+
import { DIMENSION_3D } from './dimension.js';
|
|
5
|
+
import { parseJsonObject } from './json.js';
|
|
6
|
+
import { callStepWithRetry, ProxyError, STEPS } from './model.js';
|
|
7
|
+
import { applyFix, bundleLines, clamp, planFix, SUMMARY_MAX_TOKENS } from './fixLoop.js';
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* The `feature` module's model steps, and the words around them. The loops
|
|
11
|
+
* themselves are `planFix` and `applyFix`, with their guards and budget stops,
|
|
12
|
+
* because the safety of an edit loop is the same whatever it is editing. What
|
|
13
|
+
* differs is the instruction, the frozen specification it is held to, and the
|
|
14
|
+
* write cap the caller hands the repository tools.
|
|
15
|
+
*
|
|
16
|
+
* The specification is frozen on the job by the server. This file reads the copy
|
|
17
|
+
* the job came back with, and nothing here can change what counts as done.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
/** The feature module's own ceiling, and only this module's: the large size. */
|
|
21
|
+
export const MAX_FEATURE_WRITES = 40;
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* How many files each size was priced for. The same numbers as the server's
|
|
25
|
+
* `FILES_FOR_SIZE` (src/playtest/featureSpec.js), which refuses a changeset above
|
|
26
|
+
* them, and a test holds the two together when a server checkout is named.
|
|
27
|
+
*/
|
|
28
|
+
export const FILES_FOR_SIZE = Object.freeze({ small: 8, medium: 20, large: MAX_FEATURE_WRITES });
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* The write cap for a job, from the size frozen on it. A size this CLI has never
|
|
32
|
+
* heard of gets the smallest, because a cap that is too low costs a refused write
|
|
33
|
+
* the model can read, and one that is too high costs a refused changeset.
|
|
34
|
+
*
|
|
35
|
+
* @param {string} size
|
|
36
|
+
* @returns {number}
|
|
37
|
+
*/
|
|
38
|
+
export function maxWritesForSize(size) {
|
|
39
|
+
return Object.hasOwn(FILES_FOR_SIZE, size) ? FILES_FOR_SIZE[size] : FILES_FOR_SIZE.small;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** The hard ceiling on apply turns, whatever the developer asks for. */
|
|
43
|
+
export const MAX_FEATURE_ACTIONS = 120;
|
|
44
|
+
|
|
45
|
+
const applyInstructions = primitives => Object.freeze([
|
|
46
|
+
'Build the feature now. Read before you edit. Change an existing file with `edit_file` (old_text copied verbatim, new_text in its place); use `write_file` for a new file, because it replaces the whole file.',
|
|
47
|
+
`Keep the style of the code around you and stay inside the specification. Art and audio belong to the studio: use what the project has, otherwise a placeholder primitive such as ${primitives}, and never suggest a tool or service for them. If the feature is already there or cannot be built here, change nothing and say why in one line. When you are done, say in one line what you built and call no more tools.`
|
|
48
|
+
]);
|
|
49
|
+
|
|
50
|
+
const APPLY_INSTRUCTIONS = applyInstructions('a ColorRect, a Polygon2D or a Label');
|
|
51
|
+
const APPLY_INSTRUCTIONS_3D = applyInstructions('a MeshInstance3D with a BoxMesh, a SphereMesh or a CapsuleMesh, or a Label');
|
|
52
|
+
|
|
53
|
+
/** What the job says about the engine. 3D is the Alpha, and says so to the model. */
|
|
54
|
+
export function engineLine(dimension) {
|
|
55
|
+
return dimension === DIMENSION_3D
|
|
56
|
+
? 'engine: Godot, 3D (Alpha). Where the skill says 2D, read 3D: positions are Vector3 in world units, and a persona plays with keys and input actions, so build nothing a scenario needs mouse look for'
|
|
57
|
+
: 'engine: Godot, 2D';
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* The system prompt every feature step shares.
|
|
62
|
+
*
|
|
63
|
+
* @param {{skill: string, jobId: string, branch: string, baseCommit: string, maxWrites?: number, dimension?: string}} input
|
|
64
|
+
* @returns {Array<Object>}
|
|
65
|
+
*/
|
|
66
|
+
export function featureSystem({ skill, jobId, branch, baseCommit, maxWrites = MAX_FEATURE_WRITES, dimension = '2d' }) {
|
|
67
|
+
return [
|
|
68
|
+
{ type: 'text', text: skill },
|
|
69
|
+
{
|
|
70
|
+
type: 'text',
|
|
71
|
+
text: [
|
|
72
|
+
`job_id: ${jobId}`,
|
|
73
|
+
engineLine(dimension),
|
|
74
|
+
'the paths you read and write are relative to the repository root; never absolute',
|
|
75
|
+
`branch: ${branch} (created for this attempt, from ${baseCommit || 'unknown'})`,
|
|
76
|
+
`at most ${maxWrites} files may be written`,
|
|
77
|
+
'you cannot run commands, reach the network, or commit anything, except the named checks the run_check tool offers. The developer reviews the branch.'
|
|
78
|
+
].join('\n'),
|
|
79
|
+
cache_control: { type: 'ephemeral' }
|
|
80
|
+
}
|
|
81
|
+
];
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** The frozen specification, as the data block both the plan and the apply read. */
|
|
85
|
+
export function specBlock(spec) {
|
|
86
|
+
return `<spec>\n${JSON.stringify(spec, null, 2)}\n</spec>`;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* The user message the plan step reads.
|
|
91
|
+
*
|
|
92
|
+
* @param {{spec: Object, codeBrief?: Object|null}} input
|
|
93
|
+
* @returns {Array<Object>} content blocks
|
|
94
|
+
*/
|
|
95
|
+
export function featurePlanContent({ spec, codeBrief = null, assetLines = [] }) {
|
|
96
|
+
return [{
|
|
97
|
+
type: 'text',
|
|
98
|
+
text: [
|
|
99
|
+
'A person approved this feature specification. Read the code it touches and plan how to build it.',
|
|
100
|
+
'',
|
|
101
|
+
specBlock(spec),
|
|
102
|
+
...bundleLines(codeBrief, null),
|
|
103
|
+
...assetLines,
|
|
104
|
+
'',
|
|
105
|
+
'Answer with this JSON object and nothing else:',
|
|
106
|
+
'{ "hypothesis": "one or two sentences on how you will build it", "files_to_change": ["repo/relative/path"], "out_of_scope": ["what you are deliberately not touching"], "placeholders": ["art or audio the project lacks, and the primitive that stands in for it"] }'
|
|
107
|
+
].join('\n')
|
|
108
|
+
}];
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/** The plan step, on this module's route. */
|
|
112
|
+
export function planFeature(args) {
|
|
113
|
+
return planFix({ ...args, step: STEPS.featurePlan });
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/** One apply turn set: the first, or a repair. */
|
|
117
|
+
export function applyFeature({ spec, assetInstructions = [], dimension = '2d', ...args }) {
|
|
118
|
+
return applyFix({
|
|
119
|
+
...args,
|
|
120
|
+
step: STEPS.featureApply,
|
|
121
|
+
instructions: [...(dimension === DIMENSION_3D ? APPLY_INSTRUCTIONS_3D : APPLY_INSTRUCTIONS), ...assetInstructions],
|
|
122
|
+
contract: specBlock(spec)
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* The pull request text, written from the diff that exists. `fallback` answers
|
|
128
|
+
* when the step cannot: no budget left, or no JSON in the reply.
|
|
129
|
+
*
|
|
130
|
+
* @returns {Promise<{prTitle: string, prBody: string, source: string}>}
|
|
131
|
+
*/
|
|
132
|
+
export async function summariseFeature({ client, jobId, runId, system, spec, plan, files, branch, fallback, ledger = null, log = () => {} }) {
|
|
133
|
+
const content = [{
|
|
134
|
+
type: 'text',
|
|
135
|
+
text: [
|
|
136
|
+
'Write the pull request text for the feature you just built. The reviewer owns this code. Never say it was verified: that part is added from the server\'s record.',
|
|
137
|
+
'',
|
|
138
|
+
specBlock(spec),
|
|
139
|
+
`<plan>\n${JSON.stringify(plan || {}, null, 2)}\n</plan>`,
|
|
140
|
+
`<diff_stat>\n${files.map(file => `${file.path}: +${file.additions} -${file.deletions}`).join('\n') || 'nothing'}\n</diff_stat>`,
|
|
141
|
+
`<branch>${branch}</branch>`,
|
|
142
|
+
'',
|
|
143
|
+
'Answer with this JSON object and nothing else:',
|
|
144
|
+
'{ "pr_title": "one line, at most 120 characters", "pr_body": "markdown: what was built, which requirements it meets, what the reviewer should check by hand, and what was not done" }'
|
|
145
|
+
].join('\n')
|
|
146
|
+
}];
|
|
147
|
+
let answer;
|
|
148
|
+
try {
|
|
149
|
+
answer = await callStepWithRetry({ client, jobId, runId, step: STEPS.featureSummarize, system, messages: [{ role: 'user', content }], maxTokens: SUMMARY_MAX_TOKENS });
|
|
150
|
+
} catch (error) {
|
|
151
|
+
if (error instanceof ProxyError && error.isBudget) {
|
|
152
|
+
log('There was no budget left to write the pull request text, so it was written locally instead.');
|
|
153
|
+
return { ...fallback, source: 'local' };
|
|
154
|
+
}
|
|
155
|
+
// The feature exists and has been played. Prose is the one thing left, and
|
|
156
|
+
// losing it must not lose the changeset or fail a job that worked.
|
|
157
|
+
log(`The pull request text could not be written by the model (${String(error && error.message)}), so it was written locally instead.`);
|
|
158
|
+
return { ...fallback, source: 'local' };
|
|
159
|
+
}
|
|
160
|
+
if (ledger) ledger.record({ model: answer.model, usage: answer.usage });
|
|
161
|
+
const parsed = parseJsonObject(answer.text) || {};
|
|
162
|
+
const title = typeof parsed.pr_title === 'string' && parsed.pr_title.trim() ? parsed.pr_title.trim() : fallback.prTitle;
|
|
163
|
+
const body = typeof parsed.pr_body === 'string' && parsed.pr_body.trim() ? parsed.pr_body : fallback.prBody;
|
|
164
|
+
return { prTitle: clamp(title, 120), prBody: clamp(body, 20000), source: parsed.pr_title || parsed.pr_body ? 'model' : 'local' };
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* What the persona playing one scenario is asked to do. Built from the frozen
|
|
169
|
+
* specification only, in front of the brief a persona usually plays against, so
|
|
170
|
+
* the task is the first thing it reads and the game's goals stay beside it.
|
|
171
|
+
*
|
|
172
|
+
* @param {Object|null} base the game's expectations brief
|
|
173
|
+
* @param {Object} scenario from the frozen specification
|
|
174
|
+
* @param {Object} spec
|
|
175
|
+
* @returns {Object}
|
|
176
|
+
*/
|
|
177
|
+
export function scenarioBrief(base, scenario, spec, controls = null) {
|
|
178
|
+
return {
|
|
179
|
+
...(base || {}),
|
|
180
|
+
fields: {
|
|
181
|
+
feature_under_test: `${spec.title}: ${spec.summary}`,
|
|
182
|
+
...(controls ? { controls } : {}),
|
|
183
|
+
scenario_task: scenario.title,
|
|
184
|
+
scenario_steps: scenario.steps,
|
|
185
|
+
scenario_expected: scenario.expected,
|
|
186
|
+
scenario_instruction: 'Do these steps first. If what you see is not what is expected, file it as a major finding that says what you saw. Then stop.',
|
|
187
|
+
...((base && base.fields) || {})
|
|
188
|
+
}
|
|
189
|
+
};
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/**
|
|
193
|
+
* A plain local pull request body, when the model could not write one.
|
|
194
|
+
*
|
|
195
|
+
* @returns {{prTitle: string, prBody: string}}
|
|
196
|
+
*/
|
|
197
|
+
export function localFeatureSummary({ spec, plan, files, branch }) {
|
|
198
|
+
return {
|
|
199
|
+
prTitle: `Add ${spec.title}`.slice(0, 120),
|
|
200
|
+
prBody: [
|
|
201
|
+
`# ${spec.title}`,
|
|
202
|
+
'',
|
|
203
|
+
spec.summary,
|
|
204
|
+
'',
|
|
205
|
+
plan && plan.hypothesis ? `${plan.hypothesis}\n` : '',
|
|
206
|
+
'## What changed',
|
|
207
|
+
...files.map(file => `- \`${file.path}\`: +${file.additions} -${file.deletions}`),
|
|
208
|
+
'',
|
|
209
|
+
'This summary was written locally rather than by the model, so read the diff before trusting it.',
|
|
210
|
+
'',
|
|
211
|
+
`Branch: ${branch}`,
|
|
212
|
+
''
|
|
213
|
+
].filter(line => line !== '').join('\n')
|
|
214
|
+
};
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
/**
|
|
218
|
+
* Whether each SDK event the specification lists is named in a changed file. A
|
|
219
|
+
* measurement, not a judgement: it reads the files, so "named" means the string
|
|
220
|
+
* is there, not that the game reports it at the right moment.
|
|
221
|
+
*
|
|
222
|
+
* @param {{spec: Object, files: Array<{path: string}>, repoRoot: string}} input
|
|
223
|
+
* @returns {Promise<Array<{name: string, found: boolean}>>}
|
|
224
|
+
*/
|
|
225
|
+
export async function eventsInChange({ spec, files, repoRoot }) {
|
|
226
|
+
const texts = await Promise.all(files.map(file => readFile(path.join(repoRoot, file.path), 'utf8').catch(() => '')));
|
|
227
|
+
return (spec.sdk_events || []).map(event => ({ name: event.name, found: texts.some(text => text.includes(event.name)) }));
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
function cell(value) {
|
|
231
|
+
return String(value ?? '').replace(/\|/g, '\\|').replace(/\r?\n/g, ' ');
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
/**
|
|
235
|
+
* The evidence report: the pull request body, then what was measured, kept apart
|
|
236
|
+
* from what a model wrote. The verdict is the server's words, and a placeholder
|
|
237
|
+
* is named so a reviewer knows what art or audio is still theirs to supply.
|
|
238
|
+
*
|
|
239
|
+
* @param {Object} input
|
|
240
|
+
* @returns {string} markdown
|
|
241
|
+
*/
|
|
242
|
+
export function featureEvidence({ prBody, summarySource, spec, report, files, checks, repairs, maxRepairs, placeholders, events, staleNotes = [], budgetTouched = false, wantedVerify = true, assetLines = [] }) {
|
|
243
|
+
const latest = new Map();
|
|
244
|
+
for (const result of checks) latest.set(result.name, result);
|
|
245
|
+
const out = [String(prBody || '').trimEnd(), '', '---', '', '## Evidence', ''];
|
|
246
|
+
out.push(report ? report.verification_note : 'The server\'s report could not be read, so this change reads as not verified.', '');
|
|
247
|
+
|
|
248
|
+
out.push('### Test scenarios (played by model personas, read from the server\'s own records)', '', '| Scenario | Answer | Why |', '|---|---|---|');
|
|
249
|
+
for (const scenario of (report ? report.scenarios : [])) {
|
|
250
|
+
out.push(`| ${cell(scenario.title)} | ${scenario.passed ? 'passed' : (scenario.played ? 'not passed' : 'not played')} | ${cell(scenario.reason || '')} |`);
|
|
251
|
+
}
|
|
252
|
+
if (!wantedVerify) out.push('', 'The scenarios were not played.');
|
|
253
|
+
for (const note of staleNotes) out.push('', note);
|
|
254
|
+
|
|
255
|
+
out.push('', '### Verified facts', '', '| Check | Result | Detail |', '|---|---|---|');
|
|
256
|
+
for (const result of latest.values()) out.push(`| ${cell(result.name)} | ${cell(result.status)} | ${cell(result.detail)} |`);
|
|
257
|
+
if (latest.size === 0) out.push('| none ran | | |');
|
|
258
|
+
out.push('', `${files.length} file(s) changed. ${repairs} of ${maxRepairs} repair(s) used.`, '');
|
|
259
|
+
out.push(...files.map(file => `- \`${file.path}\`: +${file.additions} -${file.deletions}`), '');
|
|
260
|
+
out.push('### SDK events the specification lists', '');
|
|
261
|
+
if (events.length === 0) out.push('None.');
|
|
262
|
+
else out.push(...events.map(event => `- \`${event.name}\`: ${event.found ? 'named in a changed file' : 'NOT named in any changed file'}`));
|
|
263
|
+
|
|
264
|
+
out.push('', ...assetLines);
|
|
265
|
+
out.push('', '### Placeholders', '');
|
|
266
|
+
if (placeholders.length === 0) out.push('None named. Studios supply their own art and audio.');
|
|
267
|
+
else out.push('These stand in for art or audio the project does not have. They are yours to replace with your own assets:', '', ...placeholders.map(item => `- ${item}`));
|
|
268
|
+
|
|
269
|
+
out.push('', '### Remaining risks', '');
|
|
270
|
+
const risks = [];
|
|
271
|
+
for (const result of latest.values()) {
|
|
272
|
+
if (result.status === 'unavailable') risks.push(`${result.name} could not run: ${result.detail}.`);
|
|
273
|
+
if (result.status === 'failed') risks.push(`${result.name} still fails on the final change: ${result.detail}.`);
|
|
274
|
+
if (result.status === 'check_refused') risks.push(`${result.name} was refused on the final change: ${result.detail}.`);
|
|
275
|
+
}
|
|
276
|
+
if (repairs > 0) risks.push(`The change needed ${repairs} repair turn set(s); read the whole diff, not just the first edit.`);
|
|
277
|
+
if (budgetTouched) risks.push('The feature stopped early on its model budget, so it may be incomplete.');
|
|
278
|
+
const missing = events.filter(event => !event.found).map(event => event.name);
|
|
279
|
+
if (missing.length) risks.push(`The specification lists SDK events no changed file names: ${missing.join(', ')}.`);
|
|
280
|
+
risks.push('A persona passing a scenario is a model playing the game, not a player. Play it yourself before you ship it.');
|
|
281
|
+
out.push(...risks.map(risk => `- ${risk}`), '');
|
|
282
|
+
|
|
283
|
+
out.push('### Model judgement', '', `The description above was written by ${summarySource === 'model' ? 'the model' : 'this CLI, because the model could not'}. The acceptance criteria below were approved by a person and frozen on the job; nothing above changes them.`, '');
|
|
284
|
+
out.push(...spec.acceptance_criteria.map(item => `- ${item}`), '');
|
|
285
|
+
return out.join('\n');
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
export default { featureSystem, featurePlanContent, planFeature, applyFeature, summariseFeature, scenarioBrief, localFeatureSummary, eventsInChange, featureEvidence, MAX_FEATURE_WRITES, MAX_FEATURE_ACTIONS };
|