ravensight-playtest 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +380 -0
- package/addons/ravensight_driver/driver.gd +836 -0
- package/addons/ravensight_driver/export_plugin.gd +51 -0
- package/addons/ravensight_driver/plugin.cfg +7 -0
- package/addons/ravensight_driver/plugin.gd +36 -0
- package/bin/ravensight-playtest.js +31 -0
- package/package.json +45 -0
- package/src/api/README.md +500 -0
- package/src/api/client.js +340 -0
- package/src/api/errors.js +115 -0
- package/src/api/http.js +194 -0
- package/src/api/index.js +107 -0
- package/src/auth/deviceCode.js +79 -0
- package/src/auth/keychain.js +159 -0
- package/src/auth/session.js +128 -0
- package/src/cli.js +335 -0
- package/src/commands/brief.js +303 -0
- package/src/commands/check.js +318 -0
- package/src/commands/fakeCore.js +379 -0
- package/src/commands/init.js +120 -0
- package/src/commands/login.js +90 -0
- package/src/commands/logout.js +70 -0
- package/src/commands/open.js +125 -0
- package/src/commands/profile.js +262 -0
- package/src/commands/resume.js +156 -0
- package/src/commands/run.js +1015 -0
- package/src/commands/upload.js +137 -0
- package/src/config.js +100 -0
- package/src/dashboard.js +97 -0
- package/src/detect.js +77 -0
- package/src/errors.js +44 -0
- package/src/fsutil.js +77 -0
- package/src/godot.js +85 -0
- package/src/packs/index.js +191 -0
- package/src/paths.js +129 -0
- package/src/run/aggregate.js +658 -0
- package/src/run/args.js +111 -0
- package/src/run/context.js +181 -0
- package/src/run/deps.js +184 -0
- package/src/run/drivers/driver.js +183 -0
- package/src/run/drivers/godot-observation.js +138 -0
- package/src/run/drivers/godot-project.js +475 -0
- package/src/run/drivers/godot-rpc.js +225 -0
- package/src/run/drivers/godot.js +587 -0
- package/src/run/drivers/index.js +52 -0
- package/src/run/drivers/web.js +385 -0
- package/src/run/exit.js +21 -0
- package/src/run/heartbeat.js +131 -0
- package/src/run/index.js +31 -0
- package/src/run/json.js +56 -0
- package/src/run/model.js +384 -0
- package/src/run/paths.js +88 -0
- package/src/run/personaLoop.js +871 -0
- package/src/run/profile.js +214 -0
- package/src/run/regenerate.js +149 -0
- package/src/run/repoTools.js +286 -0
- package/src/run/report.js +222 -0
- package/src/run/resume.js +272 -0
- package/src/run/secretScan.js +171 -0
- package/src/run/state.js +198 -0
- package/src/run/synthetic.js +206 -0
- package/src/run/tools.js +344 -0
- package/src/run/transcript.js +93 -0
- package/src/run/usage.js +115 -0
- package/src/state/index.js +105 -0
- package/src/states.js +104 -0
- package/src/ui/index.js +195 -0
- package/src/upload/allowlist.js +116 -0
- package/src/upload/index.js +467 -0
- package/src/upload/queue.js +114 -0
- package/src/version.js +63 -0
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
import { writeFile } from 'node:fs/promises';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import Ajv2020 from 'ajv/dist/2020.js';
|
|
4
|
+
import addFormats from 'ajv-formats';
|
|
5
|
+
import { callStepWithRetry, STEPS } from './model.js';
|
|
6
|
+
import { JOB_FILES } from './paths.js';
|
|
7
|
+
import { parseJsonObject } from './json.js';
|
|
8
|
+
import { createRepoTools, PROFILE_TOOLS, READ_ONLY_TOOLS } from './repoTools.js';
|
|
9
|
+
import { assertNoSecrets } from './secretScan.js';
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* The `game_profile` module: read the developer's repository and answer what
|
|
13
|
+
* Ravensight Playtest can and cannot do for this game, before they spend
|
|
14
|
+
* money on a run.
|
|
15
|
+
*
|
|
16
|
+
* Two steps, because they are priced and tooled differently:
|
|
17
|
+
*
|
|
18
|
+
* - `game_profile.triage` (cheap tier, `read_file` / `glob` / `grep`) walks
|
|
19
|
+
* the repository and gathers facts. This is the loop, and it is where
|
|
20
|
+
* nearly all the turns go.
|
|
21
|
+
* - `game_profile.synthesis` (frontier tier, `read_file` only) turns those
|
|
22
|
+
* facts into `capability-report.json`.
|
|
23
|
+
*
|
|
24
|
+
* Neither step has a `write_file` tool, so the report comes back as text and
|
|
25
|
+
* is parsed here. That is the routing table's decision, not a shortcut:
|
|
26
|
+
* profiling reads a repository, and a step that can read a repository is the
|
|
27
|
+
* last one that should also be able to write to it.
|
|
28
|
+
*
|
|
29
|
+
* The source never leaves the machine. What goes over the wire is whatever
|
|
30
|
+
* the model quotes into its own reasoning, which is why ./repoTools.js
|
|
31
|
+
* withholds any file that trips the secret scan rather than trusting the
|
|
32
|
+
* prompt's "never read .env".
|
|
33
|
+
*/
|
|
34
|
+
|
|
35
|
+
/** Turns the triage loop may spend before it has to synthesise with what it has. */
|
|
36
|
+
export const MAX_TRIAGE_TURNS = 40;
|
|
37
|
+
|
|
38
|
+
export const TRIAGE_MAX_TOKENS = 1024;
|
|
39
|
+
export const SYNTHESIS_MAX_TOKENS = 4096;
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* @param {Array<{name: string, content: Object}>} schemas
|
|
43
|
+
*/
|
|
44
|
+
export function createCapabilityValidator(schemas) {
|
|
45
|
+
const entry = (schemas || []).find(schema => schema.name === 'capability-report');
|
|
46
|
+
if (!entry) throw new Error('the content pack has no capability-report schema; run a pack sync');
|
|
47
|
+
const ajv = new Ajv2020({ allErrors: true, strict: false });
|
|
48
|
+
addFormats(ajv);
|
|
49
|
+
const compiled = ajv.compile(entry.content);
|
|
50
|
+
return {
|
|
51
|
+
validate(report) {
|
|
52
|
+
const ok = compiled(report);
|
|
53
|
+
return {
|
|
54
|
+
ok: Boolean(ok),
|
|
55
|
+
details: (compiled.errors || []).map(error => ({
|
|
56
|
+
instancePath: error.instancePath || `/${(error.params && (error.params.missingProperty || error.params.additionalProperty)) || ''}`,
|
|
57
|
+
message: error.message
|
|
58
|
+
}))
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* @param {Object} args
|
|
66
|
+
* @returns {Promise<{report: Object, lint: Array<string>, turns: number}>}
|
|
67
|
+
*/
|
|
68
|
+
export async function runGameProfile({
|
|
69
|
+
client,
|
|
70
|
+
jobId,
|
|
71
|
+
runId,
|
|
72
|
+
repoRoot,
|
|
73
|
+
skill,
|
|
74
|
+
schemas,
|
|
75
|
+
codeBrief = null,
|
|
76
|
+
brief = null,
|
|
77
|
+
engine = null,
|
|
78
|
+
commit = null,
|
|
79
|
+
dirty = false,
|
|
80
|
+
ledger = null,
|
|
81
|
+
log = () => {},
|
|
82
|
+
onProgress = () => {},
|
|
83
|
+
maxTriageTurns = MAX_TRIAGE_TURNS
|
|
84
|
+
}) {
|
|
85
|
+
const validator = createCapabilityValidator(schemas);
|
|
86
|
+
const repo = createRepoTools({ root: repoRoot, tools: PROFILE_TOOLS });
|
|
87
|
+
const system = [
|
|
88
|
+
{ type: 'text', text: skill },
|
|
89
|
+
{
|
|
90
|
+
type: 'text',
|
|
91
|
+
text: [
|
|
92
|
+
`job_id: ${jobId}`,
|
|
93
|
+
`repository root: the paths you read are relative to it; never absolute`,
|
|
94
|
+
`engine recorded on this game: ${engine || 'unknown'}`,
|
|
95
|
+
`commit: ${commit || 'unknown'}${dirty ? ' (working tree is dirty)' : ''}`,
|
|
96
|
+
codeBrief ? `<code_brief>\n${JSON.stringify(codeBrief).slice(0, 4000)}\n</code_brief>` : '<code_brief>null</code_brief>',
|
|
97
|
+
// The brief, when the game already has one. A profile is often the
|
|
98
|
+
// first thing a developer runs, in which case there is none, which is
|
|
99
|
+
// why it is optional rather than required.
|
|
100
|
+
brief ? `<expectations_brief>\n${JSON.stringify({ version: brief.version ?? null, ...(brief.fields || {}) }, null, 2).slice(0, 4000)}\n</expectations_brief>` : '<expectations_brief>null</expectations_brief>'
|
|
101
|
+
].join('\n'),
|
|
102
|
+
cache_control: { type: 'ephemeral' }
|
|
103
|
+
}
|
|
104
|
+
];
|
|
105
|
+
|
|
106
|
+
const messages = [{
|
|
107
|
+
role: 'user',
|
|
108
|
+
content: [{
|
|
109
|
+
type: 'text',
|
|
110
|
+
text: 'Profile this repository. Work through the procedure in order and read only what you need. When you have the facts you need, stop calling tools and say so in one line: the next step writes the report.'
|
|
111
|
+
}]
|
|
112
|
+
}];
|
|
113
|
+
|
|
114
|
+
let turns = 0;
|
|
115
|
+
while (turns < maxTriageTurns) {
|
|
116
|
+
turns += 1;
|
|
117
|
+
const answer = await callStepWithRetry({
|
|
118
|
+
client,
|
|
119
|
+
jobId,
|
|
120
|
+
runId,
|
|
121
|
+
step: STEPS.profileTriage,
|
|
122
|
+
system,
|
|
123
|
+
messages,
|
|
124
|
+
tools: repo.tools,
|
|
125
|
+
maxTokens: TRIAGE_MAX_TOKENS
|
|
126
|
+
});
|
|
127
|
+
if (ledger) ledger.record({ model: answer.model, usage: answer.usage });
|
|
128
|
+
if (answer.budget === 'warn') log('budget warning: the profile has used at least 80 percent of its model budget. Synthesising now.');
|
|
129
|
+
|
|
130
|
+
if (answer.toolUses.length === 0) {
|
|
131
|
+
messages.push({ role: 'assistant', content: [{ type: 'text', text: answer.text || '(done)' }] });
|
|
132
|
+
break;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
messages.push({
|
|
136
|
+
role: 'assistant',
|
|
137
|
+
content: [
|
|
138
|
+
...(answer.text.trim() ? [{ type: 'text', text: answer.text }] : []),
|
|
139
|
+
...answer.toolUses.map(toolUse => ({ type: 'tool_use', id: toolUse.id, name: toolUse.name, input: toolUse.malformed ? {} : toolUse.input }))
|
|
140
|
+
]
|
|
141
|
+
});
|
|
142
|
+
const results = [];
|
|
143
|
+
for (const toolUse of answer.toolUses) {
|
|
144
|
+
const executed = await repo.execute(toolUse);
|
|
145
|
+
results.push(executed.result);
|
|
146
|
+
onProgress({ phase: 'triage', turn: turns, tool: toolUse.name, target: toolUse.input && (toolUse.input.path || toolUse.input.pattern) });
|
|
147
|
+
}
|
|
148
|
+
messages.push({ role: 'user', content: results });
|
|
149
|
+
if (answer.budget === 'warn') break;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
messages.push({
|
|
153
|
+
role: 'user',
|
|
154
|
+
content: [{
|
|
155
|
+
type: 'text',
|
|
156
|
+
text: 'Now write the capability report. Answer with the JSON object only, nothing before or after it. Underpromise: every eligible row needs a concrete reason taken from a file you actually read.'
|
|
157
|
+
}]
|
|
158
|
+
});
|
|
159
|
+
|
|
160
|
+
let report = null;
|
|
161
|
+
let lint = [];
|
|
162
|
+
for (let attempt = 1; attempt <= 2; attempt += 1) {
|
|
163
|
+
const answer = await callStepWithRetry({
|
|
164
|
+
client,
|
|
165
|
+
jobId,
|
|
166
|
+
runId,
|
|
167
|
+
step: STEPS.profileSynthesis,
|
|
168
|
+
system,
|
|
169
|
+
messages,
|
|
170
|
+
tools: READ_ONLY_TOOLS,
|
|
171
|
+
maxTokens: SYNTHESIS_MAX_TOKENS
|
|
172
|
+
});
|
|
173
|
+
if (ledger) ledger.record({ model: answer.model, usage: answer.usage });
|
|
174
|
+
|
|
175
|
+
const parsed = parseJsonObject(answer.text);
|
|
176
|
+
const candidate = parsed ? { ...parsed, schema_version: parsed.schema_version || '1.0', job_id: jobId } : null;
|
|
177
|
+
const validation = candidate
|
|
178
|
+
? validator.validate(candidate)
|
|
179
|
+
: { ok: false, details: [{ instancePath: '/', message: 'the answer was not a JSON object' }] };
|
|
180
|
+
|
|
181
|
+
if (validation.ok) {
|
|
182
|
+
report = candidate;
|
|
183
|
+
lint = [];
|
|
184
|
+
break;
|
|
185
|
+
}
|
|
186
|
+
lint = validation.details.map(detail => `${detail.instancePath}: ${detail.message}`);
|
|
187
|
+
if (attempt === 2) throw new Error(`the capability report did not validate after one repair: ${lint.join('; ')}`);
|
|
188
|
+
|
|
189
|
+
messages.push({ role: 'assistant', content: [{ type: 'text', text: answer.text.slice(0, 2000) }] });
|
|
190
|
+
messages.push({
|
|
191
|
+
role: 'user',
|
|
192
|
+
content: [{ type: 'text', text: `That did not match capability-report.schema.json. Fix exactly these and answer with the JSON only:\n${lint.join('\n')}` }]
|
|
193
|
+
});
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
return { report, lint, turns };
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/**
|
|
200
|
+
* @param {string} jobDirectory
|
|
201
|
+
* @param {Object} report
|
|
202
|
+
* @returns {Promise<string>}
|
|
203
|
+
*/
|
|
204
|
+
export async function writeCapabilityReport(jobDirectory, report) {
|
|
205
|
+
const body = `${JSON.stringify(report, null, 2)}\n`;
|
|
206
|
+
assertNoSecrets('capability-report.json', body);
|
|
207
|
+
const target = path.join(jobDirectory, JOB_FILES.capabilityReport);
|
|
208
|
+
await writeFile(target, body, 'utf8');
|
|
209
|
+
return target;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
export { parseJsonObject };
|
|
213
|
+
|
|
214
|
+
export default { runGameProfile, writeCapabilityReport, createCapabilityValidator, parseJsonObject };
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
import { readFile } from 'node:fs/promises';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { RUN_FILES } from './paths.js';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Rebuilding a report from what a run already wrote down, with the game not
|
|
7
|
+
* running and no model turn spent.
|
|
8
|
+
*
|
|
9
|
+
* This is the point of the transcript. A run that got a long way in and then
|
|
10
|
+
* lost its heartbeat has real observations, real notes and real screenshots on
|
|
11
|
+
* disk; replaying it from the start throws all of that away, spends the action
|
|
12
|
+
* budget again, and gets a different playthrough (live game state cannot be
|
|
13
|
+
* restored, so it is not even the same run). Past a few actions, finalising
|
|
14
|
+
* what happened is worth more than starting over.
|
|
15
|
+
*
|
|
16
|
+
* What comes out is deliberately modest: a schema-valid `report.json` with no
|
|
17
|
+
* findings and a `report.md` carrying the notes. It invents no sentiment scores
|
|
18
|
+
* and no verdicts, because nobody gave any. The narrative and the findings are
|
|
19
|
+
* a model's job, and this path exists precisely because there is no model turn
|
|
20
|
+
* to spend on it.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
/** How far a run has to have got before finalising beats replaying it. */
|
|
24
|
+
export const PARTIAL_CHECKPOINT_FLOOR = 5;
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Whether this journal entry is worth finalising rather than replaying.
|
|
28
|
+
*
|
|
29
|
+
* @param {Object} entry a journal run entry
|
|
30
|
+
* @param {{restartInterrupted?: boolean}} [options]
|
|
31
|
+
* @returns {boolean}
|
|
32
|
+
*/
|
|
33
|
+
export function shouldFinalize(entry, options = {}) {
|
|
34
|
+
if (options.restartInterrupted) return false;
|
|
35
|
+
if (!entry) return false;
|
|
36
|
+
return Number(entry.checkpoint_step || 0) >= PARTIAL_CHECKPOINT_FLOOR;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Read back what a run recorded.
|
|
41
|
+
*
|
|
42
|
+
* @param {string} runDirectory
|
|
43
|
+
* @returns {Promise<{header: Object|null, notes: Array, screenshots: Array, observations: number, actions: number}>}
|
|
44
|
+
*/
|
|
45
|
+
export async function readTranscript(runDirectory) {
|
|
46
|
+
let text = '';
|
|
47
|
+
try {
|
|
48
|
+
text = await readFile(path.join(runDirectory, RUN_FILES.transcript), 'utf8');
|
|
49
|
+
} catch {
|
|
50
|
+
return { header: null, notes: [], screenshots: [], observations: 0, actions: 0 };
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
let header = null;
|
|
54
|
+
const notes = [];
|
|
55
|
+
const screenshots = [];
|
|
56
|
+
let observations = 0;
|
|
57
|
+
let actions = 0;
|
|
58
|
+
for (const line of text.split('\n')) {
|
|
59
|
+
if (!line.trim()) continue;
|
|
60
|
+
let entry;
|
|
61
|
+
try {
|
|
62
|
+
entry = JSON.parse(line);
|
|
63
|
+
} catch {
|
|
64
|
+
// A half written last line is what a killed process leaves behind. Skip
|
|
65
|
+
// it rather than refusing to read the whole transcript.
|
|
66
|
+
continue;
|
|
67
|
+
}
|
|
68
|
+
if (entry.kind === 'header') header = entry;
|
|
69
|
+
else if (entry.kind === 'note' && entry.note) notes.push(entry.note);
|
|
70
|
+
else if (entry.kind === 'screenshot' && entry.path) screenshots.push({ relativePath: entry.path, reason: entry.reason || '' });
|
|
71
|
+
else if (entry.kind === 'observation') observations += 1;
|
|
72
|
+
else if (entry.kind === 'action' && Number.isFinite(Number(entry.actions))) actions = Math.max(actions, Number(entry.actions));
|
|
73
|
+
}
|
|
74
|
+
return { header, notes, screenshots, observations, actions };
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* A schema-valid partial report built from a transcript.
|
|
79
|
+
*
|
|
80
|
+
* @param {Object} args
|
|
81
|
+
* @returns {{report: Object, markdown: string, actions: number}}
|
|
82
|
+
*/
|
|
83
|
+
export function buildPartialReport({ runId, jobId, persona, driver, target, transcript, promptVersion, usage }) {
|
|
84
|
+
const started = transcript.header && transcript.header.t ? new Date(transcript.header.t).toISOString() : new Date().toISOString();
|
|
85
|
+
const report = {
|
|
86
|
+
schema_version: '1.0',
|
|
87
|
+
run_id: runId,
|
|
88
|
+
job_id: jobId,
|
|
89
|
+
prompt_version: (transcript.header && transcript.header.prompt_version) || promptVersion || 'persona-playtest@0.1.0',
|
|
90
|
+
persona,
|
|
91
|
+
driver,
|
|
92
|
+
target,
|
|
93
|
+
started_at: started,
|
|
94
|
+
ended_at: new Date().toISOString(),
|
|
95
|
+
actions_taken: transcript.actions,
|
|
96
|
+
quit_reason: 'error',
|
|
97
|
+
quit_detail: 'the run was interrupted; this report was rebuilt from its transcript without replaying the game',
|
|
98
|
+
// No findings: a finding needs a judgement, and there was no model turn to
|
|
99
|
+
// make one. The notes are in the markdown so nothing is lost.
|
|
100
|
+
findings: [],
|
|
101
|
+
// Neutral rather than invented. The persona never gave a verdict, and a
|
|
102
|
+
// made-up 3 is still a made-up number; the markdown says so in words.
|
|
103
|
+
sentiment: { fun: 3, clarity: 3, frustration: 3 },
|
|
104
|
+
would_continue: false,
|
|
105
|
+
usage
|
|
106
|
+
};
|
|
107
|
+
return { report, markdown: renderPartialMarkdown({ persona, driver, target, transcript }), actions: transcript.actions };
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* The markdown, carrying the four headings the server lints for.
|
|
112
|
+
*
|
|
113
|
+
* @param {Object} args
|
|
114
|
+
* @returns {string}
|
|
115
|
+
*/
|
|
116
|
+
export function renderPartialMarkdown({ persona, driver, target, transcript }) {
|
|
117
|
+
const good = transcript.notes.filter(note => note.kind === 'good');
|
|
118
|
+
const bad = transcript.notes.filter(note => note.kind === 'bug' || note.kind === 'confusing');
|
|
119
|
+
const list = entries => (entries.length > 0
|
|
120
|
+
? entries.map(note => `- **${note.kind}${note.severity ? `/${note.severity}` : ''}**: ${note.text}`).join('\n')
|
|
121
|
+
: '- Nothing was logged.');
|
|
122
|
+
|
|
123
|
+
return [
|
|
124
|
+
`# Playtest: ${persona} (rebuilt from transcript)`,
|
|
125
|
+
`**Driver:** ${driver} - **Target:** ${target}`,
|
|
126
|
+
`**Session length:** ${transcript.actions} actions over ${transcript.observations} observations, then interrupted`,
|
|
127
|
+
'',
|
|
128
|
+
'## Why this report is partial',
|
|
129
|
+
'The run was interrupted before it could write a report. This was rebuilt from its transcript with the game not running, so it carries what the persona saw and noted but no narrative and no scored findings: nobody was there to judge them.',
|
|
130
|
+
'',
|
|
131
|
+
'## What worked',
|
|
132
|
+
list(good),
|
|
133
|
+
'',
|
|
134
|
+
'## Top issues',
|
|
135
|
+
list(bad),
|
|
136
|
+
'',
|
|
137
|
+
"## What this run didn't cover",
|
|
138
|
+
`Everything after action ${transcript.actions}. No persona verdict was recorded, so the sentiment scores in report.json are placeholders rather than judgements.`,
|
|
139
|
+
'',
|
|
140
|
+
'## Suggested next step',
|
|
141
|
+
'Re-run this persona to get a full report, or pass --restart-interrupted to replay it now.',
|
|
142
|
+
'',
|
|
143
|
+
'## Evidence',
|
|
144
|
+
transcript.screenshots.length > 0 ? transcript.screenshots.map(shot => `- ${shot.relativePath}`).join('\n') : '- (none)',
|
|
145
|
+
''
|
|
146
|
+
].join('\n');
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
export default { shouldFinalize, readTranscript, buildPartialReport, renderPartialMarkdown, PARTIAL_CHECKPOINT_FLOOR };
|
|
@@ -0,0 +1,286 @@
|
|
|
1
|
+
import { readFile, readdir, stat } from 'node:fs/promises';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { scanForSecrets } from './secretScan.js';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* The read-only source tools for `game_profile`: `read_file`, `glob` and
|
|
7
|
+
* `grep`, which are exactly the three names
|
|
8
|
+
* `routing.json`'s `game_profile.triage` allows (and `read_file` alone on
|
|
9
|
+
* `game_profile.synthesis` and `aggregate.synthesis`).
|
|
10
|
+
*
|
|
11
|
+
* Read-only is enforced here, not asked for in a prompt. There is no write,
|
|
12
|
+
* no shell, no network, and:
|
|
13
|
+
*
|
|
14
|
+
* - every path is resolved and refused if it leaves the root, so `../../`
|
|
15
|
+
* and a symlink out of the tree both fail;
|
|
16
|
+
* - `.env*`, key material and credential stores are refused by name,
|
|
17
|
+
* because "never read .env" is a rule the model should not be the last
|
|
18
|
+
* line of defence for;
|
|
19
|
+
* - anything whose content trips the secret scan comes back as a count of
|
|
20
|
+
* shapes instead of the content, so a key in a checked-in config never
|
|
21
|
+
* reaches the model and therefore can never be quoted into a report. Spec
|
|
22
|
+
* 02 asks for a warning without the value; this is that, enforced.
|
|
23
|
+
*
|
|
24
|
+
* Vendored directories are skipped, which is both a cost and a quality
|
|
25
|
+
* decision: `node_modules` is not the developer's game, and profiling it
|
|
26
|
+
* would spend the triage budget on somebody else's code.
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
/** Directories never walked. */
|
|
30
|
+
export const SKIP_DIRS = Object.freeze([
|
|
31
|
+
'.git', '.godot', '.ravensight', 'node_modules', 'dist', 'build', 'out',
|
|
32
|
+
'Library', 'Temp', 'Logs', 'obj', 'bin', 'ThirdParty', 'godot-cpp',
|
|
33
|
+
'.venv', 'venv', '__pycache__', '.next', '.cache', 'coverage'
|
|
34
|
+
]);
|
|
35
|
+
|
|
36
|
+
/** Files never read, whatever asks for them. */
|
|
37
|
+
export const SECRET_FILE_PATTERNS = Object.freeze([
|
|
38
|
+
/(^|\/)\.env($|\.)/i,
|
|
39
|
+
/(^|\/)\.npmrc$/i,
|
|
40
|
+
/(^|\/)id_(rsa|dsa|ecdsa|ed25519)$/i,
|
|
41
|
+
/(^|\/)credentials$/i,
|
|
42
|
+
/\.(pem|key|p12|pfx|keystore|jks)$/i
|
|
43
|
+
]);
|
|
44
|
+
|
|
45
|
+
/** How much of one file a single read returns. */
|
|
46
|
+
export const MAX_READ_BYTES = 120000;
|
|
47
|
+
|
|
48
|
+
/** How many paths a glob answers with. */
|
|
49
|
+
export const MAX_GLOB_RESULTS = 400;
|
|
50
|
+
|
|
51
|
+
/** How many matching lines a grep answers with. */
|
|
52
|
+
export const MAX_GREP_RESULTS = 200;
|
|
53
|
+
|
|
54
|
+
/** The tool definitions for `game_profile.triage`. */
|
|
55
|
+
export const PROFILE_TOOLS = Object.freeze([
|
|
56
|
+
{
|
|
57
|
+
name: 'read_file',
|
|
58
|
+
description: 'Read a text file from the repository. Paths are relative to the repository root. Read-only.',
|
|
59
|
+
input_schema: {
|
|
60
|
+
type: 'object',
|
|
61
|
+
additionalProperties: false,
|
|
62
|
+
required: ['path'],
|
|
63
|
+
properties: {
|
|
64
|
+
path: { type: 'string' },
|
|
65
|
+
start_line: { type: 'integer', minimum: 1 },
|
|
66
|
+
limit: { type: 'integer', minimum: 1, maximum: 2000 }
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
},
|
|
70
|
+
{
|
|
71
|
+
name: 'glob',
|
|
72
|
+
description: 'List repository paths matching a glob such as **/*.gd or project.godot. Vendored directories are skipped.',
|
|
73
|
+
input_schema: {
|
|
74
|
+
type: 'object',
|
|
75
|
+
additionalProperties: false,
|
|
76
|
+
required: ['pattern'],
|
|
77
|
+
properties: { pattern: { type: 'string' } }
|
|
78
|
+
}
|
|
79
|
+
},
|
|
80
|
+
{
|
|
81
|
+
name: 'grep',
|
|
82
|
+
description: 'Search the repository for a regular expression. Answers path, line number and the matching line.',
|
|
83
|
+
input_schema: {
|
|
84
|
+
type: 'object',
|
|
85
|
+
additionalProperties: false,
|
|
86
|
+
required: ['pattern'],
|
|
87
|
+
properties: {
|
|
88
|
+
pattern: { type: 'string' },
|
|
89
|
+
glob: { type: 'string', description: 'restrict to paths matching this glob' }
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
]);
|
|
94
|
+
|
|
95
|
+
/** The subset allowed on the synthesis steps. */
|
|
96
|
+
export const READ_ONLY_TOOLS = Object.freeze(PROFILE_TOOLS.filter(tool => tool.name === 'read_file'));
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* Compile a glob into a regular expression. Supports `**`, `*` and `?`,
|
|
100
|
+
* which covers every pattern the profile skill actually asks for
|
|
101
|
+
* (`project.godot`, `**\/*.gd`, `src/**`) without taking on a dependency.
|
|
102
|
+
*
|
|
103
|
+
* @param {string} pattern
|
|
104
|
+
* @returns {RegExp}
|
|
105
|
+
*/
|
|
106
|
+
export function globToRegExp(pattern) {
|
|
107
|
+
let source = '';
|
|
108
|
+
const value = String(pattern);
|
|
109
|
+
for (let i = 0; i < value.length; i += 1) {
|
|
110
|
+
const char = value[i];
|
|
111
|
+
if (char === '*') {
|
|
112
|
+
if (value[i + 1] === '*') {
|
|
113
|
+
// `**/` matches zero or more directories, so `**/x` also matches a
|
|
114
|
+
// bare `x` at the root.
|
|
115
|
+
if (value[i + 2] === '/') {
|
|
116
|
+
source += '(?:.*/)?';
|
|
117
|
+
i += 2;
|
|
118
|
+
} else {
|
|
119
|
+
source += '.*';
|
|
120
|
+
i += 1;
|
|
121
|
+
}
|
|
122
|
+
} else {
|
|
123
|
+
source += '[^/]*';
|
|
124
|
+
}
|
|
125
|
+
continue;
|
|
126
|
+
}
|
|
127
|
+
if (char === '?') {
|
|
128
|
+
source += '[^/]';
|
|
129
|
+
continue;
|
|
130
|
+
}
|
|
131
|
+
source += char.replace(/[.+^${}()|[\]\\]/g, '\\$&');
|
|
132
|
+
}
|
|
133
|
+
return new RegExp(`^${source}$`);
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* @param {string} relativePath
|
|
138
|
+
* @returns {boolean}
|
|
139
|
+
*/
|
|
140
|
+
export function isSecretPath(relativePath) {
|
|
141
|
+
const normalized = String(relativePath).split(path.sep).join('/');
|
|
142
|
+
return SECRET_FILE_PATTERNS.some(pattern => pattern.test(normalized));
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* Resolve a model-supplied path against the root, refusing anything that
|
|
147
|
+
* escapes it.
|
|
148
|
+
*
|
|
149
|
+
* @param {string} root
|
|
150
|
+
* @param {string} candidate
|
|
151
|
+
* @returns {string} an absolute path
|
|
152
|
+
*/
|
|
153
|
+
export function resolveInside(root, candidate) {
|
|
154
|
+
const base = path.resolve(root);
|
|
155
|
+
const target = path.resolve(base, String(candidate));
|
|
156
|
+
if (target !== base && !target.startsWith(base + path.sep)) {
|
|
157
|
+
throw new Error(`${candidate} is outside the repository`);
|
|
158
|
+
}
|
|
159
|
+
return target;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* @param {string} root
|
|
164
|
+
* @param {Object} [opts]
|
|
165
|
+
* @returns {Promise<Array<string>>} repository-relative paths
|
|
166
|
+
*/
|
|
167
|
+
export async function listFiles(root, { limit = 20000 } = {}) {
|
|
168
|
+
const out = [];
|
|
169
|
+
/** @param {string} dir */
|
|
170
|
+
async function walk(dir) {
|
|
171
|
+
if (out.length >= limit) return;
|
|
172
|
+
let entries;
|
|
173
|
+
try {
|
|
174
|
+
entries = await readdir(dir, { withFileTypes: true });
|
|
175
|
+
} catch {
|
|
176
|
+
return;
|
|
177
|
+
}
|
|
178
|
+
for (const entry of entries) {
|
|
179
|
+
if (out.length >= limit) return;
|
|
180
|
+
if (entry.isSymbolicLink()) continue;
|
|
181
|
+
const absolute = path.join(dir, entry.name);
|
|
182
|
+
if (entry.isDirectory()) {
|
|
183
|
+
if (SKIP_DIRS.includes(entry.name)) continue;
|
|
184
|
+
await walk(absolute);
|
|
185
|
+
continue;
|
|
186
|
+
}
|
|
187
|
+
const relative = path.relative(root, absolute).split(path.sep).join('/');
|
|
188
|
+
if (isSecretPath(relative)) continue;
|
|
189
|
+
out.push(relative);
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
await walk(path.resolve(root));
|
|
193
|
+
return out.sort();
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/**
|
|
197
|
+
* @param {Object} args
|
|
198
|
+
* @param {string} args.root the repository root
|
|
199
|
+
* @returns {{tools: Array, execute: (toolUse: Object) => Promise<{result: Object}>}}
|
|
200
|
+
*/
|
|
201
|
+
export function createRepoTools({ root, tools = PROFILE_TOOLS }) {
|
|
202
|
+
let cachedFiles = null;
|
|
203
|
+
|
|
204
|
+
async function files() {
|
|
205
|
+
if (!cachedFiles) cachedFiles = await listFiles(root);
|
|
206
|
+
return cachedFiles;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
return {
|
|
210
|
+
tools,
|
|
211
|
+
|
|
212
|
+
/**
|
|
213
|
+
* @param {Object} toolUse
|
|
214
|
+
*/
|
|
215
|
+
async execute(toolUse) {
|
|
216
|
+
const input = toolUse.input || {};
|
|
217
|
+
try {
|
|
218
|
+
if (toolUse.name === 'read_file') return { result: ok(toolUse.id, await doRead(root, input)) };
|
|
219
|
+
if (toolUse.name === 'glob') {
|
|
220
|
+
const matcher = globToRegExp(input.pattern);
|
|
221
|
+
const matched = (await files()).filter(file => matcher.test(file)).slice(0, MAX_GLOB_RESULTS);
|
|
222
|
+
return { result: ok(toolUse.id, matched.length > 0 ? matched.join('\n') : '(no matches)') };
|
|
223
|
+
}
|
|
224
|
+
if (toolUse.name === 'grep') return { result: ok(toolUse.id, await doGrep(root, await files(), input)) };
|
|
225
|
+
return { result: ok(toolUse.id, `There is no tool called ${toolUse.name}.`, true) };
|
|
226
|
+
} catch (error) {
|
|
227
|
+
return { result: ok(toolUse.id, String(error && error.message ? error.message : error), true) };
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
};
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
async function doRead(root, input) {
|
|
234
|
+
const relative = String(input.path || '');
|
|
235
|
+
if (isSecretPath(relative)) {
|
|
236
|
+
return `Refused: ${relative} is a credential file and is never read. Note in warnings that credentials are checked into the repository, without quoting anything.`;
|
|
237
|
+
}
|
|
238
|
+
const absolute = resolveInside(root, relative);
|
|
239
|
+
const info = await stat(absolute);
|
|
240
|
+
if (!info.isFile()) throw new Error(`${relative} is not a file`);
|
|
241
|
+
const text = await readFile(absolute, 'utf8');
|
|
242
|
+
const secrets = scanForSecrets(text);
|
|
243
|
+
if (secrets.length > 0) {
|
|
244
|
+
// The content is withheld rather than redacted: a partial redaction of
|
|
245
|
+
// an unknown format is a guess, and the shapes are what the warning
|
|
246
|
+
// needs anyway.
|
|
247
|
+
return `Withheld: ${relative} contains ${secrets.map(s => `${s.type} (${s.count})`).join(', ')}. Record a warning that this file holds secret shaped strings. Do not quote it and do not ask for it again.`;
|
|
248
|
+
}
|
|
249
|
+
const lines = text.split('\n');
|
|
250
|
+
const start = Math.max(1, Number(input.start_line) || 1);
|
|
251
|
+
const limit = Number(input.limit) || lines.length;
|
|
252
|
+
const slice = lines.slice(start - 1, start - 1 + limit).join('\n');
|
|
253
|
+
return slice.length > MAX_READ_BYTES ? `${slice.slice(0, MAX_READ_BYTES)}\n... [truncated]` : slice;
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
async function doGrep(root, all, input) {
|
|
257
|
+
const regex = new RegExp(String(input.pattern), 'i');
|
|
258
|
+
const matcher = input.glob ? globToRegExp(input.glob) : null;
|
|
259
|
+
const hits = [];
|
|
260
|
+
for (const relative of all) {
|
|
261
|
+
if (hits.length >= MAX_GREP_RESULTS) break;
|
|
262
|
+
if (matcher && !matcher.test(relative)) continue;
|
|
263
|
+
let text;
|
|
264
|
+
try {
|
|
265
|
+
const info = await stat(path.join(root, relative));
|
|
266
|
+
if (info.size > MAX_READ_BYTES) continue;
|
|
267
|
+
text = await readFile(path.join(root, relative), 'utf8');
|
|
268
|
+
} catch {
|
|
269
|
+
continue;
|
|
270
|
+
}
|
|
271
|
+
// A file with secrets in it is not searched either: a grep result is a
|
|
272
|
+
// quoted line, and a quoted line from such a file is the leak.
|
|
273
|
+
if (scanForSecrets(text).length > 0) continue;
|
|
274
|
+
const lines = text.split('\n');
|
|
275
|
+
for (let i = 0; i < lines.length && hits.length < MAX_GREP_RESULTS; i += 1) {
|
|
276
|
+
if (regex.test(lines[i])) hits.push(`${relative}:${i + 1}: ${lines[i].trim().slice(0, 300)}`);
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
return hits.length > 0 ? hits.join('\n') : '(no matches)';
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
function ok(id, text, isError = false) {
|
|
283
|
+
return { type: 'tool_result', tool_use_id: id, content: [{ type: 'text', text }], ...(isError ? { is_error: true } : {}) };
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
export default { createRepoTools, PROFILE_TOOLS, READ_ONLY_TOOLS, globToRegExp, listFiles, resolveInside, isSecretPath };
|