pi-supernova 0.10.2 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +365 -90
- package/docs/CHANGELOG.md +89 -5
- package/docs/TOKEN_COSTS.md +149 -324
- package/index.js +40 -11
- package/package.json +4 -3
- package/src/adapters/bash.js +40 -10
- package/src/adapters/edit.js +77 -23
- package/src/adapters/errors.js +4 -0
- package/src/adapters/index.js +1 -0
- package/src/adapters/list.js +6 -3
- package/src/adapters/read-focus.js +6 -0
- package/src/adapters/read-image.js +17 -9
- package/src/adapters/read-indexed.js +72 -0
- package/src/adapters/read-text.js +19 -2
- package/src/adapters/read.js +57 -13
- package/src/adapters/refs.js +17 -7
- package/src/adapters/write.js +14 -21
- package/src/bridge/host-bridge.js +152 -39
- package/src/bridge/invoke.js +3 -1
- package/src/bridge/tool-registry.js +132 -16
- package/src/context/evidence-graph.js +1 -0
- package/src/context/evidence-rank.js +1 -0
- package/src/context/evidence.js +4 -2
- package/src/context/fuzzy.js +31 -8
- package/src/context/outline.js +33 -9
- package/src/context/query.js +7 -2
- package/src/context/repo-index.js +8 -3
- package/src/context/search-files.js +20 -0
- package/src/context/search.js +25 -10
- package/src/context/snap-search.js +28 -6
- package/src/context/snap.js +8 -3
- package/src/context/source-entry.js +64 -28
- package/src/context/spans.js +1 -0
- package/src/contract/bash.js +16 -7
- package/src/contract/edit.js +26 -7
- package/src/contract/program.js +2 -1
- package/src/contract/read.js +46 -46
- package/src/fs/background.js +10 -2
- package/src/fs/check.js +844 -46
- package/src/fs/commit.js +89 -15
- package/src/fs/diff.js +12 -9
- package/src/fs/directory.js +14 -4
- package/src/fs/file-io.js +28 -7
- package/src/fs/glob.js +93 -11
- package/src/fs/json-read.js +3 -2
- package/src/fs/json-size.js +2 -1
- package/src/fs/lines.js +49 -2
- package/src/fs/patch.js +26 -10
- package/src/fs/process-tree.js +19 -1
- package/src/fs/read-window.js +36 -20
- package/src/fs/source-window.js +5 -10
- package/src/fs/text-ops.js +38 -18
- package/src/fs/vfs.js +296 -41
- package/src/fs/workspace.js +24 -6
- package/src/output/bottleneck.js +50 -25
- package/src/output/final.js +20 -0
- package/src/output/format.js +97 -36
- package/src/output/outcome.js +16 -4
- package/src/runtime/batch-input.js +17 -3
- package/src/runtime/guest-api.js +32 -7
- package/src/runtime/guest-worker.js +22 -4
- package/src/runtime/parallel.js +9 -49
- package/src/runtime/program-batch.js +53 -28
- package/src/runtime/program-file.js +2 -1
- package/src/runtime/program.js +94 -41
- package/src/runtime/reference.js +10 -9
- package/src/runtime/runtime.js +69 -18
- package/src/runtime/worker-pool.js +10 -1
- package/src/shared/decode.js +9 -8
- package/src/shared/image-worker.js +7 -0
- package/src/shared/image.js +58 -15
- package/src/shared/png.js +18 -0
- package/src/shared/result.js +30 -9
- package/src/shared/syntax-context.js +7 -1
- package/src/shared/utf8.js +11 -13
- package/src/ui/host-render.js +4 -0
- package/src/ui/omp-frame.js +2 -0
- package/src/ui/progress.js +2 -1
- package/src/ui/render-measure.js +11 -0
- package/src/ui/render.js +92 -31
- package/src/ui/trace.js +8 -5
package/src/runtime/runtime.js
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import {acquireWorker,killWorker} from './worker-pool.js';
|
|
2
|
+
|
|
2
3
|
export {warmGuestWorker,stopWarmGuestWorker} from './worker-pool.js';
|
|
4
|
+
|
|
3
5
|
import {admitGuest,prepareProgram,normalizeGuestLocation} from './program.js';
|
|
4
6
|
|
|
5
7
|
import { readProgramFile } from "./program-file.js";
|
|
@@ -18,7 +20,8 @@ const GUEST_COMMANDS = new Set(["read", "edit", "write", "bash"]);
|
|
|
18
20
|
// shadow it (redeclaration or TDZ); `supernova` is the tool, not a guest
|
|
19
21
|
// function; `data` can only collide when the call supplied it. Anything
|
|
20
22
|
// else gets no hint.
|
|
21
|
-
function guestMistakeHint(message) {
|
|
23
|
+
function guestMistakeHint(message, kind) {
|
|
24
|
+
if (kind !== "ReferenceError" && kind !== "SyntaxError") return "";
|
|
22
25
|
const name = /Identifier '([^']+)' has already been declared/.exec(message)?.[1] ?? /Cannot access '([^']+)' before initialization/.exec(message)?.[1];
|
|
23
26
|
|
|
24
27
|
if (GUEST_COMMANDS.has(name)) return "; `" + name + "` is a supernova command: give your variable another name (e.g. const text = await " + name + "(...))";
|
|
@@ -31,6 +34,7 @@ function guestMistakeHint(message) {
|
|
|
31
34
|
}
|
|
32
35
|
|
|
33
36
|
const ABORT_MESSAGE = "supernova aborted";
|
|
37
|
+
|
|
34
38
|
const TIMEOUT_MESSAGE = "supernova timed out: increase the outer timeoutMs (and any shorter bash timeoutMs), or split the program; sleeps count toward the deadline";
|
|
35
39
|
|
|
36
40
|
const MEMORY_SLACK = 1.5;
|
|
@@ -42,7 +46,9 @@ export function formatMemoryAttribution({ limitMb, heapBytes, externalBytes, ms,
|
|
|
42
46
|
const parts = tracked
|
|
43
47
|
? [["vfs cache", tracked.vfsCacheBytes], ["index entries", tracked.indexBytes], ["overlays", tracked.overlayBytes]]
|
|
44
48
|
: [];
|
|
49
|
+
|
|
45
50
|
const where = op ? ` during ${op} (${calls} host calls)` : ` (${calls} host calls)`;
|
|
51
|
+
|
|
46
52
|
const seen = parts.length
|
|
47
53
|
? `; supernova-tracked host bytes (not charged to guest): ${parts.map(([name, bytes]) => `${name} ${toMb(Number(bytes) || 0).toFixed(1)}MB`).join(", ")} in ${tracked.overlayFiles ?? 0} overlay files`
|
|
48
54
|
: "";
|
|
@@ -54,11 +60,6 @@ let runSeq = 0;
|
|
|
54
60
|
|
|
55
61
|
const RPC_METHODS = {
|
|
56
62
|
call: (nova, args, onItem) => nova.call(args[0], args[1], onItem),
|
|
57
|
-
callMany: async (nova, args) => {
|
|
58
|
-
const wave = await nova.callMany(args[0]);
|
|
59
|
-
|
|
60
|
-
return Array.isArray(wave) ? { results: [...wave], mode: wave.mode, reason: wave.reason } : wave;
|
|
61
|
-
},
|
|
62
63
|
speculateBegin: (nova) => nova.speculateBegin(),
|
|
63
64
|
speculateCommit: (nova) => nova.speculateCommit(),
|
|
64
65
|
speculateRollback: (nova) => nova.speculateRollback(),
|
|
@@ -149,6 +150,7 @@ class GuestRun {
|
|
|
149
150
|
|
|
150
151
|
try {
|
|
151
152
|
this.handle.worker.postMessage({ ...message, runId: this.runId });
|
|
153
|
+
|
|
152
154
|
return true;
|
|
153
155
|
} catch (err) {
|
|
154
156
|
try {
|
|
@@ -156,6 +158,7 @@ class GuestRun {
|
|
|
156
158
|
} catch (error) {
|
|
157
159
|
this.onError(error);
|
|
158
160
|
}
|
|
161
|
+
|
|
159
162
|
return false;
|
|
160
163
|
}
|
|
161
164
|
}
|
|
@@ -163,8 +166,10 @@ class GuestRun {
|
|
|
163
166
|
deliverReadItem(id, index, value) {
|
|
164
167
|
if (!this.accepting) return Promise.reject(new Error("aborted"));
|
|
165
168
|
const key = id + ":" + index;
|
|
169
|
+
|
|
166
170
|
return new Promise((resolve,reject) => {
|
|
167
171
|
this.deliveries.set(key,{resolve,reject});
|
|
172
|
+
|
|
168
173
|
if (!this.postResult({op:"rpc:item",id,index,value})) {
|
|
169
174
|
this.deliveries.delete(key);
|
|
170
175
|
reject(new Error("read result not transferable"));
|
|
@@ -175,20 +180,24 @@ class GuestRun {
|
|
|
175
180
|
onReadAck(msg) {
|
|
176
181
|
const key = msg.id + ":" + msg.index;
|
|
177
182
|
const delivery = this.deliveries.get(key);
|
|
183
|
+
|
|
178
184
|
if (!delivery) return;
|
|
179
185
|
this.deliveries.delete(key);
|
|
186
|
+
|
|
180
187
|
if (msg.error) delivery.reject(new Error(msg.error));
|
|
181
188
|
else delivery.resolve();
|
|
182
189
|
}
|
|
183
190
|
|
|
184
191
|
async drainPending(outcome) {
|
|
185
192
|
if (this.pending.size || !outcome.ok) this.cancelHost();
|
|
193
|
+
|
|
186
194
|
if (!this.pending.size) return;
|
|
187
195
|
let timer;
|
|
188
196
|
|
|
189
197
|
try {
|
|
190
198
|
await Promise.race([Promise.allSettled(this.pending), new Promise(resolve => { timer = setTimeout(resolve, 250); })]);
|
|
191
199
|
} finally { clearTimeout(timer); }
|
|
200
|
+
|
|
192
201
|
if (this.pending.size && outcome.ok) this.hostError ??= "program completed with a host call still running";
|
|
193
202
|
}
|
|
194
203
|
|
|
@@ -201,11 +210,14 @@ class GuestRun {
|
|
|
201
210
|
this.handle?.worker.off("exit", this.onExit);
|
|
202
211
|
void killWorker(this.handle);
|
|
203
212
|
await this.drainPending(outcome);
|
|
213
|
+
|
|
204
214
|
if (this.finished) return;
|
|
215
|
+
|
|
205
216
|
if (this.abortOutcome) {
|
|
206
217
|
const diagnostic = this.hostError && this.hostError !== "aborted" ? "\n" + this.hostError : "";
|
|
207
218
|
outcome = this.fail(this.abortOutcome.error + diagnostic);
|
|
208
219
|
}
|
|
220
|
+
|
|
209
221
|
this.finish(outcome.ok && this.hostError ? this.fail(this.hostError) : outcome);
|
|
210
222
|
}
|
|
211
223
|
|
|
@@ -223,11 +235,13 @@ class GuestRun {
|
|
|
223
235
|
const method = Object.hasOwn(RPC_METHODS, msg.method) && RPC_METHODS[msg.method];
|
|
224
236
|
this.rpcCount++;
|
|
225
237
|
this.lastRpcMethod = isString(msg.method) ? msg.method : null;
|
|
238
|
+
|
|
226
239
|
const work = Promise.resolve().then(() => {
|
|
227
240
|
if (!method) throw new Error("unknown nova method: " + msg.method);
|
|
228
241
|
|
|
229
242
|
const onItem = msg.streamRead && msg.method === "call" && msg.args?.[0] === "read"
|
|
230
243
|
? (index,value) => this.deliverReadItem(msg.id,index,value) : undefined;
|
|
244
|
+
|
|
231
245
|
return method(this.nova, msg.args, onItem);
|
|
232
246
|
});
|
|
233
247
|
|
|
@@ -245,8 +259,10 @@ class GuestRun {
|
|
|
245
259
|
// A completed guest cannot submit more work while attachments are decoded.
|
|
246
260
|
// The outer timer stays armed; no commit occurs before validation completes.
|
|
247
261
|
this.accepting = false;
|
|
262
|
+
|
|
248
263
|
try {
|
|
249
264
|
const packed = msg.output ?? packageFinalReturn(msg.value, this.logs, this.config);
|
|
265
|
+
|
|
250
266
|
if (packed.images?.length) await validateReturnedImages(packed.images,this.inputController.signal);
|
|
251
267
|
void this.complete({ ok: true, result: packed.returnValue, resultText: packed.returnText,
|
|
252
268
|
returnTruncated: packed.returnTruncated, images: packed.images, undefinedReturn: msg.undefinedReturn === true,
|
|
@@ -257,18 +273,25 @@ class GuestRun {
|
|
|
257
273
|
}
|
|
258
274
|
|
|
259
275
|
guestError(msg) {
|
|
260
|
-
const location = normalizeGuestLocation(this.
|
|
276
|
+
const location = normalizeGuestLocation(this.code, msg.location, this.prepared);
|
|
261
277
|
const where = location ? " (line " + location.line + ":" + location.col + ")" : "";
|
|
262
|
-
|
|
278
|
+
|
|
279
|
+
// Native errors may include source lines and carets. Annotate the headline,
|
|
280
|
+
// never the end of that diagnostic payload.
|
|
281
|
+
const message = msg.message.replace(/(\r\n|[\n\r\u2028\u2029]|$)/, where + "$1");
|
|
282
|
+
void this.complete(this.fail(message + guestMistakeHint(msg.message, msg.errorKind)));
|
|
263
283
|
}
|
|
264
284
|
|
|
265
285
|
onMemory(msg) {
|
|
266
286
|
const { heapUsed, external } = msg;
|
|
287
|
+
|
|
267
288
|
if (![heapUsed, external].every(bytes => Number.isFinite(bytes) && bytes >= 0)) return;
|
|
289
|
+
|
|
268
290
|
if (heapUsed + external <= (this.config.maxHeapMb ?? 512) * MEMORY_SLACK * 1048576) return;
|
|
269
291
|
let tracked = null;
|
|
270
292
|
|
|
271
293
|
try { tracked = isFunction(this.nova?.describeMemory) ? this.nova.describeMemory() : null; } catch {}
|
|
294
|
+
|
|
272
295
|
// Use the same cancellation/drain path as other failures, not an early
|
|
273
296
|
// finish that can return while a host command is still mutating files.
|
|
274
297
|
void this.complete(this.fail(formatMemoryAttribution({
|
|
@@ -296,8 +319,14 @@ class GuestRun {
|
|
|
296
319
|
|
|
297
320
|
async loadCode() {
|
|
298
321
|
if (this.file !== undefined) this.code = await readProgramFile(this.file, this.cwd, this.config.maxCodeChars ?? 48000, this.inputController.signal);
|
|
322
|
+
|
|
299
323
|
if (this.finished) return false;
|
|
300
|
-
|
|
324
|
+
|
|
325
|
+
if (!this.code.trim()) {
|
|
326
|
+
this.finish(this.fail("code must be a non-empty string; no commands ran"));
|
|
327
|
+
|
|
328
|
+
return false;
|
|
329
|
+
}
|
|
301
330
|
|
|
302
331
|
try { this.prepared = prepareProgram(this.code); }
|
|
303
332
|
catch (error) {
|
|
@@ -306,6 +335,7 @@ class GuestRun {
|
|
|
306
335
|
: " Fix " + this.file + " and re-run.";
|
|
307
336
|
|
|
308
337
|
this.finish(this.fail("JavaScript syntax error" + (this.file === undefined ? "" : " in " + this.file) + ": " + error.message + "; no commands ran." + guidance + errorContext(this.code, error)));
|
|
338
|
+
|
|
309
339
|
return false;
|
|
310
340
|
}
|
|
311
341
|
|
|
@@ -313,27 +343,45 @@ class GuestRun {
|
|
|
313
343
|
}
|
|
314
344
|
|
|
315
345
|
async attachWorker() {
|
|
316
|
-
if (this.wall() >= this.timeoutMs) {
|
|
346
|
+
if (this.wall() >= this.timeoutMs) {
|
|
347
|
+
this.abort(true);
|
|
348
|
+
|
|
349
|
+
return false;
|
|
350
|
+
}
|
|
351
|
+
|
|
317
352
|
this.handle = acquireWorker(this.config);
|
|
318
353
|
await this.handle.ready;
|
|
319
|
-
if (this.finished || this.signal?.aborted) { this.abort(); return false; }
|
|
320
|
-
let available = isFunction(this.nova.names) ? await this.nova.names() : [];
|
|
321
354
|
|
|
322
|
-
if (
|
|
323
|
-
|
|
355
|
+
if (this.finished || this.signal?.aborted) {
|
|
356
|
+
this.abort();
|
|
357
|
+
|
|
358
|
+
return false;
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
if (isFunction(this.nova.names)) await this.nova.names();
|
|
362
|
+
|
|
363
|
+
if (this.finished || this.signal?.aborted) {
|
|
364
|
+
this.abort();
|
|
365
|
+
|
|
366
|
+
return false;
|
|
367
|
+
}
|
|
368
|
+
|
|
324
369
|
this.handle.worker.on("message", this.onMessage);
|
|
325
370
|
this.handle.worker.on("error", this.onError);
|
|
326
371
|
this.handle.worker.on("exit", this.onExit);
|
|
327
|
-
|
|
328
|
-
this.
|
|
372
|
+
|
|
373
|
+
if (this.wall() >= this.timeoutMs) {
|
|
374
|
+
this.abort(true);
|
|
375
|
+
|
|
376
|
+
return false;
|
|
377
|
+
}
|
|
329
378
|
|
|
330
379
|
return true;
|
|
331
380
|
}
|
|
332
381
|
|
|
333
382
|
postRun() {
|
|
334
|
-
this.handle.worker.postMessage({ op: "run", runId: this.runId, prepared: this.prepared, data: this.data,
|
|
383
|
+
this.handle.worker.postMessage({ op: "run", runId: this.runId, prepared: this.prepared, data: this.data,
|
|
335
384
|
batchRead: this.nova.batchRead !== false,
|
|
336
|
-
nativeArgv: this.nova.nativeArgv === true,
|
|
337
385
|
formatResult:true,
|
|
338
386
|
limits: { maxReturnChars:this.config.maxReturnChars ?? 32000, maxLogLines: this.config.maxLogLines ?? 100, maxLogLineChars: this.config.maxLogLineChars ?? 4096 } });
|
|
339
387
|
}
|
|
@@ -341,7 +389,9 @@ class GuestRun {
|
|
|
341
389
|
async boot() {
|
|
342
390
|
try {
|
|
343
391
|
if (this.signal?.aborted) return this.abort();
|
|
392
|
+
|
|
344
393
|
if (!await this.loadCode()) return;
|
|
394
|
+
|
|
345
395
|
if (!await this.attachWorker()) return;
|
|
346
396
|
this.postRun();
|
|
347
397
|
} catch (err) {
|
|
@@ -366,6 +416,7 @@ export async function runGuestProgram({ code, file, cwd = process.cwd(), data, n
|
|
|
366
416
|
const fail = (error) => ({ ok: false, error: truncateChars(String(error), config.maxReturnChars ?? 32000, "error").text, logs: [], logTruncated: false, wallMs: 0 });
|
|
367
417
|
|
|
368
418
|
if (admitted.error) return fail(admitted.error);
|
|
419
|
+
|
|
369
420
|
if (signal?.aborted) return fail(ABORT_MESSAGE);
|
|
370
421
|
|
|
371
422
|
return new GuestRun({
|
|
@@ -4,6 +4,8 @@ const WORKER_URL = new URL("./guest-worker.js", import.meta.url);
|
|
|
4
4
|
|
|
5
5
|
let idleWorker = null;
|
|
6
6
|
|
|
7
|
+
let generation = 0;
|
|
8
|
+
|
|
7
9
|
function spawnWorker(config) {
|
|
8
10
|
const maxHeapMb = config.maxHeapMb ?? 512;
|
|
9
11
|
// An inline bootstrap accepts inherited --input-type from stdin/eval SDK hosts.
|
|
@@ -55,6 +57,7 @@ function killWorker(handle) {
|
|
|
55
57
|
}
|
|
56
58
|
|
|
57
59
|
function acquireWorker(config) {
|
|
60
|
+
const acquiredGeneration = generation;
|
|
58
61
|
const candidate = idleWorker;
|
|
59
62
|
idleWorker = null;
|
|
60
63
|
const reusable = candidate && !candidate.dead && candidate.maxHeapMb === (config.maxHeapMb ?? 512);
|
|
@@ -68,7 +71,10 @@ function acquireWorker(config) {
|
|
|
68
71
|
// so the two constructions never overlap. The finish-time warm usually
|
|
69
72
|
// becomes a no-op, so steady-state spawn count is unchanged.
|
|
70
73
|
if (reusable) warmGuestWorker(config).catch(() => {});
|
|
71
|
-
else handle.ready.then(() =>
|
|
74
|
+
else handle.ready.then(() => {
|
|
75
|
+
// A cold worker can finish starting after session_shutdown drained the pool.
|
|
76
|
+
if (generation === acquiredGeneration) return warmGuestWorker(config).catch(() => {});
|
|
77
|
+
}, () => {});
|
|
72
78
|
|
|
73
79
|
return handle;
|
|
74
80
|
}
|
|
@@ -86,6 +92,9 @@ export function warmGuestWorker(config = {}) {
|
|
|
86
92
|
}
|
|
87
93
|
|
|
88
94
|
export function stopWarmGuestWorker() {
|
|
95
|
+
generation++;
|
|
96
|
+
|
|
89
97
|
return killWorker(idleWorker);
|
|
90
98
|
}
|
|
99
|
+
|
|
91
100
|
export { acquireWorker, killWorker };
|
package/src/shared/decode.js
CHANGED
|
@@ -26,11 +26,16 @@ export function mapChangedChildren(value, visit, context) {
|
|
|
26
26
|
if (!array && !isObject(value)) return value;
|
|
27
27
|
let out = value;
|
|
28
28
|
|
|
29
|
-
|
|
29
|
+
// Array indexes can be inherited; holes and non-index properties stay unvisited.
|
|
30
|
+
const keys = array ? value.keys() : Object.keys(value);
|
|
31
|
+
|
|
32
|
+
for (const key of keys) {
|
|
33
|
+
if (array && !(key in value)) continue;
|
|
30
34
|
const before = value[key];
|
|
31
35
|
const after = visit(before, context, key);
|
|
32
36
|
|
|
33
37
|
if (Object.is(before, after)) continue;
|
|
38
|
+
|
|
34
39
|
if (out === value) out = array ? value.slice() : { ...value };
|
|
35
40
|
// Define rather than assign: "__proto__" must remain an ordinary data key.
|
|
36
41
|
Object.defineProperty(out, key, { value: after, enumerable: true, writable: true, configurable: true });
|
|
@@ -51,9 +56,11 @@ export function assertModelImageMime(mimeType) {
|
|
|
51
56
|
/** Node's base64 decoder is permissive; model attachments must not be. */
|
|
52
57
|
export function decodeImageData(data) {
|
|
53
58
|
const bytes = Buffer.from(data,"base64");
|
|
59
|
+
|
|
54
60
|
if (!data || bytes.toString("base64") !== String(data)) {
|
|
55
61
|
throw new Error("invalid image base64; use canonical padded base64 without a data-URL prefix or extra characters; no image attached");
|
|
56
62
|
}
|
|
63
|
+
|
|
57
64
|
return bytes;
|
|
58
65
|
}
|
|
59
66
|
|
|
@@ -93,7 +100,7 @@ function plainFromCollection(value, seen, depth) {
|
|
|
93
100
|
if (value instanceof Map) return plainFromMap(value, seen, depth);
|
|
94
101
|
const out = Object.create(null);
|
|
95
102
|
|
|
96
|
-
for (const k of
|
|
103
|
+
for (const k of Object.keys(value)) out[k] = toPlain(value[k], seen, depth + 1);
|
|
97
104
|
|
|
98
105
|
return out;
|
|
99
106
|
}
|
|
@@ -177,9 +184,3 @@ export function toPlain(value, seen = new Set(), depth = 0) {
|
|
|
177
184
|
export function errorMessage(error) {
|
|
178
185
|
return error instanceof Error ? error.message : String(error);
|
|
179
186
|
}
|
|
180
|
-
|
|
181
|
-
// Arrays visit present indexes (including inherited ones), not extra properties.
|
|
182
|
-
function* childKeys(value) {
|
|
183
|
-
if (!Array.isArray(value)) { yield* Object.keys(value); return; }
|
|
184
|
-
for (const key of value.keys()) if (key in value) yield key;
|
|
185
|
-
}
|
|
@@ -3,22 +3,29 @@
|
|
|
3
3
|
import sharp from "sharp";
|
|
4
4
|
|
|
5
5
|
const MAX_PIXELS = 32_000_000;
|
|
6
|
+
|
|
6
7
|
async function inputBytes() {
|
|
7
8
|
const chunks = [];
|
|
8
9
|
let size = 0;
|
|
10
|
+
|
|
9
11
|
for await (const chunk of process.stdin) {
|
|
10
12
|
size += chunk.length;
|
|
13
|
+
|
|
11
14
|
if (size > 20 * 1024 * 1024) throw new Error("encoded image exceeds 20 MiB");
|
|
12
15
|
chunks.push(chunk);
|
|
13
16
|
}
|
|
17
|
+
|
|
14
18
|
return Buffer.concat(chunks,size);
|
|
15
19
|
}
|
|
16
20
|
|
|
17
21
|
async function decode() {
|
|
18
22
|
const image = sharp(await inputBytes(),{failOn:"warning",limitInputPixels:MAX_PIXELS,pages:-1}).timeout({seconds:5});
|
|
23
|
+
|
|
19
24
|
try {
|
|
20
25
|
const meta = await image.metadata();
|
|
26
|
+
|
|
21
27
|
if ("image/" + meta.format !== process.argv[2]) throw new Error("encoded format does not match declared MIME");
|
|
28
|
+
|
|
22
29
|
if (meta.width * meta.height > MAX_PIXELS) throw new Error("decoded pixel limit exceeded");
|
|
23
30
|
// Full decode, without resizing or shrinking that could conceal corruption.
|
|
24
31
|
// No decoded pixels cross back into the agent process.
|
package/src/shared/image.js
CHANGED
|
@@ -5,16 +5,22 @@ import {assertModelImageMime,decodeImageData,errorMessage} from "./decode.js";
|
|
|
5
5
|
import {assertPng} from "./png.js";
|
|
6
6
|
|
|
7
7
|
const MAX_BYTES = 20 * 1024 * 1024;
|
|
8
|
+
|
|
8
9
|
// Cache only successful content digests, never image bytes or file paths. A file
|
|
9
10
|
// changed in place cannot reuse validation of its old contents.
|
|
10
11
|
const verified = new Set();
|
|
12
|
+
|
|
11
13
|
const workerPath = fileURLToPath(new URL("./image-worker.js",import.meta.url));
|
|
14
|
+
|
|
12
15
|
let tail = Promise.resolve();
|
|
13
16
|
|
|
14
17
|
function matchesSignature(bytes, mime) {
|
|
15
18
|
if (mime === "image/png") return bytes.subarray(0,8).equals(Buffer.from([137,80,78,71,13,10,26,10]));
|
|
19
|
+
|
|
16
20
|
if (mime === "image/jpeg") return bytes[0] === 255 && bytes[1] === 216 && bytes[2] === 255;
|
|
21
|
+
|
|
17
22
|
if (mime === "image/gif") return ["GIF87a","GIF89a"].includes(bytes.toString("ascii",0,6));
|
|
23
|
+
|
|
18
24
|
return bytes.toString("ascii",0,4) === "RIFF" && bytes.toString("ascii",8,12) === "WEBP";
|
|
19
25
|
}
|
|
20
26
|
|
|
@@ -24,29 +30,37 @@ function invalid(mime, label, reason) {
|
|
|
24
30
|
|
|
25
31
|
function decodePixels(bytes, mime, signal) {
|
|
26
32
|
signal?.throwIfAborted();
|
|
33
|
+
|
|
27
34
|
return new Promise((resolve,reject)=>{
|
|
28
35
|
// BUN_BE_BUN lets a compiled OMP executable run its normal Bun runtime,
|
|
29
36
|
// whose module resolver can load native dependencies from this package.
|
|
30
37
|
const child = childProcess.spawn(process.execPath,[workerPath,mime],{
|
|
31
38
|
env:{...process.env,BUN_BE_BUN:"1"},stdio:["pipe","ignore","pipe"],windowsHide:true,
|
|
32
39
|
});
|
|
33
|
-
|
|
40
|
+
|
|
41
|
+
let stderr = "", timedOut = false, processError;
|
|
34
42
|
const abort = ()=>child.kill("SIGKILL");
|
|
35
43
|
const timer = setTimeout(()=>{timedOut=true;abort();},5000);
|
|
44
|
+
|
|
36
45
|
const finish = error=>{
|
|
37
46
|
clearTimeout(timer);
|
|
38
47
|
signal?.removeEventListener("abort",abort);
|
|
48
|
+
|
|
39
49
|
if (error) reject(error); else resolve();
|
|
40
50
|
};
|
|
51
|
+
|
|
41
52
|
child.stderr.on("data",chunk=>{stderr += chunk.toString().slice(0,Math.max(0,2048-stderr.length));});
|
|
42
53
|
child.stdin.on("error",()=>{}); // Early decoder exit may close stdin first.
|
|
43
|
-
child.
|
|
54
|
+
// A failed spawn also closes; other child errors can precede pipe cleanup.
|
|
55
|
+
// Only close releases the raster slot, and the watchdog remains armed.
|
|
56
|
+
child.on("error",error=>{processError ??= error;});
|
|
44
57
|
child.once("close",code=>{
|
|
45
58
|
if (signal?.aborted) finish(signal.reason ?? new Error("aborted"));
|
|
46
59
|
else if (timedOut) finish(new Error("image decoding exceeded 5000 ms"));
|
|
47
|
-
else finish(code === 0 ? undefined : new Error(stderr.trim() || "image decoder exited before validation"));
|
|
60
|
+
else finish(processError ?? (code === 0 ? undefined : new Error(stderr.trim() || "image decoder exited before validation")));
|
|
48
61
|
});
|
|
49
62
|
signal?.addEventListener("abort",abort,{once:true});
|
|
63
|
+
|
|
50
64
|
if (signal?.aborted) abort();
|
|
51
65
|
child.stdin.end(bytes);
|
|
52
66
|
});
|
|
@@ -55,22 +69,51 @@ function decodePixels(bytes, mime, signal) {
|
|
|
55
69
|
export async function validateImageBytes(bytes, mime, label = "", signal) {
|
|
56
70
|
signal?.throwIfAborted();
|
|
57
71
|
assertModelImageMime(mime);
|
|
72
|
+
|
|
58
73
|
if (bytes.length > MAX_BYTES) throw invalid(mime,label,"encoded image exceeds 20 MiB");
|
|
74
|
+
|
|
59
75
|
if (!matchesSignature(bytes,mime)) throw invalid(mime,label,"signature does not match declared format");
|
|
60
76
|
const key = mime + ":" + createHash("sha256").update(bytes).digest("hex");
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
77
|
+
|
|
78
|
+
// A verified digest needs no raster allocation and cannot contend with the
|
|
79
|
+
// decoder. Recheck inside the queue for identical, concurrently admitted bytes.
|
|
80
|
+
if (verified.has(key)) return;
|
|
81
|
+
|
|
82
|
+
return new Promise((resolve,reject) => {
|
|
83
|
+
let started = false;
|
|
84
|
+
|
|
85
|
+
const abort = () => {
|
|
86
|
+
if (started) return;
|
|
87
|
+
bytes = undefined;
|
|
88
|
+
signal.removeEventListener("abort",abort);
|
|
89
|
+
reject(signal.reason);
|
|
90
|
+
};
|
|
91
|
+
|
|
92
|
+
const work = tail.then(async() => {
|
|
93
|
+
started = true;
|
|
94
|
+
signal?.removeEventListener("abort",abort);
|
|
95
|
+
signal?.throwIfAborted();
|
|
96
|
+
|
|
97
|
+
if (verified.has(key)) return;
|
|
98
|
+
|
|
99
|
+
if (mime === "image/png") assertPng(bytes,label);
|
|
100
|
+
|
|
101
|
+
try { await decodePixels(bytes,mime,signal); }
|
|
102
|
+
catch (error) { signal?.throwIfAborted(); throw invalid(mime,label,errorMessage(error)); }
|
|
103
|
+
|
|
104
|
+
verified.add(key);
|
|
105
|
+
|
|
106
|
+
if (verified.size > 16) verified.delete(verified.values().next().value);
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
// Withdrawal settles the caller and releases its bytes, not the active
|
|
110
|
+
// decoder's slot. Live successors still wait for actual raster cleanup.
|
|
111
|
+
tail = work.catch(()=>{});
|
|
112
|
+
work.then(resolve,reject);
|
|
113
|
+
signal?.addEventListener("abort",abort,{once:true});
|
|
114
|
+
|
|
115
|
+
if (signal?.aborted) abort();
|
|
69
116
|
});
|
|
70
|
-
// Serialize native raster allocations, not normal reads/guests. Failed or
|
|
71
|
-
// cancelled validation must never poison the next image's queue slot.
|
|
72
|
-
tail = work.catch(()=>{});
|
|
73
|
-
await work;
|
|
74
117
|
}
|
|
75
118
|
|
|
76
119
|
export async function validateReturnedImages(images, signal) {
|
package/src/shared/png.js
CHANGED
|
@@ -1,17 +1,22 @@
|
|
|
1
1
|
import * as zlib from "node:zlib";
|
|
2
2
|
import {isFunction} from "./decode.js";
|
|
3
|
+
|
|
3
4
|
// Fast container preflight before full decoding. Older Node/Bun versions use the
|
|
4
5
|
// portable CRC fallback instead of requiring node:zlib.crc32.
|
|
5
6
|
const signature = Buffer.from([137,80,78,71,13,10,26,10]);
|
|
7
|
+
|
|
6
8
|
const crcTable = Uint32Array.from({length:256}, (_, value) => {
|
|
7
9
|
for (let bit = 0; bit < 8; bit++) value = (value >>> 1) ^ ((value & 1) ? 0xedb88320 : 0);
|
|
10
|
+
|
|
8
11
|
return value >>> 0;
|
|
9
12
|
});
|
|
10
13
|
|
|
11
14
|
function crc32(bytes) {
|
|
12
15
|
if (isFunction(zlib.crc32)) return zlib.crc32(bytes);
|
|
13
16
|
let crc = 0xffffffff;
|
|
17
|
+
|
|
14
18
|
for (const byte of bytes) crc = (crc >>> 8) ^ crcTable[(crc ^ byte) & 255];
|
|
19
|
+
|
|
15
20
|
return (crc ^ 0xffffffff) >>> 0;
|
|
16
21
|
}
|
|
17
22
|
|
|
@@ -23,19 +28,26 @@ function chunkAt(bytes, offset, label) {
|
|
|
23
28
|
if (bytes.length - offset < 12) invalid("truncated chunk", label);
|
|
24
29
|
const length = bytes.readUInt32BE(offset);
|
|
25
30
|
const type = bytes.toString("ascii",offset+4,offset+8);
|
|
31
|
+
|
|
26
32
|
if (!/^[A-Za-z]{4}$/.test(type)) invalid("invalid chunk type", label);
|
|
27
33
|
const end = offset + 8 + length;
|
|
34
|
+
|
|
28
35
|
if (end + 4 > bytes.length) invalid("truncated " + type + " chunk", label);
|
|
36
|
+
|
|
29
37
|
if (crc32(bytes.subarray(offset+4,end)) !== bytes.readUInt32BE(end)) invalid(type + " checksum mismatch", label);
|
|
38
|
+
|
|
30
39
|
return {type,length,end:end+4};
|
|
31
40
|
}
|
|
32
41
|
|
|
33
42
|
function checkHeader(bytes, chunk, offset, label) {
|
|
34
43
|
if (offset !== 8) {
|
|
35
44
|
if (chunk.type === "IHDR") invalid("duplicate IHDR", label);
|
|
45
|
+
|
|
36
46
|
return;
|
|
37
47
|
}
|
|
48
|
+
|
|
38
49
|
if (chunk.type !== "IHDR" || chunk.length !== 13) invalid("missing or invalid IHDR", label);
|
|
50
|
+
|
|
39
51
|
if (!bytes.readUInt32BE(offset+8) || !bytes.readUInt32BE(offset+12)) invalid("empty dimensions", label);
|
|
40
52
|
}
|
|
41
53
|
|
|
@@ -43,15 +55,21 @@ function checkHeader(bytes, chunk, offset, label) {
|
|
|
43
55
|
export function assertPng(bytes, label = "") {
|
|
44
56
|
if (!bytes.subarray(0,8).equals(signature)) invalid("signature mismatch", label);
|
|
45
57
|
let offset = 8, imageData = false;
|
|
58
|
+
|
|
46
59
|
while (offset < bytes.length) {
|
|
47
60
|
const chunk = chunkAt(bytes,offset,label);
|
|
48
61
|
checkHeader(bytes,chunk,offset,label);
|
|
62
|
+
|
|
49
63
|
if (chunk.type === "IDAT") imageData = true;
|
|
64
|
+
|
|
50
65
|
if (chunk.type === "IEND") {
|
|
51
66
|
if (chunk.length || !imageData || chunk.end !== bytes.length) invalid("invalid IEND or missing IDAT", label);
|
|
67
|
+
|
|
52
68
|
return;
|
|
53
69
|
}
|
|
70
|
+
|
|
54
71
|
offset = chunk.end;
|
|
55
72
|
}
|
|
73
|
+
|
|
56
74
|
invalid("missing IEND", label);
|
|
57
75
|
}
|
package/src/shared/result.js
CHANGED
|
@@ -24,24 +24,38 @@ export function resultDiff(response) {
|
|
|
24
24
|
// Only native adapters can attach this host-local marker. It never crosses RPC;
|
|
25
25
|
// the bridge sends a typed-value flag instead of asking the guest to guess JSON.
|
|
26
26
|
export const READ_VALUE = Symbol("native read value");
|
|
27
|
+
|
|
27
28
|
export const READ_BYTES = Symbol("native read bytes");
|
|
29
|
+
|
|
28
30
|
export const READ_PREVIEW = Symbol("native read preview");
|
|
29
|
-
export const MAX_READ_VALUE_BYTES = 64 * 1024 * 1024;
|
|
30
31
|
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
}
|
|
32
|
+
// Host-only disk-read provenance, not user data or a claim about displayed text.
|
|
33
|
+
export const READ_FILES = Symbol("native disk reads");
|
|
34
34
|
|
|
35
|
-
|
|
35
|
+
export const MAX_READ_VALUE_BYTES = 64 * 1024 * 1024;
|
|
36
|
+
|
|
37
|
+
function containerBytes(value, pending, seen, remaining) {
|
|
36
38
|
if (seen.has(value)) return 8;
|
|
37
39
|
seen.add(value);
|
|
38
40
|
let bytes = 32;
|
|
41
|
+
|
|
42
|
+
if (bytes > remaining) return bytes;
|
|
39
43
|
const array = Array.isArray(value);
|
|
40
|
-
const keys = array ?
|
|
41
|
-
|
|
44
|
+
const keys = array ? null : Object.keys(value);
|
|
45
|
+
|
|
46
|
+
for (let i = 0; i < (array ? value.length : keys.length); i++) {
|
|
47
|
+
const key = array ? i : keys[i];
|
|
42
48
|
bytes += array ? 8 : 24 + 2 * key.length;
|
|
43
|
-
|
|
49
|
+
|
|
50
|
+
if (bytes > remaining) break;
|
|
51
|
+
// Charge scalar strings at their owning edge; only containers need
|
|
52
|
+
// a deferred visit for alias tracking and child traversal.
|
|
53
|
+
const item = value[key];
|
|
54
|
+
|
|
55
|
+
if (isString(item)) bytes += 2 * item.length;
|
|
56
|
+
else if (isObject(item) || Array.isArray(item)) pending.push(item);
|
|
44
57
|
}
|
|
58
|
+
|
|
45
59
|
return bytes;
|
|
46
60
|
}
|
|
47
61
|
|
|
@@ -49,22 +63,28 @@ function containerBytes(value, pending, seen) {
|
|
|
49
63
|
export function readValueBytes(value, limit = Infinity, seen = new WeakSet()) {
|
|
50
64
|
const pending = [value];
|
|
51
65
|
let bytes = 0;
|
|
66
|
+
|
|
52
67
|
while (pending.length) {
|
|
53
68
|
const item = pending.pop();
|
|
69
|
+
|
|
54
70
|
if (isString(item)) bytes += 2*item.length;
|
|
55
|
-
else if (isObject(item) || Array.isArray(item)) bytes += containerBytes(item,pending,seen);
|
|
71
|
+
else if (isObject(item) || Array.isArray(item)) bytes += containerBytes(item,pending,seen,limit-bytes);
|
|
56
72
|
else bytes += 8;
|
|
73
|
+
|
|
57
74
|
if (bytes > limit) throw new Error("read value exceeds " + limit + " bytes of remaining storage budget; select fewer fields or smaller slices");
|
|
58
75
|
}
|
|
76
|
+
|
|
59
77
|
return bytes;
|
|
60
78
|
}
|
|
61
79
|
|
|
62
80
|
export function readResult(value, details = {}, preview = isString(value) ? value : "", bytes = readValueBytes(value), render) {
|
|
63
81
|
let content;
|
|
82
|
+
|
|
64
83
|
// Native consumers retain their text-content API. The bridge/trace use the
|
|
65
84
|
// typed value/preview, so ordinary RPCs never serialize this second copy.
|
|
66
85
|
return {get content() {
|
|
67
86
|
const text = () => render ? render() : isString(value) && !details.json ? value : JSON.stringify(value);
|
|
87
|
+
|
|
68
88
|
return content ??= [{type:"text",text:text()}];
|
|
69
89
|
}, details, [READ_VALUE]:value, [READ_BYTES]:bytes, [READ_PREVIEW]:preview};
|
|
70
90
|
}
|
|
@@ -73,5 +93,6 @@ export function asReadResult(raw) {
|
|
|
73
93
|
if (Object.hasOwn(raw, READ_VALUE)) return raw;
|
|
74
94
|
const image = raw.content?.find(part => part.type === "image");
|
|
75
95
|
const value = image ?? raw.content?.filter(part => part.type === "text").map(part => part.text).join("\n") ?? "";
|
|
96
|
+
|
|
76
97
|
return { ...raw, [READ_VALUE]: value, [READ_BYTES]: readValueBytes(value) };
|
|
77
98
|
}
|
|
@@ -16,13 +16,19 @@ export function sourceContext(source, line, column, lineBreaks = JS_LINES) {
|
|
|
16
16
|
if (text === undefined) return "";
|
|
17
17
|
const located = Number.isInteger(column) && column >= 0;
|
|
18
18
|
const position = located ? Math.min(column, text.length) : 0;
|
|
19
|
+
const {shown,caret} = renderSourceWindow(text, position, located);
|
|
20
|
+
|
|
21
|
+
return "\n " + shown + (caret ? "\n " + caret : "");
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function renderSourceWindow(text, position, located) {
|
|
19
25
|
const start = Math.max(0, Math.min(position - 80, text.length - 160));
|
|
20
26
|
const end = Math.min(text.length, start + 160);
|
|
21
27
|
const prefix = start > 0 ? "…" : "";
|
|
22
28
|
const shown = prefix + displaySource(text.slice(start, end)) + (end < text.length ? "…" : "");
|
|
23
29
|
const caret = located ? " ".repeat(stringWidth(prefix + displaySource(text.slice(start, position)))) + "^" : "";
|
|
24
30
|
|
|
25
|
-
return
|
|
31
|
+
return {shown,caret};
|
|
26
32
|
}
|
|
27
33
|
|
|
28
34
|
/** acorn-style error that carries a loc, when it has one. */
|