pi-supernova 0.8.2 → 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +188 -51
- package/docs/CHANGELOG.md +86 -1
- package/docs/TOKEN_COSTS.md +38 -0
- package/index.js +10 -175
- package/package.json +2 -1
- package/src/adapters/bash.js +14 -30
- package/src/adapters/errors.js +1 -9
- package/src/adapters/read-focus.js +98 -0
- package/src/adapters/read-image.js +51 -0
- package/src/adapters/read-json.js +42 -0
- package/src/adapters/read-text.js +71 -0
- package/src/adapters/read.js +66 -635
- package/src/bridge/catalog.js +3 -2
- package/src/bridge/host-bridge.js +35 -167
- package/src/bridge/tool-registry.js +104 -0
- package/src/bridge/trace.js +41 -0
- package/src/context/evidence-graph.js +249 -0
- package/src/context/evidence-rank.js +153 -0
- package/src/context/evidence.js +10 -424
- package/src/context/fuzzy.js +116 -43
- package/src/context/query.js +80 -0
- package/src/context/repo-index.js +23 -166
- package/src/context/search-files.js +19 -0
- package/src/context/search.js +2 -24
- package/src/context/snap-search.js +203 -0
- package/src/context/snap.js +5 -266
- package/src/context/source-entry.js +112 -0
- package/src/contract/bash.js +6 -1
- package/src/contract/program.js +36 -0
- package/src/contract/read.js +8 -53
- package/src/fs/check.js +1 -1
- package/src/fs/commit.js +161 -0
- package/src/fs/diff.js +11 -15
- package/src/fs/directory.js +79 -0
- package/src/fs/file-io.js +100 -0
- package/src/fs/glob.js +54 -0
- package/src/fs/json-size.js +54 -0
- package/src/fs/lines.js +117 -0
- package/src/fs/read-window.js +74 -0
- package/src/fs/session-resource.js +50 -0
- package/src/fs/text-ops.js +7 -227
- package/src/fs/vfs.js +5 -239
- package/src/fs/workspace.js +2 -1
- package/src/output/bottleneck.js +13 -67
- package/src/output/final.js +114 -0
- package/src/output/format.js +94 -5
- package/src/output/outcome.js +91 -0
- package/src/runtime/batch-input.js +68 -0
- package/src/runtime/guest-api.js +281 -0
- package/src/runtime/guest-worker.js +62 -333
- package/src/runtime/parallel.js +41 -39
- package/src/runtime/program-batch.js +21 -75
- package/src/runtime/program-file.js +3 -11
- package/src/runtime/program.js +141 -0
- package/src/runtime/reference.js +6 -5
- package/src/runtime/runtime.js +77 -253
- package/src/runtime/worker-pool.js +91 -0
- package/src/shared/decode.js +22 -8
- package/src/shared/image-worker.js +30 -0
- package/src/shared/image.js +78 -0
- package/src/shared/png.js +57 -0
- package/src/shared/result.js +77 -0
- package/src/shared/syntax-context.js +61 -3
- package/src/ui/host-render.js +104 -0
- package/src/ui/progress.js +51 -0
- package/src/ui/render.js +21 -421
- package/src/ui/trace.js +277 -0
package/index.js
CHANGED
|
@@ -1,158 +1,23 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {programParameters} from './src/contract/program.js';
|
|
2
|
+
import {progressEmitter} from './src/ui/progress.js';
|
|
3
|
+
export {progressEmitter} from './src/ui/progress.js';
|
|
4
|
+
import {result,errorText,successText,fitOutput,attachReceipts,throwIfFailed} from './src/output/outcome.js';
|
|
5
|
+
|
|
2
6
|
import { runProgramBatch } from "./src/runtime/program-batch.js";
|
|
3
7
|
import { REFERENCE } from "./src/runtime/reference.js";
|
|
4
|
-
import {
|
|
8
|
+
import { errorMessage, isString } from "./src/shared/decode.js";
|
|
5
9
|
import { loadConfig } from "./src/config/config.js";
|
|
6
10
|
import { createHostBridge } from "./src/bridge/host-bridge.js";
|
|
7
|
-
|
|
11
|
+
|
|
8
12
|
import { runGuestProgram, warmGuestWorker, stopWarmGuestWorker } from "./src/runtime/runtime.js";
|
|
9
13
|
import { renderSupernovaCall, renderSupernovaResult } from "./src/ui/render.js";
|
|
10
14
|
|
|
11
15
|
export { renderSupernovaCall, renderSupernovaResult };
|
|
12
16
|
|
|
13
|
-
// Sync only, never top-level await. Dynamic import of host/deps hung OMP plugin load.
|
|
14
|
-
const require = createRequire(import.meta.url);
|
|
15
|
-
|
|
16
|
-
let Type;
|
|
17
|
-
|
|
18
|
-
try {
|
|
19
|
-
Type = require("typebox").Type;
|
|
20
|
-
} catch {
|
|
21
|
-
Type = {
|
|
22
|
-
Object: (props, opts) => ({ type: "object", properties: props || {}, additionalProperties: false, ...opts }),
|
|
23
|
-
String: (opts) => ({ type: "string", ...opts }),
|
|
24
|
-
Unknown: (opts) => ({ ...opts }),
|
|
25
|
-
Array: (items, opts) => ({ type: "array", items, ...opts }),
|
|
26
|
-
Integer: (opts) => ({ type: "integer", ...opts }),
|
|
27
|
-
Optional: (s) => ({ ...s }),
|
|
28
|
-
Boolean: (opts) => ({ type: "boolean", ...opts }),
|
|
29
|
-
};
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
function result(text, details) {
|
|
33
|
-
return { content: [{ type: "text", text }], details };
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
const PROGRESS_FRAME_MS = 80;
|
|
37
|
-
|
|
38
|
-
/**
|
|
39
|
-
* Live trace updates for the card. The first update is immediate (seeds the result slot);
|
|
40
|
-
* later ones are coalesced to one host re-render per frame so a tight loop of nova.calls
|
|
41
|
-
* is not throttled by the TUI. A throwing host callback must never break the run.
|
|
42
|
-
*/
|
|
43
|
-
export function progressEmitter(onUpdate) {
|
|
44
|
-
if (!isFunction(onUpdate)) return Object.assign(() => {}, { flush() {} });
|
|
45
|
-
let pending = null;
|
|
46
|
-
let timer = null;
|
|
47
|
-
let lastSent = -Infinity;
|
|
48
|
-
|
|
49
|
-
const send = () => {
|
|
50
|
-
timer = null;
|
|
51
|
-
|
|
52
|
-
if (pending === null) return;
|
|
53
|
-
// Snapshot only at emission, not on every tool event. Completed records must
|
|
54
|
-
// not mutate a previously emitted frame while Pi is still consuming it.
|
|
55
|
-
const trace = pending.map(record => ({ ...record }));
|
|
56
|
-
pending = null;
|
|
57
|
-
lastSent = performance.now();
|
|
58
|
-
|
|
59
|
-
try {
|
|
60
|
-
onUpdate({ content: [{ type: "text", text: "" }], details: { trace, running: true } });
|
|
61
|
-
} catch {}
|
|
62
|
-
};
|
|
63
|
-
|
|
64
|
-
const emit = (trace) => {
|
|
65
|
-
pending = trace;
|
|
66
|
-
|
|
67
|
-
if (timer !== null) return;
|
|
68
|
-
const wait = PROGRESS_FRAME_MS - (performance.now() - lastSent);
|
|
69
|
-
|
|
70
|
-
if (wait <= 0) send();
|
|
71
|
-
else {
|
|
72
|
-
timer = setTimeout(send, wait);
|
|
73
|
-
timer.unref?.();
|
|
74
|
-
}
|
|
75
|
-
};
|
|
76
|
-
|
|
77
|
-
emit.flush = () => {
|
|
78
|
-
if (timer !== null) clearTimeout(timer);
|
|
79
|
-
pending = null;
|
|
80
|
-
timer = null;
|
|
81
|
-
};
|
|
82
|
-
|
|
83
|
-
return emit;
|
|
84
|
-
}
|
|
85
|
-
|
|
86
17
|
function sessionStats({ programs, returnedChars }) {
|
|
87
18
|
return `this session: ${programs} programs · ${returnedChars} output characters (not token counts)`;
|
|
88
19
|
}
|
|
89
20
|
|
|
90
|
-
function logsBlock(outcome, tail = "") {
|
|
91
|
-
if (!outcome.logs?.length && !outcome.logTruncated) return "";
|
|
92
|
-
|
|
93
|
-
return `\n--- logs${outcome.logTruncated ? " [logs truncated]" : ""}\n${outcome.logs?.join("\n") ?? ""}${tail}`;
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
function mutationText(outcome) {
|
|
97
|
-
const m = outcome.mutations;
|
|
98
|
-
|
|
99
|
-
if (!m) return "";
|
|
100
|
-
const external = m.external ? "; external calls attempted=" + m.external + ", their side effects cannot be rolled back" : "";
|
|
101
|
-
const uncertain = m.pendingCommits || m.recoveryFailed ? "; filesystem outcome uncertain: inspect disk and any recovery backups before retrying" : "";
|
|
102
|
-
|
|
103
|
-
return "\nmutations: committed=" + m.committed + " rolledBack=" + m.rolledBack + " (file versions)" + external + uncertain;
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
function mutationReceipts(trace) {
|
|
107
|
-
if (!Array.isArray(trace)) return "";
|
|
108
|
-
|
|
109
|
-
return trace
|
|
110
|
-
.filter(row => row?.ok && (row.name === "write" || row.name === "edit") && isString(row.resultText) && row.resultText)
|
|
111
|
-
.map(row => row.resultText)
|
|
112
|
-
.join("\n");
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
// Corrective hint, emitted only when a turn actually split. Independent work
|
|
116
|
-
// belongs in one program: a split cannot use the single prewarmed worker and pays
|
|
117
|
-
// one extra spawn per sibling. Costs nothing until it fires, so it needs no room in
|
|
118
|
-
// the tool definition.
|
|
119
|
-
function splitTurnHint(outcome) {
|
|
120
|
-
return outcome.overlappedTurn ? ` (${outcome.overlappedTurn} supernova calls ran at once; independent work belongs in one program)` : "";
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
function errorText(outcome, call) {
|
|
124
|
-
return `error #${call} ${outcome.wallMs}ms${outcome.returnTruncated ? " [output truncated]" : ""}${mutationText(outcome)}${splitTurnHint(outcome)}
|
|
125
|
-
error: ${outcome.error}${logsBlock(outcome)}`;
|
|
126
|
-
}
|
|
127
|
-
|
|
128
|
-
function successText(outcome, call) {
|
|
129
|
-
const truncated = outcome.returnTruncated ? " [return truncated]" : "";
|
|
130
|
-
const hint = outcome.undefinedReturn ? " (no return statement; add `return` to get a value)" : "";
|
|
131
|
-
const m = outcome.mutations;
|
|
132
|
-
const showMutations = m && (m.committed || m.rolledBack || m.external || m.pendingCommits || m.recoveryFailed) ? mutationText(outcome) : "";
|
|
133
|
-
|
|
134
|
-
return `ok #${call} ${outcome.wallMs}ms${truncated}${showMutations}${splitTurnHint(outcome)}${logsBlock(outcome, "\n--- result")}\n${outcome.resultText}${hint}`;
|
|
135
|
-
}
|
|
136
|
-
|
|
137
|
-
function fitOutput(outcome, call, limit, format) {
|
|
138
|
-
let text = format(outcome, call);
|
|
139
|
-
|
|
140
|
-
if (text.length <= limit) return text;
|
|
141
|
-
outcome.returnTruncated = true;
|
|
142
|
-
const wrapper = format({ ...outcome, resultText: "", logs: [] }, call);
|
|
143
|
-
const room = Math.max(256, limit - wrapper.length);
|
|
144
|
-
|
|
145
|
-
if (Array.isArray(outcome.result) && outcome.result.length && outcome.result.every(isString)) {
|
|
146
|
-
outcome.resultText = formatBoundedStringArray(outcome.result, room);
|
|
147
|
-
} else if (isString(outcome.resultText) && outcome.resultText.length > room) {
|
|
148
|
-
outcome.resultText = truncateChars(outcome.resultText, room, "output").text;
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
text = format(outcome, call);
|
|
152
|
-
|
|
153
|
-
return text.length <= limit ? text : truncateChars(text, limit, "output").text;
|
|
154
|
-
}
|
|
155
|
-
|
|
156
21
|
const TOOL_DESCRIPTION = REFERENCE;
|
|
157
22
|
|
|
158
23
|
export default function piSupernova(pi) {
|
|
@@ -197,7 +62,7 @@ export function registerCodeMode(pi) {
|
|
|
197
62
|
|
|
198
63
|
function makeNovaApi(runBridge, cancel) {
|
|
199
64
|
return {
|
|
200
|
-
call: (name, args) => runBridge.call(name, args),
|
|
65
|
+
call: (name, args, onItem) => runBridge.call(name, args, onItem),
|
|
201
66
|
callMany: (calls) => runBridge.callMany(calls),
|
|
202
67
|
speculateBegin: () => runBridge.barrier(() => runBridge.beginSpeculation()),
|
|
203
68
|
speculateCommit: () => runBridge.barrier(() => runBridge.commitSpeculation()),
|
|
@@ -280,24 +145,6 @@ export function registerCodeMode(pi) {
|
|
|
280
145
|
scheduleWarm(runController);
|
|
281
146
|
}
|
|
282
147
|
|
|
283
|
-
function attachReceipts(outcome, trace) {
|
|
284
|
-
if (outcome.ok && outcome.result === undefined) {
|
|
285
|
-
const receipts = mutationReceipts(trace);
|
|
286
|
-
|
|
287
|
-
if (receipts) {
|
|
288
|
-
outcome.resultText = receipts;
|
|
289
|
-
outcome.undefinedReturn = false;
|
|
290
|
-
}
|
|
291
|
-
}
|
|
292
|
-
}
|
|
293
|
-
|
|
294
|
-
function throwIfFailed(outcome, visible, response) {
|
|
295
|
-
if (outcome.ok) return response;
|
|
296
|
-
const error = new Error(visible);
|
|
297
|
-
Object.defineProperty(error,"supernovaResult",{value:response});
|
|
298
|
-
throw error;
|
|
299
|
-
}
|
|
300
|
-
|
|
301
148
|
function packExecuteResult(outcome, call, runBridge, budget, runOpts, peakSeen) {
|
|
302
149
|
if (budget) budget.logLines += outcome.logs?.length ?? 0;
|
|
303
150
|
outcome.overlappedTurn = !runOpts?.parallel && peakSeen > 1 ? peakSeen : 0;
|
|
@@ -322,19 +169,7 @@ export function registerCodeMode(pi) {
|
|
|
322
169
|
label: "Supernova",
|
|
323
170
|
description: TOOL_DESCRIPTION,
|
|
324
171
|
promptSnippet: "read, write, edit, bash",
|
|
325
|
-
parameters:
|
|
326
|
-
code: Type.Optional(Type.String({ maxLength: config.maxCodeChars ?? 48000 })),
|
|
327
|
-
file: Type.Optional(Type.String({ minLength: 1 })),
|
|
328
|
-
data: Type.Optional(Type.Unknown()),
|
|
329
|
-
timeoutMs: Type.Optional(Type.Integer({ minimum: 1000 })),
|
|
330
|
-
programs: Type.Optional(Type.Array(Type.Object({
|
|
331
|
-
code: Type.Optional(Type.String({ maxLength: config.maxCodeChars ?? 48000 })),
|
|
332
|
-
file: Type.Optional(Type.String({ minLength: 1 })),
|
|
333
|
-
data: Type.Optional(Type.Unknown()),
|
|
334
|
-
}, {additionalProperties:false}), {minItems:1,maxItems:32})),
|
|
335
|
-
parallel: Type.Optional(Type.Boolean()),
|
|
336
|
-
mergeData: Type.Optional(Type.Boolean()),
|
|
337
|
-
}),
|
|
172
|
+
parameters: programParameters(config),
|
|
338
173
|
// One self-owned result frame is shared by Pi and OMP; renderCall stays empty
|
|
339
174
|
// so separate call/result slots cannot duplicate the lifecycle card.
|
|
340
175
|
renderShell: "self",
|
|
@@ -366,7 +201,7 @@ export function registerCodeMode(pi) {
|
|
|
366
201
|
runBridge.close();
|
|
367
202
|
|
|
368
203
|
while (runBridge.getOverlayDepth()) runBridge.rollbackSpeculation();
|
|
369
|
-
outcome = { ok: false, error:
|
|
204
|
+
outcome = { ok: false, error: errorMessage(error), logs: outcome?.logs ?? [], wallMs: Math.round(performance.now() - started) };
|
|
370
205
|
} finally {
|
|
371
206
|
peakSeen = Math.max(peakSeen, overlapPeak);
|
|
372
207
|
inFlight -= 1;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-supernova",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.9.1",
|
|
4
4
|
"description": "CodeMode for Pi and OMP: read, edit, write and bash, with transactional files, source views and shared-input program batches.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"author": "AdityaVG13",
|
|
@@ -76,6 +76,7 @@
|
|
|
76
76
|
"homepage": "https://github.com/AdityaVG13/pi-stack/tree/main/packages/pi-supernova#readme",
|
|
77
77
|
"dependencies": {
|
|
78
78
|
"acorn": "^8.18.0",
|
|
79
|
+
"sharp": "0.35.4",
|
|
79
80
|
"string-width": "^8.2.2"
|
|
80
81
|
}
|
|
81
82
|
}
|
package/src/adapters/bash.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import * as fs from "node:fs/promises";
|
|
2
|
-
|
|
2
|
+
|
|
3
3
|
import { normalizeBash } from "../contract/bash.js";
|
|
4
4
|
import { sourceForReferences } from "../fs/source-window.js";
|
|
5
5
|
import { resolveWorkspacePath, runCommand, clearPathCache } from "../fs/workspace.js";
|
|
@@ -10,38 +10,13 @@ export function createBash(ctx) {
|
|
|
10
10
|
return stdout && stderr ? stdout + (stdout.endsWith("\n") ? "" : "\n") + stderr : stdout || stderr;
|
|
11
11
|
}
|
|
12
12
|
|
|
13
|
-
function isLiteralArgv(params) {
|
|
14
|
-
return Array.isArray(params?.args) && process.platform !== "win32" && params.args.length === Object.keys(params.args).length && params.args.every(isString);
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
function bashCommand(params, literal) {
|
|
18
|
-
if (params?.command !== undefined && !isString(params.command)) throw new Error("bash command must be a string");
|
|
19
|
-
if (literal && (!isString(params.command) || params.args.some(arg => !isString(arg)))) throw new Error("bash argv requires a command string and an array of string args");
|
|
20
|
-
const command = String(params?.command ?? "");
|
|
21
|
-
|
|
22
|
-
if (!command.trim()) throw new Error("bash requires command");
|
|
23
|
-
|
|
24
|
-
return command;
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
function parseBash(params) {
|
|
28
|
-
const literal = isLiteralArgv(params);
|
|
29
|
-
const command = bashCommand(params, literal);
|
|
30
|
-
|
|
31
|
-
return { literal, command, argv: literal ? [command, ...params.args] : ["bash", "-c", command] };
|
|
32
|
-
}
|
|
33
|
-
|
|
34
13
|
async function bash(params, signal) {
|
|
35
14
|
params = normalizeBash(params);
|
|
36
15
|
const cwd = getCwd();
|
|
37
|
-
const
|
|
38
|
-
const
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
const st = await fs.stat(targetCwd).catch(() => null);
|
|
42
|
-
|
|
43
|
-
if (!st?.isDirectory()) throw new Error("bash cwd is not a directory: " + params.cwd);
|
|
44
|
-
}
|
|
16
|
+
const command = String(params.command);
|
|
17
|
+
const literal = Array.isArray(params.args);
|
|
18
|
+
const argv = literal ? [command, ...params.args] : ["bash", "-c", command];
|
|
19
|
+
const targetCwd = await commandCwd(params, cwd);
|
|
45
20
|
|
|
46
21
|
const transactionBarrier = await vfs.prepareExternalMutation("bash");
|
|
47
22
|
let res;
|
|
@@ -84,3 +59,12 @@ export function createBash(ctx) {
|
|
|
84
59
|
|
|
85
60
|
return { bash };
|
|
86
61
|
}
|
|
62
|
+
|
|
63
|
+
async function commandCwd(params, cwd) {
|
|
64
|
+
const target = params.cwd ? await resolveWorkspacePath(cwd, params.cwd, "bash cwd", true) : cwd;
|
|
65
|
+
if (params.cwd !== undefined) {
|
|
66
|
+
const stat = await fs.stat(target).catch(() => null);
|
|
67
|
+
if (!stat?.isDirectory()) throw new Error("bash cwd is not a directory: " + params.cwd);
|
|
68
|
+
}
|
|
69
|
+
return target;
|
|
70
|
+
}
|
package/src/adapters/errors.js
CHANGED
|
@@ -1,15 +1,7 @@
|
|
|
1
1
|
export const IMAGE_MAX_BYTES = 20 * 1024 * 1024;
|
|
2
2
|
export const LARGE_FILE_BYTES = 512 * 1024;
|
|
3
|
+
export const TEXT_MAX_BYTES = 64 * 1024 * 1024;
|
|
3
4
|
|
|
4
|
-
/** Raw path-only reads above these must use json/about/offset/complete. */
|
|
5
|
-
export const RAW_JSON_CHARS = 4096;
|
|
6
|
-
|
|
7
|
-
export const RAW_SOURCE_CHARS = 8192;
|
|
8
|
-
|
|
9
|
-
export const RAW_SOURCE_LINES = 160;
|
|
10
|
-
|
|
11
|
-
/** Routing responses above this fall back to the bound error instead of dumping. */
|
|
12
|
-
export const ROUTING_MAX_CHARS = 4096;
|
|
13
5
|
export const ABOUT_TOKEN_MAX = 16;
|
|
14
6
|
export const IMAGE_MIME = {
|
|
15
7
|
".png": "image/png",
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import * as fs from 'node:fs/promises';
|
|
2
|
+
import * as path from 'node:path';
|
|
3
|
+
import {tokenizeQuery,stem} from '../context/snap.js';
|
|
4
|
+
import {runCommand} from '../fs/workspace.js';
|
|
5
|
+
import {lineStartIndex,lineTextRange} from '../fs/lines.js';
|
|
6
|
+
import {textResult} from '../shared/result.js';
|
|
7
|
+
export function createFocusedReader(vfs, readBudget) {
|
|
8
|
+
function aboutStems(about, requireStem) {
|
|
9
|
+
const tokens = tokenizeQuery(about).tokens;
|
|
10
|
+
|
|
11
|
+
if (tokens.length > 16) throw new Error("about is too broad; use at most 16 keywords");
|
|
12
|
+
const stems = [...new Set(tokens.map(token => stem(token).slice(0, 128)))];
|
|
13
|
+
|
|
14
|
+
if (requireStem && !stems.length) throw new Error("about needs at least one searchable keyword");
|
|
15
|
+
|
|
16
|
+
return stems;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function overlayHits(overlay, stems, signal) {
|
|
20
|
+
const hits = [];
|
|
21
|
+
let line = 1;
|
|
22
|
+
let start = 0;
|
|
23
|
+
|
|
24
|
+
while (start <= overlay.length) {
|
|
25
|
+
signal?.throwIfAborted();
|
|
26
|
+
const newline = overlay.indexOf("\n", start);
|
|
27
|
+
const end = newline < 0 ? overlay.length : newline + 1;
|
|
28
|
+
const row = overlay.slice(start, newline < 0 ? end : newline).replace(/\r$/, "").toLowerCase();
|
|
29
|
+
|
|
30
|
+
if (stems.some(st => row.includes(st))) {
|
|
31
|
+
hits.push(line);
|
|
32
|
+
if (hits.length >= 200) break;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
if (end === overlay.length) break;
|
|
36
|
+
start = end;
|
|
37
|
+
line++;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
return { hits, lineCount: line };
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function overlayWindows(rel, overlay, hits, lineCount, budget) {
|
|
44
|
+
const out = [];
|
|
45
|
+
let cursor = 1;
|
|
46
|
+
let used = 0;
|
|
47
|
+
let truncated = hits.length >= 200;
|
|
48
|
+
|
|
49
|
+
for (const hit of hits) {
|
|
50
|
+
const from = Math.max(cursor, hit - 3);
|
|
51
|
+
const first = lineStartIndex(overlay, from);
|
|
52
|
+
const last = lineTextRange(overlay, Math.min(hit + 3, lineCount)).end;
|
|
53
|
+
const body = overlay.slice(first, last);
|
|
54
|
+
|
|
55
|
+
if (used + body.length > budget) { truncated = true; break; }
|
|
56
|
+
if (out.length && from > cursor) out.push("...");
|
|
57
|
+
out.push(`// ${rel}:${from}\n${body}`);
|
|
58
|
+
used += body.length;
|
|
59
|
+
cursor = Math.max(cursor, hit + 4);
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
return { out, truncated };
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
async function focusDisk(rel, about, targetPath, stems, budget, signal) {
|
|
66
|
+
const args = ["rg", "--fixed-strings", "--ignore-case", "--line-number", "--before-context", "3", "--after-context", "3"];
|
|
67
|
+
|
|
68
|
+
for (const token of stems) args.push("-e", token);
|
|
69
|
+
args.push("--", targetPath);
|
|
70
|
+
const observed = await fs.stat(targetPath);
|
|
71
|
+
const res = await runCommand(args, { cwd: path.dirname(targetPath), timeoutMs: 15000, maxOutputChars: budget, signal });
|
|
72
|
+
|
|
73
|
+
if (res.exitCode === 0 || res.exitCode === 1) await vfs.recordExpected(targetPath, observed);
|
|
74
|
+
if (res.exitCode === 1) return textResult("// " + rel + " · no matching text\n", { path: targetPath, outputTruncated: false, complete: false });
|
|
75
|
+
if (res.exitCode !== 0) throw new Error(res.stderr.trim() || `rg exited ${res.exitCode}`);
|
|
76
|
+
const marker = res.outputTruncated ? "\n[focused read truncated; narrow about or use read(path, line, count)]" : "";
|
|
77
|
+
|
|
78
|
+
return textResult("// " + rel + " · focused text windows (not a complete file); read(path, line, count) for raw text\n" + res.stdout + marker,
|
|
79
|
+
{ path: targetPath, outputTruncated: res.outputTruncated, complete: false });
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
async function focusAbout({ rel, about, overlay, targetPath, signal }) {
|
|
83
|
+
const stems = aboutStems(about, overlay === undefined);
|
|
84
|
+
const budget = readBudget(false);
|
|
85
|
+
|
|
86
|
+
if (overlay === undefined) return focusDisk(rel, about, targetPath, stems, budget, signal);
|
|
87
|
+
const { hits, lineCount } = overlayHits(overlay, stems, signal);
|
|
88
|
+
const { out, truncated } = overlayWindows(rel, overlay, hits, lineCount, budget);
|
|
89
|
+
|
|
90
|
+
if (!out.length) return textResult("// " + rel + (hits.length
|
|
91
|
+
? " · matching text exceeds view budget; first match at line " + hits[0] + "; use read(path, line, count)\n"
|
|
92
|
+
: " · no matching staged text\n"), { path: targetPath, outputTruncated: truncated, complete: false });
|
|
93
|
+
const marker = truncated ? "\n[focused read truncated; narrow about or use read(path, line, count)]" : "";
|
|
94
|
+
|
|
95
|
+
return textResult("// " + rel + " · focused staged text windows (not a complete file)\n" + out.join("\n") + marker, { path: targetPath, outputTruncated: truncated, complete: false });
|
|
96
|
+
}
|
|
97
|
+
return focusAbout;
|
|
98
|
+
}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import * as fs from 'node:fs/promises';
|
|
2
|
+
import * as path from 'node:path';
|
|
3
|
+
import {assertModelImageMime} from '../shared/decode.js';
|
|
4
|
+
import {validateImageBytes} from '../shared/image.js';
|
|
5
|
+
import {IMAGE_MIME,IMAGE_MAX_BYTES,imageTooLarge,missingFile} from './errors.js';
|
|
6
|
+
export function createImageReader(vfs) {
|
|
7
|
+
async function readImage(rel, targetPath, mime, signal) {
|
|
8
|
+
const staged = vfs.getOverlay(targetPath);
|
|
9
|
+
|
|
10
|
+
if (staged !== undefined) {
|
|
11
|
+
const size = Buffer.byteLength(staged, "utf8");
|
|
12
|
+
|
|
13
|
+
if (size > IMAGE_MAX_BYTES) throw imageTooLarge(rel, size);
|
|
14
|
+
|
|
15
|
+
return Buffer.from(staged);
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
let file;
|
|
19
|
+
|
|
20
|
+
try { file = await fs.open(targetPath, fs.constants.O_RDONLY | (fs.constants.O_NONBLOCK ?? 0)); }
|
|
21
|
+
catch (error) {
|
|
22
|
+
if (error.code === "ENOENT") throw missingFile(targetPath);
|
|
23
|
+
throw error;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
try {
|
|
27
|
+
const stat = await file.stat();
|
|
28
|
+
|
|
29
|
+
if (!stat.isFile()) throw new Error("image read requires a regular file: " + targetPath);
|
|
30
|
+
if (stat.size > IMAGE_MAX_BYTES) throw imageTooLarge(rel, stat.size);
|
|
31
|
+
const bytes = await file.readFile({ signal });
|
|
32
|
+
await vfs.recordExpected(targetPath, stat);
|
|
33
|
+
|
|
34
|
+
return bytes;
|
|
35
|
+
} finally { await file.close(); }
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
async function maybeImage(rel, targetPath, signal) {
|
|
39
|
+
const mime = IMAGE_MIME[path.extname(targetPath).toLowerCase()];
|
|
40
|
+
|
|
41
|
+
if (!mime) return null;
|
|
42
|
+
assertModelImageMime(mime);
|
|
43
|
+
const bytes = await readImage(rel, targetPath, mime, signal);
|
|
44
|
+
|
|
45
|
+
if (bytes.length > IMAGE_MAX_BYTES) throw imageTooLarge(rel, bytes.length);
|
|
46
|
+
await validateImageBytes(bytes,mime,rel,signal);
|
|
47
|
+
|
|
48
|
+
return { content: [{ type: "image", mimeType: mime, data: bytes.toString("base64") }], details: { path: targetPath } };
|
|
49
|
+
}
|
|
50
|
+
return maybeImage;
|
|
51
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import {errorMessage} from "../shared/decode.js";
|
|
2
|
+
import {MAX_JSON_BYTES,jsonProjector} from '../fs/json-read.js';
|
|
3
|
+
import {jsonErrorContext} from '../shared/syntax-context.js';
|
|
4
|
+
import {readResult,readValueBytes,MAX_READ_VALUE_BYTES} from '../shared/result.js';
|
|
5
|
+
|
|
6
|
+
function selectJsonParts(project, document, rel, selectors) {
|
|
7
|
+
const parts = [];
|
|
8
|
+
let remaining = MAX_READ_VALUE_BYTES - 32 - selectors.length * 8;
|
|
9
|
+
|
|
10
|
+
try {
|
|
11
|
+
for (const value of project(document)) {
|
|
12
|
+
remaining -= readValueBytes(value, remaining);
|
|
13
|
+
parts.push(value);
|
|
14
|
+
}
|
|
15
|
+
} catch (error) {
|
|
16
|
+
throw new Error("JSON selection failed for " + rel + " (" + selectors.join(", ") + "): " + (errorMessage(error)));
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
return {parts, bytes: MAX_READ_VALUE_BYTES - remaining};
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** RFC 8259 lets parsers ignore one leading BOM; files from Windows tools carry it. */
|
|
23
|
+
const stripBom = text => text.charCodeAt(0) === 0xfeff ? text.slice(1) : text;
|
|
24
|
+
|
|
25
|
+
async function projectJson(rel, targetPath, params, vfs) {
|
|
26
|
+
const project = jsonProjector(params.json);
|
|
27
|
+
const text = stripBom(await vfs.read(targetPath, { maxBytes: MAX_JSON_BYTES, label: "JSON input" }));
|
|
28
|
+
let document;
|
|
29
|
+
|
|
30
|
+
try { document = JSON.parse(text); }
|
|
31
|
+
catch (error) {
|
|
32
|
+
throw new Error("invalid JSON in " + rel + ": " + error.message + "; the entire document must parse before projection" + jsonErrorContext(error.message, text));
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const many = Array.isArray(params.json);
|
|
36
|
+
const selectors = many ? params.json.map(String) : [params.json === true ? "." : String(params.json)];
|
|
37
|
+
const {parts, bytes} = selectJsonParts(project, document, rel, selectors);
|
|
38
|
+
|
|
39
|
+
return readResult(many ? parts : parts[0], { path: targetPath, json: true, jsonMany: many, complete: true }, "JSON selection: " + rel + " (" + selectors.join(", ") + ")", bytes);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export { projectJson };
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import {isString,isNumber} from '../shared/decode.js';
|
|
2
|
+
import {WorkspaceIndex} from '../context/repo-index.js';
|
|
3
|
+
import {pickSpan} from '../context/spans.js';
|
|
4
|
+
import {outlineFile} from '../context/outline.js';
|
|
5
|
+
import {readResult} from '../shared/result.js';
|
|
6
|
+
import {sliceLinesRaw,sourceLines,contentLineInfo} from '../fs/lines.js';
|
|
7
|
+
import {LARGE_FILE_BYTES,TEXT_MAX_BYTES} from './errors.js';
|
|
8
|
+
import {outlineOptions,recordOutlineOrigins,createReferenceFinder} from './refs.js';
|
|
9
|
+
|
|
10
|
+
function needsSourceIndex(params,query) {
|
|
11
|
+
return isString(params.about) || (params.resolve && isString(query));
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export function createTextReader(ctx, readWindow) {
|
|
15
|
+
const {getCwd,vfs,config,index,ledger}=ctx;
|
|
16
|
+
const referenceFinder=createReferenceFinder(index,vfs);
|
|
17
|
+
|
|
18
|
+
async function loadText(targetPath, params, query, signal) {
|
|
19
|
+
const explicit = isNumber(params.offset) || isNumber(params.limit);
|
|
20
|
+
const windowed = explicit && params.complete !== true && !isString(params.about) && !isString(query);
|
|
21
|
+
if (!windowed) {
|
|
22
|
+
// Whole reads hash the bytes already loaded. Do not read once for a window
|
|
23
|
+
// and open/hash the entire file a second time just to return the same text.
|
|
24
|
+
return {text:await vfs.read(targetPath,{maxBytes:TEXT_MAX_BYTES}),windowed:false};
|
|
25
|
+
}
|
|
26
|
+
const start = isNumber(params.offset) ? Math.max(1,Math.floor(params.offset)) : 1;
|
|
27
|
+
const count = isNumber(params.limit) ? Math.max(0,Math.floor(params.limit)) : undefined;
|
|
28
|
+
const window = await readWindow(targetPath,start,count,TEXT_MAX_BYTES,signal);
|
|
29
|
+
if (!window.satisfied) throw new Error("read window exceeds " + TEXT_MAX_BYTES + " bytes: " + targetPath + "; request fewer lines or use a bounded parser through bash");
|
|
30
|
+
return {text:window.text,windowed:true,windowWhole:window.whole};
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
async function maybeOutline(cwd, rel, targetPath, text, params, entry) {
|
|
34
|
+
if (!isString(params.about)) return null;
|
|
35
|
+
const outline = entry && outlineFile(entry,rel,params.about,outlineOptions(params,await referenceFinder(cwd,targetPath),config));
|
|
36
|
+
if (!outline) return null;
|
|
37
|
+
recordOutlineOrigins(ledger,rel,outline.text);
|
|
38
|
+
return readResult(outline.text,{path:targetPath,outline:true,expanded:outline.expanded,declarations:outline.declarations});
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function resolveSpan(entry, sourceLine, query, params) {
|
|
42
|
+
if (!params.resolve || !isString(query) || !entry) return {offset:params.offset,limit:params.limit,viewComplete:undefined};
|
|
43
|
+
const span=pickSpan(WorkspaceIndex.spansOf(entry),{line:sourceLine,name:query});
|
|
44
|
+
if (!span) return {offset:params.offset,limit:params.limit,viewComplete:undefined};
|
|
45
|
+
const spanLines=span.end-span.start+1;
|
|
46
|
+
const limit=isNumber(params.limit) ? Math.min(params.limit,spanLines) : spanLines;
|
|
47
|
+
return {offset:span.start,limit,viewComplete:limit>=spanLines};
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function textFileResult(rel,targetPath,loaded,span,sliced,firstLine,explicit) {
|
|
51
|
+
const lines=sliced.length<=LARGE_FILE_BYTES ? sourceLines(sliced) : null;
|
|
52
|
+
if (lines) ledger.recordOrigin(rel,firstLine,lines,explicit);
|
|
53
|
+
return readResult(sliced,{path:targetPath,firstLine,lastLine:firstLine+(lines?.length ?? contentLineInfo(sliced).count)-1,
|
|
54
|
+
sourceChars:sliced.length,complete:loaded.windowed ? loaded.windowWhole : sliced===loaded.text,viewComplete:span.viewComplete});
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
async function readTextFile(targetPath,params,sourceLine,rel,query,signal) {
|
|
58
|
+
const explicit=isNumber(params.offset)||isNumber(params.limit);
|
|
59
|
+
const loaded=await loadText(targetPath,params,query,signal);
|
|
60
|
+
index.touch(rel);
|
|
61
|
+
const entry=needsSourceIndex(params,query) ? WorkspaceIndex.fromText(targetPath,loaded.text) : null;
|
|
62
|
+
const outlined=await maybeOutline(getCwd(),rel,targetPath,loaded.text,params,entry);
|
|
63
|
+
if (outlined) return outlined;
|
|
64
|
+
const span=resolveSpan(entry,sourceLine,query,params);
|
|
65
|
+
const firstLine=isNumber(span.offset) ? Math.max(1,Math.floor(span.offset)) : 1;
|
|
66
|
+
const sliced=loaded.windowed ? loaded.text : sliceLinesRaw(loaded.text,span.offset,span.limit);
|
|
67
|
+
if (params.complete===true && sliced!==loaded.text) throw new Error("complete:true requires the whole file; remove offset/limit for " + rel);
|
|
68
|
+
return textFileResult(rel,targetPath,loaded,span,sliced,firstLine,explicit);
|
|
69
|
+
}
|
|
70
|
+
return readTextFile;
|
|
71
|
+
}
|