pi-supernova 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -3
- package/docs/CHANGELOG.md +104 -0
- package/docs/TOKEN_COSTS.md +13 -5
- package/index.js +120 -79
- package/package.json +1 -1
- package/src/adapters/bash.js +73 -0
- package/src/adapters/edit.js +249 -0
- package/src/adapters/errors.js +31 -0
- package/src/adapters/index.js +31 -0
- package/src/adapters/list.js +102 -0
- package/src/adapters/read.js +805 -0
- package/src/adapters/refs.js +41 -0
- package/src/adapters/write.js +96 -0
- package/src/bridge/catalog.js +28 -222
- package/src/bridge/host-bridge.js +113 -1668
- package/src/bridge/invoke.js +35 -0
- package/src/bridge/native-tools.js +1 -198
- package/src/context/evidence.js +140 -76
- package/src/context/fuzzy.js +42 -24
- package/src/context/ledger.js +43 -24
- package/src/context/outline.js +23 -18
- package/src/context/repo-index.js +206 -170
- package/src/context/search.js +157 -77
- package/src/context/snap.js +240 -136
- package/src/context/spans.js +2 -1
- package/src/context/surface.js +14 -5
- package/src/contract/bash.js +31 -0
- package/src/contract/edit.js +95 -0
- package/src/contract/read.js +220 -0
- package/src/fs/check.js +12 -8
- package/src/fs/diff.js +18 -7
- package/src/fs/json-read.js +66 -35
- package/src/fs/patch.js +94 -50
- package/src/fs/source-window.js +82 -0
- package/src/fs/text-ops.js +512 -0
- package/src/fs/vfs.js +205 -175
- package/src/fs/workspace.js +119 -108
- package/src/output/bottleneck.js +195 -116
- package/src/output/format.js +101 -67
- package/src/runtime/guest-deny-imports.js +34 -0
- package/src/runtime/guest-worker.js +289 -292
- package/src/runtime/parallel.js +97 -64
- package/src/runtime/program-batch.js +178 -69
- package/src/runtime/reference.js +15 -14
- package/src/runtime/runtime.js +327 -187
- package/src/shared/decode.js +58 -36
- package/src/ui/omp-frame.js +59 -42
- package/src/ui/render-measure.js +51 -29
- package/src/ui/render.js +241 -145
package/src/context/ledger.js
CHANGED
|
@@ -39,19 +39,23 @@ function newHistory() {
|
|
|
39
39
|
// Experimental candidate collection, not proof of provider-visible retention:
|
|
40
40
|
// AgentMessage metadata and later context transformations can hide these strings.
|
|
41
41
|
// Keep this optimization opt-in until exact outgoing citation targets are validated.
|
|
42
|
-
function
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
if (value.length > MAX_OBSERVED_LINE && !value.includes("\n")) return;
|
|
42
|
+
function retainString(value, retained) {
|
|
43
|
+
// Every line, not only substantive ones. A run may span a blank or short line,
|
|
44
|
+
// so isRetained must be exact per line or the run truncates there. An oversized
|
|
45
|
+
// line is never stored: it cannot sit inside a six-line run, and this is what
|
|
46
|
+
// keeps base64 image payloads out of the retention set.
|
|
47
|
+
if (value.length > MAX_OBSERVED_LINE && !value.includes("\n")) return;
|
|
49
48
|
|
|
50
|
-
|
|
51
|
-
|
|
49
|
+
for (const line of value.split("\n")) {
|
|
50
|
+
if (retained.size >= MAX_STORED_LINES) return;
|
|
52
51
|
|
|
53
|
-
|
|
54
|
-
|
|
52
|
+
if (line.length <= MAX_OBSERVED_LINE) retained.add(line);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function collectRetained(value, retained, depth = 0) {
|
|
57
|
+
if (isString(value)) {
|
|
58
|
+
retainString(value, retained);
|
|
55
59
|
|
|
56
60
|
return;
|
|
57
61
|
}
|
|
@@ -174,7 +178,7 @@ export class SeenLedger {
|
|
|
174
178
|
return count;
|
|
175
179
|
}
|
|
176
180
|
|
|
177
|
-
|
|
181
|
+
bestCandidateRun(lines, hashes, index, call) {
|
|
178
182
|
const candidates = this.occurrences.get(hashes[index]);
|
|
179
183
|
|
|
180
184
|
if (!candidates) return null;
|
|
@@ -186,12 +190,23 @@ export class SeenLedger {
|
|
|
186
190
|
if (length >= MIN_RUN && (!best || length > best.length)) best = { ...candidate, length };
|
|
187
191
|
}
|
|
188
192
|
|
|
189
|
-
|
|
193
|
+
return best;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
substantiveCount(lines, index, length) {
|
|
190
197
|
let count = 0;
|
|
191
198
|
|
|
192
|
-
for (let i = index; i < index +
|
|
199
|
+
for (let i = index; i < index + length; i++) if (collapsible(lines[i])) count++;
|
|
193
200
|
|
|
194
|
-
return count
|
|
201
|
+
return count;
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
longestRun(lines, hashes, index, call) {
|
|
205
|
+
const best = this.bestCandidateRun(lines, hashes, index, call);
|
|
206
|
+
|
|
207
|
+
if (!best) return null;
|
|
208
|
+
|
|
209
|
+
return this.substantiveCount(lines, index, best.length) >= MIN_SUBSTANTIVE ? best : null;
|
|
195
210
|
}
|
|
196
211
|
|
|
197
212
|
citation(lines, index, run) {
|
|
@@ -245,19 +260,14 @@ export class SeenLedger {
|
|
|
245
260
|
return sent;
|
|
246
261
|
}
|
|
247
262
|
|
|
248
|
-
|
|
249
|
-
if (this.window === 0 || call <= this.history.latestCall - this.window || lines.length > MAX_STORED_LINES) return;
|
|
250
|
-
|
|
251
|
-
if (this.results.has(call)) this.forget(call);
|
|
252
|
-
|
|
263
|
+
evictForCapacity(incoming) {
|
|
253
264
|
for (const old of [...this.results.keys()].sort((a, b) => a - b)) {
|
|
254
|
-
if (this.storedLines +
|
|
265
|
+
if (this.storedLines + incoming <= MAX_STORED_LINES) break;
|
|
255
266
|
this.forget(old);
|
|
256
267
|
}
|
|
268
|
+
}
|
|
257
269
|
|
|
258
|
-
|
|
259
|
-
const origins = lines.map(line => this.origins.get(line));
|
|
260
|
-
|
|
270
|
+
indexCollapsible(lines, hashes, call) {
|
|
261
271
|
for (let i = 0; i < lines.length; i++) {
|
|
262
272
|
if (!collapsible(lines[i])) continue;
|
|
263
273
|
let list = this.occurrences.get(hashes[i]);
|
|
@@ -265,7 +275,16 @@ export class SeenLedger {
|
|
|
265
275
|
if (!list) this.occurrences.set(hashes[i], (list = []));
|
|
266
276
|
list.push({ call, index: i });
|
|
267
277
|
}
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
remember(call, lines) {
|
|
281
|
+
if (this.window === 0 || call <= this.history.latestCall - this.window || lines.length > MAX_STORED_LINES) return;
|
|
268
282
|
|
|
283
|
+
if (this.results.has(call)) this.forget(call);
|
|
284
|
+
this.evictForCapacity(lines.length);
|
|
285
|
+
const hashes = Uint32Array.from(lines, hashLine);
|
|
286
|
+
const origins = lines.map(line => this.origins.get(line));
|
|
287
|
+
this.indexCollapsible(lines, hashes, call);
|
|
269
288
|
this.results.set(call, { hashes, lines, origins });
|
|
270
289
|
this.history.storedLines += lines.length;
|
|
271
290
|
}
|
package/src/context/outline.js
CHANGED
|
@@ -97,16 +97,7 @@ function focusedText(raw, lower, stems, relPath, opts) {
|
|
|
97
97
|
* @param entry index entry (text + cached lines/surface)
|
|
98
98
|
* @param about question or symbol; empty ⇒ pure skeleton (every body folded)
|
|
99
99
|
*/
|
|
100
|
-
|
|
101
|
-
const opts = { ...OUTLINE_DEFAULTS, ...options };
|
|
102
|
-
const { raw, lower } = WorkspaceIndex.linesOf(entry);
|
|
103
|
-
const lineCount = raw.length;
|
|
104
|
-
const spans = WorkspaceIndex.spansOf(entry).map((s) => ({ ...s, signature: raw[s.start - 1].trim() }));
|
|
105
|
-
const stems = [...new Set(tokenizeQuery(about || "").tokens.map(stem))];
|
|
106
|
-
|
|
107
|
-
if (spans.length === 0) return about ? focusedText(raw, lower, stems, relPath, opts) : null;
|
|
108
|
-
const expanded = chooseExpanded(spans, lower, stems, raw, opts);
|
|
109
|
-
|
|
100
|
+
function outlineHeader(spans, raw, opts) {
|
|
110
101
|
const parts = [];
|
|
111
102
|
const headerEnd = Math.min(spans[0].start - 1, opts.headerLines);
|
|
112
103
|
|
|
@@ -118,16 +109,30 @@ export function outlineFile(entry, relPath, about, options = {}) {
|
|
|
118
109
|
if (spans[0].start - 1 > opts.headerLines) parts.push(" … " + (spans[0].start - 1 - opts.headerLines) + " more header lines");
|
|
119
110
|
}
|
|
120
111
|
|
|
112
|
+
return parts;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function clipOutline(text, title, maxChars) {
|
|
116
|
+
if (text.length <= maxChars) return text;
|
|
117
|
+
const end = text.lastIndexOf("\n", Math.max(0, maxChars - 160));
|
|
118
|
+
|
|
119
|
+
return (end > title.length ? text.slice(0, end) : text.slice(0, maxChars)) + "\n … outline truncated; use read(path, line, count) for later declarations";
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
export function outlineFile(entry, relPath, about, options = {}) {
|
|
123
|
+
const opts = { ...OUTLINE_DEFAULTS, ...options };
|
|
124
|
+
const { raw, lower } = WorkspaceIndex.linesOf(entry);
|
|
125
|
+
const lineCount = raw.length;
|
|
126
|
+
const spans = WorkspaceIndex.spansOf(entry).map((s) => ({ ...s, signature: raw[s.start - 1].trim() }));
|
|
127
|
+
const stems = [...new Set(tokenizeQuery(about || "").tokens.map(stem))];
|
|
128
|
+
|
|
129
|
+
if (spans.length === 0) return about ? focusedText(raw, lower, stems, relPath, opts) : null;
|
|
130
|
+
const expanded = chooseExpanded(spans, lower, stems, raw, opts);
|
|
131
|
+
const parts = outlineHeader(spans, raw, opts);
|
|
132
|
+
|
|
121
133
|
for (let i = 0; i < spans.length; i++) parts.push(expanded.has(i) ? expandedBlock(spans[i], raw, opts) : foldedLine(spans[i]));
|
|
122
134
|
const label = about ? String(about).replace(/\s+/g, " ").slice(0, 120) : "";
|
|
123
135
|
const title = "// " + relPath + " · " + lineCount + " lines · " + spans.length + " declarations · " + expanded.size + " expanded" + (label ? " for \"" + label + "\"" : "") + " · read(path, line, count) for a folded body";
|
|
124
|
-
let text = title + "\n" + parts.join("\n");
|
|
125
|
-
|
|
126
|
-
if (text.length > opts.maxChars) {
|
|
127
|
-
const end = text.lastIndexOf("\n", Math.max(0, opts.maxChars - 160));
|
|
128
|
-
|
|
129
|
-
text = (end > title.length ? text.slice(0, end) : text.slice(0, opts.maxChars)) + "\n … outline truncated; use read(path, line, count) for later declarations";
|
|
130
|
-
}
|
|
131
136
|
|
|
132
|
-
return { text, expanded: expanded.size, declarations: spans.length };
|
|
137
|
+
return { text: clipOutline(title + "\n" + parts.join("\n"), title, opts.maxChars), expanded: expanded.size, declarations: spans.length };
|
|
133
138
|
}
|
|
@@ -101,35 +101,41 @@ export function globToRegExp(glob) {
|
|
|
101
101
|
return new RegExp(glob.includes("/") ? "^" + body + "$" : "(?:^|/)" + body + "$");
|
|
102
102
|
}
|
|
103
103
|
|
|
104
|
-
function
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
const base = indentOf(start - 1);
|
|
108
|
-
let end = start;
|
|
109
|
-
|
|
110
|
-
for (let i = start; i < lineCount; i++) {
|
|
111
|
-
if (lower[i] === "") { end = i + 1; continue; }
|
|
112
|
-
if (indentOf(i) <= base) break;
|
|
113
|
-
end = i + 1;
|
|
114
|
-
}
|
|
104
|
+
function lineIndent(raw, i) {
|
|
105
|
+
return raw[i].length - raw[i].trimStart().length;
|
|
106
|
+
}
|
|
115
107
|
|
|
116
|
-
|
|
108
|
+
function pythonDeclarationEnd(raw, lower, start, lineCount) {
|
|
109
|
+
const base = lineIndent(raw, start - 1);
|
|
110
|
+
let end = start;
|
|
111
|
+
|
|
112
|
+
for (let i = start; i < lineCount; i++) {
|
|
113
|
+
if (lower[i] === "") { end = i + 1; continue; }
|
|
114
|
+
if (lineIndent(raw, i) <= base) break;
|
|
115
|
+
end = i + 1;
|
|
117
116
|
}
|
|
118
117
|
|
|
118
|
+
return Math.min(end, lineCount);
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function braceDelta(text) {
|
|
119
122
|
let depth = 0;
|
|
120
123
|
|
|
121
|
-
for (const ch of
|
|
124
|
+
for (const ch of text) {
|
|
122
125
|
if (ch === "{") depth++;
|
|
123
126
|
else if (ch === "}") depth--;
|
|
124
127
|
}
|
|
125
128
|
|
|
129
|
+
return depth;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
function braceDeclarationEnd(raw, start, lineCount) {
|
|
133
|
+
let depth = braceDelta(raw[start - 1] ?? "");
|
|
134
|
+
|
|
126
135
|
if (depth <= 0) return start;
|
|
127
136
|
|
|
128
137
|
for (let i = start; i < raw.length; i++) {
|
|
129
|
-
|
|
130
|
-
if (ch === "{") depth++;
|
|
131
|
-
else if (ch === "}") depth--;
|
|
132
|
-
}
|
|
138
|
+
depth += braceDelta(raw[i]);
|
|
133
139
|
|
|
134
140
|
if (depth <= 0) return i + 1;
|
|
135
141
|
}
|
|
@@ -137,6 +143,89 @@ function declarationEnd(raw, lower, start, lineCount, ext) {
|
|
|
137
143
|
return lineCount;
|
|
138
144
|
}
|
|
139
145
|
|
|
146
|
+
function declarationEnd(raw, lower, start, lineCount, ext) {
|
|
147
|
+
if (ext === ".py") return pythonDeclarationEnd(raw, lower, start, lineCount);
|
|
148
|
+
|
|
149
|
+
return braceDeclarationEnd(raw, start, lineCount);
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
function readFdBuffer(fd, buffer) {
|
|
153
|
+
let offset = 0;
|
|
154
|
+
|
|
155
|
+
while (offset < buffer.length) {
|
|
156
|
+
const read = fs.readSync(fd, buffer, offset, buffer.length - offset, offset);
|
|
157
|
+
|
|
158
|
+
if (read <= 0) break;
|
|
159
|
+
offset += read;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
return offset;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
function parseRgFiles(res, root) {
|
|
166
|
+
let error;
|
|
167
|
+
|
|
168
|
+
if (res.exitCode !== 0 && res.exitCode !== 1) error = res.stderr.trim() || "rg exited with status " + res.exitCode;
|
|
169
|
+
const truncated = res.outputTruncated === true;
|
|
170
|
+
const output = truncated && !res.stdout.endsWith("\n") ? res.stdout.slice(0, res.stdout.lastIndexOf("\n") + 1) : res.stdout;
|
|
171
|
+
|
|
172
|
+
return { files: output.split("\n").flatMap(f => f ? [path.resolve(root, f)] : []).sort(), error, truncated, missing: false };
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
function addPorcelainRow(rows, i, set) {
|
|
176
|
+
const row = rows[i];
|
|
177
|
+
|
|
178
|
+
if (row.length <= 3) return i;
|
|
179
|
+
const status = row.slice(0, 2);
|
|
180
|
+
const file = row.slice(3);
|
|
181
|
+
|
|
182
|
+
if (file) set.add(file);
|
|
183
|
+
|
|
184
|
+
if ((status.includes("R") || status.includes("C")) && i + 1 < rows.length) {
|
|
185
|
+
const target = rows[i + 1];
|
|
186
|
+
|
|
187
|
+
if (target) set.add(target);
|
|
188
|
+
|
|
189
|
+
return i + 1;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
return i;
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
function ingestPorcelain(stdout, set) {
|
|
196
|
+
const rows = stdout.split("\0");
|
|
197
|
+
|
|
198
|
+
for (let i = 0; i < rows.length; i++) i = addPorcelainRow(rows, i, set);
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
function grepPendingHuge(pending, rel, regex, out) {
|
|
202
|
+
let start = 0;
|
|
203
|
+
let line = 0;
|
|
204
|
+
|
|
205
|
+
while (start <= pending.length) {
|
|
206
|
+
const end = pending.indexOf("\n", start);
|
|
207
|
+
const stop = end === -1 ? pending.length : end;
|
|
208
|
+
const text = pending.slice(start, stop).replace(/\r$/, "");
|
|
209
|
+
|
|
210
|
+
line++;
|
|
211
|
+
if (regex.test(text)) out.push({ rel, line, text, def: false });
|
|
212
|
+
if (end === -1) break;
|
|
213
|
+
start = end + 1;
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
function grepEntryRows(e, filePath, root, regex, nameRegex, out) {
|
|
218
|
+
const lineAnchored = /\^|\$/.test(regex.source.replace(/\\[\^$]|\[[^\]]*\]/g, ""));
|
|
219
|
+
|
|
220
|
+
if (!lineAnchored && !regex.test(e.text)) return;
|
|
221
|
+
const { raw, defNames } = WorkspaceIndex.linesOf(e);
|
|
222
|
+
const rel = relativeSlash(root, filePath);
|
|
223
|
+
|
|
224
|
+
for (let i = 0; i < raw.length; i++) {
|
|
225
|
+
if (regex.test(raw[i])) out.push({ rel, line: i + 1, text: raw[i], def: defNames[i] !== "" && nameRegex.test(defNames[i]) });
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
|
|
140
229
|
export class WorkspaceIndex {
|
|
141
230
|
constructor(runCommand) {
|
|
142
231
|
this.runCommand = runCommand;
|
|
@@ -164,6 +253,8 @@ export class WorkspaceIndex {
|
|
|
164
253
|
this.lastTouched = relPath;
|
|
165
254
|
}
|
|
166
255
|
|
|
256
|
+
getEntryBytes() { return this.entryBytes; }
|
|
257
|
+
|
|
167
258
|
watch(root) {
|
|
168
259
|
if (this.watchers.has(root)) return this.watchers.get(root);
|
|
169
260
|
let ok = false;
|
|
@@ -211,24 +302,7 @@ export class WorkspaceIndex {
|
|
|
211
302
|
try {
|
|
212
303
|
const res = await this.runCommand(["git", "status", "--porcelain", "-z", "--untracked-files=all"], { cwd: root, timeoutMs: 5_000 });
|
|
213
304
|
|
|
214
|
-
if (res.exitCode === 0)
|
|
215
|
-
const rows = res.stdout.split("\0");
|
|
216
|
-
|
|
217
|
-
for (let i = 0; i < rows.length; i++) {
|
|
218
|
-
const row = rows[i];
|
|
219
|
-
|
|
220
|
-
if (row.length <= 3) continue;
|
|
221
|
-
const status = row.slice(0, 2);
|
|
222
|
-
const file = row.slice(3);
|
|
223
|
-
|
|
224
|
-
if (file) set.add(file);
|
|
225
|
-
if ((status.includes("R") || status.includes("C")) && i + 1 < rows.length) {
|
|
226
|
-
const target = rows[++i];
|
|
227
|
-
|
|
228
|
-
if (target) set.add(target);
|
|
229
|
-
}
|
|
230
|
-
}
|
|
231
|
-
}
|
|
305
|
+
if (res.exitCode === 0) ingestPorcelain(res.stdout, set);
|
|
232
306
|
} catch {}
|
|
233
307
|
|
|
234
308
|
this.gitModified.set(root, set);
|
|
@@ -248,142 +322,95 @@ export class WorkspaceIndex {
|
|
|
248
322
|
}
|
|
249
323
|
}
|
|
250
324
|
|
|
251
|
-
|
|
252
|
-
async files(root, includeHidden = false, signal) {
|
|
253
|
-
const key = root + "\0" + (includeHidden ? "h" : "");
|
|
254
|
-
const cached = this.lists.get(key);
|
|
255
|
-
const ttl = this.watch(root) ? WATCHED_TTL_MS : LIST_TTL_MS;
|
|
256
|
-
|
|
257
|
-
if (cached && Date.now() - cached.at < ttl) return cached.files;
|
|
325
|
+
async listFilesWithRg(root, includeHidden, signal) {
|
|
258
326
|
const args = ["rg", "--files"];
|
|
259
327
|
|
|
260
328
|
if (includeHidden) args.push("--hidden");
|
|
261
329
|
args.push("-g", "!.git/**", "-g", "!**/.git/**", "--", root);
|
|
262
|
-
let files = [];
|
|
263
|
-
let error;
|
|
264
|
-
let truncated = false;
|
|
265
|
-
let missing = false;
|
|
266
330
|
|
|
267
331
|
try {
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
if (res.exitCode !== 0 && res.exitCode !== 1) error = res.stderr.trim() || "rg exited with status " + res.exitCode;
|
|
271
|
-
truncated = res.outputTruncated === true;
|
|
272
|
-
const output = truncated && !res.stdout.endsWith("\n") ? res.stdout.slice(0, res.stdout.lastIndexOf("\n") + 1) : res.stdout;
|
|
273
|
-
files = output.split("\n").filter(Boolean).map(f => path.resolve(root, f)).sort();
|
|
332
|
+
return parseRgFiles(await this.runCommand(args, { cwd: root, timeoutMs: 15_000, signal }), root);
|
|
274
333
|
} catch (err) {
|
|
275
334
|
signal?.throwIfAborted();
|
|
276
|
-
error = err.message;
|
|
277
|
-
missing = !fs.existsSync(root);
|
|
278
|
-
}
|
|
279
|
-
|
|
280
|
-
this.lists.set(key, { files, at: Date.now(), error, truncated, missing });
|
|
281
335
|
|
|
282
|
-
|
|
336
|
+
return { files: [], error: err.message, truncated: false, missing: !fs.existsSync(root) };
|
|
337
|
+
}
|
|
283
338
|
}
|
|
284
339
|
|
|
285
|
-
/**
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
try {
|
|
291
|
-
stat = fs.statSync(filePath);
|
|
292
|
-
} catch {
|
|
293
|
-
const previous = this.entries.get(filePath);
|
|
294
|
-
|
|
295
|
-
if (previous) this.entryBytes -= previous.weight ?? 0;
|
|
296
|
-
this.entries.delete(filePath);
|
|
340
|
+
/** Absolute, sorted file list for a root; gitignore-aware via rg; cached for LIST_TTL_MS. */
|
|
341
|
+
async files(root, includeHidden = false, signal) {
|
|
342
|
+
const key = root + "\0" + (includeHidden ? "h" : "");
|
|
343
|
+
const cached = this.lists.get(key);
|
|
344
|
+
const ttl = this.watch(root) ? WATCHED_TTL_MS : LIST_TTL_MS;
|
|
297
345
|
|
|
298
|
-
|
|
299
|
-
|
|
346
|
+
if (cached && Date.now() - cached.at < ttl) return cached.files;
|
|
347
|
+
const listed = await this.listFilesWithRg(root, includeHidden, signal);
|
|
348
|
+
this.lists.set(key, { ...listed, at: Date.now() });
|
|
300
349
|
|
|
301
|
-
|
|
302
|
-
|
|
350
|
+
return listed.files;
|
|
351
|
+
}
|
|
303
352
|
|
|
304
|
-
|
|
305
|
-
|
|
353
|
+
dropCached(filePath) {
|
|
354
|
+
const previous = this.entries.get(filePath);
|
|
306
355
|
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
356
|
+
if (previous) this.entryBytes -= previous.weight ?? 0;
|
|
357
|
+
this.entries.delete(filePath);
|
|
358
|
+
}
|
|
310
359
|
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
this.entries.set(filePath, cached);
|
|
360
|
+
rejectEntry(filePath) {
|
|
361
|
+
this.dropCached(filePath);
|
|
314
362
|
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
let text;
|
|
318
|
-
let actual = stat;
|
|
363
|
+
return null;
|
|
364
|
+
}
|
|
319
365
|
|
|
366
|
+
statOrNull(filePath) {
|
|
320
367
|
try {
|
|
321
|
-
|
|
368
|
+
return fs.statSync(filePath);
|
|
369
|
+
} catch {
|
|
370
|
+
return null;
|
|
371
|
+
}
|
|
372
|
+
}
|
|
322
373
|
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
if (!actual.isFile() || actual.size > MAX_FILE_BYTES) {
|
|
326
|
-
const previous = this.entries.get(filePath);
|
|
374
|
+
cachedHit(filePath, stat) {
|
|
375
|
+
const cached = this.entries.get(filePath);
|
|
327
376
|
|
|
328
|
-
|
|
329
|
-
|
|
377
|
+
if (!cached || cached.mtimeMs !== stat.mtimeMs || cached.size !== stat.size) return null;
|
|
378
|
+
this.entries.delete(filePath);
|
|
379
|
+
this.entries.set(filePath, cached);
|
|
330
380
|
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
const buffer = Buffer.alloc(MAX_FILE_BYTES + 1);
|
|
334
|
-
let offset = 0;
|
|
381
|
+
return cached;
|
|
382
|
+
}
|
|
335
383
|
|
|
336
|
-
|
|
337
|
-
|
|
384
|
+
readFdText(filePath, fd) {
|
|
385
|
+
let actual = fs.fstatSync(fd);
|
|
338
386
|
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
387
|
+
if (!actual.isFile() || actual.size > MAX_FILE_BYTES) return this.rejectEntry(filePath);
|
|
388
|
+
// One reusable scratch read per index: allocating 512 KiB per file lets
|
|
389
|
+
// thousands of dead buffers pile up as RSS before a major GC notices.
|
|
390
|
+
this.scratchRead ??= Buffer.alloc(MAX_FILE_BYTES + 1);
|
|
391
|
+
const offset = readFdBuffer(fd, this.scratchRead);
|
|
342
392
|
|
|
343
|
-
|
|
344
|
-
|
|
393
|
+
if (offset > MAX_FILE_BYTES) return this.rejectEntry(filePath);
|
|
394
|
+
actual = fs.fstatSync(fd);
|
|
345
395
|
|
|
346
|
-
|
|
347
|
-
this.entries.delete(filePath);
|
|
396
|
+
if (!actual.isFile() || actual.size !== offset) return this.rejectEntry(filePath);
|
|
348
397
|
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
actual = fs.fstatSync(fd);
|
|
352
|
-
if (!actual.isFile() || actual.size !== offset) {
|
|
353
|
-
const previous = this.entries.get(filePath);
|
|
398
|
+
return { text: this.scratchRead.subarray(0, offset).toString("utf8"), actual };
|
|
399
|
+
}
|
|
354
400
|
|
|
355
|
-
|
|
356
|
-
|
|
401
|
+
readIndexedText(filePath) {
|
|
402
|
+
try {
|
|
403
|
+
const fd = fs.openSync(filePath, fs.constants.O_RDONLY | (fs.constants.O_NONBLOCK ?? 0));
|
|
357
404
|
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
text = buffer.subarray(0, offset).toString("utf8");
|
|
405
|
+
try {
|
|
406
|
+
return this.readFdText(filePath, fd);
|
|
361
407
|
} finally { fs.closeSync(fd); }
|
|
362
408
|
} catch {
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
if (previous) this.entryBytes -= previous.weight ?? 0;
|
|
366
|
-
this.entries.delete(filePath);
|
|
367
|
-
|
|
368
|
-
return null;
|
|
409
|
+
return this.rejectEntry(filePath);
|
|
369
410
|
}
|
|
411
|
+
}
|
|
370
412
|
|
|
371
|
-
|
|
372
|
-
const previous = this.entries.get(filePath);
|
|
373
|
-
|
|
374
|
-
if (previous) this.entryBytes -= previous.weight ?? 0;
|
|
375
|
-
this.entries.delete(filePath);
|
|
376
|
-
|
|
377
|
-
return null;
|
|
378
|
-
}
|
|
379
|
-
const created = { text, lower: text.toLowerCase(), mtimeMs: actual.mtimeMs, size: actual.size, weight: Math.max(1, actual.size) * 2, ext: path.extname(filePath), surface: undefined, lines: undefined, spans: undefined };
|
|
380
|
-
const previous = this.entries.get(filePath);
|
|
381
|
-
|
|
382
|
-
if (previous) this.entryBytes -= previous.weight ?? 0;
|
|
383
|
-
this.entries.delete(filePath);
|
|
384
|
-
this.entries.set(filePath, created);
|
|
385
|
-
this.entryBytes += created.weight;
|
|
386
|
-
|
|
413
|
+
evictOverflow() {
|
|
387
414
|
while (this.entryBytes > MAX_ENTRY_CACHE_BYTES && this.entries.size > 1) {
|
|
388
415
|
const oldest = this.entries.keys().next().value;
|
|
389
416
|
const evicted = this.entries.get(oldest);
|
|
@@ -391,10 +418,38 @@ export class WorkspaceIndex {
|
|
|
391
418
|
this.entries.delete(oldest);
|
|
392
419
|
this.entryBytes -= evicted?.weight ?? 0;
|
|
393
420
|
}
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
storeCreated(filePath, loaded) {
|
|
424
|
+
const created = { text: loaded.text, lower: loaded.text.toLowerCase(), mtimeMs: loaded.actual.mtimeMs, size: loaded.actual.size, weight: Math.max(1, loaded.actual.size) * 2, ext: path.extname(filePath), surface: undefined, lines: undefined, spans: undefined };
|
|
425
|
+
this.dropCached(filePath);
|
|
426
|
+
this.entries.set(filePath, created);
|
|
427
|
+
this.entryBytes += created.weight;
|
|
428
|
+
this.evictOverflow();
|
|
394
429
|
|
|
395
430
|
return created;
|
|
396
431
|
}
|
|
397
432
|
|
|
433
|
+
/** Cached {text, lower, ext, surface?} for a file, re-read when mtime/size changed. Null for unreadable, binary, or huge files. */
|
|
434
|
+
entry(filePath) {
|
|
435
|
+
if (!isTextCandidate(filePath)) return null;
|
|
436
|
+
const stat = this.statOrNull(filePath);
|
|
437
|
+
|
|
438
|
+
if (!stat) return this.rejectEntry(filePath);
|
|
439
|
+
|
|
440
|
+
if (!stat.isFile() || stat.size > MAX_FILE_BYTES) return this.rejectEntry(filePath);
|
|
441
|
+
const cached = this.cachedHit(filePath, stat);
|
|
442
|
+
|
|
443
|
+
if (cached) return cached;
|
|
444
|
+
const loaded = this.readIndexedText(filePath);
|
|
445
|
+
|
|
446
|
+
if (!loaded) return null;
|
|
447
|
+
|
|
448
|
+
if (loaded.text.includes("\0")) return this.rejectEntry(filePath);
|
|
449
|
+
|
|
450
|
+
return this.storeCreated(filePath, loaded);
|
|
451
|
+
}
|
|
452
|
+
|
|
398
453
|
static fromText(filePath, text) {
|
|
399
454
|
return { text, lower: text.toLowerCase(), ext: path.extname(filePath), surface: undefined, lines: undefined, spans: undefined };
|
|
400
455
|
}
|
|
@@ -470,46 +525,27 @@ export class WorkspaceIndex {
|
|
|
470
525
|
return hits;
|
|
471
526
|
}
|
|
472
527
|
|
|
528
|
+
resolveGrepEntry(filePath, overlayText, root, regex, out) {
|
|
529
|
+
const pending = overlayText(filePath);
|
|
530
|
+
|
|
531
|
+
if (pending === undefined) return this.entry(filePath);
|
|
532
|
+
|
|
533
|
+
if (Buffer.byteLength(pending, "utf8") <= MAX_FILE_BYTES) return WorkspaceIndex.fromText(filePath, pending);
|
|
534
|
+
grepPendingHuge(pending, relativeSlash(root, filePath), regex, out);
|
|
535
|
+
|
|
536
|
+
return null;
|
|
537
|
+
}
|
|
538
|
+
|
|
473
539
|
/** Structured grep rows {rel, line, text, def}; def marks lines whose declared name itself matches. */
|
|
474
540
|
grepRows(files, regex, root, overlayText = () => undefined) {
|
|
475
541
|
const out = [];
|
|
476
542
|
const nameRegex = new RegExp(regex.source, "i");
|
|
477
543
|
|
|
478
544
|
for (const filePath of files) {
|
|
479
|
-
const
|
|
480
|
-
let e = null;
|
|
481
|
-
|
|
482
|
-
if (pending === undefined) e = this.entry(filePath);
|
|
483
|
-
else if (Buffer.byteLength(pending, "utf8") <= MAX_FILE_BYTES) e = WorkspaceIndex.fromText(filePath, pending);
|
|
484
|
-
else {
|
|
485
|
-
const rel = relativeSlash(root, filePath);
|
|
486
|
-
let start = 0;
|
|
487
|
-
let line = 0;
|
|
488
|
-
|
|
489
|
-
while (start <= pending.length) {
|
|
490
|
-
const end = pending.indexOf("\n", start);
|
|
491
|
-
const stop = end === -1 ? pending.length : end;
|
|
492
|
-
const text = pending.slice(start, stop).replace(/\r$/, "");
|
|
493
|
-
|
|
494
|
-
line++;
|
|
495
|
-
if (regex.test(text)) out.push({ rel, line, text, def: false });
|
|
496
|
-
if (end === -1) break;
|
|
497
|
-
start = end + 1;
|
|
498
|
-
}
|
|
499
|
-
|
|
500
|
-
continue;
|
|
501
|
-
}
|
|
545
|
+
const e = this.resolveGrepEntry(filePath, overlayText, root, regex, out);
|
|
502
546
|
|
|
503
547
|
if (!e) continue;
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
if (!lineAnchored && !regex.test(e.text)) continue;
|
|
507
|
-
const { raw, defNames } = WorkspaceIndex.linesOf(e);
|
|
508
|
-
const rel = relativeSlash(root, filePath);
|
|
509
|
-
|
|
510
|
-
for (let i = 0; i < raw.length; i++) {
|
|
511
|
-
if (regex.test(raw[i])) out.push({ rel, line: i + 1, text: raw[i], def: defNames[i] !== "" && nameRegex.test(defNames[i]) });
|
|
512
|
-
}
|
|
548
|
+
grepEntryRows(e, filePath, root, regex, nameRegex, out);
|
|
513
549
|
}
|
|
514
550
|
|
|
515
551
|
return out;
|