pi-supernova 0.3.2 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +148 -29
- package/docs/CHANGELOG.md +52 -0
- package/docs/TOKEN_COSTS.md +173 -0
- package/index.js +155 -41
- package/package.json +3 -1
- package/src/bridge/catalog.js +35 -1
- package/src/bridge/host-bridge.js +411 -50
- package/src/bridge/native-tools.js +35 -1
- package/src/config/config.default.json +1 -1
- package/src/config/config.js +19 -0
- package/src/context/evidence.js +119 -5
- package/src/context/fuzzy.js +37 -0
- package/src/context/ledger.js +128 -6
- package/src/context/outline.js +24 -0
- package/src/context/repo-index.js +102 -4
- package/src/context/search.js +45 -0
- package/src/context/snap.js +118 -6
- package/src/context/spans.js +39 -0
- package/src/context/surface.js +33 -6
- package/src/fs/check.js +50 -0
- package/src/fs/diff.js +16 -0
- package/src/fs/json-read.js +91 -0
- package/src/fs/patch.js +26 -0
- package/src/fs/vfs.js +102 -5
- package/src/fs/workspace.js +62 -3
- package/src/output/bottleneck.js +63 -4
- package/src/output/format.js +136 -17
- package/src/runtime/guest-worker.js +169 -31
- package/src/runtime/parallel.js +33 -0
- package/src/runtime/program-batch.js +112 -0
- package/src/runtime/program-file.js +40 -0
- package/src/runtime/reference.js +18 -0
- package/src/runtime/runtime.js +90 -8
- package/src/shared/decode.js +40 -0
- package/src/ui/omp-frame.js +30 -1
- package/src/ui/render-measure.js +24 -0
- package/src/ui/render.js +96 -2
package/src/context/ledger.js
CHANGED
|
@@ -1,20 +1,79 @@
|
|
|
1
|
+
import { isObject, isString } from "../shared/decode.js";
|
|
2
|
+
|
|
1
3
|
const MIN_RUN = 6;
|
|
4
|
+
|
|
2
5
|
const MIN_SUBSTANTIVE = 4;
|
|
6
|
+
|
|
3
7
|
const MAX_CANDIDATES = 8;
|
|
4
|
-
|
|
8
|
+
|
|
9
|
+
const DEFAULT_WINDOW = 0;
|
|
10
|
+
|
|
5
11
|
const MAX_STORED_LINES = 200_000;
|
|
6
12
|
|
|
13
|
+
const MAX_OBSERVE_DEPTH = 64;
|
|
14
|
+
|
|
15
|
+
// A line this long is a minified blob or an inlined image, not a run of source. It
|
|
16
|
+
// cannot be part of a collapsible run anyway (MIN_RUN needs six lines), and skipping
|
|
17
|
+
// it keeps base64 payloads out of the retention set entirely.
|
|
18
|
+
const MAX_OBSERVED_LINE = 4096;
|
|
19
|
+
|
|
20
|
+
// A collapse marker is a citation, not content. Never index or re-collapse one, so
|
|
21
|
+
// citations cannot nest into a chain that points somewhere the reader must follow.
|
|
22
|
+
const COLLAPSE_MARKER = /^⋯ \d+ lines same as #\d+/;
|
|
23
|
+
|
|
7
24
|
function hashLine(line) {
|
|
8
25
|
let hash = 0x811c9dc5;
|
|
26
|
+
|
|
9
27
|
for (let i = 0; i < line.length; i++) hash = Math.imul(hash ^ line.charCodeAt(i), 0x01000193);
|
|
28
|
+
|
|
10
29
|
return hash >>> 0;
|
|
11
30
|
}
|
|
31
|
+
|
|
12
32
|
function substantive(line) { return line.trim().length >= 8; }
|
|
33
|
+
|
|
13
34
|
function newHistory() {
|
|
14
|
-
return { results: new Map(), occurrences: new Map(), storedLines: 0, latestCall: 0,
|
|
35
|
+
return { results: new Map(), occurrences: new Map(), storedLines: 0, latestCall: 0, retained: null,
|
|
15
36
|
stats: { programs: 0, returnedChars: 0, collapsedChars: 0, collapsedRuns: 0 } };
|
|
16
37
|
}
|
|
17
38
|
|
|
39
|
+
// Experimental candidate collection, not proof of provider-visible retention:
|
|
40
|
+
// AgentMessage metadata and later context transformations can hide these strings.
|
|
41
|
+
// Keep this optimization opt-in until exact outgoing citation targets are validated.
|
|
42
|
+
function collectRetained(value, retained, depth = 0) {
|
|
43
|
+
if (isString(value)) {
|
|
44
|
+
// Every line, not only substantive ones. A run may span a blank or short line,
|
|
45
|
+
// so isRetained must be exact per line or the run truncates there. An oversized
|
|
46
|
+
// line is never stored: it cannot sit inside a six-line run, and this is what
|
|
47
|
+
// keeps base64 image payloads out of the retention set.
|
|
48
|
+
if (value.length > MAX_OBSERVED_LINE && !value.includes("\n")) return;
|
|
49
|
+
|
|
50
|
+
for (const line of value.split("\n")) {
|
|
51
|
+
if (retained.size >= MAX_STORED_LINES) return;
|
|
52
|
+
|
|
53
|
+
if (line.length <= MAX_OBSERVED_LINE) retained.add(line);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
return;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
if (depth >= MAX_OBSERVE_DEPTH) return;
|
|
60
|
+
|
|
61
|
+
if (Array.isArray(value)) {
|
|
62
|
+
for (const item of value) collectRetained(item, retained, depth + 1);
|
|
63
|
+
|
|
64
|
+
return;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
// pi's AgentMessage payload is plain objects and arrays, so isObject is the whole
|
|
68
|
+
// traversal contract; anything else is not message text.
|
|
69
|
+
if (!isObject(value)) return;
|
|
70
|
+
|
|
71
|
+
for (const key of Object.keys(value)) collectRetained(value[key], retained, depth + 1);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** Content that may be collapsed out of a later result. Citations are excluded. */
|
|
75
|
+
function collapsible(line) { return substantive(line) && !COLLAPSE_MARKER.test(line); }
|
|
76
|
+
|
|
18
77
|
export class SeenLedger {
|
|
19
78
|
constructor({ window = DEFAULT_WINDOW, history } = {}) {
|
|
20
79
|
this.window = window;
|
|
@@ -26,6 +85,31 @@ export class SeenLedger {
|
|
|
26
85
|
get occurrences() { return this.history.occurrences; }
|
|
27
86
|
get storedLines() { return this.history.storedLines; }
|
|
28
87
|
get stats() { return this.history.stats; }
|
|
88
|
+
get retainedLines() { return this.history.retained; }
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Record candidate lines from the host's pre-request hook. This observation
|
|
92
|
+
* includes metadata and precedes later transformations, so it is not proof
|
|
93
|
+
* that a citation target will be present in the final provider payload.
|
|
94
|
+
* Until this runs, nothing is collapsible, so the first result of a session is
|
|
95
|
+
* always sent in full.
|
|
96
|
+
*/
|
|
97
|
+
observe(payload) {
|
|
98
|
+
if (this.window === 0) { this.history.retained = null;
|
|
99
|
+
|
|
100
|
+
return; }
|
|
101
|
+
|
|
102
|
+
const retained = new Set();
|
|
103
|
+
collectRetained(payload, retained);
|
|
104
|
+
this.history.retained = retained;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** Candidate membership only; not a final-payload retention guarantee. */
|
|
108
|
+
isRetained(line) {
|
|
109
|
+
const retained = this.history.retained;
|
|
110
|
+
|
|
111
|
+
return retained !== null && retained.has(line);
|
|
112
|
+
}
|
|
29
113
|
fork() { return new SeenLedger({ window: this.window, history: this.history }); }
|
|
30
114
|
|
|
31
115
|
reset() {
|
|
@@ -39,54 +123,74 @@ export class SeenLedger {
|
|
|
39
123
|
this.pinned.clear();
|
|
40
124
|
this.stats.programs++;
|
|
41
125
|
this.history.latestCall = Math.max(this.history.latestCall, call);
|
|
126
|
+
|
|
42
127
|
for (const old of this.results.keys()) if (old <= this.history.latestCall - this.window) this.forget(old);
|
|
43
128
|
}
|
|
44
129
|
|
|
45
130
|
forget(call) {
|
|
46
131
|
const entry = this.results.get(call);
|
|
132
|
+
|
|
47
133
|
if (!entry) return;
|
|
134
|
+
|
|
48
135
|
for (const hash of new Set(entry.hashes)) {
|
|
49
136
|
const kept = (this.occurrences.get(hash) ?? []).filter(item => item.call !== call);
|
|
137
|
+
|
|
50
138
|
if (kept.length) this.occurrences.set(hash, kept);
|
|
51
139
|
else this.occurrences.delete(hash);
|
|
52
140
|
}
|
|
141
|
+
|
|
53
142
|
this.history.storedLines -= entry.lines.length;
|
|
54
143
|
this.results.delete(call);
|
|
55
144
|
}
|
|
56
145
|
|
|
57
146
|
recordOrigin(path, firstLine, lines, pin = false) {
|
|
58
147
|
if (this.window === 0) return;
|
|
148
|
+
|
|
59
149
|
for (let i = 0; i < lines.length; i++) {
|
|
60
150
|
const text = lines[i];
|
|
151
|
+
|
|
61
152
|
if (!substantive(text)) continue;
|
|
153
|
+
|
|
62
154
|
if (this.origins.size >= MAX_STORED_LINES && !this.origins.has(text)) this.origins.delete(this.origins.keys().next().value);
|
|
63
155
|
this.origins.set(text, { path, line: firstLine + i });
|
|
156
|
+
|
|
64
157
|
if (pin) this.pinned.add(text);
|
|
65
158
|
}
|
|
66
159
|
}
|
|
67
160
|
|
|
68
161
|
runLength(lines, hashes, candidate, index) {
|
|
69
162
|
const earlier = this.results.get(candidate.call);
|
|
163
|
+
|
|
70
164
|
if (!earlier) return 0;
|
|
71
165
|
let count = 0;
|
|
166
|
+
|
|
72
167
|
while (index + count < lines.length && candidate.index + count < earlier.lines.length
|
|
73
168
|
&& hashes[index + count] === earlier.hashes[candidate.index + count]
|
|
74
169
|
&& lines[index + count] === earlier.lines[candidate.index + count]
|
|
75
|
-
&& !this.pinned.has(lines[index + count])
|
|
170
|
+
&& !this.pinned.has(lines[index + count])
|
|
171
|
+
// Experimental line membership does not establish a visible, ordered target.
|
|
172
|
+
&& this.isRetained(lines[index + count])) count++;
|
|
173
|
+
|
|
76
174
|
return count;
|
|
77
175
|
}
|
|
78
176
|
|
|
79
177
|
longestRun(lines, hashes, index, call) {
|
|
80
178
|
const candidates = this.occurrences.get(hashes[index]);
|
|
179
|
+
|
|
81
180
|
if (!candidates) return null;
|
|
82
181
|
let best = null;
|
|
182
|
+
|
|
83
183
|
for (const candidate of candidates.filter(item => item.call < call).slice(-MAX_CANDIDATES)) {
|
|
84
184
|
const length = this.runLength(lines, hashes, candidate, index);
|
|
185
|
+
|
|
85
186
|
if (length >= MIN_RUN && (!best || length > best.length)) best = { ...candidate, length };
|
|
86
187
|
}
|
|
188
|
+
|
|
87
189
|
if (!best) return null;
|
|
88
190
|
let count = 0;
|
|
89
|
-
|
|
191
|
+
|
|
192
|
+
for (let i = index; i < index + best.length; i++) if (collapsible(lines[i])) count++;
|
|
193
|
+
|
|
90
194
|
return count >= MIN_SUBSTANTIVE ? best : null;
|
|
91
195
|
}
|
|
92
196
|
|
|
@@ -94,56 +198,74 @@ export class SeenLedger {
|
|
|
94
198
|
const earlier = this.results.get(run.call);
|
|
95
199
|
const origin = offset => this.origins.get(lines[index + offset]) ?? earlier?.origins[run.index + offset];
|
|
96
200
|
const first = origin(0);
|
|
201
|
+
|
|
97
202
|
if (!first) return "";
|
|
98
203
|
let expected = first.line;
|
|
204
|
+
|
|
99
205
|
for (let i = 0; i < run.length; i++) {
|
|
100
206
|
const item = origin(i);
|
|
207
|
+
|
|
101
208
|
if (item && (item.path !== first.path || item.line !== expected)) return "";
|
|
102
209
|
expected++;
|
|
103
210
|
}
|
|
211
|
+
|
|
104
212
|
return first.path + ":" + first.line + "–" + (expected - 1);
|
|
105
213
|
}
|
|
106
214
|
|
|
107
215
|
dedupe(text, call) {
|
|
108
216
|
if (this.window === 0) {
|
|
109
217
|
this.stats.returnedChars += text.length;
|
|
218
|
+
|
|
110
219
|
return text;
|
|
111
220
|
}
|
|
221
|
+
|
|
112
222
|
const lines = text.split("\n");
|
|
113
223
|
const hashes = Uint32Array.from(lines, hashLine);
|
|
114
224
|
const out = [];
|
|
115
225
|
let collapsedChars = 0;
|
|
226
|
+
|
|
116
227
|
for (let i = 0; i < lines.length;) {
|
|
117
|
-
const run =
|
|
228
|
+
const run = collapsible(lines[i]) ? this.longestRun(lines, hashes, i, call) : null;
|
|
229
|
+
|
|
118
230
|
if (!run) { out.push(lines[i++]); continue; }
|
|
231
|
+
|
|
119
232
|
const cite = this.citation(lines, i, run);
|
|
120
233
|
out.push("⋯ " + run.length + " lines same as #" + run.call + (cite ? " · " + cite : "") + " ⋯");
|
|
234
|
+
|
|
121
235
|
for (let k = 0; k < run.length; k++) collapsedChars += lines[i + k].length + 1;
|
|
122
236
|
this.stats.collapsedRuns++;
|
|
123
237
|
i += run.length;
|
|
124
238
|
}
|
|
239
|
+
|
|
125
240
|
const sent = out.join("\n");
|
|
126
241
|
this.stats.returnedChars += sent.length;
|
|
127
242
|
this.stats.collapsedChars += collapsedChars;
|
|
128
243
|
this.remember(call, out);
|
|
244
|
+
|
|
129
245
|
return sent;
|
|
130
246
|
}
|
|
131
247
|
|
|
132
248
|
remember(call, lines) {
|
|
133
249
|
if (this.window === 0 || call <= this.history.latestCall - this.window || lines.length > MAX_STORED_LINES) return;
|
|
250
|
+
|
|
134
251
|
if (this.results.has(call)) this.forget(call);
|
|
252
|
+
|
|
135
253
|
for (const old of [...this.results.keys()].sort((a, b) => a - b)) {
|
|
136
254
|
if (this.storedLines + lines.length <= MAX_STORED_LINES) break;
|
|
137
255
|
this.forget(old);
|
|
138
256
|
}
|
|
257
|
+
|
|
139
258
|
const hashes = Uint32Array.from(lines, hashLine);
|
|
140
259
|
const origins = lines.map(line => this.origins.get(line));
|
|
260
|
+
|
|
141
261
|
for (let i = 0; i < lines.length; i++) {
|
|
142
|
-
if (!
|
|
262
|
+
if (!collapsible(lines[i])) continue;
|
|
143
263
|
let list = this.occurrences.get(hashes[i]);
|
|
264
|
+
|
|
144
265
|
if (!list) this.occurrences.set(hashes[i], (list = []));
|
|
145
266
|
list.push({ call, index: i });
|
|
146
267
|
}
|
|
268
|
+
|
|
147
269
|
this.results.set(call, { hashes, lines, origins });
|
|
148
270
|
this.history.storedLines += lines.length;
|
|
149
271
|
}
|
package/src/context/outline.js
CHANGED
|
@@ -14,12 +14,16 @@ function relevance(span, lower, stems) {
|
|
|
14
14
|
if (stems.length === 0) return 0;
|
|
15
15
|
const nameLower = span.name.toLowerCase();
|
|
16
16
|
let score = 0;
|
|
17
|
+
|
|
17
18
|
for (const s of stems) if (nameLower.includes(s)) score += 40;
|
|
19
|
+
|
|
18
20
|
for (let i = span.start - 1; i < span.end; i++) {
|
|
19
21
|
let hits = 0;
|
|
22
|
+
|
|
20
23
|
for (const s of stems) if (lower[i].includes(s)) hits++;
|
|
21
24
|
score += hits * hits * 5;
|
|
22
25
|
}
|
|
26
|
+
|
|
23
27
|
return score;
|
|
24
28
|
}
|
|
25
29
|
|
|
@@ -30,47 +34,61 @@ function chooseExpanded(spans, lower, stems, raw, opts) {
|
|
|
30
34
|
let budget = opts.maxChars;
|
|
31
35
|
// Weak-match cutoff (as fff's weak-match detector): stop once relevance falls below 40% of the best span.
|
|
32
36
|
const floor = stems.length > 0 ? Math.max(1, scored[0].r * 0.4) : 0;
|
|
37
|
+
|
|
33
38
|
for (const { i, r, chars } of scored) {
|
|
34
39
|
if (expanded.size >= opts.maxExpanded || r < floor) break;
|
|
40
|
+
|
|
35
41
|
if (chars > budget) continue;
|
|
36
42
|
expanded.add(i);
|
|
37
43
|
budget -= chars;
|
|
38
44
|
}
|
|
45
|
+
|
|
39
46
|
return expanded;
|
|
40
47
|
}
|
|
41
48
|
|
|
42
49
|
function foldedLine(span) {
|
|
43
50
|
const body = span.end - span.start;
|
|
44
51
|
const sig = span.signature.replace(/\s*\{\s*$/, "");
|
|
52
|
+
|
|
45
53
|
return String(span.start).padStart(5) + " " + sig + (body > 0 ? " … " + body + " lines" : "");
|
|
46
54
|
}
|
|
47
55
|
|
|
48
56
|
function expandedBlock(span, raw, opts) {
|
|
49
57
|
const out = [];
|
|
58
|
+
|
|
50
59
|
for (let l = span.start; l <= span.end; l++) out.push(String(l).padStart(5) + " " + raw[l - 1]);
|
|
51
60
|
const refs = opts.references ? opts.references(span.name, span.start) : [];
|
|
61
|
+
|
|
52
62
|
// Who uses this declaration: the relation a reader would otherwise grep for next.
|
|
53
63
|
if (refs.length) out.push(" // used by: " + refs.slice(0, opts.maxRefs).join(", ") + (refs.length > opts.maxRefs ? " (+" + (refs.length - opts.maxRefs) + ")" : ""));
|
|
64
|
+
|
|
54
65
|
return out.join("\n");
|
|
55
66
|
}
|
|
56
67
|
|
|
57
68
|
function focusedText(raw, lower, stems, relPath, opts) {
|
|
58
69
|
const hits = lower.map((line, i) => ({i, score: stems.filter(s => line.includes(s)).length}))
|
|
59
70
|
.filter(hit => hit.score > 0).sort((a, b) => b.score - a.score || a.i - b.i);
|
|
71
|
+
|
|
60
72
|
const parts = [];
|
|
61
73
|
const covered = new Set();
|
|
62
74
|
let budget = opts.maxChars;
|
|
75
|
+
|
|
63
76
|
for (const {i} of hits) {
|
|
64
77
|
if (parts.length >= opts.maxExpanded) break;
|
|
78
|
+
|
|
65
79
|
if (covered.has(i)) continue;
|
|
66
80
|
const start = Math.max(0, i - 3), end = Math.min(raw.length, i + 4);
|
|
67
81
|
const block = raw.slice(start, end).map((line, j) => String(start + j + 1).padStart(5) + " " + line).join("\n");
|
|
82
|
+
|
|
68
83
|
if (block.length > budget) continue;
|
|
69
84
|
parts.push(block); budget -= block.length;
|
|
85
|
+
|
|
70
86
|
for (let j = start; j < end; j++) covered.add(j);
|
|
71
87
|
}
|
|
88
|
+
|
|
72
89
|
const status = parts.length ? "focused text windows (not a complete file)"
|
|
73
90
|
: hits.length ? "matching text exceeds view budget; first match at line " + (hits[0].i + 1) : "no matching text";
|
|
91
|
+
|
|
74
92
|
return {text: "// " + relPath + " · " + status + "; read(path, line, count) for raw source\n" + parts.join("\n---\n"), expanded: parts.length, declarations: 0};
|
|
75
93
|
}
|
|
76
94
|
|
|
@@ -84,17 +102,23 @@ export function outlineFile(entry, relPath, about, options = {}) {
|
|
|
84
102
|
const lineCount = raw.length;
|
|
85
103
|
const spans = WorkspaceIndex.spansOf(entry).map((s) => ({ ...s, signature: raw[s.start - 1].trim() }));
|
|
86
104
|
const stems = [...new Set(tokenizeQuery(about || "").tokens.map(stem))];
|
|
105
|
+
|
|
87
106
|
if (spans.length === 0) return about ? focusedText(raw, lower, stems, relPath, opts) : null;
|
|
88
107
|
const expanded = chooseExpanded(spans, lower, stems, raw, opts);
|
|
89
108
|
|
|
90
109
|
const parts = [];
|
|
91
110
|
const headerEnd = Math.min(spans[0].start - 1, opts.headerLines);
|
|
111
|
+
|
|
92
112
|
if (headerEnd > 0) {
|
|
93
113
|
const header = raw.slice(0, headerEnd);
|
|
114
|
+
|
|
94
115
|
if (header.length) parts.push(header.map((l, i) => String(i + 1).padStart(5) + " " + l).join("\n"));
|
|
116
|
+
|
|
95
117
|
if (spans[0].start - 1 > opts.headerLines) parts.push(" … " + (spans[0].start - 1 - opts.headerLines) + " more header lines");
|
|
96
118
|
}
|
|
119
|
+
|
|
97
120
|
for (let i = 0; i < spans.length; i++) parts.push(expanded.has(i) ? expandedBlock(spans[i], raw, opts) : foldedLine(spans[i]));
|
|
98
121
|
const title = "// " + relPath + " · " + lineCount + " lines · " + spans.length + " declarations · " + expanded.size + " expanded" + (about ? " for \"" + about + "\"" : "") + " · read(path, line, count) for a folded body";
|
|
122
|
+
|
|
99
123
|
return { text: title + "\n" + parts.join("\n"), expanded: expanded.size, declarations: spans.length };
|
|
100
124
|
}
|
|
@@ -13,18 +13,27 @@ import { relativeSlash } from "../fs/workspace.js";
|
|
|
13
13
|
|
|
14
14
|
// With a working fs.watch the list only refreshes on change; the TTL is the fallback when watching fails.
|
|
15
15
|
const LIST_TTL_MS = 10_000;
|
|
16
|
+
|
|
16
17
|
const WATCHED_TTL_MS = 5 * 60_000;
|
|
18
|
+
|
|
17
19
|
const WATCH_DEBOUNCE_MS = 150;
|
|
20
|
+
|
|
18
21
|
const MAX_INDEXED_FILES = 4000;
|
|
22
|
+
|
|
19
23
|
const MAX_FILE_BYTES = 512 * 1024;
|
|
24
|
+
|
|
20
25
|
const BINARY_EXT = new Set([
|
|
21
26
|
".png", ".jpg", ".jpeg", ".gif", ".webp", ".ico", ".pdf", ".zip", ".gz", ".tgz", ".tar", ".bz2", ".xz", ".7z",
|
|
22
27
|
".woff", ".woff2", ".ttf", ".otf", ".eot", ".mp3", ".mp4", ".mov", ".wav", ".ogg", ".webm", ".wasm", ".class",
|
|
23
28
|
".jar", ".so", ".dylib", ".dll", ".exe", ".bin", ".o", ".a", ".node", ".lock", ".sqlite", ".sqlite3", ".db",
|
|
24
29
|
]);
|
|
30
|
+
|
|
25
31
|
const REGEX_SPECIAL = /[.+^${}()|\\]/g;
|
|
32
|
+
|
|
26
33
|
const IDENT_TOKEN = /[A-Za-z_$][\w$]*/g;
|
|
34
|
+
|
|
27
35
|
const EMPTY = Object.freeze([]);
|
|
36
|
+
|
|
28
37
|
const DEF_PATTERN = /^(?:pub\s+)?(?:export\s+)?(?:async\s+)?(?:default\s+)?(function|class|def|fn|const|let|interface|type|struct|enum)\s+([a-zA-Z0-9_$]+)/;
|
|
29
38
|
|
|
30
39
|
/** Declared identifier on a line (function/class/const/…), or ""; the same rule snap and grep use. */
|
|
@@ -39,13 +48,19 @@ function isTextCandidate(filePath) {
|
|
|
39
48
|
/** Translate one glob token at index i → [regexSource, nextIndex]. */
|
|
40
49
|
function globToken(glob, i) {
|
|
41
50
|
const ch = glob[i];
|
|
51
|
+
|
|
42
52
|
if (ch === "*" && glob[i + 1] === "*") {
|
|
43
53
|
const slashAfter = glob[i + 2] === "/";
|
|
54
|
+
|
|
44
55
|
return [slashAfter ? "(?:.*/)?" : ".*", i + (slashAfter ? 3 : 2)];
|
|
45
56
|
}
|
|
57
|
+
|
|
46
58
|
if (ch === "*") return ["[^/]*", i + 1];
|
|
59
|
+
|
|
47
60
|
if (ch === "?") return ["[^/]", i + 1];
|
|
61
|
+
|
|
48
62
|
if (ch === "{" || ch === "[") return globGroup(glob, i, ch);
|
|
63
|
+
|
|
49
64
|
return [ch.replace(REGEX_SPECIAL, "\\$&"), i + 1];
|
|
50
65
|
}
|
|
51
66
|
|
|
@@ -53,29 +68,70 @@ function globToken(glob, i) {
|
|
|
53
68
|
function globGroup(glob, i, open) {
|
|
54
69
|
const close = open === "{" ? "}" : "]";
|
|
55
70
|
const end = glob.indexOf(close, i);
|
|
71
|
+
|
|
56
72
|
if (end < 0) throw new SyntaxError("unclosed " + open + " in glob");
|
|
57
73
|
const inner = glob.slice(i + 1, end);
|
|
58
74
|
const source = open === "{" ? "(?:" + inner.split(",").map(globBody).join("|") + ")" : "[" + inner + "]";
|
|
75
|
+
|
|
59
76
|
return [source, end + 1];
|
|
60
77
|
}
|
|
61
78
|
|
|
62
79
|
function globBody(glob) {
|
|
63
80
|
let source = "";
|
|
64
81
|
let i = 0;
|
|
82
|
+
|
|
65
83
|
while (i < glob.length) {
|
|
66
84
|
const [piece, next] = globToken(glob, i);
|
|
67
85
|
source += piece;
|
|
68
86
|
i = next;
|
|
69
87
|
}
|
|
88
|
+
|
|
70
89
|
return source;
|
|
71
90
|
}
|
|
72
91
|
|
|
73
92
|
/** gitignore-style glob (rg -g) → RegExp over a "/"-separated relative path. No slash ⇒ basename match anywhere. */
|
|
74
93
|
export function globToRegExp(glob) {
|
|
75
94
|
const body = globBody(glob);
|
|
95
|
+
|
|
76
96
|
return new RegExp(glob.includes("/") ? "^" + body + "$" : "(?:^|/)" + body + "$");
|
|
77
97
|
}
|
|
78
98
|
|
|
99
|
+
function declarationEnd(raw, lower, start, lineCount, ext) {
|
|
100
|
+
if (ext === ".py") {
|
|
101
|
+
const indentOf = (i) => raw[i].length - raw[i].trimStart().length;
|
|
102
|
+
const base = indentOf(start - 1);
|
|
103
|
+
let end = start;
|
|
104
|
+
|
|
105
|
+
for (let i = start; i < lineCount; i++) {
|
|
106
|
+
if (lower[i] === "") { end = i + 1; continue; }
|
|
107
|
+
if (indentOf(i) <= base) break;
|
|
108
|
+
end = i + 1;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
return Math.min(end, lineCount);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
let depth = 0;
|
|
115
|
+
|
|
116
|
+
for (const ch of raw[start - 1] ?? "") {
|
|
117
|
+
if (ch === "{") depth++;
|
|
118
|
+
else if (ch === "}") depth--;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
if (depth <= 0) return start;
|
|
122
|
+
|
|
123
|
+
for (let i = start; i < raw.length; i++) {
|
|
124
|
+
for (const ch of raw[i]) {
|
|
125
|
+
if (ch === "{") depth++;
|
|
126
|
+
else if (ch === "}") depth--;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
if (depth <= 0) return i + 1;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
return lineCount;
|
|
133
|
+
}
|
|
134
|
+
|
|
79
135
|
export class WorkspaceIndex {
|
|
80
136
|
constructor(runCommand) {
|
|
81
137
|
this.runCommand = runCommand;
|
|
@@ -100,8 +156,10 @@ export class WorkspaceIndex {
|
|
|
100
156
|
watch(root) {
|
|
101
157
|
if (this.watchers.has(root)) return this.watchers.get(root);
|
|
102
158
|
let ok = false;
|
|
159
|
+
|
|
103
160
|
try {
|
|
104
161
|
let timer = null;
|
|
162
|
+
|
|
105
163
|
const watcher = fs.watch(root, { recursive: true, persistent: false }, () => {
|
|
106
164
|
if (timer) return;
|
|
107
165
|
timer = setTimeout(() => {
|
|
@@ -110,39 +168,50 @@ export class WorkspaceIndex {
|
|
|
110
168
|
this.gitModified.delete(root);
|
|
111
169
|
}, WATCH_DEBOUNCE_MS);
|
|
112
170
|
});
|
|
171
|
+
|
|
113
172
|
watcher.on("error", () => {
|
|
114
173
|
this.watchers.set(root, false);
|
|
115
174
|
this.lists.clear();
|
|
116
175
|
});
|
|
176
|
+
|
|
117
177
|
if (isFunction(watcher.unref)) watcher.unref();
|
|
118
178
|
ok = true;
|
|
119
179
|
} catch {
|
|
120
180
|
ok = false;
|
|
121
181
|
}
|
|
182
|
+
|
|
122
183
|
this.watchers.set(root, ok);
|
|
184
|
+
|
|
123
185
|
return ok;
|
|
124
186
|
}
|
|
125
187
|
|
|
126
188
|
/** Paths git reports as modified/added/untracked (fff's git-status boost); one spawn per list refresh. */
|
|
127
189
|
async modifiedFiles(root) {
|
|
128
190
|
const cached = this.gitModified.get(root);
|
|
191
|
+
|
|
129
192
|
if (cached) return cached;
|
|
130
193
|
const set = new Set();
|
|
194
|
+
|
|
131
195
|
try {
|
|
132
196
|
const res = await this.runCommand(["git", "status", "--porcelain", "-z", "--untracked-files=all"], { cwd: root, timeoutMs: 5_000 });
|
|
197
|
+
|
|
133
198
|
if (res.exitCode === 0) {
|
|
134
199
|
for (const row of res.stdout.split("\0")) {
|
|
135
200
|
if (row.length > 3) set.add(row.slice(3));
|
|
136
201
|
}
|
|
137
202
|
}
|
|
138
203
|
} catch {}
|
|
204
|
+
|
|
139
205
|
this.gitModified.set(root, set);
|
|
206
|
+
|
|
140
207
|
return set;
|
|
141
208
|
}
|
|
142
209
|
|
|
143
210
|
mtimeSeconds(filePath) {
|
|
144
211
|
const e = this.entries.get(filePath);
|
|
212
|
+
|
|
145
213
|
if (e) return e.mtimeMs / 1000;
|
|
214
|
+
|
|
146
215
|
try {
|
|
147
216
|
return fs.statSync(filePath).mtimeMs / 1000;
|
|
148
217
|
} catch {
|
|
@@ -155,16 +224,20 @@ export class WorkspaceIndex {
|
|
|
155
224
|
const key = root + "\0" + (includeHidden ? "h" : "");
|
|
156
225
|
const cached = this.lists.get(key);
|
|
157
226
|
const ttl = this.watch(root) ? WATCHED_TTL_MS : LIST_TTL_MS;
|
|
227
|
+
|
|
158
228
|
if (cached && Date.now() - cached.at < ttl) return cached.files;
|
|
159
229
|
const args = ["rg", "--files"];
|
|
230
|
+
|
|
160
231
|
if (includeHidden) args.push("--hidden");
|
|
161
232
|
args.push("-g", "!.git/**", "-g", "!**/.git/**", root);
|
|
162
233
|
let files = [];
|
|
163
234
|
let error;
|
|
164
235
|
let truncated = false;
|
|
165
236
|
let missing = false;
|
|
237
|
+
|
|
166
238
|
try {
|
|
167
239
|
const res = await this.runCommand(args, { cwd: root, timeoutMs: 15_000, signal });
|
|
240
|
+
|
|
168
241
|
if (res.exitCode !== 0 && res.exitCode !== 1) error = res.stderr.trim() || "rg exited with status " + res.exitCode;
|
|
169
242
|
truncated = res.outputTruncated === true;
|
|
170
243
|
const output = truncated && !res.stdout.endsWith("\n") ? res.stdout.slice(0, res.stdout.lastIndexOf("\n") + 1) : res.stdout;
|
|
@@ -174,7 +247,9 @@ export class WorkspaceIndex {
|
|
|
174
247
|
error = err.message;
|
|
175
248
|
missing = !fs.existsSync(root);
|
|
176
249
|
}
|
|
250
|
+
|
|
177
251
|
this.lists.set(key, { files, at: Date.now(), error, truncated, missing });
|
|
252
|
+
|
|
178
253
|
return files;
|
|
179
254
|
}
|
|
180
255
|
|
|
@@ -182,24 +257,31 @@ export class WorkspaceIndex {
|
|
|
182
257
|
entry(filePath) {
|
|
183
258
|
if (!isTextCandidate(filePath)) return null;
|
|
184
259
|
let stat;
|
|
260
|
+
|
|
185
261
|
try {
|
|
186
262
|
stat = fs.statSync(filePath);
|
|
187
263
|
} catch {
|
|
188
264
|
this.entries.delete(filePath);
|
|
265
|
+
|
|
189
266
|
return null;
|
|
190
267
|
}
|
|
268
|
+
|
|
191
269
|
if (!stat.isFile() || stat.size > MAX_FILE_BYTES) return null;
|
|
192
270
|
const cached = this.entries.get(filePath);
|
|
271
|
+
|
|
193
272
|
if (cached && cached.mtimeMs === stat.mtimeMs && cached.size === stat.size) return cached;
|
|
194
273
|
let text;
|
|
274
|
+
|
|
195
275
|
try {
|
|
196
276
|
text = fs.readFileSync(filePath, "utf8");
|
|
197
277
|
} catch {
|
|
198
278
|
return null;
|
|
199
279
|
}
|
|
280
|
+
|
|
200
281
|
if (text.includes("\0")) return null;
|
|
201
282
|
const created = { text, lower: text.toLowerCase(), mtimeMs: stat.mtimeMs, size: stat.size, ext: path.extname(filePath), surface: undefined, lines: undefined, spans: undefined };
|
|
202
283
|
this.entries.set(filePath, created);
|
|
284
|
+
|
|
203
285
|
return created;
|
|
204
286
|
}
|
|
205
287
|
|
|
@@ -214,37 +296,45 @@ export class WorkspaceIndex {
|
|
|
214
296
|
const lower = [];
|
|
215
297
|
const defNames = [];
|
|
216
298
|
const idents = [];
|
|
299
|
+
|
|
217
300
|
for (let i = 0; i < raw.length; i++) {
|
|
218
301
|
const trimmed = raw[i].trim();
|
|
219
302
|
lower[i] = trimmed.toLowerCase();
|
|
220
303
|
defNames[i] = DEF_PATTERN.exec(trimmed)?.[2].toLowerCase() ?? "";
|
|
221
304
|
idents[i] = trimmed.match(IDENT_TOKEN) || EMPTY;
|
|
222
305
|
}
|
|
306
|
+
|
|
223
307
|
entry.lines = { raw, lower, defNames, idents };
|
|
308
|
+
|
|
224
309
|
return entry.lines;
|
|
225
310
|
}
|
|
226
311
|
|
|
227
312
|
/**
|
|
228
|
-
* Declaration spans [start, end] (1-based, inclusive)
|
|
229
|
-
*
|
|
313
|
+
* Declaration spans [start, end] (1-based, inclusive). Nested bodies stay inside the parent
|
|
314
|
+
* (brace-matched for JS-like, indent for Python). The file's leading header is not a span.
|
|
230
315
|
*/
|
|
231
316
|
static spansOf(entry) {
|
|
232
317
|
if (entry.spans) return entry.spans;
|
|
233
318
|
const { items, lineCount } = WorkspaceIndex.surfaceOf(entry);
|
|
234
|
-
const { lower } = WorkspaceIndex.linesOf(entry);
|
|
319
|
+
const { lower, raw } = WorkspaceIndex.linesOf(entry);
|
|
235
320
|
const spans = [];
|
|
321
|
+
|
|
236
322
|
for (let i = 0; i < items.length; i++) {
|
|
237
323
|
const start = items[i].line;
|
|
238
|
-
let end =
|
|
324
|
+
let end = declarationEnd(raw, lower, start, lineCount, entry.ext);
|
|
325
|
+
|
|
239
326
|
while (end > start && lower[end - 1] === "") end--;
|
|
240
327
|
spans.push({ start, end, name: items[i].name, kind: items[i].kind, isExport: items[i].isExport === true });
|
|
241
328
|
}
|
|
329
|
+
|
|
242
330
|
entry.spans = spans;
|
|
331
|
+
|
|
243
332
|
return spans;
|
|
244
333
|
}
|
|
245
334
|
|
|
246
335
|
static surfaceOf(entry) {
|
|
247
336
|
if (!entry.surface) entry.surface = extractStructuralSurface(entry.text, entry.ext);
|
|
337
|
+
|
|
248
338
|
return entry.surface;
|
|
249
339
|
}
|
|
250
340
|
|
|
@@ -256,12 +346,16 @@ export class WorkspaceIndex {
|
|
|
256
346
|
/** Files whose lowercase text contains any (or every) needle; needles are lowercase. */
|
|
257
347
|
filesContaining(files, needles, anyOf) {
|
|
258
348
|
const hits = [];
|
|
349
|
+
|
|
259
350
|
for (const filePath of files) {
|
|
260
351
|
const e = this.entry(filePath);
|
|
352
|
+
|
|
261
353
|
if (!e) continue;
|
|
262
354
|
const found = anyOf ? needles.some((n) => e.lower.includes(n)) : needles.every((n) => e.lower.includes(n));
|
|
355
|
+
|
|
263
356
|
if (found) hits.push(filePath);
|
|
264
357
|
}
|
|
358
|
+
|
|
265
359
|
return hits;
|
|
266
360
|
}
|
|
267
361
|
|
|
@@ -269,16 +363,20 @@ export class WorkspaceIndex {
|
|
|
269
363
|
grepRows(files, regex, root, overlayText = () => undefined) {
|
|
270
364
|
const out = [];
|
|
271
365
|
const nameRegex = new RegExp(regex.source, "i");
|
|
366
|
+
|
|
272
367
|
for (const filePath of files) {
|
|
273
368
|
const pending = overlayText(filePath);
|
|
274
369
|
const e = pending === undefined ? this.entry(filePath) : WorkspaceIndex.fromText(filePath, pending);
|
|
370
|
+
|
|
275
371
|
if (!e || !regex.test(e.text)) continue;
|
|
276
372
|
const { raw, defNames } = WorkspaceIndex.linesOf(e);
|
|
277
373
|
const rel = relativeSlash(root, filePath);
|
|
374
|
+
|
|
278
375
|
for (let i = 0; i < raw.length; i++) {
|
|
279
376
|
if (regex.test(raw[i])) out.push({ rel, line: i + 1, text: raw[i], def: defNames[i] !== "" && nameRegex.test(defNames[i]) });
|
|
280
377
|
}
|
|
281
378
|
}
|
|
379
|
+
|
|
282
380
|
return out;
|
|
283
381
|
}
|
|
284
382
|
}
|