pi-supernova 0.3.2 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +148 -29
- package/docs/CHANGELOG.md +52 -0
- package/docs/TOKEN_COSTS.md +173 -0
- package/index.js +155 -41
- package/package.json +3 -1
- package/src/bridge/catalog.js +35 -1
- package/src/bridge/host-bridge.js +411 -50
- package/src/bridge/native-tools.js +35 -1
- package/src/config/config.default.json +1 -1
- package/src/config/config.js +19 -0
- package/src/context/evidence.js +119 -5
- package/src/context/fuzzy.js +37 -0
- package/src/context/ledger.js +128 -6
- package/src/context/outline.js +24 -0
- package/src/context/repo-index.js +102 -4
- package/src/context/search.js +45 -0
- package/src/context/snap.js +118 -6
- package/src/context/spans.js +39 -0
- package/src/context/surface.js +33 -6
- package/src/fs/check.js +50 -0
- package/src/fs/diff.js +16 -0
- package/src/fs/json-read.js +91 -0
- package/src/fs/patch.js +26 -0
- package/src/fs/vfs.js +102 -5
- package/src/fs/workspace.js +62 -3
- package/src/output/bottleneck.js +63 -4
- package/src/output/format.js +136 -17
- package/src/runtime/guest-worker.js +169 -31
- package/src/runtime/parallel.js +33 -0
- package/src/runtime/program-batch.js +112 -0
- package/src/runtime/program-file.js +40 -0
- package/src/runtime/reference.js +18 -0
- package/src/runtime/runtime.js +90 -8
- package/src/shared/decode.js +40 -0
- package/src/ui/omp-frame.js +30 -1
- package/src/ui/render-measure.js +24 -0
- package/src/ui/render.js +96 -2
package/src/context/search.js
CHANGED
|
@@ -16,51 +16,71 @@ function textResult(text, details) {
|
|
|
16
16
|
export async function referencesForNames({ root, names, excludePath, overlayText, pendingPaths, signal, run = runCommand }) {
|
|
17
17
|
const references = new Map(names.map(name => [name, []]));
|
|
18
18
|
const patterns = names.map(name => new RegExp("(?<![\\w$])" + name.replaceAll("$", "\\$") + "(?![\\w$])"));
|
|
19
|
+
|
|
19
20
|
const add = (file, line, text) => {
|
|
20
21
|
if (file === excludePath) return;
|
|
22
|
+
|
|
21
23
|
for (let i = 0; i < names.length; i++) {
|
|
22
24
|
const hits = references.get(names[i]);
|
|
25
|
+
|
|
23
26
|
if (hits.length < 7 && patterns[i].test(text)) hits.push(relativeSlash(root, file) + ":" + line);
|
|
24
27
|
}
|
|
25
28
|
};
|
|
29
|
+
|
|
26
30
|
const result = await run(["rg", "--json", "--fixed-strings", ...names.flatMap(name => ["-e", name]), "--", root],
|
|
27
31
|
{ cwd: root, signal, timeoutMs: 5000, maxOutputChars: 65536 });
|
|
32
|
+
|
|
28
33
|
if (result.exitCode !== 0 && result.exitCode !== 1) throw new Error(result.stderr.trim() || "reference search failed");
|
|
29
34
|
const records = result.stdout.split("\n");
|
|
35
|
+
|
|
30
36
|
for (let i = 0; i < records.length; i++) {
|
|
31
37
|
signal?.throwIfAborted();
|
|
38
|
+
|
|
32
39
|
if (!records[i]) continue;
|
|
33
40
|
let record;
|
|
41
|
+
|
|
34
42
|
try { record = JSON.parse(records[i]); }
|
|
35
43
|
catch (error) { if (result.outputTruncated && i === records.length - 1) break; throw error; }
|
|
44
|
+
|
|
36
45
|
if (record.type !== "match" || !isString(record.data?.path?.text) || !isString(record.data.lines?.text)) continue;
|
|
37
46
|
const file = path.resolve(root, record.data.path.text);
|
|
47
|
+
|
|
38
48
|
if (overlayText(file) === undefined) add(file, record.data.line_number, record.data.lines.text);
|
|
39
49
|
}
|
|
50
|
+
|
|
40
51
|
for (const file of pendingPaths) {
|
|
41
52
|
const text = overlayText(file);
|
|
53
|
+
|
|
42
54
|
if (text !== undefined) text.split("\n").forEach((line, i) => add(file, i + 1, line));
|
|
43
55
|
}
|
|
56
|
+
|
|
44
57
|
return { references, incomplete: result.outputTruncated === true };
|
|
45
58
|
}
|
|
46
59
|
|
|
47
60
|
export function rgGrepArgs(pattern, params, searchPath) {
|
|
48
61
|
const args = ["--line-number", "--no-heading", "--color", "never"];
|
|
62
|
+
|
|
49
63
|
if (params?.caseSensitive !== true) args.push("--ignore-case");
|
|
64
|
+
|
|
50
65
|
if (params?.glob) args.push("--glob", String(params.glob));
|
|
51
66
|
args.push("--", pattern, searchPath);
|
|
67
|
+
|
|
52
68
|
return args;
|
|
53
69
|
}
|
|
54
70
|
|
|
55
71
|
/** rg --files, then find(1) when rg is unavailable; both accept an optional glob/name pattern. */
|
|
56
72
|
export async function listWithTools(searchDir, pattern, cwd, signal) {
|
|
57
73
|
const args = ["--files"];
|
|
74
|
+
|
|
58
75
|
if (pattern) args.push("-g", pattern);
|
|
59
76
|
const res = await runCommand(["rg", ...args, searchDir], { cwd, timeoutMs: 30_000, signal }).catch(() => null);
|
|
77
|
+
|
|
60
78
|
if (res && (res.exitCode === 0 || res.exitCode === 1)) return textResult(res.stdout, { via: "rg" });
|
|
61
79
|
const findArgs = [searchDir];
|
|
80
|
+
|
|
62
81
|
if (pattern) findArgs.push("-name", pattern);
|
|
63
82
|
const findRes = await runCommand(["find", ...findArgs], { cwd, timeoutMs: 30_000, signal });
|
|
83
|
+
|
|
64
84
|
return textResult(findRes.stdout, { via: "find" });
|
|
65
85
|
}
|
|
66
86
|
|
|
@@ -73,6 +93,7 @@ const GLOB_CHARS = /[*?[\]{}]/;
|
|
|
73
93
|
export async function fuzzyFind(index, root, cwd, pattern, limit = 20) {
|
|
74
94
|
if (!pattern || GLOB_CHARS.test(pattern)) return null;
|
|
75
95
|
const files = await index.files(root);
|
|
96
|
+
|
|
76
97
|
if (!index.canScan(files)) return null;
|
|
77
98
|
const rel = files.map((f) => relativeSlash(cwd, f));
|
|
78
99
|
const absolute = new Map(rel.map((r, i) => [r, files[i]]));
|
|
@@ -81,23 +102,30 @@ export async function fuzzyFind(index, root, cwd, pattern, limit = 20) {
|
|
|
81
102
|
const ranked = rankPaths(pattern, rel, { frecency: index.frecency, mtimeOf, modified: await index.modifiedFiles(cwd), currentFile: index.lastTouched });
|
|
82
103
|
// fff weak-match detector: when nothing matches exactly and the best is mostly typos, say so instead of flooding.
|
|
83
104
|
const rows = ranked.slice(0, limit);
|
|
105
|
+
|
|
84
106
|
if (rows.length === 0) return "";
|
|
107
|
+
|
|
85
108
|
return rows.map((r) => r.path).join("\n") + "\n";
|
|
86
109
|
}
|
|
87
110
|
|
|
88
111
|
/** fff-style grep: smart-case, definition lines first, fuzzy fallback when the literal has no hits. */
|
|
89
112
|
export async function grepIndexed(index, pattern, params, searchPath, cwd) {
|
|
90
113
|
const compiled = grepRegex(pattern, params);
|
|
114
|
+
|
|
91
115
|
if (!compiled) return null;
|
|
92
116
|
const { regex, caseSensitive } = compiled;
|
|
93
117
|
let files = await index.files(searchPath);
|
|
118
|
+
|
|
94
119
|
if (!index.canScan(files)) return null;
|
|
120
|
+
|
|
95
121
|
if (params?.glob) {
|
|
96
122
|
const matcher = globToRegExp(String(params.glob));
|
|
97
123
|
files = files.filter((f) => matcher.test(relativeSlash(cwd, f)));
|
|
98
124
|
}
|
|
125
|
+
|
|
99
126
|
const rows = index.grepRows(files, regex, cwd);
|
|
100
127
|
const fallback = rows.length === 0 && /^[\w$.-]{4,}$/.test(pattern) ? fuzzyGrepRows(index, files, pattern, cwd, caseSensitive) : rows;
|
|
128
|
+
|
|
101
129
|
return formatGrepRows(fallback, grepLimit(params));
|
|
102
130
|
}
|
|
103
131
|
|
|
@@ -107,6 +135,7 @@ function grepLimit(params) {
|
|
|
107
135
|
|
|
108
136
|
function grepRegex(pattern, params) {
|
|
109
137
|
const caseSensitive = params?.caseSensitive === true || (params?.caseSensitive !== false && smartCase(pattern));
|
|
138
|
+
|
|
110
139
|
try {
|
|
111
140
|
return { regex: new RegExp(pattern, caseSensitive ? "" : "i"), caseSensitive };
|
|
112
141
|
} catch {
|
|
@@ -118,17 +147,22 @@ function grepRegex(pattern, params) {
|
|
|
118
147
|
function fuzzyGrepRows(index, files, pattern, cwd, caseSensitive) {
|
|
119
148
|
const maxTypos = pattern.length >= 8 ? 2 : 1;
|
|
120
149
|
const rows = [];
|
|
150
|
+
|
|
121
151
|
for (const filePath of files) {
|
|
122
152
|
const e = index.entry(filePath);
|
|
153
|
+
|
|
123
154
|
if (!e) continue;
|
|
124
155
|
const { raw, defNames } = WorkspaceIndex.linesOf(e);
|
|
125
156
|
const rel = relativeSlash(cwd, filePath);
|
|
157
|
+
|
|
126
158
|
for (let i = 0; i < raw.length && rows.length <= 400; i++) {
|
|
127
159
|
const m = fuzzyMatch(pattern, raw[i], { maxTypos, caseSensitive });
|
|
160
|
+
|
|
128
161
|
if (!m || m.end - m.start > pattern.length + 2) continue;
|
|
129
162
|
rows.push({ rel, line: i + 1, text: raw[i], def: defNames[i] !== "" && fuzzyMatch(pattern, defNames[i], { maxTypos }) !== null });
|
|
130
163
|
}
|
|
131
164
|
}
|
|
165
|
+
|
|
132
166
|
return rows;
|
|
133
167
|
}
|
|
134
168
|
|
|
@@ -136,39 +170,50 @@ function fuzzyGrepRows(index, files, pattern, cwd, caseSensitive) {
|
|
|
136
170
|
function formatGrepRows(rows, limit) {
|
|
137
171
|
if (rows.length === 0) return "";
|
|
138
172
|
const groups = new Map();
|
|
173
|
+
|
|
139
174
|
for (const r of rows) {
|
|
140
175
|
if (!groups.has(r.rel)) groups.set(r.rel, []);
|
|
141
176
|
groups.get(r.rel).push(r);
|
|
142
177
|
}
|
|
178
|
+
|
|
143
179
|
const files = [...groups.values()].sort((a, b) => Number(b.some((r) => r.def)) - Number(a.some((r) => r.def)));
|
|
144
180
|
let out = "";
|
|
145
181
|
let shown = 0;
|
|
182
|
+
|
|
146
183
|
for (const group of files) {
|
|
147
184
|
if (shown >= limit) break;
|
|
148
185
|
out += group[0].rel + "\n";
|
|
149
186
|
group.sort((a, b) => Number(b.def) - Number(a.def) || a.line - b.line);
|
|
187
|
+
|
|
150
188
|
for (const r of group) {
|
|
151
189
|
if (shown++ >= limit) break;
|
|
152
190
|
out += " " + r.line + (r.def ? "*" : ":") + " " + r.text.trim() + "\n";
|
|
153
191
|
}
|
|
154
192
|
}
|
|
193
|
+
|
|
155
194
|
if (rows.length > limit) out += "… " + (rows.length - limit) + " more matches (pass limit or narrow the pattern)\n";
|
|
195
|
+
|
|
156
196
|
return out;
|
|
157
197
|
}
|
|
158
198
|
|
|
159
199
|
/** rg --files [-g pattern] served from the index; null when the tree is too large. */
|
|
160
200
|
export async function listIndexed(index, root, cwd, pattern) {
|
|
161
201
|
const files = await index.files(root);
|
|
202
|
+
|
|
162
203
|
if (!index.canScan(files)) return null;
|
|
163
204
|
const rel = files.map((f) => path.relative(cwd, f).split(path.sep).join("/"));
|
|
205
|
+
|
|
164
206
|
if (!pattern) return rel.length ? rel.join("\n") + "\n" : "";
|
|
165
207
|
let matcher;
|
|
208
|
+
|
|
166
209
|
try {
|
|
167
210
|
matcher = globToRegExp(pattern);
|
|
168
211
|
} catch {
|
|
169
212
|
return null;
|
|
170
213
|
}
|
|
214
|
+
|
|
171
215
|
const hits = rel.filter((f) => matcher.test(f));
|
|
216
|
+
|
|
172
217
|
return hits.length ? hits.join("\n") + "\n" : "";
|
|
173
218
|
}
|
|
174
219
|
|
package/src/context/snap.js
CHANGED
|
@@ -3,6 +3,8 @@ import { isString } from "../shared/decode.js";
|
|
|
3
3
|
import { truncateChars } from "../output/format.js";
|
|
4
4
|
import * as fs from "node:fs/promises";
|
|
5
5
|
import { extractStructuralSurface } from "./surface.js";
|
|
6
|
+
import { WorkspaceIndex } from "./repo-index.js";
|
|
7
|
+
import { pickSpan, spanCandidate, spanWindow } from "./spans.js";
|
|
6
8
|
import { rankPaths } from "./fuzzy.js";
|
|
7
9
|
import { isTestPath, runCommand, relativeSlash } from "../fs/workspace.js";
|
|
8
10
|
|
|
@@ -11,20 +13,26 @@ const STOP_WORDS = new Set([
|
|
|
11
13
|
"by", "from", "is", "it", "this", "that", "where", "how", "what", "which",
|
|
12
14
|
"file", "code", "function", "class", "method", "find", "get", "look", "are", "does", "do",
|
|
13
15
|
]);
|
|
16
|
+
|
|
14
17
|
const SOURCE_EXT = new Set([".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".rs", ".py", ".go"]);
|
|
18
|
+
|
|
15
19
|
const TYPED_EXT = new Set([".ts", ".tsx", ".rs", ".go"]);
|
|
20
|
+
|
|
16
21
|
const MAX_SEARCH_CHARS = 2 * 1024 * 1024;
|
|
22
|
+
|
|
17
23
|
const MAX_ALTERNATIVES = 3;
|
|
18
24
|
|
|
19
25
|
/** Light suffix stripping so "terminated" ⊇ "terminat" matches "terminate"; deterministic, no dictionary. */
|
|
20
26
|
export function stem(token) {
|
|
21
27
|
if (token.length < 5) return token;
|
|
28
|
+
|
|
22
29
|
return token.replace(/(ations?|ings?|ed|es|e|s|ly|ers?)$/, (m) => (token.length - m.length >= 4 ? "" : m));
|
|
23
30
|
}
|
|
24
31
|
|
|
25
32
|
export function tokenizeQuery(query) {
|
|
26
33
|
if (!isString(query) || !query.trim()) return { tokens: [], wantsTest: false, wantsType: false, wantsDoc: false };
|
|
27
34
|
const words = query.replace(/([a-z])([A-Z])/g, "$1 $2").toLowerCase().split(/[^a-zA-Z0-9_]+/);
|
|
35
|
+
|
|
28
36
|
return {
|
|
29
37
|
tokens: [...new Set(words.filter(word => word.length > 1 && !STOP_WORDS.has(word)))],
|
|
30
38
|
wantsTest: words.some(word => ["test", "tests", "testing", "spec", "specs"].includes(word)),
|
|
@@ -36,28 +44,36 @@ export function tokenizeQuery(query) {
|
|
|
36
44
|
export function scorePathTopology(filePath, tokens, flags) {
|
|
37
45
|
const normalized = filePath.replaceAll("\\", "/").toLowerCase();
|
|
38
46
|
const parts = normalized.split("/");
|
|
47
|
+
|
|
39
48
|
if (parts.some(part => ["node_modules", "dist", "target"].includes(part))) return -100;
|
|
40
49
|
const test = isTestPath(normalized);
|
|
50
|
+
|
|
41
51
|
if (test && !flags.wantsTest) return -50;
|
|
52
|
+
|
|
42
53
|
if (!test && flags.wantsTest) return -20;
|
|
43
54
|
const base = path.basename(normalized);
|
|
44
55
|
const words = normalized.split(/[^a-zA-Z0-9]+/);
|
|
45
56
|
const ext = path.extname(normalized);
|
|
46
57
|
let score = SOURCE_EXT.has(ext) && !flags.wantsDoc ? 5 : 0;
|
|
58
|
+
|
|
47
59
|
if (flags.wantsType && TYPED_EXT.has(ext)) score += 10;
|
|
60
|
+
|
|
48
61
|
for (const token of tokens) {
|
|
49
62
|
if (base === token || base.startsWith(token + ".")) score += 60;
|
|
50
63
|
else if (base.includes(token)) score += 30;
|
|
51
64
|
else if (words.includes(token)) score += 15;
|
|
52
65
|
else if (normalized.includes(token)) score += 5;
|
|
53
66
|
}
|
|
67
|
+
|
|
54
68
|
return score;
|
|
55
69
|
}
|
|
56
70
|
|
|
57
71
|
function inScope(filePath, dir, includeHidden) {
|
|
58
72
|
const relative = path.relative(dir, filePath);
|
|
73
|
+
|
|
59
74
|
if (relative === ".." || relative.startsWith(".." + path.sep) || path.isAbsolute(relative)) return false;
|
|
60
75
|
const parts = relative.split(path.sep);
|
|
76
|
+
|
|
61
77
|
return !parts.includes(".git") && (includeHidden || !parts.some(part => part.startsWith(".") && part.length > 1));
|
|
62
78
|
}
|
|
63
79
|
|
|
@@ -65,33 +81,44 @@ function makeCandidate(filePath, dir, query, tokens, flags) {
|
|
|
65
81
|
const relative = path.relative(dir, filePath);
|
|
66
82
|
const lower = relative.toLowerCase();
|
|
67
83
|
const base = path.basename(lower);
|
|
84
|
+
|
|
68
85
|
const exactPath = lower === query.toLowerCase() || base === query.toLowerCase()
|
|
69
86
|
|| base.slice(0, -path.extname(base).length) === query.toLowerCase();
|
|
87
|
+
|
|
70
88
|
return { path: filePath, pathScore: scorePathTopology(relative, tokens, flags), exactPath,
|
|
71
89
|
pathCoverage: tokens.filter(token => lower.includes(token)).length,
|
|
72
90
|
matched: new Set(), exactDefinition: false, definitionCoverage: 0, lineCoverage: 0,
|
|
73
|
-
line: 1, signature: "", context: new Map(), recent: [], anchorScore: -1 };
|
|
91
|
+
line: 1, signature: "", context: new Map(), recent: [], anchorScore: -1, exactLines: new Set() };
|
|
74
92
|
}
|
|
75
93
|
|
|
76
94
|
function inspectLine(candidate, lineNumber, raw, query, tokens, isMatch) {
|
|
77
95
|
const text = raw.replace(/\r?\n$/, "");
|
|
78
96
|
const lower = text.toLowerCase();
|
|
97
|
+
|
|
79
98
|
if (isMatch) {
|
|
80
99
|
const matches = tokens.filter(token => lower.includes(token));
|
|
100
|
+
|
|
81
101
|
for (const token of matches) candidate.matched.add(token);
|
|
82
102
|
const ext = path.extname(candidate.path).toLowerCase();
|
|
83
103
|
const items = SOURCE_EXT.has(ext) ? extractStructuralSurface(text, ext).items : [];
|
|
84
104
|
let declaration;
|
|
85
105
|
let definitionCoverage = 0;
|
|
86
106
|
let exact = false;
|
|
107
|
+
|
|
87
108
|
for (const item of items) {
|
|
88
109
|
const name = item.name.toLowerCase();
|
|
89
110
|
const itemExact = name === query.toLowerCase();
|
|
90
111
|
const coverage = tokens.filter(token => name.includes(token)).length;
|
|
112
|
+
|
|
91
113
|
if (itemExact || coverage > definitionCoverage) { declaration = item; definitionCoverage = coverage; exact = itemExact; }
|
|
114
|
+
|
|
92
115
|
if (exact) break;
|
|
93
116
|
}
|
|
117
|
+
|
|
94
118
|
const score = (exact ? 10000 : 0) + definitionCoverage * 40 + matches.length;
|
|
119
|
+
|
|
120
|
+
if (exact) candidate.exactLines.add(lineNumber);
|
|
121
|
+
|
|
95
122
|
if (score > candidate.anchorScore) {
|
|
96
123
|
candidate.anchorScore = score;
|
|
97
124
|
candidate.line = lineNumber;
|
|
@@ -100,63 +127,86 @@ function inspectLine(candidate, lineNumber, raw, query, tokens, isMatch) {
|
|
|
100
127
|
candidate.definitionCoverage = definitionCoverage;
|
|
101
128
|
candidate.lineCoverage = matches.length;
|
|
102
129
|
candidate.context.clear();
|
|
130
|
+
|
|
103
131
|
for (const [number, line] of candidate.recent) if (number >= lineNumber - 2) candidate.context.set(number, line);
|
|
104
132
|
}
|
|
105
133
|
}
|
|
134
|
+
|
|
106
135
|
const excerpt = truncateChars(text, 240, "source line").text;
|
|
136
|
+
|
|
107
137
|
if (lineNumber >= candidate.line - 2 && lineNumber <= candidate.line + 4) candidate.context.set(lineNumber, excerpt);
|
|
108
138
|
candidate.recent.push([lineNumber, excerpt]);
|
|
139
|
+
|
|
109
140
|
if (candidate.recent.length > 2) candidate.recent.shift();
|
|
110
141
|
}
|
|
111
142
|
|
|
112
143
|
function inspectOverlay(candidate, text, needles, query, tokens) {
|
|
113
144
|
const lines = text.split("\n");
|
|
114
145
|
const matches = [];
|
|
146
|
+
|
|
115
147
|
for (let i = 0; i < lines.length; i++) if (needles.some(needle => lines[i].toLowerCase().includes(needle))) matches.push(i);
|
|
148
|
+
|
|
116
149
|
for (const i of matches) inspectLine(candidate, i + 1, lines[i], query, tokens, true);
|
|
117
150
|
candidate.context.clear();
|
|
151
|
+
|
|
118
152
|
for (let i = Math.max(0, candidate.line - 3); i < Math.min(lines.length, candidate.line + 4); i++) candidate.context.set(i + 1, truncateChars(lines[i], 240, "source line").text);
|
|
119
153
|
}
|
|
120
154
|
|
|
121
155
|
async function contentCandidates({ dir, includeHidden, query, tokens, flags, pendingPaths, run, overlayText, signal, exact, diskFiles }) {
|
|
122
156
|
const needles = exact ? [query.toLowerCase()] : tokens;
|
|
123
157
|
const args = ["rg", "--json", "--fixed-strings", "--ignore-case", "--before-context", "2", "--after-context", "4"];
|
|
158
|
+
|
|
124
159
|
if (includeHidden) args.push("--hidden");
|
|
125
160
|
args.push("-g", "!.git/**", "-g", "!**/.git/**");
|
|
161
|
+
|
|
126
162
|
for (const needle of needles) args.push("-e", needle);
|
|
127
163
|
args.push("--", dir);
|
|
164
|
+
|
|
128
165
|
const response = diskFiles ? await run(args, { cwd: dir, timeoutMs: 15000, maxOutputChars: MAX_SEARCH_CHARS, signal })
|
|
129
166
|
: { stdout: "", stderr: "", exitCode: 1 };
|
|
167
|
+
|
|
130
168
|
if (response.exitCode !== 0 && response.exitCode !== 1) throw new Error("source search failed: " + response.stderr.trim());
|
|
131
169
|
const candidates = new Map();
|
|
132
170
|
const records = response.stdout.split("\n");
|
|
171
|
+
|
|
133
172
|
for (let i = 0; i < records.length; i++) {
|
|
134
173
|
if ((i & 127) === 0) signal?.throwIfAborted();
|
|
174
|
+
|
|
135
175
|
if (!records[i]) continue;
|
|
136
176
|
let record;
|
|
177
|
+
|
|
137
178
|
try { record = JSON.parse(records[i]); } catch (error) {
|
|
138
179
|
if (response.outputTruncated && i === records.length - 1) break;
|
|
139
180
|
throw error;
|
|
140
181
|
}
|
|
182
|
+
|
|
141
183
|
if (record.type !== "match" && record.type !== "context") continue;
|
|
142
184
|
const data = record.data;
|
|
185
|
+
|
|
143
186
|
if (!data?.path?.text || !isString(data.lines?.text)) continue;
|
|
144
187
|
const filePath = path.resolve(dir, data.path.text);
|
|
188
|
+
|
|
145
189
|
if (!inScope(filePath, dir, includeHidden) || overlayText(filePath) !== undefined) continue;
|
|
146
190
|
let candidate = candidates.get(filePath);
|
|
191
|
+
|
|
147
192
|
if (!candidate) {
|
|
148
193
|
candidate = makeCandidate(filePath, dir, query, tokens, flags);
|
|
149
194
|
candidates.set(filePath, candidate);
|
|
150
195
|
}
|
|
196
|
+
|
|
151
197
|
inspectLine(candidate, data.line_number, data.lines.text, query, tokens, record.type === "match");
|
|
152
198
|
}
|
|
199
|
+
|
|
153
200
|
for (const filePath of pendingPaths) {
|
|
154
201
|
const pending = overlayText(filePath);
|
|
202
|
+
|
|
155
203
|
if (pending === undefined) continue;
|
|
156
204
|
const candidate = makeCandidate(filePath, dir, query, tokens, flags);
|
|
157
205
|
inspectOverlay(candidate, pending, needles, query, tokens);
|
|
206
|
+
|
|
158
207
|
if (candidate.matched.size) candidates.set(filePath, candidate);
|
|
159
208
|
}
|
|
209
|
+
|
|
160
210
|
return { candidates, truncated: response.outputTruncated === true };
|
|
161
211
|
}
|
|
162
212
|
|
|
@@ -169,69 +219,131 @@ function rankScore(candidate, tokenCount) {
|
|
|
169
219
|
|
|
170
220
|
function location(candidate, root) {
|
|
171
221
|
const context = candidate.context;
|
|
222
|
+
|
|
172
223
|
return { path: path.relative(root, candidate.path), line: candidate.line, signature: candidate.signature,
|
|
173
224
|
context: [...context].sort((a, b) => a[0] - b[0]).map(([line, text]) => (line === candidate.line ? "►" : " ") + line + " " + text) };
|
|
174
225
|
}
|
|
175
226
|
|
|
227
|
+
async function spanCandidates(filePath, lines, root, overlayText) {
|
|
228
|
+
const staged = overlayText(filePath);
|
|
229
|
+
const text = staged !== undefined ? staged : await fs.readFile(filePath, "utf8");
|
|
230
|
+
const spans = WorkspaceIndex.spansOf(WorkspaceIndex.fromText(filePath, text));
|
|
231
|
+
const rel = path.relative(root, filePath);
|
|
232
|
+
|
|
233
|
+
return lines.map(line => {
|
|
234
|
+
const span = pickSpan(spans, { line }) ?? { start: line, end: line };
|
|
235
|
+
|
|
236
|
+
return spanCandidate(rel, line, spanWindow(text, span.start, span.end));
|
|
237
|
+
});
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
async function rankedSpanCandidates(ranked, root, overlayText) {
|
|
241
|
+
const out = [];
|
|
242
|
+
|
|
243
|
+
for (const candidate of ranked) {
|
|
244
|
+
const lines = candidate.exactLines?.size ? [...candidate.exactLines].sort((a, b) => a - b) : [candidate.line];
|
|
245
|
+
out.push(...await spanCandidates(candidate.path, lines, root, overlayText));
|
|
246
|
+
if (out.length >= MAX_ALTERNATIVES) break;
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
return out.slice(0, MAX_ALTERNATIVES);
|
|
250
|
+
}
|
|
251
|
+
|
|
176
252
|
export async function executeSnap({ query, searchDir, root, includeHidden = false, run = runCommand, overlayText = () => undefined, pendingPaths = [], pathContext = {}, signal }) {
|
|
177
253
|
const flags = tokenizeQuery(query);
|
|
178
254
|
const tokens = [...new Set(flags.tokens.map(stem))];
|
|
255
|
+
|
|
179
256
|
if (tokens.length === 0) throw new Error("read requires a file path or a searchable source question");
|
|
257
|
+
|
|
180
258
|
if (tokens.length > 16) throw new Error("source question is too broad; use at most 16 keywords");
|
|
181
259
|
query = query.trim();
|
|
182
260
|
const dir = path.resolve(searchDir || process.cwd());
|
|
261
|
+
|
|
183
262
|
if (dir.split(path.sep).includes(".git")) throw new Error("cannot search Git metadata");
|
|
184
263
|
signal?.throwIfAborted();
|
|
185
264
|
flags.wantsTest ||= isTestPath(path.relative(root ?? dir, dir));
|
|
186
265
|
pendingPaths = pendingPaths.filter(file => inScope(file, dir, includeHidden));
|
|
266
|
+
|
|
187
267
|
const diskFiles = await fs.stat(dir).then(stat => stat.isDirectory(), error => {
|
|
188
268
|
if (error.code !== "ENOENT" || !pendingPaths.length) throw error;
|
|
269
|
+
|
|
189
270
|
return false;
|
|
190
271
|
});
|
|
272
|
+
|
|
191
273
|
const empty = { path: null, line: null, signature: "", confidence: 0, context: [] };
|
|
192
274
|
const exact = /^[a-zA-Z_$][\w$]*$/.test(query);
|
|
193
275
|
const search = await contentCandidates({ dir, includeHidden, query, tokens, flags, pendingPaths, run, overlayText, signal, exact, diskFiles });
|
|
194
276
|
// A declaration hit needs no prerequisite file listing or persistent index.
|
|
195
277
|
// Bare names can name files, even when callers mention the same word.
|
|
196
278
|
const needsPaths = !search.candidates.size || (exact && ![...search.candidates.values()].some(candidate => candidate.exactDefinition));
|
|
279
|
+
|
|
197
280
|
const listing = needsPaths && !search.truncated && diskFiles
|
|
198
281
|
? await run(["rg", "--files", "--null", ...(includeHidden ? ["--hidden"] : []), "-g", "!.git/**", "-g", "!**/.git/**", dir], { cwd: dir, signal, timeoutMs: 15000, maxOutputChars: MAX_SEARCH_CHARS })
|
|
199
282
|
: { stdout: "", exitCode: 1 };
|
|
283
|
+
|
|
200
284
|
if (listing.exitCode !== 0 && listing.exitCode !== 1) throw new Error("source file listing failed: " + listing.stderr.trim());
|
|
201
|
-
|
|
285
|
+
|
|
286
|
+
const paths = [...new Set([...listing.stdout.split("\0").flatMap(file => file ? [path.resolve(dir, file)] : []), ...pendingPaths])]
|
|
202
287
|
.filter(file => inScope(file, dir, includeHidden));
|
|
288
|
+
|
|
203
289
|
for (const filePath of paths) {
|
|
204
290
|
if (!inScope(filePath, dir, includeHidden) || search.candidates.has(filePath)) continue;
|
|
205
291
|
const relative = path.relative(dir, filePath).toLowerCase();
|
|
292
|
+
|
|
206
293
|
if (!tokens.some(token => relative.includes(token))) continue;
|
|
294
|
+
|
|
207
295
|
if (exact && tokens.length > 1 && !relative.includes(query.toLowerCase())) continue;
|
|
208
296
|
const candidate = makeCandidate(filePath, dir, query, tokens, flags);
|
|
297
|
+
|
|
209
298
|
if (candidate.pathScore > 0) search.candidates.set(filePath, candidate);
|
|
210
299
|
}
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
300
|
+
|
|
301
|
+
const ranked = [];
|
|
302
|
+
|
|
303
|
+
for (const candidate of search.candidates.values()) {
|
|
304
|
+
if (candidate.pathScore > -50) {
|
|
305
|
+
ranked.push({ ...candidate, score: rankScore(candidate, tokens.length) });
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
ranked.sort((a, b) => b.score - a.score || a.path.localeCompare(b.path));
|
|
310
|
+
|
|
214
311
|
const incomplete = search.truncated || listing.outputTruncated === true;
|
|
215
312
|
const relativeRoot = root ?? dir;
|
|
216
313
|
const candidates = ranked.slice(0, MAX_ALTERNATIVES).map(candidate => location(candidate, relativeRoot));
|
|
314
|
+
|
|
217
315
|
if (incomplete) return { ...empty, status: "incomplete", candidates, message: "Search output exceeded its budget. Narrow the directory with read(path, {about: question})." };
|
|
316
|
+
|
|
218
317
|
if (!ranked.length) {
|
|
219
318
|
// Reuse bounded filename discovery; fuzzy rank never authorizes a source selection.
|
|
220
319
|
const eligible = exact && query.length >= 4 && query.length <= 64;
|
|
221
320
|
const limited = eligible && paths.length > 1024;
|
|
321
|
+
|
|
222
322
|
const fuzzy = eligible ? rankPaths(query, paths.slice(0, 1024).map(file => relativeSlash(relativeRoot, file)),
|
|
223
323
|
{ ...pathContext, maxTypos: 1 }).filter(hit => hit.score > 0).slice(0, MAX_ALTERNATIVES) : [];
|
|
324
|
+
|
|
224
325
|
if (fuzzy.length || limited) return { ...empty, status: limited ? "incomplete" : "ambiguous",
|
|
225
326
|
candidates: fuzzy.map(hit => ({ path: hit.path, line: 1, context: [], match: "fuzzy" })),
|
|
226
327
|
message: limited ? "No literal match; fuzzy hints cover only 1024 paths. Narrow the directory." : "No literal match. Fuzzy filename hints are not selected source; read an explicit path." };
|
|
328
|
+
|
|
227
329
|
return { ...empty, status: "not_found" };
|
|
228
330
|
}
|
|
331
|
+
|
|
229
332
|
const best = ranked[0];
|
|
230
333
|
const second = ranked[1];
|
|
231
334
|
const margin = second ? (best.score - second.score) / Math.max(1, best.score) : 1;
|
|
232
335
|
const coverage = Math.max(best.matched.size, best.pathCoverage) / tokens.length;
|
|
233
336
|
const uniqueExact = best.exactDefinition && !second?.exactDefinition || best.exactPath && !second?.exactPath && !second?.exactDefinition;
|
|
234
|
-
|
|
337
|
+
|
|
338
|
+
if (!uniqueExact && (coverage < 0.6 || margin < 0.15 || best.definitionCoverage / tokens.length < 0.5)) {
|
|
339
|
+
return { ...empty, status: "ambiguous", candidates: await rankedSpanCandidates(ranked, relativeRoot, overlayText) };
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
if (best.exactLines.size > 1) {
|
|
343
|
+
return { ...empty, status: "ambiguous", candidates: await rankedSpanCandidates([best], relativeRoot, overlayText) };
|
|
344
|
+
}
|
|
345
|
+
|
|
235
346
|
const confidence = uniqueExact ? 0.95 : Math.min(0.85, 0.5 + coverage * 0.2 + margin * 0.15);
|
|
347
|
+
|
|
236
348
|
return { ...candidates[0], status: "found", confidence: Number(confidence.toFixed(2)) };
|
|
237
349
|
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { truncateChars } from "../output/format.js";
|
|
2
|
+
|
|
3
|
+
export function pickSpan(spans, { line, name } = {}) {
|
|
4
|
+
const needle = typeof name === "string" && /^[A-Za-z_$][\w$]*$/.test(name.trim()) ? name.trim().toLowerCase() : "";
|
|
5
|
+
const named = needle ? spans.filter(item => item.name.toLowerCase() === needle) : [];
|
|
6
|
+
|
|
7
|
+
if (named.length === 1) return named[0];
|
|
8
|
+
if (!line) return;
|
|
9
|
+
|
|
10
|
+
return spans.find(item => item.start === line)
|
|
11
|
+
?? spans.filter(item => item.start <= line && line <= item.end).sort((a, b) => (a.end - a.start) - (b.end - b.start))[0];
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export function spanWindow(text, start, end) {
|
|
15
|
+
const raw = text.split("\n");
|
|
16
|
+
|
|
17
|
+
return {
|
|
18
|
+
start,
|
|
19
|
+
end,
|
|
20
|
+
text: raw.slice(start - 1, end).join("\n"),
|
|
21
|
+
signature: truncateChars((raw[start - 1] ?? "").trim().replace(/\{.*$/, "").trim(), 240, "signature").text,
|
|
22
|
+
context: Array.from({ length: Math.max(0, end - start + 1) }, (_, i) => {
|
|
23
|
+
const n = start + i;
|
|
24
|
+
|
|
25
|
+
return { line: n, text: raw[n - 1] ?? "" };
|
|
26
|
+
}),
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export function spanCandidate(relPath, line, window) {
|
|
31
|
+
return {
|
|
32
|
+
path: relPath,
|
|
33
|
+
line,
|
|
34
|
+
lines: [window.start, window.end],
|
|
35
|
+
text: window.text,
|
|
36
|
+
signature: window.signature,
|
|
37
|
+
context: window.context.map(row => (row.line === line ? "►" : " ") + row.line + " " + row.text),
|
|
38
|
+
};
|
|
39
|
+
}
|