pi-supernova 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +97 -11
- package/docs/CHANGELOG.md +150 -0
- package/docs/TOKEN_COSTS.md +71 -29
- package/index.js +126 -82
- package/package.json +2 -2
- package/src/adapters/bash.js +73 -0
- package/src/adapters/edit.js +249 -0
- package/src/adapters/errors.js +31 -0
- package/src/adapters/index.js +31 -0
- package/src/adapters/list.js +102 -0
- package/src/adapters/read.js +805 -0
- package/src/adapters/refs.js +41 -0
- package/src/adapters/write.js +96 -0
- package/src/bridge/catalog.js +30 -220
- package/src/bridge/host-bridge.js +142 -1032
- package/src/bridge/invoke.js +35 -0
- package/src/bridge/native-tools.js +1 -188
- package/src/context/evidence.js +142 -70
- package/src/context/fuzzy.js +61 -22
- package/src/context/ledger.js +43 -24
- package/src/context/outline.js +26 -12
- package/src/context/repo-index.js +242 -71
- package/src/context/search.js +189 -56
- package/src/context/snap.js +306 -150
- package/src/context/spans.js +2 -1
- package/src/context/surface.js +29 -14
- package/src/contract/bash.js +31 -0
- package/src/contract/edit.js +95 -0
- package/src/contract/read.js +220 -0
- package/src/fs/check.js +19 -7
- package/src/fs/diff.js +18 -7
- package/src/fs/json-read.js +66 -35
- package/src/fs/patch.js +97 -51
- package/src/fs/source-window.js +82 -0
- package/src/fs/text-ops.js +512 -0
- package/src/fs/vfs.js +289 -162
- package/src/fs/workspace.js +122 -105
- package/src/output/bottleneck.js +211 -107
- package/src/output/format.js +112 -63
- package/src/runtime/guest-deny-imports.js +34 -0
- package/src/runtime/guest-worker.js +306 -213
- package/src/runtime/parallel.js +99 -63
- package/src/runtime/program-batch.js +189 -69
- package/src/runtime/program-file.js +6 -3
- package/src/runtime/reference.js +13 -12
- package/src/runtime/runtime.js +327 -176
- package/src/shared/decode.js +61 -27
- package/src/ui/omp-frame.js +70 -46
- package/src/ui/render-measure.js +51 -29
- package/src/ui/render.js +242 -146
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { isString } from "../shared/decode.js";
|
|
2
|
+
|
|
3
|
+
/** Ordered permission rules. First decisive answer wins. */
|
|
4
|
+
export function toolIsCallable(name, env) {
|
|
5
|
+
if (name === "supernova" || env.excluded.has(name)) return false;
|
|
6
|
+
if (env.sessionInvalid()) return false;
|
|
7
|
+
if (env.nativeOwned(name)) return true;
|
|
8
|
+
if (env.hostSession) return env.evalAllows(name);
|
|
9
|
+
|
|
10
|
+
return env.listed(name);
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
function isArgvOwned(name, args) {
|
|
14
|
+
return name === "bash" && Array.isArray(args?.args) && process.platform !== "win32"
|
|
15
|
+
&& args.args.length === Object.keys(args.args).length && args.args.every(isString);
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
function hostExecutor(name, env) {
|
|
19
|
+
const delegated = env.hostTool(name);
|
|
20
|
+
|
|
21
|
+
if (delegated) return { exec: delegated.execute.bind(delegated), delegated };
|
|
22
|
+
if (env.hostSession) return { exec: undefined, delegated };
|
|
23
|
+
|
|
24
|
+
return { exec: env.executors.get(name), delegated };
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export function resolveInvokeTarget(name, args, env) {
|
|
28
|
+
const argvOwned = isArgvOwned(name, args);
|
|
29
|
+
const { exec, delegated } = hostExecutor(name, env);
|
|
30
|
+
|
|
31
|
+
if (exec && !argvOwned) return { kind: "override", exec, delegated, argvOwned: false };
|
|
32
|
+
if (env.natives[name]) return { kind: "native", native: env.natives[name], argvOwned };
|
|
33
|
+
|
|
34
|
+
return { kind: "unknown" };
|
|
35
|
+
}
|
|
@@ -1,189 +1,2 @@
|
|
|
1
|
-
|
|
2
|
-
import * as path from "node:path";
|
|
3
|
-
import { homedir } from "node:os";
|
|
4
|
-
import { loadConfig } from "../config/config.js";
|
|
5
|
-
import { isString } from "../shared/decode.js";
|
|
6
|
-
import { createHostBridge } from "./host-bridge.js";
|
|
7
|
-
import { buildPatchDiff } from "../fs/diff.js";
|
|
8
|
-
import { createNativeScheduler } from "../runtime/parallel.js";
|
|
9
|
-
import { truncateChars } from "../output/format.js";
|
|
10
|
-
import { clearPathCache, resolveWorkspacePath } from "../fs/workspace.js";
|
|
11
|
-
|
|
1
|
+
/** Native command names. Host registration lives in createHostBridge. */
|
|
12
2
|
export const NATIVE_NAMES = Object.freeze(["read", "edit", "write", "bash"]);
|
|
13
|
-
|
|
14
|
-
function userPath(cwd, input) {
|
|
15
|
-
if (!isString(input) || !input.trim()) throw new Error("path must be a non-empty string");
|
|
16
|
-
const value = input.trim().replace(/^@/, "");
|
|
17
|
-
|
|
18
|
-
return path.resolve(cwd, value === "~" ? homedir() : value.startsWith("~/") ? path.join(homedir(), value.slice(2)) : value);
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
function boundText(result, maxChars) {
|
|
22
|
-
const total = result.content.reduce((sum, block) => sum + (block.type === "text" ? block.text.length : 0), 0);
|
|
23
|
-
const truncated = total > maxChars;
|
|
24
|
-
const notice = truncated ? truncateChars("\n[Truncated: read files individually or narrow the line range/question.]", maxChars, "read-result").text : "";
|
|
25
|
-
let remaining = maxChars - notice.length;
|
|
26
|
-
|
|
27
|
-
const content = result.content.map(block => {
|
|
28
|
-
if (block.type !== "text") return block; // Images remain image attachments.
|
|
29
|
-
const bounded = truncateChars(block.text, remaining, "read-result");
|
|
30
|
-
remaining -= bounded.text.length;
|
|
31
|
-
|
|
32
|
-
return { ...block, text: bounded.text };
|
|
33
|
-
});
|
|
34
|
-
|
|
35
|
-
if (notice) content.push({ type: "text", text: notice });
|
|
36
|
-
const details = { ...result.details };
|
|
37
|
-
|
|
38
|
-
if (truncated) details.outputTruncated = true;
|
|
39
|
-
|
|
40
|
-
return { ...result, content, details };
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
/** Familiar Pi tool definitions with Supernova's source engine and atomic VFS. */
|
|
44
|
-
export function registerNativeTools(pi, host, config = loadConfig()) {
|
|
45
|
-
let cwd = process.cwd();
|
|
46
|
-
const base = createHostBridge({ pi: null, config: { ...config, seenWindow: 0 }, getCwd: () => cwd });
|
|
47
|
-
const scheduler = createNativeScheduler();
|
|
48
|
-
|
|
49
|
-
const factories = {
|
|
50
|
-
read: host.createReadToolDefinition,
|
|
51
|
-
edit: host.createEditToolDefinition,
|
|
52
|
-
write: host.createWriteToolDefinition,
|
|
53
|
-
bash: host.createBashToolDefinition,
|
|
54
|
-
};
|
|
55
|
-
|
|
56
|
-
function settingsFor(ctx) {
|
|
57
|
-
return host.SettingsManager?.create(ctx.cwd, undefined, { projectTrusted: ctx.isProjectTrusted?.() === true });
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
async function readOne(args, signal, ctx, bridge, id) {
|
|
61
|
-
const target = userPath(ctx.cwd, args.path);
|
|
62
|
-
let stat;
|
|
63
|
-
|
|
64
|
-
try { stat = await fs.stat(target); }
|
|
65
|
-
catch (error) { if (error.code !== "ENOENT" && error.code !== "ENOTDIR") throw error; }
|
|
66
|
-
|
|
67
|
-
signal?.throwIfAborted();
|
|
68
|
-
const image = /\.(png|jpe?g|gif|webp|bmp)$/i.test(target);
|
|
69
|
-
|
|
70
|
-
if (stat?.isFile() && (args.about === undefined || image)) {
|
|
71
|
-
// Pi retains image handling, full text, line windows, truncation metadata,
|
|
72
|
-
// and actionable continuation offsets. Do not summarize or dedupe these.
|
|
73
|
-
return factories.read(ctx.cwd, { autoResizeImages: image ? settingsFor(ctx)?.getImageAutoResize() : undefined }).execute(id, { ...args, path: target }, signal, undefined, ctx);
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
return boundText(await bridge.natives.read({ ...args, path: stat ? target : args.path }, signal), config.maxCallResultChars);
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
async function execute(name, id, args, signal, onUpdate, context) {
|
|
80
|
-
const ctx = { ...context, cwd: context?.cwd || cwd };
|
|
81
|
-
const bridge = base.fork({ getCwd: () => ctx.cwd });
|
|
82
|
-
bridge.bindCallContext(ctx, signal);
|
|
83
|
-
signal?.throwIfAborted();
|
|
84
|
-
|
|
85
|
-
try {
|
|
86
|
-
if (name === "read") {
|
|
87
|
-
if (!Array.isArray(args.path)) return await readOne(args, signal, ctx, bridge, id);
|
|
88
|
-
|
|
89
|
-
if (args.path.some(p => !isString(p) || !p.trim())) throw new Error("read paths must be non-empty strings");
|
|
90
|
-
|
|
91
|
-
if (args.path.length > 64) throw new Error("read path arrays support at most 64 entries; use smaller batches");
|
|
92
|
-
const groups = [], errors = [];
|
|
93
|
-
|
|
94
|
-
// Bound fan-out even when a model submits a large path array.
|
|
95
|
-
for (let offset = 0; offset < args.path.length; offset += 8) {
|
|
96
|
-
const results = await Promise.all(args.path.slice(offset, offset + 8).map(async file => {
|
|
97
|
-
try {
|
|
98
|
-
const result = await readOne({ ...args, path: file }, signal, ctx, bridge, id);
|
|
99
|
-
|
|
100
|
-
return { content: [{ type: "text", text: "File: " + file }, ...result.content] };
|
|
101
|
-
} catch (error) {
|
|
102
|
-
signal?.throwIfAborted();
|
|
103
|
-
errors.push({ path: file, message: error.message });
|
|
104
|
-
|
|
105
|
-
return { content: [{ type: "text", text: "[read error: " + file + "] " + error.message }] };
|
|
106
|
-
}
|
|
107
|
-
}));
|
|
108
|
-
|
|
109
|
-
groups.push(...results);
|
|
110
|
-
}
|
|
111
|
-
|
|
112
|
-
const content = [];
|
|
113
|
-
let remaining = config.maxCallResultChars, outputTruncated = false;
|
|
114
|
-
|
|
115
|
-
for (let i = 0; i < groups.length; i++) {
|
|
116
|
-
const bounded = boundText(groups[i], Math.floor(remaining / (groups.length - i)));
|
|
117
|
-
remaining -= bounded.content.reduce((sum, block) => sum + (block.type === "text" ? block.text.length : 0), 0);
|
|
118
|
-
outputTruncated ||= bounded.details.outputTruncated === true;
|
|
119
|
-
content.push(...bounded.content);
|
|
120
|
-
}
|
|
121
|
-
|
|
122
|
-
return { content, details: { batch: true, count: args.path.length, errors, outputTruncated } };
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
if (name === "bash") {
|
|
126
|
-
const settings = settingsFor(ctx);
|
|
127
|
-
|
|
128
|
-
try { return await factories.bash(ctx.cwd, { shellPath: settings?.getShellPath(), commandPrefix: settings?.getShellCommandPrefix() }).execute(id, args, signal, onUpdate, ctx); }
|
|
129
|
-
finally { bridge.invalidateFiles(); }
|
|
130
|
-
}
|
|
131
|
-
|
|
132
|
-
clearPathCache();
|
|
133
|
-
const target = await resolveWorkspacePath(ctx.cwd, userPath(ctx.cwd, args.path), name);
|
|
134
|
-
const ops = bridge.fileOperations;
|
|
135
|
-
let before, after;
|
|
136
|
-
|
|
137
|
-
const tool = factories[name](ctx.cwd, { operations: {
|
|
138
|
-
...ops,
|
|
139
|
-
async readFile(file) { const buffer = await ops.readFile(file); before = buffer.toString("utf8");
|
|
140
|
-
|
|
141
|
-
return buffer; },
|
|
142
|
-
async writeFile(file, content) { await ops.writeFile(file, content); after = content; },
|
|
143
|
-
} });
|
|
144
|
-
|
|
145
|
-
// Pi's edit/write implementations hold its shared canonical per-file queue
|
|
146
|
-
// over the entire mutation, including our atomic replacement operation.
|
|
147
|
-
const result = await tool.execute(id, { ...args, path: target }, signal, onUpdate, ctx);
|
|
148
|
-
|
|
149
|
-
if (name === "edit" && before !== undefined && after !== undefined && result.details?.patch) {
|
|
150
|
-
const summary = await bridge.summarizeEdit(target, before, after, buildPatchDiff(target, result.details.patch));
|
|
151
|
-
result.content = [{ type: "text", text: summary }];
|
|
152
|
-
}
|
|
153
|
-
|
|
154
|
-
return result;
|
|
155
|
-
} finally { bridge.close(); }
|
|
156
|
-
}
|
|
157
|
-
|
|
158
|
-
for (const name of NATIVE_NAMES) {
|
|
159
|
-
const definition = factories[name](cwd);
|
|
160
|
-
|
|
161
|
-
const tool = {
|
|
162
|
-
...definition,
|
|
163
|
-
execute(id, args, signal, onUpdate, ctx) {
|
|
164
|
-
return scheduler.schedule(name, () => execute(name, id, args, signal, onUpdate, ctx), signal);
|
|
165
|
-
},
|
|
166
|
-
};
|
|
167
|
-
|
|
168
|
-
if (name === "read") {
|
|
169
|
-
tool.description += " Also reads directories or finds source from a symbol/question passed as path. Use about to focus a file or directory on a question. An array of paths returns all readable files and labels individual errors.";
|
|
170
|
-
tool.parameters = { ...definition.parameters, properties: {
|
|
171
|
-
...definition.parameters.properties,
|
|
172
|
-
path: { anyOf: [{ type: "string" }, { type: "array", items: { type: "string" }, maxItems: 64 }], description: "File, directory, source question, or up to 64 paths to read" },
|
|
173
|
-
about: { type: "string", description: "Focus on this question or symbol; source selection reports uncertainty instead of guessing" },
|
|
174
|
-
} };
|
|
175
|
-
tool.promptGuidelines = [...(definition.promptGuidelines || []), "read can find source from a symbol or question; check its selection status before choosing a file. Plain file reads preserve full text within the stated limits."];
|
|
176
|
-
}
|
|
177
|
-
|
|
178
|
-
pi.registerTool(tool);
|
|
179
|
-
}
|
|
180
|
-
|
|
181
|
-
pi.on("session_start", (_event, ctx) => {
|
|
182
|
-
cwd = ctx?.cwd || cwd;
|
|
183
|
-
base.invalidateFiles();
|
|
184
|
-
});
|
|
185
|
-
pi.registerCommand("supernova", {
|
|
186
|
-
description: "Show Supernova native runtime status",
|
|
187
|
-
handler: async (_args, ctx) => ctx.ui.notify("Supernova runtime: read, edit, write, bash; " + scheduler.stats.calls + " calls, " + scheduler.stats.readWaves + " read waves, peak " + scheduler.stats.peakParallelReads + " parallel reads. Mutations are ordered; plain reads are not summarized or deduplicated.", "info"),
|
|
188
|
-
});
|
|
189
|
-
}
|
package/src/context/evidence.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import * as fs from "node:fs/promises";
|
|
1
2
|
import * as path from "node:path";
|
|
2
3
|
import { WorkspaceIndex } from "./repo-index.js";
|
|
3
4
|
import { tokenizeQuery, scorePathTopology, stem } from "./snap.js";
|
|
@@ -86,36 +87,36 @@ function spanLines(span) {
|
|
|
86
87
|
|
|
87
88
|
// ---- eq.3 / eq.4: entity–context graph over candidate spans ----
|
|
88
89
|
|
|
89
|
-
function
|
|
90
|
-
const
|
|
91
|
-
|
|
92
|
-
const
|
|
90
|
+
function countSpanEntities(span, entityNames) {
|
|
91
|
+
const counts = new Map();
|
|
92
|
+
let total = 0;
|
|
93
|
+
const { idents } = span.lines;
|
|
93
94
|
|
|
94
|
-
for (
|
|
95
|
-
const
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
for (let li = span.start - 1; li < span.end; li++) {
|
|
100
|
-
for (const word of idents[li]) {
|
|
101
|
-
if (!entityNames.has(word)) continue;
|
|
102
|
-
counts.set(word, (counts.get(word) || 0) + 1);
|
|
103
|
-
total += 1;
|
|
104
|
-
}
|
|
95
|
+
for (let li = span.start - 1; li < span.end; li++) {
|
|
96
|
+
for (const word of idents[li]) {
|
|
97
|
+
if (!entityNames.has(word)) continue;
|
|
98
|
+
counts.set(word, (counts.get(word) || 0) + 1);
|
|
99
|
+
total += 1;
|
|
105
100
|
}
|
|
101
|
+
}
|
|
106
102
|
|
|
107
|
-
|
|
103
|
+
return { counts, total };
|
|
104
|
+
}
|
|
108
105
|
|
|
109
|
-
|
|
110
|
-
|
|
106
|
+
function weightsFromCounts(counts, total, entitySpans, spanId) {
|
|
107
|
+
const weights = new Map();
|
|
111
108
|
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
}
|
|
109
|
+
for (const [e, c] of counts) {
|
|
110
|
+
weights.set(e, c / total); // eq.4
|
|
115
111
|
|
|
116
|
-
|
|
112
|
+
if (!entitySpans.has(e)) entitySpans.set(e, new Set());
|
|
113
|
+
entitySpans.get(e).add(spanId);
|
|
117
114
|
}
|
|
118
115
|
|
|
116
|
+
return weights;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
function hubEntities(entitySpans, spans) {
|
|
119
120
|
// Entities present in a large share of spans (isString, path, …) carry no query signal; keep them out of propagation.
|
|
120
121
|
const hubLimit = Math.max(HUB_MIN, Math.floor(spans.length * HUB_FRACTION));
|
|
121
122
|
const hubs = new Set();
|
|
@@ -124,7 +125,20 @@ function buildGraph(spans) {
|
|
|
124
125
|
if (ids.size > hubLimit) hubs.add(entity);
|
|
125
126
|
}
|
|
126
127
|
|
|
127
|
-
return
|
|
128
|
+
return hubs;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
function buildGraph(spans) {
|
|
132
|
+
const entityNames = new Set(spans.map((s) => s.name).filter((n) => n && n.length > 2));
|
|
133
|
+
const spanEntities = new Map(); // span.id → Map(entity → w(d,e))
|
|
134
|
+
const entitySpans = new Map(); // entity → Set(span.id)
|
|
135
|
+
|
|
136
|
+
for (const span of spans) {
|
|
137
|
+
const { counts, total } = countSpanEntities(span, entityNames);
|
|
138
|
+
spanEntities.set(span.id, weightsFromCounts(counts, total, entitySpans, span.id));
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
return { entityNames, spanEntities, entitySpans, hubs: hubEntities(entitySpans, spans), byId: new Map(spans.map((s) => [s.id, s])) };
|
|
128
142
|
}
|
|
129
143
|
|
|
130
144
|
// ---- eq.8 / eq.9: entity activation and one propagation step ----
|
|
@@ -143,7 +157,7 @@ function lexicalSim(a, b) {
|
|
|
143
157
|
|
|
144
158
|
function activateEntities(profile, graph) {
|
|
145
159
|
const eta = new Map();
|
|
146
|
-
const anchors = profile.subjects.length ? profile.subjects : profile.
|
|
160
|
+
const anchors = profile.subjects.length ? profile.subjects : profile.stems;
|
|
147
161
|
|
|
148
162
|
for (const anchor of anchors) {
|
|
149
163
|
let best = null;
|
|
@@ -377,7 +391,7 @@ function normalize(scores) {
|
|
|
377
391
|
|
|
378
392
|
// ---- candidate files (boundary + topology + entity hits) ----
|
|
379
393
|
|
|
380
|
-
function
|
|
394
|
+
function topologyScored(files, profile) {
|
|
381
395
|
const scored = [];
|
|
382
396
|
|
|
383
397
|
for (const f of files) {
|
|
@@ -387,27 +401,41 @@ function candidateFiles(files, profile, index, limit, overlayText) {
|
|
|
387
401
|
}
|
|
388
402
|
|
|
389
403
|
scored.sort((a, b) => b.s - a.s);
|
|
390
|
-
const chosen = new Set();
|
|
391
|
-
const anchors = (profile.subjects.length ? profile.subjects : profile.keywords).map((a) => a.toLowerCase()).filter((a) => a.length > 2);
|
|
392
404
|
|
|
393
|
-
|
|
405
|
+
return scored;
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
function pendingAnchorHits(files, anchors, overlayText) {
|
|
409
|
+
return files.filter(file => {
|
|
394
410
|
const pending = overlayText(file);
|
|
395
411
|
|
|
396
|
-
return pending !== undefined && anchors.some(anchor => pending.toLowerCase().includes(anchor));
|
|
412
|
+
return pending !== undefined && Buffer.byteLength(pending, "utf8") <= 512 * 1024 && anchors.some(anchor => pending.toLowerCase().includes(anchor));
|
|
397
413
|
});
|
|
414
|
+
}
|
|
398
415
|
|
|
399
|
-
|
|
400
|
-
|
|
416
|
+
function chooseByHits(hits, profile, limit, chosen) {
|
|
401
417
|
for (const f of hits) {
|
|
402
418
|
if (chosen.size >= limit) break;
|
|
403
419
|
|
|
404
420
|
if (profile.flags.wantsTest || scorePathTopology(f, profile.keywords, profile.flags) > -50) chosen.add(f);
|
|
405
421
|
}
|
|
422
|
+
}
|
|
406
423
|
|
|
424
|
+
function fillFromScored(scored, limit, chosen) {
|
|
407
425
|
for (const { f } of scored) {
|
|
408
426
|
if (chosen.size >= limit) break;
|
|
409
427
|
chosen.add(f);
|
|
410
428
|
}
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
function candidateFiles(files, profile, index, limit, overlayText) {
|
|
432
|
+
const scored = topologyScored(files, profile);
|
|
433
|
+
const chosen = new Set();
|
|
434
|
+
const anchors = (profile.subjects.length ? profile.subjects : profile.stems).map((a) => a.toLowerCase()).filter((a) => a.length > 2);
|
|
435
|
+
const pendingHits = pendingAnchorHits(files, anchors, overlayText);
|
|
436
|
+
const hits = anchors.length ? [...new Set([...pendingHits, ...index.filesContaining(files, anchors, true)])] : [];
|
|
437
|
+
chooseByHits(hits, profile, limit, chosen);
|
|
438
|
+
fillFromScored(scored, limit, chosen);
|
|
411
439
|
|
|
412
440
|
return { files: [...chosen], fileScores: new Map(scored.map(({ f, s }) => [f, s])) };
|
|
413
441
|
}
|
|
@@ -494,70 +522,114 @@ function render(spans, picks, fused, opts, root) {
|
|
|
494
522
|
return out;
|
|
495
523
|
}
|
|
496
524
|
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
* @returns {{ route: string, spans: Array<{path, lines, name, kind, why, text}> }}
|
|
500
|
-
*/
|
|
501
|
-
export async function selectEvidence({ query, root, searchDir, index, overlayText = () => undefined, pendingPaths = [], options = {} }) {
|
|
502
|
-
const opts = { ...EVIDENCE_DEFAULTS, ...options };
|
|
503
|
-
const profile = profileQuery(query);
|
|
504
|
-
|
|
505
|
-
if (profile.keywords.length === 0) throw new Error("evidence requires at least one searchable concept keyword");
|
|
506
|
-
const searchRoot = path.resolve(searchDir || root);
|
|
507
|
-
|
|
508
|
-
const staged = pendingPaths.filter(file => {
|
|
525
|
+
function stagedInRoot(searchRoot, pendingPaths) {
|
|
526
|
+
return pendingPaths.filter(file => {
|
|
509
527
|
const relative = path.relative(searchRoot, file);
|
|
510
528
|
|
|
511
529
|
return relative !== ".." && !relative.startsWith(".." + path.sep) && !path.isAbsolute(relative);
|
|
512
530
|
});
|
|
531
|
+
}
|
|
513
532
|
|
|
514
|
-
|
|
533
|
+
function statCatch(error) {
|
|
534
|
+
if (error?.code === "ENOENT" || error?.code === "ENOTDIR") return null;
|
|
535
|
+
throw error;
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
async function diskFilesAt(searchRoot, index) {
|
|
539
|
+
const rootStat = await fs.stat(searchRoot).catch(statCatch);
|
|
540
|
+
|
|
541
|
+
if (rootStat?.isFile()) return [searchRoot];
|
|
542
|
+
|
|
543
|
+
if (rootStat) return index.files(searchRoot);
|
|
544
|
+
|
|
545
|
+
return [];
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
async function listedFiles(root, searchDir, pendingPaths, index) {
|
|
549
|
+
const searchRoot = path.resolve(searchDir || root);
|
|
550
|
+
const files = [...new Set([...await diskFilesAt(searchRoot, index), ...stagedInRoot(searchRoot, pendingPaths)])];
|
|
515
551
|
|
|
516
552
|
if (files.length === 0) throw new Error(`no files found to search in ${searchDir || root}`);
|
|
517
553
|
|
|
518
|
-
|
|
554
|
+
return files;
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
function overlayEntry(f, overlayText, index) {
|
|
558
|
+
const pending = overlayText(f);
|
|
559
|
+
|
|
560
|
+
return pending === undefined
|
|
561
|
+
? index.entry(f)
|
|
562
|
+
: Buffer.byteLength(pending, "utf8") <= 512 * 1024 ? WorkspaceIndex.fromText(f, pending) : null;
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
function collectSpans(chosenFiles, overlayText, index, maxSpanLines) {
|
|
519
566
|
const spans = [];
|
|
520
567
|
|
|
521
568
|
for (const f of chosenFiles) {
|
|
522
|
-
const
|
|
523
|
-
const entry = pending === undefined ? index.entry(f) : WorkspaceIndex.fromText(f, pending);
|
|
569
|
+
const entry = overlayEntry(f, overlayText, index);
|
|
524
570
|
|
|
525
571
|
if (!entry) continue;
|
|
526
|
-
spans.push(...spansOf(entry, f,
|
|
572
|
+
spans.push(...spansOf(entry, f, maxSpanLines));
|
|
527
573
|
}
|
|
528
574
|
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
const graph = buildGraph(spans);
|
|
532
|
-
const hier = hierarchicalScores(spans, fileScores, profile);
|
|
533
|
-
const hierNorm = normalize(hier);
|
|
534
|
-
const eta = propagate(activateEntities(profile, graph), spans, graph, profile);
|
|
535
|
-
const pi = pageRank(spans, graph, eta, hierNorm.map((s) => s * 0.5), opts);
|
|
536
|
-
const graphNorm = normalize([...pi]);
|
|
575
|
+
return spans;
|
|
576
|
+
}
|
|
537
577
|
|
|
578
|
+
function fuseScores(profile, graphNorm, hierNorm, rho) {
|
|
538
579
|
const [primary, secondary] = profile.route === "relational" ? [graphNorm, hierNorm] : [hierNorm, graphNorm];
|
|
539
|
-
const fused = primary.map((p, i) => opts.rho * p + (1 - opts.rho) * secondary[i]); // eq.13
|
|
540
580
|
|
|
541
|
-
|
|
542
|
-
|
|
581
|
+
return primary.map((p, i) => rho * p + (1 - rho) * secondary[i]); // eq.13
|
|
582
|
+
}
|
|
543
583
|
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
584
|
+
function spanAdmissible(span, profile) {
|
|
585
|
+
const p = span.path;
|
|
586
|
+
const isDoc = /\.(md|mdx|rst|txt)$/i.test(p);
|
|
547
587
|
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
588
|
+
return span.support > 0
|
|
589
|
+
&& (profile.flags.wantsTest || !isTestPath(p))
|
|
590
|
+
&& (profile.flags.wantsDoc || !isDoc);
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
function compareFused(spans, fused) {
|
|
594
|
+
return (a, b) => fused[b] - fused[a] || spans[a].path.localeCompare(spans[b].path) || spans[a].start - spans[b].start;
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
function dampUsageDefiners(spans, profile, fused, admissible) {
|
|
598
|
+
const usage = profile.answerType === "usage";
|
|
553
599
|
|
|
554
600
|
for (const i of admissible) {
|
|
555
601
|
if (usage && profile.subjects.includes(spans[i].name)) fused[i] *= 0.5; // a usage question is answered by callers
|
|
556
602
|
}
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
function pickEvidence(spans, fileScores, profile, opts) {
|
|
606
|
+
const graph = buildGraph(spans);
|
|
607
|
+
const hierNorm = normalize(hierarchicalScores(spans, fileScores, profile));
|
|
608
|
+
const eta = propagate(activateEntities(profile, graph), spans, graph, profile);
|
|
609
|
+
const pi = pageRank(spans, graph, eta, hierNorm.map((s) => s * 0.5), opts);
|
|
610
|
+
const fused = fuseScores(profile, normalize([...pi]), hierNorm, opts.rho);
|
|
611
|
+
// eq.15 Filter: boundary/type hard constraints and lexical support; Rank_ϕ: answer-type compatibility.
|
|
612
|
+
const admissible = spans.flatMap((span, i) => spanAdmissible(span, profile) ? [i] : []);
|
|
613
|
+
dampUsageDefiners(spans, profile, fused, admissible);
|
|
614
|
+
const main = admissible.sort(compareFused(spans, fused)).slice(0, opts.k);
|
|
615
|
+
|
|
616
|
+
return { picks: closure(main, spans, graph, fused, opts.k), fused };
|
|
617
|
+
}
|
|
557
618
|
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
619
|
+
/**
|
|
620
|
+
* R(q): top-K provenance-bearing source spans for a concept query, selected without any model call.
|
|
621
|
+
* @returns {{ route: string, spans: Array<{path, lines, name, kind, why, text}> }}
|
|
622
|
+
*/
|
|
623
|
+
export async function selectEvidence({ query, root, searchDir, index, overlayText = () => undefined, pendingPaths = [], options = {} }) {
|
|
624
|
+
const opts = { ...EVIDENCE_DEFAULTS, ...options };
|
|
625
|
+
const profile = profileQuery(query);
|
|
626
|
+
|
|
627
|
+
if (profile.keywords.length === 0) throw new Error("evidence requires at least one searchable concept keyword");
|
|
628
|
+
const { files: chosenFiles, fileScores } = candidateFiles(await listedFiles(root, searchDir, pendingPaths, index), profile, index, opts.maxCandidateFiles, overlayText);
|
|
629
|
+
const spans = collectSpans(chosenFiles, overlayText, index, opts.maxSpanLines);
|
|
630
|
+
|
|
631
|
+
if (spans.length === 0) return { route: profile.route, spans: [] };
|
|
632
|
+
const { picks, fused } = pickEvidence(spans, fileScores, profile, opts);
|
|
561
633
|
|
|
562
634
|
return { route: profile.route, spans: render(spans, picks, fused, opts, root) };
|
|
563
635
|
}
|
package/src/context/fuzzy.js
CHANGED
|
@@ -15,6 +15,7 @@ const AI_DECAY = Math.LN2 / 3; // per day
|
|
|
15
15
|
const AI_MAX_HISTORY_DAYS = 7;
|
|
16
16
|
|
|
17
17
|
const MAX_TIMESTAMPS_PER_FILE = 128;
|
|
18
|
+
const MAX_FRECENCY_FILES = 10000;
|
|
18
19
|
|
|
19
20
|
const AI_MODIFICATION_THRESHOLDS = [[16, 30], [8, 300], [4, 900], [2, 3600], [1, 14400]]; // [boost, seconds]
|
|
20
21
|
|
|
@@ -26,7 +27,10 @@ export class Frecency {
|
|
|
26
27
|
record(filePath, at = Date.now() / 1000) {
|
|
27
28
|
let list = this.access.get(filePath);
|
|
28
29
|
|
|
29
|
-
if (!list)
|
|
30
|
+
if (!list) {
|
|
31
|
+
if (this.access.size >= MAX_FRECENCY_FILES) this.access.delete(this.access.keys().next().value);
|
|
32
|
+
this.access.set(filePath, (list = []));
|
|
33
|
+
}
|
|
30
34
|
list.push(at);
|
|
31
35
|
|
|
32
36
|
if (list.length > MAX_TIMESTAMPS_PER_FILE) list.splice(0, list.length - MAX_TIMESTAMPS_PER_FILE);
|
|
@@ -118,18 +122,9 @@ function scoreAlignment(needle, nCmp, hay, hayCmp, start) {
|
|
|
118
122
|
return score;
|
|
119
123
|
}
|
|
120
124
|
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
if (direct) return { ...direct, typos: 0, exact: hay.toLowerCase() === needle.toLowerCase() };
|
|
126
|
-
|
|
127
|
-
if (maxTypos <= 0 || needle.length < 3) return null;
|
|
128
|
-
let best = null;
|
|
129
|
-
|
|
130
|
-
for (let i = 0; i < needle.length; i++) {
|
|
131
|
-
const shorter = needle.slice(0, i) + needle.slice(i + 1);
|
|
132
|
-
const m = fuzzyMatch(shorter, hay, { maxTypos: maxTypos - 1, caseSensitive });
|
|
125
|
+
function considerShorter(part, typosLeft, visit, best) {
|
|
126
|
+
for (let i = 0; i < part.length; i++) {
|
|
127
|
+
const m = visit(part.slice(0, i) + part.slice(i + 1), typosLeft - 1);
|
|
133
128
|
|
|
134
129
|
if (!m) continue;
|
|
135
130
|
const scored = { ...m, score: m.score - 12, typos: m.typos + 1, exact: false };
|
|
@@ -140,6 +135,37 @@ export function fuzzyMatch(needle, hay, { maxTypos = 0, caseSensitive = false }
|
|
|
140
135
|
return best;
|
|
141
136
|
}
|
|
142
137
|
|
|
138
|
+
function matchWithTypos(needle, hay, maxTypos, caseSensitive) {
|
|
139
|
+
const memo = new Map();
|
|
140
|
+
|
|
141
|
+
const visit = (part, typosLeft) => {
|
|
142
|
+
const key = part + "\0" + typosLeft;
|
|
143
|
+
|
|
144
|
+
if (memo.has(key)) return memo.get(key);
|
|
145
|
+
let best = matchOnce(part, hay, caseSensitive);
|
|
146
|
+
|
|
147
|
+
if (best) best = { ...best, typos: 0, exact: hay.toLowerCase() === part.toLowerCase() };
|
|
148
|
+
|
|
149
|
+
if (typosLeft > 0) best = considerShorter(part, typosLeft, visit, best);
|
|
150
|
+
memo.set(key, best);
|
|
151
|
+
|
|
152
|
+
return best;
|
|
153
|
+
};
|
|
154
|
+
|
|
155
|
+
return visit(needle, maxTypos);
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/** Best match allowing up to maxTypos skipped needle characters. */
|
|
159
|
+
export function fuzzyMatch(needle, hay, { maxTypos = 0, caseSensitive = false } = {}) {
|
|
160
|
+
const direct = matchOnce(needle, hay, caseSensitive);
|
|
161
|
+
|
|
162
|
+
if (direct) return { ...direct, typos: 0, exact: hay.toLowerCase() === needle.toLowerCase() };
|
|
163
|
+
|
|
164
|
+
if (maxTypos <= 0 || needle.length < 3 || needle.length > 128) return null;
|
|
165
|
+
|
|
166
|
+
return matchWithTypos(needle, hay, maxTypos, caseSensitive);
|
|
167
|
+
}
|
|
168
|
+
|
|
143
169
|
export function smartCase(query) {
|
|
144
170
|
return /[A-Z]/.test(query);
|
|
145
171
|
}
|
|
@@ -161,23 +187,34 @@ function distancePenalty(currentDir, candidateDir) {
|
|
|
161
187
|
* Rank file paths for a query the fff way. paths are workspace-relative "/"-joined.
|
|
162
188
|
* ctx: { frecency: Frecency, mtimeOf: (path) => sec, modified: Set(path), currentFile?: string, maxTypos }
|
|
163
189
|
*/
|
|
190
|
+
function partTypos(parts, ctx) {
|
|
191
|
+
return ctx.maxTypos ?? (parts[0].length >= 6 ? 2 : parts[0].length >= 4 ? 1 : 0);
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
function scoredPath(rel, parts, maxTypos, caseSensitive, ctx, currentDir) {
|
|
195
|
+
const matched = matchParts(parts, rel, maxTypos, caseSensitive);
|
|
196
|
+
|
|
197
|
+
if (!matched) return null;
|
|
198
|
+
const { base, first, exact } = matched;
|
|
199
|
+
const filenameStart = rel.lastIndexOf("/") + 1;
|
|
200
|
+
const boosts = filenameBonus(base, rel, filenameStart, first, parts[0]) + contextBoost(base, rel, ctx) + distancePenalty(currentDir, rel.slice(0, filenameStart));
|
|
201
|
+
|
|
202
|
+
return { path: rel, score: base + boosts, exact, typos: first.typos };
|
|
203
|
+
}
|
|
204
|
+
|
|
164
205
|
export function rankPaths(query, paths, ctx = {}) {
|
|
165
206
|
const parts = query.trim().split(/\s+/).filter((p) => p.length >= 2);
|
|
166
207
|
|
|
167
|
-
if (parts.length === 0) return [];
|
|
208
|
+
if (parts.length === 0 || parts.length > 16) return [];
|
|
168
209
|
const caseSensitive = smartCase(query);
|
|
169
|
-
const maxTypos =
|
|
210
|
+
const maxTypos = partTypos(parts, ctx);
|
|
170
211
|
const currentDir = ctx.currentFile ? ctx.currentFile.slice(0, ctx.currentFile.lastIndexOf("/") + 1) : "";
|
|
171
212
|
const out = [];
|
|
172
213
|
|
|
173
214
|
for (const rel of paths) {
|
|
174
|
-
const
|
|
215
|
+
const scored = scoredPath(rel, parts, maxTypos, caseSensitive, ctx, currentDir);
|
|
175
216
|
|
|
176
|
-
if (
|
|
177
|
-
const { base, first, exact } = matched;
|
|
178
|
-
const filenameStart = rel.lastIndexOf("/") + 1;
|
|
179
|
-
const boosts = filenameBonus(base, rel, filenameStart, first, parts[0]) + contextBoost(base, rel, ctx) + distancePenalty(currentDir, rel.slice(0, filenameStart));
|
|
180
|
-
out.push({ path: rel, score: base + boosts, exact, typos: first.typos });
|
|
217
|
+
if (scored) out.push(scored);
|
|
181
218
|
}
|
|
182
219
|
|
|
183
220
|
out.sort((a, b) => b.score - a.score || a.path.length - b.path.length || a.path.localeCompare(b.path));
|
|
@@ -212,7 +249,9 @@ function filenameBonus(base, rel, filenameStart, first, needle) {
|
|
|
212
249
|
|
|
213
250
|
/** fff: frecency boost base·f/100 and +15% for git-modified files. */
|
|
214
251
|
function contextBoost(base, rel, ctx) {
|
|
215
|
-
|
|
252
|
+
let frecency = 0;
|
|
253
|
+
|
|
254
|
+
try { frecency = ctx.frecency ? ctx.frecency.score(rel, ctx.mtimeOf?.(rel)) : 0; } catch {}
|
|
216
255
|
const gitBoost = ctx.modified?.has(rel) ? Math.floor((base * 15) / 100) : 0;
|
|
217
256
|
|
|
218
257
|
return Math.floor((base * frecency) / 100) + gitBoost;
|