syncstaff-mcp 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +86 -0
- package/dist/lib/agent-state.js +119 -0
- package/dist/lib/blast.js +462 -0
- package/dist/lib/client-config.js +81 -0
- package/dist/lib/env-compat.js +66 -0
- package/dist/lib/globs.js +0 -0
- package/dist/lib/ids.js +24 -0
- package/dist/lib/index/aliases.js +244 -0
- package/dist/lib/index/call-sites.js +178 -0
- package/dist/lib/index/checker-resolver.js +257 -0
- package/dist/lib/index/context-card.js +140 -0
- package/dist/lib/index/coverage.js +218 -0
- package/dist/lib/index/delivery.js +66 -0
- package/dist/lib/index/discovery.js +90 -0
- package/dist/lib/index/embedding.js +110 -0
- package/dist/lib/index/file-index.js +222 -0
- package/dist/lib/index/fingerprint.js +0 -0
- package/dist/lib/index/git-history.js +136 -0
- package/dist/lib/index/graph.js +234 -0
- package/dist/lib/index/impact.js +174 -0
- package/dist/lib/index/incremental.js +332 -0
- package/dist/lib/index/lexical.js +462 -0
- package/dist/lib/index/order.js +43 -0
- package/dist/lib/index/pages.js +357 -0
- package/dist/lib/index/persistence.js +233 -0
- package/dist/lib/index/pipeline.js +527 -0
- package/dist/lib/index/registry.js +106 -0
- package/dist/lib/index/resolve.js +280 -0
- package/dist/lib/index/semantic.js +381 -0
- package/dist/lib/index/surfaces.js +27 -0
- package/dist/lib/index/symbols.js +426 -0
- package/dist/lib/index/transformers-embedder.js +73 -0
- package/dist/lib/index/typescript-parser.js +532 -0
- package/dist/lib/index/vector-cache.js +176 -0
- package/dist/lib/index/verification.js +58 -0
- package/dist/lib/mcp-compaction.js +241 -0
- package/dist/lib/model-roles.js +206 -0
- package/dist/lib/path-warnings.js +90 -0
- package/dist/lib/protocol.js +95 -0
- package/dist/lib/types.js +69 -0
- package/dist/lib/version.js +21 -0
- package/dist/lib/worktree.js +211 -0
- package/dist/mcp/approval.js +0 -0
- package/dist/mcp/cloud-connector.js +99 -0
- package/dist/mcp/daemon-client.js +156 -0
- package/dist/mcp/daemon-protocol.js +100 -0
- package/dist/mcp/escalation-waiter.js +183 -0
- package/dist/mcp/graph-ops.js +169 -0
- package/dist/mcp/index.js +1151 -0
- package/dist/mcp/login.js +169 -0
- package/dist/mcp/setup.js +90 -0
- package/package.json +42 -0
|
@@ -0,0 +1,357 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { discoverSurfaceEdges } from "./surfaces.js";
|
|
3
|
+
import { byCodeUnit } from "./order.js";
|
|
4
|
+
/** Roughly 350 tokens of source, inside every supported encoder's window. */
|
|
5
|
+
export const EMBEDDED_BODY_CHARS = 1400;
|
|
6
|
+
/** Pack chunks to about this size so each embedding is close to full. */
|
|
7
|
+
const CHUNK_TARGET_CHARS = 1200;
|
|
8
|
+
/** Module context carried on every chunk. Roughly 45 tokens of the 512 budget. */
|
|
9
|
+
const SKELETON_CHARS = 180;
|
|
10
|
+
const LICENCE_BANNER = /^\s*\/\*[\s\S]*?\*\//;
|
|
11
|
+
const LEADING_IMPORTS = /^\s*(?:import\b[^\n]*|(?:export|const|let|var)\b[^\n]*\brequire\s*\([^\n]*|from\s+[^\n]*import\b[^\n]*)\n/;
|
|
12
|
+
/**
|
|
13
|
+
* Tool directives — `eslint-disable`, `@ts-nocheck`, `use strict`, shebangs.
|
|
14
|
+
*
|
|
15
|
+
* They sit above the imports, so leaving one in place stops the import
|
|
16
|
+
* stripper dead and the window fills with dependency names anyway.
|
|
17
|
+
*/
|
|
18
|
+
const LEADING_PRAGMA = /^\s*(?:#![^\n]*|["\']use strict["\'];?|\/\/\s*(?:@ts-|eslint|prettier|global\b|jshint)[^\n]*|\/\*\s*(?:eslint|prettier|global\b|@ts-|jshint)[\s\S]*?\*\/)\n?/;
|
|
19
|
+
/**
|
|
20
|
+
* Strip the opening of a file that is not about the file.
|
|
21
|
+
*
|
|
22
|
+
* Playwright opens all 3,096 of its files with the same 640-character Apache
|
|
23
|
+
* banner, so nearly half an embedding window was byte-identical across the
|
|
24
|
+
* corpus, and the rest was import lines naming other modules — the vector leg
|
|
25
|
+
* was being asked to tell files apart using the text they most have in common.
|
|
26
|
+
*
|
|
27
|
+
* A leading block comment is dropped only when it reads as a licence banner:
|
|
28
|
+
* the first comment in a file is very often the one sentence that best
|
|
29
|
+
* describes it, and discarding those would cost more than the banners do.
|
|
30
|
+
*/
|
|
31
|
+
export function distillBody(body) {
|
|
32
|
+
let text = body;
|
|
33
|
+
const banner = text.match(LICENCE_BANNER)?.[0];
|
|
34
|
+
if (banner && /copyright|licen[sc]e|SPDX-/i.test(banner))
|
|
35
|
+
text = text.slice(banner.length);
|
|
36
|
+
let previous;
|
|
37
|
+
do {
|
|
38
|
+
previous = text;
|
|
39
|
+
text = text.replace(LEADING_PRAGMA, "").replace(LEADING_IMPORTS, "");
|
|
40
|
+
} while (text !== previous);
|
|
41
|
+
return text.trimStart();
|
|
42
|
+
}
|
|
43
|
+
const digest = (value) => createHash("sha256").update(value).digest("hex");
|
|
44
|
+
const quote = (value) => `\`${value}\``;
|
|
45
|
+
/** Maximum bytes of parser-derived vocabulary carried on one file page. */
|
|
46
|
+
export const MAX_LEXICAL_VOCABULARY_CHARS = 1200;
|
|
47
|
+
const identifierWords = (value) => value
|
|
48
|
+
.split(/[^A-Za-z0-9_$]+/)
|
|
49
|
+
.flatMap((part) => part
|
|
50
|
+
.replace(/([a-z0-9])([A-Z])/g, "$1 $2")
|
|
51
|
+
.match(/[A-Za-z0-9_$]+/g) ?? [])
|
|
52
|
+
.filter((word) => word.length >= 3);
|
|
53
|
+
/**
|
|
54
|
+
* Pull only source phrases that a structural parse cannot name.
|
|
55
|
+
*
|
|
56
|
+
* The file body is already searchable, so copying every source token into the
|
|
57
|
+
* sidecar would merely count the same evidence twice. These are deliberately
|
|
58
|
+
* narrow signals instead: user-facing strings, route/flag/error text, and
|
|
59
|
+
* comments or docstrings. They are often the vocabulary a bug report repeats,
|
|
60
|
+
* while the parser facts contain none of it. The sidecar remains bounded and
|
|
61
|
+
* candidate-only, so this cannot change stable retrieval or make a long file
|
|
62
|
+
* win by volume alone.
|
|
63
|
+
*/
|
|
64
|
+
const sourceVocabulary = (source, maxChars) => {
|
|
65
|
+
if (!source || maxChars <= 0)
|
|
66
|
+
return [];
|
|
67
|
+
const phrases = [];
|
|
68
|
+
const add = (value) => {
|
|
69
|
+
const normalized = value.replace(/\\([\\'"`])/g, "$1").replace(/\s+/g, " ").trim();
|
|
70
|
+
// Ignore punctuation-only literals, generated blobs and licence banners.
|
|
71
|
+
if (normalized.length < 4 || normalized.length > 160 || !/[A-Za-z]{3}/.test(normalized))
|
|
72
|
+
return;
|
|
73
|
+
if (/^(?:copyright|spdx-license-identifier|licensed under)\b/i.test(normalized))
|
|
74
|
+
return;
|
|
75
|
+
if (!phrases.some((existing) => existing.toLowerCase() === normalized.toLowerCase()))
|
|
76
|
+
phrases.push(normalized);
|
|
77
|
+
};
|
|
78
|
+
// Triple-quoted Python docstrings need their own pass so the generic quote
|
|
79
|
+
// matcher below cannot split them into three empty strings.
|
|
80
|
+
for (const match of source.matchAll(/("""([\s\S]{4,160}?)"""|'''([\s\S]{4,160}?)''')/g)) {
|
|
81
|
+
add(match[2] ?? match[3] ?? "");
|
|
82
|
+
}
|
|
83
|
+
// Keep short literals such as flag names and error messages, but not long
|
|
84
|
+
// serialized payloads or SQL/template blobs that consume the cap without
|
|
85
|
+
// giving a useful natural-language anchor.
|
|
86
|
+
for (const match of source.matchAll(/"((?:\\.|[^"\\\r\n]){3,160})"|'((?:\\.|[^'\\\r\n]){3,160})'|`((?:\\.|[^`\\\r\n]){3,160})`/g)) {
|
|
87
|
+
add(match[1] ?? match[2] ?? match[3] ?? "");
|
|
88
|
+
}
|
|
89
|
+
// Line and block comments provide the prose that generated structural cards
|
|
90
|
+
// cannot. Strip comment markers, and stop at a reasonable sentence budget.
|
|
91
|
+
for (const match of source.matchAll(/\/\*([\s\S]{4,180}?)\*\/|\/\/([^\r\n]{4,160})|^[ \t]*#[ \t]?([^\r\n]{4,160})/gm)) {
|
|
92
|
+
add(match[1] ?? match[2] ?? match[3] ?? "");
|
|
93
|
+
}
|
|
94
|
+
return phrases;
|
|
95
|
+
};
|
|
96
|
+
/**
|
|
97
|
+
* Render a deterministic, bounded structural vocabulary for lexical search.
|
|
98
|
+
*
|
|
99
|
+
* The source body is already searchable, but BM25 length normalisation makes
|
|
100
|
+
* a long file a poor place to match a name that occurs once. This sidecar is
|
|
101
|
+
* intentionally not a code summary: it contains only parser facts that are
|
|
102
|
+
* useful query anchors, and it is capped so a large file cannot flood the
|
|
103
|
+
* index. The ordering is most-specific first, then references as a fallback.
|
|
104
|
+
*/
|
|
105
|
+
export function lexicalVocabulary(file, maxChars = MAX_LEXICAL_VOCABULARY_CHARS) {
|
|
106
|
+
const entries = [];
|
|
107
|
+
const addEntry = (value) => {
|
|
108
|
+
if (value?.trim())
|
|
109
|
+
entries.push(value.trim());
|
|
110
|
+
};
|
|
111
|
+
addEntry(file.path);
|
|
112
|
+
for (const definition of file.result.definitions ?? []) {
|
|
113
|
+
addEntry(definition.name);
|
|
114
|
+
addEntry(definition.owner ? `${definition.owner}.${definition.name}` : undefined);
|
|
115
|
+
}
|
|
116
|
+
for (const exported of file.result.exports) {
|
|
117
|
+
addEntry(exported.name);
|
|
118
|
+
addEntry(exported.from);
|
|
119
|
+
}
|
|
120
|
+
for (const imported of file.result.imports) {
|
|
121
|
+
addEntry(imported.specifier);
|
|
122
|
+
for (const name of imported.names)
|
|
123
|
+
addEntry(name);
|
|
124
|
+
for (const [local, original] of Object.entries(imported.aliases ?? {})) {
|
|
125
|
+
addEntry(local);
|
|
126
|
+
addEntry(original);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
for (const call of file.result.calls ?? [])
|
|
130
|
+
addEntry(call.callee);
|
|
131
|
+
for (const reference of file.result.references ?? [])
|
|
132
|
+
addEntry(reference.name);
|
|
133
|
+
// Parser facts lead because they are the most discriminative and stable
|
|
134
|
+
// part of the sidecar. Source phrases follow: they recover the wording of
|
|
135
|
+
// issue reports without letting comments or literals crowd out identifiers.
|
|
136
|
+
for (const phrase of sourceVocabulary(file.searchText ?? "", maxChars))
|
|
137
|
+
addEntry(phrase);
|
|
138
|
+
const seen = new Set();
|
|
139
|
+
const words = [];
|
|
140
|
+
let used = 0;
|
|
141
|
+
for (const entry of entries) {
|
|
142
|
+
for (const term of [entry, ...identifierWords(entry)]) {
|
|
143
|
+
const normalized = term.toLowerCase();
|
|
144
|
+
if (term.length < 3 || seen.has(normalized))
|
|
145
|
+
continue;
|
|
146
|
+
const cost = term.length + (words.length ? 1 : 0);
|
|
147
|
+
if (used + cost > Math.max(0, maxChars))
|
|
148
|
+
return words.join(" ");
|
|
149
|
+
seen.add(normalized);
|
|
150
|
+
words.push(term);
|
|
151
|
+
used += cost;
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
return words.join(" ");
|
|
155
|
+
}
|
|
156
|
+
function filePage(file) {
|
|
157
|
+
const imports = file.result.imports.map((item) => item.specifier).sort();
|
|
158
|
+
const exports = file.result.exports.map((item) => `${item.kind} ${item.name}`).sort();
|
|
159
|
+
const facts = [
|
|
160
|
+
`Path: ${quote(file.path)}`,
|
|
161
|
+
`Language: ${file.language ?? "unknown"}`,
|
|
162
|
+
`Parser: ${file.parser ?? "none"}`,
|
|
163
|
+
`Exports: ${exports.length ? exports.join(", ") : "none"}`,
|
|
164
|
+
`Imports: ${imports.length ? imports.join(", ") : "none"}`,
|
|
165
|
+
];
|
|
166
|
+
const summary = `${file.path} is a ${file.language ?? "source"} file with ${exports.length} export${exports.length === 1 ? "" : "s"} and ${imports.length} import${imports.length === 1 ? "" : "s"}.`;
|
|
167
|
+
const vocabulary = lexicalVocabulary(file);
|
|
168
|
+
return {
|
|
169
|
+
page_id: `file:${file.path}`,
|
|
170
|
+
target_path: file.path,
|
|
171
|
+
page_type: "file",
|
|
172
|
+
title: file.path,
|
|
173
|
+
content: facts.join("\n"),
|
|
174
|
+
summary,
|
|
175
|
+
source_hash: file.contentHash,
|
|
176
|
+
...(file.searchText ? { body: file.searchText } : {}),
|
|
177
|
+
...(vocabulary ? { lexical_vocabulary: vocabulary } : {}),
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
function modulePage(path, files) {
|
|
181
|
+
const members = files.map((file) => file.path).sort();
|
|
182
|
+
const source_hash = digest(files.map((file) => `${file.path}\0${file.contentHash}`).sort().join("\n"));
|
|
183
|
+
return {
|
|
184
|
+
page_id: `module:${path}`,
|
|
185
|
+
target_path: path,
|
|
186
|
+
page_type: "module",
|
|
187
|
+
title: path || ".",
|
|
188
|
+
content: [`Module: ${path || "."}`, `Files: ${members.join(", ")}`].join("\n"),
|
|
189
|
+
summary: `${path || "."} contains ${members.length} indexed file${members.length === 1 ? "" : "s"}.`,
|
|
190
|
+
source_hash,
|
|
191
|
+
};
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* Split one oversized file into chunks the embedder can actually read.
|
|
195
|
+
*
|
|
196
|
+
* A sentence embedder truncates hard, so a single page per file meant the
|
|
197
|
+
* vector leg saw the first 1,400 characters and nothing else — 8.7% of this
|
|
198
|
+
* repository's source and 9.3% of Playwright's. Every semantic result so far
|
|
199
|
+
* was produced through that keyhole.
|
|
200
|
+
*
|
|
201
|
+
* Chunks break at definition sites so a symbol is not split down the middle,
|
|
202
|
+
* and are packed to about a full window so no embedding is spent on a
|
|
203
|
+
* three-line local constant. Where definitions run out the split falls back to
|
|
204
|
+
* line boundaries: markdown, JSON and YAML have no symbols at all, and on this
|
|
205
|
+
* repository 97 oversized files are in that state — chunking on symbols alone
|
|
206
|
+
* reached only 41.6% coverage against 91.6% with the fallback.
|
|
207
|
+
*
|
|
208
|
+
* Chunks are `page_type: "chunk"` and carry the *file's* `target_path`, which
|
|
209
|
+
* is what lets the vector leg score spans while lexical, delivery and grading
|
|
210
|
+
* all stay file-level.
|
|
211
|
+
*/
|
|
212
|
+
function chunkPages(file, page) {
|
|
213
|
+
// Off by default, permanently, and the reason is measured.
|
|
214
|
+
//
|
|
215
|
+
// Chunking works as retrieval: +0.0224 recall@10 over 662 paired instances,
|
|
216
|
+
// 95% CI [0.0064, 0.0390], holding under a cluster bootstrap over epochs.
|
|
217
|
+
// It costs more than it is worth.
|
|
218
|
+
//
|
|
219
|
+
// Charging the benchmark for what is actually delivered — spans where a chunk
|
|
220
|
+
// won, files otherwise — the per-instance change in tokens to first gold is
|
|
221
|
+
// **+127%, CI [+76%, +195%]**, above zero on every one of six epochs. The
|
|
222
|
+
// damage is not on chunked files: on the 190 instances that received a span
|
|
223
|
+
// the effect is +31% with a CI spanning zero, while on the 472 that received
|
|
224
|
+
// none it is +166%. Chunking reranks every query but only about one result in
|
|
225
|
+
// three carries a fragment, so most queries pay the ranking cost and collect
|
|
226
|
+
// no saving.
|
|
227
|
+
//
|
|
228
|
+
// A pooled ratio-of-sums says -10.9% on the same data. That statistic is
|
|
229
|
+
// dominated by its largest terms and file sizes here span four orders of
|
|
230
|
+
// magnitude; an agent experiences one query at a time, not a ratio of sums.
|
|
231
|
+
//
|
|
232
|
+
// Span delivery is not what failed. It is built, verified (3,972 chunks, 100%
|
|
233
|
+
// of spans reproduce their chunk exactly) and cost-neutral on its own. What is
|
|
234
|
+
// rejected is the economic case for chunking on top of it. Re-open this only
|
|
235
|
+
// with a measurement that beats the per-instance figure above, not a pooled
|
|
236
|
+
// one.
|
|
237
|
+
if (process.env.KEEL_CHUNK !== "1" && process.env.CHARTER_CHUNK !== "1")
|
|
238
|
+
return [];
|
|
239
|
+
const original = page.body ?? "";
|
|
240
|
+
const body = distillBody(original);
|
|
241
|
+
if (body.length <= EMBEDDED_BODY_CHARS)
|
|
242
|
+
return [];
|
|
243
|
+
// `distillBody` strips the licence banner, pragmas and leading imports, so an
|
|
244
|
+
// offset into it is not an offset into the file. Count what was removed once,
|
|
245
|
+
// and every span below is reported in the file's own line numbers.
|
|
246
|
+
const strippedLines = original.length > body.length
|
|
247
|
+
? original.slice(0, original.length - body.length).split("\n").length - 1
|
|
248
|
+
: 0;
|
|
249
|
+
const lineStarts = [];
|
|
250
|
+
let offset = 0;
|
|
251
|
+
for (const line of body.split("\n")) {
|
|
252
|
+
lineStarts.push(offset);
|
|
253
|
+
offset += line.length + 1;
|
|
254
|
+
}
|
|
255
|
+
const atLine = (line) => lineStarts[Math.min(Math.max(line - 1, 0), lineStarts.length - 1)] ?? 0;
|
|
256
|
+
const declared = [...new Set((file.result.definitions ?? [])
|
|
257
|
+
.map((definition) => definition.line)
|
|
258
|
+
.filter((line) => typeof line === "number")
|
|
259
|
+
.map(atLine))]
|
|
260
|
+
.sort((a, b) => a - b)
|
|
261
|
+
.filter((position) => position > 0 && position < body.length);
|
|
262
|
+
// Fill any run longer than two targets with synthetic line breaks, so a file
|
|
263
|
+
// with no definitions — or one very long function — is still covered.
|
|
264
|
+
const breaks = [];
|
|
265
|
+
let previous = 0;
|
|
266
|
+
for (const boundary of [...declared, body.length]) {
|
|
267
|
+
let cursor = previous;
|
|
268
|
+
while (boundary - cursor > CHUNK_TARGET_CHARS * 2) {
|
|
269
|
+
const newline = body.indexOf("\n", cursor + CHUNK_TARGET_CHARS);
|
|
270
|
+
if (newline < 0 || newline >= boundary)
|
|
271
|
+
break;
|
|
272
|
+
breaks.push(newline + 1);
|
|
273
|
+
cursor = newline + 1;
|
|
274
|
+
}
|
|
275
|
+
if (boundary < body.length)
|
|
276
|
+
breaks.push(boundary);
|
|
277
|
+
previous = boundary;
|
|
278
|
+
}
|
|
279
|
+
const spans = [];
|
|
280
|
+
let start = 0;
|
|
281
|
+
for (const position of [...new Set(breaks)].sort((a, b) => a - b)) {
|
|
282
|
+
if (position - start >= CHUNK_TARGET_CHARS) {
|
|
283
|
+
spans.push([start, position]);
|
|
284
|
+
start = position;
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
spans.push([start, body.length]);
|
|
288
|
+
if (spans.length < 2)
|
|
289
|
+
return [];
|
|
290
|
+
const owners = (file.result.definitions ?? []).filter((definition) => typeof definition.line === "number");
|
|
291
|
+
const imports = file.result.imports.map((item) => item.specifier).sort();
|
|
292
|
+
const exports = file.result.exports.map((item) => item.name).sort();
|
|
293
|
+
return spans.map(([from, to], position) => {
|
|
294
|
+
// The skeleton stays terse on purpose: it is charged against the same
|
|
295
|
+
// window as the code, so a verbose one would crowd out the logic it exists
|
|
296
|
+
// to contextualise.
|
|
297
|
+
const first = owners.find((definition) => atLine(definition.line) >= from && atLine(definition.line) < to);
|
|
298
|
+
const label = first ? `${first.owner ? `${first.owner}.` : ""}${first.name}` : `part ${position + 1}`;
|
|
299
|
+
// Without this a chunk is a semantic orphan: `distillBody` strips the
|
|
300
|
+
// imports before the split, so every chunk after the first lost the module
|
|
301
|
+
// context that tells the embedder what kind of file it is looking at, and
|
|
302
|
+
// chunk zero had it removed on purpose. Bounded hard — the skeleton is
|
|
303
|
+
// charged against the same window as the code it is meant to explain.
|
|
304
|
+
const skeleton = [
|
|
305
|
+
imports.length ? `imports ${imports.slice(0, 8).join(" ")}` : "",
|
|
306
|
+
exports.length ? `exports ${exports.slice(0, 8).join(" ")}` : "",
|
|
307
|
+
].filter(Boolean).join(" ").slice(0, SKELETON_CHARS);
|
|
308
|
+
return {
|
|
309
|
+
page_id: `chunk:${file.path}#${position}`,
|
|
310
|
+
target_path: file.path,
|
|
311
|
+
page_type: "chunk",
|
|
312
|
+
title: file.path,
|
|
313
|
+
content: page.content,
|
|
314
|
+
summary: `${file.path} — ${label}${skeleton ? ` | ${skeleton}` : ""}`,
|
|
315
|
+
body: body.slice(from, to),
|
|
316
|
+
span: {
|
|
317
|
+
start_line: strippedLines + body.slice(0, from).split("\n").length,
|
|
318
|
+
// A chunk boundary sits just after a newline, so slicing to it leaves a
|
|
319
|
+
// trailing empty element that counts as one line too many.
|
|
320
|
+
end_line: strippedLines + body.slice(0, to).replace(/\n$/, "").split("\n").length,
|
|
321
|
+
},
|
|
322
|
+
source_hash: digest(`${file.contentHash}\0${position}`),
|
|
323
|
+
};
|
|
324
|
+
});
|
|
325
|
+
}
|
|
326
|
+
/** Render stable retrieval pages from structure only; no provider or network is used. */
|
|
327
|
+
export function generatePages(index) {
|
|
328
|
+
const pages = index.files.map(filePage);
|
|
329
|
+
const byPathForChunks = new Map(index.files.map((file, position) => [file, pages[position]]));
|
|
330
|
+
for (const [file, page] of byPathForChunks)
|
|
331
|
+
pages.push(...chunkPages(file, page));
|
|
332
|
+
const groups = new Map();
|
|
333
|
+
for (const file of index.files) {
|
|
334
|
+
const parts = file.path.split("/");
|
|
335
|
+
parts.pop();
|
|
336
|
+
const path = parts.join("/");
|
|
337
|
+
const group = groups.get(path) ?? [];
|
|
338
|
+
group.push(file);
|
|
339
|
+
groups.set(path, group);
|
|
340
|
+
}
|
|
341
|
+
for (const [path, files] of groups)
|
|
342
|
+
pages.push(modulePage(path, files));
|
|
343
|
+
return pages.sort((a, b) => byCodeUnit(a.page_id, b.page_id));
|
|
344
|
+
}
|
|
345
|
+
/** Generate pages and attach stylesheet/token/route relationships to their source pages. */
|
|
346
|
+
export function generatePagesWithSurfaces(index, sources) {
|
|
347
|
+
const pages = generatePages(index);
|
|
348
|
+
const byPath = new Map(pages.filter((page) => page.page_type === "file").map((page) => [page.target_path, page]));
|
|
349
|
+
for (const edge of discoverSurfaceEdges(sources)) {
|
|
350
|
+
const page = byPath.get(edge.from);
|
|
351
|
+
if (!page)
|
|
352
|
+
continue;
|
|
353
|
+
page.content += `\n${edge.kind}: ${edge.to}`;
|
|
354
|
+
page.summary += ` Related ${edge.kind}: ${edge.to}.`;
|
|
355
|
+
}
|
|
356
|
+
return pages;
|
|
357
|
+
}
|
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Durable, local storage for the derived index.
|
|
3
|
+
*
|
|
4
|
+
* This database is deliberately a client-side cache. It stores parser output
|
|
5
|
+
* and index identity beside the checkout; no server code needs to understand
|
|
6
|
+
* or receive the contents. A single snapshot is enough for the first tier:
|
|
7
|
+
* the incremental updater can load it, verify hashes, and reparse only the
|
|
8
|
+
* files that changed.
|
|
9
|
+
*/
|
|
10
|
+
import { DatabaseSync } from "node:sqlite";
|
|
11
|
+
// 2: index_files.search_text — the file's own text, which retrieval needs
|
|
12
|
+
// to match code and comments. A v1 database has no column for it, and
|
|
13
|
+
// restoring one would produce an index that silently searches only
|
|
14
|
+
// structural cards, so the bump forces a rebuild rather than a quiet
|
|
15
|
+
// downgrade.
|
|
16
|
+
const SCHEMA_VERSION = 2;
|
|
17
|
+
/** A local SQLite archive containing the latest FileIndex snapshot. */
|
|
18
|
+
export class IndexSnapshotArchive {
|
|
19
|
+
database;
|
|
20
|
+
constructor(databasePath) {
|
|
21
|
+
this.database = new DatabaseSync(databasePath);
|
|
22
|
+
this.database.exec(`
|
|
23
|
+
PRAGMA foreign_keys = ON;
|
|
24
|
+
-- Negative value is KB, not pages: bounds this client-side cache to
|
|
25
|
+
-- ~2MB so it can't grow unbounded per agent process
|
|
26
|
+
-- (tsk_01M1MQ01E1D7JSEHFVE2RCK8DJ).
|
|
27
|
+
PRAGMA cache_size = -2000;
|
|
28
|
+
CREATE TABLE IF NOT EXISTS index_meta (
|
|
29
|
+
key TEXT PRIMARY KEY,
|
|
30
|
+
value TEXT NOT NULL
|
|
31
|
+
);
|
|
32
|
+
CREATE TABLE IF NOT EXISTS index_files (
|
|
33
|
+
path TEXT PRIMARY KEY,
|
|
34
|
+
content_hash TEXT NOT NULL,
|
|
35
|
+
language TEXT,
|
|
36
|
+
parser TEXT,
|
|
37
|
+
result_json TEXT NOT NULL,
|
|
38
|
+
search_text TEXT
|
|
39
|
+
);
|
|
40
|
+
CREATE TABLE IF NOT EXISTS index_blind_spots (
|
|
41
|
+
path TEXT PRIMARY KEY,
|
|
42
|
+
reason TEXT NOT NULL
|
|
43
|
+
);
|
|
44
|
+
CREATE TABLE IF NOT EXISTS index_vectors (
|
|
45
|
+
cache_key TEXT PRIMARY KEY,
|
|
46
|
+
vector_json TEXT NOT NULL
|
|
47
|
+
);
|
|
48
|
+
-- Per-file coverage facts, stored so "which files in this repo are blind
|
|
49
|
+
-- spots" is answerable without running an impact query against each one.
|
|
50
|
+
-- Written separately from putIndex because coverage is derived from the
|
|
51
|
+
-- graph and the call resolution, which the index alone does not carry and
|
|
52
|
+
-- putIndex's caller does not have.
|
|
53
|
+
--
|
|
54
|
+
-- Keyed by the index fingerprint: facts describe one specific index, and
|
|
55
|
+
-- serving them beside a different one would be a confidently stale answer,
|
|
56
|
+
-- which is the failure the whole honesty contract exists to avoid.
|
|
57
|
+
CREATE TABLE IF NOT EXISTS index_coverage (
|
|
58
|
+
path TEXT PRIMARY KEY,
|
|
59
|
+
fingerprint TEXT NOT NULL,
|
|
60
|
+
fact_json TEXT NOT NULL
|
|
61
|
+
);
|
|
62
|
+
`);
|
|
63
|
+
}
|
|
64
|
+
/** Replace the current snapshot in one transaction. */
|
|
65
|
+
putIndex(index, metadata = { indexedCommit: null, indexedAt: null }) {
|
|
66
|
+
this.database.exec("BEGIN IMMEDIATE");
|
|
67
|
+
try {
|
|
68
|
+
this.database.exec("DELETE FROM index_meta; DELETE FROM index_files; DELETE FROM index_blind_spots;");
|
|
69
|
+
const putMeta = this.database.prepare("INSERT INTO index_meta (key, value) VALUES (?, ?)");
|
|
70
|
+
putMeta.run("schema_version", String(SCHEMA_VERSION));
|
|
71
|
+
putMeta.run("root", index.root);
|
|
72
|
+
putMeta.run("fingerprint", index.fingerprint);
|
|
73
|
+
putMeta.run("coverage", JSON.stringify(index.coverage));
|
|
74
|
+
putMeta.run("aliases", JSON.stringify(index.aliases ?? null));
|
|
75
|
+
putMeta.run("indexed_commit", metadata.indexedCommit ?? "");
|
|
76
|
+
putMeta.run("indexed_at", metadata.indexedAt ?? "");
|
|
77
|
+
const putFile = this.database.prepare("INSERT INTO index_files (path, content_hash, language, parser, result_json, search_text) VALUES (?, ?, ?, ?, ?, ?)");
|
|
78
|
+
for (const file of index.files) {
|
|
79
|
+
putFile.run(file.path, file.contentHash, file.language, file.parser, JSON.stringify(file.result), file.searchText ?? null);
|
|
80
|
+
}
|
|
81
|
+
const putBlindSpot = this.database.prepare("INSERT INTO index_blind_spots (path, reason) VALUES (?, ?)");
|
|
82
|
+
for (const blindSpot of index.blindSpots)
|
|
83
|
+
putBlindSpot.run(blindSpot.path, blindSpot.reason);
|
|
84
|
+
this.database.exec("COMMIT");
|
|
85
|
+
}
|
|
86
|
+
catch (error) {
|
|
87
|
+
this.database.exec("ROLLBACK");
|
|
88
|
+
throw error;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
/** Read the commit and timestamp associated with the current snapshot. */
|
|
92
|
+
getMetadata() {
|
|
93
|
+
const commit = this.meta("indexed_commit");
|
|
94
|
+
const indexedAt = this.meta("indexed_at");
|
|
95
|
+
return {
|
|
96
|
+
indexedCommit: commit || null,
|
|
97
|
+
indexedAt: indexedAt || null,
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
/** Read the latest snapshot, or null when this checkout has not been indexed. */
|
|
101
|
+
getIndex() {
|
|
102
|
+
const schema = this.meta("schema_version");
|
|
103
|
+
if (schema === null)
|
|
104
|
+
return null;
|
|
105
|
+
if (schema !== String(SCHEMA_VERSION)) {
|
|
106
|
+
throw new Error(`unsupported local index schema ${JSON.stringify(schema)}`);
|
|
107
|
+
}
|
|
108
|
+
const root = this.requiredMeta("root");
|
|
109
|
+
const fingerprint = this.requiredMeta("fingerprint");
|
|
110
|
+
const coverage = parseJson(this.requiredMeta("coverage"), "coverage");
|
|
111
|
+
const aliases = parseJson(this.requiredMeta("aliases"), "aliases");
|
|
112
|
+
const files = this.database
|
|
113
|
+
.prepare("SELECT path, content_hash, language, parser, result_json, search_text FROM index_files ORDER BY path")
|
|
114
|
+
.all()
|
|
115
|
+
.map((row) => ({
|
|
116
|
+
path: textColumn(row.path, "path"),
|
|
117
|
+
contentHash: textColumn(row.content_hash, "content_hash"),
|
|
118
|
+
language: nullableTextColumn(row.language, "language"),
|
|
119
|
+
parser: nullableTextColumn(row.parser, "parser"),
|
|
120
|
+
result: parseJson(textColumn(row.result_json, "result_json"), "file result"),
|
|
121
|
+
...(row.search_text === null || row.search_text === undefined
|
|
122
|
+
? {}
|
|
123
|
+
: { searchText: textColumn(row.search_text, "search_text") }),
|
|
124
|
+
}));
|
|
125
|
+
const blindSpots = this.database
|
|
126
|
+
.prepare("SELECT path, reason FROM index_blind_spots ORDER BY path")
|
|
127
|
+
.all()
|
|
128
|
+
.map((row) => ({
|
|
129
|
+
path: textColumn(row.path, "blind spot path"),
|
|
130
|
+
reason: textColumn(row.reason, "blind spot reason"),
|
|
131
|
+
}));
|
|
132
|
+
return { root, fingerprint, aliases: aliases ?? undefined, files, coverage, blindSpots };
|
|
133
|
+
}
|
|
134
|
+
/** Read a cached document vector, or null when it has not been embedded locally. */
|
|
135
|
+
getVector(cacheKey) {
|
|
136
|
+
const row = this.database
|
|
137
|
+
.prepare("SELECT vector_json FROM index_vectors WHERE cache_key = ?")
|
|
138
|
+
.get(cacheKey);
|
|
139
|
+
if (!row)
|
|
140
|
+
return null;
|
|
141
|
+
return vectorJson(textColumn(row.vector_json, "vector_json"));
|
|
142
|
+
}
|
|
143
|
+
/** Persist or replace a cached document vector independently of the index snapshot. */
|
|
144
|
+
putVector(cacheKey, vector) {
|
|
145
|
+
validateVector(vector);
|
|
146
|
+
this.database
|
|
147
|
+
.prepare("INSERT INTO index_vectors (cache_key, vector_json) VALUES (?, ?) " +
|
|
148
|
+
"ON CONFLICT(cache_key) DO UPDATE SET vector_json = excluded.vector_json")
|
|
149
|
+
.run(cacheKey, JSON.stringify(vector));
|
|
150
|
+
}
|
|
151
|
+
/** Close the local database. Repeated shutdown is harmless. */
|
|
152
|
+
/**
|
|
153
|
+
* Replace the stored coverage facts.
|
|
154
|
+
*
|
|
155
|
+
* Stamped with the fingerprint of the index they describe. `getCoverage`
|
|
156
|
+
* refuses to return facts stamped with any other, because coverage attached
|
|
157
|
+
* to the wrong index is worse than no coverage: it would report a file as
|
|
158
|
+
* clean on the strength of an analysis of different bytes.
|
|
159
|
+
*/
|
|
160
|
+
putCoverage(fingerprint, facts) {
|
|
161
|
+
this.database.exec("BEGIN IMMEDIATE");
|
|
162
|
+
try {
|
|
163
|
+
this.database.exec("DELETE FROM index_coverage");
|
|
164
|
+
const put = this.database.prepare("INSERT INTO index_coverage (path, fingerprint, fact_json) VALUES (?, ?, ?)");
|
|
165
|
+
// Sorted, so the stored bytes are a function of the facts and not of Map
|
|
166
|
+
// iteration order — the same determinism rule the rest of the index keeps.
|
|
167
|
+
for (const path of [...facts.keys()].sort()) {
|
|
168
|
+
put.run(path, fingerprint, JSON.stringify(facts.get(path)));
|
|
169
|
+
}
|
|
170
|
+
this.database.exec("COMMIT");
|
|
171
|
+
}
|
|
172
|
+
catch (error) {
|
|
173
|
+
this.database.exec("ROLLBACK");
|
|
174
|
+
throw error;
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
/** Stored facts for this fingerprint, or null when there are none for it. */
|
|
178
|
+
getCoverage(fingerprint) {
|
|
179
|
+
const rows = this.database
|
|
180
|
+
.prepare("SELECT path, fact_json FROM index_coverage WHERE fingerprint = ? ORDER BY path")
|
|
181
|
+
.all(fingerprint);
|
|
182
|
+
if (rows.length === 0)
|
|
183
|
+
return null;
|
|
184
|
+
const facts = new Map();
|
|
185
|
+
for (const row of rows)
|
|
186
|
+
facts.set(row.path, JSON.parse(row.fact_json));
|
|
187
|
+
return facts;
|
|
188
|
+
}
|
|
189
|
+
shutdown() {
|
|
190
|
+
if (this.database.isOpen)
|
|
191
|
+
this.database.close();
|
|
192
|
+
}
|
|
193
|
+
meta(key) {
|
|
194
|
+
const row = this.database.prepare("SELECT value FROM index_meta WHERE key = ?").get(key);
|
|
195
|
+
return row?.value === null || row?.value === undefined ? null : textColumn(row.value, key);
|
|
196
|
+
}
|
|
197
|
+
requiredMeta(key) {
|
|
198
|
+
const value = this.meta(key);
|
|
199
|
+
if (value === null)
|
|
200
|
+
throw new Error(`local index is missing metadata ${JSON.stringify(key)}`);
|
|
201
|
+
return value;
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
function textColumn(value, name) {
|
|
205
|
+
if (typeof value !== "string")
|
|
206
|
+
throw new Error(`local index column ${name} is not text`);
|
|
207
|
+
return value;
|
|
208
|
+
}
|
|
209
|
+
function nullableTextColumn(value, name) {
|
|
210
|
+
if (value === null || value === undefined)
|
|
211
|
+
return null;
|
|
212
|
+
return textColumn(value, name);
|
|
213
|
+
}
|
|
214
|
+
function parseJson(value, name) {
|
|
215
|
+
try {
|
|
216
|
+
return JSON.parse(value);
|
|
217
|
+
}
|
|
218
|
+
catch (error) {
|
|
219
|
+
throw new Error(`local index ${name} is not valid JSON`, { cause: error });
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
function vectorJson(value) {
|
|
223
|
+
const parsed = parseJson(value, "vector");
|
|
224
|
+
if (!Array.isArray(parsed) || parsed.some((component) => typeof component !== "number" || !Number.isFinite(component))) {
|
|
225
|
+
throw new Error("local index vector must be an array of finite numbers");
|
|
226
|
+
}
|
|
227
|
+
return parsed;
|
|
228
|
+
}
|
|
229
|
+
function validateVector(vector) {
|
|
230
|
+
if (vector.some((component) => !Number.isFinite(component))) {
|
|
231
|
+
throw new Error("local index vector must contain only finite numbers");
|
|
232
|
+
}
|
|
233
|
+
}
|