simt-dev 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +172 -0
- package/README.md +138 -74
- package/docs/decisions/ADR-073-a-kernel-is-a-smaller-projection-the-repository-asks-for.md +1 -1
- package/docs/decisions/ADR-077-the-command-card-leads-the-region-and-reads-committed-evidence.md +1 -1
- package/docs/decisions/ADR-083-a-budget-is-met-by-cutting-the-kernel-and-hard-is-the-floor.md +153 -0
- package/docs/decisions/ADR-084-a-workspace-carries-its-repeated-rules-once.md +100 -0
- package/docs/decisions/ADR-085-the-region-ends-with-the-names-it-kept-out.md +72 -0
- package/docs/decisions/ADR-086-the-stable-prefix-comes-first-and-the-lock-records-both-digests.md +69 -0
- package/docs/decisions/ADR-087-a-run-is-evidence-for-one-tree.md +67 -0
- package/docs/decisions/ADR-088-a-marked-line-is-enforced-by-a-checker.md +59 -0
- package/docs/decisions/ADR-089-the-local-index-is-a-cache-of-git-blobs.md +52 -0
- package/docs/decisions/ADR-090-an-optional-adapter-is-off-until-named.md +55 -0
- package/docs/decisions/ADR-091-the-map-ranks-files-a-task-mentions.md +60 -0
- package/docs/decisions/ADR-092-tests-are-imports-conventions-and-coverage.md +60 -0
- package/docs/decisions/ADR-093-why-names-what-governs-a-file.md +58 -0
- package/docs/decisions/ADR-094-a-change-runs-the-tests-that-import-it.md +44 -0
- package/docs/decisions/ADR-095-gate-output-keeps-the-failures.md +31 -0
- package/docs/decisions/ADR-096-impact-names-who-a-change-reaches.md +38 -0
- package/package.json +2 -2
- package/rules/core.md +2 -2
- package/src/agenthooks/policy.mjs +10 -2
- package/src/agenthooks/prompt.mjs +81 -0
- package/src/agenthooks/render.mjs +138 -17
- package/src/argv.mjs +34 -1
- package/src/bench/grade.mjs +3 -0
- package/src/bench/run.mjs +85 -39
- package/src/bench/variants.mjs +456 -0
- package/src/brief/assemble.mjs +346 -0
- package/src/brief/kind.mjs +134 -0
- package/src/brief/packet.mjs +197 -0
- package/src/ci/workflow.mjs +11 -1
- package/src/commands/agenthook.mjs +84 -5
- package/src/commands/bench.mjs +53 -16
- package/src/commands/brief.mjs +78 -0
- package/src/commands/compare.mjs +10 -2
- package/src/commands/decide.mjs +116 -0
- package/src/commands/evidence.mjs +96 -0
- package/src/commands/gate.mjs +165 -0
- package/src/commands/help.mjs +65 -3
- package/src/commands/hooks.mjs +45 -5
- package/src/commands/impact.mjs +205 -0
- package/src/commands/index.mjs +52 -1
- package/src/commands/lessons.mjs +127 -0
- package/src/commands/local.mjs +97 -0
- package/src/commands/map.mjs +126 -0
- package/src/commands/repoindex.mjs +55 -0
- package/src/commands/rules.mjs +17 -2
- package/src/commands/shared.mjs +398 -0
- package/src/commands/skills.mjs +3 -17
- package/src/commands/skillscompile.mjs +255 -0
- package/src/commands/sync.mjs +87 -3
- package/src/commands/tests.mjs +145 -0
- package/src/commands/tokens.mjs +30 -1
- package/src/commands/why.mjs +172 -0
- package/src/compare/learn.mjs +680 -0
- package/src/decisions/index.mjs +116 -65
- package/src/federation/client.mjs +188 -0
- package/src/federation/config.mjs +134 -0
- package/src/federation/query.mjs +48 -0
- package/src/graph/compile.mjs +475 -0
- package/src/graph/spec.mjs +1 -1
- package/src/hooks/dispatcher.mjs +15 -0
- package/src/hooks/shim.mjs +31 -0
- package/src/index/compact.mjs +228 -0
- package/src/index/extract.mjs +331 -0
- package/src/index/gate.mjs +89 -0
- package/src/index/govern.mjs +234 -0
- package/src/index/impact.mjs +186 -0
- package/src/index/map.mjs +627 -0
- package/src/index/persist.mjs +239 -0
- package/src/index/store.mjs +515 -0
- package/src/index/tests.mjs +294 -0
- package/src/kernel/budget.mjs +100 -0
- package/src/kernel/checks.mjs +378 -0
- package/src/kernel/variants.mjs +353 -0
- package/src/ledger/index.mjs +147 -0
- package/src/local/client.mjs +482 -0
- package/src/local/config.mjs +234 -0
- package/src/mcp/tools.mjs +138 -1
- package/src/measure/context.mjs +7 -5
- package/src/memory/decide.mjs +462 -0
- package/src/memory/lessons.mjs +858 -0
- package/src/projection/disclosure.mjs +92 -0
- package/src/projection/lock.mjs +125 -0
- package/src/projection/manifest.mjs +52 -0
- package/src/projection/markers.mjs +56 -17
- package/src/projection/project.mjs +24 -17
- package/src/projection/rules.mjs +407 -49
- package/src/projection/shared.mjs +428 -0
- package/src/projection/targets.mjs +2 -1
- package/src/toolwords.mjs +8 -0
|
@@ -0,0 +1,627 @@
|
|
|
1
|
+
// Ranked repo map and hybrid retrieval (R22, R28, ADR-091).
|
|
2
|
+
//
|
|
3
|
+
// PageRank over import and call edges, personalised to the task, fused with
|
|
4
|
+
// FTS5 BM25 (`bm25()`) and resolver signals via reciprocal-rank fusion (RRF).
|
|
5
|
+
// Optional local vector embeddings (stored as SQLite blobs via R37) are scored
|
|
6
|
+
// by brute-force cosine similarity and stay off by default unless they raise
|
|
7
|
+
// gold-file recall@10 by at least 10 percentage points (+0.10).
|
|
8
|
+
|
|
9
|
+
import { readdir, readFile } from 'node:fs/promises';
|
|
10
|
+
import { join } from 'node:path';
|
|
11
|
+
|
|
12
|
+
import { isTestPath, resolveVaultDir } from '../bench/mine.mjs';
|
|
13
|
+
import { readLocalConfig } from '../local/config.mjs';
|
|
14
|
+
import { localEmbed } from '../local/client.mjs';
|
|
15
|
+
import {
|
|
16
|
+
decodeVectorBlob,
|
|
17
|
+
readEdges,
|
|
18
|
+
readEmbeddings,
|
|
19
|
+
readFiles,
|
|
20
|
+
readFts,
|
|
21
|
+
readSymbols,
|
|
22
|
+
scoreFtsBm25,
|
|
23
|
+
searchFtsBm25,
|
|
24
|
+
} from './store.mjs';
|
|
25
|
+
|
|
26
|
+
export const ITERATIONS = 20;
|
|
27
|
+
export const DAMPING = 0.85;
|
|
28
|
+
export const RRF_K = 40;
|
|
29
|
+
export const EMBED_RECALL_GAIN_BAR = 0.1;
|
|
30
|
+
const TOP = 10;
|
|
31
|
+
export const DEFAULT_BUDGET = 2048;
|
|
32
|
+
|
|
33
|
+
const STOP = new Set([
|
|
34
|
+
'this', 'that', 'with', 'from', 'have', 'been', 'into', 'only', 'when', 'what',
|
|
35
|
+
'which', 'their', 'about', 'after', 'before', 'where', 'there', 'these', 'those',
|
|
36
|
+
'would', 'could', 'should', 'every', 'other', 'than', 'then', 'them', 'they',
|
|
37
|
+
'your', 'will', 'does', 'file', 'files', 'test', 'tests', 'line', 'lines', 'code',
|
|
38
|
+
'also', 'just', 'more', 'some', 'such', 'each', 'same', 'over', 'used', 'using',
|
|
39
|
+
]);
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Reciprocal-rank fusion (RRF) over multiple ranked lists.
|
|
43
|
+
*
|
|
44
|
+
* Each list entry may be a path string or `{ path }`. Ties break by path ascending
|
|
45
|
+
* so fusion is strictly deterministic.
|
|
46
|
+
*
|
|
47
|
+
* @param {{items: ({path: string}|string)[], weight?: number}[]} lists
|
|
48
|
+
* @param {number} [k]
|
|
49
|
+
* @returns {{path: string, rank: number}[]}
|
|
50
|
+
*/
|
|
51
|
+
export function rrfFuse(lists, k = RRF_K) {
|
|
52
|
+
const fused = new Map();
|
|
53
|
+
const bestScore = new Map();
|
|
54
|
+
for (const { items = [], weight = 1 } of lists) {
|
|
55
|
+
if (!Array.isArray(items) || weight <= 0) continue;
|
|
56
|
+
for (let i = 0; i < items.length; i += 1) {
|
|
57
|
+
const entry = items[i];
|
|
58
|
+
const path = typeof entry === 'string' ? entry : entry?.path;
|
|
59
|
+
if (!path) continue;
|
|
60
|
+
fused.set(path, (fused.get(path) ?? 0) + weight / (k + i + 1));
|
|
61
|
+
if (typeof entry === 'object' && typeof entry?.score === 'number') {
|
|
62
|
+
bestScore.set(path, Math.max(bestScore.get(path) ?? 0, entry.score));
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
return [...fused.entries()]
|
|
67
|
+
.map(([path, rank]) => ({ path, rank, score: bestScore.get(path) ?? 0 }))
|
|
68
|
+
.sort((a, b) => (a.rank !== b.rank ? (a.rank < b.rank ? 1 : -1) : cmp(a.path, b.path)));
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Brute-force cosine similarity between two numeric / Float32Array vectors.
|
|
73
|
+
*/
|
|
74
|
+
export function cosineSimilarity(a, b) {
|
|
75
|
+
if (!a || !b) return 0;
|
|
76
|
+
const len = Math.min(a.length ?? 0, b.length ?? 0);
|
|
77
|
+
if (len === 0) return 0;
|
|
78
|
+
let dot = 0;
|
|
79
|
+
let normA = 0;
|
|
80
|
+
let normB = 0;
|
|
81
|
+
for (let i = 0; i < len; i += 1) {
|
|
82
|
+
const va = Number(a[i]) || 0;
|
|
83
|
+
const vb = Number(b[i]) || 0;
|
|
84
|
+
dot += va * vb;
|
|
85
|
+
normA += va * va;
|
|
86
|
+
normB += vb * vb;
|
|
87
|
+
}
|
|
88
|
+
const denom = Math.sqrt(normA) * Math.sqrt(normB);
|
|
89
|
+
if (denom === 0) return 0;
|
|
90
|
+
return dot / denom;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Rank stored vector embeddings against a query vector by brute-force cosine similarity.
|
|
95
|
+
*
|
|
96
|
+
* @param {{path: string, vec: Float32Array|number[]|Buffer}[]} embeddings
|
|
97
|
+
* @param {Float32Array|number[]|Buffer} queryVector
|
|
98
|
+
* @returns {{path: string, score: number}[]}
|
|
99
|
+
*/
|
|
100
|
+
export function rankByCosine(embeddings, queryVector) {
|
|
101
|
+
const qVec = queryVector instanceof Float32Array ? queryVector : decodeVectorBlob(queryVector);
|
|
102
|
+
if (!Array.isArray(embeddings) || embeddings.length === 0 || qVec.length === 0) return [];
|
|
103
|
+
const scored = [];
|
|
104
|
+
for (const row of embeddings) {
|
|
105
|
+
if (!row?.path) continue;
|
|
106
|
+
const vec = row.vec instanceof Float32Array ? row.vec : decodeVectorBlob(row.vec);
|
|
107
|
+
if (vec.length === 0) continue;
|
|
108
|
+
const sim = cosineSimilarity(vec, qVec);
|
|
109
|
+
if (sim > 0) scored.push({ path: row.path, score: sim });
|
|
110
|
+
}
|
|
111
|
+
return scored.sort((a, b) => (a.score !== b.score ? (a.score < b.score ? 1 : -1) : cmp(a.path, b.path)));
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Rank files for one task. No task text ranks the whole graph. A task ranks the
|
|
116
|
+
* files it mentions and the files one import or call away from those, and when
|
|
117
|
+
* FTS / BM25 rows or optional vector embeddings are supplied, fuses BM25 +
|
|
118
|
+
* graph proximity + resolver signals via reciprocal-rank fusion (R28).
|
|
119
|
+
*
|
|
120
|
+
* @param {{
|
|
121
|
+
* files: {path: string}[],
|
|
122
|
+
* edges: {src: string, dst: string, kind: string, weight?: number}[],
|
|
123
|
+
* symbols: {path: string, name: string}[],
|
|
124
|
+
* fts?: {kind?: string, ref?: string, path?: string, body?: string}[],
|
|
125
|
+
* bm25?: {path: string, score?: number}[],
|
|
126
|
+
* embeddings?: {path: string, vec: Float32Array|number[]|Buffer}[],
|
|
127
|
+
* queryVector?: Float32Array|number[]|Buffer|null,
|
|
128
|
+
* task?: string
|
|
129
|
+
* }} input
|
|
130
|
+
* @returns {{path: string, rank: number, score: number}[]}
|
|
131
|
+
*/
|
|
132
|
+
export function rankFiles({
|
|
133
|
+
files,
|
|
134
|
+
edges,
|
|
135
|
+
symbols,
|
|
136
|
+
fts = undefined,
|
|
137
|
+
bm25 = undefined,
|
|
138
|
+
embeddings = undefined,
|
|
139
|
+
queryVector = null,
|
|
140
|
+
task = '',
|
|
141
|
+
}) {
|
|
142
|
+
const nodes = [...new Set(files.map((file) => file.path))].sort();
|
|
143
|
+
const symbolsByPath = groupSymbols(symbols);
|
|
144
|
+
const scores = scoreFiles(nodes, symbolsByPath, task);
|
|
145
|
+
const kept = closure(nodes, edges, scores);
|
|
146
|
+
const index = new Map(kept.map((path, i) => [path, i]));
|
|
147
|
+
const links = buildLinks(kept, edges, index);
|
|
148
|
+
const rank = pageRank(kept, links, scores);
|
|
149
|
+
// A file the task mentions stays ahead of a file it does not. PageRank on
|
|
150
|
+
// this graph otherwise returns the file every module imports, and the files
|
|
151
|
+
// the task named never reach the top 10.
|
|
152
|
+
const baseRanked = kept
|
|
153
|
+
.map((path, i) => ({ path, rank: rank[i], score: scores.get(path) ?? 0 }))
|
|
154
|
+
.sort(byRank);
|
|
155
|
+
|
|
156
|
+
if (String(task ?? '').trim().length === 0) return baseRanked;
|
|
157
|
+
|
|
158
|
+
const knownPaths = new Set(nodes);
|
|
159
|
+
const rawBm25 = Array.isArray(bm25)
|
|
160
|
+
? bm25
|
|
161
|
+
: Array.isArray(fts) && fts.length > 0
|
|
162
|
+
? scoreFtsBm25(fts, task)
|
|
163
|
+
: [];
|
|
164
|
+
const bm25Ranked = rawBm25
|
|
165
|
+
.filter((row) => knownPaths.has(row.path) && !isTestPath(row.path))
|
|
166
|
+
.slice(0, 30);
|
|
167
|
+
|
|
168
|
+
const embedRanked =
|
|
169
|
+
queryVector && Array.isArray(embeddings) && embeddings.length > 0
|
|
170
|
+
? rankByCosine(embeddings, queryVector)
|
|
171
|
+
.filter((row) => knownPaths.has(row.path) && !isTestPath(row.path))
|
|
172
|
+
.slice(0, 30)
|
|
173
|
+
: [];
|
|
174
|
+
|
|
175
|
+
if (bm25Ranked.length === 0 && embedRanked.length === 0) {
|
|
176
|
+
return baseRanked;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
const resolverRanked = baseRanked.filter((row) => (row.score ?? 0) > 0);
|
|
180
|
+
const graphRanked = [...baseRanked].sort((a, b) => (a.rank !== b.rank ? (a.rank < b.rank ? 1 : -1) : cmp(a.path, b.path)));
|
|
181
|
+
const lists = [
|
|
182
|
+
{ items: resolverRanked, weight: 2.0 },
|
|
183
|
+
{ items: graphRanked, weight: 1.0 },
|
|
184
|
+
{ items: bm25Ranked, weight: 1.0 },
|
|
185
|
+
];
|
|
186
|
+
if (embedRanked.length > 0) {
|
|
187
|
+
lists.push({ items: embedRanked, weight: 1.5 });
|
|
188
|
+
}
|
|
189
|
+
return rrfFuse(lists, RRF_K);
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/**
|
|
193
|
+
* Render the ranking within a byte budget.
|
|
194
|
+
* `outline` is paths, `names` adds exported symbols, `signatures` adds the line.
|
|
195
|
+
* A budget of 0 does not truncate. The first file is kept even when it alone
|
|
196
|
+
* exceeds the budget, so the map is never empty while the ranking is not.
|
|
197
|
+
*
|
|
198
|
+
* @param {{path: string, rank: number}[]} ranked
|
|
199
|
+
* @param {{path: string, name: string, sig?: string, exported?: number|boolean}[]} symbols
|
|
200
|
+
* @param {{level?: string, budget?: number}} [opts]
|
|
201
|
+
*/
|
|
202
|
+
export function renderMap(ranked, symbols, { level = 'outline', budget = DEFAULT_BUDGET } = {}) {
|
|
203
|
+
const byPath = groupSymbols(symbols);
|
|
204
|
+
const blocks = [];
|
|
205
|
+
let used = 0;
|
|
206
|
+
for (const row of ranked) {
|
|
207
|
+
const block = renderEntry(row.path, byPath.get(row.path) ?? [], level);
|
|
208
|
+
const gap = blocks.length === 0 ? 0 : 1;
|
|
209
|
+
const bytes = Buffer.byteLength(block);
|
|
210
|
+
if (blocks.length > 0 && budget > 0 && used + gap + bytes > budget) break;
|
|
211
|
+
blocks.push(block);
|
|
212
|
+
used += gap + bytes;
|
|
213
|
+
if (budget > 0 && used >= budget) break;
|
|
214
|
+
}
|
|
215
|
+
return {
|
|
216
|
+
text: blocks.join('\n'),
|
|
217
|
+
shown: blocks.length,
|
|
218
|
+
total: ranked.length,
|
|
219
|
+
truncated: blocks.length < ranked.length,
|
|
220
|
+
files: ranked.slice(0, blocks.length).map((row) => ({ path: row.path, rank: round6(row.rank) })),
|
|
221
|
+
};
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/** Fraction of gold paths that sit in the top 10. Null when there is nothing to recall. */
|
|
225
|
+
export function recallAt10(ranked, gold) {
|
|
226
|
+
const relevant = [...new Set(gold)];
|
|
227
|
+
if (relevant.length === 0) return null;
|
|
228
|
+
const top = new Set(ranked.slice(0, TOP).map((row) => row.path));
|
|
229
|
+
let hit = 0;
|
|
230
|
+
for (const path of relevant) if (top.has(path)) hit += 1;
|
|
231
|
+
return hit / relevant.length;
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
/** Paths a unified diff changes. The destination side is the file after the patch. */
|
|
235
|
+
export function goldPaths(patch) {
|
|
236
|
+
const out = [];
|
|
237
|
+
for (const line of String(patch).split('\n')) {
|
|
238
|
+
const match = line.match(/^diff --git a\/(.+) b\/(.+)$/);
|
|
239
|
+
if (!match || match[2] === '/dev/null') continue;
|
|
240
|
+
out.push(match[2]);
|
|
241
|
+
}
|
|
242
|
+
return [...new Set(out)].sort();
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
/**
|
|
246
|
+
* Embed a task query using the repository's configured `local.embed` endpoint.
|
|
247
|
+
*
|
|
248
|
+
* Returns `null` without opening a socket when `embed` is false, `local.embed`
|
|
249
|
+
* is unset, or a non-loopback URL is refused.
|
|
250
|
+
*/
|
|
251
|
+
export async function embedTaskQuery(root, task, { embed = false, allowRemote = false, fetchImpl = undefined } = {}) {
|
|
252
|
+
if (!embed || String(task ?? '').trim().length === 0) return null;
|
|
253
|
+
const localCfg = await readLocalConfig(root, { allowRemote });
|
|
254
|
+
if (!localCfg.embed || localCfg.embed.blocked) return null;
|
|
255
|
+
const res = await localEmbed(localCfg.embed, {
|
|
256
|
+
input: String(task).slice(0, 4096),
|
|
257
|
+
allowRemote,
|
|
258
|
+
fetchImpl,
|
|
259
|
+
});
|
|
260
|
+
if (!res.ok || !Array.isArray(res.embeddings?.[0]) || res.embeddings[0].length === 0) {
|
|
261
|
+
return null;
|
|
262
|
+
}
|
|
263
|
+
return Float32Array.from(res.embeddings[0], (v) => Number(v) || 0);
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
/**
|
|
267
|
+
* Mean recall@10 over mined tasks whose gold patch is in the vault.
|
|
268
|
+
* A missing vault is unmeasured, not a zero: the absence of a hidden patch is
|
|
269
|
+
* not evidence the map missed the file (ADR-076).
|
|
270
|
+
*
|
|
271
|
+
* When `embed: true` (or stored embeddings are provided), also evaluates recall@10
|
|
272
|
+
* with embeddings fused in and enforces the R28 acceptance gate: embeddings stay off
|
|
273
|
+
* by default unless they raise gold-file recall@10 by at least 10 percentage points
|
|
274
|
+
* (`EMBED_RECALL_GAIN_BAR = 0.1`).
|
|
275
|
+
*
|
|
276
|
+
* @param {string} root
|
|
277
|
+
* @param {{
|
|
278
|
+
* vaultRoot?: string,
|
|
279
|
+
* backend?: string,
|
|
280
|
+
* embed?: boolean,
|
|
281
|
+
* allowRemote?: boolean,
|
|
282
|
+
* fetchImpl?: typeof fetch,
|
|
283
|
+
* embeddings?: {path: string, vec: Float32Array|number[]|Buffer}[],
|
|
284
|
+
* taskVectors?: Record<string, Float32Array|number[]>|Map<string, Float32Array|number[]>
|
|
285
|
+
* }} [opts]
|
|
286
|
+
*/
|
|
287
|
+
export async function measureRecall(root, opts = {}) {
|
|
288
|
+
const tasksDir = join(root, '.simt-dev', 'bench', 'tasks');
|
|
289
|
+
let names;
|
|
290
|
+
try {
|
|
291
|
+
names = (await readdir(tasksDir)).filter((name) => name.endsWith('.json')).sort();
|
|
292
|
+
} catch {
|
|
293
|
+
return { measured: false, reason: 'no tasks', tasks: 0, missing: 0, recall: null };
|
|
294
|
+
}
|
|
295
|
+
const files = await readFiles(root, opts);
|
|
296
|
+
if (files.length === 0) {
|
|
297
|
+
return { measured: false, reason: 'no index', tasks: 0, missing: 0, recall: null };
|
|
298
|
+
}
|
|
299
|
+
const [symbols, edges, fts] = await Promise.all([
|
|
300
|
+
readSymbols(root, opts),
|
|
301
|
+
readEdges(root, opts),
|
|
302
|
+
readFts(root, opts),
|
|
303
|
+
]);
|
|
304
|
+
const storedEmbeddings =
|
|
305
|
+
opts.embeddings ?? (opts.embed ? await readEmbeddings(root, opts) : []);
|
|
306
|
+
const checkEmbed = storedEmbeddings.length > 0 || opts.embed === true;
|
|
307
|
+
|
|
308
|
+
const vault = opts.vaultRoot ?? resolveVaultDir(root);
|
|
309
|
+
let sum = 0;
|
|
310
|
+
let embedSum = 0;
|
|
311
|
+
let embedEvaluated = 0;
|
|
312
|
+
let tasks = 0;
|
|
313
|
+
let missing = 0;
|
|
314
|
+
for (const name of names) {
|
|
315
|
+
const task = JSON.parse(await readFile(join(tasksDir, name), 'utf8'));
|
|
316
|
+
let patch;
|
|
317
|
+
try {
|
|
318
|
+
patch = await readFile(join(vault, task.id, 'gold.patch'), 'utf8');
|
|
319
|
+
} catch {
|
|
320
|
+
missing += 1;
|
|
321
|
+
continue;
|
|
322
|
+
}
|
|
323
|
+
const gold = goldPaths(patch);
|
|
324
|
+
if (gold.length === 0) {
|
|
325
|
+
missing += 1;
|
|
326
|
+
continue;
|
|
327
|
+
}
|
|
328
|
+
const prompt = task.prompt ?? '';
|
|
329
|
+
const bm25 = await searchFtsBm25(root, prompt, { ...opts, fts });
|
|
330
|
+
const ranked = rankFiles({ files, edges, symbols, fts, bm25, task: prompt });
|
|
331
|
+
sum += recallAt10(ranked, gold);
|
|
332
|
+
tasks += 1;
|
|
333
|
+
|
|
334
|
+
if (checkEmbed && storedEmbeddings.length > 0) {
|
|
335
|
+
const providedVec =
|
|
336
|
+
opts.taskVectors instanceof Map
|
|
337
|
+
? opts.taskVectors.get(task.id)
|
|
338
|
+
: opts.taskVectors?.[task.id];
|
|
339
|
+
const queryVector =
|
|
340
|
+
providedVec ??
|
|
341
|
+
(await embedTaskQuery(root, prompt, {
|
|
342
|
+
embed: opts.embed === true,
|
|
343
|
+
allowRemote: opts.allowRemote === true,
|
|
344
|
+
fetchImpl: opts.fetchImpl,
|
|
345
|
+
}));
|
|
346
|
+
if (queryVector) {
|
|
347
|
+
const embedRanked = rankFiles({
|
|
348
|
+
files,
|
|
349
|
+
edges,
|
|
350
|
+
symbols,
|
|
351
|
+
fts,
|
|
352
|
+
bm25,
|
|
353
|
+
embeddings: storedEmbeddings,
|
|
354
|
+
queryVector,
|
|
355
|
+
task: prompt,
|
|
356
|
+
});
|
|
357
|
+
embedSum += recallAt10(embedRanked, gold);
|
|
358
|
+
embedEvaluated += 1;
|
|
359
|
+
} else {
|
|
360
|
+
embedSum += recallAt10(ranked, gold);
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
if (tasks === 0) return { measured: false, reason: 'no gold patches', tasks: 0, missing, recall: null };
|
|
365
|
+
const baseRecall = sum / tasks;
|
|
366
|
+
const embedRecall = checkEmbed && embedEvaluated > 0 ? embedSum / tasks : null;
|
|
367
|
+
const embedDelta = embedRecall === null ? null : embedRecall - baseRecall;
|
|
368
|
+
// R28 acceptance gate: embeddings stay off by default unless they raise
|
|
369
|
+
// gold-file recall@10 by at least 10 points (+0.10).
|
|
370
|
+
const embedAccepted = embedDelta !== null && embedDelta + 1e-9 >= EMBED_RECALL_GAIN_BAR;
|
|
371
|
+
return {
|
|
372
|
+
measured: true,
|
|
373
|
+
reason: '',
|
|
374
|
+
tasks,
|
|
375
|
+
missing,
|
|
376
|
+
recall: embedAccepted ? embedRecall : baseRecall,
|
|
377
|
+
baseRecall,
|
|
378
|
+
embedRecall,
|
|
379
|
+
embedDelta,
|
|
380
|
+
embedAccepted,
|
|
381
|
+
embeddingsDefault: 'off',
|
|
382
|
+
};
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
export function formatRecall(result) {
|
|
386
|
+
if (!result?.measured || result.recall === null) {
|
|
387
|
+
return `recall@10 unmeasured (${result?.reason ?? 'not run'})`;
|
|
388
|
+
}
|
|
389
|
+
return `recall@10 ${(result.recall * 100).toFixed(1)}% of gold files, over ${result.tasks} mined tasks`;
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
function pageRank(nodes, links, scores) {
|
|
393
|
+
const n = nodes.length;
|
|
394
|
+
const pers = new Array(n).fill(0);
|
|
395
|
+
let mass = 0;
|
|
396
|
+
for (let i = 0; i < n; i += 1) {
|
|
397
|
+
pers[i] = scores.get(nodes[i]) ?? 0;
|
|
398
|
+
mass += pers[i];
|
|
399
|
+
}
|
|
400
|
+
if (mass === 0) {
|
|
401
|
+
for (let i = 0; i < n; i += 1) pers[i] = 1 / n;
|
|
402
|
+
} else {
|
|
403
|
+
for (let i = 0; i < n; i += 1) pers[i] /= mass;
|
|
404
|
+
}
|
|
405
|
+
let rank = pers.slice();
|
|
406
|
+
for (let iter = 0; iter < ITERATIONS; iter += 1) {
|
|
407
|
+
const next = pers.map((value) => (1 - DAMPING) * value);
|
|
408
|
+
for (let i = 0; i < n; i += 1) {
|
|
409
|
+
const outs = links[i];
|
|
410
|
+
if (outs.length === 0) {
|
|
411
|
+
for (let j = 0; j < n; j += 1) next[j] += DAMPING * rank[i] * pers[j];
|
|
412
|
+
} else {
|
|
413
|
+
let sum = 0;
|
|
414
|
+
for (const [, weight] of outs) sum += weight;
|
|
415
|
+
for (const [j, weight] of outs) next[j] += (DAMPING * rank[i] * weight) / sum;
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
rank = next;
|
|
419
|
+
}
|
|
420
|
+
return rank;
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
function buildLinks(nodes, edges, index) {
|
|
424
|
+
const weight = new Map();
|
|
425
|
+
const usable = edges.filter(
|
|
426
|
+
(edge) =>
|
|
427
|
+
(edge.kind === 'import' || edge.kind === 'call') &&
|
|
428
|
+
index.has(edge.src) &&
|
|
429
|
+
index.has(edge.dst) &&
|
|
430
|
+
edge.src !== edge.dst,
|
|
431
|
+
);
|
|
432
|
+
usable.sort((a, b) => cmp(a.src, b.src) || cmp(a.dst, b.dst) || cmp(a.kind, b.kind));
|
|
433
|
+
for (const edge of usable) {
|
|
434
|
+
const lo = edge.src < edge.dst ? edge.src : edge.dst;
|
|
435
|
+
const hi = edge.src < edge.dst ? edge.dst : edge.src;
|
|
436
|
+
const key = `${lo}\0${hi}`;
|
|
437
|
+
const value = Number.isFinite(edge.weight) && edge.weight > 0 ? edge.weight : 1;
|
|
438
|
+
weight.set(key, Math.max(weight.get(key) ?? 0, value));
|
|
439
|
+
}
|
|
440
|
+
const links = nodes.map(() => []);
|
|
441
|
+
for (const [key, value] of [...weight.entries()].sort((a, b) => cmp(a[0], b[0]))) {
|
|
442
|
+
const [lo, hi] = key.split('\0');
|
|
443
|
+
const i = index.get(lo);
|
|
444
|
+
const j = index.get(hi);
|
|
445
|
+
links[i].push([j, value]);
|
|
446
|
+
links[j].push([i, value]);
|
|
447
|
+
}
|
|
448
|
+
for (const list of links) list.sort((a, b) => a[0] - b[0]);
|
|
449
|
+
return links;
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
function closure(nodes, edges, scores) {
|
|
453
|
+
const seeds = new Set(nodes.filter((path) => (scores.get(path) ?? 0) > 0));
|
|
454
|
+
if (seeds.size === 0) return nodes;
|
|
455
|
+
const known = new Set(nodes);
|
|
456
|
+
const keep = new Set(seeds);
|
|
457
|
+
for (const edge of edges) {
|
|
458
|
+
if (edge.kind !== 'import' && edge.kind !== 'call') continue;
|
|
459
|
+
if (!known.has(edge.src) || !known.has(edge.dst)) continue;
|
|
460
|
+
if (seeds.has(edge.src)) keep.add(edge.dst);
|
|
461
|
+
if (seeds.has(edge.dst)) keep.add(edge.src);
|
|
462
|
+
}
|
|
463
|
+
return [...keep].sort();
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
function scoreFiles(nodes, symbolsByPath, task) {
|
|
467
|
+
const scores = new Map(nodes.map((path) => [path, 0]));
|
|
468
|
+
const { text, terms } = termsOf(task);
|
|
469
|
+
if (terms.size === 0) return scores;
|
|
470
|
+
const segFreq = new Map();
|
|
471
|
+
for (const path of nodes) {
|
|
472
|
+
for (const segment of segments(path)) segFreq.set(segment, (segFreq.get(segment) ?? 0) + 1);
|
|
473
|
+
}
|
|
474
|
+
const cap = Math.max(3, Math.floor(nodes.length * 0.15));
|
|
475
|
+
// A term that labels every file ("src") is the tree, not the task. A long
|
|
476
|
+
// word that sits inside dozens of paths ("decision") is the same thing, so
|
|
477
|
+
// the substring count uses the same ceiling.
|
|
478
|
+
const common = (freq) => freq > cap || (nodes.length > 1 && freq === nodes.length);
|
|
479
|
+
const termFreq = new Map();
|
|
480
|
+
for (const term of terms) {
|
|
481
|
+
if (term.length < 6) continue;
|
|
482
|
+
let count = 0;
|
|
483
|
+
for (const path of nodes) if (path.toLowerCase().includes(term)) count += 1;
|
|
484
|
+
termFreq.set(term, count);
|
|
485
|
+
}
|
|
486
|
+
const symFreq = new Map();
|
|
487
|
+
for (const path of nodes) {
|
|
488
|
+
const seen = new Set();
|
|
489
|
+
for (const symbol of symbolsByPath.get(path) ?? []) {
|
|
490
|
+
const name = String(symbol.name ?? '').toLowerCase();
|
|
491
|
+
if (name.length >= 3) seen.add(name);
|
|
492
|
+
}
|
|
493
|
+
for (const name of seen) symFreq.set(name, (symFreq.get(name) ?? 0) + 1);
|
|
494
|
+
}
|
|
495
|
+
for (const path of nodes) {
|
|
496
|
+
const lower = path.toLowerCase();
|
|
497
|
+
let score = 0;
|
|
498
|
+
if (lower.length >= 3 && text.includes(lower)) score += 8;
|
|
499
|
+
const stem = stemOf(lower);
|
|
500
|
+
if (stem.length >= 3 && terms.has(stem) && !common(segFreq.get(stem) ?? 0)) score += 4;
|
|
501
|
+
for (const segment of segments(lower)) {
|
|
502
|
+
if (terms.has(segment) && !common(segFreq.get(segment) ?? 0)) score += 2;
|
|
503
|
+
}
|
|
504
|
+
for (const term of terms) {
|
|
505
|
+
const freq = termFreq.get(term) ?? 0;
|
|
506
|
+
if (term.length < 6 || freq === 0 || !lower.includes(term) || common(freq)) continue;
|
|
507
|
+
score += 3;
|
|
508
|
+
// A record cited once is context. The record the task is about is named
|
|
509
|
+
// again. A flat bonus ranked every citation with the subject, and the
|
|
510
|
+
// files the patch actually changed fell out of the top 10.
|
|
511
|
+
if (freq === 1 && /\d/.test(term)) score += 8 * Math.min(mentions(text, term), 3);
|
|
512
|
+
}
|
|
513
|
+
let hits = 0;
|
|
514
|
+
for (const symbol of symbolsByPath.get(path) ?? []) {
|
|
515
|
+
const name = String(symbol.name ?? '').toLowerCase();
|
|
516
|
+
if (name.length >= 3 && terms.has(name) && (symFreq.get(name) ?? 0) <= 5) hits += 1;
|
|
517
|
+
}
|
|
518
|
+
scores.set(path, score + Math.min(hits, 8));
|
|
519
|
+
}
|
|
520
|
+
// A task that names one file in a small directory usually edits its
|
|
521
|
+
// neighbours too. A large directory (every decision record, every command)
|
|
522
|
+
// does not get that lift: one citation would pull the whole directory in.
|
|
523
|
+
const dirMax = new Map();
|
|
524
|
+
const dirCount = new Map();
|
|
525
|
+
for (const path of nodes) {
|
|
526
|
+
const dir = directoryOf(path);
|
|
527
|
+
dirCount.set(dir, (dirCount.get(dir) ?? 0) + 1);
|
|
528
|
+
dirMax.set(dir, Math.max(dirMax.get(dir) ?? 0, scores.get(path) ?? 0));
|
|
529
|
+
}
|
|
530
|
+
for (const path of nodes) {
|
|
531
|
+
const own = scores.get(path) ?? 0;
|
|
532
|
+
if (own === 0) continue;
|
|
533
|
+
const dir = directoryOf(path);
|
|
534
|
+
if ((dirCount.get(dir) ?? 0) > 8) continue;
|
|
535
|
+
const lifted = Math.floor((dirMax.get(dir) ?? 0) / 2);
|
|
536
|
+
if (lifted > own) scores.set(path, lifted);
|
|
537
|
+
}
|
|
538
|
+
return scores;
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
function directoryOf(path) {
|
|
542
|
+
const slash = path.lastIndexOf('/');
|
|
543
|
+
return slash < 0 ? '' : path.slice(0, slash);
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
function mentions(text, term) {
|
|
547
|
+
let count = 0;
|
|
548
|
+
let from = 0;
|
|
549
|
+
while (from < text.length) {
|
|
550
|
+
const at = text.indexOf(term, from);
|
|
551
|
+
if (at < 0) return count;
|
|
552
|
+
count += 1;
|
|
553
|
+
from = at + term.length;
|
|
554
|
+
}
|
|
555
|
+
return count;
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
function termsOf(task) {
|
|
559
|
+
const text = String(task ?? '').toLowerCase();
|
|
560
|
+
const terms = new Set();
|
|
561
|
+
for (const word of text.split(/[^a-z0-9-]+/)) {
|
|
562
|
+
if (word.length >= 3 && !STOP.has(word)) terms.add(word);
|
|
563
|
+
}
|
|
564
|
+
return { text, terms };
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
function segments(path) {
|
|
568
|
+
const lower = path.toLowerCase();
|
|
569
|
+
const out = [];
|
|
570
|
+
for (const part of lower.split('/')) {
|
|
571
|
+
const stem = stemOf(part);
|
|
572
|
+
if (stem.length >= 3) out.push(stem);
|
|
573
|
+
}
|
|
574
|
+
return out;
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
function stemOf(path) {
|
|
578
|
+
const base = path.slice(path.lastIndexOf('/') + 1);
|
|
579
|
+
const dot = base.lastIndexOf('.');
|
|
580
|
+
return dot > 0 ? base.slice(0, dot) : base;
|
|
581
|
+
}
|
|
582
|
+
|
|
583
|
+
function groupSymbols(symbols) {
|
|
584
|
+
const byPath = new Map();
|
|
585
|
+
for (const symbol of symbols ?? []) {
|
|
586
|
+
const list = byPath.get(symbol.path) ?? [];
|
|
587
|
+
list.push(symbol);
|
|
588
|
+
byPath.set(symbol.path, list);
|
|
589
|
+
}
|
|
590
|
+
return byPath;
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
function renderEntry(path, symbols, level) {
|
|
594
|
+
const exported = symbols
|
|
595
|
+
.filter((symbol) => symbol.exported === 1 || symbol.exported === true)
|
|
596
|
+
.sort((a, b) => cmp(a.name, b.name) || a.line - b.line);
|
|
597
|
+
if (level === 'names') {
|
|
598
|
+
const names = exported.map((symbol) => symbol.name);
|
|
599
|
+
return names.length === 0 ? path : `${path} ${names.join(' ')}`;
|
|
600
|
+
}
|
|
601
|
+
if (level === 'signatures') {
|
|
602
|
+
const lines = [path];
|
|
603
|
+
for (const symbol of exported) {
|
|
604
|
+
const sig = String(symbol.sig ?? '').trim();
|
|
605
|
+
lines.push(sig ? ` ${symbol.name} ${sig}` : ` ${symbol.name}`);
|
|
606
|
+
}
|
|
607
|
+
return lines.join('\n');
|
|
608
|
+
}
|
|
609
|
+
return path;
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
function byRank(a, b) {
|
|
613
|
+
const score = (b.score ?? 0) - (a.score ?? 0);
|
|
614
|
+
if (score !== 0) return score;
|
|
615
|
+
if (a.rank !== b.rank) return a.rank < b.rank ? 1 : -1;
|
|
616
|
+
return cmp(a.path, b.path);
|
|
617
|
+
}
|
|
618
|
+
|
|
619
|
+
function round6(value) {
|
|
620
|
+
return Math.round(value * 1e6) / 1e6;
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
function cmp(a, b) {
|
|
624
|
+
if (a < b) return -1;
|
|
625
|
+
if (a > b) return 1;
|
|
626
|
+
return 0;
|
|
627
|
+
}
|