pi-knowledge 0.3.4 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.pi/skills/search-docs.md +24 -4
- package/CHANGELOG.md +34 -0
- package/README.md +55 -3
- package/index.ts +263 -20
- package/package.json +1 -1
- package/src/diagnostics/health.ts +84 -14
- package/src/embedding/provider.ts +50 -12
- package/src/embedding/vectors.ts +95 -12
- package/src/engine.ts +1252 -177
- package/src/indexer/chunker.ts +331 -100
- package/src/indexer/chunkers/code-ast.ts +14 -11
- package/src/search/bm25.ts +37 -22
- package/src/search/fusion.ts +33 -0
- package/src/search/query.ts +59 -0
- package/src/search/ranking.ts +201 -0
- package/src/search/reranker.ts +49 -11
- package/src/search/vector.ts +60 -0
- package/src/storage/sqlite.ts +240 -9
package/src/search/fusion.ts
CHANGED
|
@@ -3,6 +3,19 @@ export interface FusionResult {
|
|
|
3
3
|
score: number;
|
|
4
4
|
}
|
|
5
5
|
|
|
6
|
+
function normalizeScores(list: Array<{ chunkId: string; score: number }>): Map<string, number> {
|
|
7
|
+
if (list.length === 0) return new Map();
|
|
8
|
+
const scores = list.map((item) => item.score);
|
|
9
|
+
const min = Math.min(...scores);
|
|
10
|
+
const max = Math.max(...scores);
|
|
11
|
+
return new Map(
|
|
12
|
+
list.map((item) => {
|
|
13
|
+
const normalized = max === min ? 1 : (item.score - min) / (max - min);
|
|
14
|
+
return [item.chunkId, normalized];
|
|
15
|
+
}),
|
|
16
|
+
);
|
|
17
|
+
}
|
|
18
|
+
|
|
6
19
|
export function reciprocalRankFusion(lists: Array<{ chunkId: string; score: number }[]>, k = 60): FusionResult[] {
|
|
7
20
|
const scores = new Map<string, number>();
|
|
8
21
|
for (const list of lists) {
|
|
@@ -13,3 +26,23 @@ export function reciprocalRankFusion(lists: Array<{ chunkId: string; score: numb
|
|
|
13
26
|
}
|
|
14
27
|
return [...scores.entries()].map(([chunkId, score]) => ({ chunkId, score })).sort((a, b) => b.score - a.score);
|
|
15
28
|
}
|
|
29
|
+
|
|
30
|
+
export function weightedScoreFusion(
|
|
31
|
+
bm25Results: Array<{ chunkId: string; score: number }>,
|
|
32
|
+
vectorResults: Array<{ chunkId: string; score: number }>,
|
|
33
|
+
weights = { bm25: 0.45, vector: 0.55, overlap: 0.15 },
|
|
34
|
+
): FusionResult[] {
|
|
35
|
+
const bm25Scores = normalizeScores(bm25Results);
|
|
36
|
+
const vectorScores = normalizeScores(vectorResults);
|
|
37
|
+
const ids = new Set([...bm25Scores.keys(), ...vectorScores.keys()]);
|
|
38
|
+
const fused: FusionResult[] = [];
|
|
39
|
+
|
|
40
|
+
for (const chunkId of ids) {
|
|
41
|
+
const bm25 = bm25Scores.get(chunkId) ?? 0;
|
|
42
|
+
const vector = vectorScores.get(chunkId) ?? 0;
|
|
43
|
+
const overlap = bm25 > 0 && vector > 0 ? weights.overlap : 0;
|
|
44
|
+
fused.push({ chunkId, score: weights.bm25 * bm25 + weights.vector * vector + overlap });
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
return fused.sort((a, b) => b.score - a.score);
|
|
48
|
+
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { preTokenizeForFTS } from "../indexer/chunker.ts";
|
|
2
|
+
|
|
3
|
+
export const STOP_WORDS = new Set([
|
|
4
|
+
"the",
|
|
5
|
+
"and",
|
|
6
|
+
"or",
|
|
7
|
+
"to",
|
|
8
|
+
"of",
|
|
9
|
+
"in",
|
|
10
|
+
"on",
|
|
11
|
+
"for",
|
|
12
|
+
"with",
|
|
13
|
+
"how",
|
|
14
|
+
"does",
|
|
15
|
+
"what",
|
|
16
|
+
"when",
|
|
17
|
+
"where",
|
|
18
|
+
"why",
|
|
19
|
+
"this",
|
|
20
|
+
"that",
|
|
21
|
+
"from",
|
|
22
|
+
]);
|
|
23
|
+
|
|
24
|
+
const TYPO_CORRECTIONS: Record<string, string> = {
|
|
25
|
+
ho: "how",
|
|
26
|
+
setpup: "setup",
|
|
27
|
+
setpu: "setup",
|
|
28
|
+
seting: "setting",
|
|
29
|
+
configuraiton: "configuration",
|
|
30
|
+
permisson: "permission",
|
|
31
|
+
permisions: "permissions",
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
export function stemToken(token: string): string {
|
|
35
|
+
if (token.length > 4 && token.endsWith("ies")) return `${token.slice(0, -3)}y`;
|
|
36
|
+
if (token.length > 5 && token.endsWith("ing")) return token.slice(0, -3);
|
|
37
|
+
if (token.length > 4 && token.endsWith("ed")) return token.slice(0, -2);
|
|
38
|
+
if (token.length > 3 && token.endsWith("s")) return token.slice(0, -1);
|
|
39
|
+
return token;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export function tokenizeForSearch(text: string): Set<string> {
|
|
43
|
+
const normalized = preTokenizeForFTS(text)
|
|
44
|
+
.toLowerCase()
|
|
45
|
+
.replace(/[-_*"(){}[\]^~:+.#@!\\/<>|&$%?]/g, " ");
|
|
46
|
+
const tokens = normalized
|
|
47
|
+
.split(/\s+/)
|
|
48
|
+
.map((token) => TYPO_CORRECTIONS[token] ?? token)
|
|
49
|
+
.filter((token) => token.length > 1 || /[\u3400-\u4dbf\u4e00-\u9fff]/.test(token));
|
|
50
|
+
return new Set(tokens);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export function signalTokens(tokens: Set<string>): Set<string> {
|
|
54
|
+
return new Set([...tokens].filter((token) => !STOP_WORDS.has(token)).map(stemToken));
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export function normalizedQueryText(query: string): string {
|
|
58
|
+
return [...tokenizeForSearch(query)].join(" ");
|
|
59
|
+
}
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
import { buildChunkEmbeddingText } from "../indexer/chunker.ts";
|
|
2
|
+
import type { Chunk } from "../storage/sqlite.ts";
|
|
3
|
+
import { signalTokens, tokenizeForSearch } from "./query.ts";
|
|
4
|
+
|
|
5
|
+
export const MIN_HYBRID_SCORE = 0.18;
|
|
6
|
+
|
|
7
|
+
export const FILE_TYPE_ALIASES: Record<string, string> = {
|
|
8
|
+
csharp: "csharp",
|
|
9
|
+
cs: "csharp",
|
|
10
|
+
cpp: "cpp",
|
|
11
|
+
hpp: "cpp",
|
|
12
|
+
js: "javascript",
|
|
13
|
+
jsx: "javascript",
|
|
14
|
+
md: "markdown",
|
|
15
|
+
mdx: "markdown",
|
|
16
|
+
php: "php",
|
|
17
|
+
py: "python",
|
|
18
|
+
rb: "ruby",
|
|
19
|
+
sh: "shell",
|
|
20
|
+
ts: "typescript",
|
|
21
|
+
tsx: "typescript",
|
|
22
|
+
yml: "yaml",
|
|
23
|
+
};
|
|
24
|
+
|
|
25
|
+
const SOURCE_FILE_TYPES = new Set([
|
|
26
|
+
"c",
|
|
27
|
+
"cpp",
|
|
28
|
+
"csharp",
|
|
29
|
+
"go",
|
|
30
|
+
"java",
|
|
31
|
+
"javascript",
|
|
32
|
+
"php",
|
|
33
|
+
"python",
|
|
34
|
+
"ruby",
|
|
35
|
+
"rust",
|
|
36
|
+
"typescript",
|
|
37
|
+
]);
|
|
38
|
+
|
|
39
|
+
const TEST_PATH_PATTERNS = ["_test.", ".test.", ".spec.", "/test/", "/tests/", "tests/", "__tests__/", "spec/"];
|
|
40
|
+
|
|
41
|
+
export interface RankingDiagnostics {
|
|
42
|
+
adjusted_score: number;
|
|
43
|
+
base_score: number;
|
|
44
|
+
coverage: number;
|
|
45
|
+
documentation_boost: number;
|
|
46
|
+
is_localization: boolean;
|
|
47
|
+
is_test: boolean;
|
|
48
|
+
localization_penalty: number;
|
|
49
|
+
path_boost: number;
|
|
50
|
+
source_boost: number;
|
|
51
|
+
test_intent: boolean;
|
|
52
|
+
user_guide_intent: boolean;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export function normalizeFileTypeFilter(fileType: string | undefined): string | undefined {
|
|
56
|
+
if (!fileType) return undefined;
|
|
57
|
+
const normalized = fileType.trim().toLowerCase();
|
|
58
|
+
return FILE_TYPE_ALIASES[normalized] ?? normalized;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export function isTestPath(filePath: string): boolean {
|
|
62
|
+
const lower = filePath.toLowerCase();
|
|
63
|
+
return TEST_PATH_PATTERNS.some((pattern) => lower.includes(pattern));
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export function queryAsksForTests(queryTokens: Set<string>): boolean {
|
|
67
|
+
const signals = signalTokens(queryTokens);
|
|
68
|
+
return ["assert", "fixture", "spec", "test", "testing"].some((token) => signals.has(token));
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function queryAsksForUserGuide(queryTokens: Set<string>): boolean {
|
|
72
|
+
const signals = signalTokens(queryTokens);
|
|
73
|
+
return ["config", "configuration", "install", "onboard", "quickstart", "setup", "start", "usage"].some((token) =>
|
|
74
|
+
signals.has(token),
|
|
75
|
+
);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function queryAsksForLocalization(queryTokens: Set<string>): boolean {
|
|
79
|
+
const signals = signalTokens(queryTokens);
|
|
80
|
+
return ["i18n", "lang", "language", "locale", "localization", "translation"].some((token) => signals.has(token));
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export function isLocalizationPath(filePath: string, fileType: string): boolean {
|
|
84
|
+
if (!["json", "markdown", "text", "toml", "yaml"].includes(fileType)) return false;
|
|
85
|
+
const lower = filePath.toLowerCase();
|
|
86
|
+
return (
|
|
87
|
+
lower.includes("/locale/") ||
|
|
88
|
+
lower.startsWith("locale/") ||
|
|
89
|
+
lower.includes("/locales/") ||
|
|
90
|
+
lower.startsWith("locales/") ||
|
|
91
|
+
lower.includes("/lang/") ||
|
|
92
|
+
lower.startsWith("lang/") ||
|
|
93
|
+
lower.includes("/i18n/") ||
|
|
94
|
+
lower.startsWith("i18n/") ||
|
|
95
|
+
lower.includes("/translations/") ||
|
|
96
|
+
lower.startsWith("translations/")
|
|
97
|
+
);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export function queryCoverage(text: string, queryTokens: Set<string>): number {
|
|
101
|
+
const textTokens = signalTokens(tokenizeForSearch(text));
|
|
102
|
+
const signals = signalTokens(queryTokens);
|
|
103
|
+
if (signals.size === 0) return 0;
|
|
104
|
+
let matched = 0;
|
|
105
|
+
for (const token of signals) {
|
|
106
|
+
if (textTokens.has(token)) matched++;
|
|
107
|
+
}
|
|
108
|
+
return matched / signals.size;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
export function hasAnyLexicalEvidence(text: string, queryTokens: Set<string>): boolean {
|
|
112
|
+
return queryCoverage(text, queryTokens) > 0;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
export function pathTokenBoost(filePath: string, queryTokens: Set<string>): number {
|
|
116
|
+
const pathTokens = signalTokens(tokenizeForSearch(filePath.replace(/[./_-]/g, " ")));
|
|
117
|
+
const signals = signalTokens(queryTokens);
|
|
118
|
+
let matched = 0;
|
|
119
|
+
for (const token of signals) {
|
|
120
|
+
if (pathTokens.has(token)) matched++;
|
|
121
|
+
}
|
|
122
|
+
if (matched === 0) return 0;
|
|
123
|
+
return Math.min(0.45, 0.18 * matched);
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
export function basenameTokenBoost(filePath: string, queryTokens: Set<string>): number {
|
|
127
|
+
const basename = filePath.split("/").at(-1) ?? filePath;
|
|
128
|
+
const stem = basename.replace(/\.[^.]+$/, "").replace(/_test$|\.test$|\.spec$/i, "");
|
|
129
|
+
const baseTokens = signalTokens(tokenizeForSearch(stem.replace(/[._-]/g, " ")));
|
|
130
|
+
const signals = signalTokens(queryTokens);
|
|
131
|
+
if (baseTokens.size === 0 || signals.size === 0) return 0;
|
|
132
|
+
for (const token of signals) {
|
|
133
|
+
if (baseTokens.has(token)) return 0.32;
|
|
134
|
+
}
|
|
135
|
+
return 0;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
export function sourceFileBoost(chunk: Pick<Chunk, "file_path" | "file_type">, queryTokens: Set<string>): number {
|
|
139
|
+
if (!SOURCE_FILE_TYPES.has(chunk.file_type)) return 0;
|
|
140
|
+
if (queryAsksForUserGuide(queryTokens)) return Math.min(0.05, pathTokenBoost(chunk.file_path, queryTokens));
|
|
141
|
+
const boost = basenameTokenBoost(chunk.file_path, queryTokens);
|
|
142
|
+
if (boost > 0) return boost;
|
|
143
|
+
const boostFromPath = pathTokenBoost(chunk.file_path, queryTokens);
|
|
144
|
+
return Math.min(0.22, boostFromPath);
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
function documentationBoost(chunk: Pick<Chunk, "file_path" | "file_type">, queryTokens: Set<string>): number {
|
|
148
|
+
if (!queryAsksForUserGuide(queryTokens)) return 0;
|
|
149
|
+
const lower = chunk.file_path.toLowerCase();
|
|
150
|
+
if (chunk.file_type !== "markdown") return 0;
|
|
151
|
+
if (
|
|
152
|
+
lower.includes("install") ||
|
|
153
|
+
lower.includes("readme") ||
|
|
154
|
+
lower.includes("getting-started") ||
|
|
155
|
+
lower.includes("quickstart")
|
|
156
|
+
) {
|
|
157
|
+
return 0.85;
|
|
158
|
+
}
|
|
159
|
+
if (lower.startsWith("docs/")) return 0.35;
|
|
160
|
+
return 0;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
export function scoreChunkForQuery(baseScore: number, chunk: Chunk, queryTokens: Set<string>): RankingDiagnostics {
|
|
164
|
+
const guideIntent = queryAsksForUserGuide(queryTokens);
|
|
165
|
+
const pathBoost =
|
|
166
|
+
guideIntent && SOURCE_FILE_TYPES.has(chunk.file_type)
|
|
167
|
+
? Math.min(0.08, pathTokenBoost(chunk.file_path, queryTokens))
|
|
168
|
+
: pathTokenBoost(chunk.file_path, queryTokens);
|
|
169
|
+
const sourceBoost = sourceFileBoost(chunk, queryTokens);
|
|
170
|
+
const docBoost = documentationBoost(chunk, queryTokens);
|
|
171
|
+
const testIntent = queryAsksForTests(queryTokens);
|
|
172
|
+
const test = isTestPath(chunk.file_path);
|
|
173
|
+
const localization = isLocalizationPath(chunk.file_path, chunk.file_type);
|
|
174
|
+
const localizationPenalty = localization && !queryAsksForLocalization(queryTokens) ? 0.2 : 1;
|
|
175
|
+
let adjustedScore = baseScore + pathBoost + sourceBoost + docBoost;
|
|
176
|
+
if (test && !testIntent) adjustedScore *= 0.48;
|
|
177
|
+
if (!test && testIntent) adjustedScore *= 0.88;
|
|
178
|
+
adjustedScore *= localizationPenalty;
|
|
179
|
+
return {
|
|
180
|
+
adjusted_score: adjustedScore,
|
|
181
|
+
base_score: baseScore,
|
|
182
|
+
coverage: queryCoverage(buildChunkEmbeddingText(chunk), queryTokens),
|
|
183
|
+
documentation_boost: docBoost,
|
|
184
|
+
is_localization: localization,
|
|
185
|
+
is_test: test,
|
|
186
|
+
localization_penalty: localizationPenalty,
|
|
187
|
+
path_boost: pathBoost,
|
|
188
|
+
source_boost: sourceBoost,
|
|
189
|
+
test_intent: testIntent,
|
|
190
|
+
user_guide_intent: guideIntent,
|
|
191
|
+
};
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
export function hasEnoughLexicalEvidence(chunk: Chunk, queryTokens: Set<string>): boolean {
|
|
195
|
+
const signals = signalTokens(queryTokens);
|
|
196
|
+
if (signals.size <= 1) return true;
|
|
197
|
+
if (sourceFileBoost(chunk, queryTokens) >= 0.22) return true;
|
|
198
|
+
const coverage = queryCoverage(buildChunkEmbeddingText(chunk), queryTokens);
|
|
199
|
+
if (signals.size <= 3) return coverage >= 1 / signals.size;
|
|
200
|
+
return coverage >= 0.34;
|
|
201
|
+
}
|
package/src/search/reranker.ts
CHANGED
|
@@ -9,6 +9,9 @@ type RerankerPipeline = {
|
|
|
9
9
|
let rerankerPipeline: RerankerPipeline | null = null;
|
|
10
10
|
let disposeTimer: ReturnType<typeof setTimeout> | null = null;
|
|
11
11
|
let disposePromise: Promise<void> | null = null;
|
|
12
|
+
let activeRuns = 0;
|
|
13
|
+
let disposeRequested = false;
|
|
14
|
+
const idleWaiters: Array<() => void> = [];
|
|
12
15
|
const IDLE_MS = 30_000;
|
|
13
16
|
|
|
14
17
|
async function load(): Promise<RerankerPipeline> {
|
|
@@ -20,26 +23,57 @@ async function load(): Promise<RerankerPipeline> {
|
|
|
20
23
|
return rerankerPipeline;
|
|
21
24
|
}
|
|
22
25
|
|
|
23
|
-
function
|
|
26
|
+
function clearTimer(): void {
|
|
24
27
|
if (disposeTimer) clearTimeout(disposeTimer);
|
|
28
|
+
disposeTimer = null;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function scheduleDispose(): void {
|
|
32
|
+
if (activeRuns > 0 || disposeRequested) return;
|
|
33
|
+
clearTimer();
|
|
25
34
|
disposeTimer = setTimeout(() => disposeReranker(), IDLE_MS);
|
|
26
35
|
}
|
|
27
36
|
|
|
37
|
+
function beginRun(): void {
|
|
38
|
+
activeRuns++;
|
|
39
|
+
clearTimer();
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function endRun(): void {
|
|
43
|
+
activeRuns--;
|
|
44
|
+
if (activeRuns > 0) return;
|
|
45
|
+
for (const resolve of idleWaiters.splice(0)) resolve();
|
|
46
|
+
if (!disposeRequested) scheduleDispose();
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function waitForNoActiveRuns(): Promise<void> {
|
|
50
|
+
if (activeRuns === 0) return Promise.resolve();
|
|
51
|
+
return new Promise((resolve) => idleWaiters.push(resolve));
|
|
52
|
+
}
|
|
53
|
+
|
|
28
54
|
export async function disposeReranker(): Promise<void> {
|
|
29
|
-
|
|
30
|
-
clearTimeout(disposeTimer);
|
|
31
|
-
disposeTimer = null;
|
|
32
|
-
}
|
|
55
|
+
clearTimer();
|
|
33
56
|
if (disposePromise) return disposePromise;
|
|
57
|
+
disposeRequested = true;
|
|
58
|
+
await waitForNoActiveRuns();
|
|
34
59
|
const instance = rerankerPipeline;
|
|
35
60
|
rerankerPipeline = null;
|
|
36
|
-
if (!instance)
|
|
61
|
+
if (!instance) {
|
|
62
|
+
disposeRequested = false;
|
|
63
|
+
return;
|
|
64
|
+
}
|
|
37
65
|
disposePromise = Promise.resolve(instance.dispose()).finally(() => {
|
|
38
66
|
disposePromise = null;
|
|
67
|
+
disposeRequested = false;
|
|
39
68
|
});
|
|
40
69
|
return disposePromise;
|
|
41
70
|
}
|
|
42
71
|
|
|
72
|
+
export async function prepareRerankerForShutdown(): Promise<void> {
|
|
73
|
+
clearTimer();
|
|
74
|
+
await waitForNoActiveRuns();
|
|
75
|
+
}
|
|
76
|
+
|
|
43
77
|
export interface RerankCandidate {
|
|
44
78
|
chunkId: string;
|
|
45
79
|
content: string;
|
|
@@ -52,13 +86,17 @@ export async function rerank(
|
|
|
52
86
|
): Promise<Array<{ chunkId: string; score: number }>> {
|
|
53
87
|
if (candidates.length === 0) return [];
|
|
54
88
|
const pipe = await load();
|
|
55
|
-
|
|
89
|
+
beginRun();
|
|
56
90
|
|
|
57
91
|
const results: Array<{ chunkId: string; score: number }> = [];
|
|
58
|
-
|
|
59
|
-
const
|
|
60
|
-
|
|
61
|
-
|
|
92
|
+
try {
|
|
93
|
+
for (const c of candidates) {
|
|
94
|
+
const output = await pipe({ text: query, text_pair: c.content });
|
|
95
|
+
const score = Array.isArray(output) ? (output[0]?.score ?? 0) : (output?.score ?? 0);
|
|
96
|
+
results.push({ chunkId: c.chunkId, score });
|
|
97
|
+
}
|
|
98
|
+
} finally {
|
|
99
|
+
endRun();
|
|
62
100
|
}
|
|
63
101
|
|
|
64
102
|
results.sort((a, b) => b.score - a.score);
|
package/src/search/vector.ts
CHANGED
|
@@ -1,14 +1,42 @@
|
|
|
1
|
+
import { openVectorReader } from "../embedding/vectors.ts";
|
|
2
|
+
|
|
1
3
|
export interface VectorResult {
|
|
2
4
|
chunkId: string;
|
|
3
5
|
score: number;
|
|
4
6
|
}
|
|
5
7
|
|
|
8
|
+
export interface VectorFileSearchResult {
|
|
9
|
+
results: VectorResult[];
|
|
10
|
+
vectorsByChunkId: Map<string, Float32Array>;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
type ChunkIdSource = Iterable<string | { id: string }>;
|
|
14
|
+
|
|
6
15
|
function cosine(a: Float32Array, b: Float32Array): number {
|
|
7
16
|
let dot = 0;
|
|
8
17
|
for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
|
|
9
18
|
return dot;
|
|
10
19
|
}
|
|
11
20
|
|
|
21
|
+
function keepTopK(
|
|
22
|
+
top: Array<VectorResult & { vector: Float32Array }>,
|
|
23
|
+
item: VectorResult,
|
|
24
|
+
vector: Float32Array,
|
|
25
|
+
limit: number,
|
|
26
|
+
): void {
|
|
27
|
+
if (limit <= 0) return;
|
|
28
|
+
if (top.length < limit) {
|
|
29
|
+
const stored = { ...item, vector: new Float32Array(vector) };
|
|
30
|
+
top.push(stored);
|
|
31
|
+
top.sort((a, b) => a.score - b.score);
|
|
32
|
+
return;
|
|
33
|
+
}
|
|
34
|
+
if (item.score <= top[0].score) return;
|
|
35
|
+
const stored = { ...item, vector: new Float32Array(vector) };
|
|
36
|
+
top[0] = stored;
|
|
37
|
+
top.sort((a, b) => a.score - b.score);
|
|
38
|
+
}
|
|
39
|
+
|
|
12
40
|
export function searchVector(
|
|
13
41
|
queryVec: Float32Array,
|
|
14
42
|
vectors: Float32Array[],
|
|
@@ -19,3 +47,35 @@ export function searchVector(
|
|
|
19
47
|
scores.sort((a, b) => b.score - a.score);
|
|
20
48
|
return scores.slice(0, limit);
|
|
21
49
|
}
|
|
50
|
+
|
|
51
|
+
export function searchVectorFile(
|
|
52
|
+
queryVec: Float32Array,
|
|
53
|
+
vectorPath: string,
|
|
54
|
+
chunkIds: ChunkIdSource,
|
|
55
|
+
limit = 50,
|
|
56
|
+
): VectorFileSearchResult {
|
|
57
|
+
const reader = openVectorReader(vectorPath);
|
|
58
|
+
if (!reader) return { results: [], vectorsByChunkId: new Map() };
|
|
59
|
+
try {
|
|
60
|
+
if (reader.dim !== queryVec.length) return { results: [], vectorsByChunkId: new Map() };
|
|
61
|
+
const scratch = new Float32Array(reader.dim);
|
|
62
|
+
const top: Array<VectorResult & { vector: Float32Array }> = [];
|
|
63
|
+
let i = 0;
|
|
64
|
+
for (const chunkIdRef of chunkIds) {
|
|
65
|
+
if (i >= reader.count) break;
|
|
66
|
+
const vectorIndex = i;
|
|
67
|
+
i++;
|
|
68
|
+
if (!reader.readInto(vectorIndex, scratch)) continue;
|
|
69
|
+
const chunkId = typeof chunkIdRef === "string" ? chunkIdRef : chunkIdRef.id;
|
|
70
|
+
keepTopK(top, { chunkId, score: cosine(queryVec, scratch) }, scratch, limit);
|
|
71
|
+
}
|
|
72
|
+
const sorted = top.sort((a, b) => b.score - a.score);
|
|
73
|
+
const vectorsByChunkId = new Map(sorted.map((item) => [item.chunkId, item.vector]));
|
|
74
|
+
return {
|
|
75
|
+
results: sorted.map(({ chunkId, score }) => ({ chunkId, score })),
|
|
76
|
+
vectorsByChunkId,
|
|
77
|
+
};
|
|
78
|
+
} finally {
|
|
79
|
+
reader.close();
|
|
80
|
+
}
|
|
81
|
+
}
|