pi-knowledge 0.11.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/README.md +147 -33
- package/dist/index.js +15 -4
- package/dist/src/embedding/provider.d.ts +14 -0
- package/dist/src/embedding/provider.js +90 -8
- package/dist/src/engine.d.ts +3 -1
- package/dist/src/engine.js +122 -147
- package/dist/src/indexer/chunker.d.ts +1 -1
- package/dist/src/indexer/chunker.js +2 -2
- package/dist/src/indexer/embedding-safe-chunks.d.ts +5 -0
- package/dist/src/indexer/embedding-safe-chunks.js +215 -0
- package/dist/src/indexer/html.d.ts +1 -0
- package/dist/src/indexer/html.js +49 -0
- package/dist/src/model-worker-client.d.ts +4 -0
- package/dist/src/model-worker-client.js +17 -1
- package/dist/src/model-worker.js +63 -15
- package/dist/src/search/contracts.d.ts +2 -0
- package/dist/src/search/contracts.js +2 -0
- package/dist/src/search/diversity.d.ts +16 -0
- package/dist/src/search/diversity.js +183 -0
- package/dist/src/search/tuning.d.ts +10 -0
- package/dist/src/search/tuning.js +20 -0
- package/dist/src/storage/sqlite.d.ts +1 -0
- package/dist/src/storage/sqlite.js +3 -0
- package/docs/configuration.md +3 -3
- package/package.json +1 -1
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
import { LOCAL_EMBEDDING_PREFERRED_TOKENS, measureEmbeddingInputs } from "../embedding/provider.js";
|
|
2
|
+
import { buildChunkEmbeddingText, chunkIdentityHash, preTokenizeForFTS } from "./chunker.js";
|
|
3
|
+
const SENTENCE_BOUNDARIES = {
|
|
4
|
+
"。": true,
|
|
5
|
+
"!": true,
|
|
6
|
+
"?": true,
|
|
7
|
+
".": true,
|
|
8
|
+
"!": true,
|
|
9
|
+
"?": true,
|
|
10
|
+
";": true,
|
|
11
|
+
";": true,
|
|
12
|
+
};
|
|
13
|
+
function throwIfAborted(signal) {
|
|
14
|
+
if (signal?.aborted)
|
|
15
|
+
throw new Error("Cancelled");
|
|
16
|
+
}
|
|
17
|
+
function compareOrder(left, right) {
|
|
18
|
+
if (left.rootIndex !== right.rootIndex)
|
|
19
|
+
return left.rootIndex - right.rootIndex;
|
|
20
|
+
const length = Math.min(left.order.length, right.order.length);
|
|
21
|
+
for (let index = 0; index < length; index++) {
|
|
22
|
+
if (left.order[index] !== right.order[index])
|
|
23
|
+
return left.order[index] - right.order[index];
|
|
24
|
+
}
|
|
25
|
+
return left.order.length - right.order.length;
|
|
26
|
+
}
|
|
27
|
+
function nearestStringBoundary(content, desired, boundary, minimum) {
|
|
28
|
+
const before = content.lastIndexOf(boundary, desired);
|
|
29
|
+
if (before >= minimum)
|
|
30
|
+
return before + boundary.length;
|
|
31
|
+
const after = content.indexOf(boundary, desired);
|
|
32
|
+
if (after >= 0 && after <= content.length - minimum)
|
|
33
|
+
return after + boundary.length;
|
|
34
|
+
return undefined;
|
|
35
|
+
}
|
|
36
|
+
function nearestCharacterBoundary(content, desired, minimum, predicate) {
|
|
37
|
+
for (let offset = 0; offset < content.length; offset++) {
|
|
38
|
+
const before = desired - offset;
|
|
39
|
+
if (before >= minimum && predicate(content[before]))
|
|
40
|
+
return before + 1;
|
|
41
|
+
const after = desired + offset;
|
|
42
|
+
if (after < content.length - minimum && predicate(content[after]))
|
|
43
|
+
return after + 1;
|
|
44
|
+
if (before < minimum && after >= content.length - minimum)
|
|
45
|
+
break;
|
|
46
|
+
}
|
|
47
|
+
return undefined;
|
|
48
|
+
}
|
|
49
|
+
function safeCodeUnitBoundary(content, index) {
|
|
50
|
+
if (index <= 0 || index >= content.length)
|
|
51
|
+
return index;
|
|
52
|
+
const previous = content.charCodeAt(index - 1);
|
|
53
|
+
const next = content.charCodeAt(index);
|
|
54
|
+
return previous >= 0xd800 && previous <= 0xdbff && next >= 0xdc00 && next <= 0xdfff ? index - 1 : index;
|
|
55
|
+
}
|
|
56
|
+
function chooseSplitIndex(content, units, measurement) {
|
|
57
|
+
const estimatedParts = Math.max(2, Math.ceil(units / measurement.preferredMaximum));
|
|
58
|
+
const desired = Math.max(1, Math.floor(content.length / estimatedParts));
|
|
59
|
+
const minimum = Math.max(1, Math.min(64, Math.floor(content.length * 0.1)));
|
|
60
|
+
const structuralBoundary = nearestStringBoundary(content, desired, "\n\n", minimum) ??
|
|
61
|
+
nearestStringBoundary(content, desired, "\n", minimum) ??
|
|
62
|
+
nearestCharacterBoundary(content, desired, minimum, (character) => SENTENCE_BOUNDARIES[character] === true) ??
|
|
63
|
+
nearestCharacterBoundary(content, desired, minimum, (character) => /\s/u.test(character));
|
|
64
|
+
const splitIndex = safeCodeUnitBoundary(content, structuralBoundary ?? desired);
|
|
65
|
+
if (splitIndex <= 0 || splitIndex >= content.length) {
|
|
66
|
+
throw new Error(`Unable to split oversized embedding input containing ${content.length} characters`);
|
|
67
|
+
}
|
|
68
|
+
return splitIndex;
|
|
69
|
+
}
|
|
70
|
+
function lineCount(content) {
|
|
71
|
+
let count = 0;
|
|
72
|
+
for (const character of content) {
|
|
73
|
+
if (character === "\n")
|
|
74
|
+
count++;
|
|
75
|
+
}
|
|
76
|
+
return count;
|
|
77
|
+
}
|
|
78
|
+
function rebuildChunk(chunk, content, startLine, endLine) {
|
|
79
|
+
const rebuilt = {
|
|
80
|
+
...chunk,
|
|
81
|
+
content,
|
|
82
|
+
content_hash: chunkIdentityHash({
|
|
83
|
+
content,
|
|
84
|
+
filePath: chunk.file_path,
|
|
85
|
+
fileType: chunk.file_type,
|
|
86
|
+
startLine,
|
|
87
|
+
endLine,
|
|
88
|
+
metadataJson: chunk.metadata_json,
|
|
89
|
+
}),
|
|
90
|
+
content_tokenized: "",
|
|
91
|
+
start_line: startLine,
|
|
92
|
+
end_line: endLine,
|
|
93
|
+
};
|
|
94
|
+
return { ...rebuilt, content_tokenized: preTokenizeForFTS(buildChunkEmbeddingText(rebuilt)) };
|
|
95
|
+
}
|
|
96
|
+
function unsplittableEmbeddingContext(piece, measurement) {
|
|
97
|
+
return new Error(`Embedding context for ${piece.chunk.file_path}:${piece.chunk.start_line}-${piece.chunk.end_line} exceeds ${measurement.maximum} ${measurement.unit} and cannot be split safely`);
|
|
98
|
+
}
|
|
99
|
+
function splitPiece(piece, units, measurement) {
|
|
100
|
+
const content = piece.chunk.content;
|
|
101
|
+
if ([...content].length < 2)
|
|
102
|
+
throw unsplittableEmbeddingContext(piece, measurement);
|
|
103
|
+
const splitIndex = chooseSplitIndex(content, units, measurement);
|
|
104
|
+
const leftRaw = content.slice(0, splitIndex);
|
|
105
|
+
const rightRaw = content.slice(splitIndex);
|
|
106
|
+
const left = leftRaw.trim();
|
|
107
|
+
const right = rightRaw.trim();
|
|
108
|
+
if (!left || !right || left === content || right === content) {
|
|
109
|
+
throw unsplittableEmbeddingContext(piece, measurement);
|
|
110
|
+
}
|
|
111
|
+
const leftLeadingLines = lineCount(leftRaw.slice(0, leftRaw.indexOf(left)));
|
|
112
|
+
const rightLeadingLines = lineCount(rightRaw.slice(0, rightRaw.indexOf(right)));
|
|
113
|
+
const leftStart = piece.chunk.start_line + leftLeadingLines;
|
|
114
|
+
const leftEnd = leftStart + lineCount(left);
|
|
115
|
+
const rightStart = piece.chunk.start_line + lineCount(leftRaw) + rightLeadingLines;
|
|
116
|
+
const rightEnd = rightStart + lineCount(right);
|
|
117
|
+
return [
|
|
118
|
+
{
|
|
119
|
+
chunk: rebuildChunk(piece.chunk, left, leftStart, leftEnd),
|
|
120
|
+
rootIndex: piece.rootIndex,
|
|
121
|
+
order: [...piece.order, 0],
|
|
122
|
+
},
|
|
123
|
+
{
|
|
124
|
+
chunk: rebuildChunk(piece.chunk, right, rightStart, rightEnd),
|
|
125
|
+
rootIndex: piece.rootIndex,
|
|
126
|
+
order: [...piece.order, 1],
|
|
127
|
+
},
|
|
128
|
+
];
|
|
129
|
+
}
|
|
130
|
+
function withPartMetadata(chunk, part, partCount) {
|
|
131
|
+
let metadata = {};
|
|
132
|
+
try {
|
|
133
|
+
const parsed = JSON.parse(chunk.metadata_json);
|
|
134
|
+
if (parsed && typeof parsed === "object" && !Array.isArray(parsed))
|
|
135
|
+
metadata = parsed;
|
|
136
|
+
}
|
|
137
|
+
catch {
|
|
138
|
+
metadata = {};
|
|
139
|
+
}
|
|
140
|
+
const metadataJson = JSON.stringify({ ...metadata, chunk_part: part, chunk_part_count: partCount });
|
|
141
|
+
const rebuilt = { ...chunk, metadata_json: metadataJson, content_tokenized: "" };
|
|
142
|
+
rebuilt.content_hash = chunkIdentityHash({
|
|
143
|
+
content: rebuilt.content,
|
|
144
|
+
filePath: rebuilt.file_path,
|
|
145
|
+
fileType: rebuilt.file_type,
|
|
146
|
+
startLine: rebuilt.start_line,
|
|
147
|
+
endLine: rebuilt.end_line,
|
|
148
|
+
metadataJson,
|
|
149
|
+
});
|
|
150
|
+
rebuilt.content_tokenized = preTokenizeForFTS(buildChunkEmbeddingText(rebuilt));
|
|
151
|
+
return rebuilt;
|
|
152
|
+
}
|
|
153
|
+
function isProvablySafeLocalAsciiInput(text, config) {
|
|
154
|
+
if (config.provider !== "local")
|
|
155
|
+
return false;
|
|
156
|
+
const prefixLength = config.prefixes === "on" ? "passage: ".length : 0;
|
|
157
|
+
if (text.length + prefixLength + 2 > LOCAL_EMBEDDING_PREFERRED_TOKENS)
|
|
158
|
+
return false;
|
|
159
|
+
for (let index = 0; index < text.length; index++) {
|
|
160
|
+
if (text.charCodeAt(index) > 0x7f)
|
|
161
|
+
return false;
|
|
162
|
+
}
|
|
163
|
+
return true;
|
|
164
|
+
}
|
|
165
|
+
export async function finalizeChunksForEmbedding(chunks, config, signal) {
|
|
166
|
+
if (chunks.length === 0)
|
|
167
|
+
return [];
|
|
168
|
+
let pending = chunks.map((chunk, rootIndex) => ({ chunk, rootIndex, order: [] }));
|
|
169
|
+
const accepted = [];
|
|
170
|
+
while (pending.length > 0) {
|
|
171
|
+
throwIfAborted(signal);
|
|
172
|
+
const piecesToMeasure = [];
|
|
173
|
+
const textsToMeasure = [];
|
|
174
|
+
for (const piece of pending) {
|
|
175
|
+
const text = buildChunkEmbeddingText(piece.chunk);
|
|
176
|
+
if (isProvablySafeLocalAsciiInput(text, config))
|
|
177
|
+
accepted.push(piece);
|
|
178
|
+
else {
|
|
179
|
+
piecesToMeasure.push(piece);
|
|
180
|
+
textsToMeasure.push(text);
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
if (piecesToMeasure.length === 0)
|
|
184
|
+
break;
|
|
185
|
+
const measurement = await measureEmbeddingInputs(textsToMeasure, "passage", config, signal);
|
|
186
|
+
if (measurement.counts.length !== piecesToMeasure.length) {
|
|
187
|
+
throw new Error(`Embedding input measurement returned ${measurement.counts.length} counts for ${piecesToMeasure.length} chunks`);
|
|
188
|
+
}
|
|
189
|
+
const next = [];
|
|
190
|
+
for (let index = 0; index < piecesToMeasure.length; index++) {
|
|
191
|
+
const piece = piecesToMeasure[index];
|
|
192
|
+
const units = measurement.counts[index];
|
|
193
|
+
if (units === undefined)
|
|
194
|
+
throw new Error(`Missing embedding input measurement for chunk ${index}`);
|
|
195
|
+
if (units <= measurement.maximum)
|
|
196
|
+
accepted.push(piece);
|
|
197
|
+
else
|
|
198
|
+
next.push(...splitPiece(piece, units, measurement));
|
|
199
|
+
}
|
|
200
|
+
pending = next;
|
|
201
|
+
}
|
|
202
|
+
accepted.sort(compareOrder);
|
|
203
|
+
const partCounts = new Map();
|
|
204
|
+
for (const piece of accepted)
|
|
205
|
+
partCounts.set(piece.rootIndex, (partCounts.get(piece.rootIndex) ?? 0) + 1);
|
|
206
|
+
const partIndexes = new Map();
|
|
207
|
+
return accepted.map((piece) => {
|
|
208
|
+
const partCount = partCounts.get(piece.rootIndex) ?? 1;
|
|
209
|
+
if (partCount === 1)
|
|
210
|
+
return piece.chunk;
|
|
211
|
+
const part = (partIndexes.get(piece.rootIndex) ?? 0) + 1;
|
|
212
|
+
partIndexes.set(piece.rootIndex, part);
|
|
213
|
+
return withPartMetadata(piece.chunk, part, partCount);
|
|
214
|
+
});
|
|
215
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function htmlToStructuredMarkdown(html: string): string;
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
const HTML_ENTITIES = {
|
|
2
|
+
amp: "&",
|
|
3
|
+
apos: "'",
|
|
4
|
+
gt: ">",
|
|
5
|
+
lt: "<",
|
|
6
|
+
nbsp: " ",
|
|
7
|
+
quot: '"',
|
|
8
|
+
ndash: "–",
|
|
9
|
+
mdash: "—",
|
|
10
|
+
hellip: "…",
|
|
11
|
+
};
|
|
12
|
+
function decodeHtmlEntity(entity) {
|
|
13
|
+
if (!entity.startsWith("#"))
|
|
14
|
+
return HTML_ENTITIES[entity.toLowerCase()] ?? `&${entity};`;
|
|
15
|
+
const hexadecimal = entity[1]?.toLowerCase() === "x";
|
|
16
|
+
const digits = entity.slice(hexadecimal ? 2 : 1);
|
|
17
|
+
const codePoint = Number.parseInt(digits, hexadecimal ? 16 : 10);
|
|
18
|
+
if (!digits ||
|
|
19
|
+
!Number.isInteger(codePoint) ||
|
|
20
|
+
codePoint <= 0 ||
|
|
21
|
+
codePoint > 0x10ffff ||
|
|
22
|
+
(codePoint >= 0xd800 && codePoint <= 0xdfff)) {
|
|
23
|
+
return `&${entity};`;
|
|
24
|
+
}
|
|
25
|
+
return String.fromCodePoint(codePoint);
|
|
26
|
+
}
|
|
27
|
+
export function htmlToStructuredMarkdown(html) {
|
|
28
|
+
return html
|
|
29
|
+
.replace(/<!--[\s\S]*?-->/g, "")
|
|
30
|
+
.replace(/<(script|style|noscript|template|svg)\b[^>]*>[\s\S]*?(?:<\/\1\s*>|$)/gi, "")
|
|
31
|
+
.replace(/<h([1-6])\b[^>]*>/gi, (_tag, level) => `\n\n${"#".repeat(Number(level))} `)
|
|
32
|
+
.replace(/<\/h[1-6]\s*>/gi, "\n\n")
|
|
33
|
+
.replace(/<li\b[^>]*>/gi, "\n- ")
|
|
34
|
+
.replace(/<\/li\s*>/gi, "\n")
|
|
35
|
+
.replace(/<br\s*\/?\s*>/gi, "\n")
|
|
36
|
+
.replace(/<(th|td)\b[^>]*>/gi, " | ")
|
|
37
|
+
.replace(/<\/(th|td)\s*>/gi, "")
|
|
38
|
+
.replace(/<tr\b[^>]*>/gi, "\n")
|
|
39
|
+
.replace(/<\/tr\s*>/gi, "\n")
|
|
40
|
+
.replace(/<\/?(?:address|article|aside|blockquote|div|dl|dt|dd|fieldset|figcaption|figure|footer|form|header|hr|main|nav|ol|p|section|table|tbody|thead|tfoot|ul)\b[^>]*>/gi, "\n\n")
|
|
41
|
+
.replace(/<[^>]+>/g, "")
|
|
42
|
+
.replace(/&(#(?:x[0-9a-f]+|\d+)|[a-z][a-z0-9]+);/gi, (_match, entity) => decodeHtmlEntity(entity))
|
|
43
|
+
.replace(/\r\n?/g, "\n")
|
|
44
|
+
.split("\n")
|
|
45
|
+
.map((line) => line.replace(/[\t\f\v ]+/g, " ").trim())
|
|
46
|
+
.join("\n")
|
|
47
|
+
.replace(/\n{3,}/g, "\n\n")
|
|
48
|
+
.trim();
|
|
49
|
+
}
|
|
@@ -3,6 +3,10 @@ export declare function validateModelWorkerNodePath(nodePath: string): string;
|
|
|
3
3
|
export declare function resolveModelWorkerNodePath(): string;
|
|
4
4
|
export declare function configureModelWorkerNodePath(nodePath?: string): string;
|
|
5
5
|
export declare function embedInModelWorker(texts: string[], prefix: "query" | "passage", prefixesOrSignal?: "on" | "off" | AbortSignal, signal?: AbortSignal): Promise<Float32Array[]>;
|
|
6
|
+
export declare function measureEmbeddingInputInModelWorker(texts: string[], prefix: "query" | "passage", prefixes: "on" | "off", signal?: AbortSignal): Promise<{
|
|
7
|
+
counts: number[];
|
|
8
|
+
maximum: number;
|
|
9
|
+
}>;
|
|
6
10
|
export interface RerankWorkerCandidate {
|
|
7
11
|
chunkId: string;
|
|
8
12
|
content: string;
|
|
@@ -187,8 +187,8 @@ function isWorkerResponse(message) {
|
|
|
187
187
|
return !("error" in message) || message.error === undefined || typeof message.error === "string";
|
|
188
188
|
}
|
|
189
189
|
class ModelWorkerTransport {
|
|
190
|
-
child;
|
|
191
190
|
pending = new Map();
|
|
191
|
+
child;
|
|
192
192
|
stderrTail = "";
|
|
193
193
|
exited = false;
|
|
194
194
|
terminalError = null;
|
|
@@ -397,6 +397,22 @@ export async function embedInModelWorker(texts, prefix, prefixesOrSignal, signal
|
|
|
397
397
|
return new Float32Array(vector);
|
|
398
398
|
});
|
|
399
399
|
}
|
|
400
|
+
export async function measureEmbeddingInputInModelWorker(texts, prefix, prefixes, signal) {
|
|
401
|
+
const result = await getWorker().request({ type: "measure_embedding_input", texts, prefix, prefixes }, signal);
|
|
402
|
+
if (typeof result !== "object" ||
|
|
403
|
+
result === null ||
|
|
404
|
+
!("counts" in result) ||
|
|
405
|
+
!Array.isArray(result.counts) ||
|
|
406
|
+
result.counts.length !== texts.length ||
|
|
407
|
+
!result.counts.every((count) => typeof count === "number" && Number.isInteger(count) && count >= 0) ||
|
|
408
|
+
!("maximum" in result) ||
|
|
409
|
+
typeof result.maximum !== "number" ||
|
|
410
|
+
!Number.isInteger(result.maximum) ||
|
|
411
|
+
result.maximum <= 0) {
|
|
412
|
+
throw new Error("Invalid embedding input measurement from worker");
|
|
413
|
+
}
|
|
414
|
+
return { counts: result.counts, maximum: result.maximum };
|
|
415
|
+
}
|
|
400
416
|
export async function rerankInModelWorker(query, candidates, topK, reranker, signal) {
|
|
401
417
|
const result = await getWorker().request({ type: "rerank", query, candidates, topK, reranker }, signal);
|
|
402
418
|
if (!Array.isArray(result))
|
package/dist/src/model-worker.js
CHANGED
|
@@ -4,6 +4,7 @@ import { DEFAULT_RERANKER_REMOTE_HOST, DEFAULT_RERANKER_REMOTE_PATH_TEMPLATE, is
|
|
|
4
4
|
import { rerankWithSingleRawLogit } from "./search/reranker-logits.js";
|
|
5
5
|
import { getDefaultKnowledgeDir } from "./storage/sqlite.js";
|
|
6
6
|
let embeddingPipeline = null;
|
|
7
|
+
let embeddingPipelinePromise = null;
|
|
7
8
|
let rerankerPipeline = null;
|
|
8
9
|
function getModelCacheDir() {
|
|
9
10
|
return process.env.PI_KNOWLEDGE_MODEL_CACHE_DIR ?? join(getDefaultKnowledgeDir(), "models");
|
|
@@ -24,15 +25,24 @@ function configureEmbeddingTransformersEnv(env) {
|
|
|
24
25
|
async function loadEmbeddingPipeline() {
|
|
25
26
|
if (embeddingPipeline)
|
|
26
27
|
return embeddingPipeline;
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
28
|
+
if (embeddingPipelinePromise)
|
|
29
|
+
return embeddingPipelinePromise;
|
|
30
|
+
embeddingPipelinePromise = (async () => {
|
|
31
|
+
const { pipeline, env } = await import("@huggingface/transformers");
|
|
32
|
+
configureEmbeddingTransformersEnv(env);
|
|
33
|
+
const createPipeline = pipeline;
|
|
34
|
+
return (await createPipeline("feature-extraction", "Xenova/multilingual-e5-small", {
|
|
35
|
+
quantized: true,
|
|
36
|
+
dtype: "fp32",
|
|
37
|
+
}));
|
|
38
|
+
})();
|
|
39
|
+
try {
|
|
40
|
+
embeddingPipeline = await embeddingPipelinePromise;
|
|
41
|
+
return embeddingPipeline;
|
|
42
|
+
}
|
|
43
|
+
finally {
|
|
44
|
+
embeddingPipelinePromise = null;
|
|
45
|
+
}
|
|
36
46
|
}
|
|
37
47
|
async function loadRerankerPipeline(config) {
|
|
38
48
|
const key = rerankerCacheKey(config);
|
|
@@ -81,6 +91,34 @@ async function handleEmbed(request) {
|
|
|
81
91
|
}
|
|
82
92
|
return vectors;
|
|
83
93
|
}
|
|
94
|
+
function embeddingMaximumTokens(tokenizer) {
|
|
95
|
+
const configured = tokenizer.model_max_length;
|
|
96
|
+
return typeof configured === "number" && Number.isInteger(configured) && configured > 0 && configured <= 1_000_000
|
|
97
|
+
? configured
|
|
98
|
+
: 512;
|
|
99
|
+
}
|
|
100
|
+
async function handleMeasureEmbeddingInput(request) {
|
|
101
|
+
const pipe = await loadEmbeddingPipeline();
|
|
102
|
+
const texts = request.texts.map((text) => applyEmbeddingPrefix(text, request.prefix, request.prefixes));
|
|
103
|
+
const tokenized = pipe.tokenizer(texts, {
|
|
104
|
+
add_special_tokens: true,
|
|
105
|
+
truncation: false,
|
|
106
|
+
padding: false,
|
|
107
|
+
return_tensor: false,
|
|
108
|
+
});
|
|
109
|
+
const inputIds = tokenized.input_ids;
|
|
110
|
+
let counts;
|
|
111
|
+
if (texts.length === 1 && inputIds.every((token) => typeof token === "number")) {
|
|
112
|
+
counts = [inputIds.length];
|
|
113
|
+
}
|
|
114
|
+
else if (inputIds.every((tokens) => Array.isArray(tokens))) {
|
|
115
|
+
counts = inputIds.map((tokens) => tokens.length);
|
|
116
|
+
}
|
|
117
|
+
else {
|
|
118
|
+
counts = texts.map((text) => pipe.tokenizer.encode(text, { add_special_tokens: true }).length);
|
|
119
|
+
}
|
|
120
|
+
return { counts, maximum: embeddingMaximumTokens(pipe.tokenizer) };
|
|
121
|
+
}
|
|
84
122
|
async function handleRerank(request) {
|
|
85
123
|
const pipe = await loadRerankerPipeline(request.reranker);
|
|
86
124
|
const results = [];
|
|
@@ -110,17 +148,23 @@ function parseWorkerRequest(message) {
|
|
|
110
148
|
}
|
|
111
149
|
if (!("type" in message))
|
|
112
150
|
throw new Error("Invalid worker request type");
|
|
113
|
-
if (message.type === "embed") {
|
|
151
|
+
if (message.type === "embed" || message.type === "measure_embedding_input") {
|
|
114
152
|
if (!("texts" in message) || !isStringArray(message.texts))
|
|
115
|
-
throw new Error("Invalid
|
|
153
|
+
throw new Error("Invalid embedding texts");
|
|
116
154
|
if (!("prefix" in message) || (message.prefix !== "query" && message.prefix !== "passage")) {
|
|
117
|
-
throw new Error("Invalid
|
|
155
|
+
throw new Error("Invalid embedding prefix");
|
|
118
156
|
}
|
|
119
157
|
const prefixes = "prefixes" in message && (message.prefixes === "on" || message.prefixes === "off") ? message.prefixes : "on";
|
|
120
158
|
if ("prefixes" in message && message.prefixes !== "on" && message.prefixes !== "off") {
|
|
121
|
-
throw new Error("Invalid
|
|
159
|
+
throw new Error("Invalid embedding prefix strategy");
|
|
122
160
|
}
|
|
123
|
-
return {
|
|
161
|
+
return {
|
|
162
|
+
id: message.id,
|
|
163
|
+
type: message.type,
|
|
164
|
+
texts: message.texts,
|
|
165
|
+
prefix: message.prefix,
|
|
166
|
+
prefixes,
|
|
167
|
+
};
|
|
124
168
|
}
|
|
125
169
|
if (message.type === "rerank") {
|
|
126
170
|
if (!("query" in message) || typeof message.query !== "string")
|
|
@@ -151,7 +195,11 @@ async function handleRequestMessage(message) {
|
|
|
151
195
|
requestId = message.id;
|
|
152
196
|
}
|
|
153
197
|
const request = parseWorkerRequest(message);
|
|
154
|
-
const result = request.type === "embed"
|
|
198
|
+
const result = request.type === "embed"
|
|
199
|
+
? await handleEmbed(request)
|
|
200
|
+
: request.type === "measure_embedding_input"
|
|
201
|
+
? await handleMeasureEmbeddingInput(request)
|
|
202
|
+
: await handleRerank(request);
|
|
155
203
|
return { id: request.id, result };
|
|
156
204
|
}
|
|
157
205
|
catch (error) {
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { Chunk } from "../storage/sqlite.ts";
|
|
2
|
+
import type { RankingDiagnostics } from "./ranking.ts";
|
|
3
|
+
export type DiversityMode = "off" | "balanced" | "strong" | undefined;
|
|
4
|
+
export interface RankedChunk {
|
|
5
|
+
chunk: Chunk;
|
|
6
|
+
kbName: string;
|
|
7
|
+
ranking?: RankingDiagnostics;
|
|
8
|
+
score: number;
|
|
9
|
+
content: string;
|
|
10
|
+
snippet: string;
|
|
11
|
+
startLine: number;
|
|
12
|
+
endLine: number;
|
|
13
|
+
sourceChunkIds: string[];
|
|
14
|
+
}
|
|
15
|
+
export declare function diversifyRankedChunks(candidates: RankedChunk[], diversity: DiversityMode, vectorsByChunkId?: Map<string, Float32Array>, signal?: AbortSignal): Promise<RankedChunk[]>;
|
|
16
|
+
export declare function interleaveByFile(candidates: RankedChunk[], diversity: DiversityMode, signal?: AbortSignal): RankedChunk[];
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
import { setImmediate as yieldToEventLoop } from "node:timers/promises";
|
|
2
|
+
import { tokenizeForSearch } from "./query.js";
|
|
3
|
+
const VECTOR_REDUNDANCY_WEIGHT = 0.35;
|
|
4
|
+
const PAIR_CHECK_INTERVAL = 1_024;
|
|
5
|
+
const CPU_SLICE_MS = 8;
|
|
6
|
+
function throwIfAborted(signal) {
|
|
7
|
+
if (signal?.aborted)
|
|
8
|
+
throw new Error("Cancelled");
|
|
9
|
+
}
|
|
10
|
+
function cosineSimilarity(a, b) {
|
|
11
|
+
if (!a || !b || a.length !== b.length)
|
|
12
|
+
return 0;
|
|
13
|
+
let dot = 0;
|
|
14
|
+
for (let i = 0; i < a.length; i++)
|
|
15
|
+
dot += a[i] * b[i];
|
|
16
|
+
return dot;
|
|
17
|
+
}
|
|
18
|
+
function jaccardSimilarity(a, b) {
|
|
19
|
+
if (a.size === 0 || b.size === 0)
|
|
20
|
+
return 0;
|
|
21
|
+
const [small, large] = a.size <= b.size ? [a, b] : [b, a];
|
|
22
|
+
let intersection = 0;
|
|
23
|
+
for (const token of small) {
|
|
24
|
+
if (large.has(token))
|
|
25
|
+
intersection++;
|
|
26
|
+
}
|
|
27
|
+
return intersection / (a.size + b.size - intersection);
|
|
28
|
+
}
|
|
29
|
+
function proximityMeta(candidate) {
|
|
30
|
+
return {
|
|
31
|
+
filePath: candidate.chunk.file_path,
|
|
32
|
+
kbId: candidate.chunk.kb_id,
|
|
33
|
+
startLine: candidate.startLine,
|
|
34
|
+
endLine: candidate.endLine,
|
|
35
|
+
sourceIds: new Set(candidate.sourceChunkIds),
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
function lineProximity(a, b) {
|
|
39
|
+
if (a.filePath !== b.filePath || a.kbId !== b.kbId)
|
|
40
|
+
return 0;
|
|
41
|
+
const [small, large] = a.sourceIds.size <= b.sourceIds.size ? [a.sourceIds, b.sourceIds] : [b.sourceIds, a.sourceIds];
|
|
42
|
+
for (const id of small) {
|
|
43
|
+
if (large.has(id))
|
|
44
|
+
return 1;
|
|
45
|
+
}
|
|
46
|
+
const overlap = Math.max(0, Math.min(a.endLine, b.endLine) - Math.max(a.startLine, b.startLine) + 1);
|
|
47
|
+
if (overlap > 0)
|
|
48
|
+
return 1;
|
|
49
|
+
const gap = Math.max(a.startLine, b.startLine) - Math.min(a.endLine, b.endLine);
|
|
50
|
+
if (gap <= 20)
|
|
51
|
+
return 0.8;
|
|
52
|
+
if (gap <= 80)
|
|
53
|
+
return 0.45;
|
|
54
|
+
return 0.2;
|
|
55
|
+
}
|
|
56
|
+
function normalizeScores(candidates) {
|
|
57
|
+
let min = Number.POSITIVE_INFINITY;
|
|
58
|
+
let max = Number.NEGATIVE_INFINITY;
|
|
59
|
+
for (const candidate of candidates) {
|
|
60
|
+
if (candidate.score < min)
|
|
61
|
+
min = candidate.score;
|
|
62
|
+
if (candidate.score > max)
|
|
63
|
+
max = candidate.score;
|
|
64
|
+
}
|
|
65
|
+
const normalized = new Float64Array(candidates.length);
|
|
66
|
+
for (let i = 0; i < candidates.length; i++) {
|
|
67
|
+
normalized[i] = max === min ? 1 : (candidates[i].score - min) / (max - min);
|
|
68
|
+
}
|
|
69
|
+
return normalized;
|
|
70
|
+
}
|
|
71
|
+
function firstUnusedIndex(used) {
|
|
72
|
+
for (let i = 0; i < used.length; i++) {
|
|
73
|
+
if (used[i] === 0)
|
|
74
|
+
return i;
|
|
75
|
+
}
|
|
76
|
+
throw new Error("Diversity selection exhausted its candidate pool unexpectedly");
|
|
77
|
+
}
|
|
78
|
+
export async function diversifyRankedChunks(candidates, diversity, vectorsByChunkId = new Map(), signal) {
|
|
79
|
+
throwIfAborted(signal);
|
|
80
|
+
if (diversity === "off" || candidates.length <= 2)
|
|
81
|
+
return candidates;
|
|
82
|
+
const lambda = diversity === "strong" ? 0.62 : 0.76;
|
|
83
|
+
const n = candidates.length;
|
|
84
|
+
const tokenSets = new Array(n);
|
|
85
|
+
const vectors = new Array(n);
|
|
86
|
+
const metas = new Array(n);
|
|
87
|
+
const relevance = normalizeScores(candidates);
|
|
88
|
+
for (let i = 0; i < n; i++) {
|
|
89
|
+
const candidate = candidates[i];
|
|
90
|
+
tokenSets[i] = tokenizeForSearch(candidate.content);
|
|
91
|
+
vectors[i] = vectorsByChunkId.get(candidate.chunk.id);
|
|
92
|
+
metas[i] = proximityMeta(candidate);
|
|
93
|
+
}
|
|
94
|
+
const pairRedundancy = (a, b) => {
|
|
95
|
+
let best = lineProximity(metas[a], metas[b]);
|
|
96
|
+
const lexical = jaccardSimilarity(tokenSets[a], tokenSets[b]);
|
|
97
|
+
if (lexical > best)
|
|
98
|
+
best = lexical;
|
|
99
|
+
const va = vectors[a];
|
|
100
|
+
const vb = vectors[b];
|
|
101
|
+
if (va !== undefined && vb !== undefined) {
|
|
102
|
+
const vector = Math.max(0, cosineSimilarity(va, vb)) * VECTOR_REDUNDANCY_WEIGHT;
|
|
103
|
+
if (vector > best)
|
|
104
|
+
best = vector;
|
|
105
|
+
}
|
|
106
|
+
return best;
|
|
107
|
+
};
|
|
108
|
+
const redundancy = new Float64Array(n);
|
|
109
|
+
const used = new Uint8Array(n);
|
|
110
|
+
const selected = new Array(n);
|
|
111
|
+
let pairsSinceCheck = 0;
|
|
112
|
+
let sliceStartedAt = performance.now();
|
|
113
|
+
for (let round = 0; round < n; round++) {
|
|
114
|
+
throwIfAborted(signal);
|
|
115
|
+
let bestIndex = firstUnusedIndex(used);
|
|
116
|
+
let bestScore = Number.NEGATIVE_INFINITY;
|
|
117
|
+
for (let i = 0; i < n; i++) {
|
|
118
|
+
if (used[i] === 1)
|
|
119
|
+
continue;
|
|
120
|
+
const mmrScore = lambda * relevance[i] - (1 - lambda) * redundancy[i];
|
|
121
|
+
if (mmrScore > bestScore) {
|
|
122
|
+
bestScore = mmrScore;
|
|
123
|
+
bestIndex = i;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
used[bestIndex] = 1;
|
|
127
|
+
selected[round] = candidates[bestIndex];
|
|
128
|
+
for (let i = 0; i < n; i++) {
|
|
129
|
+
if (used[i] === 1)
|
|
130
|
+
continue;
|
|
131
|
+
const pair = pairRedundancy(bestIndex, i);
|
|
132
|
+
if (pair > redundancy[i])
|
|
133
|
+
redundancy[i] = pair;
|
|
134
|
+
pairsSinceCheck++;
|
|
135
|
+
if (pairsSinceCheck < PAIR_CHECK_INTERVAL)
|
|
136
|
+
continue;
|
|
137
|
+
pairsSinceCheck = 0;
|
|
138
|
+
throwIfAborted(signal);
|
|
139
|
+
if (performance.now() - sliceStartedAt < CPU_SLICE_MS)
|
|
140
|
+
continue;
|
|
141
|
+
await yieldToEventLoop();
|
|
142
|
+
throwIfAborted(signal);
|
|
143
|
+
sliceStartedAt = performance.now();
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
return selected;
|
|
147
|
+
}
|
|
148
|
+
export function interleaveByFile(candidates, diversity, signal) {
|
|
149
|
+
throwIfAborted(signal);
|
|
150
|
+
if (diversity === "off" || candidates.length <= 2)
|
|
151
|
+
return candidates;
|
|
152
|
+
const buckets = new Map();
|
|
153
|
+
const fileOrder = [];
|
|
154
|
+
for (const candidate of candidates) {
|
|
155
|
+
throwIfAborted(signal);
|
|
156
|
+
const key = `${candidate.chunk.kb_id}:${candidate.chunk.file_path}`;
|
|
157
|
+
let bucket = buckets.get(key);
|
|
158
|
+
if (!bucket) {
|
|
159
|
+
bucket = [];
|
|
160
|
+
buckets.set(key, bucket);
|
|
161
|
+
fileOrder.push(key);
|
|
162
|
+
}
|
|
163
|
+
bucket.push(candidate);
|
|
164
|
+
}
|
|
165
|
+
if (fileOrder.length <= 1)
|
|
166
|
+
return candidates;
|
|
167
|
+
const result = [];
|
|
168
|
+
let round = 0;
|
|
169
|
+
while (result.length < candidates.length) {
|
|
170
|
+
let added = false;
|
|
171
|
+
for (const key of fileOrder) {
|
|
172
|
+
const item = buckets.get(key)?.[round];
|
|
173
|
+
if (!item)
|
|
174
|
+
continue;
|
|
175
|
+
result.push(item);
|
|
176
|
+
added = true;
|
|
177
|
+
}
|
|
178
|
+
if (!added)
|
|
179
|
+
break;
|
|
180
|
+
round++;
|
|
181
|
+
}
|
|
182
|
+
return result;
|
|
183
|
+
}
|
|
@@ -22,6 +22,11 @@ export interface SearchTuningSummary {
|
|
|
22
22
|
snippet_max_length: number;
|
|
23
23
|
min_hybrid_score: number;
|
|
24
24
|
candidate_limit: number;
|
|
25
|
+
retrieval_limit_per_kb: number;
|
|
26
|
+
retrieved_candidates: number;
|
|
27
|
+
filtered_candidates: number;
|
|
28
|
+
diversity_candidates: number;
|
|
29
|
+
candidate_pool_truncated: boolean;
|
|
25
30
|
candidate_min: number;
|
|
26
31
|
candidate_multiplier: number;
|
|
27
32
|
adaptive_context_lines: number;
|
|
@@ -36,6 +41,11 @@ interface SearchTuningInput {
|
|
|
36
41
|
profile?: SearchProfile;
|
|
37
42
|
kbSourceTypes: string[];
|
|
38
43
|
}
|
|
44
|
+
export interface SearchWindow {
|
|
45
|
+
limit: number;
|
|
46
|
+
offset: number;
|
|
47
|
+
}
|
|
48
|
+
export declare function resolveSearchWindow(limit: number | undefined, offset: number | undefined, defaultLimit: number): SearchWindow;
|
|
39
49
|
export declare function resolveSearchTuning(input: SearchTuningInput): SearchTuning;
|
|
40
50
|
export declare function summarizeSearchTuning(tuning: SearchTuning, limit: number, candidateLimit: number): SearchTuningSummary;
|
|
41
51
|
export {};
|