pi-knowledge 0.11.0 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,215 @@
1
+ import { LOCAL_EMBEDDING_PREFERRED_TOKENS, measureEmbeddingInputs } from "../embedding/provider.js";
2
+ import { buildChunkEmbeddingText, chunkIdentityHash, preTokenizeForFTS } from "./chunker.js";
3
+ const SENTENCE_BOUNDARIES = {
4
+ "。": true,
5
+ "!": true,
6
+ "?": true,
7
+ ".": true,
8
+ "!": true,
9
+ "?": true,
10
+ ";": true,
11
+ ";": true,
12
+ };
13
+ function throwIfAborted(signal) {
14
+ if (signal?.aborted)
15
+ throw new Error("Cancelled");
16
+ }
17
+ function compareOrder(left, right) {
18
+ if (left.rootIndex !== right.rootIndex)
19
+ return left.rootIndex - right.rootIndex;
20
+ const length = Math.min(left.order.length, right.order.length);
21
+ for (let index = 0; index < length; index++) {
22
+ if (left.order[index] !== right.order[index])
23
+ return left.order[index] - right.order[index];
24
+ }
25
+ return left.order.length - right.order.length;
26
+ }
27
+ function nearestStringBoundary(content, desired, boundary, minimum) {
28
+ const before = content.lastIndexOf(boundary, desired);
29
+ if (before >= minimum)
30
+ return before + boundary.length;
31
+ const after = content.indexOf(boundary, desired);
32
+ if (after >= 0 && after <= content.length - minimum)
33
+ return after + boundary.length;
34
+ return undefined;
35
+ }
36
+ function nearestCharacterBoundary(content, desired, minimum, predicate) {
37
+ for (let offset = 0; offset < content.length; offset++) {
38
+ const before = desired - offset;
39
+ if (before >= minimum && predicate(content[before]))
40
+ return before + 1;
41
+ const after = desired + offset;
42
+ if (after < content.length - minimum && predicate(content[after]))
43
+ return after + 1;
44
+ if (before < minimum && after >= content.length - minimum)
45
+ break;
46
+ }
47
+ return undefined;
48
+ }
49
+ function safeCodeUnitBoundary(content, index) {
50
+ if (index <= 0 || index >= content.length)
51
+ return index;
52
+ const previous = content.charCodeAt(index - 1);
53
+ const next = content.charCodeAt(index);
54
+ return previous >= 0xd800 && previous <= 0xdbff && next >= 0xdc00 && next <= 0xdfff ? index - 1 : index;
55
+ }
56
+ function chooseSplitIndex(content, units, measurement) {
57
+ const estimatedParts = Math.max(2, Math.ceil(units / measurement.preferredMaximum));
58
+ const desired = Math.max(1, Math.floor(content.length / estimatedParts));
59
+ const minimum = Math.max(1, Math.min(64, Math.floor(content.length * 0.1)));
60
+ const structuralBoundary = nearestStringBoundary(content, desired, "\n\n", minimum) ??
61
+ nearestStringBoundary(content, desired, "\n", minimum) ??
62
+ nearestCharacterBoundary(content, desired, minimum, (character) => SENTENCE_BOUNDARIES[character] === true) ??
63
+ nearestCharacterBoundary(content, desired, minimum, (character) => /\s/u.test(character));
64
+ const splitIndex = safeCodeUnitBoundary(content, structuralBoundary ?? desired);
65
+ if (splitIndex <= 0 || splitIndex >= content.length) {
66
+ throw new Error(`Unable to split oversized embedding input containing ${content.length} characters`);
67
+ }
68
+ return splitIndex;
69
+ }
70
+ function lineCount(content) {
71
+ let count = 0;
72
+ for (const character of content) {
73
+ if (character === "\n")
74
+ count++;
75
+ }
76
+ return count;
77
+ }
78
+ function rebuildChunk(chunk, content, startLine, endLine) {
79
+ const rebuilt = {
80
+ ...chunk,
81
+ content,
82
+ content_hash: chunkIdentityHash({
83
+ content,
84
+ filePath: chunk.file_path,
85
+ fileType: chunk.file_type,
86
+ startLine,
87
+ endLine,
88
+ metadataJson: chunk.metadata_json,
89
+ }),
90
+ content_tokenized: "",
91
+ start_line: startLine,
92
+ end_line: endLine,
93
+ };
94
+ return { ...rebuilt, content_tokenized: preTokenizeForFTS(buildChunkEmbeddingText(rebuilt)) };
95
+ }
96
+ function unsplittableEmbeddingContext(piece, measurement) {
97
+ return new Error(`Embedding context for ${piece.chunk.file_path}:${piece.chunk.start_line}-${piece.chunk.end_line} exceeds ${measurement.maximum} ${measurement.unit} and cannot be split safely`);
98
+ }
99
+ function splitPiece(piece, units, measurement) {
100
+ const content = piece.chunk.content;
101
+ if ([...content].length < 2)
102
+ throw unsplittableEmbeddingContext(piece, measurement);
103
+ const splitIndex = chooseSplitIndex(content, units, measurement);
104
+ const leftRaw = content.slice(0, splitIndex);
105
+ const rightRaw = content.slice(splitIndex);
106
+ const left = leftRaw.trim();
107
+ const right = rightRaw.trim();
108
+ if (!left || !right || left === content || right === content) {
109
+ throw unsplittableEmbeddingContext(piece, measurement);
110
+ }
111
+ const leftLeadingLines = lineCount(leftRaw.slice(0, leftRaw.indexOf(left)));
112
+ const rightLeadingLines = lineCount(rightRaw.slice(0, rightRaw.indexOf(right)));
113
+ const leftStart = piece.chunk.start_line + leftLeadingLines;
114
+ const leftEnd = leftStart + lineCount(left);
115
+ const rightStart = piece.chunk.start_line + lineCount(leftRaw) + rightLeadingLines;
116
+ const rightEnd = rightStart + lineCount(right);
117
+ return [
118
+ {
119
+ chunk: rebuildChunk(piece.chunk, left, leftStart, leftEnd),
120
+ rootIndex: piece.rootIndex,
121
+ order: [...piece.order, 0],
122
+ },
123
+ {
124
+ chunk: rebuildChunk(piece.chunk, right, rightStart, rightEnd),
125
+ rootIndex: piece.rootIndex,
126
+ order: [...piece.order, 1],
127
+ },
128
+ ];
129
+ }
130
+ function withPartMetadata(chunk, part, partCount) {
131
+ let metadata = {};
132
+ try {
133
+ const parsed = JSON.parse(chunk.metadata_json);
134
+ if (parsed && typeof parsed === "object" && !Array.isArray(parsed))
135
+ metadata = parsed;
136
+ }
137
+ catch {
138
+ metadata = {};
139
+ }
140
+ const metadataJson = JSON.stringify({ ...metadata, chunk_part: part, chunk_part_count: partCount });
141
+ const rebuilt = { ...chunk, metadata_json: metadataJson, content_tokenized: "" };
142
+ rebuilt.content_hash = chunkIdentityHash({
143
+ content: rebuilt.content,
144
+ filePath: rebuilt.file_path,
145
+ fileType: rebuilt.file_type,
146
+ startLine: rebuilt.start_line,
147
+ endLine: rebuilt.end_line,
148
+ metadataJson,
149
+ });
150
+ rebuilt.content_tokenized = preTokenizeForFTS(buildChunkEmbeddingText(rebuilt));
151
+ return rebuilt;
152
+ }
153
+ function isProvablySafeLocalAsciiInput(text, config) {
154
+ if (config.provider !== "local")
155
+ return false;
156
+ const prefixLength = config.prefixes === "on" ? "passage: ".length : 0;
157
+ if (text.length + prefixLength + 2 > LOCAL_EMBEDDING_PREFERRED_TOKENS)
158
+ return false;
159
+ for (let index = 0; index < text.length; index++) {
160
+ if (text.charCodeAt(index) > 0x7f)
161
+ return false;
162
+ }
163
+ return true;
164
+ }
165
+ export async function finalizeChunksForEmbedding(chunks, config, signal) {
166
+ if (chunks.length === 0)
167
+ return [];
168
+ let pending = chunks.map((chunk, rootIndex) => ({ chunk, rootIndex, order: [] }));
169
+ const accepted = [];
170
+ while (pending.length > 0) {
171
+ throwIfAborted(signal);
172
+ const piecesToMeasure = [];
173
+ const textsToMeasure = [];
174
+ for (const piece of pending) {
175
+ const text = buildChunkEmbeddingText(piece.chunk);
176
+ if (isProvablySafeLocalAsciiInput(text, config))
177
+ accepted.push(piece);
178
+ else {
179
+ piecesToMeasure.push(piece);
180
+ textsToMeasure.push(text);
181
+ }
182
+ }
183
+ if (piecesToMeasure.length === 0)
184
+ break;
185
+ const measurement = await measureEmbeddingInputs(textsToMeasure, "passage", config, signal);
186
+ if (measurement.counts.length !== piecesToMeasure.length) {
187
+ throw new Error(`Embedding input measurement returned ${measurement.counts.length} counts for ${piecesToMeasure.length} chunks`);
188
+ }
189
+ const next = [];
190
+ for (let index = 0; index < piecesToMeasure.length; index++) {
191
+ const piece = piecesToMeasure[index];
192
+ const units = measurement.counts[index];
193
+ if (units === undefined)
194
+ throw new Error(`Missing embedding input measurement for chunk ${index}`);
195
+ if (units <= measurement.maximum)
196
+ accepted.push(piece);
197
+ else
198
+ next.push(...splitPiece(piece, units, measurement));
199
+ }
200
+ pending = next;
201
+ }
202
+ accepted.sort(compareOrder);
203
+ const partCounts = new Map();
204
+ for (const piece of accepted)
205
+ partCounts.set(piece.rootIndex, (partCounts.get(piece.rootIndex) ?? 0) + 1);
206
+ const partIndexes = new Map();
207
+ return accepted.map((piece) => {
208
+ const partCount = partCounts.get(piece.rootIndex) ?? 1;
209
+ if (partCount === 1)
210
+ return piece.chunk;
211
+ const part = (partIndexes.get(piece.rootIndex) ?? 0) + 1;
212
+ partIndexes.set(piece.rootIndex, part);
213
+ return withPartMetadata(piece.chunk, part, partCount);
214
+ });
215
+ }
@@ -0,0 +1 @@
1
+ export declare function htmlToStructuredMarkdown(html: string): string;
@@ -0,0 +1,49 @@
1
+ const HTML_ENTITIES = {
2
+ amp: "&",
3
+ apos: "'",
4
+ gt: ">",
5
+ lt: "<",
6
+ nbsp: " ",
7
+ quot: '"',
8
+ ndash: "–",
9
+ mdash: "—",
10
+ hellip: "…",
11
+ };
12
+ function decodeHtmlEntity(entity) {
13
+ if (!entity.startsWith("#"))
14
+ return HTML_ENTITIES[entity.toLowerCase()] ?? `&${entity};`;
15
+ const hexadecimal = entity[1]?.toLowerCase() === "x";
16
+ const digits = entity.slice(hexadecimal ? 2 : 1);
17
+ const codePoint = Number.parseInt(digits, hexadecimal ? 16 : 10);
18
+ if (!digits ||
19
+ !Number.isInteger(codePoint) ||
20
+ codePoint <= 0 ||
21
+ codePoint > 0x10ffff ||
22
+ (codePoint >= 0xd800 && codePoint <= 0xdfff)) {
23
+ return `&${entity};`;
24
+ }
25
+ return String.fromCodePoint(codePoint);
26
+ }
27
+ export function htmlToStructuredMarkdown(html) {
28
+ return html
29
+ .replace(/<!--[\s\S]*?-->/g, "")
30
+ .replace(/<(script|style|noscript|template|svg)\b[^>]*>[\s\S]*?(?:<\/\1\s*>|$)/gi, "")
31
+ .replace(/<h([1-6])\b[^>]*>/gi, (_tag, level) => `\n\n${"#".repeat(Number(level))} `)
32
+ .replace(/<\/h[1-6]\s*>/gi, "\n\n")
33
+ .replace(/<li\b[^>]*>/gi, "\n- ")
34
+ .replace(/<\/li\s*>/gi, "\n")
35
+ .replace(/<br\s*\/?\s*>/gi, "\n")
36
+ .replace(/<(th|td)\b[^>]*>/gi, " | ")
37
+ .replace(/<\/(th|td)\s*>/gi, "")
38
+ .replace(/<tr\b[^>]*>/gi, "\n")
39
+ .replace(/<\/tr\s*>/gi, "\n")
40
+ .replace(/<\/?(?:address|article|aside|blockquote|div|dl|dt|dd|fieldset|figcaption|figure|footer|form|header|hr|main|nav|ol|p|section|table|tbody|thead|tfoot|ul)\b[^>]*>/gi, "\n\n")
41
+ .replace(/<[^>]+>/g, "")
42
+ .replace(/&(#(?:x[0-9a-f]+|\d+)|[a-z][a-z0-9]+);/gi, (_match, entity) => decodeHtmlEntity(entity))
43
+ .replace(/\r\n?/g, "\n")
44
+ .split("\n")
45
+ .map((line) => line.replace(/[\t\f\v ]+/g, " ").trim())
46
+ .join("\n")
47
+ .replace(/\n{3,}/g, "\n\n")
48
+ .trim();
49
+ }
@@ -3,6 +3,10 @@ export declare function validateModelWorkerNodePath(nodePath: string): string;
3
3
  export declare function resolveModelWorkerNodePath(): string;
4
4
  export declare function configureModelWorkerNodePath(nodePath?: string): string;
5
5
  export declare function embedInModelWorker(texts: string[], prefix: "query" | "passage", prefixesOrSignal?: "on" | "off" | AbortSignal, signal?: AbortSignal): Promise<Float32Array[]>;
6
+ export declare function measureEmbeddingInputInModelWorker(texts: string[], prefix: "query" | "passage", prefixes: "on" | "off", signal?: AbortSignal): Promise<{
7
+ counts: number[];
8
+ maximum: number;
9
+ }>;
6
10
  export interface RerankWorkerCandidate {
7
11
  chunkId: string;
8
12
  content: string;
@@ -187,8 +187,8 @@ function isWorkerResponse(message) {
187
187
  return !("error" in message) || message.error === undefined || typeof message.error === "string";
188
188
  }
189
189
  class ModelWorkerTransport {
190
- child;
191
190
  pending = new Map();
191
+ child;
192
192
  stderrTail = "";
193
193
  exited = false;
194
194
  terminalError = null;
@@ -397,6 +397,22 @@ export async function embedInModelWorker(texts, prefix, prefixesOrSignal, signal
397
397
  return new Float32Array(vector);
398
398
  });
399
399
  }
400
+ export async function measureEmbeddingInputInModelWorker(texts, prefix, prefixes, signal) {
401
+ const result = await getWorker().request({ type: "measure_embedding_input", texts, prefix, prefixes }, signal);
402
+ if (typeof result !== "object" ||
403
+ result === null ||
404
+ !("counts" in result) ||
405
+ !Array.isArray(result.counts) ||
406
+ result.counts.length !== texts.length ||
407
+ !result.counts.every((count) => typeof count === "number" && Number.isInteger(count) && count >= 0) ||
408
+ !("maximum" in result) ||
409
+ typeof result.maximum !== "number" ||
410
+ !Number.isInteger(result.maximum) ||
411
+ result.maximum <= 0) {
412
+ throw new Error("Invalid embedding input measurement from worker");
413
+ }
414
+ return { counts: result.counts, maximum: result.maximum };
415
+ }
400
416
  export async function rerankInModelWorker(query, candidates, topK, reranker, signal) {
401
417
  const result = await getWorker().request({ type: "rerank", query, candidates, topK, reranker }, signal);
402
418
  if (!Array.isArray(result))
@@ -4,6 +4,7 @@ import { DEFAULT_RERANKER_REMOTE_HOST, DEFAULT_RERANKER_REMOTE_PATH_TEMPLATE, is
4
4
  import { rerankWithSingleRawLogit } from "./search/reranker-logits.js";
5
5
  import { getDefaultKnowledgeDir } from "./storage/sqlite.js";
6
6
  let embeddingPipeline = null;
7
+ let embeddingPipelinePromise = null;
7
8
  let rerankerPipeline = null;
8
9
  function getModelCacheDir() {
9
10
  return process.env.PI_KNOWLEDGE_MODEL_CACHE_DIR ?? join(getDefaultKnowledgeDir(), "models");
@@ -24,15 +25,24 @@ function configureEmbeddingTransformersEnv(env) {
24
25
  async function loadEmbeddingPipeline() {
25
26
  if (embeddingPipeline)
26
27
  return embeddingPipeline;
27
- const { pipeline, env } = await import("@huggingface/transformers");
28
- configureEmbeddingTransformersEnv(env);
29
- const createPipeline = pipeline;
30
- const loaded = (await createPipeline("feature-extraction", "Xenova/multilingual-e5-small", {
31
- quantized: true,
32
- dtype: "fp32",
33
- }));
34
- embeddingPipeline = loaded;
35
- return loaded;
28
+ if (embeddingPipelinePromise)
29
+ return embeddingPipelinePromise;
30
+ embeddingPipelinePromise = (async () => {
31
+ const { pipeline, env } = await import("@huggingface/transformers");
32
+ configureEmbeddingTransformersEnv(env);
33
+ const createPipeline = pipeline;
34
+ return (await createPipeline("feature-extraction", "Xenova/multilingual-e5-small", {
35
+ quantized: true,
36
+ dtype: "fp32",
37
+ }));
38
+ })();
39
+ try {
40
+ embeddingPipeline = await embeddingPipelinePromise;
41
+ return embeddingPipeline;
42
+ }
43
+ finally {
44
+ embeddingPipelinePromise = null;
45
+ }
36
46
  }
37
47
  async function loadRerankerPipeline(config) {
38
48
  const key = rerankerCacheKey(config);
@@ -81,6 +91,34 @@ async function handleEmbed(request) {
81
91
  }
82
92
  return vectors;
83
93
  }
94
+ function embeddingMaximumTokens(tokenizer) {
95
+ const configured = tokenizer.model_max_length;
96
+ return typeof configured === "number" && Number.isInteger(configured) && configured > 0 && configured <= 1_000_000
97
+ ? configured
98
+ : 512;
99
+ }
100
+ async function handleMeasureEmbeddingInput(request) {
101
+ const pipe = await loadEmbeddingPipeline();
102
+ const texts = request.texts.map((text) => applyEmbeddingPrefix(text, request.prefix, request.prefixes));
103
+ const tokenized = pipe.tokenizer(texts, {
104
+ add_special_tokens: true,
105
+ truncation: false,
106
+ padding: false,
107
+ return_tensor: false,
108
+ });
109
+ const inputIds = tokenized.input_ids;
110
+ let counts;
111
+ if (texts.length === 1 && inputIds.every((token) => typeof token === "number")) {
112
+ counts = [inputIds.length];
113
+ }
114
+ else if (inputIds.every((tokens) => Array.isArray(tokens))) {
115
+ counts = inputIds.map((tokens) => tokens.length);
116
+ }
117
+ else {
118
+ counts = texts.map((text) => pipe.tokenizer.encode(text, { add_special_tokens: true }).length);
119
+ }
120
+ return { counts, maximum: embeddingMaximumTokens(pipe.tokenizer) };
121
+ }
84
122
  async function handleRerank(request) {
85
123
  const pipe = await loadRerankerPipeline(request.reranker);
86
124
  const results = [];
@@ -110,17 +148,23 @@ function parseWorkerRequest(message) {
110
148
  }
111
149
  if (!("type" in message))
112
150
  throw new Error("Invalid worker request type");
113
- if (message.type === "embed") {
151
+ if (message.type === "embed" || message.type === "measure_embedding_input") {
114
152
  if (!("texts" in message) || !isStringArray(message.texts))
115
- throw new Error("Invalid embed texts");
153
+ throw new Error("Invalid embedding texts");
116
154
  if (!("prefix" in message) || (message.prefix !== "query" && message.prefix !== "passage")) {
117
- throw new Error("Invalid embed prefix");
155
+ throw new Error("Invalid embedding prefix");
118
156
  }
119
157
  const prefixes = "prefixes" in message && (message.prefixes === "on" || message.prefixes === "off") ? message.prefixes : "on";
120
158
  if ("prefixes" in message && message.prefixes !== "on" && message.prefixes !== "off") {
121
- throw new Error("Invalid embed prefix strategy");
159
+ throw new Error("Invalid embedding prefix strategy");
122
160
  }
123
- return { id: message.id, type: "embed", texts: message.texts, prefix: message.prefix, prefixes };
161
+ return {
162
+ id: message.id,
163
+ type: message.type,
164
+ texts: message.texts,
165
+ prefix: message.prefix,
166
+ prefixes,
167
+ };
124
168
  }
125
169
  if (message.type === "rerank") {
126
170
  if (!("query" in message) || typeof message.query !== "string")
@@ -151,7 +195,11 @@ async function handleRequestMessage(message) {
151
195
  requestId = message.id;
152
196
  }
153
197
  const request = parseWorkerRequest(message);
154
- const result = request.type === "embed" ? await handleEmbed(request) : await handleRerank(request);
198
+ const result = request.type === "embed"
199
+ ? await handleEmbed(request)
200
+ : request.type === "measure_embedding_input"
201
+ ? await handleMeasureEmbeddingInput(request)
202
+ : await handleRerank(request);
155
203
  return { id: request.id, result };
156
204
  }
157
205
  catch (error) {
@@ -0,0 +1,2 @@
1
+ export declare const MAX_SEARCH_RESULTS = 50;
2
+ export declare const MAX_SEARCH_WINDOW = 800;
@@ -0,0 +1,2 @@
1
+ export const MAX_SEARCH_RESULTS = 50;
2
+ export const MAX_SEARCH_WINDOW = 800;
@@ -0,0 +1,16 @@
1
+ import type { Chunk } from "../storage/sqlite.ts";
2
+ import type { RankingDiagnostics } from "./ranking.ts";
3
+ export type DiversityMode = "off" | "balanced" | "strong" | undefined;
4
+ export interface RankedChunk {
5
+ chunk: Chunk;
6
+ kbName: string;
7
+ ranking?: RankingDiagnostics;
8
+ score: number;
9
+ content: string;
10
+ snippet: string;
11
+ startLine: number;
12
+ endLine: number;
13
+ sourceChunkIds: string[];
14
+ }
15
+ export declare function diversifyRankedChunks(candidates: RankedChunk[], diversity: DiversityMode, vectorsByChunkId?: Map<string, Float32Array>, signal?: AbortSignal): Promise<RankedChunk[]>;
16
+ export declare function interleaveByFile(candidates: RankedChunk[], diversity: DiversityMode, signal?: AbortSignal): RankedChunk[];
@@ -0,0 +1,183 @@
1
+ import { setImmediate as yieldToEventLoop } from "node:timers/promises";
2
+ import { tokenizeForSearch } from "./query.js";
3
+ const VECTOR_REDUNDANCY_WEIGHT = 0.35;
4
+ const PAIR_CHECK_INTERVAL = 1_024;
5
+ const CPU_SLICE_MS = 8;
6
+ function throwIfAborted(signal) {
7
+ if (signal?.aborted)
8
+ throw new Error("Cancelled");
9
+ }
10
+ function cosineSimilarity(a, b) {
11
+ if (!a || !b || a.length !== b.length)
12
+ return 0;
13
+ let dot = 0;
14
+ for (let i = 0; i < a.length; i++)
15
+ dot += a[i] * b[i];
16
+ return dot;
17
+ }
18
+ function jaccardSimilarity(a, b) {
19
+ if (a.size === 0 || b.size === 0)
20
+ return 0;
21
+ const [small, large] = a.size <= b.size ? [a, b] : [b, a];
22
+ let intersection = 0;
23
+ for (const token of small) {
24
+ if (large.has(token))
25
+ intersection++;
26
+ }
27
+ return intersection / (a.size + b.size - intersection);
28
+ }
29
+ function proximityMeta(candidate) {
30
+ return {
31
+ filePath: candidate.chunk.file_path,
32
+ kbId: candidate.chunk.kb_id,
33
+ startLine: candidate.startLine,
34
+ endLine: candidate.endLine,
35
+ sourceIds: new Set(candidate.sourceChunkIds),
36
+ };
37
+ }
38
+ function lineProximity(a, b) {
39
+ if (a.filePath !== b.filePath || a.kbId !== b.kbId)
40
+ return 0;
41
+ const [small, large] = a.sourceIds.size <= b.sourceIds.size ? [a.sourceIds, b.sourceIds] : [b.sourceIds, a.sourceIds];
42
+ for (const id of small) {
43
+ if (large.has(id))
44
+ return 1;
45
+ }
46
+ const overlap = Math.max(0, Math.min(a.endLine, b.endLine) - Math.max(a.startLine, b.startLine) + 1);
47
+ if (overlap > 0)
48
+ return 1;
49
+ const gap = Math.max(a.startLine, b.startLine) - Math.min(a.endLine, b.endLine);
50
+ if (gap <= 20)
51
+ return 0.8;
52
+ if (gap <= 80)
53
+ return 0.45;
54
+ return 0.2;
55
+ }
56
+ function normalizeScores(candidates) {
57
+ let min = Number.POSITIVE_INFINITY;
58
+ let max = Number.NEGATIVE_INFINITY;
59
+ for (const candidate of candidates) {
60
+ if (candidate.score < min)
61
+ min = candidate.score;
62
+ if (candidate.score > max)
63
+ max = candidate.score;
64
+ }
65
+ const normalized = new Float64Array(candidates.length);
66
+ for (let i = 0; i < candidates.length; i++) {
67
+ normalized[i] = max === min ? 1 : (candidates[i].score - min) / (max - min);
68
+ }
69
+ return normalized;
70
+ }
71
+ function firstUnusedIndex(used) {
72
+ for (let i = 0; i < used.length; i++) {
73
+ if (used[i] === 0)
74
+ return i;
75
+ }
76
+ throw new Error("Diversity selection exhausted its candidate pool unexpectedly");
77
+ }
78
+ export async function diversifyRankedChunks(candidates, diversity, vectorsByChunkId = new Map(), signal) {
79
+ throwIfAborted(signal);
80
+ if (diversity === "off" || candidates.length <= 2)
81
+ return candidates;
82
+ const lambda = diversity === "strong" ? 0.62 : 0.76;
83
+ const n = candidates.length;
84
+ const tokenSets = new Array(n);
85
+ const vectors = new Array(n);
86
+ const metas = new Array(n);
87
+ const relevance = normalizeScores(candidates);
88
+ for (let i = 0; i < n; i++) {
89
+ const candidate = candidates[i];
90
+ tokenSets[i] = tokenizeForSearch(candidate.content);
91
+ vectors[i] = vectorsByChunkId.get(candidate.chunk.id);
92
+ metas[i] = proximityMeta(candidate);
93
+ }
94
+ const pairRedundancy = (a, b) => {
95
+ let best = lineProximity(metas[a], metas[b]);
96
+ const lexical = jaccardSimilarity(tokenSets[a], tokenSets[b]);
97
+ if (lexical > best)
98
+ best = lexical;
99
+ const va = vectors[a];
100
+ const vb = vectors[b];
101
+ if (va !== undefined && vb !== undefined) {
102
+ const vector = Math.max(0, cosineSimilarity(va, vb)) * VECTOR_REDUNDANCY_WEIGHT;
103
+ if (vector > best)
104
+ best = vector;
105
+ }
106
+ return best;
107
+ };
108
+ const redundancy = new Float64Array(n);
109
+ const used = new Uint8Array(n);
110
+ const selected = new Array(n);
111
+ let pairsSinceCheck = 0;
112
+ let sliceStartedAt = performance.now();
113
+ for (let round = 0; round < n; round++) {
114
+ throwIfAborted(signal);
115
+ let bestIndex = firstUnusedIndex(used);
116
+ let bestScore = Number.NEGATIVE_INFINITY;
117
+ for (let i = 0; i < n; i++) {
118
+ if (used[i] === 1)
119
+ continue;
120
+ const mmrScore = lambda * relevance[i] - (1 - lambda) * redundancy[i];
121
+ if (mmrScore > bestScore) {
122
+ bestScore = mmrScore;
123
+ bestIndex = i;
124
+ }
125
+ }
126
+ used[bestIndex] = 1;
127
+ selected[round] = candidates[bestIndex];
128
+ for (let i = 0; i < n; i++) {
129
+ if (used[i] === 1)
130
+ continue;
131
+ const pair = pairRedundancy(bestIndex, i);
132
+ if (pair > redundancy[i])
133
+ redundancy[i] = pair;
134
+ pairsSinceCheck++;
135
+ if (pairsSinceCheck < PAIR_CHECK_INTERVAL)
136
+ continue;
137
+ pairsSinceCheck = 0;
138
+ throwIfAborted(signal);
139
+ if (performance.now() - sliceStartedAt < CPU_SLICE_MS)
140
+ continue;
141
+ await yieldToEventLoop();
142
+ throwIfAborted(signal);
143
+ sliceStartedAt = performance.now();
144
+ }
145
+ }
146
+ return selected;
147
+ }
148
+ export function interleaveByFile(candidates, diversity, signal) {
149
+ throwIfAborted(signal);
150
+ if (diversity === "off" || candidates.length <= 2)
151
+ return candidates;
152
+ const buckets = new Map();
153
+ const fileOrder = [];
154
+ for (const candidate of candidates) {
155
+ throwIfAborted(signal);
156
+ const key = `${candidate.chunk.kb_id}:${candidate.chunk.file_path}`;
157
+ let bucket = buckets.get(key);
158
+ if (!bucket) {
159
+ bucket = [];
160
+ buckets.set(key, bucket);
161
+ fileOrder.push(key);
162
+ }
163
+ bucket.push(candidate);
164
+ }
165
+ if (fileOrder.length <= 1)
166
+ return candidates;
167
+ const result = [];
168
+ let round = 0;
169
+ while (result.length < candidates.length) {
170
+ let added = false;
171
+ for (const key of fileOrder) {
172
+ const item = buckets.get(key)?.[round];
173
+ if (!item)
174
+ continue;
175
+ result.push(item);
176
+ added = true;
177
+ }
178
+ if (!added)
179
+ break;
180
+ round++;
181
+ }
182
+ return result;
183
+ }
@@ -22,6 +22,11 @@ export interface SearchTuningSummary {
22
22
  snippet_max_length: number;
23
23
  min_hybrid_score: number;
24
24
  candidate_limit: number;
25
+ retrieval_limit_per_kb: number;
26
+ retrieved_candidates: number;
27
+ filtered_candidates: number;
28
+ diversity_candidates: number;
29
+ candidate_pool_truncated: boolean;
25
30
  candidate_min: number;
26
31
  candidate_multiplier: number;
27
32
  adaptive_context_lines: number;
@@ -36,6 +41,11 @@ interface SearchTuningInput {
36
41
  profile?: SearchProfile;
37
42
  kbSourceTypes: string[];
38
43
  }
44
+ export interface SearchWindow {
45
+ limit: number;
46
+ offset: number;
47
+ }
48
+ export declare function resolveSearchWindow(limit: number | undefined, offset: number | undefined, defaultLimit: number): SearchWindow;
39
49
  export declare function resolveSearchTuning(input: SearchTuningInput): SearchTuning;
40
50
  export declare function summarizeSearchTuning(tuning: SearchTuning, limit: number, candidateLimit: number): SearchTuningSummary;
41
51
  export {};