pi-knowledge 0.4.2 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/README.md +4 -2
- package/dist/index.d.ts +38 -0
- package/dist/index.js +435 -0
- package/dist/src/diagnostics/health.d.ts +19 -0
- package/dist/src/diagnostics/health.js +110 -0
- package/dist/src/embedding/provider.d.ts +5 -0
- package/dist/src/embedding/provider.js +99 -0
- package/dist/src/embedding/vectors.d.ts +15 -0
- package/dist/src/embedding/vectors.js +111 -0
- package/dist/src/engine.d.ts +90 -0
- package/dist/src/engine.js +1370 -0
- package/dist/src/indexer/chunker.d.ts +53 -0
- package/dist/src/indexer/chunker.js +498 -0
- package/dist/src/indexer/chunkers/code-ast.d.ts +3 -0
- package/dist/src/indexer/chunkers/code-ast.js +179 -0
- package/dist/src/model-worker-client.d.ts +10 -0
- package/dist/src/model-worker-client.js +138 -0
- package/dist/src/model-worker.d.ts +1 -0
- package/dist/src/model-worker.js +61 -0
- package/dist/src/search/bm25.d.ts +8 -0
- package/dist/src/search/bm25.js +38 -0
- package/dist/src/search/fusion.d.ts +19 -0
- package/dist/src/search/fusion.js +34 -0
- package/dist/src/search/query.d.ts +5 -0
- package/dist/src/search/query.js +57 -0
- package/dist/src/search/ranking.d.ts +27 -0
- package/dist/src/search/ranking.js +180 -0
- package/dist/src/search/reranker.d.ts +10 -0
- package/dist/src/search/reranker.js +66 -0
- package/dist/src/search/vector.d.ts +14 -0
- package/dist/src/search/vector.js +58 -0
- package/dist/src/storage/sqlite.d.ts +99 -0
- package/dist/src/storage/sqlite.js +330 -0
- package/dist/src/watcher/file-watcher.d.ts +4 -0
- package/dist/src/watcher/file-watcher.js +108 -0
- package/extension.js +7 -0
- package/package.json +10 -5
- package/index.ts +0 -500
- package/src/diagnostics/health.ts +0 -145
- package/src/embedding/provider.ts +0 -106
- package/src/embedding/vectors.ts +0 -119
- package/src/engine.ts +0 -1592
- package/src/indexer/chunker.ts +0 -612
- package/src/indexer/chunkers/code-ast.ts +0 -209
- package/src/model-worker-client.ts +0 -146
- package/src/model-worker.ts +0 -88
- package/src/search/bm25.ts +0 -55
- package/src/search/fusion.ts +0 -48
- package/src/search/query.ts +0 -59
- package/src/search/ranking.ts +0 -201
- package/src/search/reranker.ts +0 -74
- package/src/search/vector.ts +0 -81
- package/src/storage/sqlite.ts +0 -489
- package/src/watcher/file-watcher.ts +0 -116
package/src/search/ranking.ts
DELETED
|
@@ -1,201 +0,0 @@
|
|
|
1
|
-
import { buildChunkEmbeddingText } from "../indexer/chunker.ts";
|
|
2
|
-
import type { Chunk } from "../storage/sqlite.ts";
|
|
3
|
-
import { signalTokens, tokenizeForSearch } from "./query.ts";
|
|
4
|
-
|
|
5
|
-
export const MIN_HYBRID_SCORE = 0.18;
|
|
6
|
-
|
|
7
|
-
export const FILE_TYPE_ALIASES: Record<string, string> = {
|
|
8
|
-
csharp: "csharp",
|
|
9
|
-
cs: "csharp",
|
|
10
|
-
cpp: "cpp",
|
|
11
|
-
hpp: "cpp",
|
|
12
|
-
js: "javascript",
|
|
13
|
-
jsx: "javascript",
|
|
14
|
-
md: "markdown",
|
|
15
|
-
mdx: "markdown",
|
|
16
|
-
php: "php",
|
|
17
|
-
py: "python",
|
|
18
|
-
rb: "ruby",
|
|
19
|
-
sh: "shell",
|
|
20
|
-
ts: "typescript",
|
|
21
|
-
tsx: "typescript",
|
|
22
|
-
yml: "yaml",
|
|
23
|
-
};
|
|
24
|
-
|
|
25
|
-
const SOURCE_FILE_TYPES = new Set([
|
|
26
|
-
"c",
|
|
27
|
-
"cpp",
|
|
28
|
-
"csharp",
|
|
29
|
-
"go",
|
|
30
|
-
"java",
|
|
31
|
-
"javascript",
|
|
32
|
-
"php",
|
|
33
|
-
"python",
|
|
34
|
-
"ruby",
|
|
35
|
-
"rust",
|
|
36
|
-
"typescript",
|
|
37
|
-
]);
|
|
38
|
-
|
|
39
|
-
const TEST_PATH_PATTERNS = ["_test.", ".test.", ".spec.", "/test/", "/tests/", "tests/", "__tests__/", "spec/"];
|
|
40
|
-
|
|
41
|
-
export interface RankingDiagnostics {
|
|
42
|
-
adjusted_score: number;
|
|
43
|
-
base_score: number;
|
|
44
|
-
coverage: number;
|
|
45
|
-
documentation_boost: number;
|
|
46
|
-
is_localization: boolean;
|
|
47
|
-
is_test: boolean;
|
|
48
|
-
localization_penalty: number;
|
|
49
|
-
path_boost: number;
|
|
50
|
-
source_boost: number;
|
|
51
|
-
test_intent: boolean;
|
|
52
|
-
user_guide_intent: boolean;
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
export function normalizeFileTypeFilter(fileType: string | undefined): string | undefined {
|
|
56
|
-
if (!fileType) return undefined;
|
|
57
|
-
const normalized = fileType.trim().toLowerCase();
|
|
58
|
-
return FILE_TYPE_ALIASES[normalized] ?? normalized;
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
export function isTestPath(filePath: string): boolean {
|
|
62
|
-
const lower = filePath.toLowerCase();
|
|
63
|
-
return TEST_PATH_PATTERNS.some((pattern) => lower.includes(pattern));
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
export function queryAsksForTests(queryTokens: Set<string>): boolean {
|
|
67
|
-
const signals = signalTokens(queryTokens);
|
|
68
|
-
return ["assert", "fixture", "spec", "test", "testing"].some((token) => signals.has(token));
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
function queryAsksForUserGuide(queryTokens: Set<string>): boolean {
|
|
72
|
-
const signals = signalTokens(queryTokens);
|
|
73
|
-
return ["config", "configuration", "install", "onboard", "quickstart", "setup", "start", "usage"].some((token) =>
|
|
74
|
-
signals.has(token),
|
|
75
|
-
);
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
function queryAsksForLocalization(queryTokens: Set<string>): boolean {
|
|
79
|
-
const signals = signalTokens(queryTokens);
|
|
80
|
-
return ["i18n", "lang", "language", "locale", "localization", "translation"].some((token) => signals.has(token));
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
export function isLocalizationPath(filePath: string, fileType: string): boolean {
|
|
84
|
-
if (!["json", "markdown", "text", "toml", "yaml"].includes(fileType)) return false;
|
|
85
|
-
const lower = filePath.toLowerCase();
|
|
86
|
-
return (
|
|
87
|
-
lower.includes("/locale/") ||
|
|
88
|
-
lower.startsWith("locale/") ||
|
|
89
|
-
lower.includes("/locales/") ||
|
|
90
|
-
lower.startsWith("locales/") ||
|
|
91
|
-
lower.includes("/lang/") ||
|
|
92
|
-
lower.startsWith("lang/") ||
|
|
93
|
-
lower.includes("/i18n/") ||
|
|
94
|
-
lower.startsWith("i18n/") ||
|
|
95
|
-
lower.includes("/translations/") ||
|
|
96
|
-
lower.startsWith("translations/")
|
|
97
|
-
);
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
export function queryCoverage(text: string, queryTokens: Set<string>): number {
|
|
101
|
-
const textTokens = signalTokens(tokenizeForSearch(text));
|
|
102
|
-
const signals = signalTokens(queryTokens);
|
|
103
|
-
if (signals.size === 0) return 0;
|
|
104
|
-
let matched = 0;
|
|
105
|
-
for (const token of signals) {
|
|
106
|
-
if (textTokens.has(token)) matched++;
|
|
107
|
-
}
|
|
108
|
-
return matched / signals.size;
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
export function hasAnyLexicalEvidence(text: string, queryTokens: Set<string>): boolean {
|
|
112
|
-
return queryCoverage(text, queryTokens) > 0;
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
export function pathTokenBoost(filePath: string, queryTokens: Set<string>): number {
|
|
116
|
-
const pathTokens = signalTokens(tokenizeForSearch(filePath.replace(/[./_-]/g, " ")));
|
|
117
|
-
const signals = signalTokens(queryTokens);
|
|
118
|
-
let matched = 0;
|
|
119
|
-
for (const token of signals) {
|
|
120
|
-
if (pathTokens.has(token)) matched++;
|
|
121
|
-
}
|
|
122
|
-
if (matched === 0) return 0;
|
|
123
|
-
return Math.min(0.45, 0.18 * matched);
|
|
124
|
-
}
|
|
125
|
-
|
|
126
|
-
export function basenameTokenBoost(filePath: string, queryTokens: Set<string>): number {
|
|
127
|
-
const basename = filePath.split("/").at(-1) ?? filePath;
|
|
128
|
-
const stem = basename.replace(/\.[^.]+$/, "").replace(/_test$|\.test$|\.spec$/i, "");
|
|
129
|
-
const baseTokens = signalTokens(tokenizeForSearch(stem.replace(/[._-]/g, " ")));
|
|
130
|
-
const signals = signalTokens(queryTokens);
|
|
131
|
-
if (baseTokens.size === 0 || signals.size === 0) return 0;
|
|
132
|
-
for (const token of signals) {
|
|
133
|
-
if (baseTokens.has(token)) return 0.32;
|
|
134
|
-
}
|
|
135
|
-
return 0;
|
|
136
|
-
}
|
|
137
|
-
|
|
138
|
-
export function sourceFileBoost(chunk: Pick<Chunk, "file_path" | "file_type">, queryTokens: Set<string>): number {
|
|
139
|
-
if (!SOURCE_FILE_TYPES.has(chunk.file_type)) return 0;
|
|
140
|
-
if (queryAsksForUserGuide(queryTokens)) return Math.min(0.05, pathTokenBoost(chunk.file_path, queryTokens));
|
|
141
|
-
const boost = basenameTokenBoost(chunk.file_path, queryTokens);
|
|
142
|
-
if (boost > 0) return boost;
|
|
143
|
-
const boostFromPath = pathTokenBoost(chunk.file_path, queryTokens);
|
|
144
|
-
return Math.min(0.22, boostFromPath);
|
|
145
|
-
}
|
|
146
|
-
|
|
147
|
-
function documentationBoost(chunk: Pick<Chunk, "file_path" | "file_type">, queryTokens: Set<string>): number {
|
|
148
|
-
if (!queryAsksForUserGuide(queryTokens)) return 0;
|
|
149
|
-
const lower = chunk.file_path.toLowerCase();
|
|
150
|
-
if (chunk.file_type !== "markdown") return 0;
|
|
151
|
-
if (
|
|
152
|
-
lower.includes("install") ||
|
|
153
|
-
lower.includes("readme") ||
|
|
154
|
-
lower.includes("getting-started") ||
|
|
155
|
-
lower.includes("quickstart")
|
|
156
|
-
) {
|
|
157
|
-
return 0.85;
|
|
158
|
-
}
|
|
159
|
-
if (lower.startsWith("docs/")) return 0.35;
|
|
160
|
-
return 0;
|
|
161
|
-
}
|
|
162
|
-
|
|
163
|
-
export function scoreChunkForQuery(baseScore: number, chunk: Chunk, queryTokens: Set<string>): RankingDiagnostics {
|
|
164
|
-
const guideIntent = queryAsksForUserGuide(queryTokens);
|
|
165
|
-
const pathBoost =
|
|
166
|
-
guideIntent && SOURCE_FILE_TYPES.has(chunk.file_type)
|
|
167
|
-
? Math.min(0.08, pathTokenBoost(chunk.file_path, queryTokens))
|
|
168
|
-
: pathTokenBoost(chunk.file_path, queryTokens);
|
|
169
|
-
const sourceBoost = sourceFileBoost(chunk, queryTokens);
|
|
170
|
-
const docBoost = documentationBoost(chunk, queryTokens);
|
|
171
|
-
const testIntent = queryAsksForTests(queryTokens);
|
|
172
|
-
const test = isTestPath(chunk.file_path);
|
|
173
|
-
const localization = isLocalizationPath(chunk.file_path, chunk.file_type);
|
|
174
|
-
const localizationPenalty = localization && !queryAsksForLocalization(queryTokens) ? 0.2 : 1;
|
|
175
|
-
let adjustedScore = baseScore + pathBoost + sourceBoost + docBoost;
|
|
176
|
-
if (test && !testIntent) adjustedScore *= 0.48;
|
|
177
|
-
if (!test && testIntent) adjustedScore *= 0.88;
|
|
178
|
-
adjustedScore *= localizationPenalty;
|
|
179
|
-
return {
|
|
180
|
-
adjusted_score: adjustedScore,
|
|
181
|
-
base_score: baseScore,
|
|
182
|
-
coverage: queryCoverage(buildChunkEmbeddingText(chunk), queryTokens),
|
|
183
|
-
documentation_boost: docBoost,
|
|
184
|
-
is_localization: localization,
|
|
185
|
-
is_test: test,
|
|
186
|
-
localization_penalty: localizationPenalty,
|
|
187
|
-
path_boost: pathBoost,
|
|
188
|
-
source_boost: sourceBoost,
|
|
189
|
-
test_intent: testIntent,
|
|
190
|
-
user_guide_intent: guideIntent,
|
|
191
|
-
};
|
|
192
|
-
}
|
|
193
|
-
|
|
194
|
-
export function hasEnoughLexicalEvidence(chunk: Chunk, queryTokens: Set<string>): boolean {
|
|
195
|
-
const signals = signalTokens(queryTokens);
|
|
196
|
-
if (signals.size <= 1) return true;
|
|
197
|
-
if (sourceFileBoost(chunk, queryTokens) >= 0.22) return true;
|
|
198
|
-
const coverage = queryCoverage(buildChunkEmbeddingText(chunk), queryTokens);
|
|
199
|
-
if (signals.size <= 3) return coverage >= 1 / signals.size;
|
|
200
|
-
return coverage >= 0.34;
|
|
201
|
-
}
|
package/src/search/reranker.ts
DELETED
|
@@ -1,74 +0,0 @@
|
|
|
1
|
-
import { rerankInModelWorker } from "../model-worker-client.ts";
|
|
2
|
-
|
|
3
|
-
let disposeTimer: ReturnType<typeof setTimeout> | null = null;
|
|
4
|
-
let disposePromise: Promise<void> | null = null;
|
|
5
|
-
let activeRuns = 0;
|
|
6
|
-
let disposeRequested = false;
|
|
7
|
-
const idleWaiters: Array<() => void> = [];
|
|
8
|
-
const IDLE_MS = 30_000;
|
|
9
|
-
const ENABLE_NATIVE_IDLE_DISPOSE = process.env.PI_KNOWLEDGE_ENABLE_NATIVE_IDLE_DISPOSE === "true";
|
|
10
|
-
|
|
11
|
-
function clearTimer(): void {
|
|
12
|
-
if (disposeTimer) clearTimeout(disposeTimer);
|
|
13
|
-
disposeTimer = null;
|
|
14
|
-
}
|
|
15
|
-
|
|
16
|
-
function scheduleDispose(): void {
|
|
17
|
-
if (activeRuns > 0 || disposeRequested) return;
|
|
18
|
-
clearTimer();
|
|
19
|
-
if (!ENABLE_NATIVE_IDLE_DISPOSE) return;
|
|
20
|
-
disposeTimer = setTimeout(() => disposeReranker(), IDLE_MS);
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
function beginRun(): void {
|
|
24
|
-
activeRuns++;
|
|
25
|
-
clearTimer();
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
function endRun(): void {
|
|
29
|
-
activeRuns--;
|
|
30
|
-
if (activeRuns > 0) return;
|
|
31
|
-
for (const resolve of idleWaiters.splice(0)) resolve();
|
|
32
|
-
if (!disposeRequested) scheduleDispose();
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
function waitForNoActiveRuns(): Promise<void> {
|
|
36
|
-
if (activeRuns === 0) return Promise.resolve();
|
|
37
|
-
return new Promise((resolve) => idleWaiters.push(resolve));
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
export async function disposeReranker(): Promise<void> {
|
|
41
|
-
clearTimer();
|
|
42
|
-
if (disposePromise) return disposePromise;
|
|
43
|
-
disposeRequested = true;
|
|
44
|
-
await waitForNoActiveRuns();
|
|
45
|
-
disposePromise = Promise.resolve().finally(() => {
|
|
46
|
-
disposePromise = null;
|
|
47
|
-
disposeRequested = false;
|
|
48
|
-
});
|
|
49
|
-
return disposePromise;
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
export async function prepareRerankerForShutdown(): Promise<void> {
|
|
53
|
-
clearTimer();
|
|
54
|
-
await waitForNoActiveRuns();
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
export interface RerankCandidate {
|
|
58
|
-
chunkId: string;
|
|
59
|
-
content: string;
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
export async function rerank(
|
|
63
|
-
query: string,
|
|
64
|
-
candidates: RerankCandidate[],
|
|
65
|
-
topK: number,
|
|
66
|
-
): Promise<Array<{ chunkId: string; score: number }>> {
|
|
67
|
-
if (candidates.length === 0) return [];
|
|
68
|
-
beginRun();
|
|
69
|
-
try {
|
|
70
|
-
return await rerankInModelWorker(query, candidates, topK);
|
|
71
|
-
} finally {
|
|
72
|
-
endRun();
|
|
73
|
-
}
|
|
74
|
-
}
|
package/src/search/vector.ts
DELETED
|
@@ -1,81 +0,0 @@
|
|
|
1
|
-
import { openVectorReader } from "../embedding/vectors.ts";
|
|
2
|
-
|
|
3
|
-
export interface VectorResult {
|
|
4
|
-
chunkId: string;
|
|
5
|
-
score: number;
|
|
6
|
-
}
|
|
7
|
-
|
|
8
|
-
export interface VectorFileSearchResult {
|
|
9
|
-
results: VectorResult[];
|
|
10
|
-
vectorsByChunkId: Map<string, Float32Array>;
|
|
11
|
-
}
|
|
12
|
-
|
|
13
|
-
type ChunkIdSource = Iterable<string | { id: string }>;
|
|
14
|
-
|
|
15
|
-
function cosine(a: Float32Array, b: Float32Array): number {
|
|
16
|
-
let dot = 0;
|
|
17
|
-
for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
|
|
18
|
-
return dot;
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
function keepTopK(
|
|
22
|
-
top: Array<VectorResult & { vector: Float32Array }>,
|
|
23
|
-
item: VectorResult,
|
|
24
|
-
vector: Float32Array,
|
|
25
|
-
limit: number,
|
|
26
|
-
): void {
|
|
27
|
-
if (limit <= 0) return;
|
|
28
|
-
if (top.length < limit) {
|
|
29
|
-
const stored = { ...item, vector: new Float32Array(vector) };
|
|
30
|
-
top.push(stored);
|
|
31
|
-
top.sort((a, b) => a.score - b.score);
|
|
32
|
-
return;
|
|
33
|
-
}
|
|
34
|
-
if (item.score <= top[0].score) return;
|
|
35
|
-
const stored = { ...item, vector: new Float32Array(vector) };
|
|
36
|
-
top[0] = stored;
|
|
37
|
-
top.sort((a, b) => a.score - b.score);
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
export function searchVector(
|
|
41
|
-
queryVec: Float32Array,
|
|
42
|
-
vectors: Float32Array[],
|
|
43
|
-
chunkIds: string[],
|
|
44
|
-
limit = 50,
|
|
45
|
-
): VectorResult[] {
|
|
46
|
-
const scores = vectors.map((v, i) => ({ chunkId: chunkIds[i], score: cosine(queryVec, v) }));
|
|
47
|
-
scores.sort((a, b) => b.score - a.score);
|
|
48
|
-
return scores.slice(0, limit);
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
export function searchVectorFile(
|
|
52
|
-
queryVec: Float32Array,
|
|
53
|
-
vectorPath: string,
|
|
54
|
-
chunkIds: ChunkIdSource,
|
|
55
|
-
limit = 50,
|
|
56
|
-
): VectorFileSearchResult {
|
|
57
|
-
const reader = openVectorReader(vectorPath);
|
|
58
|
-
if (!reader) return { results: [], vectorsByChunkId: new Map() };
|
|
59
|
-
try {
|
|
60
|
-
if (reader.dim !== queryVec.length) return { results: [], vectorsByChunkId: new Map() };
|
|
61
|
-
const scratch = new Float32Array(reader.dim);
|
|
62
|
-
const top: Array<VectorResult & { vector: Float32Array }> = [];
|
|
63
|
-
let i = 0;
|
|
64
|
-
for (const chunkIdRef of chunkIds) {
|
|
65
|
-
if (i >= reader.count) break;
|
|
66
|
-
const vectorIndex = i;
|
|
67
|
-
i++;
|
|
68
|
-
if (!reader.readInto(vectorIndex, scratch)) continue;
|
|
69
|
-
const chunkId = typeof chunkIdRef === "string" ? chunkIdRef : chunkIdRef.id;
|
|
70
|
-
keepTopK(top, { chunkId, score: cosine(queryVec, scratch) }, scratch, limit);
|
|
71
|
-
}
|
|
72
|
-
const sorted = top.sort((a, b) => b.score - a.score);
|
|
73
|
-
const vectorsByChunkId = new Map(sorted.map((item) => [item.chunkId, item.vector]));
|
|
74
|
-
return {
|
|
75
|
-
results: sorted.map(({ chunkId, score }) => ({ chunkId, score })),
|
|
76
|
-
vectorsByChunkId,
|
|
77
|
-
};
|
|
78
|
-
} finally {
|
|
79
|
-
reader.close();
|
|
80
|
-
}
|
|
81
|
-
}
|