pi-knowledge 0.4.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +4 -2
- package/dist/index.d.ts +38 -0
- package/dist/index.js +435 -0
- package/dist/src/diagnostics/health.d.ts +19 -0
- package/dist/src/diagnostics/health.js +110 -0
- package/dist/src/embedding/provider.d.ts +5 -0
- package/dist/src/embedding/provider.js +99 -0
- package/dist/src/embedding/vectors.d.ts +15 -0
- package/dist/src/embedding/vectors.js +111 -0
- package/dist/src/engine.d.ts +90 -0
- package/dist/src/engine.js +1370 -0
- package/dist/src/indexer/chunker.d.ts +53 -0
- package/dist/src/indexer/chunker.js +498 -0
- package/dist/src/indexer/chunkers/code-ast.d.ts +3 -0
- package/dist/src/indexer/chunkers/code-ast.js +179 -0
- package/dist/src/model-worker-client.d.ts +10 -0
- package/dist/src/model-worker-client.js +138 -0
- package/dist/src/model-worker.d.ts +1 -0
- package/dist/src/model-worker.js +61 -0
- package/dist/src/search/bm25.d.ts +8 -0
- package/dist/src/search/bm25.js +38 -0
- package/dist/src/search/fusion.d.ts +19 -0
- package/dist/src/search/fusion.js +34 -0
- package/dist/src/search/query.d.ts +5 -0
- package/dist/src/search/query.js +57 -0
- package/dist/src/search/ranking.d.ts +27 -0
- package/dist/src/search/ranking.js +180 -0
- package/dist/src/search/reranker.d.ts +10 -0
- package/dist/src/search/reranker.js +66 -0
- package/dist/src/search/vector.d.ts +14 -0
- package/dist/src/search/vector.js +58 -0
- package/dist/src/storage/sqlite.d.ts +99 -0
- package/dist/src/storage/sqlite.js +330 -0
- package/dist/src/watcher/file-watcher.d.ts +4 -0
- package/dist/src/watcher/file-watcher.js +108 -0
- package/extension.js +9 -0
- package/package.json +10 -5
- package/index.ts +0 -500
- package/src/diagnostics/health.ts +0 -145
- package/src/embedding/provider.ts +0 -106
- package/src/embedding/vectors.ts +0 -119
- package/src/engine.ts +0 -1592
- package/src/indexer/chunker.ts +0 -612
- package/src/indexer/chunkers/code-ast.ts +0 -209
- package/src/model-worker-client.ts +0 -146
- package/src/model-worker.ts +0 -88
- package/src/search/bm25.ts +0 -55
- package/src/search/fusion.ts +0 -48
- package/src/search/query.ts +0 -59
- package/src/search/ranking.ts +0 -201
- package/src/search/reranker.ts +0 -74
- package/src/search/vector.ts +0 -81
- package/src/storage/sqlite.ts +0 -489
- package/src/watcher/file-watcher.ts +0 -116
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import { embedInModelWorker } from "../model-worker-client.js";
|
|
2
|
+
let disposeTimer = null;
|
|
3
|
+
let disposePromise = null;
|
|
4
|
+
let activeRuns = 0;
|
|
5
|
+
let disposeRequested = false;
|
|
6
|
+
const idleWaiters = [];
|
|
7
|
+
const IDLE_TIMEOUT_MS = Number(process.env.PI_KNOWLEDGE_EMBEDDING_IDLE_MS ?? 30_000);
|
|
8
|
+
const EMBEDDING_CONFIG = process.env.PI_KNOWLEDGE_EMBEDDING ?? "local:multilingual-e5-small";
|
|
9
|
+
const ENABLE_NATIVE_IDLE_DISPOSE = process.env.PI_KNOWLEDGE_ENABLE_NATIVE_IDLE_DISPOSE === "true";
|
|
10
|
+
function clearIdleTimer() {
|
|
11
|
+
if (disposeTimer)
|
|
12
|
+
clearTimeout(disposeTimer);
|
|
13
|
+
disposeTimer = null;
|
|
14
|
+
}
|
|
15
|
+
function scheduleIdleDispose() {
|
|
16
|
+
if (activeRuns > 0 || disposeRequested)
|
|
17
|
+
return;
|
|
18
|
+
clearIdleTimer();
|
|
19
|
+
if (!ENABLE_NATIVE_IDLE_DISPOSE)
|
|
20
|
+
return;
|
|
21
|
+
disposeTimer = setTimeout(() => dispose(), IDLE_TIMEOUT_MS);
|
|
22
|
+
}
|
|
23
|
+
function beginRun() {
|
|
24
|
+
activeRuns++;
|
|
25
|
+
clearIdleTimer();
|
|
26
|
+
}
|
|
27
|
+
function endRun() {
|
|
28
|
+
activeRuns--;
|
|
29
|
+
if (activeRuns > 0)
|
|
30
|
+
return;
|
|
31
|
+
for (const resolve of idleWaiters.splice(0))
|
|
32
|
+
resolve();
|
|
33
|
+
if (!disposeRequested)
|
|
34
|
+
scheduleIdleDispose();
|
|
35
|
+
}
|
|
36
|
+
function waitForNoActiveRuns() {
|
|
37
|
+
if (activeRuns === 0)
|
|
38
|
+
return Promise.resolve();
|
|
39
|
+
return new Promise((resolve) => idleWaiters.push(resolve));
|
|
40
|
+
}
|
|
41
|
+
export async function dispose() {
|
|
42
|
+
clearIdleTimer();
|
|
43
|
+
if (disposePromise)
|
|
44
|
+
return disposePromise;
|
|
45
|
+
disposeRequested = true;
|
|
46
|
+
await waitForNoActiveRuns();
|
|
47
|
+
disposePromise = Promise.resolve().finally(() => {
|
|
48
|
+
disposePromise = null;
|
|
49
|
+
disposeRequested = false;
|
|
50
|
+
});
|
|
51
|
+
return disposePromise;
|
|
52
|
+
}
|
|
53
|
+
export async function prepareForShutdown() {
|
|
54
|
+
clearIdleTimer();
|
|
55
|
+
await waitForNoActiveRuns();
|
|
56
|
+
}
|
|
57
|
+
async function embedViaAPI(texts, prefix) {
|
|
58
|
+
const [provider, model] = EMBEDDING_CONFIG.split(":");
|
|
59
|
+
const prefixedTexts = texts.map((t) => `${prefix}: ${t}`);
|
|
60
|
+
if (provider === "openai") {
|
|
61
|
+
const apiKey = process.env.OPENAI_API_KEY;
|
|
62
|
+
if (!apiKey)
|
|
63
|
+
throw new Error("OPENAI_API_KEY required for openai embedding");
|
|
64
|
+
const res = await fetch("https://api.openai.com/v1/embeddings", {
|
|
65
|
+
method: "POST",
|
|
66
|
+
headers: { "Content-Type": "application/json", Authorization: `Bearer ${apiKey}` },
|
|
67
|
+
body: JSON.stringify({ input: prefixedTexts, model: model || "text-embedding-3-small" }),
|
|
68
|
+
});
|
|
69
|
+
if (!res.ok)
|
|
70
|
+
throw new Error(`OpenAI API error: ${res.status}`);
|
|
71
|
+
const data = (await res.json());
|
|
72
|
+
return data.data.map((d) => new Float32Array(d.embedding));
|
|
73
|
+
}
|
|
74
|
+
throw new Error(`Unsupported embedding provider: ${provider}`);
|
|
75
|
+
}
|
|
76
|
+
export async function embedTexts(texts, prefix, signal) {
|
|
77
|
+
if (!EMBEDDING_CONFIG.startsWith("local")) {
|
|
78
|
+
try {
|
|
79
|
+
return await embedViaAPI(texts, prefix);
|
|
80
|
+
}
|
|
81
|
+
catch {
|
|
82
|
+
/* fallback to local */
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
beginRun();
|
|
86
|
+
try {
|
|
87
|
+
return await embedInModelWorker(texts, prefix, signal);
|
|
88
|
+
}
|
|
89
|
+
finally {
|
|
90
|
+
endRun();
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
export async function embedQuery(text) {
|
|
94
|
+
const [vec] = await embedTexts([text], "query");
|
|
95
|
+
return vec;
|
|
96
|
+
}
|
|
97
|
+
export async function embedDocuments(texts, signal) {
|
|
98
|
+
return embedTexts(texts, "passage", signal);
|
|
99
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
export interface VectorWriter {
|
|
2
|
+
append(vectors: Float32Array[]): void;
|
|
3
|
+
close(): void;
|
|
4
|
+
}
|
|
5
|
+
export interface VectorReader {
|
|
6
|
+
count: number;
|
|
7
|
+
dim: number;
|
|
8
|
+
read(index: number): Float32Array | undefined;
|
|
9
|
+
readInto(index: number, target: Float32Array): boolean;
|
|
10
|
+
close(): void;
|
|
11
|
+
}
|
|
12
|
+
export declare function openVectorWriter(path: string): VectorWriter;
|
|
13
|
+
export declare function openVectorReader(path: string): VectorReader | undefined;
|
|
14
|
+
export declare function saveVectors(path: string, vectors: Float32Array[]): void;
|
|
15
|
+
export declare function loadVectors(path: string): Float32Array[];
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
import { closeSync, existsSync, mkdirSync, openSync, readFileSync, readSync, writeSync } from "node:fs";
|
|
2
|
+
import { dirname } from "node:path";
|
|
3
|
+
export function openVectorWriter(path) {
|
|
4
|
+
const dir = dirname(path);
|
|
5
|
+
if (!existsSync(dir))
|
|
6
|
+
mkdirSync(dir, { recursive: true });
|
|
7
|
+
const fd = openSync(path, "w");
|
|
8
|
+
let count = 0;
|
|
9
|
+
let dim = 0;
|
|
10
|
+
let closed = false;
|
|
11
|
+
let byteOffset = 8;
|
|
12
|
+
writeSync(fd, Buffer.alloc(8), 0, 8, 0);
|
|
13
|
+
function writeHeader() {
|
|
14
|
+
const header = Buffer.alloc(8);
|
|
15
|
+
header.writeUInt32LE(count, 0);
|
|
16
|
+
header.writeUInt32LE(dim, 4);
|
|
17
|
+
writeSync(fd, header, 0, header.length, 0);
|
|
18
|
+
}
|
|
19
|
+
return {
|
|
20
|
+
append(vectors) {
|
|
21
|
+
if (closed)
|
|
22
|
+
throw new Error("Vector writer is already closed");
|
|
23
|
+
for (const vector of vectors) {
|
|
24
|
+
if (dim === 0)
|
|
25
|
+
dim = vector.length;
|
|
26
|
+
if (vector.length !== dim) {
|
|
27
|
+
throw new Error(`Vector dimension mismatch: expected ${dim}, got ${vector.length}`);
|
|
28
|
+
}
|
|
29
|
+
const bytes = Buffer.from(vector.buffer, vector.byteOffset, vector.byteLength);
|
|
30
|
+
writeSync(fd, bytes, 0, bytes.length, byteOffset);
|
|
31
|
+
byteOffset += bytes.length;
|
|
32
|
+
count++;
|
|
33
|
+
}
|
|
34
|
+
},
|
|
35
|
+
close() {
|
|
36
|
+
if (closed)
|
|
37
|
+
return;
|
|
38
|
+
writeHeader();
|
|
39
|
+
closeSync(fd);
|
|
40
|
+
closed = true;
|
|
41
|
+
},
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
export function openVectorReader(path) {
|
|
45
|
+
if (!existsSync(path))
|
|
46
|
+
return undefined;
|
|
47
|
+
const fd = openSync(path, "r");
|
|
48
|
+
let closed = false;
|
|
49
|
+
const header = Buffer.alloc(8);
|
|
50
|
+
const headerBytes = readSync(fd, header, 0, header.length, 0);
|
|
51
|
+
if (headerBytes < header.length) {
|
|
52
|
+
closeSync(fd);
|
|
53
|
+
return undefined;
|
|
54
|
+
}
|
|
55
|
+
const count = header.readUInt32LE(0);
|
|
56
|
+
const dim = header.readUInt32LE(4);
|
|
57
|
+
const vectorBytes = dim * 4;
|
|
58
|
+
const readBuffer = Buffer.alloc(vectorBytes);
|
|
59
|
+
return {
|
|
60
|
+
count,
|
|
61
|
+
dim,
|
|
62
|
+
read(index) {
|
|
63
|
+
const vector = new Float32Array(dim);
|
|
64
|
+
return this.readInto(index, vector) ? vector : undefined;
|
|
65
|
+
},
|
|
66
|
+
readInto(index, target) {
|
|
67
|
+
if (closed || index < 0 || index >= count || dim === 0 || target.length !== dim)
|
|
68
|
+
return false;
|
|
69
|
+
const bytesRead = readSync(fd, readBuffer, 0, vectorBytes, 8 + index * vectorBytes);
|
|
70
|
+
if (bytesRead < vectorBytes)
|
|
71
|
+
return false;
|
|
72
|
+
for (let i = 0; i < dim; i++)
|
|
73
|
+
target[i] = readBuffer.readFloatLE(i * 4);
|
|
74
|
+
return true;
|
|
75
|
+
},
|
|
76
|
+
close() {
|
|
77
|
+
if (closed)
|
|
78
|
+
return;
|
|
79
|
+
closeSync(fd);
|
|
80
|
+
closed = true;
|
|
81
|
+
},
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
export function saveVectors(path, vectors) {
|
|
85
|
+
const writer = openVectorWriter(path);
|
|
86
|
+
try {
|
|
87
|
+
writer.append(vectors);
|
|
88
|
+
}
|
|
89
|
+
finally {
|
|
90
|
+
writer.close();
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
export function loadVectors(path) {
|
|
94
|
+
if (!existsSync(path))
|
|
95
|
+
return [];
|
|
96
|
+
const buffer = readFileSync(path);
|
|
97
|
+
if (buffer.length < 8)
|
|
98
|
+
return [];
|
|
99
|
+
const count = buffer.readUInt32LE(0);
|
|
100
|
+
const dim = buffer.readUInt32LE(4);
|
|
101
|
+
const vectors = [];
|
|
102
|
+
for (let i = 0; i < count; i++) {
|
|
103
|
+
const offset = 8 + i * dim * 4;
|
|
104
|
+
const vec = new Float32Array(dim);
|
|
105
|
+
for (let j = 0; j < dim; j++) {
|
|
106
|
+
vec[j] = buffer.readFloatLE(offset + j * 4);
|
|
107
|
+
}
|
|
108
|
+
vectors.push(vec);
|
|
109
|
+
}
|
|
110
|
+
return vectors;
|
|
111
|
+
}
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import { type DiagnosticResult } from "./diagnostics/health.ts";
|
|
2
|
+
import { createSkippedScanStats } from "./indexer/chunker.ts";
|
|
3
|
+
import { type RankingDiagnostics } from "./search/ranking.ts";
|
|
4
|
+
import { type KnowledgeBase } from "./storage/sqlite.ts";
|
|
5
|
+
export interface SearchOptions {
|
|
6
|
+
mode?: "auto" | "fast" | "semantic" | "hybrid" | "deep" | "adaptive";
|
|
7
|
+
limit?: number;
|
|
8
|
+
offset?: number;
|
|
9
|
+
kb_id?: string;
|
|
10
|
+
filters?: {
|
|
11
|
+
file_type?: string;
|
|
12
|
+
path_pattern?: string;
|
|
13
|
+
};
|
|
14
|
+
diversity?: "off" | "balanced" | "strong";
|
|
15
|
+
}
|
|
16
|
+
export interface SearchResult {
|
|
17
|
+
content: string;
|
|
18
|
+
file_path: string;
|
|
19
|
+
file_type: string;
|
|
20
|
+
kb_name: string;
|
|
21
|
+
score: number;
|
|
22
|
+
snippet: string;
|
|
23
|
+
start_line: number;
|
|
24
|
+
end_line: number;
|
|
25
|
+
ranking?: RankingDiagnostics;
|
|
26
|
+
}
|
|
27
|
+
export declare const CURRENT_EMBEDDING_MODEL = "multilingual-e5-small";
|
|
28
|
+
export interface SearchResponse {
|
|
29
|
+
results: SearchResult[];
|
|
30
|
+
total_count: number;
|
|
31
|
+
has_more: boolean;
|
|
32
|
+
warnings?: string[];
|
|
33
|
+
mode_used?: NonNullable<SearchOptions["mode"]>;
|
|
34
|
+
retry_modes?: NonNullable<SearchOptions["mode"]>[];
|
|
35
|
+
suggestions?: string[];
|
|
36
|
+
}
|
|
37
|
+
export type ProgressCallback = (msg: string) => void;
|
|
38
|
+
export interface AddOptions {
|
|
39
|
+
include_suggested_text?: boolean;
|
|
40
|
+
include_paths?: string[];
|
|
41
|
+
exclude_paths?: string[];
|
|
42
|
+
}
|
|
43
|
+
export interface DoctorIssue {
|
|
44
|
+
severity: "blocking" | "warning" | "info";
|
|
45
|
+
kb_name?: string;
|
|
46
|
+
message: string;
|
|
47
|
+
action: string;
|
|
48
|
+
}
|
|
49
|
+
export interface DoctorReport {
|
|
50
|
+
health_score: number;
|
|
51
|
+
summary: string;
|
|
52
|
+
issues: DoctorIssue[];
|
|
53
|
+
diagnostics: DiagnosticResult[];
|
|
54
|
+
}
|
|
55
|
+
export interface IndexPlan {
|
|
56
|
+
source_type: "file" | "directory" | "text" | "url";
|
|
57
|
+
scannable_files: number;
|
|
58
|
+
scannable_bytes: number;
|
|
59
|
+
skipped: ReturnType<typeof createSkippedScanStats>;
|
|
60
|
+
summary: string;
|
|
61
|
+
}
|
|
62
|
+
export declare class KnowledgeEngine {
|
|
63
|
+
private db;
|
|
64
|
+
private knowledgeDir;
|
|
65
|
+
initialize(knowledgeDir: string): Promise<void>;
|
|
66
|
+
plan(source: string, options?: AddOptions): IndexPlan;
|
|
67
|
+
add(source: string, name: string, onProgress?: ProgressCallback, signal?: AbortSignal, options?: AddOptions): Promise<{
|
|
68
|
+
kb: KnowledgeBase;
|
|
69
|
+
chunkCount: number;
|
|
70
|
+
}>;
|
|
71
|
+
update(nameOrId: string, onProgress?: ProgressCallback, signal?: AbortSignal): Promise<{
|
|
72
|
+
added: number;
|
|
73
|
+
removed: number;
|
|
74
|
+
unchanged: number;
|
|
75
|
+
}>;
|
|
76
|
+
search(query: string, options?: SearchOptions): Promise<SearchResponse>;
|
|
77
|
+
remove(nameOrId: string): boolean;
|
|
78
|
+
list(): KnowledgeBase[];
|
|
79
|
+
clear(): void;
|
|
80
|
+
diagnose(): DiagnosticResult[];
|
|
81
|
+
doctor(): DoctorReport;
|
|
82
|
+
exportKB(nameOrId: string, outputPath: string): Promise<number>;
|
|
83
|
+
importKB(inputPath: string, onProgress?: ProgressCallback, signal?: AbortSignal): Promise<{
|
|
84
|
+
kb: KnowledgeBase;
|
|
85
|
+
chunkCount: number;
|
|
86
|
+
}>;
|
|
87
|
+
dispose(options?: {
|
|
88
|
+
disposeModels?: boolean;
|
|
89
|
+
}): Promise<void>;
|
|
90
|
+
}
|