pi-knowledge 0.3.4 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.pi/skills/search-docs.md +24 -4
- package/CHANGELOG.md +34 -0
- package/README.md +55 -3
- package/index.ts +263 -20
- package/package.json +1 -1
- package/src/diagnostics/health.ts +84 -14
- package/src/embedding/provider.ts +50 -12
- package/src/embedding/vectors.ts +95 -12
- package/src/engine.ts +1252 -177
- package/src/indexer/chunker.ts +331 -100
- package/src/indexer/chunkers/code-ast.ts +14 -11
- package/src/search/bm25.ts +37 -22
- package/src/search/fusion.ts +33 -0
- package/src/search/query.ts +59 -0
- package/src/search/ranking.ts +201 -0
- package/src/search/reranker.ts +49 -11
- package/src/search/vector.ts +60 -0
- package/src/storage/sqlite.ts +240 -9
|
@@ -1,28 +1,87 @@
|
|
|
1
1
|
import { existsSync, statSync } from "node:fs";
|
|
2
2
|
import { join } from "node:path";
|
|
3
3
|
import type Database from "better-sqlite3";
|
|
4
|
-
import {
|
|
5
|
-
|
|
4
|
+
import {
|
|
5
|
+
createSkippedScanStats,
|
|
6
|
+
iterateScannableFiles,
|
|
7
|
+
type ScanOptions,
|
|
8
|
+
type ScanResult,
|
|
9
|
+
} from "../indexer/chunker.ts";
|
|
10
|
+
import { getIndexingJob, type IndexingJob, iterateChunksByKB, type KnowledgeBase } from "../storage/sqlite.ts";
|
|
6
11
|
|
|
7
12
|
export interface DiagnosticResult {
|
|
8
13
|
kb_id: string;
|
|
9
14
|
kb_name: string;
|
|
15
|
+
status: KnowledgeBase["status"];
|
|
16
|
+
status_age_ms: number;
|
|
17
|
+
last_progress_age_ms: number;
|
|
18
|
+
stuck_indexing: boolean;
|
|
10
19
|
stale_files: string[]; // files modified after indexing
|
|
11
20
|
orphan_files: string[]; // chunks referencing deleted files
|
|
12
21
|
coverage_percent: number; // indexed files / total scannable files
|
|
13
22
|
total_source_files: number;
|
|
14
23
|
indexed_files: number;
|
|
24
|
+
skipped_files: ScanResult["skipped"];
|
|
25
|
+
job?: IndexingJob;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
const DEFAULT_STALE_INDEXING_MS = 10 * 60 * 1000;
|
|
29
|
+
|
|
30
|
+
function staleIndexingMs(): number {
|
|
31
|
+
const configured = Number(process.env.PI_KNOWLEDGE_STALE_INDEXING_MS);
|
|
32
|
+
return Number.isFinite(configured) && configured > 0 ? configured : DEFAULT_STALE_INDEXING_MS;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function scanOptionsFromSourceOptions(raw: string | null): ScanOptions {
|
|
36
|
+
if (!raw) return {};
|
|
37
|
+
try {
|
|
38
|
+
const parsed = JSON.parse(raw) as {
|
|
39
|
+
include_suggested_text?: unknown;
|
|
40
|
+
include_paths?: unknown;
|
|
41
|
+
exclude_paths?: unknown;
|
|
42
|
+
};
|
|
43
|
+
return {
|
|
44
|
+
includeSuggestedText: parsed.include_suggested_text === true,
|
|
45
|
+
includePaths: Array.isArray(parsed.include_paths)
|
|
46
|
+
? parsed.include_paths.filter((item) => typeof item === "string")
|
|
47
|
+
: undefined,
|
|
48
|
+
excludePaths: Array.isArray(parsed.exclude_paths)
|
|
49
|
+
? parsed.exclude_paths.filter((item) => typeof item === "string")
|
|
50
|
+
: undefined,
|
|
51
|
+
};
|
|
52
|
+
} catch {
|
|
53
|
+
return {};
|
|
54
|
+
}
|
|
15
55
|
}
|
|
16
56
|
|
|
17
57
|
export function diagnoseKB(db: Database.Database, kb: KnowledgeBase): DiagnosticResult {
|
|
58
|
+
const statusAgeMs = Date.now() - kb.updated_at;
|
|
59
|
+
const job = getIndexingJob(db, kb.id);
|
|
60
|
+
const lastProgressAgeMs = job?.status === "running" ? Date.now() - job.last_progress_at : statusAgeMs;
|
|
18
61
|
const result: DiagnosticResult = {
|
|
19
62
|
kb_id: kb.id,
|
|
20
63
|
kb_name: kb.name,
|
|
64
|
+
status: kb.status,
|
|
65
|
+
status_age_ms: statusAgeMs,
|
|
66
|
+
last_progress_age_ms: lastProgressAgeMs,
|
|
67
|
+
stuck_indexing: kb.status === "indexing" && lastProgressAgeMs > staleIndexingMs(),
|
|
21
68
|
stale_files: [],
|
|
22
69
|
orphan_files: [],
|
|
23
70
|
coverage_percent: 100,
|
|
24
71
|
total_source_files: 0,
|
|
25
72
|
indexed_files: kb.file_count,
|
|
73
|
+
skipped_files: {
|
|
74
|
+
total: 0,
|
|
75
|
+
by_reason: {
|
|
76
|
+
suggested_excluded: 0,
|
|
77
|
+
oversized: 0,
|
|
78
|
+
binary: 0,
|
|
79
|
+
unreadable: 0,
|
|
80
|
+
inaccessible: 0,
|
|
81
|
+
},
|
|
82
|
+
samples: [],
|
|
83
|
+
},
|
|
84
|
+
job,
|
|
26
85
|
};
|
|
27
86
|
|
|
28
87
|
if (!kb.source_path || kb.source_type === "url" || !existsSync(kb.source_path)) {
|
|
@@ -33,8 +92,19 @@ export function diagnoseKB(db: Database.Database, kb: KnowledgeBase): Diagnostic
|
|
|
33
92
|
const currentFiles = new Set<string>();
|
|
34
93
|
const isDirectory = statSync(kb.source_path).isDirectory();
|
|
35
94
|
try {
|
|
36
|
-
|
|
37
|
-
|
|
95
|
+
if (isDirectory) {
|
|
96
|
+
const skipped = createSkippedScanStats();
|
|
97
|
+
for (const file of iterateScannableFiles(
|
|
98
|
+
kb.source_path,
|
|
99
|
+
skipped,
|
|
100
|
+
scanOptionsFromSourceOptions(kb.source_options),
|
|
101
|
+
)) {
|
|
102
|
+
currentFiles.add(file.relPath);
|
|
103
|
+
}
|
|
104
|
+
result.skipped_files = skipped;
|
|
105
|
+
} else {
|
|
106
|
+
currentFiles.add(kb.source_path);
|
|
107
|
+
}
|
|
38
108
|
} catch {
|
|
39
109
|
return result;
|
|
40
110
|
}
|
|
@@ -42,9 +112,13 @@ export function diagnoseKB(db: Database.Database, kb: KnowledgeBase): Diagnostic
|
|
|
42
112
|
result.total_source_files = currentFiles.size;
|
|
43
113
|
result.coverage_percent = currentFiles.size > 0 ? Math.round((result.indexed_files / currentFiles.size) * 100) : 100;
|
|
44
114
|
|
|
45
|
-
|
|
46
|
-
const
|
|
47
|
-
const
|
|
115
|
+
const indexedFilePaths = new Set<string>();
|
|
116
|
+
const latestIndexedByFile = new Map<string, number>();
|
|
117
|
+
for (const chunk of iterateChunksByKB(db, kb.id)) {
|
|
118
|
+
indexedFilePaths.add(chunk.file_path);
|
|
119
|
+
const currentLatest = latestIndexedByFile.get(chunk.file_path) ?? 0;
|
|
120
|
+
if (chunk.indexed_at > currentLatest) latestIndexedByFile.set(chunk.file_path, chunk.indexed_at);
|
|
121
|
+
}
|
|
48
122
|
|
|
49
123
|
// Orphan detection: chunks referencing files no longer in source
|
|
50
124
|
for (const filePath of indexedFilePaths) {
|
|
@@ -58,13 +132,9 @@ export function diagnoseKB(db: Database.Database, kb: KnowledgeBase): Diagnostic
|
|
|
58
132
|
const absPath = isDirectory ? join(kb.source_path, relPath) : relPath;
|
|
59
133
|
try {
|
|
60
134
|
const mtime = statSync(absPath).mtimeMs;
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
const latestIndexed = Math.max(...fileChunks.map((c) => c.indexed_at));
|
|
65
|
-
if (mtime > latestIndexed) {
|
|
66
|
-
result.stale_files.push(relPath);
|
|
67
|
-
}
|
|
135
|
+
const latestIndexed = latestIndexedByFile.get(relPath);
|
|
136
|
+
if (latestIndexed !== undefined && mtime > latestIndexed) {
|
|
137
|
+
result.stale_files.push(relPath);
|
|
68
138
|
}
|
|
69
139
|
} catch {
|
|
70
140
|
/* file unreadable — skip */
|
|
@@ -9,8 +9,11 @@ type FeatureExtractionPipeline = {
|
|
|
9
9
|
let pipelineInstance: FeatureExtractionPipeline | null = null;
|
|
10
10
|
let disposeTimer: ReturnType<typeof setTimeout> | null = null;
|
|
11
11
|
let disposePromise: Promise<void> | null = null;
|
|
12
|
+
let activeRuns = 0;
|
|
13
|
+
let disposeRequested = false;
|
|
14
|
+
const idleWaiters: Array<() => void> = [];
|
|
12
15
|
|
|
13
|
-
const IDLE_TIMEOUT_MS = 30_000;
|
|
16
|
+
const IDLE_TIMEOUT_MS = Number(process.env.PI_KNOWLEDGE_EMBEDDING_IDLE_MS ?? 30_000);
|
|
14
17
|
const EMBEDDING_CONFIG = process.env.PI_KNOWLEDGE_EMBEDDING ?? "local:multilingual-e5-small";
|
|
15
18
|
|
|
16
19
|
function getModelCacheDir(): string {
|
|
@@ -29,26 +32,57 @@ async function loadPipeline(): Promise<FeatureExtractionPipeline> {
|
|
|
29
32
|
return pipelineInstance;
|
|
30
33
|
}
|
|
31
34
|
|
|
32
|
-
function
|
|
35
|
+
function clearIdleTimer(): void {
|
|
33
36
|
if (disposeTimer) clearTimeout(disposeTimer);
|
|
37
|
+
disposeTimer = null;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function scheduleIdleDispose(): void {
|
|
41
|
+
if (activeRuns > 0 || disposeRequested) return;
|
|
42
|
+
clearIdleTimer();
|
|
34
43
|
disposeTimer = setTimeout(() => dispose(), IDLE_TIMEOUT_MS);
|
|
35
44
|
}
|
|
36
45
|
|
|
46
|
+
function beginRun(): void {
|
|
47
|
+
activeRuns++;
|
|
48
|
+
clearIdleTimer();
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function endRun(): void {
|
|
52
|
+
activeRuns--;
|
|
53
|
+
if (activeRuns > 0) return;
|
|
54
|
+
for (const resolve of idleWaiters.splice(0)) resolve();
|
|
55
|
+
if (!disposeRequested) scheduleIdleDispose();
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function waitForNoActiveRuns(): Promise<void> {
|
|
59
|
+
if (activeRuns === 0) return Promise.resolve();
|
|
60
|
+
return new Promise((resolve) => idleWaiters.push(resolve));
|
|
61
|
+
}
|
|
62
|
+
|
|
37
63
|
export async function dispose(): Promise<void> {
|
|
38
|
-
|
|
39
|
-
clearTimeout(disposeTimer);
|
|
40
|
-
disposeTimer = null;
|
|
41
|
-
}
|
|
64
|
+
clearIdleTimer();
|
|
42
65
|
if (disposePromise) return disposePromise;
|
|
66
|
+
disposeRequested = true;
|
|
67
|
+
await waitForNoActiveRuns();
|
|
43
68
|
const instance = pipelineInstance;
|
|
44
69
|
pipelineInstance = null;
|
|
45
|
-
if (!instance)
|
|
70
|
+
if (!instance) {
|
|
71
|
+
disposeRequested = false;
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
46
74
|
disposePromise = Promise.resolve(instance.dispose()).finally(() => {
|
|
47
75
|
disposePromise = null;
|
|
76
|
+
disposeRequested = false;
|
|
48
77
|
});
|
|
49
78
|
return disposePromise;
|
|
50
79
|
}
|
|
51
80
|
|
|
81
|
+
export async function prepareForShutdown(): Promise<void> {
|
|
82
|
+
clearIdleTimer();
|
|
83
|
+
await waitForNoActiveRuns();
|
|
84
|
+
}
|
|
85
|
+
|
|
52
86
|
async function embedViaAPI(texts: string[], prefix: "query" | "passage"): Promise<Float32Array[]> {
|
|
53
87
|
const [provider, model] = EMBEDDING_CONFIG.split(":");
|
|
54
88
|
const prefixedTexts = texts.map((t) => `${prefix}: ${t}`);
|
|
@@ -82,12 +116,16 @@ export async function embedTexts(
|
|
|
82
116
|
}
|
|
83
117
|
}
|
|
84
118
|
const pipe = await loadPipeline();
|
|
85
|
-
|
|
119
|
+
beginRun();
|
|
86
120
|
const results: Float32Array[] = [];
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
121
|
+
try {
|
|
122
|
+
for (const text of texts) {
|
|
123
|
+
if (signal?.aborted) throw new Error("Cancelled");
|
|
124
|
+
const output = await pipe(`${prefix}: ${text}`, { pooling: "mean", normalize: true });
|
|
125
|
+
results.push(new Float32Array(output.data));
|
|
126
|
+
}
|
|
127
|
+
} finally {
|
|
128
|
+
endRun();
|
|
91
129
|
}
|
|
92
130
|
return results;
|
|
93
131
|
}
|
package/src/embedding/vectors.ts
CHANGED
|
@@ -1,20 +1,103 @@
|
|
|
1
|
-
import { existsSync, mkdirSync, readFileSync,
|
|
1
|
+
import { closeSync, existsSync, mkdirSync, openSync, readFileSync, readSync, writeSync } from "node:fs";
|
|
2
2
|
import { dirname } from "node:path";
|
|
3
3
|
|
|
4
|
-
export
|
|
5
|
-
|
|
4
|
+
export interface VectorWriter {
|
|
5
|
+
append(vectors: Float32Array[]): void;
|
|
6
|
+
close(): void;
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
export interface VectorReader {
|
|
10
|
+
count: number;
|
|
11
|
+
dim: number;
|
|
12
|
+
read(index: number): Float32Array | undefined;
|
|
13
|
+
readInto(index: number, target: Float32Array): boolean;
|
|
14
|
+
close(): void;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export function openVectorWriter(path: string): VectorWriter {
|
|
6
18
|
const dir = dirname(path);
|
|
7
19
|
if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
|
|
8
|
-
const
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
let
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
20
|
+
const fd = openSync(path, "w");
|
|
21
|
+
let count = 0;
|
|
22
|
+
let dim = 0;
|
|
23
|
+
let closed = false;
|
|
24
|
+
let byteOffset = 8;
|
|
25
|
+
writeSync(fd, Buffer.alloc(8), 0, 8, 0);
|
|
26
|
+
|
|
27
|
+
function writeHeader(): void {
|
|
28
|
+
const header = Buffer.alloc(8);
|
|
29
|
+
header.writeUInt32LE(count, 0);
|
|
30
|
+
header.writeUInt32LE(dim, 4);
|
|
31
|
+
writeSync(fd, header, 0, header.length, 0);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
return {
|
|
35
|
+
append(vectors: Float32Array[]): void {
|
|
36
|
+
if (closed) throw new Error("Vector writer is already closed");
|
|
37
|
+
for (const vector of vectors) {
|
|
38
|
+
if (dim === 0) dim = vector.length;
|
|
39
|
+
if (vector.length !== dim) {
|
|
40
|
+
throw new Error(`Vector dimension mismatch: expected ${dim}, got ${vector.length}`);
|
|
41
|
+
}
|
|
42
|
+
const bytes = Buffer.from(vector.buffer, vector.byteOffset, vector.byteLength);
|
|
43
|
+
writeSync(fd, bytes, 0, bytes.length, byteOffset);
|
|
44
|
+
byteOffset += bytes.length;
|
|
45
|
+
count++;
|
|
46
|
+
}
|
|
47
|
+
},
|
|
48
|
+
close(): void {
|
|
49
|
+
if (closed) return;
|
|
50
|
+
writeHeader();
|
|
51
|
+
closeSync(fd);
|
|
52
|
+
closed = true;
|
|
53
|
+
},
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export function openVectorReader(path: string): VectorReader | undefined {
|
|
58
|
+
if (!existsSync(path)) return undefined;
|
|
59
|
+
const fd = openSync(path, "r");
|
|
60
|
+
let closed = false;
|
|
61
|
+
const header = Buffer.alloc(8);
|
|
62
|
+
const headerBytes = readSync(fd, header, 0, header.length, 0);
|
|
63
|
+
if (headerBytes < header.length) {
|
|
64
|
+
closeSync(fd);
|
|
65
|
+
return undefined;
|
|
66
|
+
}
|
|
67
|
+
const count = header.readUInt32LE(0);
|
|
68
|
+
const dim = header.readUInt32LE(4);
|
|
69
|
+
const vectorBytes = dim * 4;
|
|
70
|
+
const readBuffer = Buffer.alloc(vectorBytes);
|
|
71
|
+
|
|
72
|
+
return {
|
|
73
|
+
count,
|
|
74
|
+
dim,
|
|
75
|
+
read(index: number): Float32Array | undefined {
|
|
76
|
+
const vector = new Float32Array(dim);
|
|
77
|
+
return this.readInto(index, vector) ? vector : undefined;
|
|
78
|
+
},
|
|
79
|
+
readInto(index: number, target: Float32Array): boolean {
|
|
80
|
+
if (closed || index < 0 || index >= count || dim === 0 || target.length !== dim) return false;
|
|
81
|
+
const bytesRead = readSync(fd, readBuffer, 0, vectorBytes, 8 + index * vectorBytes);
|
|
82
|
+
if (bytesRead < vectorBytes) return false;
|
|
83
|
+
for (let i = 0; i < dim; i++) target[i] = readBuffer.readFloatLE(i * 4);
|
|
84
|
+
return true;
|
|
85
|
+
},
|
|
86
|
+
close(): void {
|
|
87
|
+
if (closed) return;
|
|
88
|
+
closeSync(fd);
|
|
89
|
+
closed = true;
|
|
90
|
+
},
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
export function saveVectors(path: string, vectors: Float32Array[]): void {
|
|
95
|
+
const writer = openVectorWriter(path);
|
|
96
|
+
try {
|
|
97
|
+
writer.append(vectors);
|
|
98
|
+
} finally {
|
|
99
|
+
writer.close();
|
|
16
100
|
}
|
|
17
|
-
writeFileSync(path, buffer);
|
|
18
101
|
}
|
|
19
102
|
|
|
20
103
|
export function loadVectors(path: string): Float32Array[] {
|