pi-knowledge 0.3.6 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.pi/skills/search-docs.md +24 -4
- package/CHANGELOG.md +19 -2
- package/README.md +42 -3
- package/index.ts +201 -124
- package/package.json +1 -1
- package/src/diagnostics/health.ts +84 -14
- package/src/embedding/provider.ts +5 -39
- package/src/embedding/vectors.ts +95 -12
- package/src/engine.ts +882 -140
- package/src/indexer/chunker.ts +232 -24
- package/src/model-worker-client.ts +146 -0
- package/src/model-worker.ts +88 -0
- package/src/search/bm25.ts +15 -4
- package/src/search/ranking.ts +4 -0
- package/src/search/reranker.ts +5 -35
- package/src/search/vector.ts +60 -0
- package/src/storage/sqlite.ts +224 -9
|
@@ -1,28 +1,87 @@
|
|
|
1
1
|
import { existsSync, statSync } from "node:fs";
|
|
2
2
|
import { join } from "node:path";
|
|
3
3
|
import type Database from "better-sqlite3";
|
|
4
|
-
import {
|
|
5
|
-
|
|
4
|
+
import {
|
|
5
|
+
createSkippedScanStats,
|
|
6
|
+
iterateScannableFiles,
|
|
7
|
+
type ScanOptions,
|
|
8
|
+
type ScanResult,
|
|
9
|
+
} from "../indexer/chunker.ts";
|
|
10
|
+
import { getIndexingJob, type IndexingJob, iterateChunksByKB, type KnowledgeBase } from "../storage/sqlite.ts";
|
|
6
11
|
|
|
7
12
|
export interface DiagnosticResult {
|
|
8
13
|
kb_id: string;
|
|
9
14
|
kb_name: string;
|
|
15
|
+
status: KnowledgeBase["status"];
|
|
16
|
+
status_age_ms: number;
|
|
17
|
+
last_progress_age_ms: number;
|
|
18
|
+
stuck_indexing: boolean;
|
|
10
19
|
stale_files: string[]; // files modified after indexing
|
|
11
20
|
orphan_files: string[]; // chunks referencing deleted files
|
|
12
21
|
coverage_percent: number; // indexed files / total scannable files
|
|
13
22
|
total_source_files: number;
|
|
14
23
|
indexed_files: number;
|
|
24
|
+
skipped_files: ScanResult["skipped"];
|
|
25
|
+
job?: IndexingJob;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
const DEFAULT_STALE_INDEXING_MS = 10 * 60 * 1000;
|
|
29
|
+
|
|
30
|
+
function staleIndexingMs(): number {
|
|
31
|
+
const configured = Number(process.env.PI_KNOWLEDGE_STALE_INDEXING_MS);
|
|
32
|
+
return Number.isFinite(configured) && configured > 0 ? configured : DEFAULT_STALE_INDEXING_MS;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function scanOptionsFromSourceOptions(raw: string | null): ScanOptions {
|
|
36
|
+
if (!raw) return {};
|
|
37
|
+
try {
|
|
38
|
+
const parsed = JSON.parse(raw) as {
|
|
39
|
+
include_suggested_text?: unknown;
|
|
40
|
+
include_paths?: unknown;
|
|
41
|
+
exclude_paths?: unknown;
|
|
42
|
+
};
|
|
43
|
+
return {
|
|
44
|
+
includeSuggestedText: parsed.include_suggested_text === true,
|
|
45
|
+
includePaths: Array.isArray(parsed.include_paths)
|
|
46
|
+
? parsed.include_paths.filter((item) => typeof item === "string")
|
|
47
|
+
: undefined,
|
|
48
|
+
excludePaths: Array.isArray(parsed.exclude_paths)
|
|
49
|
+
? parsed.exclude_paths.filter((item) => typeof item === "string")
|
|
50
|
+
: undefined,
|
|
51
|
+
};
|
|
52
|
+
} catch {
|
|
53
|
+
return {};
|
|
54
|
+
}
|
|
15
55
|
}
|
|
16
56
|
|
|
17
57
|
export function diagnoseKB(db: Database.Database, kb: KnowledgeBase): DiagnosticResult {
|
|
58
|
+
const statusAgeMs = Date.now() - kb.updated_at;
|
|
59
|
+
const job = getIndexingJob(db, kb.id);
|
|
60
|
+
const lastProgressAgeMs = job?.status === "running" ? Date.now() - job.last_progress_at : statusAgeMs;
|
|
18
61
|
const result: DiagnosticResult = {
|
|
19
62
|
kb_id: kb.id,
|
|
20
63
|
kb_name: kb.name,
|
|
64
|
+
status: kb.status,
|
|
65
|
+
status_age_ms: statusAgeMs,
|
|
66
|
+
last_progress_age_ms: lastProgressAgeMs,
|
|
67
|
+
stuck_indexing: kb.status === "indexing" && lastProgressAgeMs > staleIndexingMs(),
|
|
21
68
|
stale_files: [],
|
|
22
69
|
orphan_files: [],
|
|
23
70
|
coverage_percent: 100,
|
|
24
71
|
total_source_files: 0,
|
|
25
72
|
indexed_files: kb.file_count,
|
|
73
|
+
skipped_files: {
|
|
74
|
+
total: 0,
|
|
75
|
+
by_reason: {
|
|
76
|
+
suggested_excluded: 0,
|
|
77
|
+
oversized: 0,
|
|
78
|
+
binary: 0,
|
|
79
|
+
unreadable: 0,
|
|
80
|
+
inaccessible: 0,
|
|
81
|
+
},
|
|
82
|
+
samples: [],
|
|
83
|
+
},
|
|
84
|
+
job,
|
|
26
85
|
};
|
|
27
86
|
|
|
28
87
|
if (!kb.source_path || kb.source_type === "url" || !existsSync(kb.source_path)) {
|
|
@@ -33,8 +92,19 @@ export function diagnoseKB(db: Database.Database, kb: KnowledgeBase): Diagnostic
|
|
|
33
92
|
const currentFiles = new Set<string>();
|
|
34
93
|
const isDirectory = statSync(kb.source_path).isDirectory();
|
|
35
94
|
try {
|
|
36
|
-
|
|
37
|
-
|
|
95
|
+
if (isDirectory) {
|
|
96
|
+
const skipped = createSkippedScanStats();
|
|
97
|
+
for (const file of iterateScannableFiles(
|
|
98
|
+
kb.source_path,
|
|
99
|
+
skipped,
|
|
100
|
+
scanOptionsFromSourceOptions(kb.source_options),
|
|
101
|
+
)) {
|
|
102
|
+
currentFiles.add(file.relPath);
|
|
103
|
+
}
|
|
104
|
+
result.skipped_files = skipped;
|
|
105
|
+
} else {
|
|
106
|
+
currentFiles.add(kb.source_path);
|
|
107
|
+
}
|
|
38
108
|
} catch {
|
|
39
109
|
return result;
|
|
40
110
|
}
|
|
@@ -42,9 +112,13 @@ export function diagnoseKB(db: Database.Database, kb: KnowledgeBase): Diagnostic
|
|
|
42
112
|
result.total_source_files = currentFiles.size;
|
|
43
113
|
result.coverage_percent = currentFiles.size > 0 ? Math.round((result.indexed_files / currentFiles.size) * 100) : 100;
|
|
44
114
|
|
|
45
|
-
|
|
46
|
-
const
|
|
47
|
-
const
|
|
115
|
+
const indexedFilePaths = new Set<string>();
|
|
116
|
+
const latestIndexedByFile = new Map<string, number>();
|
|
117
|
+
for (const chunk of iterateChunksByKB(db, kb.id)) {
|
|
118
|
+
indexedFilePaths.add(chunk.file_path);
|
|
119
|
+
const currentLatest = latestIndexedByFile.get(chunk.file_path) ?? 0;
|
|
120
|
+
if (chunk.indexed_at > currentLatest) latestIndexedByFile.set(chunk.file_path, chunk.indexed_at);
|
|
121
|
+
}
|
|
48
122
|
|
|
49
123
|
// Orphan detection: chunks referencing files no longer in source
|
|
50
124
|
for (const filePath of indexedFilePaths) {
|
|
@@ -58,13 +132,9 @@ export function diagnoseKB(db: Database.Database, kb: KnowledgeBase): Diagnostic
|
|
|
58
132
|
const absPath = isDirectory ? join(kb.source_path, relPath) : relPath;
|
|
59
133
|
try {
|
|
60
134
|
const mtime = statSync(absPath).mtimeMs;
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
const latestIndexed = Math.max(...fileChunks.map((c) => c.indexed_at));
|
|
65
|
-
if (mtime > latestIndexed) {
|
|
66
|
-
result.stale_files.push(relPath);
|
|
67
|
-
}
|
|
135
|
+
const latestIndexed = latestIndexedByFile.get(relPath);
|
|
136
|
+
if (latestIndexed !== undefined && mtime > latestIndexed) {
|
|
137
|
+
result.stale_files.push(relPath);
|
|
68
138
|
}
|
|
69
139
|
} catch {
|
|
70
140
|
/* file unreadable — skip */
|
|
@@ -1,12 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { getDefaultKnowledgeDir } from "../storage/sqlite.ts";
|
|
1
|
+
import { embedInModelWorker } from "../model-worker-client.ts";
|
|
3
2
|
|
|
4
|
-
type FeatureExtractionPipeline = {
|
|
5
|
-
(input: string, options: { pooling: "mean"; normalize: true }): Promise<{ data: ArrayLike<number> }>;
|
|
6
|
-
dispose(): Promise<void> | void;
|
|
7
|
-
};
|
|
8
|
-
|
|
9
|
-
let pipelineInstance: FeatureExtractionPipeline | null = null;
|
|
10
3
|
let disposeTimer: ReturnType<typeof setTimeout> | null = null;
|
|
11
4
|
let disposePromise: Promise<void> | null = null;
|
|
12
5
|
let activeRuns = 0;
|
|
@@ -15,22 +8,7 @@ const idleWaiters: Array<() => void> = [];
|
|
|
15
8
|
|
|
16
9
|
const IDLE_TIMEOUT_MS = Number(process.env.PI_KNOWLEDGE_EMBEDDING_IDLE_MS ?? 30_000);
|
|
17
10
|
const EMBEDDING_CONFIG = process.env.PI_KNOWLEDGE_EMBEDDING ?? "local:multilingual-e5-small";
|
|
18
|
-
|
|
19
|
-
function getModelCacheDir(): string {
|
|
20
|
-
return join(getDefaultKnowledgeDir(), "models");
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
async function loadPipeline(): Promise<FeatureExtractionPipeline> {
|
|
24
|
-
if (pipelineInstance) return pipelineInstance;
|
|
25
|
-
if (disposePromise) await disposePromise;
|
|
26
|
-
const { pipeline, env } = await import("@huggingface/transformers");
|
|
27
|
-
env.cacheDir = getModelCacheDir();
|
|
28
|
-
pipelineInstance = await pipeline("feature-extraction", "Xenova/multilingual-e5-small", {
|
|
29
|
-
quantized: true,
|
|
30
|
-
dtype: "fp32",
|
|
31
|
-
});
|
|
32
|
-
return pipelineInstance;
|
|
33
|
-
}
|
|
11
|
+
const ENABLE_NATIVE_IDLE_DISPOSE = process.env.PI_KNOWLEDGE_ENABLE_NATIVE_IDLE_DISPOSE === "true";
|
|
34
12
|
|
|
35
13
|
function clearIdleTimer(): void {
|
|
36
14
|
if (disposeTimer) clearTimeout(disposeTimer);
|
|
@@ -40,6 +18,7 @@ function clearIdleTimer(): void {
|
|
|
40
18
|
function scheduleIdleDispose(): void {
|
|
41
19
|
if (activeRuns > 0 || disposeRequested) return;
|
|
42
20
|
clearIdleTimer();
|
|
21
|
+
if (!ENABLE_NATIVE_IDLE_DISPOSE) return;
|
|
43
22
|
disposeTimer = setTimeout(() => dispose(), IDLE_TIMEOUT_MS);
|
|
44
23
|
}
|
|
45
24
|
|
|
@@ -65,13 +44,7 @@ export async function dispose(): Promise<void> {
|
|
|
65
44
|
if (disposePromise) return disposePromise;
|
|
66
45
|
disposeRequested = true;
|
|
67
46
|
await waitForNoActiveRuns();
|
|
68
|
-
|
|
69
|
-
pipelineInstance = null;
|
|
70
|
-
if (!instance) {
|
|
71
|
-
disposeRequested = false;
|
|
72
|
-
return;
|
|
73
|
-
}
|
|
74
|
-
disposePromise = Promise.resolve(instance.dispose()).finally(() => {
|
|
47
|
+
disposePromise = Promise.resolve().finally(() => {
|
|
75
48
|
disposePromise = null;
|
|
76
49
|
disposeRequested = false;
|
|
77
50
|
});
|
|
@@ -115,19 +88,12 @@ export async function embedTexts(
|
|
|
115
88
|
/* fallback to local */
|
|
116
89
|
}
|
|
117
90
|
}
|
|
118
|
-
const pipe = await loadPipeline();
|
|
119
91
|
beginRun();
|
|
120
|
-
const results: Float32Array[] = [];
|
|
121
92
|
try {
|
|
122
|
-
|
|
123
|
-
if (signal?.aborted) throw new Error("Cancelled");
|
|
124
|
-
const output = await pipe(`${prefix}: ${text}`, { pooling: "mean", normalize: true });
|
|
125
|
-
results.push(new Float32Array(output.data));
|
|
126
|
-
}
|
|
93
|
+
return await embedInModelWorker(texts, prefix, signal);
|
|
127
94
|
} finally {
|
|
128
95
|
endRun();
|
|
129
96
|
}
|
|
130
|
-
return results;
|
|
131
97
|
}
|
|
132
98
|
|
|
133
99
|
export async function embedQuery(text: string): Promise<Float32Array> {
|
package/src/embedding/vectors.ts
CHANGED
|
@@ -1,20 +1,103 @@
|
|
|
1
|
-
import { existsSync, mkdirSync, readFileSync,
|
|
1
|
+
import { closeSync, existsSync, mkdirSync, openSync, readFileSync, readSync, writeSync } from "node:fs";
|
|
2
2
|
import { dirname } from "node:path";
|
|
3
3
|
|
|
4
|
-
export
|
|
5
|
-
|
|
4
|
+
export interface VectorWriter {
|
|
5
|
+
append(vectors: Float32Array[]): void;
|
|
6
|
+
close(): void;
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
export interface VectorReader {
|
|
10
|
+
count: number;
|
|
11
|
+
dim: number;
|
|
12
|
+
read(index: number): Float32Array | undefined;
|
|
13
|
+
readInto(index: number, target: Float32Array): boolean;
|
|
14
|
+
close(): void;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export function openVectorWriter(path: string): VectorWriter {
|
|
6
18
|
const dir = dirname(path);
|
|
7
19
|
if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
|
|
8
|
-
const
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
let
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
20
|
+
const fd = openSync(path, "w");
|
|
21
|
+
let count = 0;
|
|
22
|
+
let dim = 0;
|
|
23
|
+
let closed = false;
|
|
24
|
+
let byteOffset = 8;
|
|
25
|
+
writeSync(fd, Buffer.alloc(8), 0, 8, 0);
|
|
26
|
+
|
|
27
|
+
function writeHeader(): void {
|
|
28
|
+
const header = Buffer.alloc(8);
|
|
29
|
+
header.writeUInt32LE(count, 0);
|
|
30
|
+
header.writeUInt32LE(dim, 4);
|
|
31
|
+
writeSync(fd, header, 0, header.length, 0);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
return {
|
|
35
|
+
append(vectors: Float32Array[]): void {
|
|
36
|
+
if (closed) throw new Error("Vector writer is already closed");
|
|
37
|
+
for (const vector of vectors) {
|
|
38
|
+
if (dim === 0) dim = vector.length;
|
|
39
|
+
if (vector.length !== dim) {
|
|
40
|
+
throw new Error(`Vector dimension mismatch: expected ${dim}, got ${vector.length}`);
|
|
41
|
+
}
|
|
42
|
+
const bytes = Buffer.from(vector.buffer, vector.byteOffset, vector.byteLength);
|
|
43
|
+
writeSync(fd, bytes, 0, bytes.length, byteOffset);
|
|
44
|
+
byteOffset += bytes.length;
|
|
45
|
+
count++;
|
|
46
|
+
}
|
|
47
|
+
},
|
|
48
|
+
close(): void {
|
|
49
|
+
if (closed) return;
|
|
50
|
+
writeHeader();
|
|
51
|
+
closeSync(fd);
|
|
52
|
+
closed = true;
|
|
53
|
+
},
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export function openVectorReader(path: string): VectorReader | undefined {
|
|
58
|
+
if (!existsSync(path)) return undefined;
|
|
59
|
+
const fd = openSync(path, "r");
|
|
60
|
+
let closed = false;
|
|
61
|
+
const header = Buffer.alloc(8);
|
|
62
|
+
const headerBytes = readSync(fd, header, 0, header.length, 0);
|
|
63
|
+
if (headerBytes < header.length) {
|
|
64
|
+
closeSync(fd);
|
|
65
|
+
return undefined;
|
|
66
|
+
}
|
|
67
|
+
const count = header.readUInt32LE(0);
|
|
68
|
+
const dim = header.readUInt32LE(4);
|
|
69
|
+
const vectorBytes = dim * 4;
|
|
70
|
+
const readBuffer = Buffer.alloc(vectorBytes);
|
|
71
|
+
|
|
72
|
+
return {
|
|
73
|
+
count,
|
|
74
|
+
dim,
|
|
75
|
+
read(index: number): Float32Array | undefined {
|
|
76
|
+
const vector = new Float32Array(dim);
|
|
77
|
+
return this.readInto(index, vector) ? vector : undefined;
|
|
78
|
+
},
|
|
79
|
+
readInto(index: number, target: Float32Array): boolean {
|
|
80
|
+
if (closed || index < 0 || index >= count || dim === 0 || target.length !== dim) return false;
|
|
81
|
+
const bytesRead = readSync(fd, readBuffer, 0, vectorBytes, 8 + index * vectorBytes);
|
|
82
|
+
if (bytesRead < vectorBytes) return false;
|
|
83
|
+
for (let i = 0; i < dim; i++) target[i] = readBuffer.readFloatLE(i * 4);
|
|
84
|
+
return true;
|
|
85
|
+
},
|
|
86
|
+
close(): void {
|
|
87
|
+
if (closed) return;
|
|
88
|
+
closeSync(fd);
|
|
89
|
+
closed = true;
|
|
90
|
+
},
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
export function saveVectors(path: string, vectors: Float32Array[]): void {
|
|
95
|
+
const writer = openVectorWriter(path);
|
|
96
|
+
try {
|
|
97
|
+
writer.append(vectors);
|
|
98
|
+
} finally {
|
|
99
|
+
writer.close();
|
|
16
100
|
}
|
|
17
|
-
writeFileSync(path, buffer);
|
|
18
101
|
}
|
|
19
102
|
|
|
20
103
|
export function loadVectors(path: string): Float32Array[] {
|