pi-knowledge 0.4.1 → 0.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/CHANGELOG.md +18 -0
  2. package/README.md +29 -8
  3. package/dist/index.d.ts +38 -0
  4. package/dist/index.js +435 -0
  5. package/dist/src/diagnostics/health.d.ts +19 -0
  6. package/dist/src/diagnostics/health.js +110 -0
  7. package/dist/src/embedding/provider.d.ts +5 -0
  8. package/dist/src/embedding/provider.js +99 -0
  9. package/dist/src/embedding/vectors.d.ts +15 -0
  10. package/dist/src/embedding/vectors.js +111 -0
  11. package/dist/src/engine.d.ts +90 -0
  12. package/dist/src/engine.js +1370 -0
  13. package/dist/src/indexer/chunker.d.ts +53 -0
  14. package/dist/src/indexer/chunker.js +498 -0
  15. package/dist/src/indexer/chunkers/code-ast.d.ts +3 -0
  16. package/dist/src/indexer/chunkers/code-ast.js +179 -0
  17. package/dist/src/model-worker-client.d.ts +10 -0
  18. package/dist/src/model-worker-client.js +138 -0
  19. package/dist/src/model-worker.d.ts +1 -0
  20. package/dist/src/model-worker.js +61 -0
  21. package/dist/src/search/bm25.d.ts +8 -0
  22. package/dist/src/search/bm25.js +38 -0
  23. package/dist/src/search/fusion.d.ts +19 -0
  24. package/dist/src/search/fusion.js +34 -0
  25. package/dist/src/search/query.d.ts +5 -0
  26. package/dist/src/search/query.js +57 -0
  27. package/dist/src/search/ranking.d.ts +27 -0
  28. package/dist/src/search/ranking.js +180 -0
  29. package/dist/src/search/reranker.d.ts +10 -0
  30. package/dist/src/search/reranker.js +66 -0
  31. package/dist/src/search/vector.d.ts +14 -0
  32. package/dist/src/search/vector.js +58 -0
  33. package/dist/src/storage/sqlite.d.ts +99 -0
  34. package/dist/src/storage/sqlite.js +330 -0
  35. package/dist/src/watcher/file-watcher.d.ts +4 -0
  36. package/dist/src/watcher/file-watcher.js +108 -0
  37. package/extension.js +9 -0
  38. package/package.json +20 -6
  39. package/index.ts +0 -500
  40. package/src/diagnostics/health.ts +0 -145
  41. package/src/embedding/provider.ts +0 -106
  42. package/src/embedding/vectors.ts +0 -119
  43. package/src/engine.ts +0 -1592
  44. package/src/indexer/chunker.ts +0 -612
  45. package/src/indexer/chunkers/code-ast.ts +0 -209
  46. package/src/model-worker-client.ts +0 -146
  47. package/src/model-worker.ts +0 -88
  48. package/src/search/bm25.ts +0 -55
  49. package/src/search/fusion.ts +0 -48
  50. package/src/search/query.ts +0 -59
  51. package/src/search/ranking.ts +0 -201
  52. package/src/search/reranker.ts +0 -74
  53. package/src/search/vector.ts +0 -81
  54. package/src/storage/sqlite.ts +0 -489
  55. package/src/watcher/file-watcher.ts +0 -116
@@ -0,0 +1,110 @@
1
+ import { existsSync, statSync } from "node:fs";
2
+ import { join } from "node:path";
3
+ import { createSkippedScanStats, iterateScannableFiles, } from "../indexer/chunker.js";
4
+ import { getIndexingJob, iterateChunksByKB } from "../storage/sqlite.js";
5
+ const DEFAULT_STALE_INDEXING_MS = 10 * 60 * 1000;
6
+ function staleIndexingMs() {
7
+ const configured = Number(process.env.PI_KNOWLEDGE_STALE_INDEXING_MS);
8
+ return Number.isFinite(configured) && configured > 0 ? configured : DEFAULT_STALE_INDEXING_MS;
9
+ }
10
+ function scanOptionsFromSourceOptions(raw) {
11
+ if (!raw)
12
+ return {};
13
+ try {
14
+ const parsed = JSON.parse(raw);
15
+ return {
16
+ includeSuggestedText: parsed.include_suggested_text === true,
17
+ includePaths: Array.isArray(parsed.include_paths)
18
+ ? parsed.include_paths.filter((item) => typeof item === "string")
19
+ : undefined,
20
+ excludePaths: Array.isArray(parsed.exclude_paths)
21
+ ? parsed.exclude_paths.filter((item) => typeof item === "string")
22
+ : undefined,
23
+ };
24
+ }
25
+ catch {
26
+ return {};
27
+ }
28
+ }
29
+ export function diagnoseKB(db, kb) {
30
+ const statusAgeMs = Date.now() - kb.updated_at;
31
+ const job = getIndexingJob(db, kb.id);
32
+ const lastProgressAgeMs = job?.status === "running" ? Date.now() - job.last_progress_at : statusAgeMs;
33
+ const result = {
34
+ kb_id: kb.id,
35
+ kb_name: kb.name,
36
+ status: kb.status,
37
+ status_age_ms: statusAgeMs,
38
+ last_progress_age_ms: lastProgressAgeMs,
39
+ stuck_indexing: kb.status === "indexing" && lastProgressAgeMs > staleIndexingMs(),
40
+ stale_files: [],
41
+ orphan_files: [],
42
+ coverage_percent: 100,
43
+ total_source_files: 0,
44
+ indexed_files: kb.file_count,
45
+ skipped_files: {
46
+ total: 0,
47
+ by_reason: {
48
+ suggested_excluded: 0,
49
+ oversized: 0,
50
+ binary: 0,
51
+ unreadable: 0,
52
+ inaccessible: 0,
53
+ },
54
+ samples: [],
55
+ },
56
+ job,
57
+ };
58
+ if (!kb.source_path || kb.source_type === "url" || !existsSync(kb.source_path)) {
59
+ return result; // text KBs or missing source — no diagnostics possible
60
+ }
61
+ // Scan current source files
62
+ const currentFiles = new Set();
63
+ const isDirectory = statSync(kb.source_path).isDirectory();
64
+ try {
65
+ if (isDirectory) {
66
+ const skipped = createSkippedScanStats();
67
+ for (const file of iterateScannableFiles(kb.source_path, skipped, scanOptionsFromSourceOptions(kb.source_options))) {
68
+ currentFiles.add(file.relPath);
69
+ }
70
+ result.skipped_files = skipped;
71
+ }
72
+ else {
73
+ currentFiles.add(kb.source_path);
74
+ }
75
+ }
76
+ catch {
77
+ return result;
78
+ }
79
+ result.total_source_files = currentFiles.size;
80
+ result.coverage_percent = currentFiles.size > 0 ? Math.round((result.indexed_files / currentFiles.size) * 100) : 100;
81
+ const indexedFilePaths = new Set();
82
+ const latestIndexedByFile = new Map();
83
+ for (const chunk of iterateChunksByKB(db, kb.id)) {
84
+ indexedFilePaths.add(chunk.file_path);
85
+ const currentLatest = latestIndexedByFile.get(chunk.file_path) ?? 0;
86
+ if (chunk.indexed_at > currentLatest)
87
+ latestIndexedByFile.set(chunk.file_path, chunk.indexed_at);
88
+ }
89
+ // Orphan detection: chunks referencing files no longer in source
90
+ for (const filePath of indexedFilePaths) {
91
+ if (!currentFiles.has(filePath)) {
92
+ result.orphan_files.push(filePath);
93
+ }
94
+ }
95
+ // Staleness detection: source files modified after last indexing
96
+ for (const relPath of currentFiles) {
97
+ const absPath = isDirectory ? join(kb.source_path, relPath) : relPath;
98
+ try {
99
+ const mtime = statSync(absPath).mtimeMs;
100
+ const latestIndexed = latestIndexedByFile.get(relPath);
101
+ if (latestIndexed !== undefined && mtime > latestIndexed) {
102
+ result.stale_files.push(relPath);
103
+ }
104
+ }
105
+ catch {
106
+ /* file unreadable — skip */
107
+ }
108
+ }
109
+ return result;
110
+ }
@@ -0,0 +1,5 @@
1
+ export declare function dispose(): Promise<void>;
2
+ export declare function prepareForShutdown(): Promise<void>;
3
+ export declare function embedTexts(texts: string[], prefix: "query" | "passage", signal?: AbortSignal): Promise<Float32Array[]>;
4
+ export declare function embedQuery(text: string): Promise<Float32Array>;
5
+ export declare function embedDocuments(texts: string[], signal?: AbortSignal): Promise<Float32Array[]>;
@@ -0,0 +1,99 @@
1
+ import { embedInModelWorker } from "../model-worker-client.js";
2
+ let disposeTimer = null;
3
+ let disposePromise = null;
4
+ let activeRuns = 0;
5
+ let disposeRequested = false;
6
+ const idleWaiters = [];
7
+ const IDLE_TIMEOUT_MS = Number(process.env.PI_KNOWLEDGE_EMBEDDING_IDLE_MS ?? 30_000);
8
+ const EMBEDDING_CONFIG = process.env.PI_KNOWLEDGE_EMBEDDING ?? "local:multilingual-e5-small";
9
+ const ENABLE_NATIVE_IDLE_DISPOSE = process.env.PI_KNOWLEDGE_ENABLE_NATIVE_IDLE_DISPOSE === "true";
10
+ function clearIdleTimer() {
11
+ if (disposeTimer)
12
+ clearTimeout(disposeTimer);
13
+ disposeTimer = null;
14
+ }
15
+ function scheduleIdleDispose() {
16
+ if (activeRuns > 0 || disposeRequested)
17
+ return;
18
+ clearIdleTimer();
19
+ if (!ENABLE_NATIVE_IDLE_DISPOSE)
20
+ return;
21
+ disposeTimer = setTimeout(() => dispose(), IDLE_TIMEOUT_MS);
22
+ }
23
+ function beginRun() {
24
+ activeRuns++;
25
+ clearIdleTimer();
26
+ }
27
+ function endRun() {
28
+ activeRuns--;
29
+ if (activeRuns > 0)
30
+ return;
31
+ for (const resolve of idleWaiters.splice(0))
32
+ resolve();
33
+ if (!disposeRequested)
34
+ scheduleIdleDispose();
35
+ }
36
+ function waitForNoActiveRuns() {
37
+ if (activeRuns === 0)
38
+ return Promise.resolve();
39
+ return new Promise((resolve) => idleWaiters.push(resolve));
40
+ }
41
+ export async function dispose() {
42
+ clearIdleTimer();
43
+ if (disposePromise)
44
+ return disposePromise;
45
+ disposeRequested = true;
46
+ await waitForNoActiveRuns();
47
+ disposePromise = Promise.resolve().finally(() => {
48
+ disposePromise = null;
49
+ disposeRequested = false;
50
+ });
51
+ return disposePromise;
52
+ }
53
+ export async function prepareForShutdown() {
54
+ clearIdleTimer();
55
+ await waitForNoActiveRuns();
56
+ }
57
+ async function embedViaAPI(texts, prefix) {
58
+ const [provider, model] = EMBEDDING_CONFIG.split(":");
59
+ const prefixedTexts = texts.map((t) => `${prefix}: ${t}`);
60
+ if (provider === "openai") {
61
+ const apiKey = process.env.OPENAI_API_KEY;
62
+ if (!apiKey)
63
+ throw new Error("OPENAI_API_KEY required for openai embedding");
64
+ const res = await fetch("https://api.openai.com/v1/embeddings", {
65
+ method: "POST",
66
+ headers: { "Content-Type": "application/json", Authorization: `Bearer ${apiKey}` },
67
+ body: JSON.stringify({ input: prefixedTexts, model: model || "text-embedding-3-small" }),
68
+ });
69
+ if (!res.ok)
70
+ throw new Error(`OpenAI API error: ${res.status}`);
71
+ const data = (await res.json());
72
+ return data.data.map((d) => new Float32Array(d.embedding));
73
+ }
74
+ throw new Error(`Unsupported embedding provider: ${provider}`);
75
+ }
76
+ export async function embedTexts(texts, prefix, signal) {
77
+ if (!EMBEDDING_CONFIG.startsWith("local")) {
78
+ try {
79
+ return await embedViaAPI(texts, prefix);
80
+ }
81
+ catch {
82
+ /* fallback to local */
83
+ }
84
+ }
85
+ beginRun();
86
+ try {
87
+ return await embedInModelWorker(texts, prefix, signal);
88
+ }
89
+ finally {
90
+ endRun();
91
+ }
92
+ }
93
+ export async function embedQuery(text) {
94
+ const [vec] = await embedTexts([text], "query");
95
+ return vec;
96
+ }
97
+ export async function embedDocuments(texts, signal) {
98
+ return embedTexts(texts, "passage", signal);
99
+ }
@@ -0,0 +1,15 @@
1
+ export interface VectorWriter {
2
+ append(vectors: Float32Array[]): void;
3
+ close(): void;
4
+ }
5
+ export interface VectorReader {
6
+ count: number;
7
+ dim: number;
8
+ read(index: number): Float32Array | undefined;
9
+ readInto(index: number, target: Float32Array): boolean;
10
+ close(): void;
11
+ }
12
+ export declare function openVectorWriter(path: string): VectorWriter;
13
+ export declare function openVectorReader(path: string): VectorReader | undefined;
14
+ export declare function saveVectors(path: string, vectors: Float32Array[]): void;
15
+ export declare function loadVectors(path: string): Float32Array[];
@@ -0,0 +1,111 @@
1
+ import { closeSync, existsSync, mkdirSync, openSync, readFileSync, readSync, writeSync } from "node:fs";
2
+ import { dirname } from "node:path";
3
+ export function openVectorWriter(path) {
4
+ const dir = dirname(path);
5
+ if (!existsSync(dir))
6
+ mkdirSync(dir, { recursive: true });
7
+ const fd = openSync(path, "w");
8
+ let count = 0;
9
+ let dim = 0;
10
+ let closed = false;
11
+ let byteOffset = 8;
12
+ writeSync(fd, Buffer.alloc(8), 0, 8, 0);
13
+ function writeHeader() {
14
+ const header = Buffer.alloc(8);
15
+ header.writeUInt32LE(count, 0);
16
+ header.writeUInt32LE(dim, 4);
17
+ writeSync(fd, header, 0, header.length, 0);
18
+ }
19
+ return {
20
+ append(vectors) {
21
+ if (closed)
22
+ throw new Error("Vector writer is already closed");
23
+ for (const vector of vectors) {
24
+ if (dim === 0)
25
+ dim = vector.length;
26
+ if (vector.length !== dim) {
27
+ throw new Error(`Vector dimension mismatch: expected ${dim}, got ${vector.length}`);
28
+ }
29
+ const bytes = Buffer.from(vector.buffer, vector.byteOffset, vector.byteLength);
30
+ writeSync(fd, bytes, 0, bytes.length, byteOffset);
31
+ byteOffset += bytes.length;
32
+ count++;
33
+ }
34
+ },
35
+ close() {
36
+ if (closed)
37
+ return;
38
+ writeHeader();
39
+ closeSync(fd);
40
+ closed = true;
41
+ },
42
+ };
43
+ }
44
+ export function openVectorReader(path) {
45
+ if (!existsSync(path))
46
+ return undefined;
47
+ const fd = openSync(path, "r");
48
+ let closed = false;
49
+ const header = Buffer.alloc(8);
50
+ const headerBytes = readSync(fd, header, 0, header.length, 0);
51
+ if (headerBytes < header.length) {
52
+ closeSync(fd);
53
+ return undefined;
54
+ }
55
+ const count = header.readUInt32LE(0);
56
+ const dim = header.readUInt32LE(4);
57
+ const vectorBytes = dim * 4;
58
+ const readBuffer = Buffer.alloc(vectorBytes);
59
+ return {
60
+ count,
61
+ dim,
62
+ read(index) {
63
+ const vector = new Float32Array(dim);
64
+ return this.readInto(index, vector) ? vector : undefined;
65
+ },
66
+ readInto(index, target) {
67
+ if (closed || index < 0 || index >= count || dim === 0 || target.length !== dim)
68
+ return false;
69
+ const bytesRead = readSync(fd, readBuffer, 0, vectorBytes, 8 + index * vectorBytes);
70
+ if (bytesRead < vectorBytes)
71
+ return false;
72
+ for (let i = 0; i < dim; i++)
73
+ target[i] = readBuffer.readFloatLE(i * 4);
74
+ return true;
75
+ },
76
+ close() {
77
+ if (closed)
78
+ return;
79
+ closeSync(fd);
80
+ closed = true;
81
+ },
82
+ };
83
+ }
84
+ export function saveVectors(path, vectors) {
85
+ const writer = openVectorWriter(path);
86
+ try {
87
+ writer.append(vectors);
88
+ }
89
+ finally {
90
+ writer.close();
91
+ }
92
+ }
93
+ export function loadVectors(path) {
94
+ if (!existsSync(path))
95
+ return [];
96
+ const buffer = readFileSync(path);
97
+ if (buffer.length < 8)
98
+ return [];
99
+ const count = buffer.readUInt32LE(0);
100
+ const dim = buffer.readUInt32LE(4);
101
+ const vectors = [];
102
+ for (let i = 0; i < count; i++) {
103
+ const offset = 8 + i * dim * 4;
104
+ const vec = new Float32Array(dim);
105
+ for (let j = 0; j < dim; j++) {
106
+ vec[j] = buffer.readFloatLE(offset + j * 4);
107
+ }
108
+ vectors.push(vec);
109
+ }
110
+ return vectors;
111
+ }
@@ -0,0 +1,90 @@
1
+ import { type DiagnosticResult } from "./diagnostics/health.ts";
2
+ import { createSkippedScanStats } from "./indexer/chunker.ts";
3
+ import { type RankingDiagnostics } from "./search/ranking.ts";
4
+ import { type KnowledgeBase } from "./storage/sqlite.ts";
5
+ export interface SearchOptions {
6
+ mode?: "auto" | "fast" | "semantic" | "hybrid" | "deep" | "adaptive";
7
+ limit?: number;
8
+ offset?: number;
9
+ kb_id?: string;
10
+ filters?: {
11
+ file_type?: string;
12
+ path_pattern?: string;
13
+ };
14
+ diversity?: "off" | "balanced" | "strong";
15
+ }
16
+ export interface SearchResult {
17
+ content: string;
18
+ file_path: string;
19
+ file_type: string;
20
+ kb_name: string;
21
+ score: number;
22
+ snippet: string;
23
+ start_line: number;
24
+ end_line: number;
25
+ ranking?: RankingDiagnostics;
26
+ }
27
+ export declare const CURRENT_EMBEDDING_MODEL = "multilingual-e5-small";
28
+ export interface SearchResponse {
29
+ results: SearchResult[];
30
+ total_count: number;
31
+ has_more: boolean;
32
+ warnings?: string[];
33
+ mode_used?: NonNullable<SearchOptions["mode"]>;
34
+ retry_modes?: NonNullable<SearchOptions["mode"]>[];
35
+ suggestions?: string[];
36
+ }
37
+ export type ProgressCallback = (msg: string) => void;
38
+ export interface AddOptions {
39
+ include_suggested_text?: boolean;
40
+ include_paths?: string[];
41
+ exclude_paths?: string[];
42
+ }
43
+ export interface DoctorIssue {
44
+ severity: "blocking" | "warning" | "info";
45
+ kb_name?: string;
46
+ message: string;
47
+ action: string;
48
+ }
49
+ export interface DoctorReport {
50
+ health_score: number;
51
+ summary: string;
52
+ issues: DoctorIssue[];
53
+ diagnostics: DiagnosticResult[];
54
+ }
55
+ export interface IndexPlan {
56
+ source_type: "file" | "directory" | "text" | "url";
57
+ scannable_files: number;
58
+ scannable_bytes: number;
59
+ skipped: ReturnType<typeof createSkippedScanStats>;
60
+ summary: string;
61
+ }
62
+ export declare class KnowledgeEngine {
63
+ private db;
64
+ private knowledgeDir;
65
+ initialize(knowledgeDir: string): Promise<void>;
66
+ plan(source: string, options?: AddOptions): IndexPlan;
67
+ add(source: string, name: string, onProgress?: ProgressCallback, signal?: AbortSignal, options?: AddOptions): Promise<{
68
+ kb: KnowledgeBase;
69
+ chunkCount: number;
70
+ }>;
71
+ update(nameOrId: string, onProgress?: ProgressCallback, signal?: AbortSignal): Promise<{
72
+ added: number;
73
+ removed: number;
74
+ unchanged: number;
75
+ }>;
76
+ search(query: string, options?: SearchOptions): Promise<SearchResponse>;
77
+ remove(nameOrId: string): boolean;
78
+ list(): KnowledgeBase[];
79
+ clear(): void;
80
+ diagnose(): DiagnosticResult[];
81
+ doctor(): DoctorReport;
82
+ exportKB(nameOrId: string, outputPath: string): Promise<number>;
83
+ importKB(inputPath: string, onProgress?: ProgressCallback, signal?: AbortSignal): Promise<{
84
+ kb: KnowledgeBase;
85
+ chunkCount: number;
86
+ }>;
87
+ dispose(options?: {
88
+ disposeModels?: boolean;
89
+ }): Promise<void>;
90
+ }