pi-knowledge 0.3.6 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,28 +1,87 @@
1
1
  import { existsSync, statSync } from "node:fs";
2
2
  import { join } from "node:path";
3
3
  import type Database from "better-sqlite3";
4
- import { walkDir } from "../indexer/chunker.ts";
5
- import { getChunksByKB, type KnowledgeBase } from "../storage/sqlite.ts";
4
+ import {
5
+ createSkippedScanStats,
6
+ iterateScannableFiles,
7
+ type ScanOptions,
8
+ type ScanResult,
9
+ } from "../indexer/chunker.ts";
10
+ import { getIndexingJob, type IndexingJob, iterateChunksByKB, type KnowledgeBase } from "../storage/sqlite.ts";
6
11
 
7
12
  export interface DiagnosticResult {
8
13
  kb_id: string;
9
14
  kb_name: string;
15
+ status: KnowledgeBase["status"];
16
+ status_age_ms: number;
17
+ last_progress_age_ms: number;
18
+ stuck_indexing: boolean;
10
19
  stale_files: string[]; // files modified after indexing
11
20
  orphan_files: string[]; // chunks referencing deleted files
12
21
  coverage_percent: number; // indexed files / total scannable files
13
22
  total_source_files: number;
14
23
  indexed_files: number;
24
+ skipped_files: ScanResult["skipped"];
25
+ job?: IndexingJob;
26
+ }
27
+
28
+ const DEFAULT_STALE_INDEXING_MS = 10 * 60 * 1000;
29
+
30
+ function staleIndexingMs(): number {
31
+ const configured = Number(process.env.PI_KNOWLEDGE_STALE_INDEXING_MS);
32
+ return Number.isFinite(configured) && configured > 0 ? configured : DEFAULT_STALE_INDEXING_MS;
33
+ }
34
+
35
+ function scanOptionsFromSourceOptions(raw: string | null): ScanOptions {
36
+ if (!raw) return {};
37
+ try {
38
+ const parsed = JSON.parse(raw) as {
39
+ include_suggested_text?: unknown;
40
+ include_paths?: unknown;
41
+ exclude_paths?: unknown;
42
+ };
43
+ return {
44
+ includeSuggestedText: parsed.include_suggested_text === true,
45
+ includePaths: Array.isArray(parsed.include_paths)
46
+ ? parsed.include_paths.filter((item) => typeof item === "string")
47
+ : undefined,
48
+ excludePaths: Array.isArray(parsed.exclude_paths)
49
+ ? parsed.exclude_paths.filter((item) => typeof item === "string")
50
+ : undefined,
51
+ };
52
+ } catch {
53
+ return {};
54
+ }
15
55
  }
16
56
 
17
57
  export function diagnoseKB(db: Database.Database, kb: KnowledgeBase): DiagnosticResult {
58
+ const statusAgeMs = Date.now() - kb.updated_at;
59
+ const job = getIndexingJob(db, kb.id);
60
+ const lastProgressAgeMs = job?.status === "running" ? Date.now() - job.last_progress_at : statusAgeMs;
18
61
  const result: DiagnosticResult = {
19
62
  kb_id: kb.id,
20
63
  kb_name: kb.name,
64
+ status: kb.status,
65
+ status_age_ms: statusAgeMs,
66
+ last_progress_age_ms: lastProgressAgeMs,
67
+ stuck_indexing: kb.status === "indexing" && lastProgressAgeMs > staleIndexingMs(),
21
68
  stale_files: [],
22
69
  orphan_files: [],
23
70
  coverage_percent: 100,
24
71
  total_source_files: 0,
25
72
  indexed_files: kb.file_count,
73
+ skipped_files: {
74
+ total: 0,
75
+ by_reason: {
76
+ suggested_excluded: 0,
77
+ oversized: 0,
78
+ binary: 0,
79
+ unreadable: 0,
80
+ inaccessible: 0,
81
+ },
82
+ samples: [],
83
+ },
84
+ job,
26
85
  };
27
86
 
28
87
  if (!kb.source_path || kb.source_type === "url" || !existsSync(kb.source_path)) {
@@ -33,8 +92,19 @@ export function diagnoseKB(db: Database.Database, kb: KnowledgeBase): Diagnostic
33
92
  const currentFiles = new Set<string>();
34
93
  const isDirectory = statSync(kb.source_path).isDirectory();
35
94
  try {
36
- const scanned = isDirectory ? walkDir(kb.source_path).map((f) => f.relPath) : [kb.source_path];
37
- for (const f of scanned) currentFiles.add(f);
95
+ if (isDirectory) {
96
+ const skipped = createSkippedScanStats();
97
+ for (const file of iterateScannableFiles(
98
+ kb.source_path,
99
+ skipped,
100
+ scanOptionsFromSourceOptions(kb.source_options),
101
+ )) {
102
+ currentFiles.add(file.relPath);
103
+ }
104
+ result.skipped_files = skipped;
105
+ } else {
106
+ currentFiles.add(kb.source_path);
107
+ }
38
108
  } catch {
39
109
  return result;
40
110
  }
@@ -42,9 +112,13 @@ export function diagnoseKB(db: Database.Database, kb: KnowledgeBase): Diagnostic
42
112
  result.total_source_files = currentFiles.size;
43
113
  result.coverage_percent = currentFiles.size > 0 ? Math.round((result.indexed_files / currentFiles.size) * 100) : 100;
44
114
 
45
- // Get indexed chunks
46
- const chunks = getChunksByKB(db, kb.id);
47
- const indexedFilePaths = new Set(chunks.map((c) => c.file_path));
115
+ const indexedFilePaths = new Set<string>();
116
+ const latestIndexedByFile = new Map<string, number>();
117
+ for (const chunk of iterateChunksByKB(db, kb.id)) {
118
+ indexedFilePaths.add(chunk.file_path);
119
+ const currentLatest = latestIndexedByFile.get(chunk.file_path) ?? 0;
120
+ if (chunk.indexed_at > currentLatest) latestIndexedByFile.set(chunk.file_path, chunk.indexed_at);
121
+ }
48
122
 
49
123
  // Orphan detection: chunks referencing files no longer in source
50
124
  for (const filePath of indexedFilePaths) {
@@ -58,13 +132,9 @@ export function diagnoseKB(db: Database.Database, kb: KnowledgeBase): Diagnostic
58
132
  const absPath = isDirectory ? join(kb.source_path, relPath) : relPath;
59
133
  try {
60
134
  const mtime = statSync(absPath).mtimeMs;
61
- // Find latest indexed_at for this file's chunks
62
- const fileChunks = chunks.filter((c) => c.file_path === relPath);
63
- if (fileChunks.length > 0) {
64
- const latestIndexed = Math.max(...fileChunks.map((c) => c.indexed_at));
65
- if (mtime > latestIndexed) {
66
- result.stale_files.push(relPath);
67
- }
135
+ const latestIndexed = latestIndexedByFile.get(relPath);
136
+ if (latestIndexed !== undefined && mtime > latestIndexed) {
137
+ result.stale_files.push(relPath);
68
138
  }
69
139
  } catch {
70
140
  /* file unreadable — skip */
@@ -1,12 +1,5 @@
1
- import { join } from "node:path";
2
- import { getDefaultKnowledgeDir } from "../storage/sqlite.ts";
1
+ import { embedInModelWorker } from "../model-worker-client.ts";
3
2
 
4
- type FeatureExtractionPipeline = {
5
- (input: string, options: { pooling: "mean"; normalize: true }): Promise<{ data: ArrayLike<number> }>;
6
- dispose(): Promise<void> | void;
7
- };
8
-
9
- let pipelineInstance: FeatureExtractionPipeline | null = null;
10
3
  let disposeTimer: ReturnType<typeof setTimeout> | null = null;
11
4
  let disposePromise: Promise<void> | null = null;
12
5
  let activeRuns = 0;
@@ -15,22 +8,7 @@ const idleWaiters: Array<() => void> = [];
15
8
 
16
9
  const IDLE_TIMEOUT_MS = Number(process.env.PI_KNOWLEDGE_EMBEDDING_IDLE_MS ?? 30_000);
17
10
  const EMBEDDING_CONFIG = process.env.PI_KNOWLEDGE_EMBEDDING ?? "local:multilingual-e5-small";
18
-
19
- function getModelCacheDir(): string {
20
- return join(getDefaultKnowledgeDir(), "models");
21
- }
22
-
23
- async function loadPipeline(): Promise<FeatureExtractionPipeline> {
24
- if (pipelineInstance) return pipelineInstance;
25
- if (disposePromise) await disposePromise;
26
- const { pipeline, env } = await import("@huggingface/transformers");
27
- env.cacheDir = getModelCacheDir();
28
- pipelineInstance = await pipeline("feature-extraction", "Xenova/multilingual-e5-small", {
29
- quantized: true,
30
- dtype: "fp32",
31
- });
32
- return pipelineInstance;
33
- }
11
+ const ENABLE_NATIVE_IDLE_DISPOSE = process.env.PI_KNOWLEDGE_ENABLE_NATIVE_IDLE_DISPOSE === "true";
34
12
 
35
13
  function clearIdleTimer(): void {
36
14
  if (disposeTimer) clearTimeout(disposeTimer);
@@ -40,6 +18,7 @@ function clearIdleTimer(): void {
40
18
  function scheduleIdleDispose(): void {
41
19
  if (activeRuns > 0 || disposeRequested) return;
42
20
  clearIdleTimer();
21
+ if (!ENABLE_NATIVE_IDLE_DISPOSE) return;
43
22
  disposeTimer = setTimeout(() => dispose(), IDLE_TIMEOUT_MS);
44
23
  }
45
24
 
@@ -65,13 +44,7 @@ export async function dispose(): Promise<void> {
65
44
  if (disposePromise) return disposePromise;
66
45
  disposeRequested = true;
67
46
  await waitForNoActiveRuns();
68
- const instance = pipelineInstance;
69
- pipelineInstance = null;
70
- if (!instance) {
71
- disposeRequested = false;
72
- return;
73
- }
74
- disposePromise = Promise.resolve(instance.dispose()).finally(() => {
47
+ disposePromise = Promise.resolve().finally(() => {
75
48
  disposePromise = null;
76
49
  disposeRequested = false;
77
50
  });
@@ -115,19 +88,12 @@ export async function embedTexts(
115
88
  /* fallback to local */
116
89
  }
117
90
  }
118
- const pipe = await loadPipeline();
119
91
  beginRun();
120
- const results: Float32Array[] = [];
121
92
  try {
122
- for (const text of texts) {
123
- if (signal?.aborted) throw new Error("Cancelled");
124
- const output = await pipe(`${prefix}: ${text}`, { pooling: "mean", normalize: true });
125
- results.push(new Float32Array(output.data));
126
- }
93
+ return await embedInModelWorker(texts, prefix, signal);
127
94
  } finally {
128
95
  endRun();
129
96
  }
130
- return results;
131
97
  }
132
98
 
133
99
  export async function embedQuery(text: string): Promise<Float32Array> {
@@ -1,20 +1,103 @@
1
- import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
1
+ import { closeSync, existsSync, mkdirSync, openSync, readFileSync, readSync, writeSync } from "node:fs";
2
2
  import { dirname } from "node:path";
3
3
 
4
- export function saveVectors(path: string, vectors: Float32Array[]): void {
5
- if (vectors.length === 0) return;
4
+ export interface VectorWriter {
5
+ append(vectors: Float32Array[]): void;
6
+ close(): void;
7
+ }
8
+
9
+ export interface VectorReader {
10
+ count: number;
11
+ dim: number;
12
+ read(index: number): Float32Array | undefined;
13
+ readInto(index: number, target: Float32Array): boolean;
14
+ close(): void;
15
+ }
16
+
17
+ export function openVectorWriter(path: string): VectorWriter {
6
18
  const dir = dirname(path);
7
19
  if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
8
- const dim = vectors[0].length;
9
- const buffer = Buffer.alloc(8 + vectors.length * dim * 4);
10
- buffer.writeUInt32LE(vectors.length, 0);
11
- buffer.writeUInt32LE(dim, 4);
12
- let offset = 8;
13
- for (const vec of vectors) {
14
- Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength).copy(buffer, offset);
15
- offset += dim * 4;
20
+ const fd = openSync(path, "w");
21
+ let count = 0;
22
+ let dim = 0;
23
+ let closed = false;
24
+ let byteOffset = 8;
25
+ writeSync(fd, Buffer.alloc(8), 0, 8, 0);
26
+
27
+ function writeHeader(): void {
28
+ const header = Buffer.alloc(8);
29
+ header.writeUInt32LE(count, 0);
30
+ header.writeUInt32LE(dim, 4);
31
+ writeSync(fd, header, 0, header.length, 0);
32
+ }
33
+
34
+ return {
35
+ append(vectors: Float32Array[]): void {
36
+ if (closed) throw new Error("Vector writer is already closed");
37
+ for (const vector of vectors) {
38
+ if (dim === 0) dim = vector.length;
39
+ if (vector.length !== dim) {
40
+ throw new Error(`Vector dimension mismatch: expected ${dim}, got ${vector.length}`);
41
+ }
42
+ const bytes = Buffer.from(vector.buffer, vector.byteOffset, vector.byteLength);
43
+ writeSync(fd, bytes, 0, bytes.length, byteOffset);
44
+ byteOffset += bytes.length;
45
+ count++;
46
+ }
47
+ },
48
+ close(): void {
49
+ if (closed) return;
50
+ writeHeader();
51
+ closeSync(fd);
52
+ closed = true;
53
+ },
54
+ };
55
+ }
56
+
57
+ export function openVectorReader(path: string): VectorReader | undefined {
58
+ if (!existsSync(path)) return undefined;
59
+ const fd = openSync(path, "r");
60
+ let closed = false;
61
+ const header = Buffer.alloc(8);
62
+ const headerBytes = readSync(fd, header, 0, header.length, 0);
63
+ if (headerBytes < header.length) {
64
+ closeSync(fd);
65
+ return undefined;
66
+ }
67
+ const count = header.readUInt32LE(0);
68
+ const dim = header.readUInt32LE(4);
69
+ const vectorBytes = dim * 4;
70
+ const readBuffer = Buffer.alloc(vectorBytes);
71
+
72
+ return {
73
+ count,
74
+ dim,
75
+ read(index: number): Float32Array | undefined {
76
+ const vector = new Float32Array(dim);
77
+ return this.readInto(index, vector) ? vector : undefined;
78
+ },
79
+ readInto(index: number, target: Float32Array): boolean {
80
+ if (closed || index < 0 || index >= count || dim === 0 || target.length !== dim) return false;
81
+ const bytesRead = readSync(fd, readBuffer, 0, vectorBytes, 8 + index * vectorBytes);
82
+ if (bytesRead < vectorBytes) return false;
83
+ for (let i = 0; i < dim; i++) target[i] = readBuffer.readFloatLE(i * 4);
84
+ return true;
85
+ },
86
+ close(): void {
87
+ if (closed) return;
88
+ closeSync(fd);
89
+ closed = true;
90
+ },
91
+ };
92
+ }
93
+
94
+ export function saveVectors(path: string, vectors: Float32Array[]): void {
95
+ const writer = openVectorWriter(path);
96
+ try {
97
+ writer.append(vectors);
98
+ } finally {
99
+ writer.close();
16
100
  }
17
- writeFileSync(path, buffer);
18
101
  }
19
102
 
20
103
  export function loadVectors(path: string): Float32Array[] {