pi-knowledge 0.2.3 → 0.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,48 @@
1
1
  # Changelog
2
2
 
3
+ ## [0.3.3] - 2026-06-15
4
+
5
+ ### Changed
6
+ - Added a mandatory async lifecycle review contract for timers, event handlers, dispose, and shutdown paths.
7
+ - Clarified review requirements for overlap analysis, guard-state updates before await points, idempotent cleanup, and recreate-after-dispose behavior.
8
+
9
+ ## [0.3.2] - 2026-06-15
10
+
11
+ ### Fixed
12
+ - Made local embedding and deep reranker model disposal idempotent to avoid concurrent native ONNX session teardown from idle timers and Pi `session_shutdown`.
13
+
14
+ ### Changed
15
+ - Updated the onnxruntime exit-crash pitfall notes with the double-dispose race mitigation.
16
+
17
+ ## [0.3.1] - 2026-06-15
18
+
19
+ ### Fixed
20
+ - Made watcher shutdown cover both native watchers and polling fallbacks without mutating the collection being iterated.
21
+ - Clarified the startup-safe render component shim contract without overclaiming Pi internals.
22
+
23
+ ### Changed
24
+ - Documented the difference between e2e smoke runs and release-grade PDF/DOCX fixture coverage.
25
+ - Added agent/contributor contracts for verification-level reporting, private fixture handling, documentation alignment, and the release/publish flow.
26
+
27
+ ## [0.3.0] - 2026-06-15
28
+
29
+ ### Fixed
30
+ - Corrected BM25 fast-mode score semantics so higher scores are consistently better after search fusion/sorting.
31
+ - Made URL knowledge bases a first-class source type and allowed `knowledge_update` to re-fetch URL sources.
32
+ - Threaded `AbortSignal` into incremental update embedding.
33
+ - Fixed stale diagnostics for single-file knowledge bases.
34
+ - Normalized `unpdf` page-array output before chunking PDF text.
35
+ - Made JSONL import cleanup partial KBs on failure and import exported KBs as portable text sources.
36
+ - Removed root extension runtime dependency on Pi virtual modules so Node strip-only startup smoke tests can run outside Pi.
37
+ - Restored `knowledge_search` custom rendering with a startup-safe local TUI component shim.
38
+ - Added polling fallback for file watching when native `fs.watch` is unavailable or fails with resource limits.
39
+ - Updated Biome 2 configuration so `npm run check` is a working quality gate.
40
+
41
+ ### Added
42
+ - Regression coverage for BM25 score direction, URL update, update cancellation, single-file diagnostics, import failure cleanup, and portable import/export behavior.
43
+ - E2E coverage for deep rerank, external PDF/DOCX fixtures, and watcher updates without committing private fixture data.
44
+ - Development contract notes for Pi runtime imports, source-type update behavior, portable exports, and release gates.
45
+
3
46
  ## [0.2.2] - 2026-06-15
4
47
 
5
48
  ### Fixed
package/README.md CHANGED
@@ -91,10 +91,32 @@ All data is stored globally at `~/.pi/knowledge/` (never in your project directo
91
91
  ```bash
92
92
  npm install
93
93
  npm test # Unit tests
94
- npm run test:e2e # Integration tests (requires pi + API key)
94
+ npm run test:e2e # Smoke integration tests; PDF/DOCX cases skip unless fixture env vars are set
95
+ PI_KNOWLEDGE_E2E_PDF=/path/to/file.pdf PI_KNOWLEDGE_E2E_DOCX=/path/to/file.docx npm run test:e2e
95
96
  npm run bench # Indexing/search benchmarks
96
97
  ```
97
98
 
99
+ PDF/DOCX fixtures should be real local files outside the repository. Do not commit private fixture files, extracted fixture text, snapshots, or machine-specific fixture paths. A release-grade e2e pass requires both fixture env vars; a run with skipped PDF/DOCX cases is only a smoke pass.
100
+
101
+ ## Release
102
+
103
+ Before publishing, update `package.json`, `package-lock.json`, and `CHANGELOG.md`, then run:
104
+
105
+ ```bash
106
+ npm run check
107
+ npm test
108
+ npm run test:e2e
109
+ PI_KNOWLEDGE_E2E_PDF=/path/to/file.pdf PI_KNOWLEDGE_E2E_DOCX=/path/to/file.docx npm run test:e2e
110
+ node --experimental-strip-types -e "import('./index.ts')"
111
+ npm pack --dry-run
112
+ pi -e ./index.ts
113
+ git push origin main
114
+ gh release create vX.Y.Z --title "vX.Y.Z" --notes-file /path/to/release-notes.md
115
+ npm publish
116
+ ```
117
+
118
+ Report any skipped or unverified gate explicitly. Do not describe smoke e2e as complete release-grade coverage.
119
+
98
120
  ## License
99
121
 
100
122
  MIT
package/index.ts CHANGED
@@ -1,9 +1,84 @@
1
1
  import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
2
- import { Text } from "@earendil-works/pi-tui";
3
- import { Type } from "typebox";
4
2
  import { KnowledgeEngine } from "./src/engine.ts";
5
3
  import { getDefaultKnowledgeDir } from "./src/storage/sqlite.ts";
6
- import { startWatcher, stopAllWatchers, getActiveWatcherCount } from "./src/watcher/file-watcher.ts";
4
+ import { getActiveWatcherCount, startWatcher, stopAllWatchers } from "./src/watcher/file-watcher.ts";
5
+
6
+ type Schema = Record<string, unknown> & { optional?: true };
7
+ type ContextMessage = { role: string; content: string };
8
+ type ThemeLike = {
9
+ fg?: (name: string, text: string) => string;
10
+ };
11
+ type RenderContextLike = {
12
+ lastComponent?: RenderableText;
13
+ };
14
+ type RenderOptionsLike = {
15
+ expanded?: boolean;
16
+ };
17
+ type ToolResultLike = {
18
+ content?: Array<{ type: string; text?: string }>;
19
+ };
20
+
21
+ /**
22
+ * Minimal duck-type shim for @earendil-works/pi-tui Text component.
23
+ * Allows startup without Pi virtual module while implementing the render host methods this extension needs.
24
+ */
25
+ class RenderableText {
26
+ private text: string;
27
+
28
+ constructor(text = "") {
29
+ this.text = text;
30
+ }
31
+
32
+ setText(text: string): void {
33
+ this.text = text;
34
+ }
35
+
36
+ invalidate(): void {}
37
+
38
+ render(): string[] {
39
+ return this.text.split("\n");
40
+ }
41
+ }
42
+
43
+ const Type = {
44
+ Object(properties: Record<string, Schema>): Schema {
45
+ const required = Object.entries(properties)
46
+ .filter(([, schema]) => !schema.optional)
47
+ .map(([name]) => name);
48
+ const normalized = Object.fromEntries(
49
+ Object.entries(properties).map(([name, schema]) => {
50
+ const { optional: _optional, ...rest } = schema;
51
+ return [name, rest];
52
+ }),
53
+ );
54
+ return { type: "object", properties: normalized, required, additionalProperties: false };
55
+ },
56
+ String(options: Record<string, unknown> = {}): Schema {
57
+ return { type: "string", ...options };
58
+ },
59
+ Number(options: Record<string, unknown> = {}): Schema {
60
+ return { type: "number", ...options };
61
+ },
62
+ Literal(value: string): Schema {
63
+ return { const: value };
64
+ },
65
+ Union(items: Schema[]): Schema {
66
+ return { anyOf: items };
67
+ },
68
+ Optional(schema: Schema): Schema {
69
+ return { ...schema, optional: true };
70
+ },
71
+ };
72
+
73
+ function themed(theme: ThemeLike, name: string, text: string): string {
74
+ return theme.fg ? theme.fg(name, text) : text;
75
+ }
76
+
77
+ function renderText(context: RenderContextLike, text: string): RenderableText {
78
+ const component = context.lastComponent ?? new RenderableText();
79
+ component.setText(text);
80
+ return component;
81
+ }
7
82
 
8
83
  const engine = new KnowledgeEngine();
9
84
  const WATCH_ENABLED = process.env.PI_KNOWLEDGE_WATCH === "true";
@@ -15,7 +90,9 @@ export default function (pi: ExtensionAPI) {
15
90
  if (WATCH_ENABLED) {
16
91
  for (const kb of engine.list()) {
17
92
  if (kb.source_path && kb.source_type === "directory") {
18
- startWatcher(kb.id, kb.source_path, (kbId) => { engine.update(kbId).catch(() => {}); });
93
+ startWatcher(kb.id, kb.source_path, (kbId) => {
94
+ engine.update(kbId).catch(() => {});
95
+ });
19
96
  }
20
97
  }
21
98
  }
@@ -24,6 +101,8 @@ export default function (pi: ExtensionAPI) {
24
101
  pi.on("session_shutdown", async () => {
25
102
  stopAllWatchers();
26
103
  await engine.dispose();
104
+ // Brief delay to let native onnxruntime threads complete cleanup before process exit
105
+ await new Promise((r) => setTimeout(r, 500));
27
106
  });
28
107
 
29
108
  // Auto-inject: search KB for relevant context before each LLM call (opt-in)
@@ -40,8 +119,11 @@ export default function (pi: ExtensionAPI) {
40
119
  const results = await engine.search(text, { mode: "fast", limit: 3 });
41
120
  if (results.results.length === 0) return;
42
121
  const context = results.results.map((r) => `[${r.file_path}]: ${r.snippet}`).join("\n\n");
43
- event.messages.unshift({ role: "user", content: `[Knowledge context]\n${context}` } as any);
44
- } catch { /* silent fail */ }
122
+ const messages = event.messages as ContextMessage[];
123
+ messages.unshift({ role: "user", content: `[Knowledge context]\n${context}` });
124
+ } catch {
125
+ /* silent fail */
126
+ }
45
127
  });
46
128
  }
47
129
 
@@ -70,15 +152,27 @@ export default function (pi: ExtensionAPI) {
70
152
  }),
71
153
  async execute(_id, params, _signal, onUpdate) {
72
154
  const { source, name } = params;
73
- const { kb, chunkCount } = await engine.add(source, name, (msg) => {
74
- onUpdate?.({ content: [{ type: "text", text: msg }] });
75
- }, _signal);
155
+ const { kb, chunkCount } = await engine.add(
156
+ source,
157
+ name,
158
+ (msg) => {
159
+ onUpdate?.({ content: [{ type: "text", text: msg }] });
160
+ },
161
+ _signal,
162
+ );
76
163
  // Start watcher for new directory KB
77
164
  if (WATCH_ENABLED && kb.source_path && kb.source_type === "directory") {
78
- startWatcher(kb.id, kb.source_path, (kbId) => { engine.update(kbId).catch(() => {}); });
165
+ startWatcher(kb.id, kb.source_path, (kbId) => {
166
+ engine.update(kbId).catch(() => {});
167
+ });
79
168
  }
80
169
  return {
81
- content: [{ type: "text", text: `Indexed "${kb.name}": ${chunkCount} chunks from ${kb.file_count} files. KB ID: ${kb.id}` }],
170
+ content: [
171
+ {
172
+ type: "text",
173
+ text: `Indexed "${kb.name}": ${chunkCount} chunks from ${kb.file_count} files. KB ID: ${kb.id}`,
174
+ },
175
+ ],
82
176
  };
83
177
  },
84
178
  });
@@ -95,20 +189,26 @@ export default function (pi: ExtensionAPI) {
95
189
  ],
96
190
  parameters: Type.Object({
97
191
  query: Type.String({ description: "Search query" }),
98
- mode: Type.Optional(Type.Union([Type.Literal("fast"), Type.Literal("semantic"), Type.Literal("hybrid"), Type.Literal("deep")])),
192
+ mode: Type.Optional(
193
+ Type.Union([Type.Literal("fast"), Type.Literal("semantic"), Type.Literal("hybrid"), Type.Literal("deep")]),
194
+ ),
99
195
  limit: Type.Optional(Type.Number({ description: "Max results (default 10)" })),
100
196
  kb_id: Type.Optional(Type.String({ description: "Limit search to specific KB" })),
101
197
  offset: Type.Optional(Type.Number({ description: "Pagination offset" })),
102
198
  file_type: Type.Optional(Type.String({ description: "Filter by file type (e.g. typescript, markdown, python)" })),
103
199
  }),
104
- renderCall(args, theme) {
105
- const mode = args.mode || "hybrid";
106
- return new Text(`${theme.fg("accent", "Search")}: "${args.query}" ${theme.fg("muted", `(${mode})`)}`, 0, 0);
200
+ renderCall(args, theme: ThemeLike, context: RenderContextLike) {
201
+ const query = typeof args.query === "string" ? args.query : "";
202
+ const mode = typeof args.mode === "string" ? args.mode : "hybrid";
203
+ return renderText(
204
+ context,
205
+ `${themed(theme, "accent", "Search")}: "${query}" ${themed(theme, "muted", `(${mode})`)}`,
206
+ );
107
207
  },
108
- renderResult(result, options, theme) {
109
- const text = result.content?.[0]?.text ?? "";
208
+ renderResult(result: ToolResultLike, options: RenderOptionsLike, _theme: ThemeLike, context: RenderContextLike) {
209
+ const text = result.content?.find((item) => item.type === "text")?.text ?? "";
110
210
  const lines = options.expanded ? text : text.split("\n").slice(0, 5).join("\n");
111
- return new Text(lines, 0, 0);
211
+ return renderText(context, lines);
112
212
  },
113
213
  async execute(_id, params) {
114
214
  const { query, mode, limit, kb_id, offset, file_type } = params;
@@ -121,9 +221,9 @@ export default function (pi: ExtensionAPI) {
121
221
  if (response.warnings?.length) {
122
222
  output = `⚠️ ${response.warnings.join("\n⚠️ ")}\n\n${output}`;
123
223
  }
124
- output += response.results.map((r, i) =>
125
- `[${i + 1}] ${r.file_path} (${r.kb_name}, score: ${r.score.toFixed(3)})\n${r.snippet}`,
126
- ).join("\n\n");
224
+ output += response.results
225
+ .map((r, i) => `[${i + 1}] ${r.file_path} (${r.kb_name}, score: ${r.score.toFixed(3)})\n${r.snippet}`)
226
+ .join("\n\n");
127
227
  return { content: [{ type: "text", text: output }] };
128
228
  },
129
229
  });
@@ -137,16 +237,19 @@ export default function (pi: ExtensionAPI) {
137
237
  target: Type.String({ description: "KB name or ID to update" }),
138
238
  }),
139
239
  async execute(_id, params, _signal, onUpdate) {
140
- const { added, removed, unchanged } = await engine.update(params.target, (msg) => {
141
- onUpdate?.({ content: [{ type: "text", text: msg }] });
142
- }, _signal);
240
+ const { added, removed, unchanged } = await engine.update(
241
+ params.target,
242
+ (msg) => {
243
+ onUpdate?.({ content: [{ type: "text", text: msg }] });
244
+ },
245
+ _signal,
246
+ );
143
247
  return {
144
248
  content: [{ type: "text", text: `Updated: +${added} added, -${removed} removed, ${unchanged} unchanged.` }],
145
249
  };
146
250
  },
147
251
  });
148
252
 
149
-
150
253
  pi.registerTool({
151
254
  name: "knowledge_status",
152
255
  label: "Knowledge Status",
@@ -156,22 +259,34 @@ export default function (pi: ExtensionAPI) {
156
259
  const kbs = engine.list();
157
260
  const watchCount = getActiveWatcherCount();
158
261
  const diagnostics = engine.diagnose();
159
- const lines = [`Storage: ${getDefaultKnowledgeDir()}`, `Knowledge bases: ${kbs.length}`, `Active watchers: ${watchCount}`, ""];
262
+ const lines = [
263
+ `Storage: ${getDefaultKnowledgeDir()}`,
264
+ `Knowledge bases: ${kbs.length}`,
265
+ `Active watchers: ${watchCount}`,
266
+ "",
267
+ ];
160
268
  for (const kb of kbs) {
161
269
  const age = Math.round((Date.now() - kb.updated_at) / 60000);
162
270
  const diag = diagnostics.find((d) => d.kb_id === kb.id);
163
- lines.push(` "${kb.name}" — ${kb.status} — ${kb.chunk_count} chunks, ${kb.file_count} files — updated ${age}m ago`);
271
+ lines.push(
272
+ ` "${kb.name}" — ${kb.status} — ${kb.chunk_count} chunks, ${kb.file_count} files — updated ${age}m ago`,
273
+ );
164
274
  if (kb.source_path) lines.push(` source: ${kb.source_path}`);
165
275
  if (diag) {
166
- lines.push(` coverage: ${diag.coverage_percent}% (${diag.indexed_files}/${diag.total_source_files} files)`);
167
- if (diag.stale_files.length > 0) lines.push(` ⚠️ stale: ${diag.stale_files.length} files modified since last index`);
168
- if (diag.orphan_files.length > 0) lines.push(` ⚠️ orphans: ${diag.orphan_files.length} chunks reference deleted files`);
276
+ lines.push(
277
+ ` coverage: ${diag.coverage_percent}% (${diag.indexed_files}/${diag.total_source_files} files)`,
278
+ );
279
+ if (diag.stale_files.length > 0)
280
+ lines.push(` ⚠️ stale: ${diag.stale_files.length} files modified since last index`);
281
+ if (diag.orphan_files.length > 0)
282
+ lines.push(` ⚠️ orphans: ${diag.orphan_files.length} chunks reference deleted files`);
169
283
  }
170
284
  }
171
285
  const totalStale = diagnostics.reduce((n, d) => n + d.stale_files.length, 0);
172
286
  const totalOrphans = diagnostics.reduce((n, d) => n + d.orphan_files.length, 0);
173
287
  if (totalStale === 0 && totalOrphans === 0 && kbs.length > 0) lines.push("", "Health: ✓ all indexes up to date");
174
- else if (totalStale > 0 || totalOrphans > 0) lines.push("", `Health: ⚠️ ${totalStale} stale, ${totalOrphans} orphans — run knowledge_update`);
288
+ else if (totalStale > 0 || totalOrphans > 0)
289
+ lines.push("", `Health: ⚠️ ${totalStale} stale, ${totalOrphans} orphans — run knowledge_update`);
175
290
  return { content: [{ type: "text", text: lines.join("\n") }] };
176
291
  },
177
292
  });
@@ -224,9 +339,13 @@ export default function (pi: ExtensionAPI) {
224
339
  input: Type.String({ description: "Input JSONL file path" }),
225
340
  }),
226
341
  async execute(_id, params, _signal, onUpdate) {
227
- const { kb, chunkCount } = await engine.importKB(params.input, (msg) => {
228
- onUpdate?.({ content: [{ type: "text", text: msg }] });
229
- }, _signal);
342
+ const { kb, chunkCount } = await engine.importKB(
343
+ params.input,
344
+ (msg) => {
345
+ onUpdate?.({ content: [{ type: "text", text: msg }] });
346
+ },
347
+ _signal,
348
+ );
230
349
  return { content: [{ type: "text", text: `Imported "${kb.name}": ${chunkCount} chunks (re-embedded)` }] };
231
350
  },
232
351
  });
package/package.json CHANGED
@@ -1,62 +1,62 @@
1
1
  {
2
- "name": "pi-knowledge",
3
- "version": "0.2.3",
4
- "description": "Vector knowledge base extension for Pi — index files, hybrid search, local embeddings",
5
- "type": "module",
6
- "main": "index.ts",
7
- "pi": {
8
- "extensions": [
9
- "./index.ts"
10
- ]
11
- },
12
- "files": [
13
- "index.ts",
14
- "src/",
15
- ".pi/",
16
- "README.md",
17
- "CHANGELOG.md",
18
- "LICENSE"
19
- ],
20
- "keywords": [
21
- "pi",
22
- "pi-extension",
23
- "knowledge-base",
24
- "vector-search",
25
- "semantic-search",
26
- "rag",
27
- "embeddings"
28
- ],
29
- "scripts": {
30
- "check": "biome check --write --error-on-warnings .",
31
- "test": "vitest --run test/unit/",
32
- "test:e2e": "vitest --run test/e2e/",
33
- "bench": "vitest bench test/bench/"
34
- },
35
- "dependencies": {
36
- "@huggingface/transformers": "3.8.1",
37
- "better-sqlite3": "11.9.1",
38
- "ignore": "7.0.5",
39
- "mammoth": "1.12.0",
40
- "tree-sitter": "0.22.4",
41
- "tree-sitter-go": "0.23.4",
42
- "tree-sitter-java": "0.23.5",
43
- "tree-sitter-python": "0.23.6",
44
- "tree-sitter-rust": "0.23.2",
45
- "tree-sitter-typescript": "0.23.2",
46
- "unpdf": "1.6.2"
47
- },
48
- "devDependencies": {
49
- "@biomejs/biome": "2.3.5",
50
- "@types/better-sqlite3": "7.6.13",
51
- "@types/node": "22.19.19",
52
- "vitest": "3.2.4"
53
- },
54
- "engines": {
55
- "node": ">=22.0.0"
56
- },
57
- "license": "MIT",
58
- "repository": {
59
- "type": "git",
60
- "url": "https://github.com/nczz/pi-knowledge.git"
61
- }
2
+ "name": "pi-knowledge",
3
+ "version": "0.3.3",
4
+ "description": "Vector knowledge base extension for Pi — index files, hybrid search, local embeddings",
5
+ "type": "module",
6
+ "main": "index.ts",
7
+ "pi": {
8
+ "extensions": [
9
+ "./index.ts"
10
+ ]
11
+ },
12
+ "files": [
13
+ "index.ts",
14
+ "src/",
15
+ ".pi/",
16
+ "README.md",
17
+ "CHANGELOG.md",
18
+ "LICENSE"
19
+ ],
20
+ "keywords": [
21
+ "pi",
22
+ "pi-extension",
23
+ "knowledge-base",
24
+ "vector-search",
25
+ "semantic-search",
26
+ "rag",
27
+ "embeddings"
28
+ ],
29
+ "scripts": {
30
+ "check": "biome check --write --error-on-warnings .",
31
+ "test": "vitest --run test/unit/",
32
+ "test:e2e": "vitest --run test/e2e/",
33
+ "bench": "vitest bench test/bench/"
34
+ },
35
+ "dependencies": {
36
+ "@huggingface/transformers": "3.8.1",
37
+ "better-sqlite3": "11.9.1",
38
+ "ignore": "7.0.5",
39
+ "mammoth": "1.12.0",
40
+ "tree-sitter": "0.22.4",
41
+ "tree-sitter-go": "0.23.4",
42
+ "tree-sitter-java": "0.23.5",
43
+ "tree-sitter-python": "0.23.6",
44
+ "tree-sitter-rust": "0.23.2",
45
+ "tree-sitter-typescript": "0.23.2",
46
+ "unpdf": "1.6.2"
47
+ },
48
+ "devDependencies": {
49
+ "@biomejs/biome": "2.3.5",
50
+ "@types/better-sqlite3": "7.6.13",
51
+ "@types/node": "22.19.19",
52
+ "vitest": "3.2.4"
53
+ },
54
+ "engines": {
55
+ "node": ">=22.0.0"
56
+ },
57
+ "license": "MIT",
58
+ "repository": {
59
+ "type": "git",
60
+ "url": "https://github.com/nczz/pi-knowledge.git"
61
+ }
62
62
  }
@@ -2,14 +2,14 @@ import { existsSync, statSync } from "node:fs";
2
2
  import { join } from "node:path";
3
3
  import type Database from "better-sqlite3";
4
4
  import { walkDir } from "../indexer/chunker.ts";
5
- import { type KnowledgeBase, getChunksByKB } from "../storage/sqlite.ts";
5
+ import { getChunksByKB, type KnowledgeBase } from "../storage/sqlite.ts";
6
6
 
7
7
  export interface DiagnosticResult {
8
8
  kb_id: string;
9
9
  kb_name: string;
10
- stale_files: string[]; // files modified after indexing
11
- orphan_files: string[]; // chunks referencing deleted files
12
- coverage_percent: number; // indexed files / total scannable files
10
+ stale_files: string[]; // files modified after indexing
11
+ orphan_files: string[]; // chunks referencing deleted files
12
+ coverage_percent: number; // indexed files / total scannable files
13
13
  total_source_files: number;
14
14
  indexed_files: number;
15
15
  }
@@ -25,16 +25,15 @@ export function diagnoseKB(db: Database.Database, kb: KnowledgeBase): Diagnostic
25
25
  indexed_files: kb.file_count,
26
26
  };
27
27
 
28
- if (!kb.source_path || !existsSync(kb.source_path)) {
28
+ if (!kb.source_path || kb.source_type === "url" || !existsSync(kb.source_path)) {
29
29
  return result; // text KBs or missing source — no diagnostics possible
30
30
  }
31
31
 
32
32
  // Scan current source files
33
33
  const currentFiles = new Set<string>();
34
+ const isDirectory = statSync(kb.source_path).isDirectory();
34
35
  try {
35
- const scanned = statSync(kb.source_path).isDirectory()
36
- ? walkDir(kb.source_path).map((f) => f.relPath)
37
- : [kb.source_path];
36
+ const scanned = isDirectory ? walkDir(kb.source_path).map((f) => f.relPath) : [kb.source_path];
38
37
  for (const f of scanned) currentFiles.add(f);
39
38
  } catch {
40
39
  return result;
@@ -56,7 +55,7 @@ export function diagnoseKB(db: Database.Database, kb: KnowledgeBase): Diagnostic
56
55
 
57
56
  // Staleness detection: source files modified after last indexing
58
57
  for (const relPath of currentFiles) {
59
- const absPath = join(kb.source_path, relPath);
58
+ const absPath = isDirectory ? join(kb.source_path, relPath) : relPath;
60
59
  try {
61
60
  const mtime = statSync(absPath).mtimeMs;
62
61
  // Find latest indexed_at for this file's chunks
@@ -67,7 +66,9 @@ export function diagnoseKB(db: Database.Database, kb: KnowledgeBase): Diagnostic
67
66
  result.stale_files.push(relPath);
68
67
  }
69
68
  }
70
- } catch { /* file unreadable — skip */ }
69
+ } catch {
70
+ /* file unreadable — skip */
71
+ }
71
72
  }
72
73
 
73
74
  return result;
@@ -1,21 +1,31 @@
1
1
  import { join } from "node:path";
2
2
  import { getDefaultKnowledgeDir } from "../storage/sqlite.ts";
3
3
 
4
- let pipelineInstance: any = null;
4
+ type FeatureExtractionPipeline = {
5
+ (input: string, options: { pooling: "mean"; normalize: true }): Promise<{ data: ArrayLike<number> }>;
6
+ dispose(): Promise<void> | void;
7
+ };
8
+
9
+ let pipelineInstance: FeatureExtractionPipeline | null = null;
5
10
  let disposeTimer: ReturnType<typeof setTimeout> | null = null;
11
+ let disposePromise: Promise<void> | null = null;
6
12
 
7
- const IDLE_TIMEOUT_MS = 60_000;
13
+ const IDLE_TIMEOUT_MS = 30_000;
8
14
  const EMBEDDING_CONFIG = process.env.PI_KNOWLEDGE_EMBEDDING ?? "local:multilingual-e5-small";
9
15
 
10
16
  function getModelCacheDir(): string {
11
17
  return join(getDefaultKnowledgeDir(), "models");
12
18
  }
13
19
 
14
- async function loadPipeline(): Promise<any> {
20
+ async function loadPipeline(): Promise<FeatureExtractionPipeline> {
15
21
  if (pipelineInstance) return pipelineInstance;
22
+ if (disposePromise) await disposePromise;
16
23
  const { pipeline, env } = await import("@huggingface/transformers");
17
24
  env.cacheDir = getModelCacheDir();
18
- pipelineInstance = await pipeline("feature-extraction", "Xenova/multilingual-e5-small", { quantized: true, dtype: "fp32" });
25
+ pipelineInstance = await pipeline("feature-extraction", "Xenova/multilingual-e5-small", {
26
+ quantized: true,
27
+ dtype: "fp32",
28
+ });
19
29
  return pipelineInstance;
20
30
  }
21
31
 
@@ -25,8 +35,18 @@ function resetIdleTimer(): void {
25
35
  }
26
36
 
27
37
  export async function dispose(): Promise<void> {
28
- if (disposeTimer) { clearTimeout(disposeTimer); disposeTimer = null; }
29
- if (pipelineInstance) { await pipelineInstance.dispose(); pipelineInstance = null; }
38
+ if (disposeTimer) {
39
+ clearTimeout(disposeTimer);
40
+ disposeTimer = null;
41
+ }
42
+ if (disposePromise) return disposePromise;
43
+ const instance = pipelineInstance;
44
+ pipelineInstance = null;
45
+ if (!instance) return;
46
+ disposePromise = Promise.resolve(instance.dispose()).finally(() => {
47
+ disposePromise = null;
48
+ });
49
+ return disposePromise;
30
50
  }
31
51
 
32
52
  async function embedViaAPI(texts: string[], prefix: "query" | "passage"): Promise<Float32Array[]> {
@@ -49,10 +69,17 @@ async function embedViaAPI(texts: string[], prefix: "query" | "passage"): Promis
49
69
  throw new Error(`Unsupported embedding provider: ${provider}`);
50
70
  }
51
71
 
52
- export async function embedTexts(texts: string[], prefix: "query" | "passage", signal?: AbortSignal): Promise<Float32Array[]> {
72
+ export async function embedTexts(
73
+ texts: string[],
74
+ prefix: "query" | "passage",
75
+ signal?: AbortSignal,
76
+ ): Promise<Float32Array[]> {
53
77
  if (!EMBEDDING_CONFIG.startsWith("local")) {
54
- try { return await embedViaAPI(texts, prefix); }
55
- catch { /* fallback to local */ }
78
+ try {
79
+ return await embedViaAPI(texts, prefix);
80
+ } catch {
81
+ /* fallback to local */
82
+ }
56
83
  }
57
84
  const pipe = await loadPipeline();
58
85
  resetIdleTimer();
@@ -1,4 +1,4 @@
1
- import { readFileSync, writeFileSync, existsSync, mkdirSync } from "node:fs";
1
+ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
2
2
  import { dirname } from "node:path";
3
3
 
4
4
  export function saveVectors(path: string, vectors: Float32Array[]): void {