pi-knowledge 0.2.3 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -0
- package/README.md +23 -1
- package/index.ts +153 -34
- package/package.json +60 -60
- package/src/diagnostics/health.ts +11 -10
- package/src/embedding/provider.ts +30 -9
- package/src/embedding/vectors.ts +1 -1
- package/src/engine.ts +212 -85
- package/src/indexer/chunker.ts +115 -39
- package/src/indexer/chunkers/code-ast.ts +77 -17
- package/src/search/bm25.ts +19 -10
- package/src/search/fusion.ts +5 -4
- package/src/search/reranker.ts +25 -7
- package/src/search/vector.ts +10 -2
- package/src/storage/sqlite.ts +27 -8
- package/src/watcher/file-watcher.ts +92 -12
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,34 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [0.3.1] - 2026-06-15
|
|
4
|
+
|
|
5
|
+
### Fixed
|
|
6
|
+
- Made watcher shutdown cover both native watchers and polling fallbacks without mutating the collection being iterated.
|
|
7
|
+
- Clarified the startup-safe render component shim contract without overclaiming Pi internals.
|
|
8
|
+
|
|
9
|
+
### Changed
|
|
10
|
+
- Documented the difference between e2e smoke runs and release-grade PDF/DOCX fixture coverage.
|
|
11
|
+
- Added agent/contributor contracts for verification-level reporting, private fixture handling, documentation alignment, and the release/publish flow.
|
|
12
|
+
|
|
13
|
+
## [0.3.0] - 2026-06-15
|
|
14
|
+
|
|
15
|
+
### Fixed
|
|
16
|
+
- Corrected BM25 fast-mode score semantics so higher scores are consistently better after search fusion/sorting.
|
|
17
|
+
- Made URL knowledge bases a first-class source type and allowed `knowledge_update` to re-fetch URL sources.
|
|
18
|
+
- Threaded `AbortSignal` into incremental update embedding.
|
|
19
|
+
- Fixed stale diagnostics for single-file knowledge bases.
|
|
20
|
+
- Normalized `unpdf` page-array output before chunking PDF text.
|
|
21
|
+
- Made JSONL import cleanup partial KBs on failure and import exported KBs as portable text sources.
|
|
22
|
+
- Removed root extension runtime dependency on Pi virtual modules so Node strip-only startup smoke tests can run outside Pi.
|
|
23
|
+
- Restored `knowledge_search` custom rendering with a startup-safe local TUI component shim.
|
|
24
|
+
- Added polling fallback for file watching when native `fs.watch` is unavailable or fails with resource limits.
|
|
25
|
+
- Updated Biome 2 configuration so `npm run check` is a working quality gate.
|
|
26
|
+
|
|
27
|
+
### Added
|
|
28
|
+
- Regression coverage for BM25 score direction, URL update, update cancellation, single-file diagnostics, import failure cleanup, and portable import/export behavior.
|
|
29
|
+
- E2E coverage for deep rerank, external PDF/DOCX fixtures, and watcher updates without committing private fixture data.
|
|
30
|
+
- Development contract notes for Pi runtime imports, source-type update behavior, portable exports, and release gates.
|
|
31
|
+
|
|
3
32
|
## [0.2.2] - 2026-06-15
|
|
4
33
|
|
|
5
34
|
### Fixed
|
package/README.md
CHANGED
|
@@ -91,10 +91,32 @@ All data is stored globally at `~/.pi/knowledge/` (never in your project directo
|
|
|
91
91
|
```bash
|
|
92
92
|
npm install
|
|
93
93
|
npm test # Unit tests
|
|
94
|
-
npm run test:e2e
|
|
94
|
+
npm run test:e2e # Smoke integration tests; PDF/DOCX cases skip unless fixture env vars are set
|
|
95
|
+
PI_KNOWLEDGE_E2E_PDF=/path/to/file.pdf PI_KNOWLEDGE_E2E_DOCX=/path/to/file.docx npm run test:e2e
|
|
95
96
|
npm run bench # Indexing/search benchmarks
|
|
96
97
|
```
|
|
97
98
|
|
|
99
|
+
PDF/DOCX fixtures should be real local files outside the repository. Do not commit private fixture files, extracted fixture text, snapshots, or machine-specific fixture paths. A release-grade e2e pass requires both fixture env vars; a run with skipped PDF/DOCX cases is only a smoke pass.
|
|
100
|
+
|
|
101
|
+
## Release
|
|
102
|
+
|
|
103
|
+
Before publishing, update `package.json`, `package-lock.json`, and `CHANGELOG.md`, then run:
|
|
104
|
+
|
|
105
|
+
```bash
|
|
106
|
+
npm run check
|
|
107
|
+
npm test
|
|
108
|
+
npm run test:e2e
|
|
109
|
+
PI_KNOWLEDGE_E2E_PDF=/path/to/file.pdf PI_KNOWLEDGE_E2E_DOCX=/path/to/file.docx npm run test:e2e
|
|
110
|
+
node --experimental-strip-types -e "import('./index.ts')"
|
|
111
|
+
npm pack --dry-run
|
|
112
|
+
pi -e ./index.ts
|
|
113
|
+
git push origin main
|
|
114
|
+
gh release create vX.Y.Z --title "vX.Y.Z" --notes-file /path/to/release-notes.md
|
|
115
|
+
npm publish
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
Report any skipped or unverified gate explicitly. Do not describe smoke e2e as complete release-grade coverage.
|
|
119
|
+
|
|
98
120
|
## License
|
|
99
121
|
|
|
100
122
|
MIT
|
package/index.ts
CHANGED
|
@@ -1,9 +1,84 @@
|
|
|
1
1
|
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
|
2
|
-
import { Text } from "@earendil-works/pi-tui";
|
|
3
|
-
import { Type } from "typebox";
|
|
4
2
|
import { KnowledgeEngine } from "./src/engine.ts";
|
|
5
3
|
import { getDefaultKnowledgeDir } from "./src/storage/sqlite.ts";
|
|
6
|
-
import { startWatcher, stopAllWatchers
|
|
4
|
+
import { getActiveWatcherCount, startWatcher, stopAllWatchers } from "./src/watcher/file-watcher.ts";
|
|
5
|
+
|
|
6
|
+
type Schema = Record<string, unknown> & { optional?: true };
|
|
7
|
+
type ContextMessage = { role: string; content: string };
|
|
8
|
+
type ThemeLike = {
|
|
9
|
+
fg?: (name: string, text: string) => string;
|
|
10
|
+
};
|
|
11
|
+
type RenderContextLike = {
|
|
12
|
+
lastComponent?: RenderableText;
|
|
13
|
+
};
|
|
14
|
+
type RenderOptionsLike = {
|
|
15
|
+
expanded?: boolean;
|
|
16
|
+
};
|
|
17
|
+
type ToolResultLike = {
|
|
18
|
+
content?: Array<{ type: string; text?: string }>;
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Minimal duck-type shim for @earendil-works/pi-tui Text component.
|
|
23
|
+
* Allows startup without Pi virtual module while implementing the render host methods this extension needs.
|
|
24
|
+
*/
|
|
25
|
+
class RenderableText {
|
|
26
|
+
private text: string;
|
|
27
|
+
|
|
28
|
+
constructor(text = "") {
|
|
29
|
+
this.text = text;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
setText(text: string): void {
|
|
33
|
+
this.text = text;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
invalidate(): void {}
|
|
37
|
+
|
|
38
|
+
render(): string[] {
|
|
39
|
+
return this.text.split("\n");
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
const Type = {
|
|
44
|
+
Object(properties: Record<string, Schema>): Schema {
|
|
45
|
+
const required = Object.entries(properties)
|
|
46
|
+
.filter(([, schema]) => !schema.optional)
|
|
47
|
+
.map(([name]) => name);
|
|
48
|
+
const normalized = Object.fromEntries(
|
|
49
|
+
Object.entries(properties).map(([name, schema]) => {
|
|
50
|
+
const { optional: _optional, ...rest } = schema;
|
|
51
|
+
return [name, rest];
|
|
52
|
+
}),
|
|
53
|
+
);
|
|
54
|
+
return { type: "object", properties: normalized, required, additionalProperties: false };
|
|
55
|
+
},
|
|
56
|
+
String(options: Record<string, unknown> = {}): Schema {
|
|
57
|
+
return { type: "string", ...options };
|
|
58
|
+
},
|
|
59
|
+
Number(options: Record<string, unknown> = {}): Schema {
|
|
60
|
+
return { type: "number", ...options };
|
|
61
|
+
},
|
|
62
|
+
Literal(value: string): Schema {
|
|
63
|
+
return { const: value };
|
|
64
|
+
},
|
|
65
|
+
Union(items: Schema[]): Schema {
|
|
66
|
+
return { anyOf: items };
|
|
67
|
+
},
|
|
68
|
+
Optional(schema: Schema): Schema {
|
|
69
|
+
return { ...schema, optional: true };
|
|
70
|
+
},
|
|
71
|
+
};
|
|
72
|
+
|
|
73
|
+
function themed(theme: ThemeLike, name: string, text: string): string {
|
|
74
|
+
return theme.fg ? theme.fg(name, text) : text;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function renderText(context: RenderContextLike, text: string): RenderableText {
|
|
78
|
+
const component = context.lastComponent ?? new RenderableText();
|
|
79
|
+
component.setText(text);
|
|
80
|
+
return component;
|
|
81
|
+
}
|
|
7
82
|
|
|
8
83
|
const engine = new KnowledgeEngine();
|
|
9
84
|
const WATCH_ENABLED = process.env.PI_KNOWLEDGE_WATCH === "true";
|
|
@@ -15,7 +90,9 @@ export default function (pi: ExtensionAPI) {
|
|
|
15
90
|
if (WATCH_ENABLED) {
|
|
16
91
|
for (const kb of engine.list()) {
|
|
17
92
|
if (kb.source_path && kb.source_type === "directory") {
|
|
18
|
-
startWatcher(kb.id, kb.source_path, (kbId) => {
|
|
93
|
+
startWatcher(kb.id, kb.source_path, (kbId) => {
|
|
94
|
+
engine.update(kbId).catch(() => {});
|
|
95
|
+
});
|
|
19
96
|
}
|
|
20
97
|
}
|
|
21
98
|
}
|
|
@@ -24,6 +101,8 @@ export default function (pi: ExtensionAPI) {
|
|
|
24
101
|
pi.on("session_shutdown", async () => {
|
|
25
102
|
stopAllWatchers();
|
|
26
103
|
await engine.dispose();
|
|
104
|
+
// Brief delay to let native onnxruntime threads complete cleanup before process exit
|
|
105
|
+
await new Promise((r) => setTimeout(r, 500));
|
|
27
106
|
});
|
|
28
107
|
|
|
29
108
|
// Auto-inject: search KB for relevant context before each LLM call (opt-in)
|
|
@@ -40,8 +119,11 @@ export default function (pi: ExtensionAPI) {
|
|
|
40
119
|
const results = await engine.search(text, { mode: "fast", limit: 3 });
|
|
41
120
|
if (results.results.length === 0) return;
|
|
42
121
|
const context = results.results.map((r) => `[${r.file_path}]: ${r.snippet}`).join("\n\n");
|
|
43
|
-
event.messages
|
|
44
|
-
|
|
122
|
+
const messages = event.messages as ContextMessage[];
|
|
123
|
+
messages.unshift({ role: "user", content: `[Knowledge context]\n${context}` });
|
|
124
|
+
} catch {
|
|
125
|
+
/* silent fail */
|
|
126
|
+
}
|
|
45
127
|
});
|
|
46
128
|
}
|
|
47
129
|
|
|
@@ -70,15 +152,27 @@ export default function (pi: ExtensionAPI) {
|
|
|
70
152
|
}),
|
|
71
153
|
async execute(_id, params, _signal, onUpdate) {
|
|
72
154
|
const { source, name } = params;
|
|
73
|
-
const { kb, chunkCount } = await engine.add(
|
|
74
|
-
|
|
75
|
-
|
|
155
|
+
const { kb, chunkCount } = await engine.add(
|
|
156
|
+
source,
|
|
157
|
+
name,
|
|
158
|
+
(msg) => {
|
|
159
|
+
onUpdate?.({ content: [{ type: "text", text: msg }] });
|
|
160
|
+
},
|
|
161
|
+
_signal,
|
|
162
|
+
);
|
|
76
163
|
// Start watcher for new directory KB
|
|
77
164
|
if (WATCH_ENABLED && kb.source_path && kb.source_type === "directory") {
|
|
78
|
-
startWatcher(kb.id, kb.source_path, (kbId) => {
|
|
165
|
+
startWatcher(kb.id, kb.source_path, (kbId) => {
|
|
166
|
+
engine.update(kbId).catch(() => {});
|
|
167
|
+
});
|
|
79
168
|
}
|
|
80
169
|
return {
|
|
81
|
-
content: [
|
|
170
|
+
content: [
|
|
171
|
+
{
|
|
172
|
+
type: "text",
|
|
173
|
+
text: `Indexed "${kb.name}": ${chunkCount} chunks from ${kb.file_count} files. KB ID: ${kb.id}`,
|
|
174
|
+
},
|
|
175
|
+
],
|
|
82
176
|
};
|
|
83
177
|
},
|
|
84
178
|
});
|
|
@@ -95,20 +189,26 @@ export default function (pi: ExtensionAPI) {
|
|
|
95
189
|
],
|
|
96
190
|
parameters: Type.Object({
|
|
97
191
|
query: Type.String({ description: "Search query" }),
|
|
98
|
-
mode: Type.Optional(
|
|
192
|
+
mode: Type.Optional(
|
|
193
|
+
Type.Union([Type.Literal("fast"), Type.Literal("semantic"), Type.Literal("hybrid"), Type.Literal("deep")]),
|
|
194
|
+
),
|
|
99
195
|
limit: Type.Optional(Type.Number({ description: "Max results (default 10)" })),
|
|
100
196
|
kb_id: Type.Optional(Type.String({ description: "Limit search to specific KB" })),
|
|
101
197
|
offset: Type.Optional(Type.Number({ description: "Pagination offset" })),
|
|
102
198
|
file_type: Type.Optional(Type.String({ description: "Filter by file type (e.g. typescript, markdown, python)" })),
|
|
103
199
|
}),
|
|
104
|
-
renderCall(args, theme) {
|
|
105
|
-
const
|
|
106
|
-
|
|
200
|
+
renderCall(args, theme: ThemeLike, context: RenderContextLike) {
|
|
201
|
+
const query = typeof args.query === "string" ? args.query : "";
|
|
202
|
+
const mode = typeof args.mode === "string" ? args.mode : "hybrid";
|
|
203
|
+
return renderText(
|
|
204
|
+
context,
|
|
205
|
+
`${themed(theme, "accent", "Search")}: "${query}" ${themed(theme, "muted", `(${mode})`)}`,
|
|
206
|
+
);
|
|
107
207
|
},
|
|
108
|
-
renderResult(result, options,
|
|
109
|
-
const text = result.content?.
|
|
208
|
+
renderResult(result: ToolResultLike, options: RenderOptionsLike, _theme: ThemeLike, context: RenderContextLike) {
|
|
209
|
+
const text = result.content?.find((item) => item.type === "text")?.text ?? "";
|
|
110
210
|
const lines = options.expanded ? text : text.split("\n").slice(0, 5).join("\n");
|
|
111
|
-
return
|
|
211
|
+
return renderText(context, lines);
|
|
112
212
|
},
|
|
113
213
|
async execute(_id, params) {
|
|
114
214
|
const { query, mode, limit, kb_id, offset, file_type } = params;
|
|
@@ -121,9 +221,9 @@ export default function (pi: ExtensionAPI) {
|
|
|
121
221
|
if (response.warnings?.length) {
|
|
122
222
|
output = `⚠️ ${response.warnings.join("\n⚠️ ")}\n\n${output}`;
|
|
123
223
|
}
|
|
124
|
-
output += response.results
|
|
125
|
-
`[${i + 1}] ${r.file_path} (${r.kb_name}, score: ${r.score.toFixed(3)})\n${r.snippet}
|
|
126
|
-
|
|
224
|
+
output += response.results
|
|
225
|
+
.map((r, i) => `[${i + 1}] ${r.file_path} (${r.kb_name}, score: ${r.score.toFixed(3)})\n${r.snippet}`)
|
|
226
|
+
.join("\n\n");
|
|
127
227
|
return { content: [{ type: "text", text: output }] };
|
|
128
228
|
},
|
|
129
229
|
});
|
|
@@ -137,16 +237,19 @@ export default function (pi: ExtensionAPI) {
|
|
|
137
237
|
target: Type.String({ description: "KB name or ID to update" }),
|
|
138
238
|
}),
|
|
139
239
|
async execute(_id, params, _signal, onUpdate) {
|
|
140
|
-
const { added, removed, unchanged } = await engine.update(
|
|
141
|
-
|
|
142
|
-
|
|
240
|
+
const { added, removed, unchanged } = await engine.update(
|
|
241
|
+
params.target,
|
|
242
|
+
(msg) => {
|
|
243
|
+
onUpdate?.({ content: [{ type: "text", text: msg }] });
|
|
244
|
+
},
|
|
245
|
+
_signal,
|
|
246
|
+
);
|
|
143
247
|
return {
|
|
144
248
|
content: [{ type: "text", text: `Updated: +${added} added, -${removed} removed, ${unchanged} unchanged.` }],
|
|
145
249
|
};
|
|
146
250
|
},
|
|
147
251
|
});
|
|
148
252
|
|
|
149
|
-
|
|
150
253
|
pi.registerTool({
|
|
151
254
|
name: "knowledge_status",
|
|
152
255
|
label: "Knowledge Status",
|
|
@@ -156,22 +259,34 @@ export default function (pi: ExtensionAPI) {
|
|
|
156
259
|
const kbs = engine.list();
|
|
157
260
|
const watchCount = getActiveWatcherCount();
|
|
158
261
|
const diagnostics = engine.diagnose();
|
|
159
|
-
const lines = [
|
|
262
|
+
const lines = [
|
|
263
|
+
`Storage: ${getDefaultKnowledgeDir()}`,
|
|
264
|
+
`Knowledge bases: ${kbs.length}`,
|
|
265
|
+
`Active watchers: ${watchCount}`,
|
|
266
|
+
"",
|
|
267
|
+
];
|
|
160
268
|
for (const kb of kbs) {
|
|
161
269
|
const age = Math.round((Date.now() - kb.updated_at) / 60000);
|
|
162
270
|
const diag = diagnostics.find((d) => d.kb_id === kb.id);
|
|
163
|
-
lines.push(
|
|
271
|
+
lines.push(
|
|
272
|
+
` "${kb.name}" — ${kb.status} — ${kb.chunk_count} chunks, ${kb.file_count} files — updated ${age}m ago`,
|
|
273
|
+
);
|
|
164
274
|
if (kb.source_path) lines.push(` source: ${kb.source_path}`);
|
|
165
275
|
if (diag) {
|
|
166
|
-
lines.push(
|
|
167
|
-
|
|
168
|
-
|
|
276
|
+
lines.push(
|
|
277
|
+
` coverage: ${diag.coverage_percent}% (${diag.indexed_files}/${diag.total_source_files} files)`,
|
|
278
|
+
);
|
|
279
|
+
if (diag.stale_files.length > 0)
|
|
280
|
+
lines.push(` ⚠️ stale: ${diag.stale_files.length} files modified since last index`);
|
|
281
|
+
if (diag.orphan_files.length > 0)
|
|
282
|
+
lines.push(` ⚠️ orphans: ${diag.orphan_files.length} chunks reference deleted files`);
|
|
169
283
|
}
|
|
170
284
|
}
|
|
171
285
|
const totalStale = diagnostics.reduce((n, d) => n + d.stale_files.length, 0);
|
|
172
286
|
const totalOrphans = diagnostics.reduce((n, d) => n + d.orphan_files.length, 0);
|
|
173
287
|
if (totalStale === 0 && totalOrphans === 0 && kbs.length > 0) lines.push("", "Health: ✓ all indexes up to date");
|
|
174
|
-
else if (totalStale > 0 || totalOrphans > 0)
|
|
288
|
+
else if (totalStale > 0 || totalOrphans > 0)
|
|
289
|
+
lines.push("", `Health: ⚠️ ${totalStale} stale, ${totalOrphans} orphans — run knowledge_update`);
|
|
175
290
|
return { content: [{ type: "text", text: lines.join("\n") }] };
|
|
176
291
|
},
|
|
177
292
|
});
|
|
@@ -224,9 +339,13 @@ export default function (pi: ExtensionAPI) {
|
|
|
224
339
|
input: Type.String({ description: "Input JSONL file path" }),
|
|
225
340
|
}),
|
|
226
341
|
async execute(_id, params, _signal, onUpdate) {
|
|
227
|
-
const { kb, chunkCount } = await engine.importKB(
|
|
228
|
-
|
|
229
|
-
|
|
342
|
+
const { kb, chunkCount } = await engine.importKB(
|
|
343
|
+
params.input,
|
|
344
|
+
(msg) => {
|
|
345
|
+
onUpdate?.({ content: [{ type: "text", text: msg }] });
|
|
346
|
+
},
|
|
347
|
+
_signal,
|
|
348
|
+
);
|
|
230
349
|
return { content: [{ type: "text", text: `Imported "${kb.name}": ${chunkCount} chunks (re-embedded)` }] };
|
|
231
350
|
},
|
|
232
351
|
});
|
package/package.json
CHANGED
|
@@ -1,62 +1,62 @@
|
|
|
1
1
|
{
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
2
|
+
"name": "pi-knowledge",
|
|
3
|
+
"version": "0.3.1",
|
|
4
|
+
"description": "Vector knowledge base extension for Pi — index files, hybrid search, local embeddings",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"main": "index.ts",
|
|
7
|
+
"pi": {
|
|
8
|
+
"extensions": [
|
|
9
|
+
"./index.ts"
|
|
10
|
+
]
|
|
11
|
+
},
|
|
12
|
+
"files": [
|
|
13
|
+
"index.ts",
|
|
14
|
+
"src/",
|
|
15
|
+
".pi/",
|
|
16
|
+
"README.md",
|
|
17
|
+
"CHANGELOG.md",
|
|
18
|
+
"LICENSE"
|
|
19
|
+
],
|
|
20
|
+
"keywords": [
|
|
21
|
+
"pi",
|
|
22
|
+
"pi-extension",
|
|
23
|
+
"knowledge-base",
|
|
24
|
+
"vector-search",
|
|
25
|
+
"semantic-search",
|
|
26
|
+
"rag",
|
|
27
|
+
"embeddings"
|
|
28
|
+
],
|
|
29
|
+
"scripts": {
|
|
30
|
+
"check": "biome check --write --error-on-warnings .",
|
|
31
|
+
"test": "vitest --run test/unit/",
|
|
32
|
+
"test:e2e": "vitest --run test/e2e/",
|
|
33
|
+
"bench": "vitest bench test/bench/"
|
|
34
|
+
},
|
|
35
|
+
"dependencies": {
|
|
36
|
+
"@huggingface/transformers": "3.8.1",
|
|
37
|
+
"better-sqlite3": "11.9.1",
|
|
38
|
+
"ignore": "7.0.5",
|
|
39
|
+
"mammoth": "1.12.0",
|
|
40
|
+
"tree-sitter": "0.22.4",
|
|
41
|
+
"tree-sitter-go": "0.23.4",
|
|
42
|
+
"tree-sitter-java": "0.23.5",
|
|
43
|
+
"tree-sitter-python": "0.23.6",
|
|
44
|
+
"tree-sitter-rust": "0.23.2",
|
|
45
|
+
"tree-sitter-typescript": "0.23.2",
|
|
46
|
+
"unpdf": "1.6.2"
|
|
47
|
+
},
|
|
48
|
+
"devDependencies": {
|
|
49
|
+
"@biomejs/biome": "2.3.5",
|
|
50
|
+
"@types/better-sqlite3": "7.6.13",
|
|
51
|
+
"@types/node": "22.19.19",
|
|
52
|
+
"vitest": "3.2.4"
|
|
53
|
+
},
|
|
54
|
+
"engines": {
|
|
55
|
+
"node": ">=22.0.0"
|
|
56
|
+
},
|
|
57
|
+
"license": "MIT",
|
|
58
|
+
"repository": {
|
|
59
|
+
"type": "git",
|
|
60
|
+
"url": "https://github.com/nczz/pi-knowledge.git"
|
|
61
|
+
}
|
|
62
62
|
}
|
|
@@ -2,14 +2,14 @@ import { existsSync, statSync } from "node:fs";
|
|
|
2
2
|
import { join } from "node:path";
|
|
3
3
|
import type Database from "better-sqlite3";
|
|
4
4
|
import { walkDir } from "../indexer/chunker.ts";
|
|
5
|
-
import { type KnowledgeBase
|
|
5
|
+
import { getChunksByKB, type KnowledgeBase } from "../storage/sqlite.ts";
|
|
6
6
|
|
|
7
7
|
export interface DiagnosticResult {
|
|
8
8
|
kb_id: string;
|
|
9
9
|
kb_name: string;
|
|
10
|
-
stale_files: string[];
|
|
11
|
-
orphan_files: string[];
|
|
12
|
-
coverage_percent: number;
|
|
10
|
+
stale_files: string[]; // files modified after indexing
|
|
11
|
+
orphan_files: string[]; // chunks referencing deleted files
|
|
12
|
+
coverage_percent: number; // indexed files / total scannable files
|
|
13
13
|
total_source_files: number;
|
|
14
14
|
indexed_files: number;
|
|
15
15
|
}
|
|
@@ -25,16 +25,15 @@ export function diagnoseKB(db: Database.Database, kb: KnowledgeBase): Diagnostic
|
|
|
25
25
|
indexed_files: kb.file_count,
|
|
26
26
|
};
|
|
27
27
|
|
|
28
|
-
if (!kb.source_path || !existsSync(kb.source_path)) {
|
|
28
|
+
if (!kb.source_path || kb.source_type === "url" || !existsSync(kb.source_path)) {
|
|
29
29
|
return result; // text KBs or missing source — no diagnostics possible
|
|
30
30
|
}
|
|
31
31
|
|
|
32
32
|
// Scan current source files
|
|
33
33
|
const currentFiles = new Set<string>();
|
|
34
|
+
const isDirectory = statSync(kb.source_path).isDirectory();
|
|
34
35
|
try {
|
|
35
|
-
const scanned =
|
|
36
|
-
? walkDir(kb.source_path).map((f) => f.relPath)
|
|
37
|
-
: [kb.source_path];
|
|
36
|
+
const scanned = isDirectory ? walkDir(kb.source_path).map((f) => f.relPath) : [kb.source_path];
|
|
38
37
|
for (const f of scanned) currentFiles.add(f);
|
|
39
38
|
} catch {
|
|
40
39
|
return result;
|
|
@@ -56,7 +55,7 @@ export function diagnoseKB(db: Database.Database, kb: KnowledgeBase): Diagnostic
|
|
|
56
55
|
|
|
57
56
|
// Staleness detection: source files modified after last indexing
|
|
58
57
|
for (const relPath of currentFiles) {
|
|
59
|
-
const absPath = join(kb.source_path, relPath);
|
|
58
|
+
const absPath = isDirectory ? join(kb.source_path, relPath) : relPath;
|
|
60
59
|
try {
|
|
61
60
|
const mtime = statSync(absPath).mtimeMs;
|
|
62
61
|
// Find latest indexed_at for this file's chunks
|
|
@@ -67,7 +66,9 @@ export function diagnoseKB(db: Database.Database, kb: KnowledgeBase): Diagnostic
|
|
|
67
66
|
result.stale_files.push(relPath);
|
|
68
67
|
}
|
|
69
68
|
}
|
|
70
|
-
} catch {
|
|
69
|
+
} catch {
|
|
70
|
+
/* file unreadable — skip */
|
|
71
|
+
}
|
|
71
72
|
}
|
|
72
73
|
|
|
73
74
|
return result;
|
|
@@ -1,21 +1,29 @@
|
|
|
1
1
|
import { join } from "node:path";
|
|
2
2
|
import { getDefaultKnowledgeDir } from "../storage/sqlite.ts";
|
|
3
3
|
|
|
4
|
-
|
|
4
|
+
type FeatureExtractionPipeline = {
|
|
5
|
+
(input: string, options: { pooling: "mean"; normalize: true }): Promise<{ data: ArrayLike<number> }>;
|
|
6
|
+
dispose(): Promise<void> | void;
|
|
7
|
+
};
|
|
8
|
+
|
|
9
|
+
let pipelineInstance: FeatureExtractionPipeline | null = null;
|
|
5
10
|
let disposeTimer: ReturnType<typeof setTimeout> | null = null;
|
|
6
11
|
|
|
7
|
-
const IDLE_TIMEOUT_MS =
|
|
12
|
+
const IDLE_TIMEOUT_MS = 30_000;
|
|
8
13
|
const EMBEDDING_CONFIG = process.env.PI_KNOWLEDGE_EMBEDDING ?? "local:multilingual-e5-small";
|
|
9
14
|
|
|
10
15
|
function getModelCacheDir(): string {
|
|
11
16
|
return join(getDefaultKnowledgeDir(), "models");
|
|
12
17
|
}
|
|
13
18
|
|
|
14
|
-
async function loadPipeline(): Promise<
|
|
19
|
+
async function loadPipeline(): Promise<FeatureExtractionPipeline> {
|
|
15
20
|
if (pipelineInstance) return pipelineInstance;
|
|
16
21
|
const { pipeline, env } = await import("@huggingface/transformers");
|
|
17
22
|
env.cacheDir = getModelCacheDir();
|
|
18
|
-
pipelineInstance = await pipeline("feature-extraction", "Xenova/multilingual-e5-small", {
|
|
23
|
+
pipelineInstance = await pipeline("feature-extraction", "Xenova/multilingual-e5-small", {
|
|
24
|
+
quantized: true,
|
|
25
|
+
dtype: "fp32",
|
|
26
|
+
});
|
|
19
27
|
return pipelineInstance;
|
|
20
28
|
}
|
|
21
29
|
|
|
@@ -25,8 +33,14 @@ function resetIdleTimer(): void {
|
|
|
25
33
|
}
|
|
26
34
|
|
|
27
35
|
export async function dispose(): Promise<void> {
|
|
28
|
-
if (disposeTimer) {
|
|
29
|
-
|
|
36
|
+
if (disposeTimer) {
|
|
37
|
+
clearTimeout(disposeTimer);
|
|
38
|
+
disposeTimer = null;
|
|
39
|
+
}
|
|
40
|
+
if (pipelineInstance) {
|
|
41
|
+
await pipelineInstance.dispose();
|
|
42
|
+
pipelineInstance = null;
|
|
43
|
+
}
|
|
30
44
|
}
|
|
31
45
|
|
|
32
46
|
async function embedViaAPI(texts: string[], prefix: "query" | "passage"): Promise<Float32Array[]> {
|
|
@@ -49,10 +63,17 @@ async function embedViaAPI(texts: string[], prefix: "query" | "passage"): Promis
|
|
|
49
63
|
throw new Error(`Unsupported embedding provider: ${provider}`);
|
|
50
64
|
}
|
|
51
65
|
|
|
52
|
-
export async function embedTexts(
|
|
66
|
+
export async function embedTexts(
|
|
67
|
+
texts: string[],
|
|
68
|
+
prefix: "query" | "passage",
|
|
69
|
+
signal?: AbortSignal,
|
|
70
|
+
): Promise<Float32Array[]> {
|
|
53
71
|
if (!EMBEDDING_CONFIG.startsWith("local")) {
|
|
54
|
-
try {
|
|
55
|
-
|
|
72
|
+
try {
|
|
73
|
+
return await embedViaAPI(texts, prefix);
|
|
74
|
+
} catch {
|
|
75
|
+
/* fallback to local */
|
|
76
|
+
}
|
|
56
77
|
}
|
|
57
78
|
const pipe = await loadPipeline();
|
|
58
79
|
resetIdleTimer();
|
package/src/embedding/vectors.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
2
2
|
import { dirname } from "node:path";
|
|
3
3
|
|
|
4
4
|
export function saveVectors(path: string, vectors: Float32Array[]): void {
|