pi-knowledge 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -0
- package/LICENSE +21 -0
- package/README.md +82 -0
- package/index.ts +202 -0
- package/package.json +58 -0
- package/src/diagnostics/health.ts +74 -0
- package/src/embedding/provider.ts +74 -0
- package/src/embedding/vectors.ts +36 -0
- package/src/engine.ts +300 -0
- package/src/indexer/chunker.ts +263 -0
- package/src/indexer/chunkers/code-ast.ts +139 -0
- package/src/search/bm25.ts +31 -0
- package/src/search/fusion.ts +14 -0
- package/src/search/reranker.ts +42 -0
- package/src/search/vector.ts +13 -0
- package/src/storage/sqlite.ts +222 -0
- package/src/watcher/file-watcher.ts +36 -0
package/CHANGELOG.md
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## [Unreleased]
|
|
4
|
+
|
|
5
|
+
### Added
|
|
6
|
+
- Project scaffold and design document
|
|
7
|
+
- Extension entry point with 7 tools: knowledge_add, knowledge_search, knowledge_update, knowledge_status, knowledge_show, knowledge_remove, knowledge_clear
|
|
8
|
+
- SQLite storage with FTS5 full-text search (WAL mode, content-sync triggers)
|
|
9
|
+
- Markdown-aware and paragraph-based chunking with camelCase/CJK pre-tokenization
|
|
10
|
+
- Local embedding via @huggingface/transformers (multilingual-e5-small, 384d, lazy load + idle dispose)
|
|
11
|
+
- Hybrid search: BM25 + vector cosine + Reciprocal Rank Fusion (RRF)
|
|
12
|
+
- Cross-encoder reranking (mode: "deep") via ms-marco-MiniLM-L-4-v2
|
|
13
|
+
- Incremental re-indexing (content-hash diff, only embeds changed chunks)
|
|
14
|
+
- File watcher (fs.watch recursive + debounce, opt-in PI_KNOWLEDGE_WATCH=true)
|
|
15
|
+
- Auto-injection per turn (opt-in PI_KNOWLEDGE_AUTO_INJECT=true, BM25 fast search)
|
|
16
|
+
- Metadata filters in search (file_type, path_pattern)
|
|
17
|
+
- Pagination (offset/limit)
|
|
18
|
+
- Vector binary storage (save/load Float32Array[])
|
|
19
|
+
- 25 unit tests (chunker + search pipeline)
|
|
20
|
+
- Comprehensive docs: competitive analysis, kiro parity mapping, embedding models, search pipeline, chunking strategies, FTS5 tokenization, offline mode, technical decisions (ADRs), Pi extension architecture
|
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
# pi-knowledge
|
|
2
|
+
|
|
3
|
+
**Vector knowledge base extension for Pi** — index any file, directory, or text into persistent, searchable knowledge bases with hybrid semantic + keyword search.
|
|
4
|
+
|
|
5
|
+
Built as a native [Pi extension](https://pi.dev/docs/latest/extensions), deeply integrated with the agent lifecycle. Designed to match and exceed what kiro-cli's built-in knowledge tool offers, while being 100% local and provider-agnostic.
|
|
6
|
+
|
|
7
|
+
## Why
|
|
8
|
+
|
|
9
|
+
Coding agents forget everything outside their context window. `pi-knowledge` gives Pi persistent, searchable long-term memory over your documentation, codebases, specs, and notes — across sessions, across projects.
|
|
10
|
+
|
|
11
|
+
Unlike `pi-memory` (which manages the agent's own notes), `pi-knowledge` indexes **your existing files** and makes them semantically searchable by the agent.
|
|
12
|
+
|
|
13
|
+
## Features
|
|
14
|
+
|
|
15
|
+
| Feature | pi-knowledge | kiro-cli knowledge | pi-memory |
|
|
16
|
+
|---------|:---:|:---:|:---:|
|
|
17
|
+
| Index arbitrary files/dirs | ✅ | ✅ | ❌ |
|
|
18
|
+
| Multiple named knowledge bases | ✅ | ✅ | ❌ |
|
|
19
|
+
| Semantic (vector) search | ✅ | ✅ | ✅ (via qmd) |
|
|
20
|
+
| BM25 keyword search | ✅ | ✅ | ✅ (via qmd) |
|
|
21
|
+
| **Hybrid search + RRF fusion** | ✅ | ❌ | partial |
|
|
22
|
+
| **Cross-encoder reranking** | ✅ | ❌ | ❌ |
|
|
23
|
+
| **Incremental re-indexing** | ✅ | ❌ | ❌ |
|
|
24
|
+
| **File watcher (auto-update)** | ✅ | ❌ | ❌ |
|
|
25
|
+
| **Code-aware chunking** | ✅ | ❌ | ❌ |
|
|
26
|
+
| **Local embeddings (zero API)** | ✅ | ❌ | ✅ (qmd) |
|
|
27
|
+
| **Index quality diagnostics** | ✅ | ❌ | ❌ |
|
|
28
|
+
| **Metadata filters in search** | ✅ | ❌ | ❌ |
|
|
29
|
+
| **Progress reporting** | ✅ | partial | ❌ |
|
|
30
|
+
| Cross-session persistence | ✅ | ✅ | ✅ |
|
|
31
|
+
| Pi extension native | ✅ | N/A | ✅ |
|
|
32
|
+
| Context injection per turn | ✅ | ❌ | ✅ |
|
|
33
|
+
| TUI custom rendering | planned | N/A | ❌ |
|
|
34
|
+
| RPC mode support | ✅ | N/A | N/A |
|
|
35
|
+
|
|
36
|
+
## Quick Start
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
# Install
|
|
40
|
+
pi install npm:pi-knowledge
|
|
41
|
+
|
|
42
|
+
# Or from source
|
|
43
|
+
pi install ./pi-knowledge
|
|
44
|
+
|
|
45
|
+
# Index a directory
|
|
46
|
+
# (agent will call knowledge_add automatically, or you can ask it)
|
|
47
|
+
> Index my project docs at ./docs as "Project Docs"
|
|
48
|
+
|
|
49
|
+
# Search
|
|
50
|
+
> Search my knowledge base for "authentication flow"
|
|
51
|
+
|
|
52
|
+
# The agent also auto-searches relevant knowledge before answering domain questions
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
## Tools
|
|
56
|
+
|
|
57
|
+
| Tool | Description |
|
|
58
|
+
|------|-------------|
|
|
59
|
+
| `knowledge_add` | Index files, directories, or inline text into a named knowledge base |
|
|
60
|
+
| `knowledge_search` | Hybrid search across one or all knowledge bases |
|
|
61
|
+
| `knowledge_remove` | Remove a knowledge base by name or ID |
|
|
62
|
+
| `knowledge_update` | Incrementally re-index changed files in a knowledge base |
|
|
63
|
+
| `knowledge_show` | List all knowledge bases with stats |
|
|
64
|
+
| `knowledge_status` | Show engine status with health diagnostics (stale, orphans, coverage) |
|
|
65
|
+
| `knowledge_clear` | Remove all knowledge bases |
|
|
66
|
+
|
|
67
|
+
## Architecture
|
|
68
|
+
|
|
69
|
+
See [DESIGN.md](DESIGN.md) for the full technical design document.
|
|
70
|
+
|
|
71
|
+
## Development
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
npm install
|
|
75
|
+
npm test # Unit tests
|
|
76
|
+
npm run test:e2e # Integration tests (requires pi + API key)
|
|
77
|
+
npm run bench # Indexing/search benchmarks
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
## License
|
|
81
|
+
|
|
82
|
+
MIT
|
package/index.ts
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
|
2
|
+
import { Type } from "typebox";
|
|
3
|
+
import { KnowledgeEngine } from "./src/engine.ts";
|
|
4
|
+
import { getDefaultKnowledgeDir } from "./src/storage/sqlite.ts";
|
|
5
|
+
import { startWatcher, stopAllWatchers, getActiveWatcherCount } from "./src/watcher/file-watcher.ts";
|
|
6
|
+
|
|
7
|
+
const engine = new KnowledgeEngine();
|
|
8
|
+
const WATCH_ENABLED = process.env.PI_KNOWLEDGE_WATCH === "true";
|
|
9
|
+
const AUTO_INJECT = process.env.PI_KNOWLEDGE_AUTO_INJECT === "true";
|
|
10
|
+
|
|
11
|
+
export default function (pi: ExtensionAPI) {
|
|
12
|
+
pi.on("session_start", async () => {
|
|
13
|
+
await engine.initialize(getDefaultKnowledgeDir());
|
|
14
|
+
if (WATCH_ENABLED) {
|
|
15
|
+
for (const kb of engine.list()) {
|
|
16
|
+
if (kb.source_path && kb.source_type === "directory") {
|
|
17
|
+
startWatcher(kb.id, kb.source_path, (kbId) => { engine.update(kbId).catch(() => {}); });
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
pi.on("session_shutdown", async () => {
|
|
24
|
+
stopAllWatchers();
|
|
25
|
+
await engine.dispose();
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
// Auto-inject: search KB for relevant context before each LLM call (opt-in)
|
|
29
|
+
if (AUTO_INJECT) {
|
|
30
|
+
pi.on("context", async (event) => {
|
|
31
|
+
const kbs = engine.list();
|
|
32
|
+
if (kbs.length === 0) return;
|
|
33
|
+
// Find last user message
|
|
34
|
+
const lastUser = [...event.messages].reverse().find((m) => m.role === "user");
|
|
35
|
+
if (!lastUser) return;
|
|
36
|
+
const text = "content" in lastUser && typeof lastUser.content === "string" ? lastUser.content : "";
|
|
37
|
+
if (!text || text.length < 5) return;
|
|
38
|
+
try {
|
|
39
|
+
const results = await engine.search(text, { mode: "fast", limit: 3 });
|
|
40
|
+
if (results.results.length === 0) return;
|
|
41
|
+
const context = results.results.map((r) => `[${r.file_path}]: ${r.snippet}`).join("\n\n");
|
|
42
|
+
event.messages.unshift({ role: "user", content: `[Knowledge context]\n${context}` } as any);
|
|
43
|
+
} catch { /* silent fail */ }
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
pi.on("before_agent_start", (event) => {
|
|
48
|
+
const kbs = engine.list();
|
|
49
|
+
if (kbs.length > 0) {
|
|
50
|
+
const desc = kbs.map((kb) => `"${kb.name}" (${kb.chunk_count} chunks, ${kb.file_count} files)`).join(", ");
|
|
51
|
+
event.systemPromptOptions.promptGuidelines?.push(
|
|
52
|
+
`Available knowledge bases: ${desc}. Use knowledge_search before answering domain questions.`,
|
|
53
|
+
);
|
|
54
|
+
}
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
pi.registerTool({
|
|
58
|
+
name: "knowledge_add",
|
|
59
|
+
label: "Knowledge Add",
|
|
60
|
+
description: "Index files, directories, or text into a named knowledge base for semantic search",
|
|
61
|
+
promptSnippet: "Index files/dirs/text into a searchable knowledge base",
|
|
62
|
+
promptGuidelines: [
|
|
63
|
+
"Use knowledge_add when the user asks to index, remember, or learn from files or documentation",
|
|
64
|
+
"Provide a descriptive name for the knowledge base",
|
|
65
|
+
],
|
|
66
|
+
parameters: Type.Object({
|
|
67
|
+
source: Type.String({ description: "File path, directory path, or inline text to index" }),
|
|
68
|
+
name: Type.String({ description: "Display name for this knowledge base" }),
|
|
69
|
+
}),
|
|
70
|
+
async execute(_id, params, _signal, onUpdate) {
|
|
71
|
+
const { source, name } = params;
|
|
72
|
+
const { kb, chunkCount } = await engine.add(source, name, (msg) => {
|
|
73
|
+
onUpdate?.({ content: [{ type: "text", text: msg }] });
|
|
74
|
+
});
|
|
75
|
+
// Start watcher for new directory KB
|
|
76
|
+
if (WATCH_ENABLED && kb.source_path && kb.source_type === "directory") {
|
|
77
|
+
startWatcher(kb.id, kb.source_path, (kbId) => { engine.update(kbId).catch(() => {}); });
|
|
78
|
+
}
|
|
79
|
+
return {
|
|
80
|
+
content: [{ type: "text", text: `Indexed "${kb.name}": ${chunkCount} chunks from ${kb.file_count} files. KB ID: ${kb.id}` }],
|
|
81
|
+
};
|
|
82
|
+
},
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
pi.registerTool({
|
|
86
|
+
name: "knowledge_search",
|
|
87
|
+
label: "Knowledge Search",
|
|
88
|
+
description: "Search indexed knowledge bases using hybrid semantic + keyword search",
|
|
89
|
+
promptSnippet: "Search knowledge bases (hybrid BM25 + semantic + RRF fusion)",
|
|
90
|
+
promptGuidelines: [
|
|
91
|
+
"Use knowledge_search to find relevant context before answering domain questions",
|
|
92
|
+
"Default mode 'hybrid' combines keyword and semantic search for best results",
|
|
93
|
+
"Use mode 'fast' for exact symbol/term lookups, 'semantic' for conceptual queries",
|
|
94
|
+
],
|
|
95
|
+
parameters: Type.Object({
|
|
96
|
+
query: Type.String({ description: "Search query" }),
|
|
97
|
+
mode: Type.Optional(Type.Union([Type.Literal("fast"), Type.Literal("semantic"), Type.Literal("hybrid"), Type.Literal("deep")])),
|
|
98
|
+
limit: Type.Optional(Type.Number({ description: "Max results (default 10)" })),
|
|
99
|
+
kb_id: Type.Optional(Type.String({ description: "Limit search to specific KB" })),
|
|
100
|
+
offset: Type.Optional(Type.Number({ description: "Pagination offset" })),
|
|
101
|
+
file_type: Type.Optional(Type.String({ description: "Filter by file type (e.g. typescript, markdown, python)" })),
|
|
102
|
+
}),
|
|
103
|
+
async execute(_id, params) {
|
|
104
|
+
const { query, mode, limit, kb_id, offset, file_type } = params;
|
|
105
|
+
const filters = file_type ? { file_type } : undefined;
|
|
106
|
+
const response = await engine.search(query, { mode, limit, kb_id, offset, filters });
|
|
107
|
+
if (response.results.length === 0) {
|
|
108
|
+
return { content: [{ type: "text", text: "No results found." }] };
|
|
109
|
+
}
|
|
110
|
+
const formatted = response.results.map((r, i) =>
|
|
111
|
+
`[${i + 1}] ${r.file_path} (${r.kb_name}, score: ${r.score.toFixed(3)})\n${r.snippet}`,
|
|
112
|
+
).join("\n\n");
|
|
113
|
+
return {
|
|
114
|
+
content: [{ type: "text", text: `${response.total_count} results (showing ${response.results.length}):\n\n${formatted}` }],
|
|
115
|
+
};
|
|
116
|
+
},
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
pi.registerTool({
|
|
120
|
+
name: "knowledge_update",
|
|
121
|
+
label: "Knowledge Update",
|
|
122
|
+
description: "Incrementally re-index a knowledge base (only re-embeds changed content)",
|
|
123
|
+
promptSnippet: "Incrementally update a knowledge base (only changed files re-embedded)",
|
|
124
|
+
parameters: Type.Object({
|
|
125
|
+
target: Type.String({ description: "KB name or ID to update" }),
|
|
126
|
+
}),
|
|
127
|
+
async execute(_id, params, _signal, onUpdate) {
|
|
128
|
+
const { added, removed, unchanged } = await engine.update(params.target, (msg) => {
|
|
129
|
+
onUpdate?.({ content: [{ type: "text", text: msg }] });
|
|
130
|
+
});
|
|
131
|
+
return {
|
|
132
|
+
content: [{ type: "text", text: `Updated: +${added} added, -${removed} removed, ${unchanged} unchanged.` }],
|
|
133
|
+
};
|
|
134
|
+
},
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
pi.registerTool({
|
|
138
|
+
name: "knowledge_status",
|
|
139
|
+
label: "Knowledge Status",
|
|
140
|
+
description: "Show knowledge engine status with health diagnostics: staleness, orphans, and coverage",
|
|
141
|
+
parameters: Type.Object({}),
|
|
142
|
+
async execute() {
|
|
143
|
+
const kbs = engine.list();
|
|
144
|
+
const watchCount = getActiveWatcherCount();
|
|
145
|
+
const diagnostics = engine.diagnose();
|
|
146
|
+
const lines = [`Storage: ${getDefaultKnowledgeDir()}`, `Knowledge bases: ${kbs.length}`, `Active watchers: ${watchCount}`, ""];
|
|
147
|
+
for (const kb of kbs) {
|
|
148
|
+
const age = Math.round((Date.now() - kb.updated_at) / 60000);
|
|
149
|
+
const diag = diagnostics.find((d) => d.kb_id === kb.id);
|
|
150
|
+
lines.push(` "${kb.name}" — ${kb.status} — ${kb.chunk_count} chunks, ${kb.file_count} files — updated ${age}m ago`);
|
|
151
|
+
if (kb.source_path) lines.push(` source: ${kb.source_path}`);
|
|
152
|
+
if (diag) {
|
|
153
|
+
lines.push(` coverage: ${diag.coverage_percent}% (${diag.indexed_files}/${diag.total_source_files} files)`);
|
|
154
|
+
if (diag.stale_files.length > 0) lines.push(` ⚠️ stale: ${diag.stale_files.length} files modified since last index`);
|
|
155
|
+
if (diag.orphan_files.length > 0) lines.push(` ⚠️ orphans: ${diag.orphan_files.length} chunks reference deleted files`);
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
const totalStale = diagnostics.reduce((n, d) => n + d.stale_files.length, 0);
|
|
159
|
+
const totalOrphans = diagnostics.reduce((n, d) => n + d.orphan_files.length, 0);
|
|
160
|
+
if (totalStale === 0 && totalOrphans === 0 && kbs.length > 0) lines.push("", "Health: ✓ all indexes up to date");
|
|
161
|
+
else if (totalStale > 0 || totalOrphans > 0) lines.push("", `Health: ⚠️ ${totalStale} stale, ${totalOrphans} orphans — run knowledge_update`);
|
|
162
|
+
return { content: [{ type: "text", text: lines.join("\n") }] };
|
|
163
|
+
},
|
|
164
|
+
});
|
|
165
|
+
|
|
166
|
+
pi.registerTool({
|
|
167
|
+
name: "knowledge_show",
|
|
168
|
+
label: "Knowledge Show",
|
|
169
|
+
description: "List all indexed knowledge bases",
|
|
170
|
+
parameters: Type.Object({}),
|
|
171
|
+
async execute() {
|
|
172
|
+
const kbs = engine.list();
|
|
173
|
+
if (kbs.length === 0) return { content: [{ type: "text", text: "No knowledge bases." }] };
|
|
174
|
+
const lines = kbs.map((kb) => `• ${kb.name} — ${kb.chunk_count} chunks, ${kb.file_count} files (${kb.status})`);
|
|
175
|
+
return { content: [{ type: "text", text: lines.join("\n") }] };
|
|
176
|
+
},
|
|
177
|
+
});
|
|
178
|
+
|
|
179
|
+
pi.registerTool({
|
|
180
|
+
name: "knowledge_remove",
|
|
181
|
+
label: "Knowledge Remove",
|
|
182
|
+
description: "Remove a knowledge base by name or ID",
|
|
183
|
+
parameters: Type.Object({
|
|
184
|
+
target: Type.String({ description: "KB name or ID to remove" }),
|
|
185
|
+
}),
|
|
186
|
+
async execute(_id, params) {
|
|
187
|
+
const ok = engine.remove(params.target);
|
|
188
|
+
return { content: [{ type: "text", text: ok ? "Removed." : "Not found." }] };
|
|
189
|
+
},
|
|
190
|
+
});
|
|
191
|
+
|
|
192
|
+
pi.registerTool({
|
|
193
|
+
name: "knowledge_clear",
|
|
194
|
+
label: "Knowledge Clear",
|
|
195
|
+
description: "Remove all knowledge bases",
|
|
196
|
+
parameters: Type.Object({}),
|
|
197
|
+
async execute() {
|
|
198
|
+
engine.clear();
|
|
199
|
+
return { content: [{ type: "text", text: "All knowledge bases cleared." }] };
|
|
200
|
+
},
|
|
201
|
+
});
|
|
202
|
+
}
|
package/package.json
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "pi-knowledge",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "Vector knowledge base extension for Pi — index files, hybrid search, local embeddings",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"main": "index.ts",
|
|
7
|
+
"pi": {
|
|
8
|
+
"extensions": [
|
|
9
|
+
"./index.ts"
|
|
10
|
+
]
|
|
11
|
+
},
|
|
12
|
+
"files": [
|
|
13
|
+
"index.ts",
|
|
14
|
+
"src/",
|
|
15
|
+
"README.md",
|
|
16
|
+
"CHANGELOG.md",
|
|
17
|
+
"LICENSE"
|
|
18
|
+
],
|
|
19
|
+
"keywords": [
|
|
20
|
+
"pi",
|
|
21
|
+
"pi-extension",
|
|
22
|
+
"knowledge-base",
|
|
23
|
+
"vector-search",
|
|
24
|
+
"semantic-search",
|
|
25
|
+
"rag",
|
|
26
|
+
"embeddings"
|
|
27
|
+
],
|
|
28
|
+
"scripts": {
|
|
29
|
+
"check": "biome check --write --error-on-warnings .",
|
|
30
|
+
"test": "vitest --run test/unit/",
|
|
31
|
+
"test:e2e": "vitest --run test/e2e/",
|
|
32
|
+
"bench": "vitest bench test/bench/"
|
|
33
|
+
},
|
|
34
|
+
"dependencies": {
|
|
35
|
+
"@huggingface/transformers": "3.8.1",
|
|
36
|
+
"better-sqlite3": "11.9.1",
|
|
37
|
+
"ignore": "7.0.5",
|
|
38
|
+
"tree-sitter": "0.22.4",
|
|
39
|
+
"tree-sitter-go": "0.23.4",
|
|
40
|
+
"tree-sitter-python": "0.23.6",
|
|
41
|
+
"tree-sitter-rust": "0.23.2",
|
|
42
|
+
"tree-sitter-typescript": "0.23.2"
|
|
43
|
+
},
|
|
44
|
+
"devDependencies": {
|
|
45
|
+
"@biomejs/biome": "2.3.5",
|
|
46
|
+
"@types/better-sqlite3": "7.6.13",
|
|
47
|
+
"@types/node": "22.19.19",
|
|
48
|
+
"vitest": "3.2.4"
|
|
49
|
+
},
|
|
50
|
+
"engines": {
|
|
51
|
+
"node": ">=22.0.0"
|
|
52
|
+
},
|
|
53
|
+
"license": "MIT",
|
|
54
|
+
"repository": {
|
|
55
|
+
"type": "git",
|
|
56
|
+
"url": "https://github.com/nczz/pi-knowledge.git"
|
|
57
|
+
}
|
|
58
|
+
}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { existsSync, statSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import type Database from "better-sqlite3";
|
|
4
|
+
import { walkDir } from "../indexer/chunker.ts";
|
|
5
|
+
import { type KnowledgeBase, getChunksByKB } from "../storage/sqlite.ts";
|
|
6
|
+
|
|
7
|
+
export interface DiagnosticResult {
|
|
8
|
+
kb_id: string;
|
|
9
|
+
kb_name: string;
|
|
10
|
+
stale_files: string[]; // files modified after indexing
|
|
11
|
+
orphan_files: string[]; // chunks referencing deleted files
|
|
12
|
+
coverage_percent: number; // indexed files / total scannable files
|
|
13
|
+
total_source_files: number;
|
|
14
|
+
indexed_files: number;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export function diagnoseKB(db: Database.Database, kb: KnowledgeBase): DiagnosticResult {
|
|
18
|
+
const result: DiagnosticResult = {
|
|
19
|
+
kb_id: kb.id,
|
|
20
|
+
kb_name: kb.name,
|
|
21
|
+
stale_files: [],
|
|
22
|
+
orphan_files: [],
|
|
23
|
+
coverage_percent: 100,
|
|
24
|
+
total_source_files: 0,
|
|
25
|
+
indexed_files: kb.file_count,
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
if (!kb.source_path || !existsSync(kb.source_path)) {
|
|
29
|
+
return result; // text KBs or missing source — no diagnostics possible
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
// Scan current source files
|
|
33
|
+
const currentFiles = new Set<string>();
|
|
34
|
+
try {
|
|
35
|
+
const scanned = statSync(kb.source_path).isDirectory()
|
|
36
|
+
? walkDir(kb.source_path).map((f) => f.relPath)
|
|
37
|
+
: [kb.source_path];
|
|
38
|
+
for (const f of scanned) currentFiles.add(f);
|
|
39
|
+
} catch {
|
|
40
|
+
return result;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
result.total_source_files = currentFiles.size;
|
|
44
|
+
result.coverage_percent = currentFiles.size > 0 ? Math.round((result.indexed_files / currentFiles.size) * 100) : 100;
|
|
45
|
+
|
|
46
|
+
// Get indexed chunks
|
|
47
|
+
const chunks = getChunksByKB(db, kb.id);
|
|
48
|
+
const indexedFilePaths = new Set(chunks.map((c) => c.file_path));
|
|
49
|
+
|
|
50
|
+
// Orphan detection: chunks referencing files no longer in source
|
|
51
|
+
for (const filePath of indexedFilePaths) {
|
|
52
|
+
if (!currentFiles.has(filePath)) {
|
|
53
|
+
result.orphan_files.push(filePath);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
// Staleness detection: source files modified after last indexing
|
|
58
|
+
for (const relPath of currentFiles) {
|
|
59
|
+
const absPath = join(kb.source_path, relPath);
|
|
60
|
+
try {
|
|
61
|
+
const mtime = statSync(absPath).mtimeMs;
|
|
62
|
+
// Find latest indexed_at for this file's chunks
|
|
63
|
+
const fileChunks = chunks.filter((c) => c.file_path === relPath);
|
|
64
|
+
if (fileChunks.length > 0) {
|
|
65
|
+
const latestIndexed = Math.max(...fileChunks.map((c) => c.indexed_at));
|
|
66
|
+
if (mtime > latestIndexed) {
|
|
67
|
+
result.stale_files.push(relPath);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
} catch { /* file unreadable — skip */ }
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
return result;
|
|
74
|
+
}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { join } from "node:path";
|
|
2
|
+
import { getDefaultKnowledgeDir } from "../storage/sqlite.ts";
|
|
3
|
+
|
|
4
|
+
let pipelineInstance: any = null;
|
|
5
|
+
let disposeTimer: ReturnType<typeof setTimeout> | null = null;
|
|
6
|
+
|
|
7
|
+
const IDLE_TIMEOUT_MS = 60_000;
|
|
8
|
+
const EMBEDDING_CONFIG = process.env.PI_KNOWLEDGE_EMBEDDING ?? "local:multilingual-e5-small";
|
|
9
|
+
|
|
10
|
+
function getModelCacheDir(): string {
|
|
11
|
+
return join(getDefaultKnowledgeDir(), "models");
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
async function loadPipeline(): Promise<any> {
|
|
15
|
+
if (pipelineInstance) return pipelineInstance;
|
|
16
|
+
const { pipeline, env } = await import("@huggingface/transformers");
|
|
17
|
+
env.cacheDir = getModelCacheDir();
|
|
18
|
+
pipelineInstance = await pipeline("feature-extraction", "Xenova/multilingual-e5-small", { quantized: true });
|
|
19
|
+
return pipelineInstance;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function resetIdleTimer(): void {
|
|
23
|
+
if (disposeTimer) clearTimeout(disposeTimer);
|
|
24
|
+
disposeTimer = setTimeout(() => dispose(), IDLE_TIMEOUT_MS);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export async function dispose(): Promise<void> {
|
|
28
|
+
if (disposeTimer) { clearTimeout(disposeTimer); disposeTimer = null; }
|
|
29
|
+
if (pipelineInstance) { await pipelineInstance.dispose(); pipelineInstance = null; }
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
async function embedViaAPI(texts: string[], prefix: "query" | "passage"): Promise<Float32Array[]> {
|
|
33
|
+
const [provider, model] = EMBEDDING_CONFIG.split(":");
|
|
34
|
+
const prefixedTexts = texts.map((t) => `${prefix}: ${t}`);
|
|
35
|
+
|
|
36
|
+
if (provider === "openai") {
|
|
37
|
+
const apiKey = process.env.OPENAI_API_KEY;
|
|
38
|
+
if (!apiKey) throw new Error("OPENAI_API_KEY required for openai embedding");
|
|
39
|
+
const res = await fetch("https://api.openai.com/v1/embeddings", {
|
|
40
|
+
method: "POST",
|
|
41
|
+
headers: { "Content-Type": "application/json", Authorization: `Bearer ${apiKey}` },
|
|
42
|
+
body: JSON.stringify({ input: prefixedTexts, model: model || "text-embedding-3-small" }),
|
|
43
|
+
});
|
|
44
|
+
if (!res.ok) throw new Error(`OpenAI API error: ${res.status}`);
|
|
45
|
+
const data = (await res.json()) as { data: Array<{ embedding: number[] }> };
|
|
46
|
+
return data.data.map((d) => new Float32Array(d.embedding));
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
throw new Error(`Unsupported embedding provider: ${provider}`);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export async function embedTexts(texts: string[], prefix: "query" | "passage"): Promise<Float32Array[]> {
|
|
53
|
+
if (!EMBEDDING_CONFIG.startsWith("local")) {
|
|
54
|
+
try { return await embedViaAPI(texts, prefix); }
|
|
55
|
+
catch { /* fallback to local */ }
|
|
56
|
+
}
|
|
57
|
+
const pipe = await loadPipeline();
|
|
58
|
+
resetIdleTimer();
|
|
59
|
+
const results: Float32Array[] = [];
|
|
60
|
+
for (const text of texts) {
|
|
61
|
+
const output = await pipe(`${prefix}: ${text}`, { pooling: "mean", normalize: true });
|
|
62
|
+
results.push(new Float32Array(output.data));
|
|
63
|
+
}
|
|
64
|
+
return results;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export async function embedQuery(text: string): Promise<Float32Array> {
|
|
68
|
+
const [vec] = await embedTexts([text], "query");
|
|
69
|
+
return vec;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export async function embedDocuments(texts: string[]): Promise<Float32Array[]> {
|
|
73
|
+
return embedTexts(texts, "passage");
|
|
74
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { readFileSync, writeFileSync, existsSync, mkdirSync } from "node:fs";
|
|
2
|
+
import { dirname } from "node:path";
|
|
3
|
+
|
|
4
|
+
export function saveVectors(path: string, vectors: Float32Array[]): void {
|
|
5
|
+
if (vectors.length === 0) return;
|
|
6
|
+
const dir = dirname(path);
|
|
7
|
+
if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
|
|
8
|
+
const dim = vectors[0].length;
|
|
9
|
+
const buffer = Buffer.alloc(8 + vectors.length * dim * 4);
|
|
10
|
+
buffer.writeUInt32LE(vectors.length, 0);
|
|
11
|
+
buffer.writeUInt32LE(dim, 4);
|
|
12
|
+
let offset = 8;
|
|
13
|
+
for (const vec of vectors) {
|
|
14
|
+
Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength).copy(buffer, offset);
|
|
15
|
+
offset += dim * 4;
|
|
16
|
+
}
|
|
17
|
+
writeFileSync(path, buffer);
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export function loadVectors(path: string): Float32Array[] {
|
|
21
|
+
if (!existsSync(path)) return [];
|
|
22
|
+
const buffer = readFileSync(path);
|
|
23
|
+
if (buffer.length < 8) return [];
|
|
24
|
+
const count = buffer.readUInt32LE(0);
|
|
25
|
+
const dim = buffer.readUInt32LE(4);
|
|
26
|
+
const vectors: Float32Array[] = [];
|
|
27
|
+
for (let i = 0; i < count; i++) {
|
|
28
|
+
const offset = 8 + i * dim * 4;
|
|
29
|
+
const vec = new Float32Array(dim);
|
|
30
|
+
for (let j = 0; j < dim; j++) {
|
|
31
|
+
vec[j] = buffer.readFloatLE(offset + j * 4);
|
|
32
|
+
}
|
|
33
|
+
vectors.push(vec);
|
|
34
|
+
}
|
|
35
|
+
return vectors;
|
|
36
|
+
}
|