pi-knowledge 0.12.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/README.md +165 -32
- package/dist/index.js +33 -64
- package/dist/src/embedding/instructions.d.ts +33 -0
- package/dist/src/embedding/instructions.js +146 -0
- package/dist/src/embedding/provider.d.ts +27 -8
- package/dist/src/embedding/provider.js +197 -64
- package/dist/src/engine.d.ts +18 -1
- package/dist/src/engine.js +136 -170
- package/dist/src/indexer/embedding-safe-chunks.js +3 -2
- package/dist/src/model-worker-client.d.ts +4 -0
- package/dist/src/model-worker-client.js +14 -2
- package/dist/src/model-worker.js +34 -3
- package/dist/src/search/contracts.d.ts +2 -0
- package/dist/src/search/contracts.js +2 -0
- package/dist/src/search/diversity.d.ts +16 -0
- package/dist/src/search/diversity.js +183 -0
- package/dist/src/search/tuning.d.ts +10 -0
- package/dist/src/search/tuning.js +20 -0
- package/dist/src/status.d.ts +8 -0
- package/dist/src/status.js +93 -0
- package/docs/configuration.md +14 -11
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,35 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [Unreleased]
|
|
4
|
+
|
|
5
|
+
## [0.14.0] - 2026-11-12
|
|
6
|
+
|
|
7
|
+
### Added
|
|
8
|
+
- Added model-aware embedding input instructions with byte-compatible E5 `on`/`off`, Jina Code task presets, and exact custom query/document instructions (#22).
|
|
9
|
+
- Added instruction-aware embedding signatures and indexed-versus-active compatibility details in `knowledge_status`, `knowledge_doctor`, search warnings, and update progress.
|
|
10
|
+
|
|
11
|
+
### Changed
|
|
12
|
+
- OpenAI-compatible embedding inputs are decorated once and reused for complete-input budget validation and the HTTP request.
|
|
13
|
+
- Local API fallback is rejected during configuration for Jina/custom instructions that the local E5 model cannot reproduce.
|
|
14
|
+
|
|
15
|
+
### Fixed
|
|
16
|
+
- Included embedding incompatibility and invalid active embedding configuration in aggregate status health instead of reporting a contradictory healthy state.
|
|
17
|
+
- Isolated PDF/DOCX parsing from Pi/OMP TUI stdio, suppressed all non-fatal PDF.js warning paths, and cleaned up explicit PDF document proxies without changing extracted text (#23).
|
|
18
|
+
|
|
19
|
+
## [0.13.0] - 2026-11-07
|
|
20
|
+
|
|
21
|
+
### Added
|
|
22
|
+
- Added bounded `knowledge_search` pagination (`limit` 1-50, `offset + limit` up to 800) and diagnostics for per-KB retrieval breadth, merged/filtered/diversity candidate counts, and global-pool truncation.
|
|
23
|
+
- Added client-side validation and L2 normalization for OpenAI-compatible embedding vectors, with a signed normalization policy and `knowledge_doctor` rebuild guidance for older API-backed KBs.
|
|
24
|
+
|
|
25
|
+
### Changed
|
|
26
|
+
- Reworked the quick start and added verified Pi and Oh My Pi upgrade guidance for npm and local sources, targeted updates, explicit-version pins, safe checkout updates, build activation, source verification, host restarts, and post-upgrade diagnostics.
|
|
27
|
+
- Clarified host-specific storage, opt-in watcher/context injection, model-cache sizing, source-only architecture links, and the distinction between structured result text and custom TUI renderers.
|
|
28
|
+
|
|
29
|
+
### Fixed
|
|
30
|
+
- Reduced MMR-style diversity ranking from O(n³) repeated redundancy scans to behavior-preserving O(n²) running maxima, with cooperative event-loop yielding and cancellation checks (#21).
|
|
31
|
+
- Bounded merged multi-KB candidates before deep reranking, adaptive expansion, and diversity so query-time work cannot grow unchecked with the number of selected KBs.
|
|
32
|
+
|
|
3
33
|
## [0.12.0] - 2026-10-19
|
|
4
34
|
|
|
5
35
|
### Added
|
package/README.md
CHANGED
|
@@ -21,7 +21,7 @@ Unlike `pi-memory` (which manages the agent's own notes), `pi-knowledge` indexes
|
|
|
21
21
|
|
|
22
22
|
## Highlights
|
|
23
23
|
|
|
24
|
-
- **Local-first project memory**: stores indexes under `~/.pi/knowledge
|
|
24
|
+
- **Local-first project memory**: stores indexes under the active host knowledge directory (`~/.pi/knowledge/` for Pi or `~/.omp/knowledge/` for OMP); no project files are modified.
|
|
25
25
|
- **Hybrid retrieval**: lexical-anchored BM25 + semantic vectors + normalized weighted score fusion, with semantic mode for vector-only conceptual recall.
|
|
26
26
|
- **Code- and document-aware indexing**: recursive AST chunking for supported code, heading-aware Markdown chunks, and an embedding-safe finalizer that preserves oversized multilingual tails without adding a heavy document runtime.
|
|
27
27
|
- **Structured URL indexing**: preserves HTML headings, paragraphs, lists, tables, and entities as lightweight Markdown-like text without a browser, Docling runtime, or new dependency.
|
|
@@ -43,7 +43,7 @@ Unlike `pi-memory` (which manages the agent's own notes), `pi-knowledge` indexes
|
|
|
43
43
|
| **Adaptive contextual search** | ✅ | ❌ | ❌ |
|
|
44
44
|
| **Diversity reranking** | ✅ | ❌ | ❌ |
|
|
45
45
|
| **Incremental re-indexing** | ✅ | ❌ | ❌ |
|
|
46
|
-
| **File watcher (auto-update)** | ✅ | ❌ | ❌ |
|
|
46
|
+
| **File watcher (opt-in auto-update)** | ✅ | ❌ | ❌ |
|
|
47
47
|
| **Code-aware chunking** | ✅ (TS/JS/Py/Go/Rust/Java/Bash/C/C++/QML) | ❌ | ❌ |
|
|
48
48
|
| **Symbol/config/heading lookup** | ✅ | ❌ | ❌ |
|
|
49
49
|
| **Local embeddings (zero API)** | ✅ | ❌ | ✅ (qmd) |
|
|
@@ -52,9 +52,9 @@ Unlike `pi-memory` (which manages the agent's own notes), `pi-knowledge` indexes
|
|
|
52
52
|
| **Progress reporting + stuck indexing diagnostics** | ✅ | partial | ❌ |
|
|
53
53
|
| Cross-session persistence | ✅ | ✅ | ✅ |
|
|
54
54
|
| Pi extension native | ✅ | N/A | ✅ |
|
|
55
|
-
| Context injection per turn | ✅ | ❌ | ✅ |
|
|
56
|
-
|
|
|
57
|
-
|
|
|
55
|
+
| Context injection per turn (opt-in) | ✅ | ❌ | ✅ |
|
|
56
|
+
| Structured search result formatting | ✅ | N/A | ❌ |
|
|
57
|
+
| Custom tool TUI renderers | ❌ | N/A | ❌ |
|
|
58
58
|
| RPC mode support | ✅ | N/A | N/A |
|
|
59
59
|
|
|
60
60
|
TUI rendering currently uses Pi's stable default tool rendering plus targeted result formatting. A fuller custom renderer is intentionally not enabled yet because custom renderers must be width-safe across terminal sizes and Pi TUI modes.
|
|
@@ -73,36 +73,137 @@ TUI rendering currently uses Pi's stable default tool rendering plus targeted re
|
|
|
73
73
|
|
|
74
74
|
This is intentionally not a heavy ColBERT-style late-interaction index yet (Khattab and Zaharia 2020). The current product chooses lightweight local embeddings, BM25, query-aware ranking, optional cross-encoder reranking, streamed vector scans, and health diagnostics for commercial usefulness with low setup cost.
|
|
75
75
|
|
|
76
|
-
|
|
76
|
+
Project dogfood drove fixes for score compression, repeated same-file results,
|
|
77
|
+
garbage-query false positives, small-module discoverability, source-versus-test
|
|
78
|
+
ranking, indexing stability, and auto-mode false positives. Existing KBs should
|
|
79
|
+
be rebuilt or updated after upgrades that change indexing text.
|
|
77
80
|
|
|
78
81
|
## Quick Start
|
|
79
82
|
|
|
83
|
+
### Install
|
|
84
|
+
|
|
85
|
+
For Pi:
|
|
86
|
+
|
|
80
87
|
```bash
|
|
81
|
-
# Install for Pi
|
|
82
88
|
pi install npm:pi-knowledge
|
|
89
|
+
```
|
|
83
90
|
|
|
84
|
-
|
|
85
|
-
omp install npm:pi-knowledge
|
|
91
|
+
For OMP:
|
|
86
92
|
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
omp install npm:pi-knowledge
|
|
93
|
+
```bash
|
|
94
|
+
omp plugin install npm:pi-knowledge
|
|
95
|
+
```
|
|
91
96
|
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
omp install ./pi-knowledge
|
|
97
|
+
On minimal Ubuntu or Debian systems, install native build tools first if OMP
|
|
98
|
+
must compile `better-sqlite3`:
|
|
95
99
|
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
100
|
+
```bash
|
|
101
|
+
sudo apt-get update
|
|
102
|
+
sudo apt-get install -y build-essential python3
|
|
103
|
+
omp plugin install npm:pi-knowledge
|
|
104
|
+
```
|
|
99
105
|
|
|
100
|
-
|
|
101
|
-
> Search my knowledge base for "authentication flow"
|
|
106
|
+
For a local checkout, install dependencies and build before registration:
|
|
102
107
|
|
|
103
|
-
|
|
108
|
+
```bash
|
|
109
|
+
cd /path/to/pi-knowledge
|
|
110
|
+
npm ci
|
|
111
|
+
npm run typecheck
|
|
112
|
+
npm test
|
|
113
|
+
npm run build
|
|
114
|
+
|
|
115
|
+
# Register with the host you use:
|
|
116
|
+
pi install ./
|
|
117
|
+
omp plugin link "$(pwd)"
|
|
104
118
|
```
|
|
105
119
|
|
|
120
|
+
Use either the npm package or the local checkout for a host, not both. Check
|
|
121
|
+
Pi with `pi list` and OMP with `omp plugin list --json` when switching sources.
|
|
122
|
+
|
|
123
|
+
### Index and search
|
|
124
|
+
|
|
125
|
+
Ask the agent:
|
|
126
|
+
|
|
127
|
+
```text
|
|
128
|
+
Index my project docs at ./docs as "Project Docs".
|
|
129
|
+
Search my knowledge bases for "authentication flow".
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
The agent receives guidance to use `knowledge_symbol_search` and
|
|
133
|
+
`knowledge_search` when knowledge bases are available. Automatic behavior is
|
|
134
|
+
opt-in. Set one or both variables before starting Pi or OMP:
|
|
135
|
+
|
|
136
|
+
```bash
|
|
137
|
+
export PI_KNOWLEDGE_AUTO_INJECT=true # retrieve context before every model call
|
|
138
|
+
export PI_KNOWLEDGE_WATCH=true # update directory KBs after file changes
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
Per-turn injection adds retrieval work and prompt context to every model call.
|
|
142
|
+
File watching applies only to directory-backed knowledge bases.
|
|
143
|
+
|
|
144
|
+
## Upgrade
|
|
145
|
+
|
|
146
|
+
### npm-installed package
|
|
147
|
+
|
|
148
|
+
Update only pi-knowledge in Pi:
|
|
149
|
+
|
|
150
|
+
```bash
|
|
151
|
+
pi update npm:pi-knowledge
|
|
152
|
+
pi list
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
Use `pi update --extensions` instead when every installed Pi extension should
|
|
156
|
+
be updated.
|
|
157
|
+
|
|
158
|
+
Update only pi-knowledge in OMP:
|
|
159
|
+
|
|
160
|
+
```bash
|
|
161
|
+
omp plugin upgrade pi-knowledge
|
|
162
|
+
omp plugin list --json
|
|
163
|
+
omp plugin doctor
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
Bare `omp plugin upgrade` and `omp update --plugins --check` report marketplace
|
|
167
|
+
plugin status; they do not confirm that this npm plugin was upgraded.
|
|
168
|
+
|
|
169
|
+
Install the unversioned Pi source (`npm:pi-knowledge`) to follow the npm
|
|
170
|
+
`latest` tag. Pi preserves explicitly versioned sources such as
|
|
171
|
+
`npm:pi-knowledge@<version>`. To remove a pin:
|
|
172
|
+
|
|
173
|
+
1. Run `pi list`.
|
|
174
|
+
2. Pass the exact pinned source shown there to `pi remove`.
|
|
175
|
+
3. Run `pi install npm:pi-knowledge`.
|
|
176
|
+
|
|
177
|
+
### Local checkout
|
|
178
|
+
|
|
179
|
+
A linked checkout is updated from Git, not from the npm registry. Finish,
|
|
180
|
+
commit, or stash local edits before pulling. Exit Pi and OMP processes that use
|
|
181
|
+
the checkout before replacing dependencies or build output.
|
|
182
|
+
|
|
183
|
+
```bash
|
|
184
|
+
cd /path/to/pi-knowledge
|
|
185
|
+
git status --short
|
|
186
|
+
git pull --ff-only # only when the checkout is clean and follows a remote branch
|
|
187
|
+
npm ci
|
|
188
|
+
npm run typecheck
|
|
189
|
+
npm test
|
|
190
|
+
npm run build
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
When actively developing local changes, skip `git pull --ff-only` and build the
|
|
194
|
+
current tree. `extension.js` prefers `dist/index.js`, so source edits are
|
|
195
|
+
inactive until the build succeeds. The local registration only needs to be
|
|
196
|
+
repeated when its path or host settings change.
|
|
197
|
+
|
|
198
|
+
Pi and OMP load extension modules once per process. Start a new host process
|
|
199
|
+
after a successful registry upgrade or local build. Then use `pi list` to
|
|
200
|
+
confirm Pi's selected source, or `omp plugin list --json` to confirm OMP's
|
|
201
|
+
installed version and path.
|
|
202
|
+
|
|
203
|
+
After releases that change indexing text, run `knowledge_doctor` and follow its
|
|
204
|
+
suggested reindex action. Source-backed knowledge bases can normally use
|
|
205
|
+
`knowledge_update`; rebuild only when diagnostics or release notes require it.
|
|
206
|
+
|
|
106
207
|
## Tools
|
|
107
208
|
|
|
108
209
|
| Tool | Description |
|
|
@@ -116,7 +217,7 @@ omp install ./pi-knowledge
|
|
|
116
217
|
| `knowledge_show` | List all knowledge bases with stats |
|
|
117
218
|
| `knowledge_status` | Show engine status with health diagnostics (stale, orphans, coverage) |
|
|
118
219
|
| `knowledge_doctor` | Diagnose health score, skipped files, stuck jobs, stale data, and recommended fixes |
|
|
119
|
-
| `knowledge_configure` | Persist runtime prerequisites such as a Windows `node.exe` path for the isolated
|
|
220
|
+
| `knowledge_configure` | Persist runtime prerequisites such as a Windows `node.exe` path for the isolated runtime worker |
|
|
120
221
|
| `knowledge_clear` | Remove all knowledge bases after `confirm: true` |
|
|
121
222
|
| `knowledge_export` | Export a KB to shareable JSONL file |
|
|
122
223
|
| `knowledge_import` | Import a KB from JSONL (re-embeds content) |
|
|
@@ -145,6 +246,8 @@ Mode selection contract:
|
|
|
145
246
|
|
|
146
247
|
Search results use balanced diversity reranking by default so near-duplicate chunks from the same file do not dominate the top results. Diversity scoring considers lexical overlap, same-file line proximity, overlapping adaptive windows, available embedding-vector similarity, file-level interleaving, and a small KB trust multiplier that favors ready file/directory sources over stale, URL, or imported text sources. In `adaptive` mode, code chunks with AST metadata can prefer same-parent/sibling chunks inside the same bounded file window; this is metadata-driven context expansion, not caller/callee analysis. Use `diversity: "off"` only when raw ranking order is needed for diagnostics. Agents can request search diagnostics to inspect mode fallback, selected profile, applied search tuning, ranking coverage, path/source/test boosts, adjusted scores, and provenance such as chunk id, chunk hash, match reason, stale flag, source freshness, and adaptive `source_chunk_ids` where available.
|
|
147
248
|
|
|
249
|
+
`limit` must be an integer from 1 to 50. `offset` must be a non-negative integer, and `offset + limit` cannot exceed the bounded 800-candidate search window. Each selected KB may contribute candidates up to the profile-derived retrieval limit, but merged candidates are globally ranked and capped before deep reranking, adaptive expansion, or diversity work. Diagnostics expose per-KB retrieval breadth, retrieved and filtered counts, the diversity pool size, and whether the global pool was truncated.
|
|
250
|
+
|
|
148
251
|
For best search quality, rebuild or update existing knowledge bases after upgrading. New indexes use contextual retrieval units: embeddings and FTS include file path, file type, Markdown heading breadcrumbs, and AST-derived code structure such as symbol names, scopes, parent symbols, and signatures while returned results keep the original chunk text readable. Before embedding, the complete contextual input is checked against the active provider budget; only oversized chunks are split again along structural boundaries.
|
|
149
252
|
Symbol/config/heading metadata and embedding-safe chunk boundaries are rebuilt during `knowledge_update`. `knowledge_doctor` reports `reindex_for_embedding_budget` for older KBs: file, directory, and URL sources can use `knowledge_update`; imported or inline-text sources should be re-imported or re-added from the original source.
|
|
150
253
|
|
|
@@ -171,13 +274,35 @@ export PI_KNOWLEDGE_EMBEDDING_BASE_URL=http://127.0.0.1:8080/v1
|
|
|
171
274
|
# Set PI_KNOWLEDGE_EMBEDDING_API_KEY or OPENAI_API_KEY only if your endpoint requires bearer auth.
|
|
172
275
|
```
|
|
173
276
|
|
|
174
|
-
|
|
277
|
+
Embedding input instructions must match the selected model. `PI_KNOWLEDGE_EMBEDDING_PREFIXES` accepts:
|
|
278
|
+
|
|
279
|
+
| Value | Query/document input | Intended use |
|
|
280
|
+
|-------|----------------------|--------------|
|
|
281
|
+
| `on` or `e5` | `query: ` / `passage: ` | Default local `multilingual-e5-small`; byte-compatible with existing indexes |
|
|
282
|
+
| `off` or `none` | Raw text | Models explicitly evaluated without instructions |
|
|
283
|
+
| `jina` or `jina-code` | Published Jina Code task prompts | `jinaai/jina-code-embeddings-1.5b` through an OpenAI-compatible endpoint |
|
|
284
|
+
| `custom` | Exact configured query/document instructions | Other instructed embedding models |
|
|
285
|
+
|
|
286
|
+
Jina Code example:
|
|
287
|
+
|
|
288
|
+
```bash
|
|
289
|
+
export PI_KNOWLEDGE_EMBEDDING=openai:jinaai/jina-code-embeddings-1.5b
|
|
290
|
+
export PI_KNOWLEDGE_EMBEDDING_BASE_URL=http://127.0.0.1:8080/v1
|
|
291
|
+
export PI_KNOWLEDGE_EMBEDDING_PREFIXES=jina-code
|
|
292
|
+
export PI_KNOWLEDGE_EMBEDDING_TASK=nl2code
|
|
293
|
+
```
|
|
294
|
+
|
|
295
|
+
The Jina preset controls input text only. The embedding server must independently implement the model's required last-token pooling and vector normalization. For custom instructions, set `PI_KNOWLEDGE_EMBEDDING_PREFIXES=custom` plus both `PI_KNOWLEDGE_EMBEDDING_QUERY_INSTRUCTION` and `PI_KNOWLEDGE_EMBEDDING_DOCUMENT_INSTRUCTION`; values are concatenated verbatim, so include delimiters such as a trailing newline.
|
|
296
|
+
|
|
297
|
+
API embedding failures are surfaced by default so configuration and context-window problems are visible. To explicitly allow query-time local fallback after API failures, set `PI_KNOWLEDGE_EMBEDDING_API_FALLBACK=local`; indexing/import/update document batches still fail instead of creating mixed-provider vector files. Local fallback is rejected during configuration when `jina-code` or `custom` instructions are selected because the local E5 model cannot reproduce those inputs. Inputs are never silently truncated.
|
|
298
|
+
|
|
299
|
+
OpenAI-compatible API vectors are validated and L2-normalized by the client before query use or persistence, so cosine-style dot products do not depend on a provider-specific vector magnitude. This normalization policy is part of the embedding signature. KBs created with an older API signature must run `knowledge_update`; `knowledge_doctor` reports `rebuild_vectors` when the active provider no longer matches stored vector semantics.
|
|
175
300
|
|
|
176
301
|
OpenAI-compatible embedding inputs use a 20000-character complete-input safety cap by default. Indexing splits oversized chunks before HTTP requests; direct oversized query/document calls fail explicitly. Adjust the cap with `PI_KNOWLEDGE_EMBEDDING_MAX_CHARS` when the remote model has a different context window. Changing the cap changes embedding compatibility and requires `knowledge_update`.
|
|
177
302
|
|
|
178
303
|
For self-hosted servers with smaller request limits, set `PI_KNOWLEDGE_EMBEDDING_BATCH_SIZE` to the maximum number of chunk inputs the server accepts per embedding request.
|
|
179
304
|
|
|
180
|
-
Stored KB metadata includes the embedding model, vector dimension, and a non-secret embedding signature. If the current query embedding is incompatible with a KB's stored vectors, or if an older KB has no signature metadata, `knowledge_search` skips vector retrieval for that KB and reports
|
|
305
|
+
Stored KB metadata includes the embedding model, vector dimension, and a non-secret embedding signature containing the exact instruction strategy/task/hash. If the current query embedding is incompatible with a KB's stored vectors, or if an older KB has no signature metadata, `knowledge_search` skips vector retrieval for that KB and reports the specific mismatch; `knowledge_status` shows indexed versus active model/decoration and includes mismatches in aggregate health. Run `knowledge_update` after changing embedding providers, models, tasks, or instructions.
|
|
181
306
|
|
|
182
307
|
## Configuration
|
|
183
308
|
|
|
@@ -186,8 +311,8 @@ Full configuration details are in [docs/configuration.md](docs/configuration.md)
|
|
|
186
311
|
| Area | Environment variables |
|
|
187
312
|
|------|-----------------------|
|
|
188
313
|
| Storage path | `PI_KNOWLEDGE_DIR`, `OMP_KNOWLEDGE_DIR`, `PI_CODING_AGENT_DIR`, `OMP_CODING_AGENT_DIR`, `OMP_PROFILE` |
|
|
189
|
-
|
|
|
190
|
-
| Embedding provider | `PI_KNOWLEDGE_EMBEDDING`, `PI_KNOWLEDGE_EMBEDDING_API_KEY`, `OPENAI_API_KEY`, `PI_KNOWLEDGE_EMBEDDING_BASE_URL`, `OPENAI_BASE_URL`, `PI_KNOWLEDGE_EMBEDDING_PREFIXES`, `PI_KNOWLEDGE_EMBEDDING_MAX_CHARS`, `PI_KNOWLEDGE_EMBEDDING_BATCH_SIZE`, `PI_KNOWLEDGE_EMBEDDING_API_FALLBACK` |
|
|
314
|
+
| Runtime worker and model cache | `PI_KNOWLEDGE_MODEL_CACHE_DIR`, `PI_KNOWLEDGE_NODE_PATH` |
|
|
315
|
+
| Embedding provider | `PI_KNOWLEDGE_EMBEDDING`, `PI_KNOWLEDGE_EMBEDDING_API_KEY`, `OPENAI_API_KEY`, `PI_KNOWLEDGE_EMBEDDING_BASE_URL`, `OPENAI_BASE_URL`, `PI_KNOWLEDGE_EMBEDDING_PREFIXES`, `PI_KNOWLEDGE_EMBEDDING_TASK`, `PI_KNOWLEDGE_EMBEDDING_QUERY_INSTRUCTION`, `PI_KNOWLEDGE_EMBEDDING_DOCUMENT_INSTRUCTION`, `PI_KNOWLEDGE_EMBEDDING_MAX_CHARS`, `PI_KNOWLEDGE_EMBEDDING_BATCH_SIZE`, `PI_KNOWLEDGE_EMBEDDING_API_FALLBACK` |
|
|
191
316
|
| Reranker provider | `PI_KNOWLEDGE_RERANKER`, `PI_KNOWLEDGE_RERANKER_REVISION`, `PI_KNOWLEDGE_RERANKER_DTYPE`, `PI_KNOWLEDGE_RERANKER_REMOTE_HOST`, `PI_KNOWLEDGE_RERANKER_REMOTE_PATH_TEMPLATE`, `PI_KNOWLEDGE_RERANKER_RAW_LOGITS`, `PI_KNOWLEDGE_RERANKER_API_ENDPOINT`, `PI_KNOWLEDGE_RERANKER_API_BASE_URL`, `PI_KNOWLEDGE_RERANKER_API_KEY`, `PI_KNOWLEDGE_RERANKER_API_FORMAT`, `PI_KNOWLEDGE_RERANKER_API_TIMEOUT_MS`, `PI_KNOWLEDGE_RERANKER_MAX_DOC_CHARS`, `PI_KNOWLEDGE_RERANKER_API_RESULTS_PATH`, `PI_KNOWLEDGE_RERANKER_API_INDEX_FIELD`, `PI_KNOWLEDGE_RERANKER_API_SCORE_FIELD`, `PI_KNOWLEDGE_RERANKER_API_SCORE_DIRECTION` |
|
|
192
317
|
| Native lifecycle | `PI_KNOWLEDGE_ENABLE_NATIVE_IDLE_DISPOSE`, `PI_KNOWLEDGE_EMBEDDING_IDLE_MS` |
|
|
193
318
|
| Runtime features | `PI_KNOWLEDGE_WATCH`, `PI_KNOWLEDGE_AUTO_INJECT`, `PI_KNOWLEDGE_STALE_INDEXING_MS`, `PI_KNOWLEDGE_OFFLINE`, `PI_KNOWLEDGE_SEARCH_PROFILE`, `PI_KNOWLEDGE_SEARCH_DEFAULT_LIMIT`, `PI_KNOWLEDGE_SNIPPET_MAX_LENGTH`, `PI_KNOWLEDGE_MIN_HYBRID_SCORE`, `PI_KNOWLEDGE_SEARCH_CANDIDATE_MIN`, `PI_KNOWLEDGE_SEARCH_CANDIDATE_MULTIPLIER`, `PI_KNOWLEDGE_ADAPTIVE_CONTEXT_LINES`, `PI_KNOWLEDGE_ADAPTIVE_MAX_CHARS`, `PI_KNOWLEDGE_ADAPTIVE_NEIGHBOR_TARGET`, `PI_KNOWLEDGE_DEEP_RERANK_CANDIDATES`, `PI_KNOWLEDGE_DEEP_RERANK_TOPK_MULTIPLIER` |
|
|
@@ -201,7 +326,7 @@ For Hugging Face Text Embeddings Inference rerankers, use `PI_KNOWLEDGE_RERANKER
|
|
|
201
326
|
|
|
202
327
|
Default storage is `~/.pi/knowledge` for Pi and `~/.omp/knowledge` for OMP. Explicit overrides are available with `PI_KNOWLEDGE_DIR` and `OMP_KNOWLEDGE_DIR`. Under the default home OMP root, existing legacy `~/.pi/knowledge` data remains visible when `~/.omp/knowledge` has not been created yet.
|
|
203
328
|
|
|
204
|
-
OMP compatibility covers path resolution, packaged entry loading, native SQLite dependency resolution, isolated
|
|
329
|
+
OMP compatibility covers path resolution, packaged entry loading, native SQLite dependency resolution, isolated worker startup, Windows-safe worker transport fallback, document parser stdio isolation, and idempotent shutdown. Local embeddings and PDF/DOCX extraction require Node 22+ for the isolated worker; set `PI_KNOWLEDGE_NODE_PATH` to `node.exe` if OMP runs from a non-Node packaged host. Compatibility-sensitive releases should validate both Pi and OMP install/runtime flows.
|
|
205
330
|
|
|
206
331
|
## Large Project Indexing
|
|
207
332
|
|
|
@@ -215,17 +340,25 @@ Update and diagnostics paths are also streaming-oriented: changed chunks are emb
|
|
|
215
340
|
|
|
216
341
|
## Architecture
|
|
217
342
|
|
|
218
|
-
[DESIGN.md](
|
|
343
|
+
[DESIGN.md](https://github.com/nczz/pi-knowledge/blob/main/DESIGN.md) is the
|
|
344
|
+
historical technical design. For current behavior contracts, prefer this
|
|
345
|
+
README, the repository's
|
|
346
|
+
[AGENTS.md](https://github.com/nczz/pi-knowledge/blob/main/AGENTS.md),
|
|
347
|
+
[docs/configuration.md](docs/configuration.md), and the source-only
|
|
348
|
+
[ADR and pitfall notes](https://github.com/nczz/pi-knowledge/tree/main/docs).
|
|
349
|
+
The npm package includes the user-facing configuration and offline-mode guides;
|
|
350
|
+
the repository retains the broader design history.
|
|
219
351
|
|
|
220
352
|
## Data Storage
|
|
221
353
|
|
|
222
|
-
All data is stored
|
|
354
|
+
All data is stored under the active host knowledge directory unless overridden;
|
|
355
|
+
indexed projects are never used for internal storage:
|
|
223
356
|
|
|
224
357
|
```
|
|
225
|
-
|
|
358
|
+
<knowledge-dir>/
|
|
226
359
|
├── knowledge.db ← SQLite (metadata + chunks + FTS5 index)
|
|
227
360
|
├── vectors/ ← Embedding vectors per KB (binary)
|
|
228
|
-
└── models/ ←
|
|
361
|
+
└── models/ ← ONNX cache (embedding ~32MB; rerankers add artifacts)
|
|
229
362
|
```
|
|
230
363
|
|
|
231
364
|
- **Backup**: copy the active knowledge directory, usually `~/.pi/knowledge/` or `~/.omp/knowledge/`
|
package/dist/index.js
CHANGED
|
@@ -6,6 +6,8 @@ var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExte
|
|
|
6
6
|
}
|
|
7
7
|
return path;
|
|
8
8
|
};
|
|
9
|
+
import { MAX_SEARCH_RESULTS, MAX_SEARCH_WINDOW } from "./src/search/contracts.js";
|
|
10
|
+
import { formatKnowledgeStatus } from "./src/status.js";
|
|
9
11
|
const Type = {
|
|
10
12
|
Object(properties) {
|
|
11
13
|
const required = Object.entries(properties)
|
|
@@ -252,7 +254,7 @@ export default function (pi) {
|
|
|
252
254
|
pi.registerTool({
|
|
253
255
|
name: "knowledge_configure",
|
|
254
256
|
label: "Knowledge Configure",
|
|
255
|
-
description: "Configure pi-knowledge runtime settings such as the
|
|
257
|
+
description: "Configure pi-knowledge runtime settings such as the isolated worker Node executable",
|
|
256
258
|
approval: "write",
|
|
257
259
|
promptSnippet: "Configure local pi-knowledge runtime prerequisites before indexing",
|
|
258
260
|
promptGuidelines: [
|
|
@@ -263,7 +265,7 @@ export default function (pi) {
|
|
|
263
265
|
],
|
|
264
266
|
parameters: Type.Object({
|
|
265
267
|
node_path: Type.Optional(Type.String({
|
|
266
|
-
description: "Optional full path to a Node 22+ executable for the isolated local
|
|
268
|
+
description: "Optional full path to a Node 22+ executable for the isolated local model and document extraction worker, such as C:\\Program Files\\nodejs\\node.exe",
|
|
267
269
|
})),
|
|
268
270
|
}),
|
|
269
271
|
async execute(_id, params, _signal) {
|
|
@@ -278,7 +280,7 @@ export default function (pi) {
|
|
|
278
280
|
content: [
|
|
279
281
|
{
|
|
280
282
|
type: "text",
|
|
281
|
-
text: `Configured
|
|
283
|
+
text: `Configured isolated worker Node executable: ${configuredPath}`,
|
|
282
284
|
},
|
|
283
285
|
],
|
|
284
286
|
details: { node_path: configuredPath },
|
|
@@ -397,9 +399,19 @@ export default function (pi) {
|
|
|
397
399
|
Type.Literal("code"),
|
|
398
400
|
Type.Literal("docs"),
|
|
399
401
|
])),
|
|
400
|
-
limit: Type.Optional(Type.Number({
|
|
402
|
+
limit: Type.Optional(Type.Number({
|
|
403
|
+
description: "Max results (default comes from search profile/env)",
|
|
404
|
+
minimum: 1,
|
|
405
|
+
maximum: MAX_SEARCH_RESULTS,
|
|
406
|
+
multipleOf: 1,
|
|
407
|
+
})),
|
|
401
408
|
kb_id: Type.Optional(Type.String({ description: "Limit search to a specific KB by ID or exact name" })),
|
|
402
|
-
offset: Type.Optional(Type.Number({
|
|
409
|
+
offset: Type.Optional(Type.Number({
|
|
410
|
+
description: `Pagination offset; offset + limit must not exceed ${MAX_SEARCH_WINDOW}`,
|
|
411
|
+
minimum: 0,
|
|
412
|
+
maximum: MAX_SEARCH_WINDOW - 1,
|
|
413
|
+
multipleOf: 1,
|
|
414
|
+
})),
|
|
403
415
|
file_type: Type.Optional(Type.String({ description: "Filter by file type (e.g. typescript, markdown, python)" })),
|
|
404
416
|
path_pattern: Type.Optional(Type.String({ description: "Filter by file path substring (for example src/engine.ts)" })),
|
|
405
417
|
diversity: Type.Optional(Type.Union([Type.Literal("off"), Type.Literal("balanced"), Type.Literal("strong")])),
|
|
@@ -417,7 +429,7 @@ export default function (pi) {
|
|
|
417
429
|
response.retry_modes?.length ? `Retried: ${response.retry_modes.join(", ")}` : "",
|
|
418
430
|
response.warnings?.length ? `Warnings: ${response.warnings.join(" | ")}` : "",
|
|
419
431
|
diagnostics && response.tuning
|
|
420
|
-
? `Tuning: profile=${response.tuning.requested_profile}->${response.tuning.selected_profile}, limit=${response.tuning.limit}, snippet=${response.tuning.snippet_max_length}, min_hybrid=${response.tuning.min_hybrid_score}, candidates=${response.tuning.candidate_limit}`
|
|
432
|
+
? `Tuning: profile=${response.tuning.requested_profile}->${response.tuning.selected_profile}, limit=${response.tuning.limit}, snippet=${response.tuning.snippet_max_length}, min_hybrid=${response.tuning.min_hybrid_score}, candidates=${response.tuning.diversity_candidates}/${response.tuning.candidate_limit}, retrieval_per_kb=${response.tuning.retrieval_limit_per_kb}, retrieved=${response.tuning.retrieved_candidates}, filtered=${response.tuning.filtered_candidates}, truncated=${response.tuning.candidate_pool_truncated}`
|
|
421
433
|
: "",
|
|
422
434
|
response.suggestions?.length ? `Suggestions: ${response.suggestions.join(" | ")}` : "",
|
|
423
435
|
]
|
|
@@ -446,7 +458,7 @@ export default function (pi) {
|
|
|
446
458
|
? `\nProvenance: chunk=${r.provenance.chunk_id}, hash=${r.provenance.chunk_hash.slice(0, 12)}, reason=${r.provenance.match_reason}, indexed_at=${r.provenance.indexed_at}, source_mtime=${r.provenance.source_mtime ?? "unknown"}, stale=${r.provenance.stale}`
|
|
447
459
|
: "";
|
|
448
460
|
const tuning = diagnostics && response.tuning && i === 0
|
|
449
|
-
? `\nTuning: limit=${response.tuning.limit}, snippet=${response.tuning.snippet_max_length}, min_hybrid=${response.tuning.min_hybrid_score}, candidates=${response.tuning.candidate_limit}`
|
|
461
|
+
? `\nTuning: limit=${response.tuning.limit}, snippet=${response.tuning.snippet_max_length}, min_hybrid=${response.tuning.min_hybrid_score}, candidates=${response.tuning.diversity_candidates}/${response.tuning.candidate_limit}, retrieval_per_kb=${response.tuning.retrieval_limit_per_kb}, retrieved=${response.tuning.retrieved_candidates}, filtered=${response.tuning.filtered_candidates}, truncated=${response.tuning.candidate_pool_truncated}`
|
|
450
462
|
: "";
|
|
451
463
|
return `[${i + 1}] ${r.file_path} (${r.kb_name}, score: ${r.score.toFixed(3)})\n${r.snippet}${diag}${provenance}${tuning}`;
|
|
452
464
|
})
|
|
@@ -546,69 +558,26 @@ export default function (pi) {
|
|
|
546
558
|
pi.registerTool({
|
|
547
559
|
name: "knowledge_status",
|
|
548
560
|
label: "Knowledge Status",
|
|
549
|
-
description: "Show knowledge engine status with
|
|
561
|
+
description: "Show knowledge engine status with embedding compatibility, staleness, orphan, and coverage diagnostics",
|
|
550
562
|
approval: "read",
|
|
551
563
|
parameters: Type.Object({}),
|
|
552
564
|
async execute(_id, _params, _signal) {
|
|
553
565
|
if (_signal?.aborted)
|
|
554
566
|
throw new Error("Cancelled");
|
|
555
567
|
const { engine, storage, watcher } = await ensureInitialized();
|
|
556
|
-
const
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
if (kb.source_path)
|
|
570
|
-
lines.push(` source: ${kb.source_path}`);
|
|
571
|
-
if (diag) {
|
|
572
|
-
if (diag.job) {
|
|
573
|
-
const jobAge = Math.round((Date.now() - diag.job.last_progress_at) / 1000);
|
|
574
|
-
const progress = [
|
|
575
|
-
`${diag.job.processed_files} files`,
|
|
576
|
-
`${diag.job.processed_chunks} chunks`,
|
|
577
|
-
diag.job.skipped_total > 0 ? `${diag.job.skipped_total} skipped` : "",
|
|
578
|
-
diag.job.added_chunks > 0 ? `+${diag.job.added_chunks}` : "",
|
|
579
|
-
diag.job.removed_chunks > 0 ? `-${diag.job.removed_chunks}` : "",
|
|
580
|
-
diag.job.unchanged_chunks > 0 ? `=${diag.job.unchanged_chunks}` : "",
|
|
581
|
-
]
|
|
582
|
-
.filter(Boolean)
|
|
583
|
-
.join(", ");
|
|
584
|
-
lines.push(` job: ${diag.job.status}/${diag.job.phase} — ${progress || "no processed items yet"} — last progress ${jobAge}s ago`);
|
|
585
|
-
if (diag.job.message)
|
|
586
|
-
lines.push(` last: ${diag.job.message}`);
|
|
587
|
-
if (diag.job.error_message)
|
|
588
|
-
lines.push(` error: ${diag.job.error_message}`);
|
|
589
|
-
}
|
|
590
|
-
lines.push(` coverage: ${diag.coverage_percent}% (${diag.indexed_files}/${diag.total_source_files} files)`);
|
|
591
|
-
if (diag.stale_files.length > 0)
|
|
592
|
-
lines.push(` ⚠️ stale: ${diag.stale_files.length} files modified since last index`);
|
|
593
|
-
if (diag.orphan_files.length > 0)
|
|
594
|
-
lines.push(` ⚠️ orphans: ${diag.orphan_files.length} chunks reference deleted files`);
|
|
595
|
-
if (diag.stuck_indexing)
|
|
596
|
-
lines.push(` ⚠️ indexing appears stuck for ${Math.round(diag.last_progress_age_ms / 60000)}m; remove and rebuild if no pi process is actively indexing it`);
|
|
597
|
-
if (diag.skipped_files.total > 0)
|
|
598
|
-
lines.push(` skipped: ${diag.skipped_files.total} files (${Object.entries(diag.skipped_files.by_reason)
|
|
599
|
-
.filter(([, count]) => count > 0)
|
|
600
|
-
.map(([reason, count]) => `${reason}: ${count}`)
|
|
601
|
-
.join(", ")})`);
|
|
602
|
-
}
|
|
603
|
-
}
|
|
604
|
-
const totalStale = diagnostics.reduce((n, d) => n + d.stale_files.length, 0);
|
|
605
|
-
const totalOrphans = diagnostics.reduce((n, d) => n + d.orphan_files.length, 0);
|
|
606
|
-
const totalStuck = diagnostics.filter((d) => d.stuck_indexing).length;
|
|
607
|
-
if (totalStale === 0 && totalOrphans === 0 && totalStuck === 0 && kbs.length > 0)
|
|
608
|
-
lines.push("", "Health: ✓ all indexes up to date");
|
|
609
|
-
else if (totalStale > 0 || totalOrphans > 0 || totalStuck > 0)
|
|
610
|
-
lines.push("", `Health: ⚠️ ${totalStale} stale, ${totalOrphans} orphans, ${totalStuck} stuck indexing — run knowledge_update or rebuild affected KBs`);
|
|
611
|
-
return { content: [{ type: "text", text: lines.join("\n") }] };
|
|
568
|
+
const snapshot = engine.statusSnapshot(_signal);
|
|
569
|
+
return {
|
|
570
|
+
content: [
|
|
571
|
+
{
|
|
572
|
+
type: "text",
|
|
573
|
+
text: formatKnowledgeStatus({
|
|
574
|
+
snapshot,
|
|
575
|
+
storagePath: storage.getDefaultKnowledgeDir(),
|
|
576
|
+
watchCount: watcher.getActiveWatcherCount(),
|
|
577
|
+
}),
|
|
578
|
+
},
|
|
579
|
+
],
|
|
580
|
+
};
|
|
612
581
|
},
|
|
613
582
|
});
|
|
614
583
|
pi.registerTool({
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
export type EmbeddingRole = "query" | "passage";
|
|
2
|
+
export type EmbeddingDecorationStrategy = "e5" | "none" | "jina-code" | "custom";
|
|
3
|
+
export type EmbeddingTask = "nl2code" | "code2code" | "code2nl" | "code2completion" | "qa";
|
|
4
|
+
export type EmbeddingPrefixMode = "on" | "off";
|
|
5
|
+
export interface EmbeddingInstructionSet {
|
|
6
|
+
query: string;
|
|
7
|
+
document: string;
|
|
8
|
+
}
|
|
9
|
+
export interface ResolvedEmbeddingDecoration {
|
|
10
|
+
strategy: EmbeddingDecorationStrategy;
|
|
11
|
+
task?: EmbeddingTask;
|
|
12
|
+
instructions: EmbeddingInstructionSet;
|
|
13
|
+
prefixMode: EmbeddingPrefixMode;
|
|
14
|
+
}
|
|
15
|
+
export declare const MAX_INSTRUCTION_CHARS = 512;
|
|
16
|
+
export declare const DEFAULT_EMBEDDING_TASK: EmbeddingTask;
|
|
17
|
+
export declare const ACCEPTED_DECORATION_VALUES: readonly ["on", "off", "e5", "none", "jina", "jina-code", "custom"];
|
|
18
|
+
export declare const ACCEPTED_EMBEDDING_TASKS: readonly ["nl2code", "code2code", "code2nl", "code2completion", "qa"];
|
|
19
|
+
export declare const E5_INSTRUCTIONS: EmbeddingInstructionSet;
|
|
20
|
+
export declare const NO_INSTRUCTIONS: EmbeddingInstructionSet;
|
|
21
|
+
/**
|
|
22
|
+
* Published prompts for jinaai/jina-code-embeddings-1.5b.
|
|
23
|
+
* Source: config_sentence_transformers.json at model revision
|
|
24
|
+
* 39aeb4fb9b60f930934c78ae5d749a46287c248a (verified 2026-11-12).
|
|
25
|
+
* Values already contain their required trailing newline and are concatenated verbatim.
|
|
26
|
+
*/
|
|
27
|
+
export declare const JINA_CODE_INSTRUCTIONS: Readonly<Record<EmbeddingTask, EmbeddingInstructionSet>>;
|
|
28
|
+
export declare function decorateEmbeddingInput(text: string, role: EmbeddingRole, instructions: EmbeddingInstructionSet): string;
|
|
29
|
+
export declare function embeddingInstructionLength(instructions: EmbeddingInstructionSet, role: EmbeddingRole): number;
|
|
30
|
+
export declare function embeddingInstructionSetHash(instructions: EmbeddingInstructionSet): string;
|
|
31
|
+
export declare function embeddingDecorationDescriptor(decoration: ResolvedEmbeddingDecoration): string;
|
|
32
|
+
export declare function resolveEmbeddingDecoration(env: NodeJS.ProcessEnv): ResolvedEmbeddingDecoration;
|
|
33
|
+
export declare function decorationSupportedByLocalModel(strategy: EmbeddingDecorationStrategy): boolean;
|