pi-knowledge 0.11.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/README.md +147 -33
- package/dist/index.js +15 -4
- package/dist/src/embedding/provider.d.ts +14 -0
- package/dist/src/embedding/provider.js +90 -8
- package/dist/src/engine.d.ts +3 -1
- package/dist/src/engine.js +122 -147
- package/dist/src/indexer/chunker.d.ts +1 -1
- package/dist/src/indexer/chunker.js +2 -2
- package/dist/src/indexer/embedding-safe-chunks.d.ts +5 -0
- package/dist/src/indexer/embedding-safe-chunks.js +215 -0
- package/dist/src/indexer/html.d.ts +1 -0
- package/dist/src/indexer/html.js +49 -0
- package/dist/src/model-worker-client.d.ts +4 -0
- package/dist/src/model-worker-client.js +17 -1
- package/dist/src/model-worker.js +63 -15
- package/dist/src/search/contracts.d.ts +2 -0
- package/dist/src/search/contracts.js +2 -0
- package/dist/src/search/diversity.d.ts +16 -0
- package/dist/src/search/diversity.js +183 -0
- package/dist/src/search/tuning.d.ts +10 -0
- package/dist/src/search/tuning.js +20 -0
- package/dist/src/storage/sqlite.d.ts +1 -0
- package/dist/src/storage/sqlite.js +3 -0
- package/docs/configuration.md +3 -3
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,32 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [Unreleased]
|
|
4
|
+
|
|
5
|
+
## [0.13.0] - 2026-11-07
|
|
6
|
+
|
|
7
|
+
### Added
|
|
8
|
+
- Added bounded `knowledge_search` pagination (`limit` 1-50, `offset + limit` up to 800) and diagnostics for per-KB retrieval breadth, merged/filtered/diversity candidate counts, and global-pool truncation.
|
|
9
|
+
- Added client-side validation and L2 normalization for OpenAI-compatible embedding vectors, with a signed normalization policy and `knowledge_doctor` rebuild guidance for older API-backed KBs.
|
|
10
|
+
|
|
11
|
+
### Changed
|
|
12
|
+
- Reworked the quick start and added verified Pi and Oh My Pi upgrade guidance for npm and local sources, targeted updates, explicit-version pins, safe checkout updates, build activation, source verification, host restarts, and post-upgrade diagnostics.
|
|
13
|
+
- Clarified host-specific storage, opt-in watcher/context injection, model-cache sizing, source-only architecture links, and the distinction between structured result text and custom TUI renderers.
|
|
14
|
+
|
|
15
|
+
### Fixed
|
|
16
|
+
- Reduced MMR-style diversity ranking from O(n³) repeated redundancy scans to behavior-preserving O(n²) running maxima, with cooperative event-loop yielding and cancellation checks (#21).
|
|
17
|
+
- Bounded merged multi-KB candidates before deep reranking, adaptive expansion, and diversity so query-time work cannot grow unchecked with the number of selected KBs.
|
|
18
|
+
|
|
19
|
+
## [0.12.0] - 2026-10-19
|
|
20
|
+
|
|
21
|
+
### Added
|
|
22
|
+
- Added embedding-safe chunk finalization for add, update, URL, inline-text, and JSONL import paths. Local inputs use the existing model worker tokenizer; OpenAI-compatible inputs use the configured complete-input character cap.
|
|
23
|
+
- Added `knowledge_doctor` action `reindex_for_embedding_budget` for KBs created before the current chunk input policy.
|
|
24
|
+
- Added dependency-free structured HTML normalization for URL sources, preserving headings, paragraphs, lists, tables, and entities for heading-aware chunking.
|
|
25
|
+
|
|
26
|
+
### Fixed
|
|
27
|
+
- Prevented local and OpenAI-compatible embedding inputs from silently discarding oversized multilingual or context-enriched chunk tails.
|
|
28
|
+
- Kept the model worker startup single-flight and Node strip-only compatible while reusing it for batched tokenizer measurement.
|
|
29
|
+
|
|
3
30
|
## [0.11.0] - 2026-09-23
|
|
4
31
|
|
|
5
32
|
### Added
|
package/README.md
CHANGED
|
@@ -21,9 +21,10 @@ Unlike `pi-memory` (which manages the agent's own notes), `pi-knowledge` indexes
|
|
|
21
21
|
|
|
22
22
|
## Highlights
|
|
23
23
|
|
|
24
|
-
- **Local-first project memory**: stores indexes under `~/.pi/knowledge
|
|
24
|
+
- **Local-first project memory**: stores indexes under the active host knowledge directory (`~/.pi/knowledge/` for Pi or `~/.omp/knowledge/` for OMP); no project files are modified.
|
|
25
25
|
- **Hybrid retrieval**: lexical-anchored BM25 + semantic vectors + normalized weighted score fusion, with semantic mode for vector-only conceptual recall.
|
|
26
|
-
- **Code-aware indexing**: recursive AST chunking for
|
|
26
|
+
- **Code- and document-aware indexing**: recursive AST chunking for supported code, heading-aware Markdown chunks, and an embedding-safe finalizer that preserves oversized multilingual tails without adding a heavy document runtime.
|
|
27
|
+
- **Structured URL indexing**: preserves HTML headings, paragraphs, lists, tables, and entities as lightweight Markdown-like text without a browser, Docling runtime, or new dependency.
|
|
27
28
|
- **Lightweight symbol lookup**: indexes functions, methods, classes, interfaces, types, variables, route-like handlers, Markdown headings, config keys, and env vars for exact agent lookup.
|
|
28
29
|
- **Better agent answers**: adaptive context windows, diversity reranking, optional cross-encoder reranking, and diagnostics.
|
|
29
30
|
- **Large-project stability**: persisted indexing progress, capped batches, streaming vector scans, and stuck-job detection.
|
|
@@ -42,7 +43,7 @@ Unlike `pi-memory` (which manages the agent's own notes), `pi-knowledge` indexes
|
|
|
42
43
|
| **Adaptive contextual search** | ✅ | ❌ | ❌ |
|
|
43
44
|
| **Diversity reranking** | ✅ | ❌ | ❌ |
|
|
44
45
|
| **Incremental re-indexing** | ✅ | ❌ | ❌ |
|
|
45
|
-
| **File watcher (auto-update)** | ✅ | ❌ | ❌ |
|
|
46
|
+
| **File watcher (opt-in auto-update)** | ✅ | ❌ | ❌ |
|
|
46
47
|
| **Code-aware chunking** | ✅ (TS/JS/Py/Go/Rust/Java/Bash/C/C++/QML) | ❌ | ❌ |
|
|
47
48
|
| **Symbol/config/heading lookup** | ✅ | ❌ | ❌ |
|
|
48
49
|
| **Local embeddings (zero API)** | ✅ | ❌ | ✅ (qmd) |
|
|
@@ -51,9 +52,9 @@ Unlike `pi-memory` (which manages the agent's own notes), `pi-knowledge` indexes
|
|
|
51
52
|
| **Progress reporting + stuck indexing diagnostics** | ✅ | partial | ❌ |
|
|
52
53
|
| Cross-session persistence | ✅ | ✅ | ✅ |
|
|
53
54
|
| Pi extension native | ✅ | N/A | ✅ |
|
|
54
|
-
| Context injection per turn | ✅ | ❌ | ✅ |
|
|
55
|
-
|
|
|
56
|
-
|
|
|
55
|
+
| Context injection per turn (opt-in) | ✅ | ❌ | ✅ |
|
|
56
|
+
| Structured search result formatting | ✅ | N/A | ❌ |
|
|
57
|
+
| Custom tool TUI renderers | ❌ | N/A | ❌ |
|
|
57
58
|
| RPC mode support | ✅ | N/A | N/A |
|
|
58
59
|
|
|
59
60
|
TUI rendering currently uses Pi's stable default tool rendering plus targeted result formatting. A fuller custom renderer is intentionally not enabled yet because custom renderers must be width-safe across terminal sizes and Pi TUI modes.
|
|
@@ -72,36 +73,137 @@ TUI rendering currently uses Pi's stable default tool rendering plus targeted re
|
|
|
72
73
|
|
|
73
74
|
This is intentionally not a heavy ColBERT-style late-interaction index yet (Khattab and Zaharia 2020). The current product chooses lightweight local embeddings, BM25, query-aware ranking, optional cross-encoder reranking, streamed vector scans, and health diagnostics for commercial usefulness with low setup cost.
|
|
74
75
|
|
|
75
|
-
|
|
76
|
+
Project dogfood drove fixes for score compression, repeated same-file results,
|
|
77
|
+
garbage-query false positives, small-module discoverability, source-versus-test
|
|
78
|
+
ranking, indexing stability, and auto-mode false positives. Existing KBs should
|
|
79
|
+
be rebuilt or updated after upgrades that change indexing text.
|
|
76
80
|
|
|
77
81
|
## Quick Start
|
|
78
82
|
|
|
83
|
+
### Install
|
|
84
|
+
|
|
85
|
+
For Pi:
|
|
86
|
+
|
|
79
87
|
```bash
|
|
80
|
-
# Install for Pi
|
|
81
88
|
pi install npm:pi-knowledge
|
|
89
|
+
```
|
|
82
90
|
|
|
83
|
-
|
|
84
|
-
omp install npm:pi-knowledge
|
|
91
|
+
For OMP:
|
|
85
92
|
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
omp install npm:pi-knowledge
|
|
93
|
+
```bash
|
|
94
|
+
omp plugin install npm:pi-knowledge
|
|
95
|
+
```
|
|
90
96
|
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
omp install ./pi-knowledge
|
|
97
|
+
On minimal Ubuntu or Debian systems, install native build tools first if OMP
|
|
98
|
+
must compile `better-sqlite3`:
|
|
94
99
|
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
100
|
+
```bash
|
|
101
|
+
sudo apt-get update
|
|
102
|
+
sudo apt-get install -y build-essential python3
|
|
103
|
+
omp plugin install npm:pi-knowledge
|
|
104
|
+
```
|
|
98
105
|
|
|
99
|
-
|
|
100
|
-
> Search my knowledge base for "authentication flow"
|
|
106
|
+
For a local checkout, install dependencies and build before registration:
|
|
101
107
|
|
|
102
|
-
|
|
108
|
+
```bash
|
|
109
|
+
cd /path/to/pi-knowledge
|
|
110
|
+
npm ci
|
|
111
|
+
npm run typecheck
|
|
112
|
+
npm test
|
|
113
|
+
npm run build
|
|
114
|
+
|
|
115
|
+
# Register with the host you use:
|
|
116
|
+
pi install ./
|
|
117
|
+
omp plugin link "$(pwd)"
|
|
103
118
|
```
|
|
104
119
|
|
|
120
|
+
Use either the npm package or the local checkout for a host, not both. Check
|
|
121
|
+
Pi with `pi list` and OMP with `omp plugin list --json` when switching sources.
|
|
122
|
+
|
|
123
|
+
### Index and search
|
|
124
|
+
|
|
125
|
+
Ask the agent:
|
|
126
|
+
|
|
127
|
+
```text
|
|
128
|
+
Index my project docs at ./docs as "Project Docs".
|
|
129
|
+
Search my knowledge bases for "authentication flow".
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
The agent receives guidance to use `knowledge_symbol_search` and
|
|
133
|
+
`knowledge_search` when knowledge bases are available. Automatic behavior is
|
|
134
|
+
opt-in. Set one or both variables before starting Pi or OMP:
|
|
135
|
+
|
|
136
|
+
```bash
|
|
137
|
+
export PI_KNOWLEDGE_AUTO_INJECT=true # retrieve context before every model call
|
|
138
|
+
export PI_KNOWLEDGE_WATCH=true # update directory KBs after file changes
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
Per-turn injection adds retrieval work and prompt context to every model call.
|
|
142
|
+
File watching applies only to directory-backed knowledge bases.
|
|
143
|
+
|
|
144
|
+
## Upgrade
|
|
145
|
+
|
|
146
|
+
### npm-installed package
|
|
147
|
+
|
|
148
|
+
Update only pi-knowledge in Pi:
|
|
149
|
+
|
|
150
|
+
```bash
|
|
151
|
+
pi update npm:pi-knowledge
|
|
152
|
+
pi list
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
Use `pi update --extensions` instead when every installed Pi extension should
|
|
156
|
+
be updated.
|
|
157
|
+
|
|
158
|
+
Update only pi-knowledge in OMP:
|
|
159
|
+
|
|
160
|
+
```bash
|
|
161
|
+
omp plugin upgrade pi-knowledge
|
|
162
|
+
omp plugin list --json
|
|
163
|
+
omp plugin doctor
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
Bare `omp plugin upgrade` and `omp update --plugins --check` report marketplace
|
|
167
|
+
plugin status; they do not confirm that this npm plugin was upgraded.
|
|
168
|
+
|
|
169
|
+
Install the unversioned Pi source (`npm:pi-knowledge`) to follow the npm
|
|
170
|
+
`latest` tag. Pi preserves explicitly versioned sources such as
|
|
171
|
+
`npm:pi-knowledge@<version>`. To remove a pin:
|
|
172
|
+
|
|
173
|
+
1. Run `pi list`.
|
|
174
|
+
2. Pass the exact pinned source shown there to `pi remove`.
|
|
175
|
+
3. Run `pi install npm:pi-knowledge`.
|
|
176
|
+
|
|
177
|
+
### Local checkout
|
|
178
|
+
|
|
179
|
+
A linked checkout is updated from Git, not from the npm registry. Finish,
|
|
180
|
+
commit, or stash local edits before pulling. Exit Pi and OMP processes that use
|
|
181
|
+
the checkout before replacing dependencies or build output.
|
|
182
|
+
|
|
183
|
+
```bash
|
|
184
|
+
cd /path/to/pi-knowledge
|
|
185
|
+
git status --short
|
|
186
|
+
git pull --ff-only # only when the checkout is clean and follows a remote branch
|
|
187
|
+
npm ci
|
|
188
|
+
npm run typecheck
|
|
189
|
+
npm test
|
|
190
|
+
npm run build
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
When actively developing local changes, skip `git pull --ff-only` and build the
|
|
194
|
+
current tree. `extension.js` prefers `dist/index.js`, so source edits are
|
|
195
|
+
inactive until the build succeeds. The local registration only needs to be
|
|
196
|
+
repeated when its path or host settings change.
|
|
197
|
+
|
|
198
|
+
Pi and OMP load extension modules once per process. Start a new host process
|
|
199
|
+
after a successful registry upgrade or local build. Then use `pi list` to
|
|
200
|
+
confirm Pi's selected source, or `omp plugin list --json` to confirm OMP's
|
|
201
|
+
installed version and path.
|
|
202
|
+
|
|
203
|
+
After releases that change indexing text, run `knowledge_doctor` and follow its
|
|
204
|
+
suggested reindex action. Source-backed knowledge bases can normally use
|
|
205
|
+
`knowledge_update`; rebuild only when diagnostics or release notes require it.
|
|
206
|
+
|
|
105
207
|
## Tools
|
|
106
208
|
|
|
107
209
|
| Tool | Description |
|
|
@@ -144,8 +246,10 @@ Mode selection contract:
|
|
|
144
246
|
|
|
145
247
|
Search results use balanced diversity reranking by default so near-duplicate chunks from the same file do not dominate the top results. Diversity scoring considers lexical overlap, same-file line proximity, overlapping adaptive windows, available embedding-vector similarity, file-level interleaving, and a small KB trust multiplier that favors ready file/directory sources over stale, URL, or imported text sources. In `adaptive` mode, code chunks with AST metadata can prefer same-parent/sibling chunks inside the same bounded file window; this is metadata-driven context expansion, not caller/callee analysis. Use `diversity: "off"` only when raw ranking order is needed for diagnostics. Agents can request search diagnostics to inspect mode fallback, selected profile, applied search tuning, ranking coverage, path/source/test boosts, adjusted scores, and provenance such as chunk id, chunk hash, match reason, stale flag, source freshness, and adaptive `source_chunk_ids` where available.
|
|
146
248
|
|
|
147
|
-
|
|
148
|
-
|
|
249
|
+
`limit` must be an integer from 1 to 50. `offset` must be a non-negative integer, and `offset + limit` cannot exceed the bounded 800-candidate search window. Each selected KB may contribute candidates up to the profile-derived retrieval limit, but merged candidates are globally ranked and capped before deep reranking, adaptive expansion, or diversity work. Diagnostics expose per-KB retrieval breadth, retrieved and filtered counts, the diversity pool size, and whether the global pool was truncated.
|
|
250
|
+
|
|
251
|
+
For best search quality, rebuild or update existing knowledge bases after upgrading. New indexes use contextual retrieval units: embeddings and FTS include file path, file type, Markdown heading breadcrumbs, and AST-derived code structure such as symbol names, scopes, parent symbols, and signatures while returned results keep the original chunk text readable. Before embedding, the complete contextual input is checked against the active provider budget; only oversized chunks are split again along structural boundaries.
|
|
252
|
+
Symbol/config/heading metadata and embedding-safe chunk boundaries are rebuilt during `knowledge_update`. `knowledge_doctor` reports `reindex_for_embedding_budget` for older KBs: file, directory, and URL sources can use `knowledge_update`; imported or inline-text sources should be re-imported or re-added from the original source.
|
|
149
253
|
|
|
150
254
|
## Embedding Configuration
|
|
151
255
|
|
|
@@ -170,9 +274,11 @@ export PI_KNOWLEDGE_EMBEDDING_BASE_URL=http://127.0.0.1:8080/v1
|
|
|
170
274
|
# Set PI_KNOWLEDGE_EMBEDDING_API_KEY or OPENAI_API_KEY only if your endpoint requires bearer auth.
|
|
171
275
|
```
|
|
172
276
|
|
|
173
|
-
API embedding failures are surfaced by default so configuration and context-window problems are visible. To explicitly allow query-time local fallback after API failures, set `PI_KNOWLEDGE_EMBEDDING_API_FALLBACK=local`; indexing/import/update document batches still fail instead of creating mixed-provider vector files.
|
|
277
|
+
API embedding failures are surfaced by default so configuration and context-window problems are visible. To explicitly allow query-time local fallback after API failures, set `PI_KNOWLEDGE_EMBEDDING_API_FALLBACK=local`; indexing/import/update document batches still fail instead of creating mixed-provider vector files. Inputs are never silently truncated.
|
|
278
|
+
|
|
279
|
+
OpenAI-compatible API vectors are validated and L2-normalized by the client before query use or persistence, so cosine-style dot products do not depend on a provider-specific vector magnitude. This normalization policy is part of the embedding signature. KBs created with an older API signature must run `knowledge_update`; `knowledge_doctor` reports `rebuild_vectors` when the active provider no longer matches stored vector semantics.
|
|
174
280
|
|
|
175
|
-
|
|
281
|
+
OpenAI-compatible embedding inputs use a 20000-character complete-input safety cap by default. Indexing splits oversized chunks before HTTP requests; direct oversized query/document calls fail explicitly. Adjust the cap with `PI_KNOWLEDGE_EMBEDDING_MAX_CHARS` when the remote model has a different context window. Changing the cap changes embedding compatibility and requires `knowledge_update`.
|
|
176
282
|
|
|
177
283
|
For self-hosted servers with smaller request limits, set `PI_KNOWLEDGE_EMBEDDING_BATCH_SIZE` to the maximum number of chunk inputs the server accepts per embedding request.
|
|
178
284
|
|
|
@@ -214,25 +320,33 @@ Update and diagnostics paths are also streaming-oriented: changed chunks are emb
|
|
|
214
320
|
|
|
215
321
|
## Architecture
|
|
216
322
|
|
|
217
|
-
[DESIGN.md](
|
|
323
|
+
[DESIGN.md](https://github.com/nczz/pi-knowledge/blob/main/DESIGN.md) is the
|
|
324
|
+
historical technical design. For current behavior contracts, prefer this
|
|
325
|
+
README, the repository's
|
|
326
|
+
[AGENTS.md](https://github.com/nczz/pi-knowledge/blob/main/AGENTS.md),
|
|
327
|
+
[docs/configuration.md](docs/configuration.md), and the source-only
|
|
328
|
+
[ADR and pitfall notes](https://github.com/nczz/pi-knowledge/tree/main/docs).
|
|
329
|
+
The npm package includes the user-facing configuration and offline-mode guides;
|
|
330
|
+
the repository retains the broader design history.
|
|
218
331
|
|
|
219
332
|
## Data Storage
|
|
220
333
|
|
|
221
|
-
All data is stored
|
|
334
|
+
All data is stored under the active host knowledge directory unless overridden;
|
|
335
|
+
indexed projects are never used for internal storage:
|
|
222
336
|
|
|
223
337
|
```
|
|
224
|
-
|
|
338
|
+
<knowledge-dir>/
|
|
225
339
|
├── knowledge.db ← SQLite (metadata + chunks + FTS5 index)
|
|
226
340
|
├── vectors/ ← Embedding vectors per KB (binary)
|
|
227
|
-
└── models/ ←
|
|
341
|
+
└── models/ ← ONNX cache (embedding ~32MB; rerankers add artifacts)
|
|
228
342
|
```
|
|
229
343
|
|
|
230
344
|
- **Backup**: copy the active knowledge directory, usually `~/.pi/knowledge/` or `~/.omp/knowledge/`
|
|
231
345
|
- **Reset**: delete the active knowledge directory to start fresh
|
|
232
346
|
- **Override**: set `PI_KNOWLEDGE_DIR` or `OMP_KNOWLEDGE_DIR`
|
|
233
347
|
- **Project safety**: pi-knowledge is read-only on indexed directories — no files are created or modified in your project
|
|
234
|
-
- **Updates**:
|
|
235
|
-
- **Symbol
|
|
348
|
+
- **Updates**: schema migrations run automatically, but index-time behavior changes are not applied to existing vectors at startup. `knowledge_doctor` reports actionable reindex requirements without loading the embedding model.
|
|
349
|
+
- **Symbol and chunk-policy indexes**: `knowledge.db` stores lightweight symbol/config/heading metadata plus an embedding input policy version. File, directory, and URL KBs with retained source paths can rebuild them with `knowledge_update`; imported portable KBs and inline text KBs should be re-imported or re-added because they intentionally do not store an active local source manifest.
|
|
236
350
|
|
|
237
351
|
## Development
|
|
238
352
|
|
package/dist/index.js
CHANGED
|
@@ -6,6 +6,7 @@ var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExte
|
|
|
6
6
|
}
|
|
7
7
|
return path;
|
|
8
8
|
};
|
|
9
|
+
import { MAX_SEARCH_RESULTS, MAX_SEARCH_WINDOW } from "./src/search/contracts.js";
|
|
9
10
|
const Type = {
|
|
10
11
|
Object(properties) {
|
|
11
12
|
const required = Object.entries(properties)
|
|
@@ -397,9 +398,19 @@ export default function (pi) {
|
|
|
397
398
|
Type.Literal("code"),
|
|
398
399
|
Type.Literal("docs"),
|
|
399
400
|
])),
|
|
400
|
-
limit: Type.Optional(Type.Number({
|
|
401
|
+
limit: Type.Optional(Type.Number({
|
|
402
|
+
description: "Max results (default comes from search profile/env)",
|
|
403
|
+
minimum: 1,
|
|
404
|
+
maximum: MAX_SEARCH_RESULTS,
|
|
405
|
+
multipleOf: 1,
|
|
406
|
+
})),
|
|
401
407
|
kb_id: Type.Optional(Type.String({ description: "Limit search to a specific KB by ID or exact name" })),
|
|
402
|
-
offset: Type.Optional(Type.Number({
|
|
408
|
+
offset: Type.Optional(Type.Number({
|
|
409
|
+
description: `Pagination offset; offset + limit must not exceed ${MAX_SEARCH_WINDOW}`,
|
|
410
|
+
minimum: 0,
|
|
411
|
+
maximum: MAX_SEARCH_WINDOW - 1,
|
|
412
|
+
multipleOf: 1,
|
|
413
|
+
})),
|
|
403
414
|
file_type: Type.Optional(Type.String({ description: "Filter by file type (e.g. typescript, markdown, python)" })),
|
|
404
415
|
path_pattern: Type.Optional(Type.String({ description: "Filter by file path substring (for example src/engine.ts)" })),
|
|
405
416
|
diversity: Type.Optional(Type.Union([Type.Literal("off"), Type.Literal("balanced"), Type.Literal("strong")])),
|
|
@@ -417,7 +428,7 @@ export default function (pi) {
|
|
|
417
428
|
response.retry_modes?.length ? `Retried: ${response.retry_modes.join(", ")}` : "",
|
|
418
429
|
response.warnings?.length ? `Warnings: ${response.warnings.join(" | ")}` : "",
|
|
419
430
|
diagnostics && response.tuning
|
|
420
|
-
? `Tuning: profile=${response.tuning.requested_profile}->${response.tuning.selected_profile}, limit=${response.tuning.limit}, snippet=${response.tuning.snippet_max_length}, min_hybrid=${response.tuning.min_hybrid_score}, candidates=${response.tuning.candidate_limit}`
|
|
431
|
+
? `Tuning: profile=${response.tuning.requested_profile}->${response.tuning.selected_profile}, limit=${response.tuning.limit}, snippet=${response.tuning.snippet_max_length}, min_hybrid=${response.tuning.min_hybrid_score}, candidates=${response.tuning.diversity_candidates}/${response.tuning.candidate_limit}, retrieval_per_kb=${response.tuning.retrieval_limit_per_kb}, retrieved=${response.tuning.retrieved_candidates}, filtered=${response.tuning.filtered_candidates}, truncated=${response.tuning.candidate_pool_truncated}`
|
|
421
432
|
: "",
|
|
422
433
|
response.suggestions?.length ? `Suggestions: ${response.suggestions.join(" | ")}` : "",
|
|
423
434
|
]
|
|
@@ -446,7 +457,7 @@ export default function (pi) {
|
|
|
446
457
|
? `\nProvenance: chunk=${r.provenance.chunk_id}, hash=${r.provenance.chunk_hash.slice(0, 12)}, reason=${r.provenance.match_reason}, indexed_at=${r.provenance.indexed_at}, source_mtime=${r.provenance.source_mtime ?? "unknown"}, stale=${r.provenance.stale}`
|
|
447
458
|
: "";
|
|
448
459
|
const tuning = diagnostics && response.tuning && i === 0
|
|
449
|
-
? `\nTuning: limit=${response.tuning.limit}, snippet=${response.tuning.snippet_max_length}, min_hybrid=${response.tuning.min_hybrid_score}, candidates=${response.tuning.candidate_limit}`
|
|
460
|
+
? `\nTuning: limit=${response.tuning.limit}, snippet=${response.tuning.snippet_max_length}, min_hybrid=${response.tuning.min_hybrid_score}, candidates=${response.tuning.diversity_candidates}/${response.tuning.candidate_limit}, retrieval_per_kb=${response.tuning.retrieval_limit_per_kb}, retrieved=${response.tuning.retrieved_candidates}, filtered=${response.tuning.filtered_candidates}, truncated=${response.tuning.candidate_pool_truncated}`
|
|
450
461
|
: "";
|
|
451
462
|
return `[${i + 1}] ${r.file_path} (${r.kb_name}, score: ${r.score.toFixed(3)})\n${r.snippet}${diag}${provenance}${tuning}`;
|
|
452
463
|
})
|
|
@@ -12,24 +12,37 @@ export interface EmbeddingConfig {
|
|
|
12
12
|
pooling: "mean";
|
|
13
13
|
normalize: true;
|
|
14
14
|
}
|
|
15
|
+
export interface EmbeddingInputMeasurement {
|
|
16
|
+
counts: number[];
|
|
17
|
+
maximum: number;
|
|
18
|
+
preferredMaximum: number;
|
|
19
|
+
unit: "tokens" | "characters";
|
|
20
|
+
}
|
|
15
21
|
export declare const CURRENT_EMBEDDING_MODEL = "multilingual-e5-small";
|
|
16
22
|
export declare const DEFAULT_LOCAL_EMBEDDING_MODEL = "multilingual-e5-small";
|
|
17
23
|
export declare const DEFAULT_OPENAI_EMBEDDING_MODEL = "text-embedding-3-small";
|
|
18
24
|
export declare const DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1";
|
|
25
|
+
export declare const LOCAL_EMBEDDING_MAX_TOKENS = 512;
|
|
26
|
+
export declare const LOCAL_EMBEDDING_PREFERRED_TOKENS = 448;
|
|
19
27
|
export declare const DEFAULT_API_MAX_EMBED_CHARS = 20000;
|
|
28
|
+
export declare const API_VECTOR_NORMALIZATION_POLICY = "client-l2-v1";
|
|
20
29
|
export declare function resolveEmbeddingConfig(env?: NodeJS.ProcessEnv): EmbeddingConfig;
|
|
21
30
|
export declare function embeddingConfigLabel(config: EmbeddingConfig): string;
|
|
31
|
+
export declare function normalizeEmbeddingVector(vector: Float32Array): Float32Array;
|
|
22
32
|
export declare function embeddingSignature(config: EmbeddingConfig, dimension: number): string;
|
|
23
33
|
export declare function dispose(): Promise<void>;
|
|
24
34
|
export declare function prepareForShutdown(): Promise<void>;
|
|
35
|
+
export declare function measureEmbeddingInputs(texts: string[], prefix: EmbeddingPrefix, config: EmbeddingConfig, signal?: AbortSignal): Promise<EmbeddingInputMeasurement>;
|
|
25
36
|
export declare function embedTextsWithConfig(texts: string[], prefix: EmbeddingPrefix, signal?: AbortSignal, options?: {
|
|
26
37
|
allowApiFallback?: boolean;
|
|
38
|
+
inputValidated?: boolean;
|
|
27
39
|
}): Promise<{
|
|
28
40
|
vectors: Float32Array[];
|
|
29
41
|
config: EmbeddingConfig;
|
|
30
42
|
}>;
|
|
31
43
|
export declare function embedTexts(texts: string[], prefix: EmbeddingPrefix, signal?: AbortSignal, options?: {
|
|
32
44
|
allowApiFallback?: boolean;
|
|
45
|
+
inputValidated?: boolean;
|
|
33
46
|
}): Promise<Float32Array[]>;
|
|
34
47
|
export declare function embedQueryWithConfig(text: string, signal?: AbortSignal): Promise<{
|
|
35
48
|
vector: Float32Array;
|
|
@@ -37,3 +50,4 @@ export declare function embedQueryWithConfig(text: string, signal?: AbortSignal)
|
|
|
37
50
|
}>;
|
|
38
51
|
export declare function embedQuery(text: string, signal?: AbortSignal): Promise<Float32Array>;
|
|
39
52
|
export declare function embedDocuments(texts: string[], signal?: AbortSignal): Promise<Float32Array[]>;
|
|
53
|
+
export declare function embedValidatedDocuments(texts: string[], signal?: AbortSignal): Promise<Float32Array[]>;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
-
import { embedInModelWorker } from "../model-worker-client.js";
|
|
2
|
+
import { embedInModelWorker, measureEmbeddingInputInModelWorker } from "../model-worker-client.js";
|
|
3
3
|
let disposeTimer = null;
|
|
4
4
|
let disposePromise = null;
|
|
5
5
|
let activeRuns = 0;
|
|
@@ -9,7 +9,10 @@ export const CURRENT_EMBEDDING_MODEL = "multilingual-e5-small";
|
|
|
9
9
|
export const DEFAULT_LOCAL_EMBEDDING_MODEL = "multilingual-e5-small";
|
|
10
10
|
export const DEFAULT_OPENAI_EMBEDDING_MODEL = "text-embedding-3-small";
|
|
11
11
|
export const DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1";
|
|
12
|
+
export const LOCAL_EMBEDDING_MAX_TOKENS = 512;
|
|
13
|
+
export const LOCAL_EMBEDDING_PREFERRED_TOKENS = 448;
|
|
12
14
|
export const DEFAULT_API_MAX_EMBED_CHARS = 20_000;
|
|
15
|
+
export const API_VECTOR_NORMALIZATION_POLICY = "client-l2-v1";
|
|
13
16
|
const IDLE_TIMEOUT_MS = Number(process.env.PI_KNOWLEDGE_EMBEDDING_IDLE_MS ?? 30_000);
|
|
14
17
|
const ENABLE_NATIVE_IDLE_DISPOSE = process.env.PI_KNOWLEDGE_ENABLE_NATIVE_IDLE_DISPOSE === "true";
|
|
15
18
|
const API_FALLBACK_TO_LOCAL = process.env.PI_KNOWLEDGE_EMBEDDING_API_FALLBACK === "local";
|
|
@@ -107,14 +110,30 @@ export function resolveEmbeddingConfig(env = process.env) {
|
|
|
107
110
|
export function embeddingConfigLabel(config) {
|
|
108
111
|
return config.provider === "local" ? config.model : `${config.provider}:${config.model}`;
|
|
109
112
|
}
|
|
113
|
+
export function normalizeEmbeddingVector(vector) {
|
|
114
|
+
let squaredMagnitude = 0;
|
|
115
|
+
for (const value of vector) {
|
|
116
|
+
if (!Number.isFinite(value))
|
|
117
|
+
throw new Error("Embedding provider returned a non-finite vector value");
|
|
118
|
+
squaredMagnitude += value * value;
|
|
119
|
+
}
|
|
120
|
+
if (!Number.isFinite(squaredMagnitude) || squaredMagnitude <= 0) {
|
|
121
|
+
throw new Error("Embedding provider returned a zero-magnitude vector");
|
|
122
|
+
}
|
|
123
|
+
const magnitude = Math.sqrt(squaredMagnitude);
|
|
124
|
+
for (let index = 0; index < vector.length; index++)
|
|
125
|
+
vector[index] /= magnitude;
|
|
126
|
+
return vector;
|
|
127
|
+
}
|
|
110
128
|
export function embeddingSignature(config, dimension) {
|
|
111
129
|
const base = config.baseUrl ? `:base-sha256=${providerInstanceHash(config.baseUrl)}` : "";
|
|
130
|
+
const normalization = config.provider === "openai" ? API_VECTOR_NORMALIZATION_POLICY : String(config.normalize);
|
|
112
131
|
const parts = [
|
|
113
132
|
`${config.provider}:${config.model}${base}`,
|
|
114
133
|
`dim=${dimension}`,
|
|
115
134
|
`apiMax=${config.provider === "openai" ? config.maxChars : "none"}`,
|
|
116
135
|
`pooling=${config.pooling}`,
|
|
117
|
-
`normalize=${
|
|
136
|
+
`normalize=${normalization}`,
|
|
118
137
|
`q=${config.queryPrefix}`,
|
|
119
138
|
`d=${config.documentPrefix}`,
|
|
120
139
|
];
|
|
@@ -141,11 +160,41 @@ export async function prepareForShutdown() {
|
|
|
141
160
|
function applyEmbeddingPrefix(text, prefix, config) {
|
|
142
161
|
return config.prefixes === "on" ? `${prefix}: ${text}` : text;
|
|
143
162
|
}
|
|
163
|
+
export async function measureEmbeddingInputs(texts, prefix, config, signal) {
|
|
164
|
+
if (signal?.aborted)
|
|
165
|
+
throw new Error("Cancelled");
|
|
166
|
+
if (config.provider === "openai") {
|
|
167
|
+
return {
|
|
168
|
+
counts: texts.map((text) => applyEmbeddingPrefix(text, prefix, config).length),
|
|
169
|
+
maximum: config.maxChars,
|
|
170
|
+
preferredMaximum: Math.max(1, Math.floor(config.maxChars * 0.875)),
|
|
171
|
+
unit: "characters",
|
|
172
|
+
};
|
|
173
|
+
}
|
|
174
|
+
beginRun();
|
|
175
|
+
try {
|
|
176
|
+
const measurement = await measureEmbeddingInputInModelWorker(texts, prefix, config.prefixes, signal);
|
|
177
|
+
return {
|
|
178
|
+
...measurement,
|
|
179
|
+
preferredMaximum: Math.max(1, Math.floor(measurement.maximum * 0.875)),
|
|
180
|
+
unit: "tokens",
|
|
181
|
+
};
|
|
182
|
+
}
|
|
183
|
+
finally {
|
|
184
|
+
endRun();
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
async function assertEmbeddingInputsWithinBudget(texts, prefix, config, signal) {
|
|
188
|
+
const measurement = await measureEmbeddingInputs(texts, prefix, config, signal);
|
|
189
|
+
for (let index = 0; index < measurement.counts.length; index++) {
|
|
190
|
+
const count = measurement.counts[index];
|
|
191
|
+
if (count !== undefined && count > measurement.maximum) {
|
|
192
|
+
throw new Error(`${config.provider} embedding ${prefix} input ${index + 1} is ${count} ${measurement.unit}; maximum is ${measurement.maximum}. Split the input before embedding.`);
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
}
|
|
144
196
|
async function embedViaAPI(texts, prefix, config, signal) {
|
|
145
|
-
const safeTexts = texts.map((text) =>
|
|
146
|
-
const embeddingText = applyEmbeddingPrefix(text, prefix, config);
|
|
147
|
-
return embeddingText.length > config.maxChars ? embeddingText.slice(0, config.maxChars) : embeddingText;
|
|
148
|
-
});
|
|
197
|
+
const safeTexts = texts.map((text) => applyEmbeddingPrefix(text, prefix, config));
|
|
149
198
|
const apiKey = cleanEnv(process.env.PI_KNOWLEDGE_EMBEDDING_API_KEY) ?? cleanEnv(process.env.OPENAI_API_KEY);
|
|
150
199
|
if (!config.baseUrl)
|
|
151
200
|
throw new Error("OpenAI embedding base URL is not configured");
|
|
@@ -163,13 +212,40 @@ async function embedViaAPI(texts, prefix, config, signal) {
|
|
|
163
212
|
const detail = await res.text().catch(() => "");
|
|
164
213
|
throw new Error(`OpenAI embedding API error: ${res.status}${detail ? ` ${detail.slice(0, 500)}` : ""}`);
|
|
165
214
|
}
|
|
166
|
-
const
|
|
167
|
-
|
|
215
|
+
const payload = (await res.json());
|
|
216
|
+
if (!payload || typeof payload !== "object" || !("data" in payload && Array.isArray(payload.data))) {
|
|
217
|
+
throw new Error("OpenAI embedding API response is missing a data array");
|
|
218
|
+
}
|
|
219
|
+
if (payload.data.length !== texts.length) {
|
|
220
|
+
throw new Error(`OpenAI embedding API returned ${payload.data.length} vectors for ${texts.length} inputs`);
|
|
221
|
+
}
|
|
222
|
+
let expectedDimension;
|
|
223
|
+
return payload.data.map((item, index) => {
|
|
224
|
+
if (!item || typeof item !== "object" || !("embedding" in item && Array.isArray(item.embedding))) {
|
|
225
|
+
throw new Error(`OpenAI embedding API response item ${index + 1} is missing an embedding array`);
|
|
226
|
+
}
|
|
227
|
+
if (expectedDimension !== undefined && item.embedding.length !== expectedDimension) {
|
|
228
|
+
throw new Error(`OpenAI embedding API response item ${index + 1} has dimension ${item.embedding.length}; expected ${expectedDimension}`);
|
|
229
|
+
}
|
|
230
|
+
expectedDimension = item.embedding.length;
|
|
231
|
+
const vector = new Float32Array(item.embedding.length);
|
|
232
|
+
for (let valueIndex = 0; valueIndex < item.embedding.length; valueIndex++) {
|
|
233
|
+
const value = item.embedding[valueIndex];
|
|
234
|
+
if (typeof value !== "number" || !Number.isFinite(value)) {
|
|
235
|
+
throw new Error(`OpenAI embedding API response item ${index + 1} contains a non-finite numeric value at position ${valueIndex}`);
|
|
236
|
+
}
|
|
237
|
+
vector[valueIndex] = value;
|
|
238
|
+
}
|
|
239
|
+
return normalizeEmbeddingVector(vector);
|
|
240
|
+
});
|
|
168
241
|
}
|
|
169
242
|
export async function embedTextsWithConfig(texts, prefix, signal, options = {}) {
|
|
170
243
|
const config = resolveEmbeddingConfig();
|
|
171
244
|
let actualConfig = config;
|
|
245
|
+
let inputValidated = options.inputValidated === true;
|
|
172
246
|
if (config.provider === "openai") {
|
|
247
|
+
if (!inputValidated)
|
|
248
|
+
await assertEmbeddingInputsWithinBudget(texts, prefix, config, signal);
|
|
173
249
|
if (signal?.aborted)
|
|
174
250
|
throw new Error("Cancelled");
|
|
175
251
|
try {
|
|
@@ -182,8 +258,11 @@ export async function embedTextsWithConfig(texts, prefix, signal, options = {})
|
|
|
182
258
|
throw error;
|
|
183
259
|
console.warn(`pi-knowledge: embedding API failed; falling back to local model because PI_KNOWLEDGE_EMBEDDING_API_FALLBACK=local (${error instanceof Error ? error.message : String(error)})`);
|
|
184
260
|
actualConfig = localEmbeddingConfig(config.maxChars, config.prefixes);
|
|
261
|
+
inputValidated = false;
|
|
185
262
|
}
|
|
186
263
|
}
|
|
264
|
+
if (!inputValidated)
|
|
265
|
+
await assertEmbeddingInputsWithinBudget(texts, prefix, actualConfig, signal);
|
|
187
266
|
beginRun();
|
|
188
267
|
try {
|
|
189
268
|
return { vectors: await embedInModelWorker(texts, prefix, actualConfig.prefixes, signal), config: actualConfig };
|
|
@@ -205,3 +284,6 @@ export async function embedQuery(text, signal) {
|
|
|
205
284
|
export async function embedDocuments(texts, signal) {
|
|
206
285
|
return embedTexts(texts, "passage", signal, { allowApiFallback: false });
|
|
207
286
|
}
|
|
287
|
+
export async function embedValidatedDocuments(texts, signal) {
|
|
288
|
+
return embedTexts(texts, "passage", signal, { allowApiFallback: false, inputValidated: true });
|
|
289
|
+
}
|
package/dist/src/engine.d.ts
CHANGED
|
@@ -58,7 +58,7 @@ export interface DoctorIssue {
|
|
|
58
58
|
kb_name?: string;
|
|
59
59
|
message: string;
|
|
60
60
|
action: string;
|
|
61
|
-
action_code?: "run_update" | "rebuild_kb" | "wait_for_indexing" | "review_skipped_scope" | "rebuild_vectors" | "check_source" | "none";
|
|
61
|
+
action_code?: "run_update" | "rebuild_kb" | "wait_for_indexing" | "review_skipped_scope" | "rebuild_vectors" | "reindex_for_embedding_budget" | "check_source" | "none";
|
|
62
62
|
}
|
|
63
63
|
export interface DoctorReport {
|
|
64
64
|
health_score: number;
|
|
@@ -97,6 +97,8 @@ export interface SymbolSearchResponse {
|
|
|
97
97
|
total_count: number;
|
|
98
98
|
has_more: boolean;
|
|
99
99
|
}
|
|
100
|
+
export declare const CURRENT_URL_STRUCTURE_POLICY = 1;
|
|
101
|
+
export declare const CURRENT_EMBEDDING_INPUT_POLICY = 2;
|
|
100
102
|
export interface IndexPlan {
|
|
101
103
|
source_type: "file" | "directory" | "text" | "url";
|
|
102
104
|
scannable_files: number;
|