pi-knowledge 0.15.0 → 0.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -0
- package/README.md +61 -23
- package/dist/index.d.ts +10 -4
- package/dist/index.js +145 -166
- package/dist/src/coverage-snapshot.d.ts +23 -0
- package/dist/src/coverage-snapshot.js +53 -0
- package/dist/src/daemon/client.d.ts +60 -0
- package/dist/src/daemon/client.js +360 -0
- package/dist/src/daemon/main.d.ts +2 -0
- package/dist/src/daemon/main.js +109 -0
- package/dist/src/daemon/protocol.d.ts +71 -0
- package/dist/src/daemon/protocol.js +170 -0
- package/dist/src/daemon/runtime.d.ts +26 -0
- package/dist/src/daemon/runtime.js +192 -0
- package/dist/src/daemon/server.d.ts +46 -0
- package/dist/src/daemon/server.js +500 -0
- package/dist/src/daemon/service.d.ts +49 -0
- package/dist/src/daemon/service.js +527 -0
- package/dist/src/diagnostics/health.d.ts +6 -0
- package/dist/src/diagnostics/health.js +135 -2
- package/dist/src/engine.d.ts +13 -8
- package/dist/src/engine.js +569 -100
- package/dist/src/filesystem-access.d.ts +9 -0
- package/dist/src/filesystem-access.js +100 -0
- package/dist/src/indexer/chunker.d.ts +12 -0
- package/dist/src/indexer/chunker.js +307 -9
- package/dist/src/mcp/http.d.ts +47 -0
- package/dist/src/mcp/http.js +564 -0
- package/dist/src/mcp/main.d.ts +2 -0
- package/dist/src/mcp/main.js +97 -0
- package/dist/src/mcp/protocol.d.ts +81 -0
- package/dist/src/mcp/protocol.js +444 -0
- package/dist/src/mcp/server.d.ts +42 -0
- package/dist/src/mcp/server.js +378 -0
- package/dist/src/model-worker-client.js +9 -0
- package/dist/src/model-worker-contract.d.ts +3 -0
- package/dist/src/model-worker-contract.js +8 -0
- package/dist/src/model-worker.js +76 -4
- package/dist/src/network/safe-fetch.d.ts +7 -0
- package/dist/src/network/safe-fetch.js +199 -0
- package/dist/src/node-runtime.d.ts +3 -0
- package/dist/src/node-runtime.js +78 -0
- package/dist/src/profiles.d.ts +15 -0
- package/dist/src/profiles.js +160 -0
- package/dist/src/runtime-config.js +5 -6
- package/dist/src/secure-files.d.ts +17 -0
- package/dist/src/secure-files.js +183 -0
- package/dist/src/status.js +12 -7
- package/dist/src/storage/sqlite.d.ts +4 -1
- package/dist/src/storage/sqlite.js +46 -3
- package/dist/src/type-guards.d.ts +1 -0
- package/dist/src/type-guards.js +3 -0
- package/dist/src/watcher/file-watcher.d.ts +3 -1
- package/dist/src/watcher/file-watcher.js +166 -32
- package/dist/src/windows-security.d.ts +4 -0
- package/dist/src/windows-security.js +67 -0
- package/docs/configuration.md +40 -9
- package/docs/daemon-profiles.md +302 -0
- package/docs/mcp-clients.md +266 -0
- package/docs/offline-mode.md +6 -6
- package/package.json +28 -2
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,35 @@
|
|
|
2
2
|
|
|
3
3
|
## [Unreleased]
|
|
4
4
|
|
|
5
|
+
## [0.16.1] - 2026-11-21
|
|
6
|
+
|
|
7
|
+
### Fixed
|
|
8
|
+
- Removed the fixed five-minute asynchronous directory-scan deadline so progressing large plan/add/update, diagnostics, and watcher traversals can complete; operators can opt into an explicit limit with `PI_KNOWLEDGE_SCAN_TIMEOUT_MS` (#26).
|
|
9
|
+
- Fixed Windows profile startup by passing ACL-hardening paths to PowerShell outside `-Command` argv, preserving clean JSON output, surfacing bounded stderr diagnostics, and handling expected atomic manifest and shutdown-file races without weakening ACL checks (#27).
|
|
10
|
+
|
|
11
|
+
## [0.16.0] - 2026-11-19
|
|
12
|
+
|
|
13
|
+
### Added
|
|
14
|
+
- Added default-first isolated knowledge profiles with one authenticated long-lived daemon per profile, explicit daemon CLI lifecycle, and automatic Pi/OMP client startup.
|
|
15
|
+
- Added a dependency-free MCP server with stdio and authenticated loopback Streamable HTTP transports, legacy and modern protocol contracts, request-scoped SSE progress, cancellation, shared public tool schemas, and explicit root/network/write/admin capabilities.
|
|
16
|
+
- Added daemon-owned atomic `coverage.json` profile snapshots for issue #25, including startup reconciliation, mutation hooks, and non-rollback failure diagnostics.
|
|
17
|
+
- Added external-agent setup for Claude Code, Codex, Cursor, VS Code Copilot, and Kiro CLI/ACP, including npm 12 install-script approvals, profile selection, token handling, and upgrade lifecycle.
|
|
18
|
+
|
|
19
|
+
### Changed
|
|
20
|
+
- Moved production SQLite, vector, model-worker, mutation, and watcher ownership out of the Pi/OMP extension into the profile daemon; extension shutdown no longer stops shared knowledge state.
|
|
21
|
+
- Stabilized default storage selection across Pi, OMP, CLI, and MCP through an atomic profile registry while preserving one existing legacy knowledge database in place.
|
|
22
|
+
- Version-pinned the reviewed native/runtime dependency install-script approvals required by npm 12.
|
|
23
|
+
|
|
24
|
+
### Fixed
|
|
25
|
+
- Enforced MCP local-root and network capabilities again at daemon operation time, with canonical immutable update targets, bounded import metadata, Windows ACL checks, and race-aware directory root binding.
|
|
26
|
+
- Bounded Unix socket fallback paths, diagnostic working sets, RPC/MCP queues and frames, and watcher polling so long scans do not trigger uninterrupted immediate rescans.
|
|
27
|
+
- Restricted capability-free MCP to indexed search/symbol/show data; live status/doctor rescans now require admin capability, and burst/slow-drip framing remains bounded without terminating active work.
|
|
28
|
+
- Added required modern MCP caching hints to cacheable complete results so strict `2026-07-28` clients such as Kiro CLI 3.5 can initialize and call tools.
|
|
29
|
+
- Made coverage `updated_at` advance only after a complete successful index refresh, including a schema migration for existing knowledge bases.
|
|
30
|
+
- Stopped permanent watcher update failures from retrying indefinitely, and refreshed legacy HTTP session age after long-running requests complete.
|
|
31
|
+
- Hardened profile storage to owner-only permissions, removed persisted KB labels from system prompts, preserved technically skipped files during updates, and resolved PowerShell from the active Windows system root.
|
|
32
|
+
|
|
33
|
+
|
|
5
34
|
## [0.15.0] - 2026-11-15
|
|
6
35
|
|
|
7
36
|
### Added
|
package/README.md
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
# pi-knowledge
|
|
2
2
|
|
|
3
|
-
Local-first RAG knowledge
|
|
3
|
+
Local-first RAG knowledge server for Pi, OMP, and MCP-compatible coding agents.
|
|
4
4
|
|
|
5
|
-
Index your codebase, docs, PDFs, URLs, and notes into persistent knowledge bases
|
|
5
|
+
Index your codebase, docs, PDFs, URLs, and notes into persistent knowledge bases shared through a profile daemon. `pi-knowledge` combines semantic embeddings, BM25 keyword search, code-aware chunking, reranking, diagnostics, and stable large-project indexing so agents can answer from your actual project knowledge instead of guessing.
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
Pi and [OMP](https://omp.sh/) use the packaged extension entry with their native hooks and tool guidance. External agents use the standalone MCP command over stdio or loopback Streamable HTTP.
|
|
8
8
|
|
|
9
9
|
## Why It Matters
|
|
10
10
|
|
|
11
|
-
Agents lose project context between sessions and cannot fit large repositories into one prompt. `pi-knowledge` gives Pi and
|
|
11
|
+
Agents lose project context between sessions and cannot fit large repositories into one prompt. `pi-knowledge` gives Pi, OMP, and MCP clients durable, searchable project memory:
|
|
12
12
|
|
|
13
13
|
- Search code and documentation by meaning, exact symbols, or both.
|
|
14
14
|
- Look up lightweight code symbols, Markdown headings, config keys, and environment variables before broader search.
|
|
@@ -21,7 +21,7 @@ Unlike `pi-memory` (which manages the agent's own notes), `pi-knowledge` indexes
|
|
|
21
21
|
|
|
22
22
|
## Highlights
|
|
23
23
|
|
|
24
|
-
- **
|
|
24
|
+
- **Default-first shared service**: Pi, OMP, CLI, and MCP share one long-lived daemon and storage domain for the zero-config `default` profile; advanced users can select isolated named profiles.
|
|
25
25
|
- **Hybrid retrieval**: lexical-anchored BM25 + semantic vectors + normalized weighted score fusion, with semantic mode for vector-only conceptual recall.
|
|
26
26
|
- **Code- and document-aware indexing**: recursive AST chunking for supported code, heading-aware Markdown chunks, and an embedding-safe finalizer that preserves oversized multilingual tails without adding a heavy document runtime.
|
|
27
27
|
- **Structured URL indexing**: preserves HTML headings, paragraphs, lists, tables, and entities as lightweight Markdown-like text without a browser, Docling runtime, or new dependency.
|
|
@@ -29,6 +29,7 @@ Unlike `pi-memory` (which manages the agent's own notes), `pi-knowledge` indexes
|
|
|
29
29
|
- **Better agent answers**: adaptive context windows, diversity reranking, optional cross-encoder reranking, and diagnostics.
|
|
30
30
|
- **Large-project stability**: persisted indexing progress, capped batches, streaming vector scans, and stuck-job detection.
|
|
31
31
|
- **Private by default**: local embeddings run without an API key.
|
|
32
|
+
- **Standalone MCP**: `pi-knowledge-mcp` provides stdio and authenticated loopback Streamable HTTP; both are read-only by default, with explicit local-root, network, write, and admin capabilities.
|
|
32
33
|
|
|
33
34
|
## Feature Comparison
|
|
34
35
|
|
|
@@ -103,6 +104,14 @@ sudo apt-get install -y build-essential python3
|
|
|
103
104
|
omp plugin install npm:pi-knowledge
|
|
104
105
|
```
|
|
105
106
|
|
|
107
|
+
For Claude Code, Codex, Cursor, VS Code Copilot, or another MCP client, install the command globally and follow [docs/mcp-clients.md](docs/mcp-clients.md):
|
|
108
|
+
|
|
109
|
+
```bash
|
|
110
|
+
npm install -g pi-knowledge
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
npm 12 requires explicit approval for native/runtime dependency install scripts; the MCP guide provides the reviewed allow-list and verification steps.
|
|
114
|
+
|
|
106
115
|
For a local checkout, install dependencies and build before registration:
|
|
107
116
|
|
|
108
117
|
```bash
|
|
@@ -138,6 +147,21 @@ export PI_KNOWLEDGE_AUTO_INJECT=true # retrieve context before every model call
|
|
|
138
147
|
export PI_KNOWLEDGE_WATCH=true # update directory KBs after file changes
|
|
139
148
|
```
|
|
140
149
|
|
|
150
|
+
The Pi/OMP extension and every MCP transport automatically ensure the selected profile daemon and leave it running when the client exits. Explicit daemon and MCP entry points are also available:
|
|
151
|
+
|
|
152
|
+
```bash
|
|
153
|
+
pi-knowledge-daemon status
|
|
154
|
+
pi-knowledge-daemon start --profile research
|
|
155
|
+
pi-knowledge-mcp # stdio; indexed search/symbol/show only
|
|
156
|
+
pi-knowledge-mcp --allow-root "$PWD" --allow-write
|
|
157
|
+
PI_KNOWLEDGE_MCP_HTTP_TOKEN="$(openssl rand -hex 32)" \
|
|
158
|
+
pi-knowledge-mcp --transport http # http://127.0.0.1:3210/mcp
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
HTTP mode is loopback-only and requires the bearer token. See [MCP Client Setup](docs/mcp-clients.md) for standard client configuration, capability flags, token handling, profiles, and lifecycle.
|
|
162
|
+
|
|
163
|
+
Most users should not set a profile. Advanced users can set `PI_KNOWLEDGE_PROFILE=research` or pass `--profile research` to isolate the database, vectors, daemon, watchers, and coverage snapshot.
|
|
164
|
+
|
|
141
165
|
Per-turn injection adds retrieval work and prompt context to every model call.
|
|
142
166
|
File watching applies only to directory-backed knowledge bases.
|
|
143
167
|
|
|
@@ -174,6 +198,14 @@ Install the unversioned Pi source (`npm:pi-knowledge`) to follow the npm
|
|
|
174
198
|
2. Pass the exact pinned source shown there to `pi remove`.
|
|
175
199
|
3. Run `pi install npm:pi-knowledge`.
|
|
176
200
|
|
|
201
|
+
|
|
202
|
+
For a globally installed standalone MCP command:
|
|
203
|
+
|
|
204
|
+
```bash
|
|
205
|
+
npm install -g pi-knowledge
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
With npm 12, repeat the reviewed `--allow-scripts` list from [docs/mcp-clients.md](docs/mcp-clients.md). Restart persistent HTTP adapters and reconnect stdio clients after upgrading. A compatible client safely replaces an older profile daemon when the package/build protocol requires it; Pi/OMP session shutdown and MCP client shutdown do not stop the shared daemon.
|
|
177
209
|
### Local checkout
|
|
178
210
|
|
|
179
211
|
A linked checkout is updated from Git, not from the npm registry. Finish,
|
|
@@ -244,7 +276,7 @@ Mode selection contract:
|
|
|
244
276
|
- Use `deep` for high-stakes answers, ambiguous top results, or final verification when slower reranking is acceptable.
|
|
245
277
|
- If results are empty, follow the returned classified cause and action. `auto` records its attempted modes; do not retry a mode already listed.
|
|
246
278
|
|
|
247
|
-
Search results use balanced diversity reranking by default so near-duplicate chunks from the same file do not dominate the top results. Diversity scoring considers lexical overlap, same-file line proximity, overlapping adaptive windows, available embedding-vector similarity, file-level interleaving, and a small KB trust multiplier that favors ready file/directory sources over stale, URL, or imported text sources. In `adaptive` mode, code chunks with AST metadata can prefer same-parent/sibling chunks inside the same bounded file window; this is metadata-driven context expansion, not caller/callee analysis. Use `diversity: "off"` only when raw ranking order is needed for diagnostics. Agents can request search diagnostics to inspect mode fallback, selected profile, applied search tuning, ranking coverage, path/source/test boosts, adjusted scores, and provenance such as chunk id, chunk hash,
|
|
279
|
+
Search results use balanced diversity reranking by default so near-duplicate chunks from the same file do not dominate the top results. Diversity scoring considers lexical overlap, same-file line proximity, overlapping adaptive windows, available embedding-vector similarity, file-level interleaving, and a small KB trust multiplier that favors ready file/directory sources over stale, URL, or imported text sources. In `adaptive` mode, code chunks with AST metadata can prefer same-parent/sibling chunks inside the same bounded file window; this is metadata-driven context expansion, not caller/callee analysis. Use `diversity: "off"` only when raw ranking order is needed for diagnostics. Agents can request search diagnostics to inspect mode fallback, selected profile, applied search tuning, ranking coverage, path/source/test boosts, adjusted scores, and indexed-only provenance such as chunk id, chunk hash, indexed timestamp, persisted stale flag, match reason, and adaptive `source_chunk_ids` where available.
|
|
248
280
|
|
|
249
281
|
Zero-result responses are self-classifying. `knowledge_search` distinguishes missing or unavailable KBs, KBs with no indexed chunks, embedding incompatibility, retrieval misses, score-threshold rejection, lexical-evidence rejection, metadata filtering, empty reranker output, and out-of-range pages. `knowledge_symbol_search` distinguishes missing symbol metadata, exact-name misses, symbol filters, unavailable KBs, and pagination. Both return concise two-line guidance plus a structured `details.miss` object; classification uses request and search-pipeline facts without running a source diagnostics scan.
|
|
250
282
|
|
|
@@ -312,27 +344,29 @@ Full configuration details are in [docs/configuration.md](docs/configuration.md)
|
|
|
312
344
|
|
|
313
345
|
| Area | Environment variables |
|
|
314
346
|
|------|-----------------------|
|
|
315
|
-
|
|
|
316
|
-
| Runtime worker and model cache | `PI_KNOWLEDGE_MODEL_CACHE_DIR`, `PI_KNOWLEDGE_NODE_PATH` |
|
|
347
|
+
| Profile and storage | `PI_KNOWLEDGE_PROFILE`, `PI_KNOWLEDGE_PROFILE_ROOT`, `PI_KNOWLEDGE_DIR`, `OMP_KNOWLEDGE_DIR`, `PI_CODING_AGENT_DIR`, `OMP_CODING_AGENT_DIR` |
|
|
348
|
+
| Runtime worker and model cache | `PI_KNOWLEDGE_MODEL_CACHE_DIR`, `PI_KNOWLEDGE_NODE_PATH`, `PI_KNOWLEDGE_RUNTIME_DIR` |
|
|
317
349
|
| Embedding provider | `PI_KNOWLEDGE_EMBEDDING`, `PI_KNOWLEDGE_EMBEDDING_API_KEY`, `OPENAI_API_KEY`, `PI_KNOWLEDGE_EMBEDDING_BASE_URL`, `OPENAI_BASE_URL`, `PI_KNOWLEDGE_EMBEDDING_PREFIXES`, `PI_KNOWLEDGE_EMBEDDING_TASK`, `PI_KNOWLEDGE_EMBEDDING_QUERY_INSTRUCTION`, `PI_KNOWLEDGE_EMBEDDING_DOCUMENT_INSTRUCTION`, `PI_KNOWLEDGE_EMBEDDING_MAX_CHARS`, `PI_KNOWLEDGE_EMBEDDING_BATCH_SIZE`, `PI_KNOWLEDGE_EMBEDDING_API_FALLBACK` |
|
|
350
|
+
| MCP HTTP | `PI_KNOWLEDGE_MCP_HTTP_TOKEN` |
|
|
318
351
|
| Reranker provider | `PI_KNOWLEDGE_RERANKER`, `PI_KNOWLEDGE_RERANKER_REVISION`, `PI_KNOWLEDGE_RERANKER_DTYPE`, `PI_KNOWLEDGE_RERANKER_REMOTE_HOST`, `PI_KNOWLEDGE_RERANKER_REMOTE_PATH_TEMPLATE`, `PI_KNOWLEDGE_RERANKER_RAW_LOGITS`, `PI_KNOWLEDGE_RERANKER_API_ENDPOINT`, `PI_KNOWLEDGE_RERANKER_API_BASE_URL`, `PI_KNOWLEDGE_RERANKER_API_KEY`, `PI_KNOWLEDGE_RERANKER_API_FORMAT`, `PI_KNOWLEDGE_RERANKER_API_TIMEOUT_MS`, `PI_KNOWLEDGE_RERANKER_MAX_DOC_CHARS`, `PI_KNOWLEDGE_RERANKER_API_RESULTS_PATH`, `PI_KNOWLEDGE_RERANKER_API_INDEX_FIELD`, `PI_KNOWLEDGE_RERANKER_API_SCORE_FIELD`, `PI_KNOWLEDGE_RERANKER_API_SCORE_DIRECTION` |
|
|
319
352
|
| Native lifecycle | `PI_KNOWLEDGE_ENABLE_NATIVE_IDLE_DISPOSE`, `PI_KNOWLEDGE_EMBEDDING_IDLE_MS` |
|
|
320
|
-
| Runtime features | `PI_KNOWLEDGE_WATCH`, `PI_KNOWLEDGE_AUTO_INJECT`, `PI_KNOWLEDGE_STALE_INDEXING_MS`, `PI_KNOWLEDGE_OFFLINE`, `PI_KNOWLEDGE_SEARCH_PROFILE`, `PI_KNOWLEDGE_SEARCH_DEFAULT_LIMIT`, `PI_KNOWLEDGE_SNIPPET_MAX_LENGTH`, `PI_KNOWLEDGE_MIN_HYBRID_SCORE`, `PI_KNOWLEDGE_SEARCH_CANDIDATE_MIN`, `PI_KNOWLEDGE_SEARCH_CANDIDATE_MULTIPLIER`, `PI_KNOWLEDGE_ADAPTIVE_CONTEXT_LINES`, `PI_KNOWLEDGE_ADAPTIVE_MAX_CHARS`, `PI_KNOWLEDGE_ADAPTIVE_NEIGHBOR_TARGET`, `PI_KNOWLEDGE_DEEP_RERANK_CANDIDATES`, `PI_KNOWLEDGE_DEEP_RERANK_TOPK_MULTIPLIER` |
|
|
353
|
+
| Runtime features | `PI_KNOWLEDGE_WATCH`, `PI_KNOWLEDGE_AUTO_INJECT`, `PI_KNOWLEDGE_STALE_INDEXING_MS`, `PI_KNOWLEDGE_SCAN_TIMEOUT_MS`, `PI_KNOWLEDGE_OFFLINE`, `PI_KNOWLEDGE_SEARCH_PROFILE`, `PI_KNOWLEDGE_SEARCH_DEFAULT_LIMIT`, `PI_KNOWLEDGE_SNIPPET_MAX_LENGTH`, `PI_KNOWLEDGE_MIN_HYBRID_SCORE`, `PI_KNOWLEDGE_SEARCH_CANDIDATE_MIN`, `PI_KNOWLEDGE_SEARCH_CANDIDATE_MULTIPLIER`, `PI_KNOWLEDGE_ADAPTIVE_CONTEXT_LINES`, `PI_KNOWLEDGE_ADAPTIVE_MAX_CHARS`, `PI_KNOWLEDGE_ADAPTIVE_NEIGHBOR_TARGET`, `PI_KNOWLEDGE_DEEP_RERANK_CANDIDATES`, `PI_KNOWLEDGE_DEEP_RERANK_TOPK_MULTIPLIER` |
|
|
321
354
|
| Release fixtures | `PI_KNOWLEDGE_E2E_PDF`, `PI_KNOWLEDGE_E2E_DOCX` |
|
|
322
355
|
|
|
323
356
|
For Hugging Face Text Embeddings Inference rerankers, use `PI_KNOWLEDGE_RERANKER=api:<model>`, `PI_KNOWLEDGE_RERANKER_API_FORMAT=tei`, and `PI_KNOWLEDGE_RERANKER_API_ENDPOINT=http://127.0.0.1:8080/rerank`; TEI exposes OpenAI-compatible embeddings under `/v1`, but reranking uses `/rerank`.
|
|
324
357
|
|
|
325
358
|
## Pi and OMP Support
|
|
326
359
|
|
|
327
|
-
`pi-knowledge` supports Pi and [OMP](https://omp.sh/)
|
|
360
|
+
`pi-knowledge` supports Pi and [OMP](https://omp.sh/) through the packaged `extension.js` entry shim. The startup-light extension lazily connects to the selected profile daemon; it no longer opens SQLite, vectors, model workers, or watchers inside the host process.
|
|
328
361
|
|
|
329
|
-
|
|
362
|
+
The default profile atomically adopts one existing legacy Pi/OMP knowledge directory or creates `~/.pi-knowledge/profiles/default`. `~/.pi-knowledge/profiles.json` keeps that choice stable across Pi, OMP, CLI, and MCP. Explicit `PI_KNOWLEDGE_DIR` / `OMP_KNOWLEDGE_DIR` overrides apply only to the default profile. Named profiles are isolated under `~/.pi-knowledge/profiles/<name>`.
|
|
330
363
|
|
|
331
|
-
OMP compatibility covers
|
|
364
|
+
OMP compatibility covers packaged entry loading, native SQLite dependency resolution in the daemon, trusted Node 22+ startup, Windows-safe worker transport, parser stdio isolation, and idempotent shutdown. `knowledge_configure` persists an explicit trusted Node path when a packaged host cannot launch the daemon directly.
|
|
332
365
|
|
|
333
366
|
## Large Project Indexing
|
|
334
367
|
|
|
335
368
|
Indexing is designed as a stable long-running operation, not a quick background trick. `knowledge_add`, `knowledge_update`, and `knowledge_import` scan directories incrementally, embed and store chunks in hard-capped batches, stream vector files to disk, and report progress with file/chunk counts, chunks/sec, skipped file counts, elapsed time, and file ETA where available. Directory indexing starts with a metadata-only planning scan so large repositories can show total scannable files and skipped counts before expensive embedding starts.
|
|
369
|
+
Directory scans have no wall-clock timeout by default; entry-count, total-byte, depth, file-size, access, and cancellation bounds remain enforced. Operators who need an absolute limit for unusually slow or remote filesystems can set `PI_KNOWLEDGE_SCAN_TIMEOUT_MS` to a positive integer in milliseconds, or `0` to keep the default unbounded-duration policy. Restart the selected profile daemon after changing this runtime setting.
|
|
336
370
|
|
|
337
371
|
Directory indexing separates technical skips from user-confirmable suggestions. `knowledge_plan` inspects a source without writing a KB, so agents can show scannable files, suggested exclusions, and technical skips before asking the user to confirm scope. Unsupported binary/non-text files, oversized files, unreadable files, inaccessible paths, and documents that cannot be extracted are skipped for stability. Text files that may be private or low-signal, such as `.env`, credential-named files, generated reports, lockfiles, vendor text, build output text, and runtime/cache text, are suggested exclusions by default rather than permanent blocks. Agents should explain the privacy and search-precision tradeoff, ask the user when the choice is ambiguous, and then use `include_suggested_text` or focused `include_paths` when the user confirms those text files belong in the KB. Ordinary project configuration files such as `settings.json` or `appsettings.json` remain indexable because they often describe real system behavior.
|
|
338
372
|
|
|
@@ -353,21 +387,25 @@ the repository retains the broader design history.
|
|
|
353
387
|
|
|
354
388
|
## Data Storage
|
|
355
389
|
|
|
356
|
-
|
|
357
|
-
indexed projects are never used for internal storage:
|
|
390
|
+
Storage is profile-scoped and never placed inside an indexed project:
|
|
358
391
|
|
|
359
392
|
```
|
|
360
|
-
<
|
|
361
|
-
├── knowledge.db ← SQLite
|
|
362
|
-
├── vectors/ ← Embedding vectors per KB
|
|
363
|
-
|
|
393
|
+
<profile-storage>/
|
|
394
|
+
├── knowledge.db ← SQLite metadata, chunks, FTS, symbols, and job state
|
|
395
|
+
├── vectors/ ← Embedding vectors per KB
|
|
396
|
+
├── models/ ← Local model cache
|
|
397
|
+
├── config.json ← Profile runtime configuration
|
|
398
|
+
├── coverage.json ← Advisory atomic profile scope/status snapshot
|
|
399
|
+
└── .runtime/ ← Private daemon ownership and manifest state
|
|
364
400
|
```
|
|
365
401
|
|
|
366
|
-
- **
|
|
367
|
-
- **
|
|
368
|
-
- **
|
|
369
|
-
- **
|
|
370
|
-
- **
|
|
402
|
+
- **Default**: stable mapping recorded in `~/.pi-knowledge/profiles.json`
|
|
403
|
+
- **Named profiles**: `~/.pi-knowledge/profiles/<name>` unless `PI_KNOWLEDGE_PROFILE_ROOT` overrides the root
|
|
404
|
+
- **Backup**: stop the profile daemon, then copy the profile storage directory
|
|
405
|
+
- **Reset**: use confirmed `knowledge_clear`; do not delete live daemon files
|
|
406
|
+
- **Project safety**: indexed directories remain read-only
|
|
407
|
+
- **Coverage snapshot**: `coverage.json` contains absolute source paths and is sensitive, advisory point-in-time state; consumers must handle absence, staleness, and unknown schema versions
|
|
408
|
+
- **Updates**: schema migrations run automatically, but index-time behavior changes are reported by `knowledge_doctor` rather than silently rebuilding vectors at startup
|
|
371
409
|
- **Symbol and chunk-policy indexes**: `knowledge.db` stores lightweight symbol/config/heading metadata plus an embedding input policy version. File, directory, and URL KBs with retained source paths can rebuild them with `knowledge_update`; imported portable KBs and inline text KBs should be re-imported or re-added because they intentionally do not store an active local source manifest.
|
|
372
410
|
|
|
373
411
|
## Development
|
package/dist/index.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { FilesystemAccessPolicy } from "./src/filesystem-access.ts";
|
|
1
2
|
type ToolResult = {
|
|
2
3
|
content: Array<{
|
|
3
4
|
type: "text";
|
|
@@ -8,7 +9,11 @@ type ToolResult = {
|
|
|
8
9
|
};
|
|
9
10
|
type ToolUpdate = (result: ToolResult) => void;
|
|
10
11
|
type ToolApproval = "read" | "write" | "exec";
|
|
11
|
-
|
|
12
|
+
export interface ToolExecutionContext {
|
|
13
|
+
profile?: string;
|
|
14
|
+
filesystemAccess?: FilesystemAccessPolicy;
|
|
15
|
+
}
|
|
16
|
+
export type ToolDefinition = {
|
|
12
17
|
name: string;
|
|
13
18
|
label: string;
|
|
14
19
|
description: string;
|
|
@@ -16,7 +21,7 @@ type ToolDefinition = {
|
|
|
16
21
|
promptSnippet?: string;
|
|
17
22
|
promptGuidelines?: string[];
|
|
18
23
|
parameters: Schema;
|
|
19
|
-
execute?: (toolCallId: string, params: Record<string, unknown>, signal: AbortSignal | undefined, onUpdate: ToolUpdate | undefined, ctx:
|
|
24
|
+
execute?: (toolCallId: string, params: Record<string, unknown>, signal: AbortSignal | undefined, onUpdate: ToolUpdate | undefined, ctx: ToolExecutionContext | undefined) => ToolResult | Promise<ToolResult>;
|
|
20
25
|
};
|
|
21
26
|
type ContextEvent = {
|
|
22
27
|
messages: Array<{
|
|
@@ -36,5 +41,6 @@ type ExtensionAPI = {
|
|
|
36
41
|
type Schema = Record<string, unknown> & {
|
|
37
42
|
optional?: true;
|
|
38
43
|
};
|
|
39
|
-
|
|
40
|
-
export
|
|
44
|
+
declare function registerKnowledgeExtension(pi: ExtensionAPI): void;
|
|
45
|
+
export declare function getKnowledgeToolDefinitions(): ToolDefinition[];
|
|
46
|
+
export default registerKnowledgeExtension;
|