pi-knowledge 0.13.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/README.md +30 -8
- package/dist/index.js +176 -84
- package/dist/src/embedding/instructions.d.ts +33 -0
- package/dist/src/embedding/instructions.js +146 -0
- package/dist/src/embedding/provider.d.ts +25 -8
- package/dist/src/embedding/provider.js +155 -63
- package/dist/src/engine.d.ts +68 -2
- package/dist/src/engine.js +483 -128
- package/dist/src/indexer/embedding-safe-chunks.js +3 -2
- package/dist/src/model-worker-client.d.ts +4 -0
- package/dist/src/model-worker-client.js +14 -2
- package/dist/src/model-worker.js +34 -3
- package/dist/src/search/tuning.d.ts +2 -0
- package/dist/src/search/tuning.js +2 -0
- package/dist/src/status.d.ts +8 -0
- package/dist/src/status.js +93 -0
- package/docs/configuration.md +16 -11
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,29 @@
|
|
|
2
2
|
|
|
3
3
|
## [Unreleased]
|
|
4
4
|
|
|
5
|
+
## [0.15.0] - 2026-11-15
|
|
6
|
+
|
|
7
|
+
### Added
|
|
8
|
+
- Added deterministic, engine-owned zero-result classifications for `knowledge_search` and `knowledge_symbol_search`, with structured reasons/actions, searched and skipped KB summaries, stage-specific candidate counts, and bounded `auto` attempt details (#24).
|
|
9
|
+
|
|
10
|
+
### Changed
|
|
11
|
+
- Replaced hypothetical zero-result guidance with concise fact-first actions while keeping miss classification free of source diagnostics scans and absolute source-path enumeration.
|
|
12
|
+
- Split search diagnostics into retrieved, score-threshold-passed, eligibility-passed, metadata-passed, and diversity candidate counts so threshold, lexical-evidence, and file/path rejection remain distinct.
|
|
13
|
+
|
|
14
|
+
## [0.14.0] - 2026-11-12
|
|
15
|
+
|
|
16
|
+
### Added
|
|
17
|
+
- Added model-aware embedding input instructions with byte-compatible E5 `on`/`off`, Jina Code task presets, and exact custom query/document instructions (#22).
|
|
18
|
+
- Added instruction-aware embedding signatures and indexed-versus-active compatibility details in `knowledge_status`, `knowledge_doctor`, search warnings, and update progress.
|
|
19
|
+
|
|
20
|
+
### Changed
|
|
21
|
+
- OpenAI-compatible embedding inputs are decorated once and reused for complete-input budget validation and the HTTP request.
|
|
22
|
+
- Local API fallback is rejected during configuration for Jina/custom instructions that the local E5 model cannot reproduce.
|
|
23
|
+
|
|
24
|
+
### Fixed
|
|
25
|
+
- Included embedding incompatibility and invalid active embedding configuration in aggregate status health instead of reporting a contradictory healthy state.
|
|
26
|
+
- Isolated PDF/DOCX parsing from Pi/OMP TUI stdio, suppressed all non-fatal PDF.js warning paths, and cleaned up explicit PDF document proxies without changing extracted text (#23).
|
|
27
|
+
|
|
5
28
|
## [0.13.0] - 2026-11-07
|
|
6
29
|
|
|
7
30
|
### Added
|
package/README.md
CHANGED
|
@@ -217,7 +217,7 @@ suggested reindex action. Source-backed knowledge bases can normally use
|
|
|
217
217
|
| `knowledge_show` | List all knowledge bases with stats |
|
|
218
218
|
| `knowledge_status` | Show engine status with health diagnostics (stale, orphans, coverage) |
|
|
219
219
|
| `knowledge_doctor` | Diagnose health score, skipped files, stuck jobs, stale data, and recommended fixes |
|
|
220
|
-
| `knowledge_configure` | Persist runtime prerequisites such as a Windows `node.exe` path for the isolated
|
|
220
|
+
| `knowledge_configure` | Persist runtime prerequisites such as a Windows `node.exe` path for the isolated runtime worker |
|
|
221
221
|
| `knowledge_clear` | Remove all knowledge bases after `confirm: true` |
|
|
222
222
|
| `knowledge_export` | Export a KB to shareable JSONL file |
|
|
223
223
|
| `knowledge_import` | Import a KB from JSONL (re-embeds content) |
|
|
@@ -242,11 +242,13 @@ Mode selection contract:
|
|
|
242
242
|
- Use `semantic` when the query is conceptual and exact terms may differ from indexed wording; use `semantic_hybrid` when weak lexical overlap should not block vector-backed fused results.
|
|
243
243
|
- Use `adaptive` when the answer needs nearby code, neighboring documentation sections, AST-related sibling methods where indexed, or enough context to make a safe edit.
|
|
244
244
|
- Use `deep` for high-stakes answers, ambiguous top results, or final verification when slower reranking is acceptable.
|
|
245
|
-
- If results are empty
|
|
245
|
+
- If results are empty, follow the returned classified cause and action. `auto` records its attempted modes; do not retry a mode already listed.
|
|
246
246
|
|
|
247
247
|
Search results use balanced diversity reranking by default so near-duplicate chunks from the same file do not dominate the top results. Diversity scoring considers lexical overlap, same-file line proximity, overlapping adaptive windows, available embedding-vector similarity, file-level interleaving, and a small KB trust multiplier that favors ready file/directory sources over stale, URL, or imported text sources. In `adaptive` mode, code chunks with AST metadata can prefer same-parent/sibling chunks inside the same bounded file window; this is metadata-driven context expansion, not caller/callee analysis. Use `diversity: "off"` only when raw ranking order is needed for diagnostics. Agents can request search diagnostics to inspect mode fallback, selected profile, applied search tuning, ranking coverage, path/source/test boosts, adjusted scores, and provenance such as chunk id, chunk hash, match reason, stale flag, source freshness, and adaptive `source_chunk_ids` where available.
|
|
248
248
|
|
|
249
|
-
|
|
249
|
+
Zero-result responses are self-classifying. `knowledge_search` distinguishes missing or unavailable KBs, KBs with no indexed chunks, embedding incompatibility, retrieval misses, score-threshold rejection, lexical-evidence rejection, metadata filtering, empty reranker output, and out-of-range pages. `knowledge_symbol_search` distinguishes missing symbol metadata, exact-name misses, symbol filters, unavailable KBs, and pagination. Both return concise two-line guidance plus a structured `details.miss` object; classification uses request and search-pipeline facts without running a source diagnostics scan.
|
|
250
|
+
|
|
251
|
+
`limit` must be an integer from 1 to 50. `offset` must be a non-negative integer, and `offset + limit` cannot exceed the bounded 800-candidate search window. Each selected KB may contribute candidates up to the profile-derived retrieval limit, but merged candidates are globally ranked and capped before deep reranking, adaptive expansion, or diversity work. Diagnostics expose per-KB retrieval breadth, retrieved, score-threshold-passed, eligibility-passed, metadata-passed, and diversity counts plus whether the global pool was truncated.
|
|
250
252
|
|
|
251
253
|
For best search quality, rebuild or update existing knowledge bases after upgrading. New indexes use contextual retrieval units: embeddings and FTS include file path, file type, Markdown heading breadcrumbs, and AST-derived code structure such as symbol names, scopes, parent symbols, and signatures while returned results keep the original chunk text readable. Before embedding, the complete contextual input is checked against the active provider budget; only oversized chunks are split again along structural boundaries.
|
|
252
254
|
Symbol/config/heading metadata and embedding-safe chunk boundaries are rebuilt during `knowledge_update`. `knowledge_doctor` reports `reindex_for_embedding_budget` for older KBs: file, directory, and URL sources can use `knowledge_update`; imported or inline-text sources should be re-imported or re-added from the original source.
|
|
@@ -274,7 +276,27 @@ export PI_KNOWLEDGE_EMBEDDING_BASE_URL=http://127.0.0.1:8080/v1
|
|
|
274
276
|
# Set PI_KNOWLEDGE_EMBEDDING_API_KEY or OPENAI_API_KEY only if your endpoint requires bearer auth.
|
|
275
277
|
```
|
|
276
278
|
|
|
277
|
-
|
|
279
|
+
Embedding input instructions must match the selected model. `PI_KNOWLEDGE_EMBEDDING_PREFIXES` accepts:
|
|
280
|
+
|
|
281
|
+
| Value | Query/document input | Intended use |
|
|
282
|
+
|-------|----------------------|--------------|
|
|
283
|
+
| `on` or `e5` | `query: ` / `passage: ` | Default local `multilingual-e5-small`; byte-compatible with existing indexes |
|
|
284
|
+
| `off` or `none` | Raw text | Models explicitly evaluated without instructions |
|
|
285
|
+
| `jina` or `jina-code` | Published Jina Code task prompts | `jinaai/jina-code-embeddings-1.5b` through an OpenAI-compatible endpoint |
|
|
286
|
+
| `custom` | Exact configured query/document instructions | Other instructed embedding models |
|
|
287
|
+
|
|
288
|
+
Jina Code example:
|
|
289
|
+
|
|
290
|
+
```bash
|
|
291
|
+
export PI_KNOWLEDGE_EMBEDDING=openai:jinaai/jina-code-embeddings-1.5b
|
|
292
|
+
export PI_KNOWLEDGE_EMBEDDING_BASE_URL=http://127.0.0.1:8080/v1
|
|
293
|
+
export PI_KNOWLEDGE_EMBEDDING_PREFIXES=jina-code
|
|
294
|
+
export PI_KNOWLEDGE_EMBEDDING_TASK=nl2code
|
|
295
|
+
```
|
|
296
|
+
|
|
297
|
+
The Jina preset controls input text only. The embedding server must independently implement the model's required last-token pooling and vector normalization. For custom instructions, set `PI_KNOWLEDGE_EMBEDDING_PREFIXES=custom` plus both `PI_KNOWLEDGE_EMBEDDING_QUERY_INSTRUCTION` and `PI_KNOWLEDGE_EMBEDDING_DOCUMENT_INSTRUCTION`; values are concatenated verbatim, so include delimiters such as a trailing newline.
|
|
298
|
+
|
|
299
|
+
API embedding failures are surfaced by default so configuration and context-window problems are visible. To explicitly allow query-time local fallback after API failures, set `PI_KNOWLEDGE_EMBEDDING_API_FALLBACK=local`; indexing/import/update document batches still fail instead of creating mixed-provider vector files. Local fallback is rejected during configuration when `jina-code` or `custom` instructions are selected because the local E5 model cannot reproduce those inputs. Inputs are never silently truncated.
|
|
278
300
|
|
|
279
301
|
OpenAI-compatible API vectors are validated and L2-normalized by the client before query use or persistence, so cosine-style dot products do not depend on a provider-specific vector magnitude. This normalization policy is part of the embedding signature. KBs created with an older API signature must run `knowledge_update`; `knowledge_doctor` reports `rebuild_vectors` when the active provider no longer matches stored vector semantics.
|
|
280
302
|
|
|
@@ -282,7 +304,7 @@ OpenAI-compatible embedding inputs use a 20000-character complete-input safety c
|
|
|
282
304
|
|
|
283
305
|
For self-hosted servers with smaller request limits, set `PI_KNOWLEDGE_EMBEDDING_BATCH_SIZE` to the maximum number of chunk inputs the server accepts per embedding request.
|
|
284
306
|
|
|
285
|
-
Stored KB metadata includes the embedding model, vector dimension, and a non-secret embedding signature. If the current query embedding is incompatible with a KB's stored vectors, or if an older KB has no signature metadata, `knowledge_search` skips vector retrieval for that KB and reports
|
|
307
|
+
Stored KB metadata includes the embedding model, vector dimension, and a non-secret embedding signature containing the exact instruction strategy/task/hash. If the current query embedding is incompatible with a KB's stored vectors, or if an older KB has no signature metadata, `knowledge_search` skips vector retrieval for that KB and reports the specific mismatch; `knowledge_status` shows indexed versus active model/decoration and includes mismatches in aggregate health. Run `knowledge_update` after changing embedding providers, models, tasks, or instructions.
|
|
286
308
|
|
|
287
309
|
## Configuration
|
|
288
310
|
|
|
@@ -291,8 +313,8 @@ Full configuration details are in [docs/configuration.md](docs/configuration.md)
|
|
|
291
313
|
| Area | Environment variables |
|
|
292
314
|
|------|-----------------------|
|
|
293
315
|
| Storage path | `PI_KNOWLEDGE_DIR`, `OMP_KNOWLEDGE_DIR`, `PI_CODING_AGENT_DIR`, `OMP_CODING_AGENT_DIR`, `OMP_PROFILE` |
|
|
294
|
-
|
|
|
295
|
-
| Embedding provider | `PI_KNOWLEDGE_EMBEDDING`, `PI_KNOWLEDGE_EMBEDDING_API_KEY`, `OPENAI_API_KEY`, `PI_KNOWLEDGE_EMBEDDING_BASE_URL`, `OPENAI_BASE_URL`, `PI_KNOWLEDGE_EMBEDDING_PREFIXES`, `PI_KNOWLEDGE_EMBEDDING_MAX_CHARS`, `PI_KNOWLEDGE_EMBEDDING_BATCH_SIZE`, `PI_KNOWLEDGE_EMBEDDING_API_FALLBACK` |
|
|
316
|
+
| Runtime worker and model cache | `PI_KNOWLEDGE_MODEL_CACHE_DIR`, `PI_KNOWLEDGE_NODE_PATH` |
|
|
317
|
+
| Embedding provider | `PI_KNOWLEDGE_EMBEDDING`, `PI_KNOWLEDGE_EMBEDDING_API_KEY`, `OPENAI_API_KEY`, `PI_KNOWLEDGE_EMBEDDING_BASE_URL`, `OPENAI_BASE_URL`, `PI_KNOWLEDGE_EMBEDDING_PREFIXES`, `PI_KNOWLEDGE_EMBEDDING_TASK`, `PI_KNOWLEDGE_EMBEDDING_QUERY_INSTRUCTION`, `PI_KNOWLEDGE_EMBEDDING_DOCUMENT_INSTRUCTION`, `PI_KNOWLEDGE_EMBEDDING_MAX_CHARS`, `PI_KNOWLEDGE_EMBEDDING_BATCH_SIZE`, `PI_KNOWLEDGE_EMBEDDING_API_FALLBACK` |
|
|
296
318
|
| Reranker provider | `PI_KNOWLEDGE_RERANKER`, `PI_KNOWLEDGE_RERANKER_REVISION`, `PI_KNOWLEDGE_RERANKER_DTYPE`, `PI_KNOWLEDGE_RERANKER_REMOTE_HOST`, `PI_KNOWLEDGE_RERANKER_REMOTE_PATH_TEMPLATE`, `PI_KNOWLEDGE_RERANKER_RAW_LOGITS`, `PI_KNOWLEDGE_RERANKER_API_ENDPOINT`, `PI_KNOWLEDGE_RERANKER_API_BASE_URL`, `PI_KNOWLEDGE_RERANKER_API_KEY`, `PI_KNOWLEDGE_RERANKER_API_FORMAT`, `PI_KNOWLEDGE_RERANKER_API_TIMEOUT_MS`, `PI_KNOWLEDGE_RERANKER_MAX_DOC_CHARS`, `PI_KNOWLEDGE_RERANKER_API_RESULTS_PATH`, `PI_KNOWLEDGE_RERANKER_API_INDEX_FIELD`, `PI_KNOWLEDGE_RERANKER_API_SCORE_FIELD`, `PI_KNOWLEDGE_RERANKER_API_SCORE_DIRECTION` |
|
|
297
319
|
| Native lifecycle | `PI_KNOWLEDGE_ENABLE_NATIVE_IDLE_DISPOSE`, `PI_KNOWLEDGE_EMBEDDING_IDLE_MS` |
|
|
298
320
|
| Runtime features | `PI_KNOWLEDGE_WATCH`, `PI_KNOWLEDGE_AUTO_INJECT`, `PI_KNOWLEDGE_STALE_INDEXING_MS`, `PI_KNOWLEDGE_OFFLINE`, `PI_KNOWLEDGE_SEARCH_PROFILE`, `PI_KNOWLEDGE_SEARCH_DEFAULT_LIMIT`, `PI_KNOWLEDGE_SNIPPET_MAX_LENGTH`, `PI_KNOWLEDGE_MIN_HYBRID_SCORE`, `PI_KNOWLEDGE_SEARCH_CANDIDATE_MIN`, `PI_KNOWLEDGE_SEARCH_CANDIDATE_MULTIPLIER`, `PI_KNOWLEDGE_ADAPTIVE_CONTEXT_LINES`, `PI_KNOWLEDGE_ADAPTIVE_MAX_CHARS`, `PI_KNOWLEDGE_ADAPTIVE_NEIGHBOR_TARGET`, `PI_KNOWLEDGE_DEEP_RERANK_CANDIDATES`, `PI_KNOWLEDGE_DEEP_RERANK_TOPK_MULTIPLIER` |
|
|
@@ -306,7 +328,7 @@ For Hugging Face Text Embeddings Inference rerankers, use `PI_KNOWLEDGE_RERANKER
|
|
|
306
328
|
|
|
307
329
|
Default storage is `~/.pi/knowledge` for Pi and `~/.omp/knowledge` for OMP. Explicit overrides are available with `PI_KNOWLEDGE_DIR` and `OMP_KNOWLEDGE_DIR`. Under the default home OMP root, existing legacy `~/.pi/knowledge` data remains visible when `~/.omp/knowledge` has not been created yet.
|
|
308
330
|
|
|
309
|
-
OMP compatibility covers path resolution, packaged entry loading, native SQLite dependency resolution, isolated
|
|
331
|
+
OMP compatibility covers path resolution, packaged entry loading, native SQLite dependency resolution, isolated worker startup, Windows-safe worker transport fallback, document parser stdio isolation, and idempotent shutdown. Local embeddings and PDF/DOCX extraction require Node 22+ for the isolated worker; set `PI_KNOWLEDGE_NODE_PATH` to `node.exe` if OMP runs from a non-Node packaged host. Compatibility-sensitive releases should validate both Pi and OMP install/runtime flows.
|
|
310
332
|
|
|
311
333
|
## Large Project Indexing
|
|
312
334
|
|
package/dist/index.js
CHANGED
|
@@ -7,6 +7,7 @@ var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExte
|
|
|
7
7
|
return path;
|
|
8
8
|
};
|
|
9
9
|
import { MAX_SEARCH_RESULTS, MAX_SEARCH_WINDOW } from "./src/search/contracts.js";
|
|
10
|
+
import { formatKnowledgeStatus } from "./src/status.js";
|
|
10
11
|
const Type = {
|
|
11
12
|
Object(properties) {
|
|
12
13
|
const required = Object.entries(properties)
|
|
@@ -67,6 +68,155 @@ function scanOptionsFromSourceOptions(raw) {
|
|
|
67
68
|
return {};
|
|
68
69
|
}
|
|
69
70
|
}
|
|
71
|
+
const SEARCH_MISS_ACTION_TEXT = {
|
|
72
|
+
add_knowledge_base: "Use knowledge_add to index a directory, file, URL, or text source.",
|
|
73
|
+
fix_kb_id: "Fix kb_id or omit it to search all knowledge bases.",
|
|
74
|
+
wait_for_indexing: "Retry after indexing completes.",
|
|
75
|
+
fix_knowledge_base: "Run knowledge_doctor and follow the action for the affected knowledge base.",
|
|
76
|
+
fix_embedding: "Run knowledge_doctor to resolve the embedding compatibility problem.",
|
|
77
|
+
try_semantic: 'Try mode "semantic", or rephrase with terms from the target files.',
|
|
78
|
+
try_semantic_hybrid: 'Try mode "semantic_hybrid", or rephrase with terms from the target files.',
|
|
79
|
+
relax_filter: "Relax the file type or path filter.",
|
|
80
|
+
reset_offset: "Retry with offset 0.",
|
|
81
|
+
rephrase: "Rephrase with exact terms from the target files.",
|
|
82
|
+
};
|
|
83
|
+
const SYMBOL_MISS_ACTION_TEXT = {
|
|
84
|
+
add_knowledge_base: "Use knowledge_add to index a directory, file, URL, or text source.",
|
|
85
|
+
fix_kb_id: "Fix kb_id or omit it to search all knowledge bases.",
|
|
86
|
+
wait_for_indexing: "Retry after indexing completes.",
|
|
87
|
+
fix_knowledge_base: "Run knowledge_doctor and follow the action for the affected knowledge base.",
|
|
88
|
+
rebuild_symbol_index: "Run knowledge_update for a source-backed KB, or re-add an inline/imported source to rebuild symbol metadata.",
|
|
89
|
+
broaden_symbol_search: "Retry with exact=false.",
|
|
90
|
+
relax_filter: "Relax the kind or file_pattern filter.",
|
|
91
|
+
reset_offset: "Retry with offset 0.",
|
|
92
|
+
use_text_search: 'Try knowledge_search mode "fast" for source text or "adaptive" for surrounding context.',
|
|
93
|
+
};
|
|
94
|
+
function inlineLabel(value, fallback) {
|
|
95
|
+
const normalized = value
|
|
96
|
+
.replace(/[\p{Cc}\p{Cf}]+/gu, " ")
|
|
97
|
+
.replace(/\s+/g, " ")
|
|
98
|
+
.trim()
|
|
99
|
+
.replaceAll('"', "'");
|
|
100
|
+
return normalized && normalized.length <= 80 ? `"${normalized}"` : fallback;
|
|
101
|
+
}
|
|
102
|
+
function requestedKbSuffix(requestedKbId) {
|
|
103
|
+
return requestedKbId ? ` in ${inlineLabel(requestedKbId, "the requested KB")}` : "";
|
|
104
|
+
}
|
|
105
|
+
function compactFilter(miss) {
|
|
106
|
+
const filters = [];
|
|
107
|
+
if (miss.file_type)
|
|
108
|
+
filters.push(`file_type ${inlineLabel(miss.file_type, "with the requested value")}`);
|
|
109
|
+
if (miss.path_pattern)
|
|
110
|
+
filters.push(`path_pattern ${inlineLabel(miss.path_pattern, "with the requested value")}`);
|
|
111
|
+
return filters.join(" and ") || "the requested metadata filters";
|
|
112
|
+
}
|
|
113
|
+
function formatSearchMiss(miss) {
|
|
114
|
+
const scope = requestedKbSuffix(miss.requested_kb_id);
|
|
115
|
+
const indexedChunks = miss.searched_kbs.reduce((count, kb) => count + kb.chunk_count, 0);
|
|
116
|
+
const attemptedModes = miss.attempts?.map((attempt) => attempt.mode);
|
|
117
|
+
const attemptSuffix = attemptedModes && attemptedModes.length > 1 ? ` Modes tried: ${attemptedModes.join(", ")}.` : "";
|
|
118
|
+
let cause;
|
|
119
|
+
switch (miss.reason) {
|
|
120
|
+
case "no_knowledge_bases":
|
|
121
|
+
cause = "No results: no knowledge bases are indexed.";
|
|
122
|
+
break;
|
|
123
|
+
case "knowledge_base_not_found":
|
|
124
|
+
cause = `No results: knowledge base ${inlineLabel(miss.requested_kb_id ?? "", "from the requested kb_id")} does not exist.`;
|
|
125
|
+
break;
|
|
126
|
+
case "no_searchable_knowledge_bases": {
|
|
127
|
+
const indexing = miss.skipped_kbs.find((kb) => kb.status === "indexing");
|
|
128
|
+
const errored = miss.skipped_kbs.find((kb) => kb.status === "error");
|
|
129
|
+
if (miss.action === "wait_for_indexing" && indexing) {
|
|
130
|
+
cause = `No results yet: ${inlineLabel(indexing.name, "the selected KB")} is still indexing (${indexing.chunk_count} chunks persisted).`;
|
|
131
|
+
}
|
|
132
|
+
else if (errored) {
|
|
133
|
+
cause = `No results: ${inlineLabel(errored.name, "the selected KB")} is in error state and is not searchable.`;
|
|
134
|
+
}
|
|
135
|
+
else if (indexing) {
|
|
136
|
+
cause = `No results: one or more selected knowledge bases are unavailable while ${inlineLabel(indexing.name, "one KB")} indexes.`;
|
|
137
|
+
}
|
|
138
|
+
else {
|
|
139
|
+
cause = "No results: no selected knowledge base is currently searchable.";
|
|
140
|
+
}
|
|
141
|
+
break;
|
|
142
|
+
}
|
|
143
|
+
case "no_indexed_chunks":
|
|
144
|
+
cause = "No results: the searchable knowledge bases contain no indexed chunks.";
|
|
145
|
+
break;
|
|
146
|
+
case "embedding_incompatible":
|
|
147
|
+
cause = "No results: the selected knowledge bases are incompatible with the active embedding configuration.";
|
|
148
|
+
break;
|
|
149
|
+
case "no_retrieval_candidates":
|
|
150
|
+
cause = `No results: no retrieval candidates matched${scope} (${indexedChunks} indexed chunks).${attemptSuffix}`;
|
|
151
|
+
break;
|
|
152
|
+
case "threshold_filtered": {
|
|
153
|
+
const threshold = miss.min_hybrid_score === undefined ? "the score threshold" : `the ${miss.min_hybrid_score} score threshold`;
|
|
154
|
+
cause = `No results: ${miss.retrieved_candidates} candidates were retrieved, but none passed ${threshold}.${attemptSuffix}`;
|
|
155
|
+
break;
|
|
156
|
+
}
|
|
157
|
+
case "evidence_filtered":
|
|
158
|
+
cause = `No results: ${miss.threshold_candidates} candidates passed the score threshold, but none had enough lexical evidence.${attemptSuffix}`;
|
|
159
|
+
break;
|
|
160
|
+
case "metadata_filtered":
|
|
161
|
+
cause = `No results: ${miss.eligible_candidates} eligible candidates matched the query, but none matched ${compactFilter(miss)}.`;
|
|
162
|
+
break;
|
|
163
|
+
case "reranker_empty":
|
|
164
|
+
cause = `No results: ${miss.filtered_candidates} candidates reached deep reranking, but the reranker returned none.`;
|
|
165
|
+
break;
|
|
166
|
+
case "page_out_of_range":
|
|
167
|
+
cause = `No results on this page: the requested offset exceeds ${miss.total_results} available results.`;
|
|
168
|
+
break;
|
|
169
|
+
case "no_match":
|
|
170
|
+
cause = `No results: candidates were processed but no result remained${scope}.${attemptSuffix}`;
|
|
171
|
+
break;
|
|
172
|
+
}
|
|
173
|
+
return `${cause}\n${SEARCH_MISS_ACTION_TEXT[miss.action]}`;
|
|
174
|
+
}
|
|
175
|
+
function formatSymbolMiss(miss) {
|
|
176
|
+
const scope = requestedKbSuffix(miss.requested_kb_id);
|
|
177
|
+
let cause;
|
|
178
|
+
switch (miss.reason) {
|
|
179
|
+
case "no_knowledge_bases":
|
|
180
|
+
cause = "No symbols found: no knowledge bases are indexed.";
|
|
181
|
+
break;
|
|
182
|
+
case "knowledge_base_not_found":
|
|
183
|
+
cause = `No symbols found: knowledge base ${inlineLabel(miss.requested_kb_id ?? "", "from the requested kb_id")} does not exist.`;
|
|
184
|
+
break;
|
|
185
|
+
case "no_searchable_knowledge_bases": {
|
|
186
|
+
const indexing = miss.skipped_kbs.find((kb) => kb.status === "indexing");
|
|
187
|
+
const errored = miss.skipped_kbs.find((kb) => kb.status === "error");
|
|
188
|
+
if (miss.action === "wait_for_indexing" && indexing) {
|
|
189
|
+
cause = `No symbols yet: ${inlineLabel(indexing.name, "the selected KB")} is still indexing (${indexing.chunk_count} chunks persisted).`;
|
|
190
|
+
}
|
|
191
|
+
else if (errored) {
|
|
192
|
+
cause = `No symbols found: ${inlineLabel(errored.name, "the selected KB")} is in error state and is not searchable.`;
|
|
193
|
+
}
|
|
194
|
+
else if (indexing) {
|
|
195
|
+
cause = `No symbols found: one or more selected knowledge bases are unavailable while ${inlineLabel(indexing.name, "one KB")} indexes.`;
|
|
196
|
+
}
|
|
197
|
+
else {
|
|
198
|
+
cause = "No symbols found: no selected knowledge base is currently searchable.";
|
|
199
|
+
}
|
|
200
|
+
break;
|
|
201
|
+
}
|
|
202
|
+
case "symbol_index_empty":
|
|
203
|
+
cause = `No symbols are indexed${scope}.`;
|
|
204
|
+
break;
|
|
205
|
+
case "exact_name_not_found":
|
|
206
|
+
cause = `No symbol matched the exact normalized name${scope}.`;
|
|
207
|
+
break;
|
|
208
|
+
case "metadata_filtered":
|
|
209
|
+
cause = `No symbols matched the requested kind or file_pattern${scope}; ${miss.unfiltered_matches} unfiltered matches exist.`;
|
|
210
|
+
break;
|
|
211
|
+
case "page_out_of_range":
|
|
212
|
+
cause = "No symbols on this page: the requested offset exceeds the available matches.";
|
|
213
|
+
break;
|
|
214
|
+
case "no_match":
|
|
215
|
+
cause = `No indexed symbol matched${scope}.`;
|
|
216
|
+
break;
|
|
217
|
+
}
|
|
218
|
+
return `${cause}\n${SYMBOL_MISS_ACTION_TEXT[miss.action]}`;
|
|
219
|
+
}
|
|
70
220
|
function runtimeModule(modulePath) {
|
|
71
221
|
return `${modulePath}${RUNTIME_EXTENSION}`;
|
|
72
222
|
}
|
|
@@ -253,7 +403,7 @@ export default function (pi) {
|
|
|
253
403
|
pi.registerTool({
|
|
254
404
|
name: "knowledge_configure",
|
|
255
405
|
label: "Knowledge Configure",
|
|
256
|
-
description: "Configure pi-knowledge runtime settings such as the
|
|
406
|
+
description: "Configure pi-knowledge runtime settings such as the isolated worker Node executable",
|
|
257
407
|
approval: "write",
|
|
258
408
|
promptSnippet: "Configure local pi-knowledge runtime prerequisites before indexing",
|
|
259
409
|
promptGuidelines: [
|
|
@@ -264,7 +414,7 @@ export default function (pi) {
|
|
|
264
414
|
],
|
|
265
415
|
parameters: Type.Object({
|
|
266
416
|
node_path: Type.Optional(Type.String({
|
|
267
|
-
description: "Optional full path to a Node 22+ executable for the isolated local
|
|
417
|
+
description: "Optional full path to a Node 22+ executable for the isolated local model and document extraction worker, such as C:\\Program Files\\nodejs\\node.exe",
|
|
268
418
|
})),
|
|
269
419
|
}),
|
|
270
420
|
async execute(_id, params, _signal) {
|
|
@@ -279,7 +429,7 @@ export default function (pi) {
|
|
|
279
429
|
content: [
|
|
280
430
|
{
|
|
281
431
|
type: "text",
|
|
282
|
-
text: `Configured
|
|
432
|
+
text: `Configured isolated worker Node executable: ${configuredPath}`,
|
|
283
433
|
},
|
|
284
434
|
],
|
|
285
435
|
details: { node_path: configuredPath },
|
|
@@ -369,7 +519,7 @@ export default function (pi) {
|
|
|
369
519
|
"Use mode 'adaptive' when the user needs surrounding implementation context, related nearby sections, or enough context to make a code change",
|
|
370
520
|
"Use mode 'deep' for high-stakes answers, ambiguous top results, or final verification where slower cross-encoder reranking is acceptable",
|
|
371
521
|
"Use profile 'low_token' for slow local models that need fewer, stricter, longer snippets; use profile 'precision' for identifiers/errors and 'recall' or 'long_context' for broad prose/document research",
|
|
372
|
-
"If a search returns no results
|
|
522
|
+
"If a search returns no results, follow its classified action; do not repeat modes listed in the auto attempt summary",
|
|
373
523
|
"If top results are repetitive, retry with diversity 'strong' or mode 'adaptive' instead of increasing the limit first",
|
|
374
524
|
],
|
|
375
525
|
parameters: Type.Object({
|
|
@@ -422,19 +572,9 @@ export default function (pi) {
|
|
|
422
572
|
const filters = file_type || path_pattern ? { file_type, path_pattern } : undefined;
|
|
423
573
|
const response = await engine.search(query, { mode, profile, limit, kb_id, offset, filters, diversity }, _signal);
|
|
424
574
|
if (response.results.length === 0) {
|
|
425
|
-
|
|
426
|
-
"
|
|
427
|
-
|
|
428
|
-
response.retry_modes?.length ? `Retried: ${response.retry_modes.join(", ")}` : "",
|
|
429
|
-
response.warnings?.length ? `Warnings: ${response.warnings.join(" | ")}` : "",
|
|
430
|
-
diagnostics && response.tuning
|
|
431
|
-
? `Tuning: profile=${response.tuning.requested_profile}->${response.tuning.selected_profile}, limit=${response.tuning.limit}, snippet=${response.tuning.snippet_max_length}, min_hybrid=${response.tuning.min_hybrid_score}, candidates=${response.tuning.diversity_candidates}/${response.tuning.candidate_limit}, retrieval_per_kb=${response.tuning.retrieval_limit_per_kb}, retrieved=${response.tuning.retrieved_candidates}, filtered=${response.tuning.filtered_candidates}, truncated=${response.tuning.candidate_pool_truncated}`
|
|
432
|
-
: "",
|
|
433
|
-
response.suggestions?.length ? `Suggestions: ${response.suggestions.join(" | ")}` : "",
|
|
434
|
-
]
|
|
435
|
-
.filter(Boolean)
|
|
436
|
-
.join("\n");
|
|
437
|
-
return { content: [{ type: "text", text: details }], details: diagnostics ? response : undefined };
|
|
575
|
+
if (!response.miss)
|
|
576
|
+
throw new Error("Search returned no results without a miss classification");
|
|
577
|
+
return { content: [{ type: "text", text: formatSearchMiss(response.miss) }], details: response };
|
|
438
578
|
}
|
|
439
579
|
let output = `${response.total_count} results (showing ${response.results.length})`;
|
|
440
580
|
if (response.mode_used)
|
|
@@ -457,7 +597,7 @@ export default function (pi) {
|
|
|
457
597
|
? `\nProvenance: chunk=${r.provenance.chunk_id}, hash=${r.provenance.chunk_hash.slice(0, 12)}, reason=${r.provenance.match_reason}, indexed_at=${r.provenance.indexed_at}, source_mtime=${r.provenance.source_mtime ?? "unknown"}, stale=${r.provenance.stale}`
|
|
458
598
|
: "";
|
|
459
599
|
const tuning = diagnostics && response.tuning && i === 0
|
|
460
|
-
? `\nTuning: limit=${response.tuning.limit}, snippet=${response.tuning.snippet_max_length}, min_hybrid=${response.tuning.min_hybrid_score}, candidates=${response.tuning.diversity_candidates}/${response.tuning.candidate_limit}, retrieval_per_kb=${response.tuning.retrieval_limit_per_kb}, retrieved=${response.tuning.retrieved_candidates},
|
|
600
|
+
? `\nTuning: limit=${response.tuning.limit}, snippet=${response.tuning.snippet_max_length}, min_hybrid=${response.tuning.min_hybrid_score}, candidates=${response.tuning.diversity_candidates}/${response.tuning.candidate_limit}, retrieval_per_kb=${response.tuning.retrieval_limit_per_kb}, retrieved=${response.tuning.retrieved_candidates}, threshold=${response.tuning.threshold_candidates}, eligible=${response.tuning.eligible_candidates}, metadata=${response.tuning.filtered_candidates}, truncated=${response.tuning.candidate_pool_truncated}`
|
|
461
601
|
: "";
|
|
462
602
|
return `[${i + 1}] ${r.file_path} (${r.kb_name}, score: ${r.score.toFixed(3)})\n${r.snippet}${diag}${provenance}${tuning}`;
|
|
463
603
|
})
|
|
@@ -474,7 +614,7 @@ export default function (pi) {
|
|
|
474
614
|
promptGuidelines: [
|
|
475
615
|
"Use knowledge_symbol_search before knowledge_search when looking for a function, class, env var, config key, route, or Markdown heading",
|
|
476
616
|
"Use exact=true when the user provides a precise symbol or key",
|
|
477
|
-
"This is a lightweight declaration-pattern index, not full LSP;
|
|
617
|
+
"This is a lightweight declaration-pattern index, not full LSP; on a miss, follow its classified action and use knowledge_search mode 'fast' or 'adaptive' only when directed to source text",
|
|
478
618
|
"After finding a symbol, use knowledge_search mode 'adaptive' with the file path or symbol name when surrounding implementation context is needed",
|
|
479
619
|
],
|
|
480
620
|
parameters: Type.Object({
|
|
@@ -511,14 +651,9 @@ export default function (pi) {
|
|
|
511
651
|
exact: params.exact === true,
|
|
512
652
|
}, _signal);
|
|
513
653
|
if (response.results.length === 0) {
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
type: "text",
|
|
518
|
-
text: "No symbols found. This may mean the symbol is absent, symbol metadata is missing/stale, or the lightweight extractor does not cover this syntax. For important lookups, run knowledge_doctor or fallback to knowledge_search mode 'fast'/'adaptive'.",
|
|
519
|
-
},
|
|
520
|
-
],
|
|
521
|
-
};
|
|
654
|
+
if (!response.miss)
|
|
655
|
+
throw new Error("Symbol search returned no results without a miss classification");
|
|
656
|
+
return { content: [{ type: "text", text: formatSymbolMiss(response.miss) }], details: response };
|
|
522
657
|
}
|
|
523
658
|
const lines = response.results.map((result, index) => {
|
|
524
659
|
const signature = result.signature ? ` — ${result.signature}` : "";
|
|
@@ -557,69 +692,26 @@ export default function (pi) {
|
|
|
557
692
|
pi.registerTool({
|
|
558
693
|
name: "knowledge_status",
|
|
559
694
|
label: "Knowledge Status",
|
|
560
|
-
description: "Show knowledge engine status with
|
|
695
|
+
description: "Show knowledge engine status with embedding compatibility, staleness, orphan, and coverage diagnostics",
|
|
561
696
|
approval: "read",
|
|
562
697
|
parameters: Type.Object({}),
|
|
563
698
|
async execute(_id, _params, _signal) {
|
|
564
699
|
if (_signal?.aborted)
|
|
565
700
|
throw new Error("Cancelled");
|
|
566
701
|
const { engine, storage, watcher } = await ensureInitialized();
|
|
567
|
-
const
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
if (kb.source_path)
|
|
581
|
-
lines.push(` source: ${kb.source_path}`);
|
|
582
|
-
if (diag) {
|
|
583
|
-
if (diag.job) {
|
|
584
|
-
const jobAge = Math.round((Date.now() - diag.job.last_progress_at) / 1000);
|
|
585
|
-
const progress = [
|
|
586
|
-
`${diag.job.processed_files} files`,
|
|
587
|
-
`${diag.job.processed_chunks} chunks`,
|
|
588
|
-
diag.job.skipped_total > 0 ? `${diag.job.skipped_total} skipped` : "",
|
|
589
|
-
diag.job.added_chunks > 0 ? `+${diag.job.added_chunks}` : "",
|
|
590
|
-
diag.job.removed_chunks > 0 ? `-${diag.job.removed_chunks}` : "",
|
|
591
|
-
diag.job.unchanged_chunks > 0 ? `=${diag.job.unchanged_chunks}` : "",
|
|
592
|
-
]
|
|
593
|
-
.filter(Boolean)
|
|
594
|
-
.join(", ");
|
|
595
|
-
lines.push(` job: ${diag.job.status}/${diag.job.phase} — ${progress || "no processed items yet"} — last progress ${jobAge}s ago`);
|
|
596
|
-
if (diag.job.message)
|
|
597
|
-
lines.push(` last: ${diag.job.message}`);
|
|
598
|
-
if (diag.job.error_message)
|
|
599
|
-
lines.push(` error: ${diag.job.error_message}`);
|
|
600
|
-
}
|
|
601
|
-
lines.push(` coverage: ${diag.coverage_percent}% (${diag.indexed_files}/${diag.total_source_files} files)`);
|
|
602
|
-
if (diag.stale_files.length > 0)
|
|
603
|
-
lines.push(` ⚠️ stale: ${diag.stale_files.length} files modified since last index`);
|
|
604
|
-
if (diag.orphan_files.length > 0)
|
|
605
|
-
lines.push(` ⚠️ orphans: ${diag.orphan_files.length} chunks reference deleted files`);
|
|
606
|
-
if (diag.stuck_indexing)
|
|
607
|
-
lines.push(` ⚠️ indexing appears stuck for ${Math.round(diag.last_progress_age_ms / 60000)}m; remove and rebuild if no pi process is actively indexing it`);
|
|
608
|
-
if (diag.skipped_files.total > 0)
|
|
609
|
-
lines.push(` skipped: ${diag.skipped_files.total} files (${Object.entries(diag.skipped_files.by_reason)
|
|
610
|
-
.filter(([, count]) => count > 0)
|
|
611
|
-
.map(([reason, count]) => `${reason}: ${count}`)
|
|
612
|
-
.join(", ")})`);
|
|
613
|
-
}
|
|
614
|
-
}
|
|
615
|
-
const totalStale = diagnostics.reduce((n, d) => n + d.stale_files.length, 0);
|
|
616
|
-
const totalOrphans = diagnostics.reduce((n, d) => n + d.orphan_files.length, 0);
|
|
617
|
-
const totalStuck = diagnostics.filter((d) => d.stuck_indexing).length;
|
|
618
|
-
if (totalStale === 0 && totalOrphans === 0 && totalStuck === 0 && kbs.length > 0)
|
|
619
|
-
lines.push("", "Health: ✓ all indexes up to date");
|
|
620
|
-
else if (totalStale > 0 || totalOrphans > 0 || totalStuck > 0)
|
|
621
|
-
lines.push("", `Health: ⚠️ ${totalStale} stale, ${totalOrphans} orphans, ${totalStuck} stuck indexing — run knowledge_update or rebuild affected KBs`);
|
|
622
|
-
return { content: [{ type: "text", text: lines.join("\n") }] };
|
|
702
|
+
const snapshot = engine.statusSnapshot(_signal);
|
|
703
|
+
return {
|
|
704
|
+
content: [
|
|
705
|
+
{
|
|
706
|
+
type: "text",
|
|
707
|
+
text: formatKnowledgeStatus({
|
|
708
|
+
snapshot,
|
|
709
|
+
storagePath: storage.getDefaultKnowledgeDir(),
|
|
710
|
+
watchCount: watcher.getActiveWatcherCount(),
|
|
711
|
+
}),
|
|
712
|
+
},
|
|
713
|
+
],
|
|
714
|
+
};
|
|
623
715
|
},
|
|
624
716
|
});
|
|
625
717
|
pi.registerTool({
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
export type EmbeddingRole = "query" | "passage";
|
|
2
|
+
export type EmbeddingDecorationStrategy = "e5" | "none" | "jina-code" | "custom";
|
|
3
|
+
export type EmbeddingTask = "nl2code" | "code2code" | "code2nl" | "code2completion" | "qa";
|
|
4
|
+
export type EmbeddingPrefixMode = "on" | "off";
|
|
5
|
+
export interface EmbeddingInstructionSet {
|
|
6
|
+
query: string;
|
|
7
|
+
document: string;
|
|
8
|
+
}
|
|
9
|
+
export interface ResolvedEmbeddingDecoration {
|
|
10
|
+
strategy: EmbeddingDecorationStrategy;
|
|
11
|
+
task?: EmbeddingTask;
|
|
12
|
+
instructions: EmbeddingInstructionSet;
|
|
13
|
+
prefixMode: EmbeddingPrefixMode;
|
|
14
|
+
}
|
|
15
|
+
export declare const MAX_INSTRUCTION_CHARS = 512;
|
|
16
|
+
export declare const DEFAULT_EMBEDDING_TASK: EmbeddingTask;
|
|
17
|
+
export declare const ACCEPTED_DECORATION_VALUES: readonly ["on", "off", "e5", "none", "jina", "jina-code", "custom"];
|
|
18
|
+
export declare const ACCEPTED_EMBEDDING_TASKS: readonly ["nl2code", "code2code", "code2nl", "code2completion", "qa"];
|
|
19
|
+
export declare const E5_INSTRUCTIONS: EmbeddingInstructionSet;
|
|
20
|
+
export declare const NO_INSTRUCTIONS: EmbeddingInstructionSet;
|
|
21
|
+
/**
|
|
22
|
+
* Published prompts for jinaai/jina-code-embeddings-1.5b.
|
|
23
|
+
* Source: config_sentence_transformers.json at model revision
|
|
24
|
+
* 39aeb4fb9b60f930934c78ae5d749a46287c248a (verified 2026-11-12).
|
|
25
|
+
* Values already contain their required trailing newline and are concatenated verbatim.
|
|
26
|
+
*/
|
|
27
|
+
export declare const JINA_CODE_INSTRUCTIONS: Readonly<Record<EmbeddingTask, EmbeddingInstructionSet>>;
|
|
28
|
+
export declare function decorateEmbeddingInput(text: string, role: EmbeddingRole, instructions: EmbeddingInstructionSet): string;
|
|
29
|
+
export declare function embeddingInstructionLength(instructions: EmbeddingInstructionSet, role: EmbeddingRole): number;
|
|
30
|
+
export declare function embeddingInstructionSetHash(instructions: EmbeddingInstructionSet): string;
|
|
31
|
+
export declare function embeddingDecorationDescriptor(decoration: ResolvedEmbeddingDecoration): string;
|
|
32
|
+
export declare function resolveEmbeddingDecoration(env: NodeJS.ProcessEnv): ResolvedEmbeddingDecoration;
|
|
33
|
+
export declare function decorationSupportedByLocalModel(strategy: EmbeddingDecorationStrategy): boolean;
|