pi-knowledge 0.12.0 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,183 @@
1
+ import { setImmediate as yieldToEventLoop } from "node:timers/promises";
2
+ import { tokenizeForSearch } from "./query.js";
3
+ const VECTOR_REDUNDANCY_WEIGHT = 0.35;
4
+ const PAIR_CHECK_INTERVAL = 1_024;
5
+ const CPU_SLICE_MS = 8;
6
+ function throwIfAborted(signal) {
7
+ if (signal?.aborted)
8
+ throw new Error("Cancelled");
9
+ }
10
+ function cosineSimilarity(a, b) {
11
+ if (!a || !b || a.length !== b.length)
12
+ return 0;
13
+ let dot = 0;
14
+ for (let i = 0; i < a.length; i++)
15
+ dot += a[i] * b[i];
16
+ return dot;
17
+ }
18
+ function jaccardSimilarity(a, b) {
19
+ if (a.size === 0 || b.size === 0)
20
+ return 0;
21
+ const [small, large] = a.size <= b.size ? [a, b] : [b, a];
22
+ let intersection = 0;
23
+ for (const token of small) {
24
+ if (large.has(token))
25
+ intersection++;
26
+ }
27
+ return intersection / (a.size + b.size - intersection);
28
+ }
29
+ function proximityMeta(candidate) {
30
+ return {
31
+ filePath: candidate.chunk.file_path,
32
+ kbId: candidate.chunk.kb_id,
33
+ startLine: candidate.startLine,
34
+ endLine: candidate.endLine,
35
+ sourceIds: new Set(candidate.sourceChunkIds),
36
+ };
37
+ }
38
+ function lineProximity(a, b) {
39
+ if (a.filePath !== b.filePath || a.kbId !== b.kbId)
40
+ return 0;
41
+ const [small, large] = a.sourceIds.size <= b.sourceIds.size ? [a.sourceIds, b.sourceIds] : [b.sourceIds, a.sourceIds];
42
+ for (const id of small) {
43
+ if (large.has(id))
44
+ return 1;
45
+ }
46
+ const overlap = Math.max(0, Math.min(a.endLine, b.endLine) - Math.max(a.startLine, b.startLine) + 1);
47
+ if (overlap > 0)
48
+ return 1;
49
+ const gap = Math.max(a.startLine, b.startLine) - Math.min(a.endLine, b.endLine);
50
+ if (gap <= 20)
51
+ return 0.8;
52
+ if (gap <= 80)
53
+ return 0.45;
54
+ return 0.2;
55
+ }
56
+ function normalizeScores(candidates) {
57
+ let min = Number.POSITIVE_INFINITY;
58
+ let max = Number.NEGATIVE_INFINITY;
59
+ for (const candidate of candidates) {
60
+ if (candidate.score < min)
61
+ min = candidate.score;
62
+ if (candidate.score > max)
63
+ max = candidate.score;
64
+ }
65
+ const normalized = new Float64Array(candidates.length);
66
+ for (let i = 0; i < candidates.length; i++) {
67
+ normalized[i] = max === min ? 1 : (candidates[i].score - min) / (max - min);
68
+ }
69
+ return normalized;
70
+ }
71
+ function firstUnusedIndex(used) {
72
+ for (let i = 0; i < used.length; i++) {
73
+ if (used[i] === 0)
74
+ return i;
75
+ }
76
+ throw new Error("Diversity selection exhausted its candidate pool unexpectedly");
77
+ }
78
+ export async function diversifyRankedChunks(candidates, diversity, vectorsByChunkId = new Map(), signal) {
79
+ throwIfAborted(signal);
80
+ if (diversity === "off" || candidates.length <= 2)
81
+ return candidates;
82
+ const lambda = diversity === "strong" ? 0.62 : 0.76;
83
+ const n = candidates.length;
84
+ const tokenSets = new Array(n);
85
+ const vectors = new Array(n);
86
+ const metas = new Array(n);
87
+ const relevance = normalizeScores(candidates);
88
+ for (let i = 0; i < n; i++) {
89
+ const candidate = candidates[i];
90
+ tokenSets[i] = tokenizeForSearch(candidate.content);
91
+ vectors[i] = vectorsByChunkId.get(candidate.chunk.id);
92
+ metas[i] = proximityMeta(candidate);
93
+ }
94
+ const pairRedundancy = (a, b) => {
95
+ let best = lineProximity(metas[a], metas[b]);
96
+ const lexical = jaccardSimilarity(tokenSets[a], tokenSets[b]);
97
+ if (lexical > best)
98
+ best = lexical;
99
+ const va = vectors[a];
100
+ const vb = vectors[b];
101
+ if (va !== undefined && vb !== undefined) {
102
+ const vector = Math.max(0, cosineSimilarity(va, vb)) * VECTOR_REDUNDANCY_WEIGHT;
103
+ if (vector > best)
104
+ best = vector;
105
+ }
106
+ return best;
107
+ };
108
+ const redundancy = new Float64Array(n);
109
+ const used = new Uint8Array(n);
110
+ const selected = new Array(n);
111
+ let pairsSinceCheck = 0;
112
+ let sliceStartedAt = performance.now();
113
+ for (let round = 0; round < n; round++) {
114
+ throwIfAborted(signal);
115
+ let bestIndex = firstUnusedIndex(used);
116
+ let bestScore = Number.NEGATIVE_INFINITY;
117
+ for (let i = 0; i < n; i++) {
118
+ if (used[i] === 1)
119
+ continue;
120
+ const mmrScore = lambda * relevance[i] - (1 - lambda) * redundancy[i];
121
+ if (mmrScore > bestScore) {
122
+ bestScore = mmrScore;
123
+ bestIndex = i;
124
+ }
125
+ }
126
+ used[bestIndex] = 1;
127
+ selected[round] = candidates[bestIndex];
128
+ for (let i = 0; i < n; i++) {
129
+ if (used[i] === 1)
130
+ continue;
131
+ const pair = pairRedundancy(bestIndex, i);
132
+ if (pair > redundancy[i])
133
+ redundancy[i] = pair;
134
+ pairsSinceCheck++;
135
+ if (pairsSinceCheck < PAIR_CHECK_INTERVAL)
136
+ continue;
137
+ pairsSinceCheck = 0;
138
+ throwIfAborted(signal);
139
+ if (performance.now() - sliceStartedAt < CPU_SLICE_MS)
140
+ continue;
141
+ await yieldToEventLoop();
142
+ throwIfAborted(signal);
143
+ sliceStartedAt = performance.now();
144
+ }
145
+ }
146
+ return selected;
147
+ }
148
+ export function interleaveByFile(candidates, diversity, signal) {
149
+ throwIfAborted(signal);
150
+ if (diversity === "off" || candidates.length <= 2)
151
+ return candidates;
152
+ const buckets = new Map();
153
+ const fileOrder = [];
154
+ for (const candidate of candidates) {
155
+ throwIfAborted(signal);
156
+ const key = `${candidate.chunk.kb_id}:${candidate.chunk.file_path}`;
157
+ let bucket = buckets.get(key);
158
+ if (!bucket) {
159
+ bucket = [];
160
+ buckets.set(key, bucket);
161
+ fileOrder.push(key);
162
+ }
163
+ bucket.push(candidate);
164
+ }
165
+ if (fileOrder.length <= 1)
166
+ return candidates;
167
+ const result = [];
168
+ let round = 0;
169
+ while (result.length < candidates.length) {
170
+ let added = false;
171
+ for (const key of fileOrder) {
172
+ const item = buckets.get(key)?.[round];
173
+ if (!item)
174
+ continue;
175
+ result.push(item);
176
+ added = true;
177
+ }
178
+ if (!added)
179
+ break;
180
+ round++;
181
+ }
182
+ return result;
183
+ }
@@ -22,6 +22,11 @@ export interface SearchTuningSummary {
22
22
  snippet_max_length: number;
23
23
  min_hybrid_score: number;
24
24
  candidate_limit: number;
25
+ retrieval_limit_per_kb: number;
26
+ retrieved_candidates: number;
27
+ filtered_candidates: number;
28
+ diversity_candidates: number;
29
+ candidate_pool_truncated: boolean;
25
30
  candidate_min: number;
26
31
  candidate_multiplier: number;
27
32
  adaptive_context_lines: number;
@@ -36,6 +41,11 @@ interface SearchTuningInput {
36
41
  profile?: SearchProfile;
37
42
  kbSourceTypes: string[];
38
43
  }
44
+ export interface SearchWindow {
45
+ limit: number;
46
+ offset: number;
47
+ }
48
+ export declare function resolveSearchWindow(limit: number | undefined, offset: number | undefined, defaultLimit: number): SearchWindow;
39
49
  export declare function resolveSearchTuning(input: SearchTuningInput): SearchTuning;
40
50
  export declare function summarizeSearchTuning(tuning: SearchTuning, limit: number, candidateLimit: number): SearchTuningSummary;
41
51
  export {};
@@ -1,3 +1,4 @@
1
+ import { MAX_SEARCH_RESULTS, MAX_SEARCH_WINDOW } from "./contracts.js";
1
2
  import { MIN_HYBRID_SCORE } from "./ranking.js";
2
3
  const PROFILE_DEFAULTS = {
3
4
  balanced: {
@@ -152,6 +153,20 @@ function chooseRuntimeProfile(input) {
152
153
  return "long_context";
153
154
  return "balanced";
154
155
  }
156
+ export function resolveSearchWindow(limit, offset, defaultLimit) {
157
+ const resolvedLimit = limit ?? defaultLimit;
158
+ if (!Number.isInteger(resolvedLimit) || resolvedLimit < 1 || resolvedLimit > MAX_SEARCH_RESULTS) {
159
+ throw new Error(`Search limit must be an integer from 1 to ${MAX_SEARCH_RESULTS}`);
160
+ }
161
+ const resolvedOffset = offset ?? 0;
162
+ if (!Number.isInteger(resolvedOffset) || resolvedOffset < 0) {
163
+ throw new Error("Search offset must be a non-negative integer");
164
+ }
165
+ if (resolvedOffset + resolvedLimit > MAX_SEARCH_WINDOW) {
166
+ throw new Error(`Search offset + limit must not exceed ${MAX_SEARCH_WINDOW}; narrow the query or use a more specific knowledge base`);
167
+ }
168
+ return { limit: resolvedLimit, offset: resolvedOffset };
169
+ }
155
170
  export function resolveSearchTuning(input) {
156
171
  const requestedProfile = input.profile ?? parseProfile(process.env.PI_KNOWLEDGE_SEARCH_PROFILE) ?? "auto";
157
172
  const selectedProfile = requestedProfile === "auto" ? chooseRuntimeProfile(input) : requestedProfile;
@@ -179,6 +194,11 @@ export function summarizeSearchTuning(tuning, limit, candidateLimit) {
179
194
  snippet_max_length: tuning.snippetMaxLength,
180
195
  min_hybrid_score: tuning.minHybridScore,
181
196
  candidate_limit: candidateLimit,
197
+ retrieval_limit_per_kb: candidateLimit,
198
+ retrieved_candidates: 0,
199
+ filtered_candidates: 0,
200
+ diversity_candidates: 0,
201
+ candidate_pool_truncated: false,
182
202
  candidate_min: tuning.candidateMin,
183
203
  candidate_multiplier: tuning.candidateMultiplier,
184
204
  adaptive_context_lines: tuning.adaptiveContextLines,
@@ -0,0 +1,8 @@
1
+ import type { KnowledgeStatusSnapshot } from "./engine.ts";
2
+ export interface KnowledgeStatusFormatOptions {
3
+ snapshot: KnowledgeStatusSnapshot;
4
+ storagePath: string;
5
+ watchCount: number;
6
+ now?: number;
7
+ }
8
+ export declare function formatKnowledgeStatus(options: KnowledgeStatusFormatOptions): string;
@@ -0,0 +1,93 @@
1
+ export function formatKnowledgeStatus(options) {
2
+ const { snapshot, storagePath, watchCount } = options;
3
+ const now = options.now ?? Date.now();
4
+ const embeddingByKb = new Map(snapshot.embeddings.map((embedding) => [embedding.kb_id, embedding]));
5
+ const diagnosticsByKb = new Map(snapshot.diagnostics.map((diagnostic) => [diagnostic.kb_id, diagnostic]));
6
+ const lines = [
7
+ `Storage: ${storagePath}`,
8
+ `Knowledge bases: ${snapshot.knowledge_bases.length}`,
9
+ `Active watchers: ${watchCount}`,
10
+ ];
11
+ if (snapshot.embedding_error)
12
+ lines.push(`Embedding configuration: ⚠️ ${snapshot.embedding_error}`);
13
+ lines.push("");
14
+ for (const kb of snapshot.knowledge_bases) {
15
+ const age = Math.round((now - kb.updated_at) / 60_000);
16
+ const diagnostic = diagnosticsByKb.get(kb.id);
17
+ const embedding = embeddingByKb.get(kb.id);
18
+ lines.push(` "${kb.name}" — ${kb.status} — ${kb.chunk_count} chunks, ${kb.file_count} files — updated ${age}m ago`);
19
+ if (kb.source_path)
20
+ lines.push(` source: ${kb.source_path}`);
21
+ if (embedding) {
22
+ lines.push(` embedding: indexed=${embedding.indexed_model ?? "unrecorded"} · active=${embedding.active_model} · decoration=${embedding.active_decoration}`);
23
+ const requiresCompatibleVectors = kb.chunk_count > 0 && (kb.status === "ready" || kb.status === "stale");
24
+ if (requiresCompatibleVectors && !embedding.compatibility.compatible) {
25
+ lines.push(` ⚠️ vectors: ${embedding.detail}`);
26
+ }
27
+ }
28
+ else {
29
+ lines.push(` embedding: indexed=${kb.embedding_model ?? "unrecorded"} · active=unavailable`);
30
+ }
31
+ if (!diagnostic)
32
+ continue;
33
+ if (diagnostic.job) {
34
+ const jobAge = Math.round((now - diagnostic.job.last_progress_at) / 1_000);
35
+ const progress = [
36
+ `${diagnostic.job.processed_files} files`,
37
+ `${diagnostic.job.processed_chunks} chunks`,
38
+ diagnostic.job.skipped_total > 0 ? `${diagnostic.job.skipped_total} skipped` : "",
39
+ diagnostic.job.added_chunks > 0 ? `+${diagnostic.job.added_chunks}` : "",
40
+ diagnostic.job.removed_chunks > 0 ? `-${diagnostic.job.removed_chunks}` : "",
41
+ diagnostic.job.unchanged_chunks > 0 ? `=${diagnostic.job.unchanged_chunks}` : "",
42
+ ]
43
+ .filter(Boolean)
44
+ .join(", ");
45
+ lines.push(` job: ${diagnostic.job.status}/${diagnostic.job.phase} — ${progress || "no processed items yet"} — last progress ${jobAge}s ago`);
46
+ if (diagnostic.job.message)
47
+ lines.push(` last: ${diagnostic.job.message}`);
48
+ if (diagnostic.job.error_message)
49
+ lines.push(` error: ${diagnostic.job.error_message}`);
50
+ }
51
+ lines.push(` coverage: ${diagnostic.coverage_percent}% (${diagnostic.indexed_files}/${diagnostic.total_source_files} files)`);
52
+ if (diagnostic.stale_files.length > 0)
53
+ lines.push(` ⚠️ stale: ${diagnostic.stale_files.length} files modified since last index`);
54
+ if (diagnostic.orphan_files.length > 0)
55
+ lines.push(` ⚠️ orphans: ${diagnostic.orphan_files.length} chunks reference deleted files`);
56
+ if (diagnostic.stuck_indexing)
57
+ lines.push(` ⚠️ indexing appears stuck for ${Math.round(diagnostic.last_progress_age_ms / 60_000)}m; remove and rebuild if no pi process is actively indexing it`);
58
+ if (diagnostic.skipped_files.total > 0)
59
+ lines.push(` skipped: ${diagnostic.skipped_files.total} files (${Object.entries(diagnostic.skipped_files.by_reason)
60
+ .filter(([, count]) => count > 0)
61
+ .map(([reason, count]) => `${reason}: ${count}`)
62
+ .join(", ")})`);
63
+ }
64
+ const totalStale = snapshot.diagnostics.reduce((count, diagnostic) => count + diagnostic.stale_files.length, 0);
65
+ const totalOrphans = snapshot.diagnostics.reduce((count, diagnostic) => count + diagnostic.orphan_files.length, 0);
66
+ const totalStuck = snapshot.diagnostics.filter((diagnostic) => diagnostic.stuck_indexing).length;
67
+ const totalIndexing = snapshot.knowledge_bases.filter((kb) => kb.status === "indexing" && !diagnosticsByKb.get(kb.id)?.stuck_indexing).length;
68
+ const totalErrors = snapshot.knowledge_bases.filter((kb) => kb.status === "error").length;
69
+ const totalEmbedding = snapshot.knowledge_bases.reduce((count, kb) => {
70
+ if (kb.chunk_count === 0 || (kb.status !== "ready" && kb.status !== "stale"))
71
+ return count;
72
+ const embedding = embeddingByKb.get(kb.id);
73
+ return count + (embedding && !embedding.compatibility.compatible ? 1 : 0);
74
+ }, 0) + (snapshot.embedding_error ? 1 : 0);
75
+ if (totalStale === 0 &&
76
+ totalOrphans === 0 &&
77
+ totalStuck === 0 &&
78
+ totalIndexing === 0 &&
79
+ totalErrors === 0 &&
80
+ totalEmbedding === 0 &&
81
+ snapshot.knowledge_bases.length > 0) {
82
+ lines.push("", "Health: ✓ all indexes up to date and embedding-compatible");
83
+ }
84
+ else if (totalStale > 0 ||
85
+ totalOrphans > 0 ||
86
+ totalStuck > 0 ||
87
+ totalIndexing > 0 ||
88
+ totalErrors > 0 ||
89
+ totalEmbedding > 0) {
90
+ lines.push("", `Health: ⚠️ ${totalStale} stale, ${totalOrphans} orphans, ${totalStuck} stuck indexing, ${totalIndexing} indexing, ${totalErrors} errors, ${totalEmbedding} embedding issues — run knowledge_update, correct embedding configuration, or rebuild affected KBs`);
91
+ }
92
+ return lines.join("\n");
93
+ }
@@ -12,7 +12,7 @@
12
12
  | `OMP_CODING_AGENT_DIR` | unset | Derive the OMP host root from a configured agent directory. Used when `PI_CODING_AGENT_DIR` is not set. |
13
13
  | `OMP_PROFILE` | unset | Treat the current process as OMP-hosted for default path selection. |
14
14
  | `PI_KNOWLEDGE_MODEL_CACHE_DIR` | `<knowledge-dir>/models` | Override the local Transformers.js model cache directory. |
15
- | `PI_KNOWLEDGE_NODE_PATH` | current Node when possible, then `NODE`, then `node` on `PATH` | Override the Node 22+ executable used to run the isolated model worker. Useful for Windows OMP packaged runtimes where the host process is not Node. |
15
+ | `PI_KNOWLEDGE_NODE_PATH` | current Node when possible, then `NODE`, then `node` on `PATH` | Override the Node 22+ executable used by the isolated runtime worker for local models and PDF/DOCX extraction. Useful for Windows OMP packaged runtimes where the host process is not Node. |
16
16
 
17
17
  Default data storage is `~/.pi/knowledge` under Pi and `~/.omp/knowledge` under OMP. For the default home OMP root, `pi-knowledge` preserves an existing legacy `~/.pi/knowledge` directory when `~/.omp/knowledge` does not exist, so existing Pi knowledge bases remain visible during migration.
18
18
 
@@ -25,15 +25,18 @@ Default data storage is `~/.pi/knowledge` under Pi and `~/.omp/knowledge` under
25
25
  | `OPENAI_API_KEY` | unset | Fallback API key used when `PI_KNOWLEDGE_EMBEDDING=openai:<model>`. Required by OpenAI's hosted API; many local OpenAI-compatible servers do not need it. |
26
26
  | `PI_KNOWLEDGE_EMBEDDING_BASE_URL` | unset | OpenAI-compatible embedding API root, such as `http://127.0.0.1:8080/v1`. Takes precedence over `OPENAI_BASE_URL`. |
27
27
  | `OPENAI_BASE_URL` | `https://api.openai.com/v1` | Common OpenAI-compatible API root fallback. |
28
- | `PI_KNOWLEDGE_EMBEDDING_PREFIXES` | `on` | Controls whether `query: ` and `passage: ` prefixes are prepended before embedding. Values: `on`, `off`. Keep `on` for the default E5 local model; use `off` only for models or self-hosted APIs that are evaluated without E5-style prefixes. |
29
- | `PI_KNOWLEDGE_EMBEDDING_MAX_CHARS` | `20000` | Final per-input API embedding safety cap for OpenAI-compatible servers with smaller context windows. This does not replace chunker bounds. |
28
+ | `PI_KNOWLEDGE_EMBEDDING_PREFIXES` | `on` | Embedding input decoration strategy. `on`/`e5` use `query: ` and `passage: `; `off`/`none` use raw text; `jina`/`jina-code` use published Jina Code prompts; `custom` uses the two instruction variables below. |
29
+ | `PI_KNOWLEDGE_EMBEDDING_TASK` | `nl2code` for `jina-code` | Jina Code prompt task: `nl2code`, `code2code`, `code2nl`, `code2completion`, or `qa`. Rejected for other strategies. |
30
+ | `PI_KNOWLEDGE_EMBEDDING_QUERY_INSTRUCTION` | unset | Exact query instruction for `custom`. Required and non-empty; maximum 512 characters. Whitespace is preserved. |
31
+ | `PI_KNOWLEDGE_EMBEDDING_DOCUMENT_INSTRUCTION` | unset | Exact document instruction for `custom`. Required; may be an explicit empty string; maximum 512 characters. Whitespace is preserved. |
32
+ | `PI_KNOWLEDGE_EMBEDDING_MAX_CHARS` | `20000` | Final complete-input API embedding safety cap after instructions are added. Configuration fails if an instruction alone leaves no room for text. This does not replace chunker bounds. |
30
33
  | `PI_KNOWLEDGE_EMBEDDING_BATCH_SIZE` | `64` | Maximum number of document chunks sent in one embedding request during add, update, and import. Lower this for self-hosted embedding servers with smaller request batch limits. |
31
- | `PI_KNOWLEDGE_EMBEDDING_API_FALLBACK` | unset | Set to `local` to explicitly fall back to local embeddings for query embeddings after API failures. Indexing/import/update document batches do not fall back because mixed-provider vector files are unsafe. |
34
+ | `PI_KNOWLEDGE_EMBEDDING_API_FALLBACK` | unset | Set to `local` to explicitly fall back to local embeddings for query embeddings after API failures. Indexing/import/update document batches do not fall back. `jina-code` and `custom` reject local fallback during configuration because the local E5 model cannot reproduce those inputs. |
32
35
  | `PI_KNOWLEDGE_ENABLE_NATIVE_IDLE_DISPOSE` | unset | Enables idle timers for embedding/reranker run coordination. It does not guarantee immediate worker memory release; model-worker shutdown remains tied to engine/session shutdown. Disabled by default for stable shutdown. |
33
36
  | `PI_KNOWLEDGE_EMBEDDING_IDLE_MS` | `30000` | Idle timer used only when native idle disposal coordination is enabled. Mainly for lifecycle stress tests. |
34
37
  | `PI_KNOWLEDGE_OFFLINE` | unset | Use with a pre-populated model cache for offline local model operation. See `docs/offline-mode.md`. |
35
38
 
36
- Each KB stores the embedding model label, vector dimension, and a non-secret embedding signature that includes provider/model semantics, query/document prefixes, non-default prefix strategy, pooling, normalization, API input cap, and a hash of the API base URL when applicable. `knowledge_search` skips vector retrieval for KBs whose stored signature or dimension is missing or incompatible with the current query embedding and uses BM25-only fallback for hybrid searches. Run `knowledge_update` to rebuild vectors after changing embedding provider/model/max-character/prefix settings or after upgrading older KBs with no signature metadata.
39
+ Each KB stores the embedding model label, vector dimension, and a non-secret embedding signature that includes provider/model semantics, the input decoration descriptor, pooling, normalization policy, API input cap, and a hash of the API base URL when applicable. E5 `on` and raw-text `off` descriptors remain byte-compatible with existing signatures; Jina signatures include the task and instruction hash, while custom signatures contain only an instruction hash rather than the raw instruction text. OpenAI-compatible response vectors are validated and L2-normalized by the client before use or persistence; the `client-l2-v1` policy is part of API signatures. `knowledge_search` skips vector retrieval for missing or incompatible signatures/dimensions and uses BM25-only fallback for hybrid searches. `knowledge_status` reports indexed and active embedding settings and counts incompatibility in aggregate health. `knowledge_doctor` reports configuration failures and rebuild actions. Run `knowledge_update` after changing provider, model, endpoint, input cap, task, or instructions; re-add inline text or re-import portable sources that cannot be updated.
37
40
 
38
41
  ## Reranker
39
42
 
@@ -60,9 +63,9 @@ Reranker settings affect query-time `deep` search only; they do not change index
60
63
 
61
64
  For Hugging Face Text Embeddings Inference rerankers, set `PI_KNOWLEDGE_RERANKER_API_FORMAT=tei` and point `PI_KNOWLEDGE_RERANKER_API_ENDPOINT` at the TEI `/rerank` endpoint, for example `http://127.0.0.1:8080/rerank`. Do not reuse an embeddings `OPENAI_BASE_URL` ending in `/v1` unless the reranker endpoint actually exists there; TEI's OpenAI-compatible route covers embeddings, while reranking uses `/rerank`.
62
65
 
63
- Local embeddings require a real Node 22+ executable for the isolated model worker. `pi-knowledge` first tries Node IPC and automatically falls back to a stdin/stdout JSONL worker transport when the host runtime does not expose `child_process.fork().send()`, which can happen in Windows OMP compatibility layers. On Windows it searches `PI_KNOWLEDGE_NODE_PATH`, persisted `knowledge_configure` settings, common Node, Volta, NVM, Codex `cua_node`, and `PATH` locations before failing. If worker startup still fails, call `knowledge_configure` with the full `node.exe` path such as `C:\Program Files\nodejs\node.exe`; accidental surrounding quotes are ignored. OpenAI-compatible embeddings avoid the local worker entirely.
66
+ Local embeddings, local/HF reranking, and PDF/DOCX extraction require a real Node 22+ executable for the isolated worker. `pi-knowledge` first tries Node IPC and automatically falls back to a stdin/stdout JSONL worker transport when the host runtime does not expose `child_process.fork().send()`, which can happen in Windows OMP compatibility layers. On Windows it searches `PI_KNOWLEDGE_NODE_PATH`, persisted `knowledge_configure` settings, common Node, Volta, NVM, Codex `cua_node`, and `PATH` locations before failing. If worker startup still fails, call `knowledge_configure` with the full `node.exe` path such as `C:\Program Files\nodejs\node.exe`; accidental surrounding quotes are ignored. OpenAI-compatible embeddings avoid local model loading, but document extraction still uses the worker to isolate parser stdio from the host.
64
67
 
65
- API embedding failures intentionally surface by default. Silent fallback can hide bad API keys, wrong base URLs, unsupported model names, context-window errors, or server batch limits and can produce a KB with a different embedding model than intended. Query-time fallback is opt-in; indexing fallback is disabled to prevent mixed vector spaces in one KB. OpenAI-compatible embedding requests send `Authorization: Bearer <key>` only when `PI_KNOWLEDGE_EMBEDDING_API_KEY` or `OPENAI_API_KEY` is configured.
68
+ API embedding failures intentionally surface by default. Silent fallback can hide bad API keys, wrong base URLs, unsupported model names, context-window errors, server batch limits, or instruction mismatches and can produce a KB with a different embedding model than intended. Query-time fallback is opt-in and supports only E5 `on`/`off` semantics; indexing fallback is disabled to prevent mixed vector spaces in one KB. OpenAI-compatible embedding requests send `Authorization: Bearer <key>` only when `PI_KNOWLEDGE_EMBEDDING_API_KEY` or `OPENAI_API_KEY` is configured. Instruction decoration is applied once before both safety-cap validation and the HTTP request.
66
69
 
67
70
  ## Runtime Features and Diagnostics
68
71
 
@@ -83,11 +86,11 @@ API embedding failures intentionally surface by default. Silent fallback can hid
83
86
  | `PI_KNOWLEDGE_DEEP_RERANK_CANDIDATES` | profile default | Override how many filtered candidates `deep` mode sends to the reranker. Clamped to 5-100. |
84
87
  | `PI_KNOWLEDGE_DEEP_RERANK_TOPK_MULTIPLIER` | profile default | Override deep rerank top-k multiplier relative to the effective result limit. Clamped to 1-10. |
85
88
 
86
- Search diagnostics include result provenance and applied search tuning in both text output and structured `details`: selected profile, effective limit, snippet length, hybrid threshold, candidate pool size, chunk id, chunk hash, match reason, indexed timestamp, stale flag, and source mtime when the original source file is available. This provenance is stored in SQLite and derived from indexed source metadata; it does not create or modify files in the indexed project.
89
+ Search diagnostics include result provenance and applied search tuning in both text output and structured `details`: selected profile, effective limit, snippet length, hybrid threshold, global candidate limit, per-KB retrieval limit, retrieved/filtered/diversity candidate counts, global-pool truncation, chunk id, chunk hash, match reason, indexed timestamp, stale flag, and source mtime when the original source file is available. This provenance is stored in SQLite and derived from indexed source metadata; it does not create or modify files in the indexed project.
87
90
 
88
- `knowledge_search` supports `file_type`, `path_pattern`, `profile`, and explicit `limit` filters. Search profiles tune result count, snippets, hybrid strictness, candidate breadth, adaptive context, and deep rerank breadth at runtime. `auto` chooses from query shape, mode, and KB source type; explicit tool parameters win over env overrides, env overrides win over profile defaults, and profile defaults win over built-in compatibility values. `hybrid` mode remains lexical-anchored to prevent low-evidence semantic false positives; use `semantic` for pure vector search, or `semantic_hybrid` when weak lexical overlap should not block vector-backed weighted fusion. `adaptive` uses bounded same-file context expansion; when AST metadata is present it can prefer same-parent/sibling chunks, but it remains metadata-driven context expansion rather than caller/callee analysis.
91
+ `knowledge_search` supports `file_type`, substring `path_pattern`, `profile`, integer `limit` from 1 to 50, and non-negative integer `offset`. `offset + limit` cannot exceed the 800-candidate global search window. Each KB retrieves up to the profile-derived candidate limit, then merged results are ranked, filtered, and globally capped before deep reranking, adaptive expansion, or diversity. Search profiles tune result count, snippets, hybrid strictness, candidate breadth, adaptive context, and deep rerank breadth at runtime. `auto` chooses from query shape, mode, and KB source type; explicit tool parameters win over env overrides, env overrides win over profile defaults, and profile defaults win over built-in compatibility values. `hybrid` mode remains lexical-anchored to prevent low-evidence semantic false positives; use `semantic` for pure vector search, or `semantic_hybrid` when weak lexical overlap should not block vector-backed weighted fusion. `adaptive` uses bounded same-file context expansion; when AST metadata is present it can prefer same-parent/sibling chunks without claiming caller/callee graph semantics.
89
92
 
90
- `knowledge_doctor` emits both human-readable issues and machine-readable actions in structured `details`. Action codes include `run_update`, `rebuild_kb`, `wait_for_indexing`, `review_skipped_scope`, `rebuild_vectors`, `check_source`, and `none`. Agents should prefer these action codes over parsing the English action text.
93
+ `knowledge_doctor` emits both human-readable issues and machine-readable actions in structured `details`. Action codes include `run_update`, `rebuild_kb`, `wait_for_indexing`, `review_skipped_scope`, `rebuild_vectors`, `reindex_for_embedding_budget`, `fix_embedding_config`, `check_source`, and `none`. Agents should prefer these action codes over parsing the English action text.
91
94
 
92
95
  `knowledge_configure` writes runtime configuration to the knowledge directory's `config.json`. It validates Node 22+ before persisting `node_path`, so agents can configure a Codex- or user-installed `node.exe` once without relying on environment variables being injected into an already-running OMP process.
93
96
 
@@ -119,7 +122,7 @@ Compatibility guarantees:
119
122
 
120
123
  - The packaged `extension.js` loads built `dist/index.js` when present and falls back to source `index.ts` for local development.
121
124
  - Runtime modules are imported lazily, after the extension is actually used.
122
- - Local embedding and reranking models run in an isolated Node model worker, not in the Pi or OMP TUI process. Node IPC is preferred; stdin/stdout JSONL transport is used automatically when host IPC is unavailable.
125
+ - Local embedding/reranking models and PDF/DOCX parsers run in an isolated Node worker, not in the Pi or OMP TUI process. Node IPC is preferred; stdin/stdout JSONL transport is used automatically when host IPC is unavailable.
123
126
  - Native SQLite loading includes a fallback for hoisted plugin dependency layouts.
124
127
  - OMP path resolution can use `OMP_KNOWLEDGE_DIR`, `OMP_CODING_AGENT_DIR`, and `OMP_PROFILE`.
125
128
  - Existing default Pi knowledge data can remain visible under the default OMP home root through the legacy `~/.pi/knowledge` fallback described above.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "pi-knowledge",
3
- "version": "0.12.0",
3
+ "version": "0.14.0",
4
4
  "description": "Local-first RAG knowledge base for Pi and OMP coding agents with hybrid code/doc search, reranking, diagnostics, and large-project indexing",
5
5
  "type": "module",
6
6
  "main": "dist/index.js",