pi-knowledge 0.13.0 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,3 +1,4 @@
1
+ import { embeddingInstructionLength } from "../embedding/instructions.js";
1
2
  import { LOCAL_EMBEDDING_PREFERRED_TOKENS, measureEmbeddingInputs } from "../embedding/provider.js";
2
3
  import { buildChunkEmbeddingText, chunkIdentityHash, preTokenizeForFTS } from "./chunker.js";
3
4
  const SENTENCE_BOUNDARIES = {
@@ -153,8 +154,8 @@ function withPartMetadata(chunk, part, partCount) {
153
154
  function isProvablySafeLocalAsciiInput(text, config) {
154
155
  if (config.provider !== "local")
155
156
  return false;
156
- const prefixLength = config.prefixes === "on" ? "passage: ".length : 0;
157
- if (text.length + prefixLength + 2 > LOCAL_EMBEDDING_PREFERRED_TOKENS)
157
+ const instructionLength = embeddingInstructionLength(config.instructions, "passage");
158
+ if (text.length + instructionLength + 2 > LOCAL_EMBEDDING_PREFERRED_TOKENS)
158
159
  return false;
159
160
  for (let index = 0; index < text.length; index++) {
160
161
  if (text.charCodeAt(index) > 0x7f)
@@ -7,6 +7,10 @@ export declare function measureEmbeddingInputInModelWorker(texts: string[], pref
7
7
  counts: number[];
8
8
  maximum: number;
9
9
  }>;
10
+ export declare function extractDocumentInModelWorker(filePath: string, fileType: "pdf" | "docx", signal?: AbortSignal): Promise<{
11
+ content: string;
12
+ fileType: "pdf" | "docx";
13
+ }>;
10
14
  export interface RerankWorkerCandidate {
11
15
  chunkId: string;
12
16
  content: string;
@@ -39,7 +39,7 @@ function validateNodeExecPath(candidate) {
39
39
  }).trim();
40
40
  const major = parseNodeMajor(version);
41
41
  if (major === null || major < MIN_NODE_MAJOR) {
42
- throw new Error(`Node ${MIN_NODE_MAJOR}+ is required for the model worker; ${candidate} reported ${version}`);
42
+ throw new Error(`Node ${MIN_NODE_MAJOR}+ is required for the isolated runtime worker; ${candidate} reported ${version}`);
43
43
  }
44
44
  return candidate;
45
45
  }
@@ -149,7 +149,7 @@ function getNodeExecPath() {
149
149
  // Try the next candidate before surfacing a deterministic error below.
150
150
  }
151
151
  }
152
- throw new Error(`Node ${MIN_NODE_MAJOR}+ is required for local pi-knowledge embeddings, but no usable node executable was found. Install Node ${MIN_NODE_MAJOR}+ or set PI_KNOWLEDGE_NODE_PATH to the full node.exe path.`);
152
+ throw new Error(`Node ${MIN_NODE_MAJOR}+ is required for isolated pi-knowledge worker operations such as local models and PDF/DOCX extraction, but no usable node executable was found. Install Node ${MIN_NODE_MAJOR}+ or set PI_KNOWLEDGE_NODE_PATH to the full node.exe path.`);
153
153
  }
154
154
  export function validateModelWorkerNodePath(nodePath) {
155
155
  return validateNodeExecPath(normalizeExecutableCandidate(nodePath));
@@ -413,6 +413,18 @@ export async function measureEmbeddingInputInModelWorker(texts, prefix, prefixes
413
413
  }
414
414
  return { counts: result.counts, maximum: result.maximum };
415
415
  }
416
+ export async function extractDocumentInModelWorker(filePath, fileType, signal) {
417
+ const result = await getWorker().request({ type: "extract_document", filePath, fileType }, signal);
418
+ if (typeof result !== "object" ||
419
+ result === null ||
420
+ !("content" in result) ||
421
+ typeof result.content !== "string" ||
422
+ !("fileType" in result) ||
423
+ result.fileType !== fileType) {
424
+ throw new Error("Invalid document extraction worker response");
425
+ }
426
+ return { content: result.content, fileType };
427
+ }
416
428
  export async function rerankInModelWorker(query, candidates, topK, reranker, signal) {
417
429
  const result = await getWorker().request({ type: "rerank", query, candidates, topK, reranker }, signal);
418
430
  if (!Array.isArray(result))
@@ -1,5 +1,7 @@
1
+ import { readFile } from "node:fs/promises";
1
2
  import { join } from "node:path";
2
3
  import { createInterface } from "node:readline";
4
+ import { decorateEmbeddingInput, E5_INSTRUCTIONS, NO_INSTRUCTIONS, } from "./embedding/instructions.js";
3
5
  import { DEFAULT_RERANKER_REMOTE_HOST, DEFAULT_RERANKER_REMOTE_PATH_TEMPLATE, isHfRerankerConfig, rerankerCacheKey, resolveRerankerConfig, } from "./search/reranker-config.js";
4
6
  import { rerankWithSingleRawLogit } from "./search/reranker-logits.js";
5
7
  import { getDefaultKnowledgeDir } from "./storage/sqlite.js";
@@ -76,8 +78,8 @@ async function loadRerankerPipeline(config) {
76
78
  rerankerPipeline = { key, pipe: loaded };
77
79
  return loaded;
78
80
  }
79
- function applyEmbeddingPrefix(text, prefix, prefixes) {
80
- return prefixes === "on" ? `${prefix}: ${text}` : text;
81
+ function applyEmbeddingPrefix(text, role, prefixes) {
82
+ return decorateEmbeddingInput(text, role, prefixes === "on" ? E5_INSTRUCTIONS : NO_INSTRUCTIONS);
81
83
  }
82
84
  async function handleEmbed(request) {
83
85
  const pipe = await loadEmbeddingPipeline();
@@ -130,6 +132,24 @@ async function handleRerank(request) {
130
132
  results.sort((a, b) => b.score - a.score);
131
133
  return results.slice(0, request.topK);
132
134
  }
135
+ const PDFJS_ERRORS_ONLY_VERBOSITY = 0;
136
+ async function handleExtractDocument(request) {
137
+ if (request.fileType === "pdf") {
138
+ const { extractText, getDocumentProxy } = await import("unpdf");
139
+ const bytes = await readFile(request.filePath);
140
+ const pdf = await getDocumentProxy(new Uint8Array(bytes), { verbosity: PDFJS_ERRORS_ONLY_VERBOSITY });
141
+ try {
142
+ const { text } = await extractText(pdf);
143
+ return { content: Array.isArray(text) ? text.join("\n\n") : text, fileType: "pdf" };
144
+ }
145
+ finally {
146
+ await pdf.destroy();
147
+ }
148
+ }
149
+ const mammoth = await import("mammoth");
150
+ const result = await mammoth.extractRawText({ path: request.filePath });
151
+ return { content: result.value, fileType: "docx" };
152
+ }
133
153
  function isStringArray(value) {
134
154
  return Array.isArray(value) && value.every((item) => typeof item === "string");
135
155
  }
@@ -166,6 +186,15 @@ function parseWorkerRequest(message) {
166
186
  prefixes,
167
187
  };
168
188
  }
189
+ if (message.type === "extract_document") {
190
+ if (!("filePath" in message) || typeof message.filePath !== "string") {
191
+ throw new Error("Invalid document file path");
192
+ }
193
+ if (!("fileType" in message) || (message.fileType !== "pdf" && message.fileType !== "docx")) {
194
+ throw new Error("Invalid document file type");
195
+ }
196
+ return { id: message.id, type: "extract_document", filePath: message.filePath, fileType: message.fileType };
197
+ }
169
198
  if (message.type === "rerank") {
170
199
  if (!("query" in message) || typeof message.query !== "string")
171
200
  throw new Error("Invalid rerank query");
@@ -199,7 +228,9 @@ async function handleRequestMessage(message) {
199
228
  ? await handleEmbed(request)
200
229
  : request.type === "measure_embedding_input"
201
230
  ? await handleMeasureEmbeddingInput(request)
202
- : await handleRerank(request);
231
+ : request.type === "extract_document"
232
+ ? await handleExtractDocument(request)
233
+ : await handleRerank(request);
203
234
  return { id: request.id, result };
204
235
  }
205
236
  catch (error) {
@@ -24,6 +24,8 @@ export interface SearchTuningSummary {
24
24
  candidate_limit: number;
25
25
  retrieval_limit_per_kb: number;
26
26
  retrieved_candidates: number;
27
+ threshold_candidates: number;
28
+ eligible_candidates: number;
27
29
  filtered_candidates: number;
28
30
  diversity_candidates: number;
29
31
  candidate_pool_truncated: boolean;
@@ -196,6 +196,8 @@ export function summarizeSearchTuning(tuning, limit, candidateLimit) {
196
196
  candidate_limit: candidateLimit,
197
197
  retrieval_limit_per_kb: candidateLimit,
198
198
  retrieved_candidates: 0,
199
+ threshold_candidates: 0,
200
+ eligible_candidates: 0,
199
201
  filtered_candidates: 0,
200
202
  diversity_candidates: 0,
201
203
  candidate_pool_truncated: false,
@@ -0,0 +1,8 @@
1
+ import type { KnowledgeStatusSnapshot } from "./engine.ts";
2
+ export interface KnowledgeStatusFormatOptions {
3
+ snapshot: KnowledgeStatusSnapshot;
4
+ storagePath: string;
5
+ watchCount: number;
6
+ now?: number;
7
+ }
8
+ export declare function formatKnowledgeStatus(options: KnowledgeStatusFormatOptions): string;
@@ -0,0 +1,93 @@
1
+ export function formatKnowledgeStatus(options) {
2
+ const { snapshot, storagePath, watchCount } = options;
3
+ const now = options.now ?? Date.now();
4
+ const embeddingByKb = new Map(snapshot.embeddings.map((embedding) => [embedding.kb_id, embedding]));
5
+ const diagnosticsByKb = new Map(snapshot.diagnostics.map((diagnostic) => [diagnostic.kb_id, diagnostic]));
6
+ const lines = [
7
+ `Storage: ${storagePath}`,
8
+ `Knowledge bases: ${snapshot.knowledge_bases.length}`,
9
+ `Active watchers: ${watchCount}`,
10
+ ];
11
+ if (snapshot.embedding_error)
12
+ lines.push(`Embedding configuration: ⚠️ ${snapshot.embedding_error}`);
13
+ lines.push("");
14
+ for (const kb of snapshot.knowledge_bases) {
15
+ const age = Math.round((now - kb.updated_at) / 60_000);
16
+ const diagnostic = diagnosticsByKb.get(kb.id);
17
+ const embedding = embeddingByKb.get(kb.id);
18
+ lines.push(` "${kb.name}" — ${kb.status} — ${kb.chunk_count} chunks, ${kb.file_count} files — updated ${age}m ago`);
19
+ if (kb.source_path)
20
+ lines.push(` source: ${kb.source_path}`);
21
+ if (embedding) {
22
+ lines.push(` embedding: indexed=${embedding.indexed_model ?? "unrecorded"} · active=${embedding.active_model} · decoration=${embedding.active_decoration}`);
23
+ const requiresCompatibleVectors = kb.chunk_count > 0 && (kb.status === "ready" || kb.status === "stale");
24
+ if (requiresCompatibleVectors && !embedding.compatibility.compatible) {
25
+ lines.push(` ⚠️ vectors: ${embedding.detail}`);
26
+ }
27
+ }
28
+ else {
29
+ lines.push(` embedding: indexed=${kb.embedding_model ?? "unrecorded"} · active=unavailable`);
30
+ }
31
+ if (!diagnostic)
32
+ continue;
33
+ if (diagnostic.job) {
34
+ const jobAge = Math.round((now - diagnostic.job.last_progress_at) / 1_000);
35
+ const progress = [
36
+ `${diagnostic.job.processed_files} files`,
37
+ `${diagnostic.job.processed_chunks} chunks`,
38
+ diagnostic.job.skipped_total > 0 ? `${diagnostic.job.skipped_total} skipped` : "",
39
+ diagnostic.job.added_chunks > 0 ? `+${diagnostic.job.added_chunks}` : "",
40
+ diagnostic.job.removed_chunks > 0 ? `-${diagnostic.job.removed_chunks}` : "",
41
+ diagnostic.job.unchanged_chunks > 0 ? `=${diagnostic.job.unchanged_chunks}` : "",
42
+ ]
43
+ .filter(Boolean)
44
+ .join(", ");
45
+ lines.push(` job: ${diagnostic.job.status}/${diagnostic.job.phase} — ${progress || "no processed items yet"} — last progress ${jobAge}s ago`);
46
+ if (diagnostic.job.message)
47
+ lines.push(` last: ${diagnostic.job.message}`);
48
+ if (diagnostic.job.error_message)
49
+ lines.push(` error: ${diagnostic.job.error_message}`);
50
+ }
51
+ lines.push(` coverage: ${diagnostic.coverage_percent}% (${diagnostic.indexed_files}/${diagnostic.total_source_files} files)`);
52
+ if (diagnostic.stale_files.length > 0)
53
+ lines.push(` ⚠️ stale: ${diagnostic.stale_files.length} files modified since last index`);
54
+ if (diagnostic.orphan_files.length > 0)
55
+ lines.push(` ⚠️ orphans: ${diagnostic.orphan_files.length} chunks reference deleted files`);
56
+ if (diagnostic.stuck_indexing)
57
+ lines.push(` ⚠️ indexing appears stuck for ${Math.round(diagnostic.last_progress_age_ms / 60_000)}m; remove and rebuild if no pi process is actively indexing it`);
58
+ if (diagnostic.skipped_files.total > 0)
59
+ lines.push(` skipped: ${diagnostic.skipped_files.total} files (${Object.entries(diagnostic.skipped_files.by_reason)
60
+ .filter(([, count]) => count > 0)
61
+ .map(([reason, count]) => `${reason}: ${count}`)
62
+ .join(", ")})`);
63
+ }
64
+ const totalStale = snapshot.diagnostics.reduce((count, diagnostic) => count + diagnostic.stale_files.length, 0);
65
+ const totalOrphans = snapshot.diagnostics.reduce((count, diagnostic) => count + diagnostic.orphan_files.length, 0);
66
+ const totalStuck = snapshot.diagnostics.filter((diagnostic) => diagnostic.stuck_indexing).length;
67
+ const totalIndexing = snapshot.knowledge_bases.filter((kb) => kb.status === "indexing" && !diagnosticsByKb.get(kb.id)?.stuck_indexing).length;
68
+ const totalErrors = snapshot.knowledge_bases.filter((kb) => kb.status === "error").length;
69
+ const totalEmbedding = snapshot.knowledge_bases.reduce((count, kb) => {
70
+ if (kb.chunk_count === 0 || (kb.status !== "ready" && kb.status !== "stale"))
71
+ return count;
72
+ const embedding = embeddingByKb.get(kb.id);
73
+ return count + (embedding && !embedding.compatibility.compatible ? 1 : 0);
74
+ }, 0) + (snapshot.embedding_error ? 1 : 0);
75
+ if (totalStale === 0 &&
76
+ totalOrphans === 0 &&
77
+ totalStuck === 0 &&
78
+ totalIndexing === 0 &&
79
+ totalErrors === 0 &&
80
+ totalEmbedding === 0 &&
81
+ snapshot.knowledge_bases.length > 0) {
82
+ lines.push("", "Health: ✓ all indexes up to date and embedding-compatible");
83
+ }
84
+ else if (totalStale > 0 ||
85
+ totalOrphans > 0 ||
86
+ totalStuck > 0 ||
87
+ totalIndexing > 0 ||
88
+ totalErrors > 0 ||
89
+ totalEmbedding > 0) {
90
+ lines.push("", `Health: ⚠️ ${totalStale} stale, ${totalOrphans} orphans, ${totalStuck} stuck indexing, ${totalIndexing} indexing, ${totalErrors} errors, ${totalEmbedding} embedding issues — run knowledge_update, correct embedding configuration, or rebuild affected KBs`);
91
+ }
92
+ return lines.join("\n");
93
+ }
@@ -12,7 +12,7 @@
12
12
  | `OMP_CODING_AGENT_DIR` | unset | Derive the OMP host root from a configured agent directory. Used when `PI_CODING_AGENT_DIR` is not set. |
13
13
  | `OMP_PROFILE` | unset | Treat the current process as OMP-hosted for default path selection. |
14
14
  | `PI_KNOWLEDGE_MODEL_CACHE_DIR` | `<knowledge-dir>/models` | Override the local Transformers.js model cache directory. |
15
- | `PI_KNOWLEDGE_NODE_PATH` | current Node when possible, then `NODE`, then `node` on `PATH` | Override the Node 22+ executable used to run the isolated model worker. Useful for Windows OMP packaged runtimes where the host process is not Node. |
15
+ | `PI_KNOWLEDGE_NODE_PATH` | current Node when possible, then `NODE`, then `node` on `PATH` | Override the Node 22+ executable used by the isolated runtime worker for local models and PDF/DOCX extraction. Useful for Windows OMP packaged runtimes where the host process is not Node. |
16
16
 
17
17
  Default data storage is `~/.pi/knowledge` under Pi and `~/.omp/knowledge` under OMP. For the default home OMP root, `pi-knowledge` preserves an existing legacy `~/.pi/knowledge` directory when `~/.omp/knowledge` does not exist, so existing Pi knowledge bases remain visible during migration.
18
18
 
@@ -25,15 +25,18 @@ Default data storage is `~/.pi/knowledge` under Pi and `~/.omp/knowledge` under
25
25
  | `OPENAI_API_KEY` | unset | Fallback API key used when `PI_KNOWLEDGE_EMBEDDING=openai:<model>`. Required by OpenAI's hosted API; many local OpenAI-compatible servers do not need it. |
26
26
  | `PI_KNOWLEDGE_EMBEDDING_BASE_URL` | unset | OpenAI-compatible embedding API root, such as `http://127.0.0.1:8080/v1`. Takes precedence over `OPENAI_BASE_URL`. |
27
27
  | `OPENAI_BASE_URL` | `https://api.openai.com/v1` | Common OpenAI-compatible API root fallback. |
28
- | `PI_KNOWLEDGE_EMBEDDING_PREFIXES` | `on` | Controls whether `query: ` and `passage: ` prefixes are prepended before embedding. Values: `on`, `off`. Keep `on` for the default E5 local model; use `off` only for models or self-hosted APIs that are evaluated without E5-style prefixes. |
29
- | `PI_KNOWLEDGE_EMBEDDING_MAX_CHARS` | `20000` | Final per-input API embedding safety cap for OpenAI-compatible servers with smaller context windows. This does not replace chunker bounds. |
28
+ | `PI_KNOWLEDGE_EMBEDDING_PREFIXES` | `on` | Embedding input decoration strategy. `on`/`e5` use `query: ` and `passage: `; `off`/`none` use raw text; `jina`/`jina-code` use published Jina Code prompts; `custom` uses the two instruction variables below. |
29
+ | `PI_KNOWLEDGE_EMBEDDING_TASK` | `nl2code` for `jina-code` | Jina Code prompt task: `nl2code`, `code2code`, `code2nl`, `code2completion`, or `qa`. Rejected for other strategies. |
30
+ | `PI_KNOWLEDGE_EMBEDDING_QUERY_INSTRUCTION` | unset | Exact query instruction for `custom`. Required and non-empty; maximum 512 characters. Whitespace is preserved. |
31
+ | `PI_KNOWLEDGE_EMBEDDING_DOCUMENT_INSTRUCTION` | unset | Exact document instruction for `custom`. Required; may be an explicit empty string; maximum 512 characters. Whitespace is preserved. |
32
+ | `PI_KNOWLEDGE_EMBEDDING_MAX_CHARS` | `20000` | Final complete-input API embedding safety cap after instructions are added. Configuration fails if an instruction alone leaves no room for text. This does not replace chunker bounds. |
30
33
  | `PI_KNOWLEDGE_EMBEDDING_BATCH_SIZE` | `64` | Maximum number of document chunks sent in one embedding request during add, update, and import. Lower this for self-hosted embedding servers with smaller request batch limits. |
31
- | `PI_KNOWLEDGE_EMBEDDING_API_FALLBACK` | unset | Set to `local` to explicitly fall back to local embeddings for query embeddings after API failures. Indexing/import/update document batches do not fall back because mixed-provider vector files are unsafe. |
34
+ | `PI_KNOWLEDGE_EMBEDDING_API_FALLBACK` | unset | Set to `local` to explicitly fall back to local embeddings for query embeddings after API failures. Indexing/import/update document batches do not fall back. `jina-code` and `custom` reject local fallback during configuration because the local E5 model cannot reproduce those inputs. |
32
35
  | `PI_KNOWLEDGE_ENABLE_NATIVE_IDLE_DISPOSE` | unset | Enables idle timers for embedding/reranker run coordination. It does not guarantee immediate worker memory release; model-worker shutdown remains tied to engine/session shutdown. Disabled by default for stable shutdown. |
33
36
  | `PI_KNOWLEDGE_EMBEDDING_IDLE_MS` | `30000` | Idle timer used only when native idle disposal coordination is enabled. Mainly for lifecycle stress tests. |
34
37
  | `PI_KNOWLEDGE_OFFLINE` | unset | Use with a pre-populated model cache for offline local model operation. See `docs/offline-mode.md`. |
35
38
 
36
- Each KB stores the embedding model label, vector dimension, and a non-secret embedding signature that includes provider/model semantics, query/document prefixes, non-default prefix strategy, pooling, normalization policy, API input cap, and a hash of the API base URL when applicable. OpenAI-compatible response vectors are validated and L2-normalized by the client before use or persistence; the `client-l2-v1` policy is part of API signatures. `knowledge_search` skips vector retrieval for KBs whose stored signature or dimension is missing or incompatible with the current query embedding and uses BM25-only fallback for hybrid searches. Run `knowledge_update` to rebuild vectors after changing embedding provider/model/max-character/prefix settings or after upgrading an API-backed KB created before client-side normalization was signed.
39
+ Each KB stores the embedding model label, vector dimension, and a non-secret embedding signature that includes provider/model semantics, the input decoration descriptor, pooling, normalization policy, API input cap, and a hash of the API base URL when applicable. E5 `on` and raw-text `off` descriptors remain byte-compatible with existing signatures; Jina signatures include the task and instruction hash, while custom signatures contain only an instruction hash rather than the raw instruction text. OpenAI-compatible response vectors are validated and L2-normalized by the client before use or persistence; the `client-l2-v1` policy is part of API signatures. `knowledge_search` skips vector retrieval for missing or incompatible signatures/dimensions and uses BM25-only fallback for hybrid searches. `knowledge_status` reports indexed and active embedding settings and counts incompatibility in aggregate health. `knowledge_doctor` reports configuration failures and rebuild actions. Run `knowledge_update` after changing provider, model, endpoint, input cap, task, or instructions; re-add inline text or re-import portable sources that cannot be updated.
37
40
 
38
41
  ## Reranker
39
42
 
@@ -60,9 +63,9 @@ Reranker settings affect query-time `deep` search only; they do not change index
60
63
 
61
64
  For Hugging Face Text Embeddings Inference rerankers, set `PI_KNOWLEDGE_RERANKER_API_FORMAT=tei` and point `PI_KNOWLEDGE_RERANKER_API_ENDPOINT` at the TEI `/rerank` endpoint, for example `http://127.0.0.1:8080/rerank`. Do not reuse an embeddings `OPENAI_BASE_URL` ending in `/v1` unless the reranker endpoint actually exists there; TEI's OpenAI-compatible route covers embeddings, while reranking uses `/rerank`.
62
65
 
63
- Local embeddings require a real Node 22+ executable for the isolated model worker. `pi-knowledge` first tries Node IPC and automatically falls back to a stdin/stdout JSONL worker transport when the host runtime does not expose `child_process.fork().send()`, which can happen in Windows OMP compatibility layers. On Windows it searches `PI_KNOWLEDGE_NODE_PATH`, persisted `knowledge_configure` settings, common Node, Volta, NVM, Codex `cua_node`, and `PATH` locations before failing. If worker startup still fails, call `knowledge_configure` with the full `node.exe` path such as `C:\Program Files\nodejs\node.exe`; accidental surrounding quotes are ignored. OpenAI-compatible embeddings avoid the local worker entirely.
66
+ Local embeddings, local/HF reranking, and PDF/DOCX extraction require a real Node 22+ executable for the isolated worker. `pi-knowledge` first tries Node IPC and automatically falls back to a stdin/stdout JSONL worker transport when the host runtime does not expose `child_process.fork().send()`, which can happen in Windows OMP compatibility layers. On Windows it searches `PI_KNOWLEDGE_NODE_PATH`, persisted `knowledge_configure` settings, common Node, Volta, NVM, Codex `cua_node`, and `PATH` locations before failing. If worker startup still fails, call `knowledge_configure` with the full `node.exe` path such as `C:\Program Files\nodejs\node.exe`; accidental surrounding quotes are ignored. OpenAI-compatible embeddings avoid local model loading, but document extraction still uses the worker to isolate parser stdio from the host.
64
67
 
65
- API embedding failures intentionally surface by default. Silent fallback can hide bad API keys, wrong base URLs, unsupported model names, context-window errors, or server batch limits and can produce a KB with a different embedding model than intended. Query-time fallback is opt-in; indexing fallback is disabled to prevent mixed vector spaces in one KB. OpenAI-compatible embedding requests send `Authorization: Bearer <key>` only when `PI_KNOWLEDGE_EMBEDDING_API_KEY` or `OPENAI_API_KEY` is configured.
68
+ API embedding failures intentionally surface by default. Silent fallback can hide bad API keys, wrong base URLs, unsupported model names, context-window errors, server batch limits, or instruction mismatches and can produce a KB with a different embedding model than intended. Query-time fallback is opt-in and supports only E5 `on`/`off` semantics; indexing fallback is disabled to prevent mixed vector spaces in one KB. OpenAI-compatible embedding requests send `Authorization: Bearer <key>` only when `PI_KNOWLEDGE_EMBEDDING_API_KEY` or `OPENAI_API_KEY` is configured. Instruction decoration is applied once before both safety-cap validation and the HTTP request.
66
69
 
67
70
  ## Runtime Features and Diagnostics
68
71
 
@@ -83,15 +86,17 @@ API embedding failures intentionally surface by default. Silent fallback can hid
83
86
  | `PI_KNOWLEDGE_DEEP_RERANK_CANDIDATES` | profile default | Override how many filtered candidates `deep` mode sends to the reranker. Clamped to 5-100. |
84
87
  | `PI_KNOWLEDGE_DEEP_RERANK_TOPK_MULTIPLIER` | profile default | Override deep rerank top-k multiplier relative to the effective result limit. Clamped to 1-10. |
85
88
 
86
- Search diagnostics include result provenance and applied search tuning in both text output and structured `details`: selected profile, effective limit, snippet length, hybrid threshold, global candidate limit, per-KB retrieval limit, retrieved/filtered/diversity candidate counts, global-pool truncation, chunk id, chunk hash, match reason, indexed timestamp, stale flag, and source mtime when the original source file is available. This provenance is stored in SQLite and derived from indexed source metadata; it does not create or modify files in the indexed project.
89
+ Search diagnostics include result provenance and applied search tuning in both text output and structured `details`: selected profile, effective limit, snippet length, hybrid threshold, global candidate limit, per-KB retrieval limit, retrieved/threshold-passed/eligibility-passed/metadata-passed/diversity candidate counts, global-pool truncation, chunk id, chunk hash, match reason, indexed timestamp, stale flag, and source mtime when the original source file is available. This provenance is stored in SQLite and derived from indexed source metadata; it does not create or modify files in the indexed project.
87
90
 
88
91
  `knowledge_search` supports `file_type`, substring `path_pattern`, `profile`, integer `limit` from 1 to 50, and non-negative integer `offset`. `offset + limit` cannot exceed the 800-candidate global search window. Each KB retrieves up to the profile-derived candidate limit, then merged results are ranked, filtered, and globally capped before deep reranking, adaptive expansion, or diversity. Search profiles tune result count, snippets, hybrid strictness, candidate breadth, adaptive context, and deep rerank breadth at runtime. `auto` chooses from query shape, mode, and KB source type; explicit tool parameters win over env overrides, env overrides win over profile defaults, and profile defaults win over built-in compatibility values. `hybrid` mode remains lexical-anchored to prevent low-evidence semantic false positives; use `semantic` for pure vector search, or `semantic_hybrid` when weak lexical overlap should not block vector-backed weighted fusion. `adaptive` uses bounded same-file context expansion; when AST metadata is present it can prefer same-parent/sibling chunks without claiming caller/callee graph semantics.
89
92
 
90
- `knowledge_doctor` emits both human-readable issues and machine-readable actions in structured `details`. Action codes include `run_update`, `rebuild_kb`, `wait_for_indexing`, `review_skipped_scope`, `rebuild_vectors`, `check_source`, and `none`. Agents should prefer these action codes over parsing the English action text.
93
+ Empty `knowledge_search` responses always include structured `details.miss` with a deterministic reason, action, searched/skipped KB summaries, stage counts, requested filters, and bounded `auto` attempt summaries. Empty `knowledge_symbol_search` responses expose the corresponding symbol miss contract. The default text is a two-line rendering of these facts. Miss classification does not run source diagnostics, enumerate absolute source paths, change ranking, or mutate a KB.
94
+
95
+ `knowledge_doctor` emits both human-readable issues and machine-readable actions in structured `details`. Action codes include `run_update`, `rebuild_kb`, `wait_for_indexing`, `review_skipped_scope`, `rebuild_vectors`, `reindex_for_embedding_budget`, `fix_embedding_config`, `check_source`, and `none`. Agents should prefer these action codes over parsing the English action text.
91
96
 
92
97
  `knowledge_configure` writes runtime configuration to the knowledge directory's `config.json`. It validates Node 22+ before persisting `node_path`, so agents can configure a Codex- or user-installed `node.exe` once without relying on environment variables being injected into an already-running OMP process.
93
98
 
94
- `knowledge_symbol_search` uses a lightweight symbol index stored in `knowledge.db`. It is rebuilt during `knowledge_add`, `knowledge_update`, and `knowledge_import`, and can find AST-backed methods/functions/classes/interfaces/types/variables for supported code files, route-like handlers, Markdown headings, config keys, and environment variables. It is not a full LSP or code graph; it does not answer caller/callee or rename-safety questions. For empty important lookups, use `knowledge_search` mode `fast`/`adaptive` and check `knowledge_doctor` for stale or missing symbol metadata. Older file/directory/URL KBs without symbol metadata should be updated before relying on symbol lookup. Imported portable KBs and inline text KBs do not retain an active source path; rebuild their symbols by re-importing or re-adding the content.
99
+ `knowledge_symbol_search` uses a lightweight symbol index stored in `knowledge.db`. It is rebuilt during `knowledge_add`, `knowledge_update`, and `knowledge_import`, and can find AST-backed methods/functions/classes/interfaces/types/variables for supported code files, route-like handlers, Markdown headings, config keys, and environment variables. It is not a full LSP or code graph; it does not answer caller/callee or rename-safety questions. Empty lookups distinguish missing KBs, unavailable KBs, empty symbol metadata, exact-name misses, metadata filters, pagination, and ordinary no-match results. If the symbol index itself is empty, use `knowledge_update` for source-backed KBs; imported portable KBs and inline text KBs do not retain an active source path and must be re-imported or re-added from the original content.
95
100
 
96
101
  ## Test and Release Fixtures
97
102
 
@@ -119,7 +124,7 @@ Compatibility guarantees:
119
124
 
120
125
  - The packaged `extension.js` loads built `dist/index.js` when present and falls back to source `index.ts` for local development.
121
126
  - Runtime modules are imported lazily, after the extension is actually used.
122
- - Local embedding and reranking models run in an isolated Node model worker, not in the Pi or OMP TUI process. Node IPC is preferred; stdin/stdout JSONL transport is used automatically when host IPC is unavailable.
127
+ - Local embedding/reranking models and PDF/DOCX parsers run in an isolated Node worker, not in the Pi or OMP TUI process. Node IPC is preferred; stdin/stdout JSONL transport is used automatically when host IPC is unavailable.
123
128
  - Native SQLite loading includes a fallback for hoisted plugin dependency layouts.
124
129
  - OMP path resolution can use `OMP_KNOWLEDGE_DIR`, `OMP_CODING_AGENT_DIR`, and `OMP_PROFILE`.
125
130
  - Existing default Pi knowledge data can remain visible under the default OMP home root through the legacy `~/.pi/knowledge` fallback described above.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "pi-knowledge",
3
- "version": "0.13.0",
3
+ "version": "0.15.0",
4
4
  "description": "Local-first RAG knowledge base for Pi and OMP coding agents with hybrid code/doc search, reranking, diagnostics, and large-project indexing",
5
5
  "type": "module",
6
6
  "main": "dist/index.js",