pi-knowledge 0.13.0 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,13 +1,13 @@
1
1
  import { existsSync, readFileSync, renameSync, rmSync, statSync } from "node:fs";
2
2
  import { join, resolve } from "node:path";
3
3
  import { diagnoseKB } from "./diagnostics/health.js";
4
- import { dispose as disposeEmbedding, embedDocuments, embeddingConfigLabel, embeddingSignature, embedQueryWithConfig, embedValidatedDocuments, prepareForShutdown as prepareEmbeddingForShutdown, resolveEmbeddingConfig, } from "./embedding/provider.js";
4
+ import { describeEmbeddingCompatibility, describeEmbeddingDecoration, dispose as disposeEmbedding, embedDocuments, embeddingConfigLabel, embeddingSignature, embedQueryWithConfig, embedValidatedDocuments, inspectEmbeddingCompatibility, prepareForShutdown as prepareEmbeddingForShutdown, resolveEmbeddingConfig, } from "./embedding/provider.js";
5
5
  import { openVectorReader, openVectorWriter } from "./embedding/vectors.js";
6
6
  import { addSkippedScanEntry, analyzeIndexableContent, buildChunkEmbeddingText, chunkFile, chunkIdentityHash, chunkMarkdown, createSkippedScanStats, isReadableTextFile, isSupportedDocumentFile, iterateScannableFiles, preTokenizeForFTS, summarizeSkippedScan, } from "./indexer/chunker.js";
7
7
  import { finalizeChunksForEmbedding } from "./indexer/embedding-safe-chunks.js";
8
8
  import { htmlToStructuredMarkdown } from "./indexer/html.js";
9
9
  import { extractSymbols } from "./indexer/symbols.js";
10
- import { shutdownModelWorker } from "./model-worker-client.js";
10
+ import { extractDocumentInModelWorker, shutdownModelWorker } from "./model-worker-client.js";
11
11
  import { searchBM25 } from "./search/bm25.js";
12
12
  import { MAX_SEARCH_WINDOW } from "./search/contracts.js";
13
13
  import { diversifyRankedChunks, interleaveByFile } from "./search/diversity.js";
@@ -370,30 +370,19 @@ async function chunkUrl(source, signal) {
370
370
  const structuredChunks = chunkMarkdown(text, source, "html");
371
371
  return structuredChunks.length > 0 ? structuredChunks : chunkFile(text, source);
372
372
  }
373
- function normalizeExtractedText(text) {
374
- return Array.isArray(text) ? text.join("\n\n") : text;
375
- }
376
373
  async function extractSourceFileContent(filePath, signal) {
377
374
  const lowerPath = filePath.toLowerCase();
378
375
  if (lowerPath.endsWith(".pdf")) {
379
376
  throwIfAborted(signal);
380
- // PDF extraction is an optional heavy runtime path; keep parser loading out of extension startup.
381
- const { extractText } = await import("unpdf");
382
- throwIfAborted(signal);
383
- const buf = readFileSync(filePath);
377
+ const extracted = await extractDocumentInModelWorker(filePath, "pdf", signal);
384
378
  throwIfAborted(signal);
385
- const { text } = await extractText(new Uint8Array(buf));
386
- throwIfAborted(signal);
387
- return { content: normalizeExtractedText(text), fileType: "pdf" };
379
+ return extracted;
388
380
  }
389
381
  if (lowerPath.endsWith(".docx") || lowerPath.endsWith(".doc")) {
390
382
  throwIfAborted(signal);
391
- // DOCX extraction is an optional heavy runtime path; keep parser loading out of extension startup.
392
- const mammoth = await import("mammoth");
393
- throwIfAborted(signal);
394
- const result = await mammoth.extractRawText({ path: filePath });
383
+ const extracted = await extractDocumentInModelWorker(filePath, "docx", signal);
395
384
  throwIfAborted(signal);
396
- return { content: result.value, fileType: "docx" };
385
+ return extracted;
397
386
  }
398
387
  if (!isReadableTextFile(filePath)) {
399
388
  throw new Error(`File is not readable text and has no supported extractor: ${filePath}`);
@@ -451,15 +440,8 @@ function persistEmbeddingMetadata(db, kbId, config, vectors) {
451
440
  updateKBEmbeddingMetadata(db, kbId, embeddingConfigLabel(config), embeddingSignature(config, dimension), dimension);
452
441
  return dimension;
453
442
  }
454
- function embeddingMismatchWarning(kb) {
455
- return `"${kb.name}" has incompatible embedding metadata; vector retrieval was skipped and knowledge_update should rebuild it`;
456
- }
457
- function canSearchVectors(kb, config, queryDimension) {
458
- if (kb.embedding_signature === null || kb.embedding_dimension === null)
459
- return false;
460
- if (kb.embedding_dimension !== queryDimension)
461
- return false;
462
- return kb.embedding_signature === embeddingSignature(config, queryDimension);
443
+ function embeddingMismatchWarning(kb, compatibility) {
444
+ return `"${kb.name}" has incompatible embedding vectors (${describeEmbeddingCompatibility(compatibility)}); vector retrieval was skipped and knowledge_update should rebuild them`;
463
445
  }
464
446
  function throwIfAborted(signal) {
465
447
  if (signal?.aborted)
@@ -844,14 +826,17 @@ export class KnowledgeEngine {
844
826
  if (!this.db)
845
827
  throw new Error("Engine not initialized");
846
828
  throwIfAborted(signal);
847
- updateKBStatus(this.db, kb.id, "indexing");
848
- startIndexingJob(this.db, kb.id, "update", `Starting update for "${kb.name}"`);
849
829
  const scanOptions = toScanOptions(parseAddOptions(kb.source_options));
850
830
  const embeddingConfig = resolveEmbeddingConfig();
851
831
  const embeddingBatchSize = resolveIndexEmbedBatchSize();
832
+ updateKBStatus(this.db, kb.id, "indexing");
833
+ startIndexingJob(this.db, kb.id, "update", `Starting update for "${kb.name}"`);
852
834
  const embeddingModel = embeddingConfigLabel(embeddingConfig);
853
- const currentSignature = kb.embedding_dimension === null ? undefined : embeddingSignature(embeddingConfig, kb.embedding_dimension);
854
- let canReuseExistingVectors = kb.embedding_signature !== null && kb.embedding_dimension !== null && kb.embedding_signature === currentSignature;
835
+ const compatibility = inspectEmbeddingCompatibility(kb.embedding_signature, kb.embedding_dimension, embeddingConfig, kb.embedding_dimension ?? 0);
836
+ let canReuseExistingVectors = compatibility.compatible;
837
+ let rebuildMessage = compatibility.compatible
838
+ ? undefined
839
+ : `Embedding vectors for "${kb.name}" are incompatible (${describeEmbeddingCompatibility(compatibility)}); rebuilding all vectors`;
855
840
  let replacementVectorPath;
856
841
  let addedVectorPath;
857
842
  let addedVectorWriter;
@@ -880,9 +865,7 @@ export class KnowledgeEngine {
880
865
  canReuseExistingVectors = false;
881
866
  const vectorCount = oldVectorReader?.count ?? 0;
882
867
  const vectorDimension = oldVectorReader?.dim ?? 0;
883
- const message = `Stored vector file for "${kb.name}" is incomplete or incompatible (${vectorCount} vectors/${vectorDimension}d for ${existingIndex} chunks/${kb.embedding_dimension}d); rebuilding all vectors`;
884
- updateIndexingJob(this.db, kb.id, { phase: "embedding", message });
885
- onProgress?.(message);
868
+ rebuildMessage = `Stored vector file for "${kb.name}" is incomplete or incompatible (${vectorCount} vectors/${vectorDimension}d for ${existingIndex} chunks/${kb.embedding_dimension}d); rebuilding all vectors`;
886
869
  }
887
870
  }
888
871
  finally {
@@ -892,10 +875,9 @@ export class KnowledgeEngine {
892
875
  const reusableHashes = canReuseExistingVectors
893
876
  ? existingHashes
894
877
  : new Map();
895
- if (!canReuseExistingVectors) {
896
- const message = `Embedding metadata changed or missing for "${kb.name}"; rebuilding all vectors`;
897
- updateIndexingJob(this.db, kb.id, { phase: "embedding", message });
898
- onProgress?.(message);
878
+ if (!canReuseExistingVectors && rebuildMessage) {
879
+ updateIndexingJob(this.db, kb.id, { phase: "embedding", message: rebuildMessage });
880
+ onProgress?.(rebuildMessage);
899
881
  }
900
882
  const oldVectorIndexByHash = new Map();
901
883
  const newVectorIndexByHash = new Map();
@@ -1274,8 +1256,9 @@ export class KnowledgeEngine {
1274
1256
  queryEmbedding ??= await embedQueryWithConfig(query, signal);
1275
1257
  const { vector: queryVec, config: queryEmbeddingConfig } = queryEmbedding;
1276
1258
  throwIfAborted(signal);
1277
- if (!canSearchVectors(kb, queryEmbeddingConfig, queryVec.length)) {
1278
- warnings.push(embeddingMismatchWarning(kb));
1259
+ const compatibility = inspectEmbeddingCompatibility(kb.embedding_signature, kb.embedding_dimension, queryEmbeddingConfig, queryVec.length);
1260
+ if (!compatibility.compatible) {
1261
+ warnings.push(embeddingMismatchWarning(kb, compatibility));
1279
1262
  continue;
1280
1263
  }
1281
1264
  const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), retrievalLimitPerKb, signal);
@@ -1292,14 +1275,15 @@ export class KnowledgeEngine {
1292
1275
  queryEmbedding ??= await embedQueryWithConfig(query, signal);
1293
1276
  const { vector: queryVec, config: queryEmbeddingConfig } = queryEmbedding;
1294
1277
  throwIfAborted(signal);
1295
- if (canSearchVectors(kb, queryEmbeddingConfig, queryVec.length)) {
1278
+ const compatibility = inspectEmbeddingCompatibility(kb.embedding_signature, kb.embedding_dimension, queryEmbeddingConfig, queryVec.length);
1279
+ if (compatibility.compatible) {
1296
1280
  const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), retrievalLimitPerKb, signal);
1297
1281
  vecResults = vectorResults.results;
1298
1282
  for (const [chunkId, vector] of vectorResults.vectorsByChunkId)
1299
1283
  vectorsByChunkId.set(chunkId, vector);
1300
1284
  }
1301
1285
  else {
1302
- warnings.push(embeddingMismatchWarning(kb));
1286
+ warnings.push(embeddingMismatchWarning(kb, compatibility));
1303
1287
  }
1304
1288
  const fused = weightedScoreFusion(bm25Results, vecResults);
1305
1289
  allResults.push(...fused.map((result) => ({ ...result, score: result.score * kbTrustMultiplier(kb) })));
@@ -1562,6 +1546,46 @@ export class KnowledgeEngine {
1562
1546
  return diagnoseKB(db, kb, signal);
1563
1547
  });
1564
1548
  }
1549
+ statusSnapshot(signal) {
1550
+ if (!this.db)
1551
+ return { knowledge_bases: [], diagnostics: [], embeddings: [] };
1552
+ throwIfAborted(signal);
1553
+ const db = this.db;
1554
+ const knowledgeBases = listKBs(db);
1555
+ const diagnostics = knowledgeBases.map((kb) => {
1556
+ throwIfAborted(signal);
1557
+ return diagnoseKB(db, kb, signal);
1558
+ });
1559
+ try {
1560
+ const config = resolveEmbeddingConfig();
1561
+ const activeModel = embeddingConfigLabel(config);
1562
+ const activeDecoration = describeEmbeddingDecoration(config);
1563
+ return {
1564
+ knowledge_bases: knowledgeBases,
1565
+ diagnostics,
1566
+ embeddings: knowledgeBases.map((kb) => {
1567
+ const compatibility = inspectEmbeddingCompatibility(kb.embedding_signature, kb.embedding_dimension, config, kb.embedding_dimension ?? 0);
1568
+ return {
1569
+ kb_id: kb.id,
1570
+ kb_name: kb.name,
1571
+ indexed_model: kb.embedding_model,
1572
+ active_model: activeModel,
1573
+ active_decoration: activeDecoration,
1574
+ compatibility,
1575
+ detail: describeEmbeddingCompatibility(compatibility),
1576
+ };
1577
+ }),
1578
+ };
1579
+ }
1580
+ catch (error) {
1581
+ return {
1582
+ knowledge_bases: knowledgeBases,
1583
+ diagnostics,
1584
+ embeddings: [],
1585
+ embedding_error: error instanceof Error ? error.message : String(error),
1586
+ };
1587
+ }
1588
+ }
1565
1589
  symbolSearch(query, options = {}, signal) {
1566
1590
  if (!this.db)
1567
1591
  throw new Error("Engine not initialized");
@@ -1683,23 +1707,34 @@ export class KnowledgeEngine {
1683
1707
  }
1684
1708
  }
1685
1709
  if (this.db) {
1686
- const activeEmbeddingConfig = resolveEmbeddingConfig();
1710
+ let activeEmbeddingConfig;
1711
+ try {
1712
+ activeEmbeddingConfig = resolveEmbeddingConfig();
1713
+ }
1714
+ catch (error) {
1715
+ issues.push({
1716
+ severity: "blocking",
1717
+ message: `Active embedding configuration is invalid: ${error instanceof Error ? error.message : String(error)}`,
1718
+ action: "Correct the embedding environment variables, then run knowledge_doctor again.",
1719
+ action_code: "fix_embedding_config",
1720
+ });
1721
+ }
1687
1722
  for (const kb of kbs) {
1688
1723
  throwIfAborted(signal);
1689
- if ((kb.status === "ready" || kb.status === "stale") &&
1690
- kb.chunk_count > 0 &&
1691
- kb.embedding_dimension !== null &&
1692
- kb.embedding_signature !== embeddingSignature(activeEmbeddingConfig, kb.embedding_dimension)) {
1693
- const canUpdate = kb.source_path !== null && kb.source_type !== "text";
1694
- issues.push({
1695
- severity: "warning",
1696
- kb_name: kb.name,
1697
- message: "This KB's vectors are incompatible with the active embedding provider or normalization policy.",
1698
- action: canUpdate
1699
- ? "Run knowledge_update to rebuild vectors with the active embedding configuration."
1700
- : "Re-add inline text or re-import the original source with the active embedding configuration.",
1701
- action_code: "rebuild_vectors",
1702
- });
1724
+ if (activeEmbeddingConfig && (kb.status === "ready" || kb.status === "stale") && kb.chunk_count > 0) {
1725
+ const compatibility = inspectEmbeddingCompatibility(kb.embedding_signature, kb.embedding_dimension, activeEmbeddingConfig, kb.embedding_dimension ?? 0);
1726
+ if (!compatibility.compatible) {
1727
+ const canUpdate = kb.source_path !== null && kb.source_type !== "text";
1728
+ issues.push({
1729
+ severity: "warning",
1730
+ kb_name: kb.name,
1731
+ message: `This KB's vectors are incompatible with the active embedding configuration: ${describeEmbeddingCompatibility(compatibility)}.`,
1732
+ action: canUpdate
1733
+ ? "Run knowledge_update to rebuild vectors with the active embedding configuration."
1734
+ : "Re-add inline text or re-import the original source with the active embedding configuration.",
1735
+ action_code: "rebuild_vectors",
1736
+ });
1737
+ }
1703
1738
  }
1704
1739
  if ((kb.status === "ready" || kb.status === "stale") &&
1705
1740
  (storedSourcePolicyVersion(kb.source_options, "embedding_input_policy") ?? 0) < CURRENT_EMBEDDING_INPUT_POLICY) {
@@ -1824,10 +1859,10 @@ export class KnowledgeEngine {
1824
1859
  throwIfAborted(signal);
1825
1860
  const { createReadStream } = await import("node:fs");
1826
1861
  const { createInterface } = await import("node:readline");
1827
- const stream = createReadStream(inputPath, { encoding: "utf-8" });
1828
- const lines = createInterface({ input: stream, crlfDelay: Infinity });
1829
1862
  const embeddingConfig = resolveEmbeddingConfig();
1830
1863
  const embeddingBatchSize = resolveIndexEmbedBatchSize();
1864
+ const stream = createReadStream(inputPath, { encoding: "utf-8" });
1865
+ const lines = createInterface({ input: stream, crlfDelay: Infinity });
1831
1866
  let header;
1832
1867
  let kb;
1833
1868
  let vectorWriter;
@@ -1,3 +1,4 @@
1
+ import { embeddingInstructionLength } from "../embedding/instructions.js";
1
2
  import { LOCAL_EMBEDDING_PREFERRED_TOKENS, measureEmbeddingInputs } from "../embedding/provider.js";
2
3
  import { buildChunkEmbeddingText, chunkIdentityHash, preTokenizeForFTS } from "./chunker.js";
3
4
  const SENTENCE_BOUNDARIES = {
@@ -153,8 +154,8 @@ function withPartMetadata(chunk, part, partCount) {
153
154
  function isProvablySafeLocalAsciiInput(text, config) {
154
155
  if (config.provider !== "local")
155
156
  return false;
156
- const prefixLength = config.prefixes === "on" ? "passage: ".length : 0;
157
- if (text.length + prefixLength + 2 > LOCAL_EMBEDDING_PREFERRED_TOKENS)
157
+ const instructionLength = embeddingInstructionLength(config.instructions, "passage");
158
+ if (text.length + instructionLength + 2 > LOCAL_EMBEDDING_PREFERRED_TOKENS)
158
159
  return false;
159
160
  for (let index = 0; index < text.length; index++) {
160
161
  if (text.charCodeAt(index) > 0x7f)
@@ -7,6 +7,10 @@ export declare function measureEmbeddingInputInModelWorker(texts: string[], pref
7
7
  counts: number[];
8
8
  maximum: number;
9
9
  }>;
10
+ export declare function extractDocumentInModelWorker(filePath: string, fileType: "pdf" | "docx", signal?: AbortSignal): Promise<{
11
+ content: string;
12
+ fileType: "pdf" | "docx";
13
+ }>;
10
14
  export interface RerankWorkerCandidate {
11
15
  chunkId: string;
12
16
  content: string;
@@ -39,7 +39,7 @@ function validateNodeExecPath(candidate) {
39
39
  }).trim();
40
40
  const major = parseNodeMajor(version);
41
41
  if (major === null || major < MIN_NODE_MAJOR) {
42
- throw new Error(`Node ${MIN_NODE_MAJOR}+ is required for the model worker; ${candidate} reported ${version}`);
42
+ throw new Error(`Node ${MIN_NODE_MAJOR}+ is required for the isolated runtime worker; ${candidate} reported ${version}`);
43
43
  }
44
44
  return candidate;
45
45
  }
@@ -149,7 +149,7 @@ function getNodeExecPath() {
149
149
  // Try the next candidate before surfacing a deterministic error below.
150
150
  }
151
151
  }
152
- throw new Error(`Node ${MIN_NODE_MAJOR}+ is required for local pi-knowledge embeddings, but no usable node executable was found. Install Node ${MIN_NODE_MAJOR}+ or set PI_KNOWLEDGE_NODE_PATH to the full node.exe path.`);
152
+ throw new Error(`Node ${MIN_NODE_MAJOR}+ is required for isolated pi-knowledge worker operations such as local models and PDF/DOCX extraction, but no usable node executable was found. Install Node ${MIN_NODE_MAJOR}+ or set PI_KNOWLEDGE_NODE_PATH to the full node.exe path.`);
153
153
  }
154
154
  export function validateModelWorkerNodePath(nodePath) {
155
155
  return validateNodeExecPath(normalizeExecutableCandidate(nodePath));
@@ -413,6 +413,18 @@ export async function measureEmbeddingInputInModelWorker(texts, prefix, prefixes
413
413
  }
414
414
  return { counts: result.counts, maximum: result.maximum };
415
415
  }
416
+ export async function extractDocumentInModelWorker(filePath, fileType, signal) {
417
+ const result = await getWorker().request({ type: "extract_document", filePath, fileType }, signal);
418
+ if (typeof result !== "object" ||
419
+ result === null ||
420
+ !("content" in result) ||
421
+ typeof result.content !== "string" ||
422
+ !("fileType" in result) ||
423
+ result.fileType !== fileType) {
424
+ throw new Error("Invalid document extraction worker response");
425
+ }
426
+ return { content: result.content, fileType };
427
+ }
416
428
  export async function rerankInModelWorker(query, candidates, topK, reranker, signal) {
417
429
  const result = await getWorker().request({ type: "rerank", query, candidates, topK, reranker }, signal);
418
430
  if (!Array.isArray(result))
@@ -1,5 +1,7 @@
1
+ import { readFile } from "node:fs/promises";
1
2
  import { join } from "node:path";
2
3
  import { createInterface } from "node:readline";
4
+ import { decorateEmbeddingInput, E5_INSTRUCTIONS, NO_INSTRUCTIONS, } from "./embedding/instructions.js";
3
5
  import { DEFAULT_RERANKER_REMOTE_HOST, DEFAULT_RERANKER_REMOTE_PATH_TEMPLATE, isHfRerankerConfig, rerankerCacheKey, resolveRerankerConfig, } from "./search/reranker-config.js";
4
6
  import { rerankWithSingleRawLogit } from "./search/reranker-logits.js";
5
7
  import { getDefaultKnowledgeDir } from "./storage/sqlite.js";
@@ -76,8 +78,8 @@ async function loadRerankerPipeline(config) {
76
78
  rerankerPipeline = { key, pipe: loaded };
77
79
  return loaded;
78
80
  }
79
- function applyEmbeddingPrefix(text, prefix, prefixes) {
80
- return prefixes === "on" ? `${prefix}: ${text}` : text;
81
+ function applyEmbeddingPrefix(text, role, prefixes) {
82
+ return decorateEmbeddingInput(text, role, prefixes === "on" ? E5_INSTRUCTIONS : NO_INSTRUCTIONS);
81
83
  }
82
84
  async function handleEmbed(request) {
83
85
  const pipe = await loadEmbeddingPipeline();
@@ -130,6 +132,24 @@ async function handleRerank(request) {
130
132
  results.sort((a, b) => b.score - a.score);
131
133
  return results.slice(0, request.topK);
132
134
  }
135
+ const PDFJS_ERRORS_ONLY_VERBOSITY = 0;
136
+ async function handleExtractDocument(request) {
137
+ if (request.fileType === "pdf") {
138
+ const { extractText, getDocumentProxy } = await import("unpdf");
139
+ const bytes = await readFile(request.filePath);
140
+ const pdf = await getDocumentProxy(new Uint8Array(bytes), { verbosity: PDFJS_ERRORS_ONLY_VERBOSITY });
141
+ try {
142
+ const { text } = await extractText(pdf);
143
+ return { content: Array.isArray(text) ? text.join("\n\n") : text, fileType: "pdf" };
144
+ }
145
+ finally {
146
+ await pdf.destroy();
147
+ }
148
+ }
149
+ const mammoth = await import("mammoth");
150
+ const result = await mammoth.extractRawText({ path: request.filePath });
151
+ return { content: result.value, fileType: "docx" };
152
+ }
133
153
  function isStringArray(value) {
134
154
  return Array.isArray(value) && value.every((item) => typeof item === "string");
135
155
  }
@@ -166,6 +186,15 @@ function parseWorkerRequest(message) {
166
186
  prefixes,
167
187
  };
168
188
  }
189
+ if (message.type === "extract_document") {
190
+ if (!("filePath" in message) || typeof message.filePath !== "string") {
191
+ throw new Error("Invalid document file path");
192
+ }
193
+ if (!("fileType" in message) || (message.fileType !== "pdf" && message.fileType !== "docx")) {
194
+ throw new Error("Invalid document file type");
195
+ }
196
+ return { id: message.id, type: "extract_document", filePath: message.filePath, fileType: message.fileType };
197
+ }
169
198
  if (message.type === "rerank") {
170
199
  if (!("query" in message) || typeof message.query !== "string")
171
200
  throw new Error("Invalid rerank query");
@@ -199,7 +228,9 @@ async function handleRequestMessage(message) {
199
228
  ? await handleEmbed(request)
200
229
  : request.type === "measure_embedding_input"
201
230
  ? await handleMeasureEmbeddingInput(request)
202
- : await handleRerank(request);
231
+ : request.type === "extract_document"
232
+ ? await handleExtractDocument(request)
233
+ : await handleRerank(request);
203
234
  return { id: request.id, result };
204
235
  }
205
236
  catch (error) {
@@ -0,0 +1,8 @@
1
+ import type { KnowledgeStatusSnapshot } from "./engine.ts";
2
+ export interface KnowledgeStatusFormatOptions {
3
+ snapshot: KnowledgeStatusSnapshot;
4
+ storagePath: string;
5
+ watchCount: number;
6
+ now?: number;
7
+ }
8
+ export declare function formatKnowledgeStatus(options: KnowledgeStatusFormatOptions): string;
@@ -0,0 +1,93 @@
1
+ export function formatKnowledgeStatus(options) {
2
+ const { snapshot, storagePath, watchCount } = options;
3
+ const now = options.now ?? Date.now();
4
+ const embeddingByKb = new Map(snapshot.embeddings.map((embedding) => [embedding.kb_id, embedding]));
5
+ const diagnosticsByKb = new Map(snapshot.diagnostics.map((diagnostic) => [diagnostic.kb_id, diagnostic]));
6
+ const lines = [
7
+ `Storage: ${storagePath}`,
8
+ `Knowledge bases: ${snapshot.knowledge_bases.length}`,
9
+ `Active watchers: ${watchCount}`,
10
+ ];
11
+ if (snapshot.embedding_error)
12
+ lines.push(`Embedding configuration: ⚠️ ${snapshot.embedding_error}`);
13
+ lines.push("");
14
+ for (const kb of snapshot.knowledge_bases) {
15
+ const age = Math.round((now - kb.updated_at) / 60_000);
16
+ const diagnostic = diagnosticsByKb.get(kb.id);
17
+ const embedding = embeddingByKb.get(kb.id);
18
+ lines.push(` "${kb.name}" — ${kb.status} — ${kb.chunk_count} chunks, ${kb.file_count} files — updated ${age}m ago`);
19
+ if (kb.source_path)
20
+ lines.push(` source: ${kb.source_path}`);
21
+ if (embedding) {
22
+ lines.push(` embedding: indexed=${embedding.indexed_model ?? "unrecorded"} · active=${embedding.active_model} · decoration=${embedding.active_decoration}`);
23
+ const requiresCompatibleVectors = kb.chunk_count > 0 && (kb.status === "ready" || kb.status === "stale");
24
+ if (requiresCompatibleVectors && !embedding.compatibility.compatible) {
25
+ lines.push(` ⚠️ vectors: ${embedding.detail}`);
26
+ }
27
+ }
28
+ else {
29
+ lines.push(` embedding: indexed=${kb.embedding_model ?? "unrecorded"} · active=unavailable`);
30
+ }
31
+ if (!diagnostic)
32
+ continue;
33
+ if (diagnostic.job) {
34
+ const jobAge = Math.round((now - diagnostic.job.last_progress_at) / 1_000);
35
+ const progress = [
36
+ `${diagnostic.job.processed_files} files`,
37
+ `${diagnostic.job.processed_chunks} chunks`,
38
+ diagnostic.job.skipped_total > 0 ? `${diagnostic.job.skipped_total} skipped` : "",
39
+ diagnostic.job.added_chunks > 0 ? `+${diagnostic.job.added_chunks}` : "",
40
+ diagnostic.job.removed_chunks > 0 ? `-${diagnostic.job.removed_chunks}` : "",
41
+ diagnostic.job.unchanged_chunks > 0 ? `=${diagnostic.job.unchanged_chunks}` : "",
42
+ ]
43
+ .filter(Boolean)
44
+ .join(", ");
45
+ lines.push(` job: ${diagnostic.job.status}/${diagnostic.job.phase} — ${progress || "no processed items yet"} — last progress ${jobAge}s ago`);
46
+ if (diagnostic.job.message)
47
+ lines.push(` last: ${diagnostic.job.message}`);
48
+ if (diagnostic.job.error_message)
49
+ lines.push(` error: ${diagnostic.job.error_message}`);
50
+ }
51
+ lines.push(` coverage: ${diagnostic.coverage_percent}% (${diagnostic.indexed_files}/${diagnostic.total_source_files} files)`);
52
+ if (diagnostic.stale_files.length > 0)
53
+ lines.push(` ⚠️ stale: ${diagnostic.stale_files.length} files modified since last index`);
54
+ if (diagnostic.orphan_files.length > 0)
55
+ lines.push(` ⚠️ orphans: ${diagnostic.orphan_files.length} chunks reference deleted files`);
56
+ if (diagnostic.stuck_indexing)
57
+ lines.push(` ⚠️ indexing appears stuck for ${Math.round(diagnostic.last_progress_age_ms / 60_000)}m; remove and rebuild if no pi process is actively indexing it`);
58
+ if (diagnostic.skipped_files.total > 0)
59
+ lines.push(` skipped: ${diagnostic.skipped_files.total} files (${Object.entries(diagnostic.skipped_files.by_reason)
60
+ .filter(([, count]) => count > 0)
61
+ .map(([reason, count]) => `${reason}: ${count}`)
62
+ .join(", ")})`);
63
+ }
64
+ const totalStale = snapshot.diagnostics.reduce((count, diagnostic) => count + diagnostic.stale_files.length, 0);
65
+ const totalOrphans = snapshot.diagnostics.reduce((count, diagnostic) => count + diagnostic.orphan_files.length, 0);
66
+ const totalStuck = snapshot.diagnostics.filter((diagnostic) => diagnostic.stuck_indexing).length;
67
+ const totalIndexing = snapshot.knowledge_bases.filter((kb) => kb.status === "indexing" && !diagnosticsByKb.get(kb.id)?.stuck_indexing).length;
68
+ const totalErrors = snapshot.knowledge_bases.filter((kb) => kb.status === "error").length;
69
+ const totalEmbedding = snapshot.knowledge_bases.reduce((count, kb) => {
70
+ if (kb.chunk_count === 0 || (kb.status !== "ready" && kb.status !== "stale"))
71
+ return count;
72
+ const embedding = embeddingByKb.get(kb.id);
73
+ return count + (embedding && !embedding.compatibility.compatible ? 1 : 0);
74
+ }, 0) + (snapshot.embedding_error ? 1 : 0);
75
+ if (totalStale === 0 &&
76
+ totalOrphans === 0 &&
77
+ totalStuck === 0 &&
78
+ totalIndexing === 0 &&
79
+ totalErrors === 0 &&
80
+ totalEmbedding === 0 &&
81
+ snapshot.knowledge_bases.length > 0) {
82
+ lines.push("", "Health: ✓ all indexes up to date and embedding-compatible");
83
+ }
84
+ else if (totalStale > 0 ||
85
+ totalOrphans > 0 ||
86
+ totalStuck > 0 ||
87
+ totalIndexing > 0 ||
88
+ totalErrors > 0 ||
89
+ totalEmbedding > 0) {
90
+ lines.push("", `Health: ⚠️ ${totalStale} stale, ${totalOrphans} orphans, ${totalStuck} stuck indexing, ${totalIndexing} indexing, ${totalErrors} errors, ${totalEmbedding} embedding issues — run knowledge_update, correct embedding configuration, or rebuild affected KBs`);
91
+ }
92
+ return lines.join("\n");
93
+ }
@@ -12,7 +12,7 @@
12
12
  | `OMP_CODING_AGENT_DIR` | unset | Derive the OMP host root from a configured agent directory. Used when `PI_CODING_AGENT_DIR` is not set. |
13
13
  | `OMP_PROFILE` | unset | Treat the current process as OMP-hosted for default path selection. |
14
14
  | `PI_KNOWLEDGE_MODEL_CACHE_DIR` | `<knowledge-dir>/models` | Override the local Transformers.js model cache directory. |
15
- | `PI_KNOWLEDGE_NODE_PATH` | current Node when possible, then `NODE`, then `node` on `PATH` | Override the Node 22+ executable used to run the isolated model worker. Useful for Windows OMP packaged runtimes where the host process is not Node. |
15
+ | `PI_KNOWLEDGE_NODE_PATH` | current Node when possible, then `NODE`, then `node` on `PATH` | Override the Node 22+ executable used by the isolated runtime worker for local models and PDF/DOCX extraction. Useful for Windows OMP packaged runtimes where the host process is not Node. |
16
16
 
17
17
  Default data storage is `~/.pi/knowledge` under Pi and `~/.omp/knowledge` under OMP. For the default home OMP root, `pi-knowledge` preserves an existing legacy `~/.pi/knowledge` directory when `~/.omp/knowledge` does not exist, so existing Pi knowledge bases remain visible during migration.
18
18
 
@@ -25,15 +25,18 @@ Default data storage is `~/.pi/knowledge` under Pi and `~/.omp/knowledge` under
25
25
  | `OPENAI_API_KEY` | unset | Fallback API key used when `PI_KNOWLEDGE_EMBEDDING=openai:<model>`. Required by OpenAI's hosted API; many local OpenAI-compatible servers do not need it. |
26
26
  | `PI_KNOWLEDGE_EMBEDDING_BASE_URL` | unset | OpenAI-compatible embedding API root, such as `http://127.0.0.1:8080/v1`. Takes precedence over `OPENAI_BASE_URL`. |
27
27
  | `OPENAI_BASE_URL` | `https://api.openai.com/v1` | Common OpenAI-compatible API root fallback. |
28
- | `PI_KNOWLEDGE_EMBEDDING_PREFIXES` | `on` | Controls whether `query: ` and `passage: ` prefixes are prepended before embedding. Values: `on`, `off`. Keep `on` for the default E5 local model; use `off` only for models or self-hosted APIs that are evaluated without E5-style prefixes. |
29
- | `PI_KNOWLEDGE_EMBEDDING_MAX_CHARS` | `20000` | Final per-input API embedding safety cap for OpenAI-compatible servers with smaller context windows. This does not replace chunker bounds. |
28
+ | `PI_KNOWLEDGE_EMBEDDING_PREFIXES` | `on` | Embedding input decoration strategy. `on`/`e5` use `query: ` and `passage: `; `off`/`none` use raw text; `jina`/`jina-code` use published Jina Code prompts; `custom` uses the two instruction variables below. |
29
+ | `PI_KNOWLEDGE_EMBEDDING_TASK` | `nl2code` for `jina-code` | Jina Code prompt task: `nl2code`, `code2code`, `code2nl`, `code2completion`, or `qa`. Rejected for other strategies. |
30
+ | `PI_KNOWLEDGE_EMBEDDING_QUERY_INSTRUCTION` | unset | Exact query instruction for `custom`. Required and non-empty; maximum 512 characters. Whitespace is preserved. |
31
+ | `PI_KNOWLEDGE_EMBEDDING_DOCUMENT_INSTRUCTION` | unset | Exact document instruction for `custom`. Required; may be an explicit empty string; maximum 512 characters. Whitespace is preserved. |
32
+ | `PI_KNOWLEDGE_EMBEDDING_MAX_CHARS` | `20000` | Final complete-input API embedding safety cap after instructions are added. Configuration fails if an instruction alone leaves no room for text. This does not replace chunker bounds. |
30
33
  | `PI_KNOWLEDGE_EMBEDDING_BATCH_SIZE` | `64` | Maximum number of document chunks sent in one embedding request during add, update, and import. Lower this for self-hosted embedding servers with smaller request batch limits. |
31
- | `PI_KNOWLEDGE_EMBEDDING_API_FALLBACK` | unset | Set to `local` to explicitly fall back to local embeddings for query embeddings after API failures. Indexing/import/update document batches do not fall back because mixed-provider vector files are unsafe. |
34
+ | `PI_KNOWLEDGE_EMBEDDING_API_FALLBACK` | unset | Set to `local` to explicitly fall back to local embeddings for query embeddings after API failures. Indexing/import/update document batches do not fall back. `jina-code` and `custom` reject local fallback during configuration because the local E5 model cannot reproduce those inputs. |
32
35
  | `PI_KNOWLEDGE_ENABLE_NATIVE_IDLE_DISPOSE` | unset | Enables idle timers for embedding/reranker run coordination. It does not guarantee immediate worker memory release; model-worker shutdown remains tied to engine/session shutdown. Disabled by default for stable shutdown. |
33
36
  | `PI_KNOWLEDGE_EMBEDDING_IDLE_MS` | `30000` | Idle timer used only when native idle disposal coordination is enabled. Mainly for lifecycle stress tests. |
34
37
  | `PI_KNOWLEDGE_OFFLINE` | unset | Use with a pre-populated model cache for offline local model operation. See `docs/offline-mode.md`. |
35
38
 
36
- Each KB stores the embedding model label, vector dimension, and a non-secret embedding signature that includes provider/model semantics, query/document prefixes, non-default prefix strategy, pooling, normalization policy, API input cap, and a hash of the API base URL when applicable. OpenAI-compatible response vectors are validated and L2-normalized by the client before use or persistence; the `client-l2-v1` policy is part of API signatures. `knowledge_search` skips vector retrieval for KBs whose stored signature or dimension is missing or incompatible with the current query embedding and uses BM25-only fallback for hybrid searches. Run `knowledge_update` to rebuild vectors after changing embedding provider/model/max-character/prefix settings or after upgrading an API-backed KB created before client-side normalization was signed.
39
+ Each KB stores the embedding model label, vector dimension, and a non-secret embedding signature that includes provider/model semantics, the input decoration descriptor, pooling, normalization policy, API input cap, and a hash of the API base URL when applicable. E5 `on` and raw-text `off` descriptors remain byte-compatible with existing signatures; Jina signatures include the task and instruction hash, while custom signatures contain only an instruction hash rather than the raw instruction text. OpenAI-compatible response vectors are validated and L2-normalized by the client before use or persistence; the `client-l2-v1` policy is part of API signatures. `knowledge_search` skips vector retrieval for missing or incompatible signatures/dimensions and uses BM25-only fallback for hybrid searches. `knowledge_status` reports indexed and active embedding settings and counts incompatibility in aggregate health. `knowledge_doctor` reports configuration failures and rebuild actions. Run `knowledge_update` after changing provider, model, endpoint, input cap, task, or instructions; re-add inline text or re-import portable sources that cannot be updated.
37
40
 
38
41
  ## Reranker
39
42
 
@@ -60,9 +63,9 @@ Reranker settings affect query-time `deep` search only; they do not change index
60
63
 
61
64
  For Hugging Face Text Embeddings Inference rerankers, set `PI_KNOWLEDGE_RERANKER_API_FORMAT=tei` and point `PI_KNOWLEDGE_RERANKER_API_ENDPOINT` at the TEI `/rerank` endpoint, for example `http://127.0.0.1:8080/rerank`. Do not reuse an embeddings `OPENAI_BASE_URL` ending in `/v1` unless the reranker endpoint actually exists there; TEI's OpenAI-compatible route covers embeddings, while reranking uses `/rerank`.
62
65
 
63
- Local embeddings require a real Node 22+ executable for the isolated model worker. `pi-knowledge` first tries Node IPC and automatically falls back to a stdin/stdout JSONL worker transport when the host runtime does not expose `child_process.fork().send()`, which can happen in Windows OMP compatibility layers. On Windows it searches `PI_KNOWLEDGE_NODE_PATH`, persisted `knowledge_configure` settings, common Node, Volta, NVM, Codex `cua_node`, and `PATH` locations before failing. If worker startup still fails, call `knowledge_configure` with the full `node.exe` path such as `C:\Program Files\nodejs\node.exe`; accidental surrounding quotes are ignored. OpenAI-compatible embeddings avoid the local worker entirely.
66
+ Local embeddings, local/HF reranking, and PDF/DOCX extraction require a real Node 22+ executable for the isolated worker. `pi-knowledge` first tries Node IPC and automatically falls back to a stdin/stdout JSONL worker transport when the host runtime does not expose `child_process.fork().send()`, which can happen in Windows OMP compatibility layers. On Windows it searches `PI_KNOWLEDGE_NODE_PATH`, persisted `knowledge_configure` settings, common Node, Volta, NVM, Codex `cua_node`, and `PATH` locations before failing. If worker startup still fails, call `knowledge_configure` with the full `node.exe` path such as `C:\Program Files\nodejs\node.exe`; accidental surrounding quotes are ignored. OpenAI-compatible embeddings avoid local model loading, but document extraction still uses the worker to isolate parser stdio from the host.
64
67
 
65
- API embedding failures intentionally surface by default. Silent fallback can hide bad API keys, wrong base URLs, unsupported model names, context-window errors, or server batch limits and can produce a KB with a different embedding model than intended. Query-time fallback is opt-in; indexing fallback is disabled to prevent mixed vector spaces in one KB. OpenAI-compatible embedding requests send `Authorization: Bearer <key>` only when `PI_KNOWLEDGE_EMBEDDING_API_KEY` or `OPENAI_API_KEY` is configured.
68
+ API embedding failures intentionally surface by default. Silent fallback can hide bad API keys, wrong base URLs, unsupported model names, context-window errors, server batch limits, or instruction mismatches and can produce a KB with a different embedding model than intended. Query-time fallback is opt-in and supports only E5 `on`/`off` semantics; indexing fallback is disabled to prevent mixed vector spaces in one KB. OpenAI-compatible embedding requests send `Authorization: Bearer <key>` only when `PI_KNOWLEDGE_EMBEDDING_API_KEY` or `OPENAI_API_KEY` is configured. Instruction decoration is applied once before both safety-cap validation and the HTTP request.
66
69
 
67
70
  ## Runtime Features and Diagnostics
68
71
 
@@ -87,7 +90,7 @@ Search diagnostics include result provenance and applied search tuning in both t
87
90
 
88
91
  `knowledge_search` supports `file_type`, substring `path_pattern`, `profile`, integer `limit` from 1 to 50, and non-negative integer `offset`. `offset + limit` cannot exceed the 800-candidate global search window. Each KB retrieves up to the profile-derived candidate limit, then merged results are ranked, filtered, and globally capped before deep reranking, adaptive expansion, or diversity. Search profiles tune result count, snippets, hybrid strictness, candidate breadth, adaptive context, and deep rerank breadth at runtime. `auto` chooses from query shape, mode, and KB source type; explicit tool parameters win over env overrides, env overrides win over profile defaults, and profile defaults win over built-in compatibility values. `hybrid` mode remains lexical-anchored to prevent low-evidence semantic false positives; use `semantic` for pure vector search, or `semantic_hybrid` when weak lexical overlap should not block vector-backed weighted fusion. `adaptive` uses bounded same-file context expansion; when AST metadata is present it can prefer same-parent/sibling chunks without claiming caller/callee graph semantics.
89
92
 
90
- `knowledge_doctor` emits both human-readable issues and machine-readable actions in structured `details`. Action codes include `run_update`, `rebuild_kb`, `wait_for_indexing`, `review_skipped_scope`, `rebuild_vectors`, `check_source`, and `none`. Agents should prefer these action codes over parsing the English action text.
93
+ `knowledge_doctor` emits both human-readable issues and machine-readable actions in structured `details`. Action codes include `run_update`, `rebuild_kb`, `wait_for_indexing`, `review_skipped_scope`, `rebuild_vectors`, `reindex_for_embedding_budget`, `fix_embedding_config`, `check_source`, and `none`. Agents should prefer these action codes over parsing the English action text.
91
94
 
92
95
  `knowledge_configure` writes runtime configuration to the knowledge directory's `config.json`. It validates Node 22+ before persisting `node_path`, so agents can configure a Codex- or user-installed `node.exe` once without relying on environment variables being injected into an already-running OMP process.
93
96
 
@@ -119,7 +122,7 @@ Compatibility guarantees:
119
122
 
120
123
  - The packaged `extension.js` loads built `dist/index.js` when present and falls back to source `index.ts` for local development.
121
124
  - Runtime modules are imported lazily, after the extension is actually used.
122
- - Local embedding and reranking models run in an isolated Node model worker, not in the Pi or OMP TUI process. Node IPC is preferred; stdin/stdout JSONL transport is used automatically when host IPC is unavailable.
125
+ - Local embedding/reranking models and PDF/DOCX parsers run in an isolated Node worker, not in the Pi or OMP TUI process. Node IPC is preferred; stdin/stdout JSONL transport is used automatically when host IPC is unavailable.
123
126
  - Native SQLite loading includes a fallback for hoisted plugin dependency layouts.
124
127
  - OMP path resolution can use `OMP_KNOWLEDGE_DIR`, `OMP_CODING_AGENT_DIR`, and `OMP_PROFILE`.
125
128
  - Existing default Pi knowledge data can remain visible under the default OMP home root through the legacy `~/.pi/knowledge` fallback described above.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "pi-knowledge",
3
- "version": "0.13.0",
3
+ "version": "0.14.0",
4
4
  "description": "Local-first RAG knowledge base for Pi and OMP coding agents with hybrid code/doc search, reranking, diagnostics, and large-project indexing",
5
5
  "type": "module",
6
6
  "main": "dist/index.js",