pi-knowledge 0.7.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,6 +1,12 @@
1
1
  # Changelog
2
2
 
3
3
 
4
+ ## [0.8.0] - 2026-08-04
5
+
6
+ ### Added
7
+ - Added Hugging Face Text Embeddings Inference reranker API format support through `PI_KNOWLEDGE_RERANKER_API_FORMAT=tei`.
8
+
9
+
4
10
  ## [0.7.0] - 2026-07-28
5
11
 
6
12
  ### Added
package/README.md CHANGED
@@ -184,6 +184,8 @@ Full configuration details are in [docs/configuration.md](docs/configuration.md)
184
184
  | Runtime features | `PI_KNOWLEDGE_WATCH`, `PI_KNOWLEDGE_AUTO_INJECT`, `PI_KNOWLEDGE_STALE_INDEXING_MS`, `PI_KNOWLEDGE_OFFLINE`, `PI_KNOWLEDGE_SEARCH_PROFILE`, `PI_KNOWLEDGE_SEARCH_DEFAULT_LIMIT`, `PI_KNOWLEDGE_SNIPPET_MAX_LENGTH`, `PI_KNOWLEDGE_MIN_HYBRID_SCORE`, `PI_KNOWLEDGE_SEARCH_CANDIDATE_MIN`, `PI_KNOWLEDGE_SEARCH_CANDIDATE_MULTIPLIER`, `PI_KNOWLEDGE_ADAPTIVE_CONTEXT_LINES`, `PI_KNOWLEDGE_ADAPTIVE_MAX_CHARS`, `PI_KNOWLEDGE_ADAPTIVE_NEIGHBOR_TARGET`, `PI_KNOWLEDGE_DEEP_RERANK_CANDIDATES`, `PI_KNOWLEDGE_DEEP_RERANK_TOPK_MULTIPLIER` |
185
185
  | Release fixtures | `PI_KNOWLEDGE_E2E_PDF`, `PI_KNOWLEDGE_E2E_DOCX` |
186
186
 
187
+ For Hugging Face Text Embeddings Inference rerankers, use `PI_KNOWLEDGE_RERANKER=api:<model>`, `PI_KNOWLEDGE_RERANKER_API_FORMAT=tei`, and `PI_KNOWLEDGE_RERANKER_API_ENDPOINT=http://127.0.0.1:8080/rerank`; TEI exposes OpenAI-compatible embeddings under `/v1`, but reranking uses `/rerank`.
188
+
187
189
  ## Pi and OMP Support
188
190
 
189
191
  `pi-knowledge` supports Pi and [OMP](https://omp.sh/) extension loading through the packaged `extension.js` entry shim. The entry stays startup-light: install-time validation can inspect the extension without resolving native runtime dependencies, and runtime modules load lazily only when tools or lifecycle hooks need them.
@@ -37,12 +37,12 @@ function mapApiResult(result, candidates, config) {
37
37
  return { chunkId: candidates[index].chunkId, score };
38
38
  }
39
39
  function requestBody(query, candidates, topK, config) {
40
- const documents = candidates.map((candidate) => {
41
- const text = candidate.content.length > config.maxDocumentChars
42
- ? candidate.content.slice(0, config.maxDocumentChars)
43
- : candidate.content;
44
- return config.format === "custom-json" ? { id: candidate.chunkId, text } : text;
45
- });
40
+ const texts = candidates.map((candidate) => candidate.content.length > config.maxDocumentChars
41
+ ? candidate.content.slice(0, config.maxDocumentChars)
42
+ : candidate.content);
43
+ if (config.format === "tei")
44
+ return JSON.stringify({ query, texts });
45
+ const documents = config.format === "custom-json" ? texts.map((text, index) => ({ id: candidates[index].chunkId, text })) : texts;
46
46
  return JSON.stringify({
47
47
  model: config.model,
48
48
  query,
@@ -1,5 +1,5 @@
1
1
  export type RerankerProvider = "hf" | "api";
2
- export type ApiRerankerFormat = "cohere" | "jina" | "custom-json";
2
+ export type ApiRerankerFormat = "cohere" | "jina" | "custom-json" | "tei";
3
3
  export type ApiScoreDirection = "desc" | "asc";
4
4
  export interface HfRerankerConfig {
5
5
  provider: "hf";
@@ -60,8 +60,9 @@ function normalizeApiFormat(value) {
60
60
  if (value === undefined)
61
61
  return "cohere";
62
62
  const normalized = value.toLowerCase();
63
- if (normalized === "cohere" || normalized === "jina" || normalized === "custom-json")
63
+ if (normalized === "cohere" || normalized === "jina" || normalized === "custom-json" || normalized === "tei") {
64
64
  return normalized;
65
+ }
65
66
  throw new Error(`Unsupported reranker API format: ${value}`);
66
67
  }
67
68
  function normalizeScoreDirection(value) {
@@ -87,17 +88,18 @@ function resolveApiRerankerConfig(model, env) {
87
88
  endpointFromBaseUrl(cleanEnv(env.PI_KNOWLEDGE_RERANKER_API_BASE_URL) ??
88
89
  cleanEnv(env.OPENAI_BASE_URL) ??
89
90
  "https://api.cohere.com/v2"));
91
+ const format = normalizeApiFormat(cleanEnv(env.PI_KNOWLEDGE_RERANKER_API_FORMAT));
90
92
  return {
91
93
  provider: "api",
92
94
  model,
93
95
  endpoint,
94
96
  apiKey: cleanEnv(env.PI_KNOWLEDGE_RERANKER_API_KEY) ?? cleanEnv(env.OPENAI_API_KEY),
95
- format: normalizeApiFormat(cleanEnv(env.PI_KNOWLEDGE_RERANKER_API_FORMAT)),
97
+ format,
96
98
  timeoutMs: parseNumber(cleanEnv(env.PI_KNOWLEDGE_RERANKER_API_TIMEOUT_MS), DEFAULT_RERANKER_API_TIMEOUT_MS, 1_000, 300_000),
97
99
  maxDocumentChars: parseNumber(cleanEnv(env.PI_KNOWLEDGE_RERANKER_MAX_DOC_CHARS), DEFAULT_RERANKER_MAX_DOC_CHARS, 100, 200_000),
98
- resultsPath: cleanEnv(env.PI_KNOWLEDGE_RERANKER_API_RESULTS_PATH) ?? "results",
100
+ resultsPath: cleanEnv(env.PI_KNOWLEDGE_RERANKER_API_RESULTS_PATH) ?? (format === "tei" ? "" : "results"),
99
101
  indexField: cleanEnv(env.PI_KNOWLEDGE_RERANKER_API_INDEX_FIELD) ?? "index",
100
- scoreField: cleanEnv(env.PI_KNOWLEDGE_RERANKER_API_SCORE_FIELD) ?? "relevance_score",
102
+ scoreField: cleanEnv(env.PI_KNOWLEDGE_RERANKER_API_SCORE_FIELD) ?? (format === "tei" ? "score" : "relevance_score"),
101
103
  scoreDirection: normalizeScoreDirection(cleanEnv(env.PI_KNOWLEDGE_RERANKER_API_SCORE_DIRECTION)),
102
104
  };
103
105
  }
@@ -45,16 +45,18 @@ Each KB stores the embedding model label, vector dimension, and a non-secret emb
45
45
  | `PI_KNOWLEDGE_RERANKER_API_ENDPOINT` | derived from base URL | Full HTTP rerank endpoint. Takes precedence over `PI_KNOWLEDGE_RERANKER_API_BASE_URL` and `OPENAI_BASE_URL`. |
46
46
  | `PI_KNOWLEDGE_RERANKER_API_BASE_URL` | `https://api.cohere.com/v2` | Base URL for Cohere/Jina-compatible `/rerank` APIs. `OPENAI_BASE_URL` is accepted only as a convenience fallback for gateways exposing the same schema. |
47
47
  | `PI_KNOWLEDGE_RERANKER_API_KEY` | `OPENAI_API_KEY` fallback | Bearer token for API rerankers. Omit only for trusted local services that do not require auth. |
48
- | `PI_KNOWLEDGE_RERANKER_API_FORMAT` | `cohere` | Request/response preset: `cohere`, `jina`, or `custom-json`. All presets send `model`, `query`, `documents`, `top_n`, and `return_documents:false`. |
48
+ | `PI_KNOWLEDGE_RERANKER_API_FORMAT` | `cohere` | Request/response preset: `cohere`, `jina`, `custom-json`, or `tei`. Cohere/Jina-compatible presets send `model`, `query`, `documents`, `top_n`, and `return_documents:false`; `tei` sends `query` plus `texts` and parses a root response array with `index`/`score`. |
49
49
  | `PI_KNOWLEDGE_RERANKER_API_TIMEOUT_MS` | `30000` | Per-request timeout for API reranking. |
50
50
  | `PI_KNOWLEDGE_RERANKER_MAX_DOC_CHARS` | `12000` | Per-candidate document safety cap before sending content to the API. |
51
- | `PI_KNOWLEDGE_RERANKER_API_RESULTS_PATH` | `results` | Dot path to the result array for `custom-json` or compatible responses. |
51
+ | `PI_KNOWLEDGE_RERANKER_API_RESULTS_PATH` | `results` (`tei`: root response array) | Dot path to the result array for `custom-json` or compatible responses. |
52
52
  | `PI_KNOWLEDGE_RERANKER_API_INDEX_FIELD` | `index` | Field name or dot path holding the original candidate index. |
53
- | `PI_KNOWLEDGE_RERANKER_API_SCORE_FIELD` | `relevance_score` | Field name or dot path holding the rerank score. |
53
+ | `PI_KNOWLEDGE_RERANKER_API_SCORE_FIELD` | `relevance_score` (`tei`: `score`) | Field name or dot path holding the rerank score. |
54
54
  | `PI_KNOWLEDGE_RERANKER_API_SCORE_DIRECTION` | `desc` | Sort direction for returned scores: `desc` when higher is better, `asc` when lower is better. |
55
55
 
56
56
  Reranker settings affect query-time `deep` search only; they do not change indexed KB vectors. Local/HF rerankers run in the isolated model worker. Offline mode sets the model worker to local-only loading, so a local/HF reranker must already exist in the model cache. `PI_KNOWLEDGE_RERANKER_RAW_LOGITS=true` switches local/HF rerankers from the Transformers.js text-classification pipeline to direct sequence-classification logits and is intended only for trusted single-logit reranker models; use trusted model repositories or pinned revisions for non-default artifacts. API rerankers send the query and selected candidate chunk content to the configured external service; failures surface by default and do not silently fall back to the local worker.
57
57
 
58
+ For Hugging Face Text Embeddings Inference rerankers, set `PI_KNOWLEDGE_RERANKER_API_FORMAT=tei` and point `PI_KNOWLEDGE_RERANKER_API_ENDPOINT` at the TEI `/rerank` endpoint, for example `http://127.0.0.1:8080/rerank`. Do not reuse an embeddings `OPENAI_BASE_URL` ending in `/v1` unless the reranker endpoint actually exists there; TEI's OpenAI-compatible route covers embeddings, while reranking uses `/rerank`.
59
+
58
60
  Local embeddings require a real Node 22+ executable for the isolated model worker. `pi-knowledge` first tries Node IPC and automatically falls back to a stdin/stdout JSONL worker transport when the host runtime does not expose `child_process.fork().send()`, which can happen in Windows OMP compatibility layers. On Windows it searches `PI_KNOWLEDGE_NODE_PATH`, persisted `knowledge_configure` settings, common Node, Volta, NVM, Codex `cua_node`, and `PATH` locations before failing. If worker startup still fails, call `knowledge_configure` with the full `node.exe` path such as `C:\Program Files\nodejs\node.exe`; accidental surrounding quotes are ignored. OpenAI-compatible embeddings avoid the local worker entirely.
59
61
 
60
62
  API embedding failures intentionally surface by default. Silent fallback can hide bad API keys, wrong base URLs, unsupported model names, or context-window errors and can produce a KB with a different embedding model than intended. Query-time fallback is opt-in; indexing fallback is disabled to prevent mixed vector spaces in one KB.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "pi-knowledge",
3
- "version": "0.7.0",
3
+ "version": "0.8.0",
4
4
  "description": "Local-first RAG knowledge base for Pi and OMP coding agents with hybrid code/doc search, reranking, diagnostics, and large-project indexing",
5
5
  "type": "module",
6
6
  "main": "dist/index.js",