pi-knowledge 0.5.5 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,28 @@
1
1
  # Changelog
2
2
 
3
+ ## [0.6.0] - 2026-07-25
4
+
5
+ ### Added
6
+ - Added configurable local/Hugging Face reranker selection through `PI_KNOWLEDGE_RERANKER`, including revision, dtype, and trusted mirror settings.
7
+ - Added API reranker support for `deep` search with Cohere/Jina-compatible requests and declarative custom JSON response mapping.
8
+ - Added embedding metadata signatures and dimensions so knowledge bases can detect incompatible vector spaces after provider, API base URL, preprocessing, or dimension changes.
9
+
10
+ ### Changed
11
+ - Document embedding batches now surface OpenAI-compatible API failures instead of falling back to local embeddings during add, update, or import, preventing mixed-provider vector files.
12
+ - `knowledge_search` skips incompatible or legacy unsigned vectors with warnings, while hybrid search can fall back to BM25-only results until `knowledge_update` rebuilds vectors.
13
+
14
+ ### Fixed
15
+ - Rebuilds unchanged chunks during `knowledge_update` when embedding metadata is missing or incompatible, rather than reusing stale vectors.
16
+ - Resets Transformers.js remote model settings before embedding loads so custom reranker mirrors do not affect the embedding pipeline.
17
+
18
+ ## [0.5.6] - 2026-07-23
19
+
20
+ ### Added
21
+ - Added runtime search tuning profiles (`auto`, `low_token`, `precision`, `recall`, `long_context`, `code`, `docs`, `balanced`) with diagnostics that report the applied result limit, snippet length, hybrid threshold, candidate pool, adaptive context, and deep rerank breadth.
22
+
23
+ ### Changed
24
+ - Added bounded environment overrides for search defaults so slow local model users can reduce result count, lengthen snippets, tighten hybrid precision, or tune candidate breadth without editing source.
25
+
3
26
  ## [0.5.5] - 2026-07-23
4
27
 
5
28
  ### Added
package/README.md CHANGED
@@ -125,6 +125,8 @@ omp install ./pi-knowledge
125
125
  - `auto`: selects a primary mode from the query shape and retries alternate modes when results are empty or weak.
126
126
  - `code`, `config`, `errors`, `docs`, `decision`: intent aliases for agents. `code`, `config`, and `errors` bias toward exact lexical/symbol evidence; `docs` and `decision` keep the intended source type explicit.
127
127
 
128
+ Search profiles tune result count, snippet length, hybrid strictness, candidate breadth, adaptive context, and deep rerank breadth. `profile: "auto"` selects a runtime profile from query shape, mode, and KB source type; explicit tool parameters still win over profile defaults. Use `profile: "low_token"` for slow local models that need fewer, stricter, longer results in one call; use `precision` for identifiers/errors, `recall` for broad discovery, `long_context` for prose/specs, and `balanced` for compatibility.
129
+
128
130
  Mode selection contract:
129
131
 
130
132
  - Start with `hybrid` for most project questions that contain useful lexical anchors.
@@ -134,7 +136,7 @@ Mode selection contract:
134
136
  - Use `deep` for high-stakes answers, ambiguous top results, or final verification when slower reranking is acceptable.
135
137
  - If results are empty or weak but the KB should contain the answer, retry once with a different mode before concluding no answer exists.
136
138
 
137
- Search results use balanced diversity reranking by default so near-duplicate chunks from the same file do not dominate the top results. Diversity scoring considers lexical overlap, same-file line proximity, overlapping adaptive windows, available embedding-vector similarity, file-level interleaving, and a small KB trust multiplier that favors ready file/directory sources over stale, URL, or imported text sources. Use `diversity: "off"` only when raw ranking order is needed for diagnostics. Agents can request search diagnostics to inspect mode fallback, ranking coverage, path/source/test boosts, adjusted scores, and provenance such as chunk id, chunk hash, match reason, stale flag, and source freshness where available.
139
+ Search results use balanced diversity reranking by default so near-duplicate chunks from the same file do not dominate the top results. Diversity scoring considers lexical overlap, same-file line proximity, overlapping adaptive windows, available embedding-vector similarity, file-level interleaving, and a small KB trust multiplier that favors ready file/directory sources over stale, URL, or imported text sources. Use `diversity: "off"` only when raw ranking order is needed for diagnostics. Agents can request search diagnostics to inspect mode fallback, selected profile, applied search tuning, ranking coverage, path/source/test boosts, adjusted scores, and provenance such as chunk id, chunk hash, match reason, stale flag, and source freshness where available.
138
140
 
139
141
  For best search quality, rebuild or update existing knowledge bases after upgrading. New indexes use contextual retrieval units: embeddings and FTS include file path, file type, Markdown heading breadcrumbs, and code symbol names while returned results keep the original chunk text readable. This improves queries that mention project structure, filenames, sections, or functions, and reduces duplicate-looking chunk hits.
140
142
  Symbol/config/heading metadata is also rebuilt during `knowledge_update`; older KBs created before this feature may show a doctor action recommending update.
@@ -162,9 +164,11 @@ export PI_KNOWLEDGE_EMBEDDING_BASE_URL=http://127.0.0.1:8080/v1
162
164
  export OPENAI_API_KEY=local-placeholder
163
165
  ```
164
166
 
165
- API embedding failures are surfaced by default so configuration and context-window problems are visible. To explicitly allow a local fallback after API failures, set `PI_KNOWLEDGE_EMBEDDING_API_FALLBACK=local`.
167
+ API embedding failures are surfaced by default so configuration and context-window problems are visible. To explicitly allow query-time local fallback after API failures, set `PI_KNOWLEDGE_EMBEDDING_API_FALLBACK=local`; indexing/import/update document batches still fail instead of creating mixed-provider vector files.
168
+
169
+ API embedding requests are capped at 20000 characters per input by default as a final context-window safety guard for OpenAI-compatible servers. Adjust this with `PI_KNOWLEDGE_EMBEDDING_MAX_CHARS` when your embedding model has a different context window; changing the cap changes embedding compatibility and requires `knowledge_update`.
166
170
 
167
- API embedding requests are capped at 20000 characters per input by default as a final context-window safety guard for OpenAI-compatible servers. Adjust this with `PI_KNOWLEDGE_EMBEDDING_MAX_CHARS` when your embedding model has a different context window.
171
+ Stored KB metadata includes the embedding model, vector dimension, and a non-secret embedding signature. If the current query embedding is incompatible with a KB's stored vectors, or if an older KB has no signature metadata, `knowledge_search` skips vector retrieval for that KB and reports a warning; run `knowledge_update` after changing embedding providers or upgrading older KBs.
168
172
 
169
173
  ## Configuration
170
174
 
@@ -175,8 +179,9 @@ Full configuration details are in [docs/configuration.md](docs/configuration.md)
175
179
  | Storage path | `PI_KNOWLEDGE_DIR`, `OMP_KNOWLEDGE_DIR`, `PI_CODING_AGENT_DIR`, `OMP_CODING_AGENT_DIR`, `OMP_PROFILE` |
176
180
  | Model worker and cache | `PI_KNOWLEDGE_MODEL_CACHE_DIR`, `PI_KNOWLEDGE_NODE_PATH` |
177
181
  | Embedding provider | `PI_KNOWLEDGE_EMBEDDING`, `OPENAI_API_KEY`, `PI_KNOWLEDGE_EMBEDDING_BASE_URL`, `OPENAI_BASE_URL`, `PI_KNOWLEDGE_EMBEDDING_MAX_CHARS`, `PI_KNOWLEDGE_EMBEDDING_API_FALLBACK` |
182
+ | Reranker provider | `PI_KNOWLEDGE_RERANKER`, `PI_KNOWLEDGE_RERANKER_REVISION`, `PI_KNOWLEDGE_RERANKER_DTYPE`, `PI_KNOWLEDGE_RERANKER_REMOTE_HOST`, `PI_KNOWLEDGE_RERANKER_REMOTE_PATH_TEMPLATE`, `PI_KNOWLEDGE_RERANKER_API_ENDPOINT`, `PI_KNOWLEDGE_RERANKER_API_BASE_URL`, `PI_KNOWLEDGE_RERANKER_API_KEY`, `PI_KNOWLEDGE_RERANKER_API_FORMAT`, `PI_KNOWLEDGE_RERANKER_API_TIMEOUT_MS`, `PI_KNOWLEDGE_RERANKER_MAX_DOC_CHARS`, `PI_KNOWLEDGE_RERANKER_API_RESULTS_PATH`, `PI_KNOWLEDGE_RERANKER_API_INDEX_FIELD`, `PI_KNOWLEDGE_RERANKER_API_SCORE_FIELD`, `PI_KNOWLEDGE_RERANKER_API_SCORE_DIRECTION` |
178
183
  | Native lifecycle | `PI_KNOWLEDGE_ENABLE_NATIVE_IDLE_DISPOSE`, `PI_KNOWLEDGE_EMBEDDING_IDLE_MS` |
179
- | Runtime features | `PI_KNOWLEDGE_WATCH`, `PI_KNOWLEDGE_AUTO_INJECT`, `PI_KNOWLEDGE_STALE_INDEXING_MS`, `PI_KNOWLEDGE_OFFLINE` |
184
+ | Runtime features | `PI_KNOWLEDGE_WATCH`, `PI_KNOWLEDGE_AUTO_INJECT`, `PI_KNOWLEDGE_STALE_INDEXING_MS`, `PI_KNOWLEDGE_OFFLINE`, `PI_KNOWLEDGE_SEARCH_PROFILE`, `PI_KNOWLEDGE_SEARCH_DEFAULT_LIMIT`, `PI_KNOWLEDGE_SNIPPET_MAX_LENGTH`, `PI_KNOWLEDGE_MIN_HYBRID_SCORE`, `PI_KNOWLEDGE_SEARCH_CANDIDATE_MIN`, `PI_KNOWLEDGE_SEARCH_CANDIDATE_MULTIPLIER`, `PI_KNOWLEDGE_ADAPTIVE_CONTEXT_LINES`, `PI_KNOWLEDGE_ADAPTIVE_MAX_CHARS`, `PI_KNOWLEDGE_ADAPTIVE_NEIGHBOR_TARGET`, `PI_KNOWLEDGE_DEEP_RERANK_CANDIDATES`, `PI_KNOWLEDGE_DEEP_RERANK_TOPK_MULTIPLIER` |
180
185
  | Release fixtures | `PI_KNOWLEDGE_E2E_PDF`, `PI_KNOWLEDGE_E2E_DOCX` |
181
186
 
182
187
  ## Pi and OMP Support
package/dist/index.js CHANGED
@@ -363,6 +363,7 @@ export default function (pi) {
363
363
  "Use mode 'semantic' for broad conceptual questions when exact terms may differ from the indexed wording or hybrid returns no lexical matches",
364
364
  "Use mode 'adaptive' when the user needs surrounding implementation context, related nearby sections, or enough context to make a code change",
365
365
  "Use mode 'deep' for high-stakes answers, ambiguous top results, or final verification where slower cross-encoder reranking is acceptable",
366
+ "Use profile 'low_token' for slow local models that need fewer, stricter, longer snippets; use profile 'precision' for identifiers/errors and 'recall' or 'long_context' for broad prose/document research",
366
367
  "If a search returns no results or obviously weak results, retry once with a different mode before concluding the KB lacks the answer",
367
368
  "If top results are repetitive, retry with diversity 'strong' or mode 'adaptive' instead of increasing the limit first",
368
369
  ],
@@ -381,7 +382,17 @@ export default function (pi) {
381
382
  Type.Literal("errors"),
382
383
  Type.Literal("decision"),
383
384
  ])),
384
- limit: Type.Optional(Type.Number({ description: "Max results (default 10)" })),
385
+ profile: Type.Optional(Type.Union([
386
+ Type.Literal("auto"),
387
+ Type.Literal("balanced"),
388
+ Type.Literal("low_token"),
389
+ Type.Literal("precision"),
390
+ Type.Literal("recall"),
391
+ Type.Literal("long_context"),
392
+ Type.Literal("code"),
393
+ Type.Literal("docs"),
394
+ ])),
395
+ limit: Type.Optional(Type.Number({ description: "Max results (default comes from search profile/env)" })),
385
396
  kb_id: Type.Optional(Type.String({ description: "Limit search to a specific KB by ID or exact name" })),
386
397
  offset: Type.Optional(Type.Number({ description: "Pagination offset" })),
387
398
  file_type: Type.Optional(Type.String({ description: "Filter by file type (e.g. typescript, markdown, python)" })),
@@ -391,15 +402,18 @@ export default function (pi) {
391
402
  }),
392
403
  async execute(_id, params, _signal) {
393
404
  const { engine } = await ensureInitialized();
394
- const { query, mode, limit, kb_id, offset, file_type, path_pattern, diversity, diagnostics } = params;
405
+ const { query, mode, profile, limit, kb_id, offset, file_type, path_pattern, diversity, diagnostics } = params;
395
406
  const filters = file_type || path_pattern ? { file_type, path_pattern } : undefined;
396
- const response = await engine.search(query, { mode, limit, kb_id, offset, filters, diversity }, _signal);
407
+ const response = await engine.search(query, { mode, profile, limit, kb_id, offset, filters, diversity }, _signal);
397
408
  if (response.results.length === 0) {
398
409
  const details = [
399
410
  "No results found.",
400
411
  response.mode_used ? `Mode used: ${response.mode_used}` : "",
401
412
  response.retry_modes?.length ? `Retried: ${response.retry_modes.join(", ")}` : "",
402
413
  response.warnings?.length ? `Warnings: ${response.warnings.join(" | ")}` : "",
414
+ diagnostics && response.tuning
415
+ ? `Tuning: profile=${response.tuning.requested_profile}->${response.tuning.selected_profile}, limit=${response.tuning.limit}, snippet=${response.tuning.snippet_max_length}, min_hybrid=${response.tuning.min_hybrid_score}, candidates=${response.tuning.candidate_limit}`
416
+ : "",
403
417
  response.suggestions?.length ? `Suggestions: ${response.suggestions.join(" | ")}` : "",
404
418
  ]
405
419
  .filter(Boolean)
@@ -411,6 +425,9 @@ export default function (pi) {
411
425
  output += ` — mode: ${response.mode_used}`;
412
426
  if (response.retry_modes?.length)
413
427
  output += ` — retried: ${response.retry_modes.join(", ")}`;
428
+ if (diagnostics && response.tuning) {
429
+ output += ` — profile: ${response.tuning.requested_profile}->${response.tuning.selected_profile}`;
430
+ }
414
431
  output += ":\n\n";
415
432
  if (response.warnings?.length) {
416
433
  output = `Warnings:\n- ${response.warnings.join("\n- ")}\n\n${output}`;
@@ -423,7 +440,10 @@ export default function (pi) {
423
440
  const provenance = diagnostics && r.provenance
424
441
  ? `\nProvenance: chunk=${r.provenance.chunk_id}, hash=${r.provenance.chunk_hash.slice(0, 12)}, reason=${r.provenance.match_reason}, indexed_at=${r.provenance.indexed_at}, source_mtime=${r.provenance.source_mtime ?? "unknown"}, stale=${r.provenance.stale}`
425
442
  : "";
426
- return `[${i + 1}] ${r.file_path} (${r.kb_name}, score: ${r.score.toFixed(3)})\n${r.snippet}${diag}${provenance}`;
443
+ const tuning = diagnostics && response.tuning && i === 0
444
+ ? `\nTuning: limit=${response.tuning.limit}, snippet=${response.tuning.snippet_max_length}, min_hybrid=${response.tuning.min_hybrid_score}, candidates=${response.tuning.candidate_limit}`
445
+ : "";
446
+ return `[${i + 1}] ${r.file_path} (${r.kb_name}, score: ${r.score.toFixed(3)})\n${r.snippet}${diag}${provenance}${tuning}`;
427
447
  })
428
448
  .join("\n\n");
429
449
  return { content: [{ type: "text", text: output }], details: diagnostics ? response : undefined };
@@ -1,5 +1,37 @@
1
+ export type EmbeddingProvider = "local" | "openai";
2
+ export type EmbeddingPrefix = "query" | "passage";
3
+ export interface EmbeddingConfig {
4
+ provider: EmbeddingProvider;
5
+ model: string;
6
+ baseUrl?: string;
7
+ maxChars: number;
8
+ queryPrefix: "query";
9
+ documentPrefix: "passage";
10
+ pooling: "mean";
11
+ normalize: true;
12
+ }
13
+ export declare const CURRENT_EMBEDDING_MODEL = "multilingual-e5-small";
14
+ export declare const DEFAULT_LOCAL_EMBEDDING_MODEL = "multilingual-e5-small";
15
+ export declare const DEFAULT_OPENAI_EMBEDDING_MODEL = "text-embedding-3-small";
16
+ export declare const DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1";
17
+ export declare const DEFAULT_API_MAX_EMBED_CHARS = 20000;
18
+ export declare function resolveEmbeddingConfig(env?: NodeJS.ProcessEnv): EmbeddingConfig;
19
+ export declare function embeddingConfigLabel(config: EmbeddingConfig): string;
20
+ export declare function embeddingSignature(config: EmbeddingConfig, dimension: number): string;
1
21
  export declare function dispose(): Promise<void>;
2
22
  export declare function prepareForShutdown(): Promise<void>;
3
- export declare function embedTexts(texts: string[], prefix: "query" | "passage", signal?: AbortSignal): Promise<Float32Array[]>;
23
+ export declare function embedTextsWithConfig(texts: string[], prefix: EmbeddingPrefix, signal?: AbortSignal, options?: {
24
+ allowApiFallback?: boolean;
25
+ }): Promise<{
26
+ vectors: Float32Array[];
27
+ config: EmbeddingConfig;
28
+ }>;
29
+ export declare function embedTexts(texts: string[], prefix: EmbeddingPrefix, signal?: AbortSignal, options?: {
30
+ allowApiFallback?: boolean;
31
+ }): Promise<Float32Array[]>;
32
+ export declare function embedQueryWithConfig(text: string, signal?: AbortSignal): Promise<{
33
+ vector: Float32Array;
34
+ config: EmbeddingConfig;
35
+ }>;
4
36
  export declare function embedQuery(text: string, signal?: AbortSignal): Promise<Float32Array>;
5
37
  export declare function embedDocuments(texts: string[], signal?: AbortSignal): Promise<Float32Array[]>;
@@ -1,15 +1,33 @@
1
+ import { createHash } from "node:crypto";
1
2
  import { embedInModelWorker } from "../model-worker-client.js";
2
3
  let disposeTimer = null;
3
4
  let disposePromise = null;
4
5
  let activeRuns = 0;
5
6
  let disposeRequested = false;
6
7
  const idleWaiters = [];
8
+ export const CURRENT_EMBEDDING_MODEL = "multilingual-e5-small";
9
+ export const DEFAULT_LOCAL_EMBEDDING_MODEL = "multilingual-e5-small";
10
+ export const DEFAULT_OPENAI_EMBEDDING_MODEL = "text-embedding-3-small";
11
+ export const DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1";
12
+ export const DEFAULT_API_MAX_EMBED_CHARS = 20_000;
7
13
  const IDLE_TIMEOUT_MS = Number(process.env.PI_KNOWLEDGE_EMBEDDING_IDLE_MS ?? 30_000);
8
- const EMBEDDING_CONFIG = process.env.PI_KNOWLEDGE_EMBEDDING ?? "local:multilingual-e5-small";
9
14
  const ENABLE_NATIVE_IDLE_DISPOSE = process.env.PI_KNOWLEDGE_ENABLE_NATIVE_IDLE_DISPOSE === "true";
10
15
  const API_FALLBACK_TO_LOCAL = process.env.PI_KNOWLEDGE_EMBEDDING_API_FALLBACK === "local";
11
- const DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1";
12
- const DEFAULT_API_MAX_EMBED_CHARS = 20_000;
16
+ function localEmbeddingConfig(maxChars) {
17
+ return {
18
+ provider: "local",
19
+ model: DEFAULT_LOCAL_EMBEDDING_MODEL,
20
+ maxChars,
21
+ queryPrefix: "query",
22
+ documentPrefix: "passage",
23
+ pooling: "mean",
24
+ normalize: true,
25
+ };
26
+ }
27
+ function cleanEnv(value) {
28
+ const trimmed = value?.trim();
29
+ return trimmed ? trimmed : undefined;
30
+ }
13
31
  function isAbortError(error) {
14
32
  return typeof error === "object" && error !== null && "name" in error && error.name === "AbortError";
15
33
  }
@@ -44,6 +62,52 @@ function waitForNoActiveRuns() {
44
62
  return Promise.resolve();
45
63
  return new Promise((resolve) => idleWaiters.push(resolve));
46
64
  }
65
+ function providerInstanceHash(value) {
66
+ return createHash("sha256").update(value).digest("hex").slice(0, 12);
67
+ }
68
+ export function resolveEmbeddingConfig(env = process.env) {
69
+ const raw = cleanEnv(env.PI_KNOWLEDGE_EMBEDDING) ?? `local:${DEFAULT_LOCAL_EMBEDDING_MODEL}`;
70
+ const separator = raw.indexOf(":");
71
+ const provider = separator >= 0 ? raw.slice(0, separator).toLowerCase() : raw.toLowerCase();
72
+ const model = separator >= 0 ? raw.slice(separator + 1).trim() : "";
73
+ const maxCharsValue = Number(cleanEnv(env.PI_KNOWLEDGE_EMBEDDING_MAX_CHARS) ?? DEFAULT_API_MAX_EMBED_CHARS);
74
+ const maxChars = Number.isFinite(maxCharsValue) && maxCharsValue > 0 ? Math.trunc(maxCharsValue) : DEFAULT_API_MAX_EMBED_CHARS;
75
+ if (provider === "local") {
76
+ const localModel = model || DEFAULT_LOCAL_EMBEDDING_MODEL;
77
+ if (localModel !== DEFAULT_LOCAL_EMBEDDING_MODEL) {
78
+ throw new Error(`Unsupported local embedding model: ${localModel}`);
79
+ }
80
+ return localEmbeddingConfig(maxChars);
81
+ }
82
+ if (provider === "openai") {
83
+ return {
84
+ provider: "openai",
85
+ model: model || DEFAULT_OPENAI_EMBEDDING_MODEL,
86
+ baseUrl: cleanEnv(env.PI_KNOWLEDGE_EMBEDDING_BASE_URL) ?? cleanEnv(env.OPENAI_BASE_URL) ?? DEFAULT_OPENAI_BASE_URL,
87
+ maxChars,
88
+ queryPrefix: "query",
89
+ documentPrefix: "passage",
90
+ pooling: "mean",
91
+ normalize: true,
92
+ };
93
+ }
94
+ throw new Error(`Unsupported embedding provider: ${provider}`);
95
+ }
96
+ export function embeddingConfigLabel(config) {
97
+ return config.provider === "local" ? config.model : `${config.provider}:${config.model}`;
98
+ }
99
+ export function embeddingSignature(config, dimension) {
100
+ const base = config.baseUrl ? `:base-sha256=${providerInstanceHash(config.baseUrl)}` : "";
101
+ return [
102
+ `${config.provider}:${config.model}${base}`,
103
+ `dim=${dimension}`,
104
+ `apiMax=${config.provider === "openai" ? config.maxChars : "none"}`,
105
+ `pooling=${config.pooling}`,
106
+ `normalize=${config.normalize}`,
107
+ `q=${config.queryPrefix}`,
108
+ `d=${config.documentPrefix}`,
109
+ ].join(":");
110
+ }
47
111
  export async function dispose() {
48
112
  clearIdleTimer();
49
113
  if (disposePromise)
@@ -60,62 +124,64 @@ export async function prepareForShutdown() {
60
124
  clearIdleTimer();
61
125
  await waitForNoActiveRuns();
62
126
  }
63
- async function embedViaAPI(texts, prefix, signal) {
64
- const [provider, model] = EMBEDDING_CONFIG.split(":");
65
- const configuredMaxChars = Number(process.env.PI_KNOWLEDGE_EMBEDDING_MAX_CHARS ?? DEFAULT_API_MAX_EMBED_CHARS);
66
- const maxChars = Number.isFinite(configuredMaxChars) && configuredMaxChars > 0 ? configuredMaxChars : DEFAULT_API_MAX_EMBED_CHARS;
127
+ async function embedViaAPI(texts, prefix, config, signal) {
67
128
  const prefixedTexts = texts.map((t) => `${prefix}: ${t}`);
68
- const safeTexts = prefixedTexts.map((text) => (text.length > maxChars ? text.slice(0, maxChars) : text));
69
- if (provider === "openai") {
70
- const apiKey = process.env.OPENAI_API_KEY;
71
- if (!apiKey)
72
- throw new Error("OPENAI_API_KEY required for openai embedding");
73
- const baseUrl = process.env.PI_KNOWLEDGE_EMBEDDING_BASE_URL?.trim() ||
74
- process.env.OPENAI_BASE_URL?.trim() ||
75
- DEFAULT_OPENAI_BASE_URL;
76
- const endpoint = new URL("embeddings", `${baseUrl.replace(/\/+$/, "")}/`);
77
- const res = await fetch(endpoint, {
78
- method: "POST",
79
- headers: { "Content-Type": "application/json", Authorization: `Bearer ${apiKey}` },
80
- body: JSON.stringify({ input: safeTexts, model: model || "text-embedding-3-small" }),
81
- signal,
82
- });
83
- if (!res.ok) {
84
- const detail = await res.text().catch(() => "");
85
- throw new Error(`OpenAI embedding API error: ${res.status}${detail ? ` ${detail.slice(0, 500)}` : ""}`);
86
- }
87
- const data = (await res.json());
88
- return data.data.map((d) => new Float32Array(d.embedding));
129
+ const safeTexts = prefixedTexts.map((text) => text.length > config.maxChars ? text.slice(0, config.maxChars) : text);
130
+ const apiKey = process.env.OPENAI_API_KEY;
131
+ if (!apiKey)
132
+ throw new Error("OPENAI_API_KEY required for openai embedding");
133
+ if (!config.baseUrl)
134
+ throw new Error("OpenAI embedding base URL is not configured");
135
+ const endpoint = new URL("embeddings", `${config.baseUrl.replace(/\/+$/, "")}/`);
136
+ const res = await fetch(endpoint, {
137
+ method: "POST",
138
+ headers: { "Content-Type": "application/json", Authorization: `Bearer ${apiKey}` },
139
+ body: JSON.stringify({ input: safeTexts, model: config.model }),
140
+ signal,
141
+ });
142
+ if (!res.ok) {
143
+ const detail = await res.text().catch(() => "");
144
+ throw new Error(`OpenAI embedding API error: ${res.status}${detail ? ` ${detail.slice(0, 500)}` : ""}`);
89
145
  }
90
- throw new Error(`Unsupported embedding provider: ${provider}`);
146
+ const data = (await res.json());
147
+ return data.data.map((d) => new Float32Array(d.embedding));
91
148
  }
92
- export async function embedTexts(texts, prefix, signal) {
93
- if (!EMBEDDING_CONFIG.startsWith("local")) {
149
+ export async function embedTextsWithConfig(texts, prefix, signal, options = {}) {
150
+ const config = resolveEmbeddingConfig();
151
+ let actualConfig = config;
152
+ if (config.provider === "openai") {
94
153
  if (signal?.aborted)
95
154
  throw new Error("Cancelled");
96
155
  try {
97
- return await embedViaAPI(texts, prefix, signal);
156
+ return { vectors: await embedViaAPI(texts, prefix, config, signal), config };
98
157
  }
99
158
  catch (error) {
100
159
  if (signal?.aborted || isAbortError(error))
101
160
  throw new Error("Cancelled");
102
- if (!API_FALLBACK_TO_LOCAL)
161
+ if (!API_FALLBACK_TO_LOCAL || !options.allowApiFallback)
103
162
  throw error;
104
163
  console.warn(`pi-knowledge: embedding API failed; falling back to local model because PI_KNOWLEDGE_EMBEDDING_API_FALLBACK=local (${error instanceof Error ? error.message : String(error)})`);
164
+ actualConfig = localEmbeddingConfig(config.maxChars);
105
165
  }
106
166
  }
107
167
  beginRun();
108
168
  try {
109
- return await embedInModelWorker(texts, prefix, signal);
169
+ return { vectors: await embedInModelWorker(texts, prefix, signal), config: actualConfig };
110
170
  }
111
171
  finally {
112
172
  endRun();
113
173
  }
114
174
  }
175
+ export async function embedTexts(texts, prefix, signal, options = {}) {
176
+ return (await embedTextsWithConfig(texts, prefix, signal, options)).vectors;
177
+ }
178
+ export async function embedQueryWithConfig(text, signal) {
179
+ const { vectors, config } = await embedTextsWithConfig([text], "query", signal, { allowApiFallback: true });
180
+ return { vector: vectors[0], config };
181
+ }
115
182
  export async function embedQuery(text, signal) {
116
- const [vec] = await embedTexts([text], "query", signal);
117
- return vec;
183
+ return (await embedQueryWithConfig(text, signal)).vector;
118
184
  }
119
185
  export async function embedDocuments(texts, signal) {
120
- return embedTexts(texts, "passage", signal);
186
+ return embedTexts(texts, "passage", signal, { allowApiFallback: false });
121
187
  }
@@ -1,10 +1,12 @@
1
1
  import { type DiagnosticResult } from "./diagnostics/health.ts";
2
2
  import { createSkippedScanStats } from "./indexer/chunker.ts";
3
3
  import { type RankingDiagnostics } from "./search/ranking.ts";
4
+ import { type SearchProfile, type SearchTuningSummary } from "./search/tuning.ts";
4
5
  import { type KnowledgeBase, type KnowledgeSymbol } from "./storage/sqlite.ts";
5
6
  export type SearchMode = "auto" | "fast" | "semantic" | "hybrid" | "deep" | "adaptive" | "code" | "config" | "docs" | "errors" | "decision";
6
7
  export interface SearchOptions {
7
8
  mode?: SearchMode;
9
+ profile?: SearchProfile;
8
10
  limit?: number;
9
11
  offset?: number;
10
12
  kb_id?: string;
@@ -34,7 +36,7 @@ export interface SearchResult {
34
36
  source_chunk_ids?: string[];
35
37
  };
36
38
  }
37
- export declare const CURRENT_EMBEDDING_MODEL = "multilingual-e5-small";
39
+ export { CURRENT_EMBEDDING_MODEL } from "./embedding/provider.ts";
38
40
  export interface SearchResponse {
39
41
  results: SearchResult[];
40
42
  total_count: number;
@@ -43,6 +45,7 @@ export interface SearchResponse {
43
45
  mode_used?: SearchMode;
44
46
  retry_modes?: SearchMode[];
45
47
  suggestions?: string[];
48
+ tuning?: SearchTuningSummary;
46
49
  }
47
50
  export type ProgressCallback = (msg: string) => void;
48
51
  export interface AddOptions {