pi-knowledge 0.13.0 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,6 @@
1
1
  import { createHash } from "node:crypto";
2
2
  import { embedInModelWorker, measureEmbeddingInputInModelWorker } from "../model-worker-client.js";
3
+ import { decorateEmbeddingInput, decorationSupportedByLocalModel, embeddingDecorationDescriptor, embeddingInstructionLength, resolveEmbeddingDecoration, } from "./instructions.js";
3
4
  let disposeTimer = null;
4
5
  let disposePromise = null;
5
6
  let activeRuns = 0;
@@ -15,15 +16,16 @@ export const DEFAULT_API_MAX_EMBED_CHARS = 20_000;
15
16
  export const API_VECTOR_NORMALIZATION_POLICY = "client-l2-v1";
16
17
  const IDLE_TIMEOUT_MS = Number(process.env.PI_KNOWLEDGE_EMBEDDING_IDLE_MS ?? 30_000);
17
18
  const ENABLE_NATIVE_IDLE_DISPOSE = process.env.PI_KNOWLEDGE_ENABLE_NATIVE_IDLE_DISPOSE === "true";
18
- const API_FALLBACK_TO_LOCAL = process.env.PI_KNOWLEDGE_EMBEDDING_API_FALLBACK === "local";
19
- function localEmbeddingConfig(maxChars, prefixes) {
19
+ function localEmbeddingConfig(maxChars, decoration) {
20
20
  return {
21
21
  provider: "local",
22
22
  model: DEFAULT_LOCAL_EMBEDDING_MODEL,
23
23
  maxChars,
24
- queryPrefix: "query",
25
- documentPrefix: "passage",
26
- prefixes,
24
+ strategy: decoration.strategy,
25
+ task: decoration.task,
26
+ instructions: decoration.instructions,
27
+ prefixMode: decoration.prefixMode,
28
+ apiFallbackToLocal: false,
27
29
  pooling: "mean",
28
30
  normalize: true,
29
31
  };
@@ -36,8 +38,7 @@ function isAbortError(error) {
36
38
  return typeof error === "object" && error !== null && "name" in error && error.name === "AbortError";
37
39
  }
38
40
  function clearIdleTimer() {
39
- if (disposeTimer)
40
- clearTimeout(disposeTimer);
41
+ clearTimeout(disposeTimer ?? undefined);
41
42
  disposeTimer = null;
42
43
  }
43
44
  function scheduleIdleDispose() {
@@ -61,21 +62,23 @@ function endRun() {
61
62
  if (!disposeRequested)
62
63
  scheduleIdleDispose();
63
64
  }
64
- function waitForNoActiveRuns() {
65
+ async function waitForNoActiveRuns() {
65
66
  if (activeRuns === 0)
66
- return Promise.resolve();
67
- return new Promise((resolve) => idleWaiters.push(resolve));
67
+ return;
68
+ const { promise, resolve } = Promise.withResolvers();
69
+ idleWaiters.push(resolve);
70
+ await promise;
68
71
  }
69
72
  function providerInstanceHash(value) {
70
73
  return createHash("sha256").update(value).digest("hex").slice(0, 12);
71
74
  }
72
- function resolveEmbeddingPrefixStrategy(env) {
73
- const raw = cleanEnv(env.PI_KNOWLEDGE_EMBEDDING_PREFIXES)?.toLowerCase();
74
- if (!raw)
75
- return "on";
76
- if (raw === "on" || raw === "off")
77
- return raw;
78
- throw new Error(`Unsupported embedding prefix strategy: ${raw}. Use PI_KNOWLEDGE_EMBEDDING_PREFIXES=on or off.`);
75
+ function assertApiInstructionBudget(config) {
76
+ for (const role of ["query", "passage"]) {
77
+ const instructionLength = embeddingInstructionLength(config.instructions, role);
78
+ if (instructionLength >= config.maxChars) {
79
+ throw new Error(`Embedding ${role} instruction is ${instructionLength} characters; PI_KNOWLEDGE_EMBEDDING_MAX_CHARS=${config.maxChars} must leave room for input text.`);
80
+ }
81
+ }
79
82
  }
80
83
  export function resolveEmbeddingConfig(env = process.env) {
81
84
  const raw = cleanEnv(env.PI_KNOWLEDGE_EMBEDDING) ?? `local:${DEFAULT_LOCAL_EMBEDDING_MODEL}`;
@@ -84,26 +87,41 @@ export function resolveEmbeddingConfig(env = process.env) {
84
87
  const model = separator >= 0 ? raw.slice(separator + 1).trim() : "";
85
88
  const maxCharsValue = Number(cleanEnv(env.PI_KNOWLEDGE_EMBEDDING_MAX_CHARS) ?? DEFAULT_API_MAX_EMBED_CHARS);
86
89
  const maxChars = Number.isFinite(maxCharsValue) && maxCharsValue > 0 ? Math.trunc(maxCharsValue) : DEFAULT_API_MAX_EMBED_CHARS;
87
- const prefixes = resolveEmbeddingPrefixStrategy(env);
90
+ const decoration = resolveEmbeddingDecoration(env);
91
+ const fallback = cleanEnv(env.PI_KNOWLEDGE_EMBEDDING_API_FALLBACK)?.toLowerCase();
92
+ if (fallback && fallback !== "local") {
93
+ throw new Error("Unsupported embedding API fallback. Use PI_KNOWLEDGE_EMBEDDING_API_FALLBACK=local or unset it.");
94
+ }
95
+ const apiFallbackToLocal = fallback === "local";
88
96
  if (provider === "local") {
89
97
  const localModel = model || DEFAULT_LOCAL_EMBEDDING_MODEL;
90
98
  if (localModel !== DEFAULT_LOCAL_EMBEDDING_MODEL) {
91
99
  throw new Error(`Unsupported local embedding model: ${localModel}`);
92
100
  }
93
- return localEmbeddingConfig(maxChars, prefixes);
101
+ if (!decorationSupportedByLocalModel(decoration.strategy)) {
102
+ throw new Error(`Embedding decoration ${decoration.strategy} is not supported for the local embedding model. Use PI_KNOWLEDGE_EMBEDDING_PREFIXES=on|off|e5|none, or an OpenAI-compatible provider.`);
103
+ }
104
+ return localEmbeddingConfig(maxChars, decoration);
94
105
  }
95
106
  if (provider === "openai") {
96
- return {
107
+ if (apiFallbackToLocal && !decorationSupportedByLocalModel(decoration.strategy)) {
108
+ throw new Error(`PI_KNOWLEDGE_EMBEDDING_API_FALLBACK=local is incompatible with embedding decoration ${decoration.strategy}; the local model supports only E5 prefixes or none.`);
109
+ }
110
+ const config = {
97
111
  provider: "openai",
98
112
  model: model || DEFAULT_OPENAI_EMBEDDING_MODEL,
99
113
  baseUrl: cleanEnv(env.PI_KNOWLEDGE_EMBEDDING_BASE_URL) ?? cleanEnv(env.OPENAI_BASE_URL) ?? DEFAULT_OPENAI_BASE_URL,
100
114
  maxChars,
101
- queryPrefix: "query",
102
- documentPrefix: "passage",
103
- prefixes,
115
+ strategy: decoration.strategy,
116
+ task: decoration.task,
117
+ instructions: decoration.instructions,
118
+ prefixMode: decoration.prefixMode,
119
+ apiFallbackToLocal,
104
120
  pooling: "mean",
105
121
  normalize: true,
106
122
  };
123
+ assertApiInstructionBudget(config);
124
+ return config;
107
125
  }
108
126
  throw new Error(`Unsupported embedding provider: ${provider}`);
109
127
  }
@@ -125,29 +143,86 @@ export function normalizeEmbeddingVector(vector) {
125
143
  vector[index] /= magnitude;
126
144
  return vector;
127
145
  }
146
+ export function describeEmbeddingDecoration(config) {
147
+ return embeddingDecorationDescriptor({
148
+ strategy: config.strategy,
149
+ task: config.task,
150
+ instructions: config.instructions,
151
+ prefixMode: config.prefixMode,
152
+ });
153
+ }
128
154
  export function embeddingSignature(config, dimension) {
129
155
  const base = config.baseUrl ? `:base-sha256=${providerInstanceHash(config.baseUrl)}` : "";
130
156
  const normalization = config.provider === "openai" ? API_VECTOR_NORMALIZATION_POLICY : String(config.normalize);
131
- const parts = [
157
+ return [
132
158
  `${config.provider}:${config.model}${base}`,
133
159
  `dim=${dimension}`,
134
160
  `apiMax=${config.provider === "openai" ? config.maxChars : "none"}`,
135
161
  `pooling=${config.pooling}`,
136
162
  `normalize=${normalization}`,
137
- `q=${config.queryPrefix}`,
138
- `d=${config.documentPrefix}`,
139
- ];
140
- if (config.prefixes === "off")
141
- parts.push("prefixes=off");
142
- return parts.join(":");
143
- }
144
- export async function dispose() {
163
+ describeEmbeddingDecoration(config),
164
+ ].join(":");
165
+ }
166
+ const RECORDED_DECORATION_PATTERN = /:(q=query:d=passage(?::prefixes=off)?|strategy=(?:jina-code|custom)(?::task=[a-z0-9-]+)?:ih=[0-9a-f]{12})$/;
167
+ export function recordedEmbeddingDecoration(signature) {
168
+ if (!signature)
169
+ return undefined;
170
+ return RECORDED_DECORATION_PATTERN.exec(signature)?.[1];
171
+ }
172
+ export function inspectEmbeddingCompatibility(recordedSignature, recordedDimension, config, activeDimension) {
173
+ const activeSignature = embeddingSignature(config, activeDimension);
174
+ const activeDecoration = describeEmbeddingDecoration(config);
175
+ const base = {
176
+ activeSignature,
177
+ recordedSignature,
178
+ activeDecoration,
179
+ activeDimension,
180
+ recordedDimension,
181
+ };
182
+ if (!recordedSignature || recordedDimension === null) {
183
+ return { ...base, compatible: false, cause: "unrecorded" };
184
+ }
185
+ const recordedDecoration = recordedEmbeddingDecoration(recordedSignature);
186
+ if (recordedDimension !== activeDimension) {
187
+ return { ...base, compatible: false, cause: "dimension", recordedDecoration };
188
+ }
189
+ if (recordedSignature === activeSignature) {
190
+ return { ...base, compatible: true, cause: "match", recordedDecoration };
191
+ }
192
+ if (!recordedDecoration) {
193
+ return { ...base, compatible: false, cause: "signature_format" };
194
+ }
195
+ if (recordedDecoration !== activeDecoration) {
196
+ const recordedBase = recordedSignature.slice(0, recordedSignature.length - recordedDecoration.length - 1);
197
+ const activeBase = activeSignature.slice(0, activeSignature.length - activeDecoration.length - 1);
198
+ if (recordedBase === activeBase) {
199
+ return { ...base, compatible: false, cause: "decoration", recordedDecoration };
200
+ }
201
+ }
202
+ return { ...base, compatible: false, cause: "configuration", recordedDecoration };
203
+ }
204
+ export function describeEmbeddingCompatibility(compatibility) {
205
+ switch (compatibility.cause) {
206
+ case "match":
207
+ return "indexed vectors match the active embedding configuration";
208
+ case "unrecorded":
209
+ return "the indexed embedding signature or dimension is missing";
210
+ case "dimension":
211
+ return `indexed dimension ${compatibility.recordedDimension ?? "unknown"} differs from active dimension ${compatibility.activeDimension}`;
212
+ case "decoration":
213
+ return `indexed decoration ${compatibility.recordedDecoration ?? "unknown"} differs from active decoration ${compatibility.activeDecoration}`;
214
+ case "configuration":
215
+ return "the indexed provider, model, endpoint, input limit, pooling, or normalization differs from the active configuration";
216
+ case "signature_format":
217
+ return "the indexed embedding signature cannot be interpreted";
218
+ }
219
+ }
220
+ export function dispose() {
145
221
  clearIdleTimer();
146
222
  if (disposePromise)
147
223
  return disposePromise;
148
224
  disposeRequested = true;
149
- await waitForNoActiveRuns();
150
- disposePromise = Promise.resolve().finally(() => {
225
+ disposePromise = waitForNoActiveRuns().finally(() => {
151
226
  disposePromise = null;
152
227
  disposeRequested = false;
153
228
  });
@@ -157,23 +232,26 @@ export async function prepareForShutdown() {
157
232
  clearIdleTimer();
158
233
  await waitForNoActiveRuns();
159
234
  }
160
- function applyEmbeddingPrefix(text, prefix, config) {
161
- return config.prefixes === "on" ? `${prefix}: ${text}` : text;
235
+ function prepareEmbeddingInputs(texts, role, config) {
236
+ return texts.map((text) => decorateEmbeddingInput(text, role, config.instructions));
162
237
  }
163
- export async function measureEmbeddingInputs(texts, prefix, config, signal) {
238
+ function apiInputMeasurement(preparedInputs, config) {
239
+ return {
240
+ counts: preparedInputs.map((text) => text.length),
241
+ maximum: config.maxChars,
242
+ preferredMaximum: Math.max(1, Math.floor(config.maxChars * 0.875)),
243
+ unit: "characters",
244
+ };
245
+ }
246
+ export async function measureEmbeddingInputs(texts, role, config, signal) {
164
247
  if (signal?.aborted)
165
248
  throw new Error("Cancelled");
166
249
  if (config.provider === "openai") {
167
- return {
168
- counts: texts.map((text) => applyEmbeddingPrefix(text, prefix, config).length),
169
- maximum: config.maxChars,
170
- preferredMaximum: Math.max(1, Math.floor(config.maxChars * 0.875)),
171
- unit: "characters",
172
- };
250
+ return apiInputMeasurement(prepareEmbeddingInputs(texts, role, config), config);
173
251
  }
174
252
  beginRun();
175
253
  try {
176
- const measurement = await measureEmbeddingInputInModelWorker(texts, prefix, config.prefixes, signal);
254
+ const measurement = await measureEmbeddingInputInModelWorker(texts, role, config.prefixMode, signal);
177
255
  return {
178
256
  ...measurement,
179
257
  preferredMaximum: Math.max(1, Math.floor(measurement.maximum * 0.875)),
@@ -184,17 +262,18 @@ export async function measureEmbeddingInputs(texts, prefix, config, signal) {
184
262
  endRun();
185
263
  }
186
264
  }
187
- async function assertEmbeddingInputsWithinBudget(texts, prefix, config, signal) {
188
- const measurement = await measureEmbeddingInputs(texts, prefix, config, signal);
265
+ function assertMeasurementWithinBudget(measurement, role, provider) {
189
266
  for (let index = 0; index < measurement.counts.length; index++) {
190
267
  const count = measurement.counts[index];
191
268
  if (count !== undefined && count > measurement.maximum) {
192
- throw new Error(`${config.provider} embedding ${prefix} input ${index + 1} is ${count} ${measurement.unit}; maximum is ${measurement.maximum}. Split the input before embedding.`);
269
+ throw new Error(`${provider} embedding ${role} input ${index + 1} is ${count} ${measurement.unit}; maximum is ${measurement.maximum}. Split the input before embedding.`);
193
270
  }
194
271
  }
195
272
  }
196
- async function embedViaAPI(texts, prefix, config, signal) {
197
- const safeTexts = texts.map((text) => applyEmbeddingPrefix(text, prefix, config));
273
+ async function assertLocalInputsWithinBudget(texts, role, config, signal) {
274
+ assertMeasurementWithinBudget(await measureEmbeddingInputs(texts, role, config, signal), role, config.provider);
275
+ }
276
+ async function embedPreparedInputsViaAPI(preparedInputs, config, signal) {
198
277
  const apiKey = cleanEnv(process.env.PI_KNOWLEDGE_EMBEDDING_API_KEY) ?? cleanEnv(process.env.OPENAI_API_KEY);
199
278
  if (!config.baseUrl)
200
279
  throw new Error("OpenAI embedding base URL is not configured");
@@ -205,7 +284,7 @@ async function embedViaAPI(texts, prefix, config, signal) {
205
284
  const res = await fetch(endpoint, {
206
285
  method: "POST",
207
286
  headers,
208
- body: JSON.stringify({ input: safeTexts, model: config.model }),
287
+ body: JSON.stringify({ input: preparedInputs, model: config.model }),
209
288
  signal,
210
289
  });
211
290
  if (!res.ok) {
@@ -216,8 +295,8 @@ async function embedViaAPI(texts, prefix, config, signal) {
216
295
  if (!payload || typeof payload !== "object" || !("data" in payload && Array.isArray(payload.data))) {
217
296
  throw new Error("OpenAI embedding API response is missing a data array");
218
297
  }
219
- if (payload.data.length !== texts.length) {
220
- throw new Error(`OpenAI embedding API returned ${payload.data.length} vectors for ${texts.length} inputs`);
298
+ if (payload.data.length !== preparedInputs.length) {
299
+ throw new Error(`OpenAI embedding API returned ${payload.data.length} vectors for ${preparedInputs.length} inputs`);
221
300
  }
222
301
  let expectedDimension;
223
302
  return payload.data.map((item, index) => {
@@ -239,44 +318,57 @@ async function embedViaAPI(texts, prefix, config, signal) {
239
318
  return normalizeEmbeddingVector(vector);
240
319
  });
241
320
  }
242
- export async function embedTextsWithConfig(texts, prefix, signal, options = {}) {
321
+ export async function embedTextsWithConfig(texts, role, signal, options = {}) {
243
322
  const config = resolveEmbeddingConfig();
244
323
  let actualConfig = config;
245
324
  let inputValidated = options.inputValidated === true;
246
325
  if (config.provider === "openai") {
247
- if (!inputValidated)
248
- await assertEmbeddingInputsWithinBudget(texts, prefix, config, signal);
326
+ const preparedInputs = prepareEmbeddingInputs(texts, role, config);
327
+ if (!inputValidated) {
328
+ assertMeasurementWithinBudget(apiInputMeasurement(preparedInputs, config), role, config.provider);
329
+ }
249
330
  if (signal?.aborted)
250
331
  throw new Error("Cancelled");
251
332
  try {
252
- return { vectors: await embedViaAPI(texts, prefix, config, signal), config };
333
+ return { vectors: await embedPreparedInputsViaAPI(preparedInputs, config, signal), config };
253
334
  }
254
335
  catch (error) {
255
336
  if (signal?.aborted || isAbortError(error))
256
337
  throw new Error("Cancelled");
257
- if (!API_FALLBACK_TO_LOCAL || !options.allowApiFallback)
338
+ if (!config.apiFallbackToLocal || !options.allowApiFallback)
258
339
  throw error;
259
340
  console.warn(`pi-knowledge: embedding API failed; falling back to local model because PI_KNOWLEDGE_EMBEDDING_API_FALLBACK=local (${error instanceof Error ? error.message : String(error)})`);
260
- actualConfig = localEmbeddingConfig(config.maxChars, config.prefixes);
341
+ actualConfig = localEmbeddingConfig(config.maxChars, {
342
+ strategy: config.strategy,
343
+ task: config.task,
344
+ instructions: config.instructions,
345
+ prefixMode: config.prefixMode,
346
+ });
261
347
  inputValidated = false;
262
348
  }
263
349
  }
264
350
  if (!inputValidated)
265
- await assertEmbeddingInputsWithinBudget(texts, prefix, actualConfig, signal);
351
+ await assertLocalInputsWithinBudget(texts, role, actualConfig, signal);
266
352
  beginRun();
267
353
  try {
268
- return { vectors: await embedInModelWorker(texts, prefix, actualConfig.prefixes, signal), config: actualConfig };
354
+ return {
355
+ vectors: await embedInModelWorker(texts, role, actualConfig.prefixMode, signal),
356
+ config: actualConfig,
357
+ };
269
358
  }
270
359
  finally {
271
360
  endRun();
272
361
  }
273
362
  }
274
- export async function embedTexts(texts, prefix, signal, options = {}) {
275
- return (await embedTextsWithConfig(texts, prefix, signal, options)).vectors;
363
+ export async function embedTexts(texts, role, signal, options = {}) {
364
+ return (await embedTextsWithConfig(texts, role, signal, options)).vectors;
276
365
  }
277
366
  export async function embedQueryWithConfig(text, signal) {
278
367
  const { vectors, config } = await embedTextsWithConfig([text], "query", signal, { allowApiFallback: true });
279
- return { vector: vectors[0], config };
368
+ const vector = vectors[0];
369
+ if (!vector)
370
+ throw new Error("Embedding provider returned no query vector");
371
+ return { vector, config };
280
372
  }
281
373
  export async function embedQuery(text, signal) {
282
374
  return (await embedQueryWithConfig(text, signal)).vector;
@@ -1,4 +1,5 @@
1
1
  import { type DiagnosticResult } from "./diagnostics/health.ts";
2
+ import type { EmbeddingCompatibility } from "./embedding/provider.ts";
2
3
  import { createSkippedScanStats } from "./indexer/chunker.ts";
3
4
  import { type RankingDiagnostics } from "./search/ranking.ts";
4
5
  import { type SearchProfile, type SearchTuningSummary } from "./search/tuning.ts";
@@ -58,7 +59,7 @@ export interface DoctorIssue {
58
59
  kb_name?: string;
59
60
  message: string;
60
61
  action: string;
61
- action_code?: "run_update" | "rebuild_kb" | "wait_for_indexing" | "review_skipped_scope" | "rebuild_vectors" | "reindex_for_embedding_budget" | "check_source" | "none";
62
+ action_code?: "run_update" | "rebuild_kb" | "wait_for_indexing" | "review_skipped_scope" | "fix_embedding_config" | "rebuild_vectors" | "reindex_for_embedding_budget" | "check_source" | "none";
62
63
  }
63
64
  export interface DoctorReport {
64
65
  health_score: number;
@@ -72,6 +73,21 @@ export interface DoctorReport {
72
73
  description: string;
73
74
  }>;
74
75
  }
76
+ export interface KnowledgeBaseEmbeddingStatus {
77
+ kb_id: string;
78
+ kb_name: string;
79
+ indexed_model: string | null;
80
+ active_model: string;
81
+ active_decoration: string;
82
+ compatibility: EmbeddingCompatibility;
83
+ detail: string;
84
+ }
85
+ export interface KnowledgeStatusSnapshot {
86
+ knowledge_bases: KnowledgeBase[];
87
+ diagnostics: DiagnosticResult[];
88
+ embeddings: KnowledgeBaseEmbeddingStatus[];
89
+ embedding_error?: string;
90
+ }
75
91
  export interface SymbolSearchOptions {
76
92
  kb_id?: string;
77
93
  kind?: KnowledgeSymbol["kind"];
@@ -133,6 +149,7 @@ export declare class KnowledgeEngine {
133
149
  list(signal?: AbortSignal): KnowledgeBase[];
134
150
  clear(): void;
135
151
  diagnose(signal?: AbortSignal): DiagnosticResult[];
152
+ statusSnapshot(signal?: AbortSignal): KnowledgeStatusSnapshot;
136
153
  symbolSearch(query: string, options?: SymbolSearchOptions, signal?: AbortSignal): SymbolSearchResponse;
137
154
  doctor(signal?: AbortSignal): DoctorReport;
138
155
  exportKB(nameOrId: string, outputPath: string, signal?: AbortSignal, onProgress?: ProgressCallback): Promise<number>;