pi-knowledge 0.12.0 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,146 @@
1
+ import { createHash } from "node:crypto";
2
+ export const MAX_INSTRUCTION_CHARS = 512;
3
+ export const DEFAULT_EMBEDDING_TASK = "nl2code";
4
+ export const ACCEPTED_DECORATION_VALUES = ["on", "off", "e5", "none", "jina", "jina-code", "custom"];
5
+ export const ACCEPTED_EMBEDDING_TASKS = ["nl2code", "code2code", "code2nl", "code2completion", "qa"];
6
+ const STRATEGY_ALIASES = {
7
+ on: "e5",
8
+ e5: "e5",
9
+ off: "none",
10
+ none: "none",
11
+ jina: "jina-code",
12
+ "jina-code": "jina-code",
13
+ custom: "custom",
14
+ };
15
+ const STRATEGY_ENV = "PI_KNOWLEDGE_EMBEDDING_PREFIXES";
16
+ const TASK_ENV = "PI_KNOWLEDGE_EMBEDDING_TASK";
17
+ const QUERY_INSTRUCTION_ENV = "PI_KNOWLEDGE_EMBEDDING_QUERY_INSTRUCTION";
18
+ const DOCUMENT_INSTRUCTION_ENV = "PI_KNOWLEDGE_EMBEDDING_DOCUMENT_INSTRUCTION";
19
+ export const E5_INSTRUCTIONS = Object.freeze({ query: "query: ", document: "passage: " });
20
+ export const NO_INSTRUCTIONS = Object.freeze({ query: "", document: "" });
21
+ /**
22
+ * Published prompts for jinaai/jina-code-embeddings-1.5b.
23
+ * Source: config_sentence_transformers.json at model revision
24
+ * 39aeb4fb9b60f930934c78ae5d749a46287c248a (verified 2026-11-12).
25
+ * Values already contain their required trailing newline and are concatenated verbatim.
26
+ */
27
+ export const JINA_CODE_INSTRUCTIONS = Object.freeze({
28
+ nl2code: Object.freeze({
29
+ query: "Find the most relevant code snippet given the following query:\n",
30
+ document: "Candidate code snippet:\n",
31
+ }),
32
+ code2code: Object.freeze({
33
+ query: "Find an equivalent code snippet given the following code snippet:\n",
34
+ document: "Candidate code snippet:\n",
35
+ }),
36
+ code2nl: Object.freeze({
37
+ query: "Find the most relevant comment given the following code snippet:\n",
38
+ document: "Candidate comment:\n",
39
+ }),
40
+ code2completion: Object.freeze({
41
+ query: "Find the most relevant completion given the following start of code snippet:\n",
42
+ document: "Candidate completion:\n",
43
+ }),
44
+ qa: Object.freeze({
45
+ query: "Find the most relevant answer given the following question:\n",
46
+ document: "Candidate answer:\n",
47
+ }),
48
+ });
49
+ export function decorateEmbeddingInput(text, role, instructions) {
50
+ return `${role === "query" ? instructions.query : instructions.document}${text}`;
51
+ }
52
+ export function embeddingInstructionLength(instructions, role) {
53
+ return (role === "query" ? instructions.query : instructions.document).length;
54
+ }
55
+ export function embeddingInstructionSetHash(instructions) {
56
+ return createHash("sha256")
57
+ .update(JSON.stringify([instructions.query, instructions.document]))
58
+ .digest("hex")
59
+ .slice(0, 12);
60
+ }
61
+ export function embeddingDecorationDescriptor(decoration) {
62
+ if (decoration.strategy === "e5")
63
+ return "q=query:d=passage";
64
+ if (decoration.strategy === "none")
65
+ return "q=query:d=passage:prefixes=off";
66
+ const instructionHash = embeddingInstructionSetHash(decoration.instructions);
67
+ if (decoration.strategy === "jina-code") {
68
+ return `strategy=jina-code:task=${decoration.task ?? DEFAULT_EMBEDDING_TASK}:ih=${instructionHash}`;
69
+ }
70
+ return `strategy=custom:ih=${instructionHash}`;
71
+ }
72
+ function cleanValue(value) {
73
+ const trimmed = value?.trim();
74
+ return trimmed ? trimmed : undefined;
75
+ }
76
+ function assertInstructionValue(name, value) {
77
+ if (value.length > MAX_INSTRUCTION_CHARS) {
78
+ throw new Error(`Embedding instruction ${name} is ${value.length} characters; maximum is ${MAX_INSTRUCTION_CHARS}.`);
79
+ }
80
+ for (const character of value) {
81
+ const code = character.codePointAt(0) ?? 0;
82
+ if (character !== "\n" && (code < 0x20 || (code >= 0x7f && code <= 0x9f))) {
83
+ throw new Error(`Embedding instruction ${name} contains an unsupported control character; only printable text and newline are allowed.`);
84
+ }
85
+ }
86
+ }
87
+ function resolveCustomInstructions(env) {
88
+ const query = env[QUERY_INSTRUCTION_ENV];
89
+ const document = env[DOCUMENT_INSTRUCTION_ENV];
90
+ const missing = [
91
+ query === undefined ? QUERY_INSTRUCTION_ENV : "",
92
+ document === undefined ? DOCUMENT_INSTRUCTION_ENV : "",
93
+ ].filter(Boolean);
94
+ if (missing.length > 0)
95
+ throw new Error(`${STRATEGY_ENV}=custom requires ${missing.join(" and ")}.`);
96
+ if (query === undefined || document === undefined)
97
+ throw new Error("Embedding instruction resolution failed");
98
+ if (!query.trim())
99
+ throw new Error(`${QUERY_INSTRUCTION_ENV} must be a non-empty string.`);
100
+ assertInstructionValue(QUERY_INSTRUCTION_ENV, query);
101
+ assertInstructionValue(DOCUMENT_INSTRUCTION_ENV, document);
102
+ return Object.freeze({ query, document });
103
+ }
104
+ export function resolveEmbeddingDecoration(env) {
105
+ const rawStrategy = cleanValue(env[STRATEGY_ENV])?.toLowerCase();
106
+ const strategy = rawStrategy ? STRATEGY_ALIASES[rawStrategy] : "e5";
107
+ if (!strategy) {
108
+ throw new Error(`Unsupported embedding decoration strategy: ${rawStrategy}. Use ${STRATEGY_ENV}=${ACCEPTED_DECORATION_VALUES.join("|")}.`);
109
+ }
110
+ const customVariablesPresent = env[QUERY_INSTRUCTION_ENV] !== undefined || env[DOCUMENT_INSTRUCTION_ENV] !== undefined;
111
+ if (strategy !== "custom" && customVariablesPresent) {
112
+ throw new Error(`${QUERY_INSTRUCTION_ENV} and ${DOCUMENT_INSTRUCTION_ENV} are valid only when ${STRATEGY_ENV}=custom.`);
113
+ }
114
+ const rawTask = cleanValue(env[TASK_ENV])?.toLowerCase();
115
+ let task;
116
+ if (rawTask) {
117
+ if (strategy !== "jina-code") {
118
+ throw new Error(`${TASK_ENV} is valid only when ${STRATEGY_ENV}=jina-code.`);
119
+ }
120
+ if (!ACCEPTED_EMBEDDING_TASKS.includes(rawTask)) {
121
+ throw new Error(`Unsupported embedding task: ${rawTask}. Use ${TASK_ENV}=${ACCEPTED_EMBEDDING_TASKS.join("|")}.`);
122
+ }
123
+ task = rawTask;
124
+ }
125
+ else if (strategy === "jina-code") {
126
+ task = DEFAULT_EMBEDDING_TASK;
127
+ }
128
+ let instructions;
129
+ if (strategy === "e5")
130
+ instructions = E5_INSTRUCTIONS;
131
+ else if (strategy === "none")
132
+ instructions = NO_INSTRUCTIONS;
133
+ else if (strategy === "jina-code")
134
+ instructions = JINA_CODE_INSTRUCTIONS[task ?? DEFAULT_EMBEDDING_TASK];
135
+ else
136
+ instructions = resolveCustomInstructions(env);
137
+ return Object.freeze({
138
+ strategy,
139
+ task,
140
+ instructions,
141
+ prefixMode: strategy === "e5" ? "on" : "off",
142
+ });
143
+ }
144
+ export function decorationSupportedByLocalModel(strategy) {
145
+ return strategy === "e5" || strategy === "none";
146
+ }
@@ -1,14 +1,16 @@
1
+ import { type EmbeddingDecorationStrategy, type EmbeddingInstructionSet, type EmbeddingPrefixMode, type EmbeddingRole, type EmbeddingTask } from "./instructions.ts";
2
+ export type { EmbeddingDecorationStrategy, EmbeddingInstructionSet, EmbeddingRole, EmbeddingTask, } from "./instructions.ts";
1
3
  export type EmbeddingProvider = "local" | "openai";
2
- export type EmbeddingPrefix = "query" | "passage";
3
- export type EmbeddingPrefixStrategy = "on" | "off";
4
4
  export interface EmbeddingConfig {
5
5
  provider: EmbeddingProvider;
6
6
  model: string;
7
7
  baseUrl?: string;
8
8
  maxChars: number;
9
- queryPrefix: "query";
10
- documentPrefix: "passage";
11
- prefixes: EmbeddingPrefixStrategy;
9
+ strategy: EmbeddingDecorationStrategy;
10
+ task?: EmbeddingTask;
11
+ instructions: EmbeddingInstructionSet;
12
+ prefixMode: EmbeddingPrefixMode;
13
+ apiFallbackToLocal: boolean;
12
14
  pooling: "mean";
13
15
  normalize: true;
14
16
  }
@@ -18,6 +20,17 @@ export interface EmbeddingInputMeasurement {
18
20
  preferredMaximum: number;
19
21
  unit: "tokens" | "characters";
20
22
  }
23
+ export type EmbeddingCompatibilityCause = "match" | "unrecorded" | "dimension" | "decoration" | "configuration" | "signature_format";
24
+ export interface EmbeddingCompatibility {
25
+ compatible: boolean;
26
+ cause: EmbeddingCompatibilityCause;
27
+ activeSignature: string;
28
+ recordedSignature: string | null;
29
+ activeDecoration: string;
30
+ recordedDecoration?: string;
31
+ activeDimension: number;
32
+ recordedDimension: number | null;
33
+ }
21
34
  export declare const CURRENT_EMBEDDING_MODEL = "multilingual-e5-small";
22
35
  export declare const DEFAULT_LOCAL_EMBEDDING_MODEL = "multilingual-e5-small";
23
36
  export declare const DEFAULT_OPENAI_EMBEDDING_MODEL = "text-embedding-3-small";
@@ -25,20 +38,26 @@ export declare const DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1";
25
38
  export declare const LOCAL_EMBEDDING_MAX_TOKENS = 512;
26
39
  export declare const LOCAL_EMBEDDING_PREFERRED_TOKENS = 448;
27
40
  export declare const DEFAULT_API_MAX_EMBED_CHARS = 20000;
41
+ export declare const API_VECTOR_NORMALIZATION_POLICY = "client-l2-v1";
28
42
  export declare function resolveEmbeddingConfig(env?: NodeJS.ProcessEnv): EmbeddingConfig;
29
43
  export declare function embeddingConfigLabel(config: EmbeddingConfig): string;
44
+ export declare function normalizeEmbeddingVector(vector: Float32Array): Float32Array;
45
+ export declare function describeEmbeddingDecoration(config: EmbeddingConfig): string;
30
46
  export declare function embeddingSignature(config: EmbeddingConfig, dimension: number): string;
47
+ export declare function recordedEmbeddingDecoration(signature: string | null): string | undefined;
48
+ export declare function inspectEmbeddingCompatibility(recordedSignature: string | null, recordedDimension: number | null, config: EmbeddingConfig, activeDimension: number): EmbeddingCompatibility;
49
+ export declare function describeEmbeddingCompatibility(compatibility: EmbeddingCompatibility): string;
31
50
  export declare function dispose(): Promise<void>;
32
51
  export declare function prepareForShutdown(): Promise<void>;
33
- export declare function measureEmbeddingInputs(texts: string[], prefix: EmbeddingPrefix, config: EmbeddingConfig, signal?: AbortSignal): Promise<EmbeddingInputMeasurement>;
34
- export declare function embedTextsWithConfig(texts: string[], prefix: EmbeddingPrefix, signal?: AbortSignal, options?: {
52
+ export declare function measureEmbeddingInputs(texts: string[], role: EmbeddingRole, config: EmbeddingConfig, signal?: AbortSignal): Promise<EmbeddingInputMeasurement>;
53
+ export declare function embedTextsWithConfig(texts: string[], role: EmbeddingRole, signal?: AbortSignal, options?: {
35
54
  allowApiFallback?: boolean;
36
55
  inputValidated?: boolean;
37
56
  }): Promise<{
38
57
  vectors: Float32Array[];
39
58
  config: EmbeddingConfig;
40
59
  }>;
41
- export declare function embedTexts(texts: string[], prefix: EmbeddingPrefix, signal?: AbortSignal, options?: {
60
+ export declare function embedTexts(texts: string[], role: EmbeddingRole, signal?: AbortSignal, options?: {
42
61
  allowApiFallback?: boolean;
43
62
  inputValidated?: boolean;
44
63
  }): Promise<Float32Array[]>;
@@ -1,5 +1,6 @@
1
1
  import { createHash } from "node:crypto";
2
2
  import { embedInModelWorker, measureEmbeddingInputInModelWorker } from "../model-worker-client.js";
3
+ import { decorateEmbeddingInput, decorationSupportedByLocalModel, embeddingDecorationDescriptor, embeddingInstructionLength, resolveEmbeddingDecoration, } from "./instructions.js";
3
4
  let disposeTimer = null;
4
5
  let disposePromise = null;
5
6
  let activeRuns = 0;
@@ -12,17 +13,19 @@ export const DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1";
12
13
  export const LOCAL_EMBEDDING_MAX_TOKENS = 512;
13
14
  export const LOCAL_EMBEDDING_PREFERRED_TOKENS = 448;
14
15
  export const DEFAULT_API_MAX_EMBED_CHARS = 20_000;
16
+ export const API_VECTOR_NORMALIZATION_POLICY = "client-l2-v1";
15
17
  const IDLE_TIMEOUT_MS = Number(process.env.PI_KNOWLEDGE_EMBEDDING_IDLE_MS ?? 30_000);
16
18
  const ENABLE_NATIVE_IDLE_DISPOSE = process.env.PI_KNOWLEDGE_ENABLE_NATIVE_IDLE_DISPOSE === "true";
17
- const API_FALLBACK_TO_LOCAL = process.env.PI_KNOWLEDGE_EMBEDDING_API_FALLBACK === "local";
18
- function localEmbeddingConfig(maxChars, prefixes) {
19
+ function localEmbeddingConfig(maxChars, decoration) {
19
20
  return {
20
21
  provider: "local",
21
22
  model: DEFAULT_LOCAL_EMBEDDING_MODEL,
22
23
  maxChars,
23
- queryPrefix: "query",
24
- documentPrefix: "passage",
25
- prefixes,
24
+ strategy: decoration.strategy,
25
+ task: decoration.task,
26
+ instructions: decoration.instructions,
27
+ prefixMode: decoration.prefixMode,
28
+ apiFallbackToLocal: false,
26
29
  pooling: "mean",
27
30
  normalize: true,
28
31
  };
@@ -35,8 +38,7 @@ function isAbortError(error) {
35
38
  return typeof error === "object" && error !== null && "name" in error && error.name === "AbortError";
36
39
  }
37
40
  function clearIdleTimer() {
38
- if (disposeTimer)
39
- clearTimeout(disposeTimer);
41
+ clearTimeout(disposeTimer ?? undefined);
40
42
  disposeTimer = null;
41
43
  }
42
44
  function scheduleIdleDispose() {
@@ -60,21 +62,23 @@ function endRun() {
60
62
  if (!disposeRequested)
61
63
  scheduleIdleDispose();
62
64
  }
63
- function waitForNoActiveRuns() {
65
+ async function waitForNoActiveRuns() {
64
66
  if (activeRuns === 0)
65
- return Promise.resolve();
66
- return new Promise((resolve) => idleWaiters.push(resolve));
67
+ return;
68
+ const { promise, resolve } = Promise.withResolvers();
69
+ idleWaiters.push(resolve);
70
+ await promise;
67
71
  }
68
72
  function providerInstanceHash(value) {
69
73
  return createHash("sha256").update(value).digest("hex").slice(0, 12);
70
74
  }
71
- function resolveEmbeddingPrefixStrategy(env) {
72
- const raw = cleanEnv(env.PI_KNOWLEDGE_EMBEDDING_PREFIXES)?.toLowerCase();
73
- if (!raw)
74
- return "on";
75
- if (raw === "on" || raw === "off")
76
- return raw;
77
- throw new Error(`Unsupported embedding prefix strategy: ${raw}. Use PI_KNOWLEDGE_EMBEDDING_PREFIXES=on or off.`);
75
+ function assertApiInstructionBudget(config) {
76
+ for (const role of ["query", "passage"]) {
77
+ const instructionLength = embeddingInstructionLength(config.instructions, role);
78
+ if (instructionLength >= config.maxChars) {
79
+ throw new Error(`Embedding ${role} instruction is ${instructionLength} characters; PI_KNOWLEDGE_EMBEDDING_MAX_CHARS=${config.maxChars} must leave room for input text.`);
80
+ }
81
+ }
78
82
  }
79
83
  export function resolveEmbeddingConfig(env = process.env) {
80
84
  const raw = cleanEnv(env.PI_KNOWLEDGE_EMBEDDING) ?? `local:${DEFAULT_LOCAL_EMBEDDING_MODEL}`;
@@ -83,54 +87,142 @@ export function resolveEmbeddingConfig(env = process.env) {
83
87
  const model = separator >= 0 ? raw.slice(separator + 1).trim() : "";
84
88
  const maxCharsValue = Number(cleanEnv(env.PI_KNOWLEDGE_EMBEDDING_MAX_CHARS) ?? DEFAULT_API_MAX_EMBED_CHARS);
85
89
  const maxChars = Number.isFinite(maxCharsValue) && maxCharsValue > 0 ? Math.trunc(maxCharsValue) : DEFAULT_API_MAX_EMBED_CHARS;
86
- const prefixes = resolveEmbeddingPrefixStrategy(env);
90
+ const decoration = resolveEmbeddingDecoration(env);
91
+ const fallback = cleanEnv(env.PI_KNOWLEDGE_EMBEDDING_API_FALLBACK)?.toLowerCase();
92
+ if (fallback && fallback !== "local") {
93
+ throw new Error("Unsupported embedding API fallback. Use PI_KNOWLEDGE_EMBEDDING_API_FALLBACK=local or unset it.");
94
+ }
95
+ const apiFallbackToLocal = fallback === "local";
87
96
  if (provider === "local") {
88
97
  const localModel = model || DEFAULT_LOCAL_EMBEDDING_MODEL;
89
98
  if (localModel !== DEFAULT_LOCAL_EMBEDDING_MODEL) {
90
99
  throw new Error(`Unsupported local embedding model: ${localModel}`);
91
100
  }
92
- return localEmbeddingConfig(maxChars, prefixes);
101
+ if (!decorationSupportedByLocalModel(decoration.strategy)) {
102
+ throw new Error(`Embedding decoration ${decoration.strategy} is not supported for the local embedding model. Use PI_KNOWLEDGE_EMBEDDING_PREFIXES=on|off|e5|none, or an OpenAI-compatible provider.`);
103
+ }
104
+ return localEmbeddingConfig(maxChars, decoration);
93
105
  }
94
106
  if (provider === "openai") {
95
- return {
107
+ if (apiFallbackToLocal && !decorationSupportedByLocalModel(decoration.strategy)) {
108
+ throw new Error(`PI_KNOWLEDGE_EMBEDDING_API_FALLBACK=local is incompatible with embedding decoration ${decoration.strategy}; the local model supports only E5 prefixes or none.`);
109
+ }
110
+ const config = {
96
111
  provider: "openai",
97
112
  model: model || DEFAULT_OPENAI_EMBEDDING_MODEL,
98
113
  baseUrl: cleanEnv(env.PI_KNOWLEDGE_EMBEDDING_BASE_URL) ?? cleanEnv(env.OPENAI_BASE_URL) ?? DEFAULT_OPENAI_BASE_URL,
99
114
  maxChars,
100
- queryPrefix: "query",
101
- documentPrefix: "passage",
102
- prefixes,
115
+ strategy: decoration.strategy,
116
+ task: decoration.task,
117
+ instructions: decoration.instructions,
118
+ prefixMode: decoration.prefixMode,
119
+ apiFallbackToLocal,
103
120
  pooling: "mean",
104
121
  normalize: true,
105
122
  };
123
+ assertApiInstructionBudget(config);
124
+ return config;
106
125
  }
107
126
  throw new Error(`Unsupported embedding provider: ${provider}`);
108
127
  }
109
128
  export function embeddingConfigLabel(config) {
110
129
  return config.provider === "local" ? config.model : `${config.provider}:${config.model}`;
111
130
  }
131
+ export function normalizeEmbeddingVector(vector) {
132
+ let squaredMagnitude = 0;
133
+ for (const value of vector) {
134
+ if (!Number.isFinite(value))
135
+ throw new Error("Embedding provider returned a non-finite vector value");
136
+ squaredMagnitude += value * value;
137
+ }
138
+ if (!Number.isFinite(squaredMagnitude) || squaredMagnitude <= 0) {
139
+ throw new Error("Embedding provider returned a zero-magnitude vector");
140
+ }
141
+ const magnitude = Math.sqrt(squaredMagnitude);
142
+ for (let index = 0; index < vector.length; index++)
143
+ vector[index] /= magnitude;
144
+ return vector;
145
+ }
146
+ export function describeEmbeddingDecoration(config) {
147
+ return embeddingDecorationDescriptor({
148
+ strategy: config.strategy,
149
+ task: config.task,
150
+ instructions: config.instructions,
151
+ prefixMode: config.prefixMode,
152
+ });
153
+ }
112
154
  export function embeddingSignature(config, dimension) {
113
155
  const base = config.baseUrl ? `:base-sha256=${providerInstanceHash(config.baseUrl)}` : "";
114
- const parts = [
156
+ const normalization = config.provider === "openai" ? API_VECTOR_NORMALIZATION_POLICY : String(config.normalize);
157
+ return [
115
158
  `${config.provider}:${config.model}${base}`,
116
159
  `dim=${dimension}`,
117
160
  `apiMax=${config.provider === "openai" ? config.maxChars : "none"}`,
118
161
  `pooling=${config.pooling}`,
119
- `normalize=${config.normalize}`,
120
- `q=${config.queryPrefix}`,
121
- `d=${config.documentPrefix}`,
122
- ];
123
- if (config.prefixes === "off")
124
- parts.push("prefixes=off");
125
- return parts.join(":");
126
- }
127
- export async function dispose() {
162
+ `normalize=${normalization}`,
163
+ describeEmbeddingDecoration(config),
164
+ ].join(":");
165
+ }
166
+ const RECORDED_DECORATION_PATTERN = /:(q=query:d=passage(?::prefixes=off)?|strategy=(?:jina-code|custom)(?::task=[a-z0-9-]+)?:ih=[0-9a-f]{12})$/;
167
+ export function recordedEmbeddingDecoration(signature) {
168
+ if (!signature)
169
+ return undefined;
170
+ return RECORDED_DECORATION_PATTERN.exec(signature)?.[1];
171
+ }
172
+ export function inspectEmbeddingCompatibility(recordedSignature, recordedDimension, config, activeDimension) {
173
+ const activeSignature = embeddingSignature(config, activeDimension);
174
+ const activeDecoration = describeEmbeddingDecoration(config);
175
+ const base = {
176
+ activeSignature,
177
+ recordedSignature,
178
+ activeDecoration,
179
+ activeDimension,
180
+ recordedDimension,
181
+ };
182
+ if (!recordedSignature || recordedDimension === null) {
183
+ return { ...base, compatible: false, cause: "unrecorded" };
184
+ }
185
+ const recordedDecoration = recordedEmbeddingDecoration(recordedSignature);
186
+ if (recordedDimension !== activeDimension) {
187
+ return { ...base, compatible: false, cause: "dimension", recordedDecoration };
188
+ }
189
+ if (recordedSignature === activeSignature) {
190
+ return { ...base, compatible: true, cause: "match", recordedDecoration };
191
+ }
192
+ if (!recordedDecoration) {
193
+ return { ...base, compatible: false, cause: "signature_format" };
194
+ }
195
+ if (recordedDecoration !== activeDecoration) {
196
+ const recordedBase = recordedSignature.slice(0, recordedSignature.length - recordedDecoration.length - 1);
197
+ const activeBase = activeSignature.slice(0, activeSignature.length - activeDecoration.length - 1);
198
+ if (recordedBase === activeBase) {
199
+ return { ...base, compatible: false, cause: "decoration", recordedDecoration };
200
+ }
201
+ }
202
+ return { ...base, compatible: false, cause: "configuration", recordedDecoration };
203
+ }
204
+ export function describeEmbeddingCompatibility(compatibility) {
205
+ switch (compatibility.cause) {
206
+ case "match":
207
+ return "indexed vectors match the active embedding configuration";
208
+ case "unrecorded":
209
+ return "the indexed embedding signature or dimension is missing";
210
+ case "dimension":
211
+ return `indexed dimension ${compatibility.recordedDimension ?? "unknown"} differs from active dimension ${compatibility.activeDimension}`;
212
+ case "decoration":
213
+ return `indexed decoration ${compatibility.recordedDecoration ?? "unknown"} differs from active decoration ${compatibility.activeDecoration}`;
214
+ case "configuration":
215
+ return "the indexed provider, model, endpoint, input limit, pooling, or normalization differs from the active configuration";
216
+ case "signature_format":
217
+ return "the indexed embedding signature cannot be interpreted";
218
+ }
219
+ }
220
+ export function dispose() {
128
221
  clearIdleTimer();
129
222
  if (disposePromise)
130
223
  return disposePromise;
131
224
  disposeRequested = true;
132
- await waitForNoActiveRuns();
133
- disposePromise = Promise.resolve().finally(() => {
225
+ disposePromise = waitForNoActiveRuns().finally(() => {
134
226
  disposePromise = null;
135
227
  disposeRequested = false;
136
228
  });
@@ -140,23 +232,26 @@ export async function prepareForShutdown() {
140
232
  clearIdleTimer();
141
233
  await waitForNoActiveRuns();
142
234
  }
143
- function applyEmbeddingPrefix(text, prefix, config) {
144
- return config.prefixes === "on" ? `${prefix}: ${text}` : text;
235
+ function prepareEmbeddingInputs(texts, role, config) {
236
+ return texts.map((text) => decorateEmbeddingInput(text, role, config.instructions));
145
237
  }
146
- export async function measureEmbeddingInputs(texts, prefix, config, signal) {
238
+ function apiInputMeasurement(preparedInputs, config) {
239
+ return {
240
+ counts: preparedInputs.map((text) => text.length),
241
+ maximum: config.maxChars,
242
+ preferredMaximum: Math.max(1, Math.floor(config.maxChars * 0.875)),
243
+ unit: "characters",
244
+ };
245
+ }
246
+ export async function measureEmbeddingInputs(texts, role, config, signal) {
147
247
  if (signal?.aborted)
148
248
  throw new Error("Cancelled");
149
249
  if (config.provider === "openai") {
150
- return {
151
- counts: texts.map((text) => applyEmbeddingPrefix(text, prefix, config).length),
152
- maximum: config.maxChars,
153
- preferredMaximum: Math.max(1, Math.floor(config.maxChars * 0.875)),
154
- unit: "characters",
155
- };
250
+ return apiInputMeasurement(prepareEmbeddingInputs(texts, role, config), config);
156
251
  }
157
252
  beginRun();
158
253
  try {
159
- const measurement = await measureEmbeddingInputInModelWorker(texts, prefix, config.prefixes, signal);
254
+ const measurement = await measureEmbeddingInputInModelWorker(texts, role, config.prefixMode, signal);
160
255
  return {
161
256
  ...measurement,
162
257
  preferredMaximum: Math.max(1, Math.floor(measurement.maximum * 0.875)),
@@ -167,17 +262,18 @@ export async function measureEmbeddingInputs(texts, prefix, config, signal) {
167
262
  endRun();
168
263
  }
169
264
  }
170
- async function assertEmbeddingInputsWithinBudget(texts, prefix, config, signal) {
171
- const measurement = await measureEmbeddingInputs(texts, prefix, config, signal);
265
+ function assertMeasurementWithinBudget(measurement, role, provider) {
172
266
  for (let index = 0; index < measurement.counts.length; index++) {
173
267
  const count = measurement.counts[index];
174
268
  if (count !== undefined && count > measurement.maximum) {
175
- throw new Error(`${config.provider} embedding ${prefix} input ${index + 1} is ${count} ${measurement.unit}; maximum is ${measurement.maximum}. Split the input before embedding.`);
269
+ throw new Error(`${provider} embedding ${role} input ${index + 1} is ${count} ${measurement.unit}; maximum is ${measurement.maximum}. Split the input before embedding.`);
176
270
  }
177
271
  }
178
272
  }
179
- async function embedViaAPI(texts, prefix, config, signal) {
180
- const safeTexts = texts.map((text) => applyEmbeddingPrefix(text, prefix, config));
273
+ async function assertLocalInputsWithinBudget(texts, role, config, signal) {
274
+ assertMeasurementWithinBudget(await measureEmbeddingInputs(texts, role, config, signal), role, config.provider);
275
+ }
276
+ async function embedPreparedInputsViaAPI(preparedInputs, config, signal) {
181
277
  const apiKey = cleanEnv(process.env.PI_KNOWLEDGE_EMBEDDING_API_KEY) ?? cleanEnv(process.env.OPENAI_API_KEY);
182
278
  if (!config.baseUrl)
183
279
  throw new Error("OpenAI embedding base URL is not configured");
@@ -188,54 +284,91 @@ async function embedViaAPI(texts, prefix, config, signal) {
188
284
  const res = await fetch(endpoint, {
189
285
  method: "POST",
190
286
  headers,
191
- body: JSON.stringify({ input: safeTexts, model: config.model }),
287
+ body: JSON.stringify({ input: preparedInputs, model: config.model }),
192
288
  signal,
193
289
  });
194
290
  if (!res.ok) {
195
291
  const detail = await res.text().catch(() => "");
196
292
  throw new Error(`OpenAI embedding API error: ${res.status}${detail ? ` ${detail.slice(0, 500)}` : ""}`);
197
293
  }
198
- const data = (await res.json());
199
- return data.data.map((d) => new Float32Array(d.embedding));
294
+ const payload = (await res.json());
295
+ if (!payload || typeof payload !== "object" || !("data" in payload && Array.isArray(payload.data))) {
296
+ throw new Error("OpenAI embedding API response is missing a data array");
297
+ }
298
+ if (payload.data.length !== preparedInputs.length) {
299
+ throw new Error(`OpenAI embedding API returned ${payload.data.length} vectors for ${preparedInputs.length} inputs`);
300
+ }
301
+ let expectedDimension;
302
+ return payload.data.map((item, index) => {
303
+ if (!item || typeof item !== "object" || !("embedding" in item && Array.isArray(item.embedding))) {
304
+ throw new Error(`OpenAI embedding API response item ${index + 1} is missing an embedding array`);
305
+ }
306
+ if (expectedDimension !== undefined && item.embedding.length !== expectedDimension) {
307
+ throw new Error(`OpenAI embedding API response item ${index + 1} has dimension ${item.embedding.length}; expected ${expectedDimension}`);
308
+ }
309
+ expectedDimension = item.embedding.length;
310
+ const vector = new Float32Array(item.embedding.length);
311
+ for (let valueIndex = 0; valueIndex < item.embedding.length; valueIndex++) {
312
+ const value = item.embedding[valueIndex];
313
+ if (typeof value !== "number" || !Number.isFinite(value)) {
314
+ throw new Error(`OpenAI embedding API response item ${index + 1} contains a non-finite numeric value at position ${valueIndex}`);
315
+ }
316
+ vector[valueIndex] = value;
317
+ }
318
+ return normalizeEmbeddingVector(vector);
319
+ });
200
320
  }
201
- export async function embedTextsWithConfig(texts, prefix, signal, options = {}) {
321
+ export async function embedTextsWithConfig(texts, role, signal, options = {}) {
202
322
  const config = resolveEmbeddingConfig();
203
323
  let actualConfig = config;
204
324
  let inputValidated = options.inputValidated === true;
205
325
  if (config.provider === "openai") {
206
- if (!inputValidated)
207
- await assertEmbeddingInputsWithinBudget(texts, prefix, config, signal);
326
+ const preparedInputs = prepareEmbeddingInputs(texts, role, config);
327
+ if (!inputValidated) {
328
+ assertMeasurementWithinBudget(apiInputMeasurement(preparedInputs, config), role, config.provider);
329
+ }
208
330
  if (signal?.aborted)
209
331
  throw new Error("Cancelled");
210
332
  try {
211
- return { vectors: await embedViaAPI(texts, prefix, config, signal), config };
333
+ return { vectors: await embedPreparedInputsViaAPI(preparedInputs, config, signal), config };
212
334
  }
213
335
  catch (error) {
214
336
  if (signal?.aborted || isAbortError(error))
215
337
  throw new Error("Cancelled");
216
- if (!API_FALLBACK_TO_LOCAL || !options.allowApiFallback)
338
+ if (!config.apiFallbackToLocal || !options.allowApiFallback)
217
339
  throw error;
218
340
  console.warn(`pi-knowledge: embedding API failed; falling back to local model because PI_KNOWLEDGE_EMBEDDING_API_FALLBACK=local (${error instanceof Error ? error.message : String(error)})`);
219
- actualConfig = localEmbeddingConfig(config.maxChars, config.prefixes);
341
+ actualConfig = localEmbeddingConfig(config.maxChars, {
342
+ strategy: config.strategy,
343
+ task: config.task,
344
+ instructions: config.instructions,
345
+ prefixMode: config.prefixMode,
346
+ });
220
347
  inputValidated = false;
221
348
  }
222
349
  }
223
350
  if (!inputValidated)
224
- await assertEmbeddingInputsWithinBudget(texts, prefix, actualConfig, signal);
351
+ await assertLocalInputsWithinBudget(texts, role, actualConfig, signal);
225
352
  beginRun();
226
353
  try {
227
- return { vectors: await embedInModelWorker(texts, prefix, actualConfig.prefixes, signal), config: actualConfig };
354
+ return {
355
+ vectors: await embedInModelWorker(texts, role, actualConfig.prefixMode, signal),
356
+ config: actualConfig,
357
+ };
228
358
  }
229
359
  finally {
230
360
  endRun();
231
361
  }
232
362
  }
233
- export async function embedTexts(texts, prefix, signal, options = {}) {
234
- return (await embedTextsWithConfig(texts, prefix, signal, options)).vectors;
363
+ export async function embedTexts(texts, role, signal, options = {}) {
364
+ return (await embedTextsWithConfig(texts, role, signal, options)).vectors;
235
365
  }
236
366
  export async function embedQueryWithConfig(text, signal) {
237
367
  const { vectors, config } = await embedTextsWithConfig([text], "query", signal, { allowApiFallback: true });
238
- return { vector: vectors[0], config };
368
+ const vector = vectors[0];
369
+ if (!vector)
370
+ throw new Error("Embedding provider returned no query vector");
371
+ return { vector, config };
239
372
  }
240
373
  export async function embedQuery(text, signal) {
241
374
  return (await embedQueryWithConfig(text, signal)).vector;
@@ -1,4 +1,5 @@
1
1
  import { type DiagnosticResult } from "./diagnostics/health.ts";
2
+ import type { EmbeddingCompatibility } from "./embedding/provider.ts";
2
3
  import { createSkippedScanStats } from "./indexer/chunker.ts";
3
4
  import { type RankingDiagnostics } from "./search/ranking.ts";
4
5
  import { type SearchProfile, type SearchTuningSummary } from "./search/tuning.ts";
@@ -58,7 +59,7 @@ export interface DoctorIssue {
58
59
  kb_name?: string;
59
60
  message: string;
60
61
  action: string;
61
- action_code?: "run_update" | "rebuild_kb" | "wait_for_indexing" | "review_skipped_scope" | "rebuild_vectors" | "reindex_for_embedding_budget" | "check_source" | "none";
62
+ action_code?: "run_update" | "rebuild_kb" | "wait_for_indexing" | "review_skipped_scope" | "fix_embedding_config" | "rebuild_vectors" | "reindex_for_embedding_budget" | "check_source" | "none";
62
63
  }
63
64
  export interface DoctorReport {
64
65
  health_score: number;
@@ -72,6 +73,21 @@ export interface DoctorReport {
72
73
  description: string;
73
74
  }>;
74
75
  }
76
+ export interface KnowledgeBaseEmbeddingStatus {
77
+ kb_id: string;
78
+ kb_name: string;
79
+ indexed_model: string | null;
80
+ active_model: string;
81
+ active_decoration: string;
82
+ compatibility: EmbeddingCompatibility;
83
+ detail: string;
84
+ }
85
+ export interface KnowledgeStatusSnapshot {
86
+ knowledge_bases: KnowledgeBase[];
87
+ diagnostics: DiagnosticResult[];
88
+ embeddings: KnowledgeBaseEmbeddingStatus[];
89
+ embedding_error?: string;
90
+ }
75
91
  export interface SymbolSearchOptions {
76
92
  kb_id?: string;
77
93
  kind?: KnowledgeSymbol["kind"];
@@ -133,6 +149,7 @@ export declare class KnowledgeEngine {
133
149
  list(signal?: AbortSignal): KnowledgeBase[];
134
150
  clear(): void;
135
151
  diagnose(signal?: AbortSignal): DiagnosticResult[];
152
+ statusSnapshot(signal?: AbortSignal): KnowledgeStatusSnapshot;
136
153
  symbolSearch(query: string, options?: SymbolSearchOptions, signal?: AbortSignal): SymbolSearchResponse;
137
154
  doctor(signal?: AbortSignal): DoctorReport;
138
155
  exportKB(nameOrId: string, outputPath: string, signal?: AbortSignal, onProgress?: ProgressCallback): Promise<number>;