pi-knowledge 0.13.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/README.md +30 -8
- package/dist/index.js +176 -84
- package/dist/src/embedding/instructions.d.ts +33 -0
- package/dist/src/embedding/instructions.js +146 -0
- package/dist/src/embedding/provider.d.ts +25 -8
- package/dist/src/embedding/provider.js +155 -63
- package/dist/src/engine.d.ts +68 -2
- package/dist/src/engine.js +483 -128
- package/dist/src/indexer/embedding-safe-chunks.js +3 -2
- package/dist/src/model-worker-client.d.ts +4 -0
- package/dist/src/model-worker-client.js +14 -2
- package/dist/src/model-worker.js +34 -3
- package/dist/src/search/tuning.d.ts +2 -0
- package/dist/src/search/tuning.js +2 -0
- package/dist/src/status.d.ts +8 -0
- package/dist/src/status.js +93 -0
- package/docs/configuration.md +16 -11
- package/package.json +1 -1
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
export const MAX_INSTRUCTION_CHARS = 512;
|
|
3
|
+
export const DEFAULT_EMBEDDING_TASK = "nl2code";
|
|
4
|
+
export const ACCEPTED_DECORATION_VALUES = ["on", "off", "e5", "none", "jina", "jina-code", "custom"];
|
|
5
|
+
export const ACCEPTED_EMBEDDING_TASKS = ["nl2code", "code2code", "code2nl", "code2completion", "qa"];
|
|
6
|
+
const STRATEGY_ALIASES = {
|
|
7
|
+
on: "e5",
|
|
8
|
+
e5: "e5",
|
|
9
|
+
off: "none",
|
|
10
|
+
none: "none",
|
|
11
|
+
jina: "jina-code",
|
|
12
|
+
"jina-code": "jina-code",
|
|
13
|
+
custom: "custom",
|
|
14
|
+
};
|
|
15
|
+
const STRATEGY_ENV = "PI_KNOWLEDGE_EMBEDDING_PREFIXES";
|
|
16
|
+
const TASK_ENV = "PI_KNOWLEDGE_EMBEDDING_TASK";
|
|
17
|
+
const QUERY_INSTRUCTION_ENV = "PI_KNOWLEDGE_EMBEDDING_QUERY_INSTRUCTION";
|
|
18
|
+
const DOCUMENT_INSTRUCTION_ENV = "PI_KNOWLEDGE_EMBEDDING_DOCUMENT_INSTRUCTION";
|
|
19
|
+
export const E5_INSTRUCTIONS = Object.freeze({ query: "query: ", document: "passage: " });
|
|
20
|
+
export const NO_INSTRUCTIONS = Object.freeze({ query: "", document: "" });
|
|
21
|
+
/**
|
|
22
|
+
* Published prompts for jinaai/jina-code-embeddings-1.5b.
|
|
23
|
+
* Source: config_sentence_transformers.json at model revision
|
|
24
|
+
* 39aeb4fb9b60f930934c78ae5d749a46287c248a (verified 2026-11-12).
|
|
25
|
+
* Values already contain their required trailing newline and are concatenated verbatim.
|
|
26
|
+
*/
|
|
27
|
+
export const JINA_CODE_INSTRUCTIONS = Object.freeze({
|
|
28
|
+
nl2code: Object.freeze({
|
|
29
|
+
query: "Find the most relevant code snippet given the following query:\n",
|
|
30
|
+
document: "Candidate code snippet:\n",
|
|
31
|
+
}),
|
|
32
|
+
code2code: Object.freeze({
|
|
33
|
+
query: "Find an equivalent code snippet given the following code snippet:\n",
|
|
34
|
+
document: "Candidate code snippet:\n",
|
|
35
|
+
}),
|
|
36
|
+
code2nl: Object.freeze({
|
|
37
|
+
query: "Find the most relevant comment given the following code snippet:\n",
|
|
38
|
+
document: "Candidate comment:\n",
|
|
39
|
+
}),
|
|
40
|
+
code2completion: Object.freeze({
|
|
41
|
+
query: "Find the most relevant completion given the following start of code snippet:\n",
|
|
42
|
+
document: "Candidate completion:\n",
|
|
43
|
+
}),
|
|
44
|
+
qa: Object.freeze({
|
|
45
|
+
query: "Find the most relevant answer given the following question:\n",
|
|
46
|
+
document: "Candidate answer:\n",
|
|
47
|
+
}),
|
|
48
|
+
});
|
|
49
|
+
export function decorateEmbeddingInput(text, role, instructions) {
|
|
50
|
+
return `${role === "query" ? instructions.query : instructions.document}${text}`;
|
|
51
|
+
}
|
|
52
|
+
export function embeddingInstructionLength(instructions, role) {
|
|
53
|
+
return (role === "query" ? instructions.query : instructions.document).length;
|
|
54
|
+
}
|
|
55
|
+
export function embeddingInstructionSetHash(instructions) {
|
|
56
|
+
return createHash("sha256")
|
|
57
|
+
.update(JSON.stringify([instructions.query, instructions.document]))
|
|
58
|
+
.digest("hex")
|
|
59
|
+
.slice(0, 12);
|
|
60
|
+
}
|
|
61
|
+
export function embeddingDecorationDescriptor(decoration) {
|
|
62
|
+
if (decoration.strategy === "e5")
|
|
63
|
+
return "q=query:d=passage";
|
|
64
|
+
if (decoration.strategy === "none")
|
|
65
|
+
return "q=query:d=passage:prefixes=off";
|
|
66
|
+
const instructionHash = embeddingInstructionSetHash(decoration.instructions);
|
|
67
|
+
if (decoration.strategy === "jina-code") {
|
|
68
|
+
return `strategy=jina-code:task=${decoration.task ?? DEFAULT_EMBEDDING_TASK}:ih=${instructionHash}`;
|
|
69
|
+
}
|
|
70
|
+
return `strategy=custom:ih=${instructionHash}`;
|
|
71
|
+
}
|
|
72
|
+
function cleanValue(value) {
|
|
73
|
+
const trimmed = value?.trim();
|
|
74
|
+
return trimmed ? trimmed : undefined;
|
|
75
|
+
}
|
|
76
|
+
function assertInstructionValue(name, value) {
|
|
77
|
+
if (value.length > MAX_INSTRUCTION_CHARS) {
|
|
78
|
+
throw new Error(`Embedding instruction ${name} is ${value.length} characters; maximum is ${MAX_INSTRUCTION_CHARS}.`);
|
|
79
|
+
}
|
|
80
|
+
for (const character of value) {
|
|
81
|
+
const code = character.codePointAt(0) ?? 0;
|
|
82
|
+
if (character !== "\n" && (code < 0x20 || (code >= 0x7f && code <= 0x9f))) {
|
|
83
|
+
throw new Error(`Embedding instruction ${name} contains an unsupported control character; only printable text and newline are allowed.`);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
function resolveCustomInstructions(env) {
|
|
88
|
+
const query = env[QUERY_INSTRUCTION_ENV];
|
|
89
|
+
const document = env[DOCUMENT_INSTRUCTION_ENV];
|
|
90
|
+
const missing = [
|
|
91
|
+
query === undefined ? QUERY_INSTRUCTION_ENV : "",
|
|
92
|
+
document === undefined ? DOCUMENT_INSTRUCTION_ENV : "",
|
|
93
|
+
].filter(Boolean);
|
|
94
|
+
if (missing.length > 0)
|
|
95
|
+
throw new Error(`${STRATEGY_ENV}=custom requires ${missing.join(" and ")}.`);
|
|
96
|
+
if (query === undefined || document === undefined)
|
|
97
|
+
throw new Error("Embedding instruction resolution failed");
|
|
98
|
+
if (!query.trim())
|
|
99
|
+
throw new Error(`${QUERY_INSTRUCTION_ENV} must be a non-empty string.`);
|
|
100
|
+
assertInstructionValue(QUERY_INSTRUCTION_ENV, query);
|
|
101
|
+
assertInstructionValue(DOCUMENT_INSTRUCTION_ENV, document);
|
|
102
|
+
return Object.freeze({ query, document });
|
|
103
|
+
}
|
|
104
|
+
export function resolveEmbeddingDecoration(env) {
|
|
105
|
+
const rawStrategy = cleanValue(env[STRATEGY_ENV])?.toLowerCase();
|
|
106
|
+
const strategy = rawStrategy ? STRATEGY_ALIASES[rawStrategy] : "e5";
|
|
107
|
+
if (!strategy) {
|
|
108
|
+
throw new Error(`Unsupported embedding decoration strategy: ${rawStrategy}. Use ${STRATEGY_ENV}=${ACCEPTED_DECORATION_VALUES.join("|")}.`);
|
|
109
|
+
}
|
|
110
|
+
const customVariablesPresent = env[QUERY_INSTRUCTION_ENV] !== undefined || env[DOCUMENT_INSTRUCTION_ENV] !== undefined;
|
|
111
|
+
if (strategy !== "custom" && customVariablesPresent) {
|
|
112
|
+
throw new Error(`${QUERY_INSTRUCTION_ENV} and ${DOCUMENT_INSTRUCTION_ENV} are valid only when ${STRATEGY_ENV}=custom.`);
|
|
113
|
+
}
|
|
114
|
+
const rawTask = cleanValue(env[TASK_ENV])?.toLowerCase();
|
|
115
|
+
let task;
|
|
116
|
+
if (rawTask) {
|
|
117
|
+
if (strategy !== "jina-code") {
|
|
118
|
+
throw new Error(`${TASK_ENV} is valid only when ${STRATEGY_ENV}=jina-code.`);
|
|
119
|
+
}
|
|
120
|
+
if (!ACCEPTED_EMBEDDING_TASKS.includes(rawTask)) {
|
|
121
|
+
throw new Error(`Unsupported embedding task: ${rawTask}. Use ${TASK_ENV}=${ACCEPTED_EMBEDDING_TASKS.join("|")}.`);
|
|
122
|
+
}
|
|
123
|
+
task = rawTask;
|
|
124
|
+
}
|
|
125
|
+
else if (strategy === "jina-code") {
|
|
126
|
+
task = DEFAULT_EMBEDDING_TASK;
|
|
127
|
+
}
|
|
128
|
+
let instructions;
|
|
129
|
+
if (strategy === "e5")
|
|
130
|
+
instructions = E5_INSTRUCTIONS;
|
|
131
|
+
else if (strategy === "none")
|
|
132
|
+
instructions = NO_INSTRUCTIONS;
|
|
133
|
+
else if (strategy === "jina-code")
|
|
134
|
+
instructions = JINA_CODE_INSTRUCTIONS[task ?? DEFAULT_EMBEDDING_TASK];
|
|
135
|
+
else
|
|
136
|
+
instructions = resolveCustomInstructions(env);
|
|
137
|
+
return Object.freeze({
|
|
138
|
+
strategy,
|
|
139
|
+
task,
|
|
140
|
+
instructions,
|
|
141
|
+
prefixMode: strategy === "e5" ? "on" : "off",
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
export function decorationSupportedByLocalModel(strategy) {
|
|
145
|
+
return strategy === "e5" || strategy === "none";
|
|
146
|
+
}
|
|
@@ -1,14 +1,16 @@
|
|
|
1
|
+
import { type EmbeddingDecorationStrategy, type EmbeddingInstructionSet, type EmbeddingPrefixMode, type EmbeddingRole, type EmbeddingTask } from "./instructions.ts";
|
|
2
|
+
export type { EmbeddingDecorationStrategy, EmbeddingInstructionSet, EmbeddingRole, EmbeddingTask, } from "./instructions.ts";
|
|
1
3
|
export type EmbeddingProvider = "local" | "openai";
|
|
2
|
-
export type EmbeddingPrefix = "query" | "passage";
|
|
3
|
-
export type EmbeddingPrefixStrategy = "on" | "off";
|
|
4
4
|
export interface EmbeddingConfig {
|
|
5
5
|
provider: EmbeddingProvider;
|
|
6
6
|
model: string;
|
|
7
7
|
baseUrl?: string;
|
|
8
8
|
maxChars: number;
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
9
|
+
strategy: EmbeddingDecorationStrategy;
|
|
10
|
+
task?: EmbeddingTask;
|
|
11
|
+
instructions: EmbeddingInstructionSet;
|
|
12
|
+
prefixMode: EmbeddingPrefixMode;
|
|
13
|
+
apiFallbackToLocal: boolean;
|
|
12
14
|
pooling: "mean";
|
|
13
15
|
normalize: true;
|
|
14
16
|
}
|
|
@@ -18,6 +20,17 @@ export interface EmbeddingInputMeasurement {
|
|
|
18
20
|
preferredMaximum: number;
|
|
19
21
|
unit: "tokens" | "characters";
|
|
20
22
|
}
|
|
23
|
+
export type EmbeddingCompatibilityCause = "match" | "unrecorded" | "dimension" | "decoration" | "configuration" | "signature_format";
|
|
24
|
+
export interface EmbeddingCompatibility {
|
|
25
|
+
compatible: boolean;
|
|
26
|
+
cause: EmbeddingCompatibilityCause;
|
|
27
|
+
activeSignature: string;
|
|
28
|
+
recordedSignature: string | null;
|
|
29
|
+
activeDecoration: string;
|
|
30
|
+
recordedDecoration?: string;
|
|
31
|
+
activeDimension: number;
|
|
32
|
+
recordedDimension: number | null;
|
|
33
|
+
}
|
|
21
34
|
export declare const CURRENT_EMBEDDING_MODEL = "multilingual-e5-small";
|
|
22
35
|
export declare const DEFAULT_LOCAL_EMBEDDING_MODEL = "multilingual-e5-small";
|
|
23
36
|
export declare const DEFAULT_OPENAI_EMBEDDING_MODEL = "text-embedding-3-small";
|
|
@@ -29,18 +42,22 @@ export declare const API_VECTOR_NORMALIZATION_POLICY = "client-l2-v1";
|
|
|
29
42
|
export declare function resolveEmbeddingConfig(env?: NodeJS.ProcessEnv): EmbeddingConfig;
|
|
30
43
|
export declare function embeddingConfigLabel(config: EmbeddingConfig): string;
|
|
31
44
|
export declare function normalizeEmbeddingVector(vector: Float32Array): Float32Array;
|
|
45
|
+
export declare function describeEmbeddingDecoration(config: EmbeddingConfig): string;
|
|
32
46
|
export declare function embeddingSignature(config: EmbeddingConfig, dimension: number): string;
|
|
47
|
+
export declare function recordedEmbeddingDecoration(signature: string | null): string | undefined;
|
|
48
|
+
export declare function inspectEmbeddingCompatibility(recordedSignature: string | null, recordedDimension: number | null, config: EmbeddingConfig, activeDimension: number): EmbeddingCompatibility;
|
|
49
|
+
export declare function describeEmbeddingCompatibility(compatibility: EmbeddingCompatibility): string;
|
|
33
50
|
export declare function dispose(): Promise<void>;
|
|
34
51
|
export declare function prepareForShutdown(): Promise<void>;
|
|
35
|
-
export declare function measureEmbeddingInputs(texts: string[],
|
|
36
|
-
export declare function embedTextsWithConfig(texts: string[],
|
|
52
|
+
export declare function measureEmbeddingInputs(texts: string[], role: EmbeddingRole, config: EmbeddingConfig, signal?: AbortSignal): Promise<EmbeddingInputMeasurement>;
|
|
53
|
+
export declare function embedTextsWithConfig(texts: string[], role: EmbeddingRole, signal?: AbortSignal, options?: {
|
|
37
54
|
allowApiFallback?: boolean;
|
|
38
55
|
inputValidated?: boolean;
|
|
39
56
|
}): Promise<{
|
|
40
57
|
vectors: Float32Array[];
|
|
41
58
|
config: EmbeddingConfig;
|
|
42
59
|
}>;
|
|
43
|
-
export declare function embedTexts(texts: string[],
|
|
60
|
+
export declare function embedTexts(texts: string[], role: EmbeddingRole, signal?: AbortSignal, options?: {
|
|
44
61
|
allowApiFallback?: boolean;
|
|
45
62
|
inputValidated?: boolean;
|
|
46
63
|
}): Promise<Float32Array[]>;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
2
|
import { embedInModelWorker, measureEmbeddingInputInModelWorker } from "../model-worker-client.js";
|
|
3
|
+
import { decorateEmbeddingInput, decorationSupportedByLocalModel, embeddingDecorationDescriptor, embeddingInstructionLength, resolveEmbeddingDecoration, } from "./instructions.js";
|
|
3
4
|
let disposeTimer = null;
|
|
4
5
|
let disposePromise = null;
|
|
5
6
|
let activeRuns = 0;
|
|
@@ -15,15 +16,16 @@ export const DEFAULT_API_MAX_EMBED_CHARS = 20_000;
|
|
|
15
16
|
export const API_VECTOR_NORMALIZATION_POLICY = "client-l2-v1";
|
|
16
17
|
const IDLE_TIMEOUT_MS = Number(process.env.PI_KNOWLEDGE_EMBEDDING_IDLE_MS ?? 30_000);
|
|
17
18
|
const ENABLE_NATIVE_IDLE_DISPOSE = process.env.PI_KNOWLEDGE_ENABLE_NATIVE_IDLE_DISPOSE === "true";
|
|
18
|
-
|
|
19
|
-
function localEmbeddingConfig(maxChars, prefixes) {
|
|
19
|
+
function localEmbeddingConfig(maxChars, decoration) {
|
|
20
20
|
return {
|
|
21
21
|
provider: "local",
|
|
22
22
|
model: DEFAULT_LOCAL_EMBEDDING_MODEL,
|
|
23
23
|
maxChars,
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
24
|
+
strategy: decoration.strategy,
|
|
25
|
+
task: decoration.task,
|
|
26
|
+
instructions: decoration.instructions,
|
|
27
|
+
prefixMode: decoration.prefixMode,
|
|
28
|
+
apiFallbackToLocal: false,
|
|
27
29
|
pooling: "mean",
|
|
28
30
|
normalize: true,
|
|
29
31
|
};
|
|
@@ -36,8 +38,7 @@ function isAbortError(error) {
|
|
|
36
38
|
return typeof error === "object" && error !== null && "name" in error && error.name === "AbortError";
|
|
37
39
|
}
|
|
38
40
|
function clearIdleTimer() {
|
|
39
|
-
|
|
40
|
-
clearTimeout(disposeTimer);
|
|
41
|
+
clearTimeout(disposeTimer ?? undefined);
|
|
41
42
|
disposeTimer = null;
|
|
42
43
|
}
|
|
43
44
|
function scheduleIdleDispose() {
|
|
@@ -61,21 +62,23 @@ function endRun() {
|
|
|
61
62
|
if (!disposeRequested)
|
|
62
63
|
scheduleIdleDispose();
|
|
63
64
|
}
|
|
64
|
-
function waitForNoActiveRuns() {
|
|
65
|
+
async function waitForNoActiveRuns() {
|
|
65
66
|
if (activeRuns === 0)
|
|
66
|
-
return
|
|
67
|
-
|
|
67
|
+
return;
|
|
68
|
+
const { promise, resolve } = Promise.withResolvers();
|
|
69
|
+
idleWaiters.push(resolve);
|
|
70
|
+
await promise;
|
|
68
71
|
}
|
|
69
72
|
function providerInstanceHash(value) {
|
|
70
73
|
return createHash("sha256").update(value).digest("hex").slice(0, 12);
|
|
71
74
|
}
|
|
72
|
-
function
|
|
73
|
-
const
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
75
|
+
function assertApiInstructionBudget(config) {
|
|
76
|
+
for (const role of ["query", "passage"]) {
|
|
77
|
+
const instructionLength = embeddingInstructionLength(config.instructions, role);
|
|
78
|
+
if (instructionLength >= config.maxChars) {
|
|
79
|
+
throw new Error(`Embedding ${role} instruction is ${instructionLength} characters; PI_KNOWLEDGE_EMBEDDING_MAX_CHARS=${config.maxChars} must leave room for input text.`);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
79
82
|
}
|
|
80
83
|
export function resolveEmbeddingConfig(env = process.env) {
|
|
81
84
|
const raw = cleanEnv(env.PI_KNOWLEDGE_EMBEDDING) ?? `local:${DEFAULT_LOCAL_EMBEDDING_MODEL}`;
|
|
@@ -84,26 +87,41 @@ export function resolveEmbeddingConfig(env = process.env) {
|
|
|
84
87
|
const model = separator >= 0 ? raw.slice(separator + 1).trim() : "";
|
|
85
88
|
const maxCharsValue = Number(cleanEnv(env.PI_KNOWLEDGE_EMBEDDING_MAX_CHARS) ?? DEFAULT_API_MAX_EMBED_CHARS);
|
|
86
89
|
const maxChars = Number.isFinite(maxCharsValue) && maxCharsValue > 0 ? Math.trunc(maxCharsValue) : DEFAULT_API_MAX_EMBED_CHARS;
|
|
87
|
-
const
|
|
90
|
+
const decoration = resolveEmbeddingDecoration(env);
|
|
91
|
+
const fallback = cleanEnv(env.PI_KNOWLEDGE_EMBEDDING_API_FALLBACK)?.toLowerCase();
|
|
92
|
+
if (fallback && fallback !== "local") {
|
|
93
|
+
throw new Error("Unsupported embedding API fallback. Use PI_KNOWLEDGE_EMBEDDING_API_FALLBACK=local or unset it.");
|
|
94
|
+
}
|
|
95
|
+
const apiFallbackToLocal = fallback === "local";
|
|
88
96
|
if (provider === "local") {
|
|
89
97
|
const localModel = model || DEFAULT_LOCAL_EMBEDDING_MODEL;
|
|
90
98
|
if (localModel !== DEFAULT_LOCAL_EMBEDDING_MODEL) {
|
|
91
99
|
throw new Error(`Unsupported local embedding model: ${localModel}`);
|
|
92
100
|
}
|
|
93
|
-
|
|
101
|
+
if (!decorationSupportedByLocalModel(decoration.strategy)) {
|
|
102
|
+
throw new Error(`Embedding decoration ${decoration.strategy} is not supported for the local embedding model. Use PI_KNOWLEDGE_EMBEDDING_PREFIXES=on|off|e5|none, or an OpenAI-compatible provider.`);
|
|
103
|
+
}
|
|
104
|
+
return localEmbeddingConfig(maxChars, decoration);
|
|
94
105
|
}
|
|
95
106
|
if (provider === "openai") {
|
|
96
|
-
|
|
107
|
+
if (apiFallbackToLocal && !decorationSupportedByLocalModel(decoration.strategy)) {
|
|
108
|
+
throw new Error(`PI_KNOWLEDGE_EMBEDDING_API_FALLBACK=local is incompatible with embedding decoration ${decoration.strategy}; the local model supports only E5 prefixes or none.`);
|
|
109
|
+
}
|
|
110
|
+
const config = {
|
|
97
111
|
provider: "openai",
|
|
98
112
|
model: model || DEFAULT_OPENAI_EMBEDDING_MODEL,
|
|
99
113
|
baseUrl: cleanEnv(env.PI_KNOWLEDGE_EMBEDDING_BASE_URL) ?? cleanEnv(env.OPENAI_BASE_URL) ?? DEFAULT_OPENAI_BASE_URL,
|
|
100
114
|
maxChars,
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
115
|
+
strategy: decoration.strategy,
|
|
116
|
+
task: decoration.task,
|
|
117
|
+
instructions: decoration.instructions,
|
|
118
|
+
prefixMode: decoration.prefixMode,
|
|
119
|
+
apiFallbackToLocal,
|
|
104
120
|
pooling: "mean",
|
|
105
121
|
normalize: true,
|
|
106
122
|
};
|
|
123
|
+
assertApiInstructionBudget(config);
|
|
124
|
+
return config;
|
|
107
125
|
}
|
|
108
126
|
throw new Error(`Unsupported embedding provider: ${provider}`);
|
|
109
127
|
}
|
|
@@ -125,29 +143,86 @@ export function normalizeEmbeddingVector(vector) {
|
|
|
125
143
|
vector[index] /= magnitude;
|
|
126
144
|
return vector;
|
|
127
145
|
}
|
|
146
|
+
export function describeEmbeddingDecoration(config) {
|
|
147
|
+
return embeddingDecorationDescriptor({
|
|
148
|
+
strategy: config.strategy,
|
|
149
|
+
task: config.task,
|
|
150
|
+
instructions: config.instructions,
|
|
151
|
+
prefixMode: config.prefixMode,
|
|
152
|
+
});
|
|
153
|
+
}
|
|
128
154
|
export function embeddingSignature(config, dimension) {
|
|
129
155
|
const base = config.baseUrl ? `:base-sha256=${providerInstanceHash(config.baseUrl)}` : "";
|
|
130
156
|
const normalization = config.provider === "openai" ? API_VECTOR_NORMALIZATION_POLICY : String(config.normalize);
|
|
131
|
-
|
|
157
|
+
return [
|
|
132
158
|
`${config.provider}:${config.model}${base}`,
|
|
133
159
|
`dim=${dimension}`,
|
|
134
160
|
`apiMax=${config.provider === "openai" ? config.maxChars : "none"}`,
|
|
135
161
|
`pooling=${config.pooling}`,
|
|
136
162
|
`normalize=${normalization}`,
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
163
|
+
describeEmbeddingDecoration(config),
|
|
164
|
+
].join(":");
|
|
165
|
+
}
|
|
166
|
+
const RECORDED_DECORATION_PATTERN = /:(q=query:d=passage(?::prefixes=off)?|strategy=(?:jina-code|custom)(?::task=[a-z0-9-]+)?:ih=[0-9a-f]{12})$/;
|
|
167
|
+
export function recordedEmbeddingDecoration(signature) {
|
|
168
|
+
if (!signature)
|
|
169
|
+
return undefined;
|
|
170
|
+
return RECORDED_DECORATION_PATTERN.exec(signature)?.[1];
|
|
171
|
+
}
|
|
172
|
+
export function inspectEmbeddingCompatibility(recordedSignature, recordedDimension, config, activeDimension) {
|
|
173
|
+
const activeSignature = embeddingSignature(config, activeDimension);
|
|
174
|
+
const activeDecoration = describeEmbeddingDecoration(config);
|
|
175
|
+
const base = {
|
|
176
|
+
activeSignature,
|
|
177
|
+
recordedSignature,
|
|
178
|
+
activeDecoration,
|
|
179
|
+
activeDimension,
|
|
180
|
+
recordedDimension,
|
|
181
|
+
};
|
|
182
|
+
if (!recordedSignature || recordedDimension === null) {
|
|
183
|
+
return { ...base, compatible: false, cause: "unrecorded" };
|
|
184
|
+
}
|
|
185
|
+
const recordedDecoration = recordedEmbeddingDecoration(recordedSignature);
|
|
186
|
+
if (recordedDimension !== activeDimension) {
|
|
187
|
+
return { ...base, compatible: false, cause: "dimension", recordedDecoration };
|
|
188
|
+
}
|
|
189
|
+
if (recordedSignature === activeSignature) {
|
|
190
|
+
return { ...base, compatible: true, cause: "match", recordedDecoration };
|
|
191
|
+
}
|
|
192
|
+
if (!recordedDecoration) {
|
|
193
|
+
return { ...base, compatible: false, cause: "signature_format" };
|
|
194
|
+
}
|
|
195
|
+
if (recordedDecoration !== activeDecoration) {
|
|
196
|
+
const recordedBase = recordedSignature.slice(0, recordedSignature.length - recordedDecoration.length - 1);
|
|
197
|
+
const activeBase = activeSignature.slice(0, activeSignature.length - activeDecoration.length - 1);
|
|
198
|
+
if (recordedBase === activeBase) {
|
|
199
|
+
return { ...base, compatible: false, cause: "decoration", recordedDecoration };
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
return { ...base, compatible: false, cause: "configuration", recordedDecoration };
|
|
203
|
+
}
|
|
204
|
+
export function describeEmbeddingCompatibility(compatibility) {
|
|
205
|
+
switch (compatibility.cause) {
|
|
206
|
+
case "match":
|
|
207
|
+
return "indexed vectors match the active embedding configuration";
|
|
208
|
+
case "unrecorded":
|
|
209
|
+
return "the indexed embedding signature or dimension is missing";
|
|
210
|
+
case "dimension":
|
|
211
|
+
return `indexed dimension ${compatibility.recordedDimension ?? "unknown"} differs from active dimension ${compatibility.activeDimension}`;
|
|
212
|
+
case "decoration":
|
|
213
|
+
return `indexed decoration ${compatibility.recordedDecoration ?? "unknown"} differs from active decoration ${compatibility.activeDecoration}`;
|
|
214
|
+
case "configuration":
|
|
215
|
+
return "the indexed provider, model, endpoint, input limit, pooling, or normalization differs from the active configuration";
|
|
216
|
+
case "signature_format":
|
|
217
|
+
return "the indexed embedding signature cannot be interpreted";
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
export function dispose() {
|
|
145
221
|
clearIdleTimer();
|
|
146
222
|
if (disposePromise)
|
|
147
223
|
return disposePromise;
|
|
148
224
|
disposeRequested = true;
|
|
149
|
-
|
|
150
|
-
disposePromise = Promise.resolve().finally(() => {
|
|
225
|
+
disposePromise = waitForNoActiveRuns().finally(() => {
|
|
151
226
|
disposePromise = null;
|
|
152
227
|
disposeRequested = false;
|
|
153
228
|
});
|
|
@@ -157,23 +232,26 @@ export async function prepareForShutdown() {
|
|
|
157
232
|
clearIdleTimer();
|
|
158
233
|
await waitForNoActiveRuns();
|
|
159
234
|
}
|
|
160
|
-
function
|
|
161
|
-
return
|
|
235
|
+
function prepareEmbeddingInputs(texts, role, config) {
|
|
236
|
+
return texts.map((text) => decorateEmbeddingInput(text, role, config.instructions));
|
|
162
237
|
}
|
|
163
|
-
|
|
238
|
+
function apiInputMeasurement(preparedInputs, config) {
|
|
239
|
+
return {
|
|
240
|
+
counts: preparedInputs.map((text) => text.length),
|
|
241
|
+
maximum: config.maxChars,
|
|
242
|
+
preferredMaximum: Math.max(1, Math.floor(config.maxChars * 0.875)),
|
|
243
|
+
unit: "characters",
|
|
244
|
+
};
|
|
245
|
+
}
|
|
246
|
+
export async function measureEmbeddingInputs(texts, role, config, signal) {
|
|
164
247
|
if (signal?.aborted)
|
|
165
248
|
throw new Error("Cancelled");
|
|
166
249
|
if (config.provider === "openai") {
|
|
167
|
-
return
|
|
168
|
-
counts: texts.map((text) => applyEmbeddingPrefix(text, prefix, config).length),
|
|
169
|
-
maximum: config.maxChars,
|
|
170
|
-
preferredMaximum: Math.max(1, Math.floor(config.maxChars * 0.875)),
|
|
171
|
-
unit: "characters",
|
|
172
|
-
};
|
|
250
|
+
return apiInputMeasurement(prepareEmbeddingInputs(texts, role, config), config);
|
|
173
251
|
}
|
|
174
252
|
beginRun();
|
|
175
253
|
try {
|
|
176
|
-
const measurement = await measureEmbeddingInputInModelWorker(texts,
|
|
254
|
+
const measurement = await measureEmbeddingInputInModelWorker(texts, role, config.prefixMode, signal);
|
|
177
255
|
return {
|
|
178
256
|
...measurement,
|
|
179
257
|
preferredMaximum: Math.max(1, Math.floor(measurement.maximum * 0.875)),
|
|
@@ -184,17 +262,18 @@ export async function measureEmbeddingInputs(texts, prefix, config, signal) {
|
|
|
184
262
|
endRun();
|
|
185
263
|
}
|
|
186
264
|
}
|
|
187
|
-
|
|
188
|
-
const measurement = await measureEmbeddingInputs(texts, prefix, config, signal);
|
|
265
|
+
function assertMeasurementWithinBudget(measurement, role, provider) {
|
|
189
266
|
for (let index = 0; index < measurement.counts.length; index++) {
|
|
190
267
|
const count = measurement.counts[index];
|
|
191
268
|
if (count !== undefined && count > measurement.maximum) {
|
|
192
|
-
throw new Error(`${
|
|
269
|
+
throw new Error(`${provider} embedding ${role} input ${index + 1} is ${count} ${measurement.unit}; maximum is ${measurement.maximum}. Split the input before embedding.`);
|
|
193
270
|
}
|
|
194
271
|
}
|
|
195
272
|
}
|
|
196
|
-
async function
|
|
197
|
-
|
|
273
|
+
async function assertLocalInputsWithinBudget(texts, role, config, signal) {
|
|
274
|
+
assertMeasurementWithinBudget(await measureEmbeddingInputs(texts, role, config, signal), role, config.provider);
|
|
275
|
+
}
|
|
276
|
+
async function embedPreparedInputsViaAPI(preparedInputs, config, signal) {
|
|
198
277
|
const apiKey = cleanEnv(process.env.PI_KNOWLEDGE_EMBEDDING_API_KEY) ?? cleanEnv(process.env.OPENAI_API_KEY);
|
|
199
278
|
if (!config.baseUrl)
|
|
200
279
|
throw new Error("OpenAI embedding base URL is not configured");
|
|
@@ -205,7 +284,7 @@ async function embedViaAPI(texts, prefix, config, signal) {
|
|
|
205
284
|
const res = await fetch(endpoint, {
|
|
206
285
|
method: "POST",
|
|
207
286
|
headers,
|
|
208
|
-
body: JSON.stringify({ input:
|
|
287
|
+
body: JSON.stringify({ input: preparedInputs, model: config.model }),
|
|
209
288
|
signal,
|
|
210
289
|
});
|
|
211
290
|
if (!res.ok) {
|
|
@@ -216,8 +295,8 @@ async function embedViaAPI(texts, prefix, config, signal) {
|
|
|
216
295
|
if (!payload || typeof payload !== "object" || !("data" in payload && Array.isArray(payload.data))) {
|
|
217
296
|
throw new Error("OpenAI embedding API response is missing a data array");
|
|
218
297
|
}
|
|
219
|
-
if (payload.data.length !==
|
|
220
|
-
throw new Error(`OpenAI embedding API returned ${payload.data.length} vectors for ${
|
|
298
|
+
if (payload.data.length !== preparedInputs.length) {
|
|
299
|
+
throw new Error(`OpenAI embedding API returned ${payload.data.length} vectors for ${preparedInputs.length} inputs`);
|
|
221
300
|
}
|
|
222
301
|
let expectedDimension;
|
|
223
302
|
return payload.data.map((item, index) => {
|
|
@@ -239,44 +318,57 @@ async function embedViaAPI(texts, prefix, config, signal) {
|
|
|
239
318
|
return normalizeEmbeddingVector(vector);
|
|
240
319
|
});
|
|
241
320
|
}
|
|
242
|
-
export async function embedTextsWithConfig(texts,
|
|
321
|
+
export async function embedTextsWithConfig(texts, role, signal, options = {}) {
|
|
243
322
|
const config = resolveEmbeddingConfig();
|
|
244
323
|
let actualConfig = config;
|
|
245
324
|
let inputValidated = options.inputValidated === true;
|
|
246
325
|
if (config.provider === "openai") {
|
|
247
|
-
|
|
248
|
-
|
|
326
|
+
const preparedInputs = prepareEmbeddingInputs(texts, role, config);
|
|
327
|
+
if (!inputValidated) {
|
|
328
|
+
assertMeasurementWithinBudget(apiInputMeasurement(preparedInputs, config), role, config.provider);
|
|
329
|
+
}
|
|
249
330
|
if (signal?.aborted)
|
|
250
331
|
throw new Error("Cancelled");
|
|
251
332
|
try {
|
|
252
|
-
return { vectors: await
|
|
333
|
+
return { vectors: await embedPreparedInputsViaAPI(preparedInputs, config, signal), config };
|
|
253
334
|
}
|
|
254
335
|
catch (error) {
|
|
255
336
|
if (signal?.aborted || isAbortError(error))
|
|
256
337
|
throw new Error("Cancelled");
|
|
257
|
-
if (!
|
|
338
|
+
if (!config.apiFallbackToLocal || !options.allowApiFallback)
|
|
258
339
|
throw error;
|
|
259
340
|
console.warn(`pi-knowledge: embedding API failed; falling back to local model because PI_KNOWLEDGE_EMBEDDING_API_FALLBACK=local (${error instanceof Error ? error.message : String(error)})`);
|
|
260
|
-
actualConfig = localEmbeddingConfig(config.maxChars,
|
|
341
|
+
actualConfig = localEmbeddingConfig(config.maxChars, {
|
|
342
|
+
strategy: config.strategy,
|
|
343
|
+
task: config.task,
|
|
344
|
+
instructions: config.instructions,
|
|
345
|
+
prefixMode: config.prefixMode,
|
|
346
|
+
});
|
|
261
347
|
inputValidated = false;
|
|
262
348
|
}
|
|
263
349
|
}
|
|
264
350
|
if (!inputValidated)
|
|
265
|
-
await
|
|
351
|
+
await assertLocalInputsWithinBudget(texts, role, actualConfig, signal);
|
|
266
352
|
beginRun();
|
|
267
353
|
try {
|
|
268
|
-
return {
|
|
354
|
+
return {
|
|
355
|
+
vectors: await embedInModelWorker(texts, role, actualConfig.prefixMode, signal),
|
|
356
|
+
config: actualConfig,
|
|
357
|
+
};
|
|
269
358
|
}
|
|
270
359
|
finally {
|
|
271
360
|
endRun();
|
|
272
361
|
}
|
|
273
362
|
}
|
|
274
|
-
export async function embedTexts(texts,
|
|
275
|
-
return (await embedTextsWithConfig(texts,
|
|
363
|
+
export async function embedTexts(texts, role, signal, options = {}) {
|
|
364
|
+
return (await embedTextsWithConfig(texts, role, signal, options)).vectors;
|
|
276
365
|
}
|
|
277
366
|
export async function embedQueryWithConfig(text, signal) {
|
|
278
367
|
const { vectors, config } = await embedTextsWithConfig([text], "query", signal, { allowApiFallback: true });
|
|
279
|
-
|
|
368
|
+
const vector = vectors[0];
|
|
369
|
+
if (!vector)
|
|
370
|
+
throw new Error("Embedding provider returned no query vector");
|
|
371
|
+
return { vector, config };
|
|
280
372
|
}
|
|
281
373
|
export async function embedQuery(text, signal) {
|
|
282
374
|
return (await embedQueryWithConfig(text, signal)).vector;
|