pi-knowledge 0.5.5 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/README.md +9 -4
- package/dist/index.js +24 -4
- package/dist/src/embedding/provider.d.ts +33 -1
- package/dist/src/embedding/provider.js +103 -37
- package/dist/src/engine.d.ts +4 -1
- package/dist/src/engine.js +98 -34
- package/dist/src/model-worker-client.d.ts +2 -1
- package/dist/src/model-worker-client.js +2 -2
- package/dist/src/model-worker.js +25 -8
- package/dist/src/search/reranker-api.d.ts +7 -0
- package/dist/src/search/reranker-api.js +89 -0
- package/dist/src/search/reranker-config.d.ts +34 -0
- package/dist/src/search/reranker-config.js +151 -0
- package/dist/src/search/reranker.js +7 -1
- package/dist/src/search/tuning.d.ts +41 -0
- package/dist/src/search/tuning.js +190 -0
- package/dist/src/storage/sqlite.d.ts +6 -0
- package/dist/src/storage/sqlite.js +23 -3
- package/docs/configuration.md +41 -6
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,28 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [0.6.0] - 2026-07-25
|
|
4
|
+
|
|
5
|
+
### Added
|
|
6
|
+
- Added configurable local/Hugging Face reranker selection through `PI_KNOWLEDGE_RERANKER`, including revision, dtype, and trusted mirror settings.
|
|
7
|
+
- Added API reranker support for `deep` search with Cohere/Jina-compatible requests and declarative custom JSON response mapping.
|
|
8
|
+
- Added embedding metadata signatures and dimensions so knowledge bases can detect incompatible vector spaces after provider, API base URL, preprocessing, or dimension changes.
|
|
9
|
+
|
|
10
|
+
### Changed
|
|
11
|
+
- Document embedding batches now surface OpenAI-compatible API failures instead of falling back to local embeddings during add, update, or import, preventing mixed-provider vector files.
|
|
12
|
+
- `knowledge_search` skips incompatible or legacy unsigned vectors with warnings, while hybrid search can fall back to BM25-only results until `knowledge_update` rebuilds vectors.
|
|
13
|
+
|
|
14
|
+
### Fixed
|
|
15
|
+
- Rebuilds unchanged chunks during `knowledge_update` when embedding metadata is missing or incompatible, rather than reusing stale vectors.
|
|
16
|
+
- Resets Transformers.js remote model settings before embedding loads so custom reranker mirrors do not affect the embedding pipeline.
|
|
17
|
+
|
|
18
|
+
## [0.5.6] - 2026-07-23
|
|
19
|
+
|
|
20
|
+
### Added
|
|
21
|
+
- Added runtime search tuning profiles (`auto`, `low_token`, `precision`, `recall`, `long_context`, `code`, `docs`, `balanced`) with diagnostics that report the applied result limit, snippet length, hybrid threshold, candidate pool, adaptive context, and deep rerank breadth.
|
|
22
|
+
|
|
23
|
+
### Changed
|
|
24
|
+
- Added bounded environment overrides for search defaults so slow local model users can reduce result count, lengthen snippets, tighten hybrid precision, or tune candidate breadth without editing source.
|
|
25
|
+
|
|
3
26
|
## [0.5.5] - 2026-07-23
|
|
4
27
|
|
|
5
28
|
### Added
|
package/README.md
CHANGED
|
@@ -125,6 +125,8 @@ omp install ./pi-knowledge
|
|
|
125
125
|
- `auto`: selects a primary mode from the query shape and retries alternate modes when results are empty or weak.
|
|
126
126
|
- `code`, `config`, `errors`, `docs`, `decision`: intent aliases for agents. `code`, `config`, and `errors` bias toward exact lexical/symbol evidence; `docs` and `decision` keep the intended source type explicit.
|
|
127
127
|
|
|
128
|
+
Search profiles tune result count, snippet length, hybrid strictness, candidate breadth, adaptive context, and deep rerank breadth. `profile: "auto"` selects a runtime profile from query shape, mode, and KB source type; explicit tool parameters still win over profile defaults. Use `profile: "low_token"` for slow local models that need fewer, stricter, longer results in one call; use `precision` for identifiers/errors, `recall` for broad discovery, `long_context` for prose/specs, and `balanced` for compatibility.
|
|
129
|
+
|
|
128
130
|
Mode selection contract:
|
|
129
131
|
|
|
130
132
|
- Start with `hybrid` for most project questions that contain useful lexical anchors.
|
|
@@ -134,7 +136,7 @@ Mode selection contract:
|
|
|
134
136
|
- Use `deep` for high-stakes answers, ambiguous top results, or final verification when slower reranking is acceptable.
|
|
135
137
|
- If results are empty or weak but the KB should contain the answer, retry once with a different mode before concluding no answer exists.
|
|
136
138
|
|
|
137
|
-
Search results use balanced diversity reranking by default so near-duplicate chunks from the same file do not dominate the top results. Diversity scoring considers lexical overlap, same-file line proximity, overlapping adaptive windows, available embedding-vector similarity, file-level interleaving, and a small KB trust multiplier that favors ready file/directory sources over stale, URL, or imported text sources. Use `diversity: "off"` only when raw ranking order is needed for diagnostics. Agents can request search diagnostics to inspect mode fallback, ranking coverage, path/source/test boosts, adjusted scores, and provenance such as chunk id, chunk hash, match reason, stale flag, and source freshness where available.
|
|
139
|
+
Search results use balanced diversity reranking by default so near-duplicate chunks from the same file do not dominate the top results. Diversity scoring considers lexical overlap, same-file line proximity, overlapping adaptive windows, available embedding-vector similarity, file-level interleaving, and a small KB trust multiplier that favors ready file/directory sources over stale, URL, or imported text sources. Use `diversity: "off"` only when raw ranking order is needed for diagnostics. Agents can request search diagnostics to inspect mode fallback, selected profile, applied search tuning, ranking coverage, path/source/test boosts, adjusted scores, and provenance such as chunk id, chunk hash, match reason, stale flag, and source freshness where available.
|
|
138
140
|
|
|
139
141
|
For best search quality, rebuild or update existing knowledge bases after upgrading. New indexes use contextual retrieval units: embeddings and FTS include file path, file type, Markdown heading breadcrumbs, and code symbol names while returned results keep the original chunk text readable. This improves queries that mention project structure, filenames, sections, or functions, and reduces duplicate-looking chunk hits.
|
|
140
142
|
Symbol/config/heading metadata is also rebuilt during `knowledge_update`; older KBs created before this feature may show a doctor action recommending update.
|
|
@@ -162,9 +164,11 @@ export PI_KNOWLEDGE_EMBEDDING_BASE_URL=http://127.0.0.1:8080/v1
|
|
|
162
164
|
export OPENAI_API_KEY=local-placeholder
|
|
163
165
|
```
|
|
164
166
|
|
|
165
|
-
API embedding failures are surfaced by default so configuration and context-window problems are visible. To explicitly allow
|
|
167
|
+
API embedding failures are surfaced by default so configuration and context-window problems are visible. To explicitly allow query-time local fallback after API failures, set `PI_KNOWLEDGE_EMBEDDING_API_FALLBACK=local`; indexing/import/update document batches still fail instead of creating mixed-provider vector files.
|
|
168
|
+
|
|
169
|
+
API embedding requests are capped at 20000 characters per input by default as a final context-window safety guard for OpenAI-compatible servers. Adjust this with `PI_KNOWLEDGE_EMBEDDING_MAX_CHARS` when your embedding model has a different context window; changing the cap changes embedding compatibility and requires `knowledge_update`.
|
|
166
170
|
|
|
167
|
-
|
|
171
|
+
Stored KB metadata includes the embedding model, vector dimension, and a non-secret embedding signature. If the current query embedding is incompatible with a KB's stored vectors, or if an older KB has no signature metadata, `knowledge_search` skips vector retrieval for that KB and reports a warning; run `knowledge_update` after changing embedding providers or upgrading older KBs.
|
|
168
172
|
|
|
169
173
|
## Configuration
|
|
170
174
|
|
|
@@ -175,8 +179,9 @@ Full configuration details are in [docs/configuration.md](docs/configuration.md)
|
|
|
175
179
|
| Storage path | `PI_KNOWLEDGE_DIR`, `OMP_KNOWLEDGE_DIR`, `PI_CODING_AGENT_DIR`, `OMP_CODING_AGENT_DIR`, `OMP_PROFILE` |
|
|
176
180
|
| Model worker and cache | `PI_KNOWLEDGE_MODEL_CACHE_DIR`, `PI_KNOWLEDGE_NODE_PATH` |
|
|
177
181
|
| Embedding provider | `PI_KNOWLEDGE_EMBEDDING`, `OPENAI_API_KEY`, `PI_KNOWLEDGE_EMBEDDING_BASE_URL`, `OPENAI_BASE_URL`, `PI_KNOWLEDGE_EMBEDDING_MAX_CHARS`, `PI_KNOWLEDGE_EMBEDDING_API_FALLBACK` |
|
|
182
|
+
| Reranker provider | `PI_KNOWLEDGE_RERANKER`, `PI_KNOWLEDGE_RERANKER_REVISION`, `PI_KNOWLEDGE_RERANKER_DTYPE`, `PI_KNOWLEDGE_RERANKER_REMOTE_HOST`, `PI_KNOWLEDGE_RERANKER_REMOTE_PATH_TEMPLATE`, `PI_KNOWLEDGE_RERANKER_API_ENDPOINT`, `PI_KNOWLEDGE_RERANKER_API_BASE_URL`, `PI_KNOWLEDGE_RERANKER_API_KEY`, `PI_KNOWLEDGE_RERANKER_API_FORMAT`, `PI_KNOWLEDGE_RERANKER_API_TIMEOUT_MS`, `PI_KNOWLEDGE_RERANKER_MAX_DOC_CHARS`, `PI_KNOWLEDGE_RERANKER_API_RESULTS_PATH`, `PI_KNOWLEDGE_RERANKER_API_INDEX_FIELD`, `PI_KNOWLEDGE_RERANKER_API_SCORE_FIELD`, `PI_KNOWLEDGE_RERANKER_API_SCORE_DIRECTION` |
|
|
178
183
|
| Native lifecycle | `PI_KNOWLEDGE_ENABLE_NATIVE_IDLE_DISPOSE`, `PI_KNOWLEDGE_EMBEDDING_IDLE_MS` |
|
|
179
|
-
| Runtime features | `PI_KNOWLEDGE_WATCH`, `PI_KNOWLEDGE_AUTO_INJECT`, `PI_KNOWLEDGE_STALE_INDEXING_MS`, `PI_KNOWLEDGE_OFFLINE` |
|
|
184
|
+
| Runtime features | `PI_KNOWLEDGE_WATCH`, `PI_KNOWLEDGE_AUTO_INJECT`, `PI_KNOWLEDGE_STALE_INDEXING_MS`, `PI_KNOWLEDGE_OFFLINE`, `PI_KNOWLEDGE_SEARCH_PROFILE`, `PI_KNOWLEDGE_SEARCH_DEFAULT_LIMIT`, `PI_KNOWLEDGE_SNIPPET_MAX_LENGTH`, `PI_KNOWLEDGE_MIN_HYBRID_SCORE`, `PI_KNOWLEDGE_SEARCH_CANDIDATE_MIN`, `PI_KNOWLEDGE_SEARCH_CANDIDATE_MULTIPLIER`, `PI_KNOWLEDGE_ADAPTIVE_CONTEXT_LINES`, `PI_KNOWLEDGE_ADAPTIVE_MAX_CHARS`, `PI_KNOWLEDGE_ADAPTIVE_NEIGHBOR_TARGET`, `PI_KNOWLEDGE_DEEP_RERANK_CANDIDATES`, `PI_KNOWLEDGE_DEEP_RERANK_TOPK_MULTIPLIER` |
|
|
180
185
|
| Release fixtures | `PI_KNOWLEDGE_E2E_PDF`, `PI_KNOWLEDGE_E2E_DOCX` |
|
|
181
186
|
|
|
182
187
|
## Pi and OMP Support
|
package/dist/index.js
CHANGED
|
@@ -363,6 +363,7 @@ export default function (pi) {
|
|
|
363
363
|
"Use mode 'semantic' for broad conceptual questions when exact terms may differ from the indexed wording or hybrid returns no lexical matches",
|
|
364
364
|
"Use mode 'adaptive' when the user needs surrounding implementation context, related nearby sections, or enough context to make a code change",
|
|
365
365
|
"Use mode 'deep' for high-stakes answers, ambiguous top results, or final verification where slower cross-encoder reranking is acceptable",
|
|
366
|
+
"Use profile 'low_token' for slow local models that need fewer, stricter, longer snippets; use profile 'precision' for identifiers/errors and 'recall' or 'long_context' for broad prose/document research",
|
|
366
367
|
"If a search returns no results or obviously weak results, retry once with a different mode before concluding the KB lacks the answer",
|
|
367
368
|
"If top results are repetitive, retry with diversity 'strong' or mode 'adaptive' instead of increasing the limit first",
|
|
368
369
|
],
|
|
@@ -381,7 +382,17 @@ export default function (pi) {
|
|
|
381
382
|
Type.Literal("errors"),
|
|
382
383
|
Type.Literal("decision"),
|
|
383
384
|
])),
|
|
384
|
-
|
|
385
|
+
profile: Type.Optional(Type.Union([
|
|
386
|
+
Type.Literal("auto"),
|
|
387
|
+
Type.Literal("balanced"),
|
|
388
|
+
Type.Literal("low_token"),
|
|
389
|
+
Type.Literal("precision"),
|
|
390
|
+
Type.Literal("recall"),
|
|
391
|
+
Type.Literal("long_context"),
|
|
392
|
+
Type.Literal("code"),
|
|
393
|
+
Type.Literal("docs"),
|
|
394
|
+
])),
|
|
395
|
+
limit: Type.Optional(Type.Number({ description: "Max results (default comes from search profile/env)" })),
|
|
385
396
|
kb_id: Type.Optional(Type.String({ description: "Limit search to a specific KB by ID or exact name" })),
|
|
386
397
|
offset: Type.Optional(Type.Number({ description: "Pagination offset" })),
|
|
387
398
|
file_type: Type.Optional(Type.String({ description: "Filter by file type (e.g. typescript, markdown, python)" })),
|
|
@@ -391,15 +402,18 @@ export default function (pi) {
|
|
|
391
402
|
}),
|
|
392
403
|
async execute(_id, params, _signal) {
|
|
393
404
|
const { engine } = await ensureInitialized();
|
|
394
|
-
const { query, mode, limit, kb_id, offset, file_type, path_pattern, diversity, diagnostics } = params;
|
|
405
|
+
const { query, mode, profile, limit, kb_id, offset, file_type, path_pattern, diversity, diagnostics } = params;
|
|
395
406
|
const filters = file_type || path_pattern ? { file_type, path_pattern } : undefined;
|
|
396
|
-
const response = await engine.search(query, { mode, limit, kb_id, offset, filters, diversity }, _signal);
|
|
407
|
+
const response = await engine.search(query, { mode, profile, limit, kb_id, offset, filters, diversity }, _signal);
|
|
397
408
|
if (response.results.length === 0) {
|
|
398
409
|
const details = [
|
|
399
410
|
"No results found.",
|
|
400
411
|
response.mode_used ? `Mode used: ${response.mode_used}` : "",
|
|
401
412
|
response.retry_modes?.length ? `Retried: ${response.retry_modes.join(", ")}` : "",
|
|
402
413
|
response.warnings?.length ? `Warnings: ${response.warnings.join(" | ")}` : "",
|
|
414
|
+
diagnostics && response.tuning
|
|
415
|
+
? `Tuning: profile=${response.tuning.requested_profile}->${response.tuning.selected_profile}, limit=${response.tuning.limit}, snippet=${response.tuning.snippet_max_length}, min_hybrid=${response.tuning.min_hybrid_score}, candidates=${response.tuning.candidate_limit}`
|
|
416
|
+
: "",
|
|
403
417
|
response.suggestions?.length ? `Suggestions: ${response.suggestions.join(" | ")}` : "",
|
|
404
418
|
]
|
|
405
419
|
.filter(Boolean)
|
|
@@ -411,6 +425,9 @@ export default function (pi) {
|
|
|
411
425
|
output += ` — mode: ${response.mode_used}`;
|
|
412
426
|
if (response.retry_modes?.length)
|
|
413
427
|
output += ` — retried: ${response.retry_modes.join(", ")}`;
|
|
428
|
+
if (diagnostics && response.tuning) {
|
|
429
|
+
output += ` — profile: ${response.tuning.requested_profile}->${response.tuning.selected_profile}`;
|
|
430
|
+
}
|
|
414
431
|
output += ":\n\n";
|
|
415
432
|
if (response.warnings?.length) {
|
|
416
433
|
output = `Warnings:\n- ${response.warnings.join("\n- ")}\n\n${output}`;
|
|
@@ -423,7 +440,10 @@ export default function (pi) {
|
|
|
423
440
|
const provenance = diagnostics && r.provenance
|
|
424
441
|
? `\nProvenance: chunk=${r.provenance.chunk_id}, hash=${r.provenance.chunk_hash.slice(0, 12)}, reason=${r.provenance.match_reason}, indexed_at=${r.provenance.indexed_at}, source_mtime=${r.provenance.source_mtime ?? "unknown"}, stale=${r.provenance.stale}`
|
|
425
442
|
: "";
|
|
426
|
-
|
|
443
|
+
const tuning = diagnostics && response.tuning && i === 0
|
|
444
|
+
? `\nTuning: limit=${response.tuning.limit}, snippet=${response.tuning.snippet_max_length}, min_hybrid=${response.tuning.min_hybrid_score}, candidates=${response.tuning.candidate_limit}`
|
|
445
|
+
: "";
|
|
446
|
+
return `[${i + 1}] ${r.file_path} (${r.kb_name}, score: ${r.score.toFixed(3)})\n${r.snippet}${diag}${provenance}${tuning}`;
|
|
427
447
|
})
|
|
428
448
|
.join("\n\n");
|
|
429
449
|
return { content: [{ type: "text", text: output }], details: diagnostics ? response : undefined };
|
|
@@ -1,5 +1,37 @@
|
|
|
1
|
+
export type EmbeddingProvider = "local" | "openai";
|
|
2
|
+
export type EmbeddingPrefix = "query" | "passage";
|
|
3
|
+
export interface EmbeddingConfig {
|
|
4
|
+
provider: EmbeddingProvider;
|
|
5
|
+
model: string;
|
|
6
|
+
baseUrl?: string;
|
|
7
|
+
maxChars: number;
|
|
8
|
+
queryPrefix: "query";
|
|
9
|
+
documentPrefix: "passage";
|
|
10
|
+
pooling: "mean";
|
|
11
|
+
normalize: true;
|
|
12
|
+
}
|
|
13
|
+
export declare const CURRENT_EMBEDDING_MODEL = "multilingual-e5-small";
|
|
14
|
+
export declare const DEFAULT_LOCAL_EMBEDDING_MODEL = "multilingual-e5-small";
|
|
15
|
+
export declare const DEFAULT_OPENAI_EMBEDDING_MODEL = "text-embedding-3-small";
|
|
16
|
+
export declare const DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1";
|
|
17
|
+
export declare const DEFAULT_API_MAX_EMBED_CHARS = 20000;
|
|
18
|
+
export declare function resolveEmbeddingConfig(env?: NodeJS.ProcessEnv): EmbeddingConfig;
|
|
19
|
+
export declare function embeddingConfigLabel(config: EmbeddingConfig): string;
|
|
20
|
+
export declare function embeddingSignature(config: EmbeddingConfig, dimension: number): string;
|
|
1
21
|
export declare function dispose(): Promise<void>;
|
|
2
22
|
export declare function prepareForShutdown(): Promise<void>;
|
|
3
|
-
export declare function
|
|
23
|
+
export declare function embedTextsWithConfig(texts: string[], prefix: EmbeddingPrefix, signal?: AbortSignal, options?: {
|
|
24
|
+
allowApiFallback?: boolean;
|
|
25
|
+
}): Promise<{
|
|
26
|
+
vectors: Float32Array[];
|
|
27
|
+
config: EmbeddingConfig;
|
|
28
|
+
}>;
|
|
29
|
+
export declare function embedTexts(texts: string[], prefix: EmbeddingPrefix, signal?: AbortSignal, options?: {
|
|
30
|
+
allowApiFallback?: boolean;
|
|
31
|
+
}): Promise<Float32Array[]>;
|
|
32
|
+
export declare function embedQueryWithConfig(text: string, signal?: AbortSignal): Promise<{
|
|
33
|
+
vector: Float32Array;
|
|
34
|
+
config: EmbeddingConfig;
|
|
35
|
+
}>;
|
|
4
36
|
export declare function embedQuery(text: string, signal?: AbortSignal): Promise<Float32Array>;
|
|
5
37
|
export declare function embedDocuments(texts: string[], signal?: AbortSignal): Promise<Float32Array[]>;
|
|
@@ -1,15 +1,33 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
1
2
|
import { embedInModelWorker } from "../model-worker-client.js";
|
|
2
3
|
let disposeTimer = null;
|
|
3
4
|
let disposePromise = null;
|
|
4
5
|
let activeRuns = 0;
|
|
5
6
|
let disposeRequested = false;
|
|
6
7
|
const idleWaiters = [];
|
|
8
|
+
export const CURRENT_EMBEDDING_MODEL = "multilingual-e5-small";
|
|
9
|
+
export const DEFAULT_LOCAL_EMBEDDING_MODEL = "multilingual-e5-small";
|
|
10
|
+
export const DEFAULT_OPENAI_EMBEDDING_MODEL = "text-embedding-3-small";
|
|
11
|
+
export const DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1";
|
|
12
|
+
export const DEFAULT_API_MAX_EMBED_CHARS = 20_000;
|
|
7
13
|
const IDLE_TIMEOUT_MS = Number(process.env.PI_KNOWLEDGE_EMBEDDING_IDLE_MS ?? 30_000);
|
|
8
|
-
const EMBEDDING_CONFIG = process.env.PI_KNOWLEDGE_EMBEDDING ?? "local:multilingual-e5-small";
|
|
9
14
|
const ENABLE_NATIVE_IDLE_DISPOSE = process.env.PI_KNOWLEDGE_ENABLE_NATIVE_IDLE_DISPOSE === "true";
|
|
10
15
|
const API_FALLBACK_TO_LOCAL = process.env.PI_KNOWLEDGE_EMBEDDING_API_FALLBACK === "local";
|
|
11
|
-
|
|
12
|
-
|
|
16
|
+
function localEmbeddingConfig(maxChars) {
|
|
17
|
+
return {
|
|
18
|
+
provider: "local",
|
|
19
|
+
model: DEFAULT_LOCAL_EMBEDDING_MODEL,
|
|
20
|
+
maxChars,
|
|
21
|
+
queryPrefix: "query",
|
|
22
|
+
documentPrefix: "passage",
|
|
23
|
+
pooling: "mean",
|
|
24
|
+
normalize: true,
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
function cleanEnv(value) {
|
|
28
|
+
const trimmed = value?.trim();
|
|
29
|
+
return trimmed ? trimmed : undefined;
|
|
30
|
+
}
|
|
13
31
|
function isAbortError(error) {
|
|
14
32
|
return typeof error === "object" && error !== null && "name" in error && error.name === "AbortError";
|
|
15
33
|
}
|
|
@@ -44,6 +62,52 @@ function waitForNoActiveRuns() {
|
|
|
44
62
|
return Promise.resolve();
|
|
45
63
|
return new Promise((resolve) => idleWaiters.push(resolve));
|
|
46
64
|
}
|
|
65
|
+
function providerInstanceHash(value) {
|
|
66
|
+
return createHash("sha256").update(value).digest("hex").slice(0, 12);
|
|
67
|
+
}
|
|
68
|
+
export function resolveEmbeddingConfig(env = process.env) {
|
|
69
|
+
const raw = cleanEnv(env.PI_KNOWLEDGE_EMBEDDING) ?? `local:${DEFAULT_LOCAL_EMBEDDING_MODEL}`;
|
|
70
|
+
const separator = raw.indexOf(":");
|
|
71
|
+
const provider = separator >= 0 ? raw.slice(0, separator).toLowerCase() : raw.toLowerCase();
|
|
72
|
+
const model = separator >= 0 ? raw.slice(separator + 1).trim() : "";
|
|
73
|
+
const maxCharsValue = Number(cleanEnv(env.PI_KNOWLEDGE_EMBEDDING_MAX_CHARS) ?? DEFAULT_API_MAX_EMBED_CHARS);
|
|
74
|
+
const maxChars = Number.isFinite(maxCharsValue) && maxCharsValue > 0 ? Math.trunc(maxCharsValue) : DEFAULT_API_MAX_EMBED_CHARS;
|
|
75
|
+
if (provider === "local") {
|
|
76
|
+
const localModel = model || DEFAULT_LOCAL_EMBEDDING_MODEL;
|
|
77
|
+
if (localModel !== DEFAULT_LOCAL_EMBEDDING_MODEL) {
|
|
78
|
+
throw new Error(`Unsupported local embedding model: ${localModel}`);
|
|
79
|
+
}
|
|
80
|
+
return localEmbeddingConfig(maxChars);
|
|
81
|
+
}
|
|
82
|
+
if (provider === "openai") {
|
|
83
|
+
return {
|
|
84
|
+
provider: "openai",
|
|
85
|
+
model: model || DEFAULT_OPENAI_EMBEDDING_MODEL,
|
|
86
|
+
baseUrl: cleanEnv(env.PI_KNOWLEDGE_EMBEDDING_BASE_URL) ?? cleanEnv(env.OPENAI_BASE_URL) ?? DEFAULT_OPENAI_BASE_URL,
|
|
87
|
+
maxChars,
|
|
88
|
+
queryPrefix: "query",
|
|
89
|
+
documentPrefix: "passage",
|
|
90
|
+
pooling: "mean",
|
|
91
|
+
normalize: true,
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
throw new Error(`Unsupported embedding provider: ${provider}`);
|
|
95
|
+
}
|
|
96
|
+
export function embeddingConfigLabel(config) {
|
|
97
|
+
return config.provider === "local" ? config.model : `${config.provider}:${config.model}`;
|
|
98
|
+
}
|
|
99
|
+
export function embeddingSignature(config, dimension) {
|
|
100
|
+
const base = config.baseUrl ? `:base-sha256=${providerInstanceHash(config.baseUrl)}` : "";
|
|
101
|
+
return [
|
|
102
|
+
`${config.provider}:${config.model}${base}`,
|
|
103
|
+
`dim=${dimension}`,
|
|
104
|
+
`apiMax=${config.provider === "openai" ? config.maxChars : "none"}`,
|
|
105
|
+
`pooling=${config.pooling}`,
|
|
106
|
+
`normalize=${config.normalize}`,
|
|
107
|
+
`q=${config.queryPrefix}`,
|
|
108
|
+
`d=${config.documentPrefix}`,
|
|
109
|
+
].join(":");
|
|
110
|
+
}
|
|
47
111
|
export async function dispose() {
|
|
48
112
|
clearIdleTimer();
|
|
49
113
|
if (disposePromise)
|
|
@@ -60,62 +124,64 @@ export async function prepareForShutdown() {
|
|
|
60
124
|
clearIdleTimer();
|
|
61
125
|
await waitForNoActiveRuns();
|
|
62
126
|
}
|
|
63
|
-
async function embedViaAPI(texts, prefix, signal) {
|
|
64
|
-
const [provider, model] = EMBEDDING_CONFIG.split(":");
|
|
65
|
-
const configuredMaxChars = Number(process.env.PI_KNOWLEDGE_EMBEDDING_MAX_CHARS ?? DEFAULT_API_MAX_EMBED_CHARS);
|
|
66
|
-
const maxChars = Number.isFinite(configuredMaxChars) && configuredMaxChars > 0 ? configuredMaxChars : DEFAULT_API_MAX_EMBED_CHARS;
|
|
127
|
+
async function embedViaAPI(texts, prefix, config, signal) {
|
|
67
128
|
const prefixedTexts = texts.map((t) => `${prefix}: ${t}`);
|
|
68
|
-
const safeTexts = prefixedTexts.map((text) =>
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
const detail = await res.text().catch(() => "");
|
|
85
|
-
throw new Error(`OpenAI embedding API error: ${res.status}${detail ? ` ${detail.slice(0, 500)}` : ""}`);
|
|
86
|
-
}
|
|
87
|
-
const data = (await res.json());
|
|
88
|
-
return data.data.map((d) => new Float32Array(d.embedding));
|
|
129
|
+
const safeTexts = prefixedTexts.map((text) => text.length > config.maxChars ? text.slice(0, config.maxChars) : text);
|
|
130
|
+
const apiKey = process.env.OPENAI_API_KEY;
|
|
131
|
+
if (!apiKey)
|
|
132
|
+
throw new Error("OPENAI_API_KEY required for openai embedding");
|
|
133
|
+
if (!config.baseUrl)
|
|
134
|
+
throw new Error("OpenAI embedding base URL is not configured");
|
|
135
|
+
const endpoint = new URL("embeddings", `${config.baseUrl.replace(/\/+$/, "")}/`);
|
|
136
|
+
const res = await fetch(endpoint, {
|
|
137
|
+
method: "POST",
|
|
138
|
+
headers: { "Content-Type": "application/json", Authorization: `Bearer ${apiKey}` },
|
|
139
|
+
body: JSON.stringify({ input: safeTexts, model: config.model }),
|
|
140
|
+
signal,
|
|
141
|
+
});
|
|
142
|
+
if (!res.ok) {
|
|
143
|
+
const detail = await res.text().catch(() => "");
|
|
144
|
+
throw new Error(`OpenAI embedding API error: ${res.status}${detail ? ` ${detail.slice(0, 500)}` : ""}`);
|
|
89
145
|
}
|
|
90
|
-
|
|
146
|
+
const data = (await res.json());
|
|
147
|
+
return data.data.map((d) => new Float32Array(d.embedding));
|
|
91
148
|
}
|
|
92
|
-
export async function
|
|
93
|
-
|
|
149
|
+
export async function embedTextsWithConfig(texts, prefix, signal, options = {}) {
|
|
150
|
+
const config = resolveEmbeddingConfig();
|
|
151
|
+
let actualConfig = config;
|
|
152
|
+
if (config.provider === "openai") {
|
|
94
153
|
if (signal?.aborted)
|
|
95
154
|
throw new Error("Cancelled");
|
|
96
155
|
try {
|
|
97
|
-
return await embedViaAPI(texts, prefix, signal);
|
|
156
|
+
return { vectors: await embedViaAPI(texts, prefix, config, signal), config };
|
|
98
157
|
}
|
|
99
158
|
catch (error) {
|
|
100
159
|
if (signal?.aborted || isAbortError(error))
|
|
101
160
|
throw new Error("Cancelled");
|
|
102
|
-
if (!API_FALLBACK_TO_LOCAL)
|
|
161
|
+
if (!API_FALLBACK_TO_LOCAL || !options.allowApiFallback)
|
|
103
162
|
throw error;
|
|
104
163
|
console.warn(`pi-knowledge: embedding API failed; falling back to local model because PI_KNOWLEDGE_EMBEDDING_API_FALLBACK=local (${error instanceof Error ? error.message : String(error)})`);
|
|
164
|
+
actualConfig = localEmbeddingConfig(config.maxChars);
|
|
105
165
|
}
|
|
106
166
|
}
|
|
107
167
|
beginRun();
|
|
108
168
|
try {
|
|
109
|
-
return await embedInModelWorker(texts, prefix, signal);
|
|
169
|
+
return { vectors: await embedInModelWorker(texts, prefix, signal), config: actualConfig };
|
|
110
170
|
}
|
|
111
171
|
finally {
|
|
112
172
|
endRun();
|
|
113
173
|
}
|
|
114
174
|
}
|
|
175
|
+
export async function embedTexts(texts, prefix, signal, options = {}) {
|
|
176
|
+
return (await embedTextsWithConfig(texts, prefix, signal, options)).vectors;
|
|
177
|
+
}
|
|
178
|
+
export async function embedQueryWithConfig(text, signal) {
|
|
179
|
+
const { vectors, config } = await embedTextsWithConfig([text], "query", signal, { allowApiFallback: true });
|
|
180
|
+
return { vector: vectors[0], config };
|
|
181
|
+
}
|
|
115
182
|
export async function embedQuery(text, signal) {
|
|
116
|
-
|
|
117
|
-
return vec;
|
|
183
|
+
return (await embedQueryWithConfig(text, signal)).vector;
|
|
118
184
|
}
|
|
119
185
|
export async function embedDocuments(texts, signal) {
|
|
120
|
-
return embedTexts(texts, "passage", signal);
|
|
186
|
+
return embedTexts(texts, "passage", signal, { allowApiFallback: false });
|
|
121
187
|
}
|
package/dist/src/engine.d.ts
CHANGED
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
import { type DiagnosticResult } from "./diagnostics/health.ts";
|
|
2
2
|
import { createSkippedScanStats } from "./indexer/chunker.ts";
|
|
3
3
|
import { type RankingDiagnostics } from "./search/ranking.ts";
|
|
4
|
+
import { type SearchProfile, type SearchTuningSummary } from "./search/tuning.ts";
|
|
4
5
|
import { type KnowledgeBase, type KnowledgeSymbol } from "./storage/sqlite.ts";
|
|
5
6
|
export type SearchMode = "auto" | "fast" | "semantic" | "hybrid" | "deep" | "adaptive" | "code" | "config" | "docs" | "errors" | "decision";
|
|
6
7
|
export interface SearchOptions {
|
|
7
8
|
mode?: SearchMode;
|
|
9
|
+
profile?: SearchProfile;
|
|
8
10
|
limit?: number;
|
|
9
11
|
offset?: number;
|
|
10
12
|
kb_id?: string;
|
|
@@ -34,7 +36,7 @@ export interface SearchResult {
|
|
|
34
36
|
source_chunk_ids?: string[];
|
|
35
37
|
};
|
|
36
38
|
}
|
|
37
|
-
export
|
|
39
|
+
export { CURRENT_EMBEDDING_MODEL } from "./embedding/provider.ts";
|
|
38
40
|
export interface SearchResponse {
|
|
39
41
|
results: SearchResult[];
|
|
40
42
|
total_count: number;
|
|
@@ -43,6 +45,7 @@ export interface SearchResponse {
|
|
|
43
45
|
mode_used?: SearchMode;
|
|
44
46
|
retry_modes?: SearchMode[];
|
|
45
47
|
suggestions?: string[];
|
|
48
|
+
tuning?: SearchTuningSummary;
|
|
46
49
|
}
|
|
47
50
|
export type ProgressCallback = (msg: string) => void;
|
|
48
51
|
export interface AddOptions {
|