pi-knowledge 0.5.5 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/README.md +9 -4
- package/dist/index.js +24 -4
- package/dist/src/embedding/provider.d.ts +33 -1
- package/dist/src/embedding/provider.js +103 -37
- package/dist/src/engine.d.ts +4 -1
- package/dist/src/engine.js +98 -34
- package/dist/src/model-worker-client.d.ts +2 -1
- package/dist/src/model-worker-client.js +2 -2
- package/dist/src/model-worker.js +25 -8
- package/dist/src/search/reranker-api.d.ts +7 -0
- package/dist/src/search/reranker-api.js +89 -0
- package/dist/src/search/reranker-config.d.ts +34 -0
- package/dist/src/search/reranker-config.js +151 -0
- package/dist/src/search/reranker.js +7 -1
- package/dist/src/search/tuning.d.ts +41 -0
- package/dist/src/search/tuning.js +190 -0
- package/dist/src/storage/sqlite.d.ts +6 -0
- package/dist/src/storage/sqlite.js +23 -3
- package/docs/configuration.md +41 -6
- package/package.json +1 -1
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
export const DEFAULT_RERANKER_MODEL = "Xenova/ms-marco-MiniLM-L-4-v2";
|
|
2
|
+
export const DEFAULT_RERANKER_REVISION = "main";
|
|
3
|
+
export const DEFAULT_RERANKER_REMOTE_HOST = "https://huggingface.co/";
|
|
4
|
+
export const DEFAULT_RERANKER_REMOTE_PATH_TEMPLATE = "{model}/resolve/{revision}/";
|
|
5
|
+
export const DEFAULT_RERANKER_API_TIMEOUT_MS = 30_000;
|
|
6
|
+
export const DEFAULT_RERANKER_MAX_DOC_CHARS = 12_000;
|
|
7
|
+
function cleanEnv(value) {
|
|
8
|
+
const trimmed = value?.trim();
|
|
9
|
+
return trimmed ? trimmed : undefined;
|
|
10
|
+
}
|
|
11
|
+
function stripProviderPrefix(raw) {
|
|
12
|
+
const separator = raw.indexOf(":");
|
|
13
|
+
if (separator <= 0)
|
|
14
|
+
return { provider: undefined, value: raw };
|
|
15
|
+
const prefix = raw.slice(0, separator).toLowerCase();
|
|
16
|
+
if (prefix !== "hf" && prefix !== "api")
|
|
17
|
+
return { provider: undefined, value: raw };
|
|
18
|
+
return { provider: prefix, value: raw.slice(separator + 1) };
|
|
19
|
+
}
|
|
20
|
+
function parseHuggingFaceUrl(raw) {
|
|
21
|
+
let url;
|
|
22
|
+
try {
|
|
23
|
+
url = new URL(raw);
|
|
24
|
+
}
|
|
25
|
+
catch {
|
|
26
|
+
return undefined;
|
|
27
|
+
}
|
|
28
|
+
if (url.protocol !== "https:" && url.protocol !== "http:") {
|
|
29
|
+
throw new Error(`Unsupported reranker model URL protocol: ${url.protocol}`);
|
|
30
|
+
}
|
|
31
|
+
const parts = url.pathname.split("/").filter(Boolean);
|
|
32
|
+
if (parts.length < 2)
|
|
33
|
+
throw new Error(`Invalid Hugging Face reranker URL: ${raw}`);
|
|
34
|
+
const model = `${decodeURIComponent(parts[0])}/${decodeURIComponent(parts[1])}`;
|
|
35
|
+
let revision;
|
|
36
|
+
const treeIndex = parts.indexOf("tree");
|
|
37
|
+
const resolveIndex = parts.indexOf("resolve");
|
|
38
|
+
if (treeIndex >= 0 && parts[treeIndex + 1])
|
|
39
|
+
revision = decodeURIComponent(parts[treeIndex + 1]);
|
|
40
|
+
if (resolveIndex >= 0 && parts[resolveIndex + 1])
|
|
41
|
+
revision = decodeURIComponent(parts[resolveIndex + 1]);
|
|
42
|
+
return { model, revision, remoteHost: `${url.origin}/` };
|
|
43
|
+
}
|
|
44
|
+
function resolveHfModel(raw) {
|
|
45
|
+
const parsedUrl = parseHuggingFaceUrl(raw);
|
|
46
|
+
if (parsedUrl)
|
|
47
|
+
return parsedUrl;
|
|
48
|
+
if (!raw.includes("/") || raw.startsWith("/") || raw.endsWith("/")) {
|
|
49
|
+
throw new Error(`Invalid reranker model id: ${raw}`);
|
|
50
|
+
}
|
|
51
|
+
return { model: raw };
|
|
52
|
+
}
|
|
53
|
+
function parseNumber(value, fallback, min, max) {
|
|
54
|
+
const parsed = Number(value);
|
|
55
|
+
if (!Number.isFinite(parsed))
|
|
56
|
+
return fallback;
|
|
57
|
+
return Math.min(Math.max(Math.trunc(parsed), min), max);
|
|
58
|
+
}
|
|
59
|
+
function normalizeApiFormat(value) {
|
|
60
|
+
if (value === undefined)
|
|
61
|
+
return "cohere";
|
|
62
|
+
const normalized = value.toLowerCase();
|
|
63
|
+
if (normalized === "cohere" || normalized === "jina" || normalized === "custom-json")
|
|
64
|
+
return normalized;
|
|
65
|
+
throw new Error(`Unsupported reranker API format: ${value}`);
|
|
66
|
+
}
|
|
67
|
+
function normalizeScoreDirection(value) {
|
|
68
|
+
if (value === undefined)
|
|
69
|
+
return "desc";
|
|
70
|
+
const normalized = value.toLowerCase();
|
|
71
|
+
if (normalized === "desc" || normalized === "asc")
|
|
72
|
+
return normalized;
|
|
73
|
+
throw new Error(`Unsupported reranker score direction: ${value}`);
|
|
74
|
+
}
|
|
75
|
+
function endpointFromBaseUrl(baseUrl) {
|
|
76
|
+
return new URL("rerank", `${baseUrl.replace(/\/+$/, "")}/`).toString();
|
|
77
|
+
}
|
|
78
|
+
function normalizeHttpUrl(raw) {
|
|
79
|
+
const url = new URL(raw);
|
|
80
|
+
if (url.protocol !== "http:" && url.protocol !== "https:") {
|
|
81
|
+
throw new Error(`Unsupported reranker API endpoint protocol: ${url.protocol}`);
|
|
82
|
+
}
|
|
83
|
+
return url.toString();
|
|
84
|
+
}
|
|
85
|
+
function resolveApiRerankerConfig(model, env) {
|
|
86
|
+
const endpoint = normalizeHttpUrl(cleanEnv(env.PI_KNOWLEDGE_RERANKER_API_ENDPOINT) ??
|
|
87
|
+
endpointFromBaseUrl(cleanEnv(env.PI_KNOWLEDGE_RERANKER_API_BASE_URL) ??
|
|
88
|
+
cleanEnv(env.OPENAI_BASE_URL) ??
|
|
89
|
+
"https://api.cohere.com/v2"));
|
|
90
|
+
return {
|
|
91
|
+
provider: "api",
|
|
92
|
+
model,
|
|
93
|
+
endpoint,
|
|
94
|
+
apiKey: cleanEnv(env.PI_KNOWLEDGE_RERANKER_API_KEY) ?? cleanEnv(env.OPENAI_API_KEY),
|
|
95
|
+
format: normalizeApiFormat(cleanEnv(env.PI_KNOWLEDGE_RERANKER_API_FORMAT)),
|
|
96
|
+
timeoutMs: parseNumber(cleanEnv(env.PI_KNOWLEDGE_RERANKER_API_TIMEOUT_MS), DEFAULT_RERANKER_API_TIMEOUT_MS, 1_000, 300_000),
|
|
97
|
+
maxDocumentChars: parseNumber(cleanEnv(env.PI_KNOWLEDGE_RERANKER_MAX_DOC_CHARS), DEFAULT_RERANKER_MAX_DOC_CHARS, 100, 200_000),
|
|
98
|
+
resultsPath: cleanEnv(env.PI_KNOWLEDGE_RERANKER_API_RESULTS_PATH) ?? "results",
|
|
99
|
+
indexField: cleanEnv(env.PI_KNOWLEDGE_RERANKER_API_INDEX_FIELD) ?? "index",
|
|
100
|
+
scoreField: cleanEnv(env.PI_KNOWLEDGE_RERANKER_API_SCORE_FIELD) ?? "relevance_score",
|
|
101
|
+
scoreDirection: normalizeScoreDirection(cleanEnv(env.PI_KNOWLEDGE_RERANKER_API_SCORE_DIRECTION)),
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
export function resolveRerankerConfig(env = process.env) {
|
|
105
|
+
const raw = cleanEnv(env.PI_KNOWLEDGE_RERANKER) ?? `hf:${DEFAULT_RERANKER_MODEL}`;
|
|
106
|
+
const { provider, value } = stripProviderPrefix(raw);
|
|
107
|
+
if (provider === "api") {
|
|
108
|
+
const model = value.trim();
|
|
109
|
+
if (!model)
|
|
110
|
+
throw new Error("PI_KNOWLEDGE_RERANKER=api:<model> requires a model name");
|
|
111
|
+
return resolveApiRerankerConfig(model, env);
|
|
112
|
+
}
|
|
113
|
+
if (provider !== undefined && provider !== "hf")
|
|
114
|
+
throw new Error(`Unsupported reranker provider: ${provider}`);
|
|
115
|
+
const modelSource = value.trim();
|
|
116
|
+
if (!modelSource)
|
|
117
|
+
throw new Error("PI_KNOWLEDGE_RERANKER requires a model id or URL");
|
|
118
|
+
const resolved = resolveHfModel(modelSource);
|
|
119
|
+
const revision = cleanEnv(env.PI_KNOWLEDGE_RERANKER_REVISION) ?? resolved.revision ?? DEFAULT_RERANKER_REVISION;
|
|
120
|
+
return {
|
|
121
|
+
provider: "hf",
|
|
122
|
+
model: resolved.model,
|
|
123
|
+
revision,
|
|
124
|
+
dtype: cleanEnv(env.PI_KNOWLEDGE_RERANKER_DTYPE),
|
|
125
|
+
remoteHost: cleanEnv(env.PI_KNOWLEDGE_RERANKER_REMOTE_HOST) ?? resolved.remoteHost,
|
|
126
|
+
remotePathTemplate: cleanEnv(env.PI_KNOWLEDGE_RERANKER_REMOTE_PATH_TEMPLATE),
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
export function isHfRerankerConfig(value) {
|
|
130
|
+
if (typeof value !== "object" || value === null)
|
|
131
|
+
return false;
|
|
132
|
+
const config = value;
|
|
133
|
+
return (config.provider === "hf" &&
|
|
134
|
+
typeof config.model === "string" &&
|
|
135
|
+
config.model.length > 0 &&
|
|
136
|
+
typeof config.revision === "string" &&
|
|
137
|
+
config.revision.length > 0 &&
|
|
138
|
+
(config.dtype === undefined || typeof config.dtype === "string") &&
|
|
139
|
+
(config.remoteHost === undefined || typeof config.remoteHost === "string") &&
|
|
140
|
+
(config.remotePathTemplate === undefined || typeof config.remotePathTemplate === "string"));
|
|
141
|
+
}
|
|
142
|
+
export function rerankerCacheKey(config) {
|
|
143
|
+
return [
|
|
144
|
+
config.provider,
|
|
145
|
+
config.model,
|
|
146
|
+
config.revision,
|
|
147
|
+
config.dtype ?? "",
|
|
148
|
+
config.remoteHost ?? "",
|
|
149
|
+
config.remotePathTemplate ?? "",
|
|
150
|
+
].join("\u0000");
|
|
151
|
+
}
|
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { rerankInModelWorker } from "../model-worker-client.js";
|
|
2
|
+
import { rerankViaApi } from "./reranker-api.js";
|
|
3
|
+
import { resolveRerankerConfig } from "./reranker-config.js";
|
|
2
4
|
let disposeTimer = null;
|
|
3
5
|
let disposePromise = null;
|
|
4
6
|
let activeRuns = 0;
|
|
@@ -56,9 +58,13 @@ export async function prepareRerankerForShutdown() {
|
|
|
56
58
|
export async function rerank(query, candidates, topK, signal) {
|
|
57
59
|
if (candidates.length === 0)
|
|
58
60
|
return [];
|
|
61
|
+
const rerankerConfig = resolveRerankerConfig();
|
|
59
62
|
beginRun();
|
|
60
63
|
try {
|
|
61
|
-
|
|
64
|
+
if (rerankerConfig.provider === "api") {
|
|
65
|
+
return await rerankViaApi(query, candidates, topK, rerankerConfig, signal);
|
|
66
|
+
}
|
|
67
|
+
return await rerankInModelWorker(query, candidates, topK, rerankerConfig, signal);
|
|
62
68
|
}
|
|
63
69
|
finally {
|
|
64
70
|
endRun();
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
export type SearchProfile = "auto" | "balanced" | "low_token" | "precision" | "recall" | "long_context" | "code" | "docs";
|
|
2
|
+
export type RuntimeProfile = Exclude<SearchProfile, "auto">;
|
|
3
|
+
type SearchModeForTuning = "auto" | "fast" | "semantic" | "hybrid" | "deep" | "adaptive" | "code" | "config" | "docs" | "errors" | "decision";
|
|
4
|
+
export interface SearchTuning {
|
|
5
|
+
requestedProfile: SearchProfile;
|
|
6
|
+
selectedProfile: RuntimeProfile;
|
|
7
|
+
defaultLimit: number;
|
|
8
|
+
snippetMaxLength: number;
|
|
9
|
+
minHybridScore: number;
|
|
10
|
+
candidateMin: number;
|
|
11
|
+
candidateMultiplier: number;
|
|
12
|
+
adaptiveContextLines: number;
|
|
13
|
+
adaptiveMaxContextChars: number;
|
|
14
|
+
adaptiveNeighborTarget: number;
|
|
15
|
+
deepRerankCandidates: number;
|
|
16
|
+
deepRerankTopKMultiplier: number;
|
|
17
|
+
}
|
|
18
|
+
export interface SearchTuningSummary {
|
|
19
|
+
requested_profile: SearchProfile;
|
|
20
|
+
selected_profile: RuntimeProfile;
|
|
21
|
+
limit: number;
|
|
22
|
+
snippet_max_length: number;
|
|
23
|
+
min_hybrid_score: number;
|
|
24
|
+
candidate_limit: number;
|
|
25
|
+
candidate_min: number;
|
|
26
|
+
candidate_multiplier: number;
|
|
27
|
+
adaptive_context_lines: number;
|
|
28
|
+
adaptive_max_context_chars: number;
|
|
29
|
+
adaptive_neighbor_target: number;
|
|
30
|
+
deep_rerank_candidates: number;
|
|
31
|
+
deep_rerank_top_k_multiplier: number;
|
|
32
|
+
}
|
|
33
|
+
interface SearchTuningInput {
|
|
34
|
+
query: string;
|
|
35
|
+
mode: SearchModeForTuning;
|
|
36
|
+
profile?: SearchProfile;
|
|
37
|
+
kbSourceTypes: string[];
|
|
38
|
+
}
|
|
39
|
+
export declare function resolveSearchTuning(input: SearchTuningInput): SearchTuning;
|
|
40
|
+
export declare function summarizeSearchTuning(tuning: SearchTuning, limit: number, candidateLimit: number): SearchTuningSummary;
|
|
41
|
+
export {};
|
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
import { MIN_HYBRID_SCORE } from "./ranking.js";
|
|
2
|
+
const PROFILE_DEFAULTS = {
|
|
3
|
+
balanced: {
|
|
4
|
+
defaultLimit: 10,
|
|
5
|
+
snippetMaxLength: 240,
|
|
6
|
+
minHybridScore: MIN_HYBRID_SCORE,
|
|
7
|
+
candidateMin: 50,
|
|
8
|
+
candidateMultiplier: 12,
|
|
9
|
+
adaptiveContextLines: 80,
|
|
10
|
+
adaptiveMaxContextChars: 6_000,
|
|
11
|
+
adaptiveNeighborTarget: 5,
|
|
12
|
+
deepRerankCandidates: 30,
|
|
13
|
+
deepRerankTopKMultiplier: 3,
|
|
14
|
+
},
|
|
15
|
+
low_token: {
|
|
16
|
+
defaultLimit: 5,
|
|
17
|
+
snippetMaxLength: 768,
|
|
18
|
+
minHybridScore: 0.4,
|
|
19
|
+
candidateMin: 35,
|
|
20
|
+
candidateMultiplier: 8,
|
|
21
|
+
adaptiveContextLines: 80,
|
|
22
|
+
adaptiveMaxContextChars: 6_000,
|
|
23
|
+
adaptiveNeighborTarget: 5,
|
|
24
|
+
deepRerankCandidates: 20,
|
|
25
|
+
deepRerankTopKMultiplier: 2,
|
|
26
|
+
},
|
|
27
|
+
precision: {
|
|
28
|
+
defaultLimit: 5,
|
|
29
|
+
snippetMaxLength: 400,
|
|
30
|
+
minHybridScore: 0.4,
|
|
31
|
+
candidateMin: 30,
|
|
32
|
+
candidateMultiplier: 6,
|
|
33
|
+
adaptiveContextLines: 60,
|
|
34
|
+
adaptiveMaxContextChars: 4_000,
|
|
35
|
+
adaptiveNeighborTarget: 4,
|
|
36
|
+
deepRerankCandidates: 20,
|
|
37
|
+
deepRerankTopKMultiplier: 2,
|
|
38
|
+
},
|
|
39
|
+
recall: {
|
|
40
|
+
defaultLimit: 15,
|
|
41
|
+
snippetMaxLength: 320,
|
|
42
|
+
minHybridScore: 0.12,
|
|
43
|
+
candidateMin: 75,
|
|
44
|
+
candidateMultiplier: 16,
|
|
45
|
+
adaptiveContextLines: 100,
|
|
46
|
+
adaptiveMaxContextChars: 8_000,
|
|
47
|
+
adaptiveNeighborTarget: 6,
|
|
48
|
+
deepRerankCandidates: 40,
|
|
49
|
+
deepRerankTopKMultiplier: 3,
|
|
50
|
+
},
|
|
51
|
+
long_context: {
|
|
52
|
+
defaultLimit: 5,
|
|
53
|
+
snippetMaxLength: 1_000,
|
|
54
|
+
minHybridScore: 0.25,
|
|
55
|
+
candidateMin: 50,
|
|
56
|
+
candidateMultiplier: 10,
|
|
57
|
+
adaptiveContextLines: 120,
|
|
58
|
+
adaptiveMaxContextChars: 10_000,
|
|
59
|
+
adaptiveNeighborTarget: 7,
|
|
60
|
+
deepRerankCandidates: 30,
|
|
61
|
+
deepRerankTopKMultiplier: 2,
|
|
62
|
+
},
|
|
63
|
+
code: {
|
|
64
|
+
defaultLimit: 10,
|
|
65
|
+
snippetMaxLength: 320,
|
|
66
|
+
minHybridScore: 0.25,
|
|
67
|
+
candidateMin: 50,
|
|
68
|
+
candidateMultiplier: 12,
|
|
69
|
+
adaptiveContextLines: 80,
|
|
70
|
+
adaptiveMaxContextChars: 6_000,
|
|
71
|
+
adaptiveNeighborTarget: 5,
|
|
72
|
+
deepRerankCandidates: 30,
|
|
73
|
+
deepRerankTopKMultiplier: 3,
|
|
74
|
+
},
|
|
75
|
+
docs: {
|
|
76
|
+
defaultLimit: 8,
|
|
77
|
+
snippetMaxLength: 768,
|
|
78
|
+
minHybridScore: 0.22,
|
|
79
|
+
candidateMin: 60,
|
|
80
|
+
candidateMultiplier: 12,
|
|
81
|
+
adaptiveContextLines: 120,
|
|
82
|
+
adaptiveMaxContextChars: 9_000,
|
|
83
|
+
adaptiveNeighborTarget: 7,
|
|
84
|
+
deepRerankCandidates: 30,
|
|
85
|
+
deepRerankTopKMultiplier: 3,
|
|
86
|
+
},
|
|
87
|
+
};
|
|
88
|
+
const SEARCH_PROFILES = new Set([
|
|
89
|
+
"auto",
|
|
90
|
+
"balanced",
|
|
91
|
+
"low_token",
|
|
92
|
+
"precision",
|
|
93
|
+
"recall",
|
|
94
|
+
"long_context",
|
|
95
|
+
"code",
|
|
96
|
+
"docs",
|
|
97
|
+
]);
|
|
98
|
+
function parseProfile(value) {
|
|
99
|
+
const normalized = value?.trim().toLowerCase();
|
|
100
|
+
if (!normalized)
|
|
101
|
+
return undefined;
|
|
102
|
+
return SEARCH_PROFILES.has(normalized) ? normalized : undefined;
|
|
103
|
+
}
|
|
104
|
+
function envNumber(name) {
|
|
105
|
+
const raw = process.env[name]?.trim();
|
|
106
|
+
if (!raw)
|
|
107
|
+
return undefined;
|
|
108
|
+
const value = Number(raw);
|
|
109
|
+
return Number.isFinite(value) ? value : undefined;
|
|
110
|
+
}
|
|
111
|
+
function clampInt(value, fallback, min, max) {
|
|
112
|
+
if (value === undefined)
|
|
113
|
+
return fallback;
|
|
114
|
+
return Math.min(max, Math.max(min, Math.trunc(value)));
|
|
115
|
+
}
|
|
116
|
+
function clampNumber(value, fallback, min, max) {
|
|
117
|
+
if (value === undefined)
|
|
118
|
+
return fallback;
|
|
119
|
+
return Math.min(max, Math.max(min, value));
|
|
120
|
+
}
|
|
121
|
+
function looksExactOrCodeQuery(query) {
|
|
122
|
+
const trimmed = query.trim();
|
|
123
|
+
if (trimmed.length === 0)
|
|
124
|
+
return false;
|
|
125
|
+
const lower = trimmed.toLowerCase();
|
|
126
|
+
if (/^[A-Z][A-Z0-9_]{2,}$/.test(trimmed))
|
|
127
|
+
return true;
|
|
128
|
+
if (/^[\w./-]+\.(ts|tsx|js|jsx|json|md|py|go|rs|java|yml|yaml|toml)$/.test(lower))
|
|
129
|
+
return true;
|
|
130
|
+
return /[A-Za-z_$][\w$]*\([^)]*\)|[A-Za-z_$][\w$]*\.[A-Za-z_$][\w$]*|\b[A-Z][A-Za-z0-9_$]*[A-Z][A-Za-z0-9_$]*\b|\b[a-z]+[A-Z][A-Za-z0-9_$]*\b/.test(trimmed);
|
|
131
|
+
}
|
|
132
|
+
function looksErrorQuery(query) {
|
|
133
|
+
return /\b(error|exception|failed|failure|stack|trace|enoent|eacces|timeout|abort|cancelled|錯誤|失敗)\b/i.test(query);
|
|
134
|
+
}
|
|
135
|
+
function looksLongContextQuery(query) {
|
|
136
|
+
const wordCount = query.trim().split(/\s+/).filter(Boolean).length;
|
|
137
|
+
return (wordCount >= 10 ||
|
|
138
|
+
/\b(explain|why|how|architecture|design|overview|docs?|spec|manual|guide)\b|流程|架構|概念|為什麼|如何|文件|規格/i.test(query));
|
|
139
|
+
}
|
|
140
|
+
function chooseRuntimeProfile(input) {
|
|
141
|
+
if (input.mode === "code" || input.mode === "config" || input.mode === "errors" || input.mode === "fast") {
|
|
142
|
+
return "precision";
|
|
143
|
+
}
|
|
144
|
+
if (input.mode === "docs" || input.mode === "decision" || input.mode === "adaptive")
|
|
145
|
+
return "long_context";
|
|
146
|
+
if (input.mode === "semantic")
|
|
147
|
+
return "recall";
|
|
148
|
+
if (looksErrorQuery(input.query) || looksExactOrCodeQuery(input.query))
|
|
149
|
+
return "precision";
|
|
150
|
+
const hasDocumentSource = input.kbSourceTypes.some((sourceType) => sourceType === "url" || sourceType === "text");
|
|
151
|
+
if (looksLongContextQuery(input.query) || hasDocumentSource)
|
|
152
|
+
return "long_context";
|
|
153
|
+
return "balanced";
|
|
154
|
+
}
|
|
155
|
+
export function resolveSearchTuning(input) {
|
|
156
|
+
const requestedProfile = input.profile ?? parseProfile(process.env.PI_KNOWLEDGE_SEARCH_PROFILE) ?? "auto";
|
|
157
|
+
const selectedProfile = requestedProfile === "auto" ? chooseRuntimeProfile(input) : requestedProfile;
|
|
158
|
+
const base = PROFILE_DEFAULTS[selectedProfile];
|
|
159
|
+
return {
|
|
160
|
+
requestedProfile,
|
|
161
|
+
selectedProfile,
|
|
162
|
+
defaultLimit: clampInt(envNumber("PI_KNOWLEDGE_SEARCH_DEFAULT_LIMIT"), base.defaultLimit, 1, 50),
|
|
163
|
+
snippetMaxLength: clampInt(envNumber("PI_KNOWLEDGE_SNIPPET_MAX_LENGTH"), base.snippetMaxLength, 80, 4_000),
|
|
164
|
+
minHybridScore: clampNumber(envNumber("PI_KNOWLEDGE_MIN_HYBRID_SCORE"), base.minHybridScore, 0, 1),
|
|
165
|
+
candidateMin: clampInt(envNumber("PI_KNOWLEDGE_SEARCH_CANDIDATE_MIN"), base.candidateMin, 10, 500),
|
|
166
|
+
candidateMultiplier: clampInt(envNumber("PI_KNOWLEDGE_SEARCH_CANDIDATE_MULTIPLIER"), base.candidateMultiplier, 2, 30),
|
|
167
|
+
adaptiveContextLines: clampInt(envNumber("PI_KNOWLEDGE_ADAPTIVE_CONTEXT_LINES"), base.adaptiveContextLines, 10, 500),
|
|
168
|
+
adaptiveMaxContextChars: clampInt(envNumber("PI_KNOWLEDGE_ADAPTIVE_MAX_CHARS"), base.adaptiveMaxContextChars, 1_000, 50_000),
|
|
169
|
+
adaptiveNeighborTarget: clampInt(envNumber("PI_KNOWLEDGE_ADAPTIVE_NEIGHBOR_TARGET"), base.adaptiveNeighborTarget, 1, 20),
|
|
170
|
+
deepRerankCandidates: clampInt(envNumber("PI_KNOWLEDGE_DEEP_RERANK_CANDIDATES"), base.deepRerankCandidates, 5, 100),
|
|
171
|
+
deepRerankTopKMultiplier: clampInt(envNumber("PI_KNOWLEDGE_DEEP_RERANK_TOPK_MULTIPLIER"), base.deepRerankTopKMultiplier, 1, 10),
|
|
172
|
+
};
|
|
173
|
+
}
|
|
174
|
+
export function summarizeSearchTuning(tuning, limit, candidateLimit) {
|
|
175
|
+
return {
|
|
176
|
+
requested_profile: tuning.requestedProfile,
|
|
177
|
+
selected_profile: tuning.selectedProfile,
|
|
178
|
+
limit,
|
|
179
|
+
snippet_max_length: tuning.snippetMaxLength,
|
|
180
|
+
min_hybrid_score: tuning.minHybridScore,
|
|
181
|
+
candidate_limit: candidateLimit,
|
|
182
|
+
candidate_min: tuning.candidateMin,
|
|
183
|
+
candidate_multiplier: tuning.candidateMultiplier,
|
|
184
|
+
adaptive_context_lines: tuning.adaptiveContextLines,
|
|
185
|
+
adaptive_max_context_chars: tuning.adaptiveMaxContextChars,
|
|
186
|
+
adaptive_neighbor_target: tuning.adaptiveNeighborTarget,
|
|
187
|
+
deep_rerank_candidates: tuning.deepRerankCandidates,
|
|
188
|
+
deep_rerank_top_k_multiplier: tuning.deepRerankTopKMultiplier,
|
|
189
|
+
};
|
|
190
|
+
}
|
|
@@ -11,6 +11,8 @@ export interface KnowledgeBase {
|
|
|
11
11
|
chunk_count: number;
|
|
12
12
|
file_count: number;
|
|
13
13
|
embedding_model: string;
|
|
14
|
+
embedding_signature: string | null;
|
|
15
|
+
embedding_dimension: number | null;
|
|
14
16
|
status: "ready" | "indexing" | "error" | "stale";
|
|
15
17
|
}
|
|
16
18
|
export interface IndexingJob {
|
|
@@ -75,6 +77,9 @@ export declare function createKB(db: Database.Database, opts: {
|
|
|
75
77
|
source_path?: string;
|
|
76
78
|
source_type: KnowledgeBase["source_type"];
|
|
77
79
|
source_options?: string;
|
|
80
|
+
embedding_model?: string;
|
|
81
|
+
embedding_signature?: string;
|
|
82
|
+
embedding_dimension?: number;
|
|
78
83
|
}): KnowledgeBase;
|
|
79
84
|
export declare function getKB(db: Database.Database, id: string): KnowledgeBase | undefined;
|
|
80
85
|
export declare function getKBByName(db: Database.Database, name: string): KnowledgeBase | undefined;
|
|
@@ -82,6 +87,7 @@ export declare function listKBs(db: Database.Database): KnowledgeBase[];
|
|
|
82
87
|
export declare function deleteKB(db: Database.Database, id: string): void;
|
|
83
88
|
export declare function updateKBStatus(db: Database.Database, id: string, status: KnowledgeBase["status"]): void;
|
|
84
89
|
export declare function updateKBCounts(db: Database.Database, id: string, chunkCount: number, fileCount: number): void;
|
|
90
|
+
export declare function updateKBEmbeddingMetadata(db: Database.Database, id: string, embeddingModel: string, embeddingSignature: string | null, embeddingDimension: number | null): void;
|
|
85
91
|
export declare function startIndexingJob(db: Database.Database, kbId: string, operation: IndexingJob["operation"], message: string): void;
|
|
86
92
|
export declare function updateIndexingJob(db: Database.Database, kbId: string, progress: {
|
|
87
93
|
phase?: string;
|
|
@@ -4,7 +4,7 @@ import { createRequire } from "node:module";
|
|
|
4
4
|
import { homedir } from "node:os";
|
|
5
5
|
import { dirname, join } from "node:path";
|
|
6
6
|
import { fileURLToPath } from "node:url";
|
|
7
|
-
const SCHEMA_VERSION =
|
|
7
|
+
const SCHEMA_VERSION = 5;
|
|
8
8
|
const ITERATION_BATCH_SIZE = 500;
|
|
9
9
|
const SCHEMA_SQL = `
|
|
10
10
|
CREATE TABLE IF NOT EXISTS schema_version (
|
|
@@ -23,6 +23,8 @@ CREATE TABLE IF NOT EXISTS knowledge_bases (
|
|
|
23
23
|
chunk_count INTEGER NOT NULL DEFAULT 0,
|
|
24
24
|
file_count INTEGER NOT NULL DEFAULT 0,
|
|
25
25
|
embedding_model TEXT NOT NULL DEFAULT 'multilingual-e5-small',
|
|
26
|
+
embedding_signature TEXT,
|
|
27
|
+
embedding_dimension INTEGER,
|
|
26
28
|
status TEXT NOT NULL DEFAULT 'ready'
|
|
27
29
|
);
|
|
28
30
|
|
|
@@ -304,6 +306,16 @@ CREATE INDEX IF NOT EXISTS idx_symbols_file ON symbols(kb_id, file_path);
|
|
|
304
306
|
`);
|
|
305
307
|
continue;
|
|
306
308
|
}
|
|
309
|
+
if (v === 5) {
|
|
310
|
+
const columns = db.prepare("PRAGMA table_info(knowledge_bases)").all();
|
|
311
|
+
if (!columns.some((column) => column.name === "embedding_signature")) {
|
|
312
|
+
db.exec("ALTER TABLE knowledge_bases ADD COLUMN embedding_signature TEXT");
|
|
313
|
+
}
|
|
314
|
+
if (!columns.some((column) => column.name === "embedding_dimension")) {
|
|
315
|
+
db.exec("ALTER TABLE knowledge_bases ADD COLUMN embedding_dimension INTEGER");
|
|
316
|
+
}
|
|
317
|
+
continue;
|
|
318
|
+
}
|
|
307
319
|
if (migrations[v])
|
|
308
320
|
db.exec(migrations[v]);
|
|
309
321
|
}
|
|
@@ -312,8 +324,11 @@ CREATE INDEX IF NOT EXISTS idx_symbols_file ON symbols(kb_id, file_path);
|
|
|
312
324
|
export function createKB(db, opts) {
|
|
313
325
|
const id = randomUUID();
|
|
314
326
|
const now = Date.now();
|
|
315
|
-
db.prepare(`INSERT INTO knowledge_bases (
|
|
316
|
-
|
|
327
|
+
db.prepare(`INSERT INTO knowledge_bases (
|
|
328
|
+
id, name, description, source_path, source_type, source_options, created_at, updated_at,
|
|
329
|
+
embedding_model, embedding_signature, embedding_dimension
|
|
330
|
+
)
|
|
331
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(id, opts.name, opts.description ?? null, opts.source_path ?? null, opts.source_type, opts.source_options ?? null, now, now, opts.embedding_model ?? "multilingual-e5-small", opts.embedding_signature ?? null, opts.embedding_dimension ?? null);
|
|
317
332
|
const kb = getKB(db, id);
|
|
318
333
|
if (!kb)
|
|
319
334
|
throw new Error(`Failed to create knowledge base: ${id}`);
|
|
@@ -340,6 +355,11 @@ export function updateKBStatus(db, id, status) {
|
|
|
340
355
|
export function updateKBCounts(db, id, chunkCount, fileCount) {
|
|
341
356
|
db.prepare("UPDATE knowledge_bases SET chunk_count = ?, file_count = ?, updated_at = ? WHERE id = ?").run(chunkCount, fileCount, Date.now(), id);
|
|
342
357
|
}
|
|
358
|
+
export function updateKBEmbeddingMetadata(db, id, embeddingModel, embeddingSignature, embeddingDimension) {
|
|
359
|
+
db.prepare(`UPDATE knowledge_bases
|
|
360
|
+
SET embedding_model = ?, embedding_signature = ?, embedding_dimension = ?, updated_at = ?
|
|
361
|
+
WHERE id = ?`).run(embeddingModel, embeddingSignature, embeddingDimension, Date.now(), id);
|
|
362
|
+
}
|
|
343
363
|
// --- Indexing Jobs ---
|
|
344
364
|
export function startIndexingJob(db, kbId, operation, message) {
|
|
345
365
|
const now = Date.now();
|
package/docs/configuration.md
CHANGED
|
@@ -25,14 +25,38 @@ Default data storage is `~/.pi/knowledge` under Pi and `~/.omp/knowledge` under
|
|
|
25
25
|
| `PI_KNOWLEDGE_EMBEDDING_BASE_URL` | unset | OpenAI-compatible embedding API root, such as `http://127.0.0.1:8080/v1`. Takes precedence over `OPENAI_BASE_URL`. |
|
|
26
26
|
| `OPENAI_BASE_URL` | `https://api.openai.com/v1` | Common OpenAI-compatible API root fallback. |
|
|
27
27
|
| `PI_KNOWLEDGE_EMBEDDING_MAX_CHARS` | `20000` | Final per-input API embedding safety cap for OpenAI-compatible servers with smaller context windows. This does not replace chunker bounds. |
|
|
28
|
-
| `PI_KNOWLEDGE_EMBEDDING_API_FALLBACK` | unset | Set to `local` to explicitly fall back to local embeddings after API failures.
|
|
28
|
+
| `PI_KNOWLEDGE_EMBEDDING_API_FALLBACK` | unset | Set to `local` to explicitly fall back to local embeddings for query embeddings after API failures. Indexing/import/update document batches do not fall back because mixed-provider vector files are unsafe. |
|
|
29
29
|
| `PI_KNOWLEDGE_ENABLE_NATIVE_IDLE_DISPOSE` | unset | Enables idle timers for embedding/reranker run coordination. It does not guarantee immediate worker memory release; model-worker shutdown remains tied to engine/session shutdown. Disabled by default for stable shutdown. |
|
|
30
30
|
| `PI_KNOWLEDGE_EMBEDDING_IDLE_MS` | `30000` | Idle timer used only when native idle disposal coordination is enabled. Mainly for lifecycle stress tests. |
|
|
31
31
|
| `PI_KNOWLEDGE_OFFLINE` | unset | Use with a pre-populated model cache for offline local model operation. See `docs/offline-mode.md`. |
|
|
32
32
|
|
|
33
|
+
Each KB stores the embedding model label, vector dimension, and a non-secret embedding signature that includes provider/model semantics, query/document prefixes, pooling, normalization, API input cap, and a hash of the API base URL when applicable. `knowledge_search` skips vector retrieval for KBs whose stored signature or dimension is missing or incompatible with the current query embedding and uses BM25-only fallback for hybrid searches. Run `knowledge_update` to rebuild vectors after changing embedding provider/model/max-character settings or after upgrading older KBs with no signature metadata.
|
|
34
|
+
|
|
35
|
+
## Reranker
|
|
36
|
+
|
|
37
|
+
| Variable | Default | Purpose |
|
|
38
|
+
|----------|---------|---------|
|
|
39
|
+
| `PI_KNOWLEDGE_RERANKER` | `Xenova/ms-marco-MiniLM-L-4-v2` | Reranker used by `deep` mode. Accepts local/HF forms (`<model>`, `hf:<model>`, Hugging Face model URLs) or `api:<model>` for an external HTTP rerank service. |
|
|
40
|
+
| `PI_KNOWLEDGE_RERANKER_REVISION` | `main` | Hugging Face revision passed to Transformers.js for local/HF rerankers. Explicit env value overrides a revision parsed from a model URL. |
|
|
41
|
+
| `PI_KNOWLEDGE_RERANKER_DTYPE` | unset | Optional Transformers.js dtype, such as `fp32`, for local/HF rerankers. |
|
|
42
|
+
| `PI_KNOWLEDGE_RERANKER_REMOTE_HOST` | Hugging Face host or URL-derived host | Remote artifact host for HF-compatible model repositories. Use with trusted mirrors only. |
|
|
43
|
+
| `PI_KNOWLEDGE_RERANKER_REMOTE_PATH_TEMPLATE` | `{model}/resolve/{revision}/` | Transformers.js remote path template for HF-compatible mirrors. |
|
|
44
|
+
| `PI_KNOWLEDGE_RERANKER_API_ENDPOINT` | derived from base URL | Full HTTP rerank endpoint. Takes precedence over `PI_KNOWLEDGE_RERANKER_API_BASE_URL` and `OPENAI_BASE_URL`. |
|
|
45
|
+
| `PI_KNOWLEDGE_RERANKER_API_BASE_URL` | `https://api.cohere.com/v2` | Base URL for Cohere/Jina-compatible `/rerank` APIs. `OPENAI_BASE_URL` is accepted only as a convenience fallback for gateways exposing the same schema. |
|
|
46
|
+
| `PI_KNOWLEDGE_RERANKER_API_KEY` | `OPENAI_API_KEY` fallback | Bearer token for API rerankers. Omit only for trusted local services that do not require auth. |
|
|
47
|
+
| `PI_KNOWLEDGE_RERANKER_API_FORMAT` | `cohere` | Request/response preset: `cohere`, `jina`, or `custom-json`. All presets send `model`, `query`, `documents`, `top_n`, and `return_documents:false`. |
|
|
48
|
+
| `PI_KNOWLEDGE_RERANKER_API_TIMEOUT_MS` | `30000` | Per-request timeout for API reranking. |
|
|
49
|
+
| `PI_KNOWLEDGE_RERANKER_MAX_DOC_CHARS` | `12000` | Per-candidate document safety cap before sending content to the API. |
|
|
50
|
+
| `PI_KNOWLEDGE_RERANKER_API_RESULTS_PATH` | `results` | Dot path to the result array for `custom-json` or compatible responses. |
|
|
51
|
+
| `PI_KNOWLEDGE_RERANKER_API_INDEX_FIELD` | `index` | Field name or dot path holding the original candidate index. |
|
|
52
|
+
| `PI_KNOWLEDGE_RERANKER_API_SCORE_FIELD` | `relevance_score` | Field name or dot path holding the rerank score. |
|
|
53
|
+
| `PI_KNOWLEDGE_RERANKER_API_SCORE_DIRECTION` | `desc` | Sort direction for returned scores: `desc` when higher is better, `asc` when lower is better. |
|
|
54
|
+
|
|
55
|
+
Reranker settings affect query-time `deep` search only; they do not change indexed KB vectors. Local/HF rerankers run in the isolated model worker. Offline mode sets the model worker to local-only loading, so a local/HF reranker must already exist in the model cache. API rerankers send the query and selected candidate chunk content to the configured external service; failures surface by default and do not silently fall back to the local worker.
|
|
56
|
+
|
|
33
57
|
Local embeddings require a real Node 22+ executable for the isolated model worker. `pi-knowledge` first tries Node IPC and automatically falls back to a stdin/stdout JSONL worker transport when the host runtime does not expose `child_process.fork().send()`, which can happen in Windows OMP compatibility layers. On Windows it searches `PI_KNOWLEDGE_NODE_PATH`, persisted `knowledge_configure` settings, common Node, Volta, NVM, Codex `cua_node`, and `PATH` locations before failing. If worker startup still fails, call `knowledge_configure` with the full `node.exe` path such as `C:\Program Files\nodejs\node.exe`; accidental surrounding quotes are ignored. OpenAI-compatible embeddings avoid the local worker entirely.
|
|
34
58
|
|
|
35
|
-
API embedding failures intentionally surface by default. Silent fallback can hide bad API keys, wrong base URLs, unsupported model names, or context-window errors and can produce a KB with a different embedding model than intended.
|
|
59
|
+
API embedding failures intentionally surface by default. Silent fallback can hide bad API keys, wrong base URLs, unsupported model names, or context-window errors and can produce a KB with a different embedding model than intended. Query-time fallback is opt-in; indexing fallback is disabled to prevent mixed vector spaces in one KB.
|
|
36
60
|
|
|
37
61
|
## Runtime Features and Diagnostics
|
|
38
62
|
|
|
@@ -41,10 +65,21 @@ API embedding failures intentionally surface by default. Silent fallback can hid
|
|
|
41
65
|
| `PI_KNOWLEDGE_WATCH` | unset | Set to `true` to start file watchers for directory KBs. The polling fallback remains available when native `fs.watch` fails. |
|
|
42
66
|
| `PI_KNOWLEDGE_AUTO_INJECT` | unset | Set to `true` to auto-search KB context before model calls. This is opt-in. |
|
|
43
67
|
| `PI_KNOWLEDGE_STALE_INDEXING_MS` | built-in stale threshold | Override the stale indexing threshold used by diagnostics and `knowledge_doctor`. |
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
`
|
|
68
|
+
| `PI_KNOWLEDGE_SEARCH_PROFILE` | `auto` | Select the default search tuning profile. Values: `auto`, `balanced`, `low_token`, `precision`, `recall`, `long_context`, `code`, `docs`. Tool `profile` parameters override this. |
|
|
69
|
+
| `PI_KNOWLEDGE_SEARCH_DEFAULT_LIMIT` | profile default | Override the result limit used only when `knowledge_search` does not pass `limit`. Clamped to 1-50. |
|
|
70
|
+
| `PI_KNOWLEDGE_SNIPPET_MAX_LENGTH` | profile default | Override returned snippet length. Clamped to 80-4000 characters. |
|
|
71
|
+
| `PI_KNOWLEDGE_MIN_HYBRID_SCORE` | profile default | Override the hybrid confidence gate. Higher values reduce noisy results; lower values increase recall. Clamped to 0-1. |
|
|
72
|
+
| `PI_KNOWLEDGE_SEARCH_CANDIDATE_MIN` | profile default | Override the minimum internal retrieval candidate pool. Clamped to 10-500. |
|
|
73
|
+
| `PI_KNOWLEDGE_SEARCH_CANDIDATE_MULTIPLIER` | profile default | Override the internal candidate multiplier applied to the effective result limit. Clamped to 2-30. |
|
|
74
|
+
| `PI_KNOWLEDGE_ADAPTIVE_CONTEXT_LINES` | profile default | Override surrounding lines fetched for `adaptive` context expansion. Clamped to 10-500. |
|
|
75
|
+
| `PI_KNOWLEDGE_ADAPTIVE_MAX_CHARS` | profile default | Override maximum adaptive context characters per result. Clamped to 1000-50000. |
|
|
76
|
+
| `PI_KNOWLEDGE_ADAPTIVE_NEIGHBOR_TARGET` | profile default | Override the number of nearby/query-relevant chunks considered for adaptive windows. Clamped to 1-20. |
|
|
77
|
+
| `PI_KNOWLEDGE_DEEP_RERANK_CANDIDATES` | profile default | Override how many filtered candidates `deep` mode sends to the reranker. Clamped to 5-100. |
|
|
78
|
+
| `PI_KNOWLEDGE_DEEP_RERANK_TOPK_MULTIPLIER` | profile default | Override deep rerank top-k multiplier relative to the effective result limit. Clamped to 1-10. |
|
|
79
|
+
|
|
80
|
+
Search diagnostics include result provenance and applied search tuning in both text output and structured `details`: selected profile, effective limit, snippet length, hybrid threshold, candidate pool size, chunk id, chunk hash, match reason, indexed timestamp, stale flag, and source mtime when the original source file is available. This provenance is stored in SQLite and derived from indexed source metadata; it does not create or modify files in the indexed project.
|
|
81
|
+
|
|
82
|
+
`knowledge_search` supports `file_type`, `path_pattern`, `profile`, and explicit `limit` filters. Search profiles tune result count, snippets, hybrid strictness, candidate breadth, adaptive context, and deep rerank breadth at runtime. `auto` chooses from query shape, mode, and KB source type; explicit tool parameters win over env overrides, env overrides win over profile defaults, and profile defaults win over built-in compatibility values. `hybrid` mode remains lexical-anchored to prevent low-evidence semantic false positives; use `semantic` when the query is conceptual and exact terms may differ from indexed wording.
|
|
48
83
|
|
|
49
84
|
`knowledge_doctor` emits both human-readable issues and machine-readable actions in structured `details`. Action codes include `run_update`, `rebuild_kb`, `wait_for_indexing`, `review_skipped_scope`, `rebuild_vectors`, `check_source`, and `none`. Agents should prefer these action codes over parsing the English action text.
|
|
50
85
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-knowledge",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.6.0",
|
|
4
4
|
"description": "Local-first RAG knowledge base for Pi and OMP coding agents with hybrid code/doc search, reranking, diagnostics, and large-project indexing",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|