pi-knowledge 0.4.7 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/README.md +19 -12
- package/dist/index.js +186 -42
- package/dist/src/diagnostics/health.d.ts +1 -1
- package/dist/src/diagnostics/health.js +13 -12
- package/dist/src/embedding/provider.d.ts +1 -1
- package/dist/src/embedding/provider.js +12 -4
- package/dist/src/engine.d.ts +61 -10
- package/dist/src/engine.js +591 -186
- package/dist/src/indexer/chunker.d.ts +3 -1
- package/dist/src/indexer/chunker.js +57 -48
- package/dist/src/indexer/symbols.d.ts +2 -0
- package/dist/src/indexer/symbols.js +175 -0
- package/dist/src/model-worker-client.d.ts +1 -1
- package/dist/src/model-worker-client.js +7 -7
- package/dist/src/search/reranker.d.ts +1 -1
- package/dist/src/search/reranker.js +2 -2
- package/dist/src/search/vector.d.ts +1 -1
- package/dist/src/search/vector.js +5 -1
- package/dist/src/storage/sqlite.d.ts +36 -1
- package/dist/src/storage/sqlite.js +159 -2
- package/dist/src/watcher/file-watcher.d.ts +2 -1
- package/dist/src/watcher/file-watcher.js +20 -30
- package/docs/configuration.md +10 -2
- package/package.json +1 -1
|
@@ -10,6 +10,9 @@ const ENABLE_NATIVE_IDLE_DISPOSE = process.env.PI_KNOWLEDGE_ENABLE_NATIVE_IDLE_D
|
|
|
10
10
|
const API_FALLBACK_TO_LOCAL = process.env.PI_KNOWLEDGE_EMBEDDING_API_FALLBACK === "local";
|
|
11
11
|
const DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1";
|
|
12
12
|
const DEFAULT_API_MAX_EMBED_CHARS = 20_000;
|
|
13
|
+
function isAbortError(error) {
|
|
14
|
+
return typeof error === "object" && error !== null && "name" in error && error.name === "AbortError";
|
|
15
|
+
}
|
|
13
16
|
function clearIdleTimer() {
|
|
14
17
|
if (disposeTimer)
|
|
15
18
|
clearTimeout(disposeTimer);
|
|
@@ -57,7 +60,7 @@ export async function prepareForShutdown() {
|
|
|
57
60
|
clearIdleTimer();
|
|
58
61
|
await waitForNoActiveRuns();
|
|
59
62
|
}
|
|
60
|
-
async function embedViaAPI(texts, prefix) {
|
|
63
|
+
async function embedViaAPI(texts, prefix, signal) {
|
|
61
64
|
const [provider, model] = EMBEDDING_CONFIG.split(":");
|
|
62
65
|
const configuredMaxChars = Number(process.env.PI_KNOWLEDGE_EMBEDDING_MAX_CHARS ?? DEFAULT_API_MAX_EMBED_CHARS);
|
|
63
66
|
const maxChars = Number.isFinite(configuredMaxChars) && configuredMaxChars > 0 ? configuredMaxChars : DEFAULT_API_MAX_EMBED_CHARS;
|
|
@@ -75,6 +78,7 @@ async function embedViaAPI(texts, prefix) {
|
|
|
75
78
|
method: "POST",
|
|
76
79
|
headers: { "Content-Type": "application/json", Authorization: `Bearer ${apiKey}` },
|
|
77
80
|
body: JSON.stringify({ input: safeTexts, model: model || "text-embedding-3-small" }),
|
|
81
|
+
signal,
|
|
78
82
|
});
|
|
79
83
|
if (!res.ok) {
|
|
80
84
|
const detail = await res.text().catch(() => "");
|
|
@@ -87,10 +91,14 @@ async function embedViaAPI(texts, prefix) {
|
|
|
87
91
|
}
|
|
88
92
|
export async function embedTexts(texts, prefix, signal) {
|
|
89
93
|
if (!EMBEDDING_CONFIG.startsWith("local")) {
|
|
94
|
+
if (signal?.aborted)
|
|
95
|
+
throw new Error("Cancelled");
|
|
90
96
|
try {
|
|
91
|
-
return await embedViaAPI(texts, prefix);
|
|
97
|
+
return await embedViaAPI(texts, prefix, signal);
|
|
92
98
|
}
|
|
93
99
|
catch (error) {
|
|
100
|
+
if (signal?.aborted || isAbortError(error))
|
|
101
|
+
throw new Error("Cancelled");
|
|
94
102
|
if (!API_FALLBACK_TO_LOCAL)
|
|
95
103
|
throw error;
|
|
96
104
|
console.warn(`pi-knowledge: embedding API failed; falling back to local model because PI_KNOWLEDGE_EMBEDDING_API_FALLBACK=local (${error instanceof Error ? error.message : String(error)})`);
|
|
@@ -104,8 +112,8 @@ export async function embedTexts(texts, prefix, signal) {
|
|
|
104
112
|
endRun();
|
|
105
113
|
}
|
|
106
114
|
}
|
|
107
|
-
export async function embedQuery(text) {
|
|
108
|
-
const [vec] = await embedTexts([text], "query");
|
|
115
|
+
export async function embedQuery(text, signal) {
|
|
116
|
+
const [vec] = await embedTexts([text], "query", signal);
|
|
109
117
|
return vec;
|
|
110
118
|
}
|
|
111
119
|
export async function embedDocuments(texts, signal) {
|
package/dist/src/engine.d.ts
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { type DiagnosticResult } from "./diagnostics/health.ts";
|
|
2
2
|
import { createSkippedScanStats } from "./indexer/chunker.ts";
|
|
3
3
|
import { type RankingDiagnostics } from "./search/ranking.ts";
|
|
4
|
-
import { type KnowledgeBase } from "./storage/sqlite.ts";
|
|
4
|
+
import { type KnowledgeBase, type KnowledgeSymbol } from "./storage/sqlite.ts";
|
|
5
|
+
export type SearchMode = "auto" | "fast" | "semantic" | "hybrid" | "deep" | "adaptive" | "code" | "config" | "docs" | "errors" | "decision";
|
|
5
6
|
export interface SearchOptions {
|
|
6
|
-
mode?:
|
|
7
|
+
mode?: SearchMode;
|
|
7
8
|
limit?: number;
|
|
8
9
|
offset?: number;
|
|
9
10
|
kb_id?: string;
|
|
@@ -23,6 +24,15 @@ export interface SearchResult {
|
|
|
23
24
|
start_line: number;
|
|
24
25
|
end_line: number;
|
|
25
26
|
ranking?: RankingDiagnostics;
|
|
27
|
+
provenance?: {
|
|
28
|
+
chunk_id: string;
|
|
29
|
+
chunk_hash: string;
|
|
30
|
+
indexed_at: number;
|
|
31
|
+
source_mtime?: number;
|
|
32
|
+
stale: boolean;
|
|
33
|
+
match_reason: "bm25" | "vector" | "hybrid" | "rerank" | "symbol" | "adaptive";
|
|
34
|
+
source_chunk_ids?: string[];
|
|
35
|
+
};
|
|
26
36
|
}
|
|
27
37
|
export declare const CURRENT_EMBEDDING_MODEL = "multilingual-e5-small";
|
|
28
38
|
export interface SearchResponse {
|
|
@@ -30,8 +40,8 @@ export interface SearchResponse {
|
|
|
30
40
|
total_count: number;
|
|
31
41
|
has_more: boolean;
|
|
32
42
|
warnings?: string[];
|
|
33
|
-
mode_used?:
|
|
34
|
-
retry_modes?:
|
|
43
|
+
mode_used?: SearchMode;
|
|
44
|
+
retry_modes?: SearchMode[];
|
|
35
45
|
suggestions?: string[];
|
|
36
46
|
}
|
|
37
47
|
export type ProgressCallback = (msg: string) => void;
|
|
@@ -45,12 +55,44 @@ export interface DoctorIssue {
|
|
|
45
55
|
kb_name?: string;
|
|
46
56
|
message: string;
|
|
47
57
|
action: string;
|
|
58
|
+
action_code?: "run_update" | "rebuild_kb" | "wait_for_indexing" | "review_skipped_scope" | "rebuild_vectors" | "check_source" | "none";
|
|
48
59
|
}
|
|
49
60
|
export interface DoctorReport {
|
|
50
61
|
health_score: number;
|
|
51
62
|
summary: string;
|
|
52
63
|
issues: DoctorIssue[];
|
|
53
64
|
diagnostics: DiagnosticResult[];
|
|
65
|
+
actions: Array<{
|
|
66
|
+
code: NonNullable<DoctorIssue["action_code"]>;
|
|
67
|
+
kb_name?: string;
|
|
68
|
+
target?: string;
|
|
69
|
+
description: string;
|
|
70
|
+
}>;
|
|
71
|
+
}
|
|
72
|
+
export interface SymbolSearchOptions {
|
|
73
|
+
kb_id?: string;
|
|
74
|
+
kind?: KnowledgeSymbol["kind"];
|
|
75
|
+
file_pattern?: string;
|
|
76
|
+
limit?: number;
|
|
77
|
+
offset?: number;
|
|
78
|
+
exact?: boolean;
|
|
79
|
+
}
|
|
80
|
+
export interface SymbolSearchResponse {
|
|
81
|
+
results: Array<{
|
|
82
|
+
name: string;
|
|
83
|
+
kind: KnowledgeSymbol["kind"];
|
|
84
|
+
file_path: string;
|
|
85
|
+
file_type: string;
|
|
86
|
+
kb_name: string;
|
|
87
|
+
start_line: number;
|
|
88
|
+
end_line: number;
|
|
89
|
+
signature?: string;
|
|
90
|
+
container_name?: string;
|
|
91
|
+
text: string;
|
|
92
|
+
indexed_at: number;
|
|
93
|
+
}>;
|
|
94
|
+
total_count: number;
|
|
95
|
+
has_more: boolean;
|
|
54
96
|
}
|
|
55
97
|
export interface IndexPlan {
|
|
56
98
|
source_type: "file" | "directory" | "text" | "url";
|
|
@@ -63,29 +105,38 @@ export declare class KnowledgeEngine {
|
|
|
63
105
|
private db;
|
|
64
106
|
private knowledgeDir;
|
|
65
107
|
private activeUpdates;
|
|
108
|
+
private activeMutations;
|
|
109
|
+
private disposing;
|
|
110
|
+
private runExclusive;
|
|
66
111
|
initialize(knowledgeDir: string): Promise<void>;
|
|
67
|
-
|
|
112
|
+
private vectorPathFor;
|
|
113
|
+
private deleteVectorFile;
|
|
114
|
+
plan(source: string, options?: AddOptions, signal?: AbortSignal): IndexPlan;
|
|
68
115
|
add(source: string, name: string, onProgress?: ProgressCallback, signal?: AbortSignal, options?: AddOptions): Promise<{
|
|
69
116
|
kb: KnowledgeBase;
|
|
70
117
|
chunkCount: number;
|
|
71
118
|
}>;
|
|
119
|
+
private addUnlocked;
|
|
72
120
|
update(nameOrId: string, onProgress?: ProgressCallback, signal?: AbortSignal): Promise<{
|
|
73
121
|
added: number;
|
|
74
122
|
removed: number;
|
|
75
123
|
unchanged: number;
|
|
76
124
|
}>;
|
|
77
125
|
private runUpdate;
|
|
78
|
-
search(query: string, options?: SearchOptions): Promise<SearchResponse>;
|
|
126
|
+
search(query: string, options?: SearchOptions, signal?: AbortSignal): Promise<SearchResponse>;
|
|
79
127
|
remove(nameOrId: string): boolean;
|
|
80
|
-
list(): KnowledgeBase[];
|
|
128
|
+
list(signal?: AbortSignal): KnowledgeBase[];
|
|
81
129
|
clear(): void;
|
|
82
|
-
diagnose(): DiagnosticResult[];
|
|
83
|
-
|
|
84
|
-
|
|
130
|
+
diagnose(signal?: AbortSignal): DiagnosticResult[];
|
|
131
|
+
symbolSearch(query: string, options?: SymbolSearchOptions, signal?: AbortSignal): SymbolSearchResponse;
|
|
132
|
+
doctor(signal?: AbortSignal): DoctorReport;
|
|
133
|
+
exportKB(nameOrId: string, outputPath: string, signal?: AbortSignal, onProgress?: ProgressCallback): Promise<number>;
|
|
134
|
+
private exportKBUnlocked;
|
|
85
135
|
importKB(inputPath: string, onProgress?: ProgressCallback, signal?: AbortSignal): Promise<{
|
|
86
136
|
kb: KnowledgeBase;
|
|
87
137
|
chunkCount: number;
|
|
88
138
|
}>;
|
|
139
|
+
private importKBUnlocked;
|
|
89
140
|
dispose(options?: {
|
|
90
141
|
disposeModels?: boolean;
|
|
91
142
|
}): Promise<void>;
|