pi-knowledge 0.4.7 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -18,7 +18,7 @@ export interface ScannableFile {
18
18
  }
19
19
  export interface SkippedScanEntry {
20
20
  path: string;
21
- reason: "suggested_excluded" | "oversized" | "binary" | "unreadable" | "inaccessible";
21
+ reason: "suggested_excluded" | "oversized" | "binary" | "unreadable" | "inaccessible" | "extraction_failed";
22
22
  size?: number;
23
23
  }
24
24
  export interface ScanResult {
@@ -30,7 +30,9 @@ export interface ScanResult {
30
30
  };
31
31
  }
32
32
  export declare function createSkippedScanStats(): ScanResult["skipped"];
33
+ export declare function addSkippedScanEntry(skipped: ScanResult["skipped"], entry: SkippedScanEntry): void;
33
34
  export declare function summarizeSkippedScan(skipped: ScanResult["skipped"]): string;
35
+ export declare function isSupportedDocumentFile(filePath: string): boolean;
34
36
  export declare function iterateScannableFiles(dirPath: string, skipped?: ScanResult["skipped"], options?: ScanOptions): Generator<ScannableFile>;
35
37
  export declare function iterateScannedFiles(dirPath: string, skipped?: ScanResult["skipped"], options?: ScanOptions): Generator<ScannedFile>;
36
38
  export declare function iterateScannedFilesAsync(dirPath: string, onFile: (file: ScannedFile) => Promise<void> | void, skipped?: ScanResult["skipped"], options?: ScanOptions): Promise<ScanResult["skipped"]>;
@@ -3,53 +3,50 @@ import { closeSync, existsSync, openSync, readdirSync, readFileSync, readSync, s
3
3
  import { extname, join, relative, sep } from "node:path";
4
4
  import ignore from "ignore";
5
5
  const MAX_FILE_SIZE = 10 * 1024 * 1024; // 10 MB
6
- const BINARY_EXTENSIONS = new Set([
7
- ".png",
8
- ".jpg",
9
- ".jpeg",
10
- ".gif",
11
- ".ico",
12
- ".bmp",
13
- ".webp",
14
- ".svg",
15
- ".woff",
16
- ".woff2",
17
- ".ttf",
18
- ".eot",
19
- ".otf",
20
- ".zip",
21
- ".gz",
22
- ".tar",
23
- ".bz2",
24
- ".7z",
25
- ".rar",
26
- ".pdf",
27
- ".doc",
28
- ".docx",
29
- ".xls",
30
- ".xlsx",
31
- ".ppt",
32
- ".exe",
33
- ".dll",
34
- ".so",
35
- ".dylib",
36
- ".node",
37
- ".db",
38
- ".sqlite",
39
- ".bin",
40
- ".dat",
41
- ".lock",
42
- ".mp3",
43
- ".mp4",
44
- ".wav",
45
- ".avi",
46
- ".mov",
47
- ".webm",
48
- ".wasm",
49
- ".o",
50
- ".a",
51
- ".lib",
52
- ]);
6
+ const DOCUMENT_EXTENSIONS = { ".doc": true, ".docx": true, ".pdf": true };
7
+ const BINARY_EXTENSIONS = {
8
+ ".7z": true,
9
+ ".a": true,
10
+ ".avi": true,
11
+ ".bin": true,
12
+ ".bmp": true,
13
+ ".bz2": true,
14
+ ".dat": true,
15
+ ".db": true,
16
+ ".dll": true,
17
+ ".dylib": true,
18
+ ".eot": true,
19
+ ".exe": true,
20
+ ".gif": true,
21
+ ".gz": true,
22
+ ".ico": true,
23
+ ".jpg": true,
24
+ ".jpeg": true,
25
+ ".lib": true,
26
+ ".lock": true,
27
+ ".mov": true,
28
+ ".mp3": true,
29
+ ".mp4": true,
30
+ ".node": true,
31
+ ".o": true,
32
+ ".otf": true,
33
+ ".ppt": true,
34
+ ".rar": true,
35
+ ".so": true,
36
+ ".sqlite": true,
37
+ ".svg": true,
38
+ ".tar": true,
39
+ ".ttf": true,
40
+ ".wasm": true,
41
+ ".wav": true,
42
+ ".webm": true,
43
+ ".webp": true,
44
+ ".woff": true,
45
+ ".woff2": true,
46
+ ".xls": true,
47
+ ".xlsx": true,
48
+ ".zip": true,
49
+ };
53
50
  const DEFAULT_SUGGESTED_EXCLUDE = [
54
51
  "node_modules",
55
52
  ".git",
@@ -111,6 +108,7 @@ export function createSkippedScanStats() {
111
108
  binary: 0,
112
109
  unreadable: 0,
113
110
  inaccessible: 0,
111
+ extraction_failed: 0,
114
112
  },
115
113
  samples: [],
116
114
  };
@@ -121,6 +119,9 @@ function addSkipped(skipped, entry) {
121
119
  if (skipped.samples.length < MAX_SKIPPED_SAMPLES)
122
120
  skipped.samples.push(entry);
123
121
  }
122
+ export function addSkippedScanEntry(skipped, entry) {
123
+ addSkipped(skipped, entry);
124
+ }
124
125
  export function summarizeSkippedScan(skipped) {
125
126
  return (Object.entries(skipped.by_reason)
126
127
  .filter(([, count]) => count > 0)
@@ -157,14 +158,22 @@ function detectFileType(filePath) {
157
158
  ".zsh": "shell",
158
159
  ".sql": "sql",
159
160
  ".graphql": "graphql",
161
+ ".pdf": "pdf",
162
+ ".doc": "docx",
163
+ ".docx": "docx",
160
164
  ".txt": "text",
161
165
  ".csv": "text",
162
166
  ".log": "text",
163
167
  };
164
168
  return map[ext] ?? "text";
165
169
  }
170
+ export function isSupportedDocumentFile(filePath) {
171
+ return DOCUMENT_EXTENSIONS[extname(filePath).toLowerCase()] === true;
172
+ }
166
173
  function isBinaryFile(filePath) {
167
- if (BINARY_EXTENSIONS.has(extname(filePath).toLowerCase()))
174
+ if (isSupportedDocumentFile(filePath))
175
+ return false;
176
+ if (BINARY_EXTENSIONS[extname(filePath).toLowerCase()] === true)
168
177
  return true;
169
178
  let fd;
170
179
  try {
@@ -0,0 +1,2 @@
1
+ import type { KnowledgeSymbolInsert } from "../storage/sqlite.ts";
2
+ export declare function extractSymbols(content: string, filePath: string, fileType: string): KnowledgeSymbolInsert[];
@@ -0,0 +1,175 @@
1
+ import { extname } from "node:path";
2
+ const CODE_EXTENSIONS = new Set([".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".py", ".go", ".rs", ".java"]);
3
+ const CONFIG_EXTENSIONS = new Set([".json", ".yaml", ".yml", ".toml", ".env"]);
4
+ function lineNumberAt(content, offset) {
5
+ let line = 1;
6
+ for (let i = 0; i < offset; i++) {
7
+ if (content.charCodeAt(i) === 10)
8
+ line++;
9
+ }
10
+ return line;
11
+ }
12
+ function lineEnd(content, startLine, maxLines = 12) {
13
+ return Math.min(startLine + maxLines, content.split("\n").length);
14
+ }
15
+ function makeSymbol(name, kind, filePath, fileType, startLine, options = {}) {
16
+ const trimmed = name.trim();
17
+ if (!trimmed)
18
+ return undefined;
19
+ return {
20
+ name: trimmed,
21
+ kind,
22
+ file_path: filePath,
23
+ file_type: fileType,
24
+ start_line: startLine,
25
+ end_line: options.endLine ?? startLine,
26
+ container_name: options.containerName ?? null,
27
+ signature: options.signature ?? null,
28
+ text: options.text ?? trimmed,
29
+ metadata_json: JSON.stringify(options.metadata ?? {}),
30
+ };
31
+ }
32
+ function pushSymbol(symbols, symbol) {
33
+ if (symbol)
34
+ symbols.push(symbol);
35
+ }
36
+ function extractMarkdownSymbols(content, filePath, fileType) {
37
+ const symbols = [];
38
+ const lines = content.split("\n");
39
+ for (let index = 0; index < lines.length; index++) {
40
+ const match = /^(#{1,6})\s+(.+?)\s*$/.exec(lines[index]);
41
+ if (!match)
42
+ continue;
43
+ pushSymbol(symbols, makeSymbol(match[2].replace(/\s+#*$/, ""), "heading", filePath, fileType, index + 1, {
44
+ text: lines[index],
45
+ metadata: { depth: match[1].length },
46
+ }));
47
+ }
48
+ return symbols;
49
+ }
50
+ function extractConfigSymbols(content, filePath, fileType) {
51
+ const symbols = [];
52
+ const lines = content.split("\n");
53
+ const ext = extname(filePath).toLowerCase();
54
+ for (let index = 0; index < lines.length; index++) {
55
+ const line = lines[index];
56
+ if (ext === ".json") {
57
+ const match = /^\s*"([^"]+)"\s*:/.exec(line);
58
+ pushSymbol(symbols, match ? makeSymbol(match[1], "config_key", filePath, fileType, index + 1, { text: line.trim() }) : undefined);
59
+ continue;
60
+ }
61
+ if (ext === ".yaml" || ext === ".yml") {
62
+ const match = /^\s*([A-Za-z0-9_.-]+)\s*:/.exec(line);
63
+ pushSymbol(symbols, match ? makeSymbol(match[1], "config_key", filePath, fileType, index + 1, { text: line.trim() }) : undefined);
64
+ continue;
65
+ }
66
+ if (ext === ".toml") {
67
+ const section = /^\s*\[+([A-Za-z0-9_.-]+)\]+/.exec(line);
68
+ const key = /^\s*([A-Za-z0-9_.-]+)\s*=/.exec(line);
69
+ const name = section?.[1] ?? key?.[1];
70
+ pushSymbol(symbols, name ? makeSymbol(name, "config_key", filePath, fileType, index + 1, { text: line.trim() }) : undefined);
71
+ continue;
72
+ }
73
+ const env = /^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=/.exec(line);
74
+ pushSymbol(symbols, env ? makeSymbol(env[1], "env_var", filePath, fileType, index + 1, { text: line.trim() }) : undefined);
75
+ }
76
+ return symbols;
77
+ }
78
+ function extractCodeSymbols(content, filePath, fileType) {
79
+ const symbols = [];
80
+ const patterns = [
81
+ {
82
+ regex: /\b(?:export\s+)?(?:async\s+)?function\s+([A-Za-z_$][\w$]*)\s*\([^)]*\)/g,
83
+ kind: "function",
84
+ nameIndex: 1,
85
+ },
86
+ { regex: /\b(?:export\s+)?class\s+([A-Za-z_$][\w$]*)\b/g, kind: "class", nameIndex: 1 },
87
+ { regex: /\b(?:export\s+)?interface\s+([A-Za-z_$][\w$]*)\b/g, kind: "interface", nameIndex: 1 },
88
+ { regex: /\b(?:export\s+)?type\s+([A-Za-z_$][\w$]*)\b/g, kind: "type", nameIndex: 1 },
89
+ {
90
+ regex: /\b(?:export\s+)?(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*(?:async\s*)?(?:\([^)]*\)|[A-Za-z_$][\w$]*)\s*=>/g,
91
+ kind: "function",
92
+ nameIndex: 1,
93
+ },
94
+ { regex: /\b(?:export\s+)?(?:const|let|var)\s+([A-Za-z_$][\w$]*)\b/g, kind: "variable", nameIndex: 1 },
95
+ { regex: /\bfunc\s+(?:\([^)]+\)\s*)?([A-Za-z_][\w]*)\s*\([^)]*\)/g, kind: "function", nameIndex: 1 },
96
+ { regex: /\btype\s+([A-Za-z_][\w]*)\s+(?:struct|interface)\b/g, kind: "type", nameIndex: 1 },
97
+ { regex: /\bdef\s+([A-Za-z_][\w]*)\s*\([^)]*\)\s*:/g, kind: "function", nameIndex: 1 },
98
+ { regex: /\bclass\s+([A-Za-z_][\w]*)\b/g, kind: "class", nameIndex: 1 },
99
+ { regex: /\bfn\s+([A-Za-z_][\w]*)\s*\([^)]*\)/g, kind: "function", nameIndex: 1 },
100
+ { regex: /\b(?:struct|enum|trait)\s+([A-Za-z_][\w]*)\b/g, kind: "type", nameIndex: 1 },
101
+ ];
102
+ const seen = new Set();
103
+ for (const pattern of patterns) {
104
+ for (const match of content.matchAll(pattern.regex)) {
105
+ const name = match[pattern.nameIndex];
106
+ const index = match.index ?? 0;
107
+ const startLine = lineNumberAt(content, index);
108
+ const key = `${pattern.kind}:${name}:${startLine}`;
109
+ if (seen.has(key))
110
+ continue;
111
+ seen.add(key);
112
+ pushSymbol(symbols, makeSymbol(name, pattern.kind, filePath, fileType, startLine, {
113
+ endLine: lineEnd(content, startLine),
114
+ signature: match[0].split("\n")[0].trim(),
115
+ text: match[0].split("\n")[0].trim(),
116
+ }));
117
+ }
118
+ }
119
+ return symbols;
120
+ }
121
+ function extractRouteSymbols(content, filePath, fileType) {
122
+ const symbols = [];
123
+ const patterns = [
124
+ {
125
+ regex: /\b(?:app|router|route|server|api|hono)\s*\.\s*(get|post|put|patch|delete|del|options|head|all|use)\s*\(\s*["'`]([^"'`]+)["'`]/g,
126
+ methodIndex: 1,
127
+ pathIndex: 2,
128
+ },
129
+ {
130
+ regex: /\b(?:http\.)?HandleFunc\s*\(\s*["'`]([^"'`]+)["'`]/g,
131
+ pathIndex: 1,
132
+ defaultMethod: "ANY",
133
+ },
134
+ {
135
+ regex: /\b(?:http\.)?Handle\s*\(\s*["'`]([^"'`]+)["'`]/g,
136
+ pathIndex: 1,
137
+ defaultMethod: "ANY",
138
+ },
139
+ ];
140
+ const seen = new Set();
141
+ for (const pattern of patterns) {
142
+ for (const match of content.matchAll(pattern.regex)) {
143
+ const rawMethod = pattern.methodIndex ? match[pattern.methodIndex] : pattern.defaultMethod;
144
+ const method = rawMethod === "del" ? "DELETE" : (rawMethod ?? "ANY").toUpperCase();
145
+ const path = match[pattern.pathIndex];
146
+ const index = match.index ?? 0;
147
+ const startLine = lineNumberAt(content, index);
148
+ const name = `${method} ${path}`;
149
+ const key = `${name}:${startLine}`;
150
+ if (seen.has(key))
151
+ continue;
152
+ seen.add(key);
153
+ pushSymbol(symbols, makeSymbol(name, "route", filePath, fileType, startLine, {
154
+ signature: match[0].split("\n")[0].trim(),
155
+ text: match[0].split("\n")[0].trim(),
156
+ metadata: { method, path },
157
+ }));
158
+ }
159
+ }
160
+ return symbols;
161
+ }
162
+ export function extractSymbols(content, filePath, fileType) {
163
+ const ext = extname(filePath).toLowerCase();
164
+ const symbols = [];
165
+ if (fileType === "markdown" || ext === ".md" || ext === ".mdx")
166
+ symbols.push(...extractMarkdownSymbols(content, filePath, fileType));
167
+ if (CONFIG_EXTENSIONS.has(ext) || filePath.split("/").pop()?.startsWith(".env")) {
168
+ symbols.push(...extractConfigSymbols(content, filePath, fileType));
169
+ }
170
+ if (CODE_EXTENSIONS.has(ext) || ["typescript", "javascript", "python", "go", "rust", "java"].includes(fileType)) {
171
+ symbols.push(...extractCodeSymbols(content, filePath, fileType));
172
+ symbols.push(...extractRouteSymbols(content, filePath, fileType));
173
+ }
174
+ return symbols;
175
+ }
@@ -3,7 +3,7 @@ export interface RerankWorkerCandidate {
3
3
  chunkId: string;
4
4
  content: string;
5
5
  }
6
- export declare function rerankInModelWorker(query: string, candidates: RerankWorkerCandidate[], topK: number): Promise<Array<{
6
+ export declare function rerankInModelWorker(query: string, candidates: RerankWorkerCandidate[], topK: number, signal?: AbortSignal): Promise<Array<{
7
7
  chunkId: string;
8
8
  score: number;
9
9
  }>>;
@@ -86,19 +86,19 @@ async function requestModelWorker(message, signal) {
86
86
  const id = nextRequestId++;
87
87
  return new Promise((resolve, reject) => {
88
88
  let abortHandler;
89
+ const cleanup = () => {
90
+ if (abortHandler)
91
+ signal?.removeEventListener("abort", abortHandler);
92
+ };
89
93
  pending.set(id, { resolve, reject });
90
94
  if (signal) {
91
95
  abortHandler = () => {
92
96
  pending.delete(id);
97
+ cleanup();
93
98
  reject(new Error("Cancelled"));
94
- shutdownModelWorker();
95
99
  };
96
100
  signal.addEventListener("abort", abortHandler, { once: true });
97
101
  }
98
- const cleanup = () => {
99
- if (abortHandler)
100
- signal?.removeEventListener("abort", abortHandler);
101
- };
102
102
  const originalResolve = resolve;
103
103
  const originalReject = reject;
104
104
  pending.set(id, {
@@ -130,8 +130,8 @@ export async function embedInModelWorker(texts, prefix, signal) {
130
130
  return new Float32Array(vector);
131
131
  });
132
132
  }
133
- export async function rerankInModelWorker(query, candidates, topK) {
134
- const result = await requestModelWorker({ type: "rerank", query, candidates, topK });
133
+ export async function rerankInModelWorker(query, candidates, topK, signal) {
134
+ const result = await requestModelWorker({ type: "rerank", query, candidates, topK }, signal);
135
135
  if (!Array.isArray(result))
136
136
  throw new Error("Invalid reranker worker response");
137
137
  return result.map((item) => {
@@ -4,7 +4,7 @@ export interface RerankCandidate {
4
4
  chunkId: string;
5
5
  content: string;
6
6
  }
7
- export declare function rerank(query: string, candidates: RerankCandidate[], topK: number): Promise<Array<{
7
+ export declare function rerank(query: string, candidates: RerankCandidate[], topK: number, signal?: AbortSignal): Promise<Array<{
8
8
  chunkId: string;
9
9
  score: number;
10
10
  }>>;
@@ -53,12 +53,12 @@ export async function prepareRerankerForShutdown() {
53
53
  clearTimer();
54
54
  await waitForNoActiveRuns();
55
55
  }
56
- export async function rerank(query, candidates, topK) {
56
+ export async function rerank(query, candidates, topK, signal) {
57
57
  if (candidates.length === 0)
58
58
  return [];
59
59
  beginRun();
60
60
  try {
61
- return await rerankInModelWorker(query, candidates, topK);
61
+ return await rerankInModelWorker(query, candidates, topK, signal);
62
62
  }
63
63
  finally {
64
64
  endRun();
@@ -10,5 +10,5 @@ type ChunkIdSource = Iterable<string | {
10
10
  id: string;
11
11
  }>;
12
12
  export declare function searchVector(queryVec: Float32Array, vectors: Float32Array[], chunkIds: string[], limit?: number): VectorResult[];
13
- export declare function searchVectorFile(queryVec: Float32Array, vectorPath: string, chunkIds: ChunkIdSource, limit?: number): VectorFileSearchResult;
13
+ export declare function searchVectorFile(queryVec: Float32Array, vectorPath: string, chunkIds: ChunkIdSource, limit?: number, signal?: AbortSignal): VectorFileSearchResult;
14
14
  export {};
@@ -25,7 +25,9 @@ export function searchVector(queryVec, vectors, chunkIds, limit = 50) {
25
25
  scores.sort((a, b) => b.score - a.score);
26
26
  return scores.slice(0, limit);
27
27
  }
28
- export function searchVectorFile(queryVec, vectorPath, chunkIds, limit = 50) {
28
+ export function searchVectorFile(queryVec, vectorPath, chunkIds, limit = 50, signal) {
29
+ if (signal?.aborted)
30
+ throw new Error("Cancelled");
29
31
  const reader = openVectorReader(vectorPath);
30
32
  if (!reader)
31
33
  return { results: [], vectorsByChunkId: new Map() };
@@ -36,6 +38,8 @@ export function searchVectorFile(queryVec, vectorPath, chunkIds, limit = 50) {
36
38
  const top = [];
37
39
  let i = 0;
38
40
  for (const chunkIdRef of chunkIds) {
41
+ if (signal?.aborted)
42
+ throw new Error("Cancelled");
39
43
  if (i >= reader.count)
40
44
  break;
41
45
  const vectorIndex = i;
@@ -45,6 +45,23 @@ export interface Chunk {
45
45
  indexed_at: number;
46
46
  }
47
47
  export type ChunkInsert = Omit<Chunk, "id" | "kb_id" | "indexed_at">;
48
+ export interface KnowledgeSymbol {
49
+ id: string;
50
+ kb_id: string;
51
+ name: string;
52
+ normalized_name: string;
53
+ kind: "function" | "class" | "interface" | "type" | "variable" | "heading" | "config_key" | "env_var" | "route";
54
+ file_path: string;
55
+ file_type: string;
56
+ start_line: number;
57
+ end_line: number;
58
+ container_name: string | null;
59
+ signature: string | null;
60
+ text: string;
61
+ metadata_json: string;
62
+ indexed_at: number;
63
+ }
64
+ export type KnowledgeSymbolInsert = Omit<KnowledgeSymbol, "id" | "kb_id" | "indexed_at" | "normalized_name">;
48
65
  export declare function getDefaultKnowledgeDir(): string;
49
66
  export declare function resolveHostKnowledgeDir(hostRoot: string, options?: {
50
67
  legacyPiDir?: string;
@@ -78,7 +95,7 @@ export declare function updateIndexingJob(db: Database.Database, kbId: string, p
78
95
  }): void;
79
96
  export declare function finishIndexingJob(db: Database.Database, kbId: string, status: Extract<IndexingJob["status"], "succeeded" | "failed" | "cancelled">, message: string, errorMessage?: string): void;
80
97
  export declare function getIndexingJob(db: Database.Database, kbId: string): IndexingJob | undefined;
81
- export declare function insertChunks(db: Database.Database, kbId: string, chunks: ChunkInsert[]): void;
98
+ export declare function insertChunks(db: Database.Database, kbId: string, chunks: ChunkInsert[]): string[];
82
99
  export declare function getChunksByKB(db: Database.Database, kbId: string): Chunk[];
83
100
  export declare function iterateChunksByKB(db: Database.Database, kbId: string): IterableIterator<Chunk>;
84
101
  export declare function getChunkIdsByKB(db: Database.Database, kbId: string): string[];
@@ -97,3 +114,21 @@ export declare function iterateChunkHashesByKB(db: Database.Database, kbId: stri
97
114
  }>;
98
115
  export declare function getChunkCount(db: Database.Database, kbId: string): number;
99
116
  export declare function getFileCount(db: Database.Database, kbId: string): number;
117
+ export declare function insertSymbols(db: Database.Database, kbId: string, symbols: KnowledgeSymbolInsert[]): void;
118
+ export declare function deleteSymbolsByKB(db: Database.Database, kbId: string): void;
119
+ export declare function deleteSymbolsByFile(db: Database.Database, kbId: string, filePath: string): void;
120
+ export declare function searchSymbols(db: Database.Database, query: string, options?: {
121
+ kbId?: string;
122
+ kind?: KnowledgeSymbol["kind"];
123
+ filePattern?: string;
124
+ limit?: number;
125
+ offset?: number;
126
+ exact?: boolean;
127
+ }): KnowledgeSymbol[];
128
+ export declare function countSymbols(db: Database.Database, query: string, options?: {
129
+ kbId?: string;
130
+ kind?: KnowledgeSymbol["kind"];
131
+ filePattern?: string;
132
+ exact?: boolean;
133
+ }): number;
134
+ export declare function getSymbolCount(db: Database.Database, kbId: string): number;