pi-knowledge 0.4.7 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -4,7 +4,7 @@ import { createRequire } from "node:module";
4
4
  import { homedir } from "node:os";
5
5
  import { basename, dirname, join } from "node:path";
6
6
  import { fileURLToPath } from "node:url";
7
- const SCHEMA_VERSION = 3;
7
+ const SCHEMA_VERSION = 4;
8
8
  const ITERATION_BATCH_SIZE = 500;
9
9
  const SCHEMA_SQL = `
10
10
  CREATE TABLE IF NOT EXISTS schema_version (
@@ -64,6 +64,28 @@ CREATE TABLE IF NOT EXISTS indexing_jobs (
64
64
  CREATE INDEX IF NOT EXISTS idx_chunks_kb_id ON chunks(kb_id);
65
65
  CREATE INDEX IF NOT EXISTS idx_chunks_content_hash ON chunks(content_hash);
66
66
 
67
+ CREATE TABLE IF NOT EXISTS symbols (
68
+ id TEXT PRIMARY KEY,
69
+ kb_id TEXT NOT NULL,
70
+ name TEXT NOT NULL,
71
+ normalized_name TEXT NOT NULL,
72
+ kind TEXT NOT NULL,
73
+ file_path TEXT NOT NULL,
74
+ file_type TEXT NOT NULL DEFAULT 'text',
75
+ start_line INTEGER NOT NULL DEFAULT 0,
76
+ end_line INTEGER NOT NULL DEFAULT 0,
77
+ container_name TEXT,
78
+ signature TEXT,
79
+ text TEXT NOT NULL DEFAULT '',
80
+ metadata_json TEXT NOT NULL DEFAULT '{}',
81
+ indexed_at INTEGER NOT NULL,
82
+ FOREIGN KEY (kb_id) REFERENCES knowledge_bases(id) ON DELETE CASCADE
83
+ );
84
+
85
+ CREATE INDEX IF NOT EXISTS idx_symbols_kb_name ON symbols(kb_id, normalized_name);
86
+ CREATE INDEX IF NOT EXISTS idx_symbols_kb_kind ON symbols(kb_id, kind);
87
+ CREATE INDEX IF NOT EXISTS idx_symbols_file ON symbols(kb_id, file_path);
88
+
67
89
  CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
68
90
  content_tokenized,
69
91
  content=chunks,
@@ -239,6 +261,31 @@ CREATE TABLE IF NOT EXISTS indexing_jobs (
239
261
  }
240
262
  continue;
241
263
  }
264
+ if (v === 4) {
265
+ db.exec(`
266
+ CREATE TABLE IF NOT EXISTS symbols (
267
+ id TEXT PRIMARY KEY,
268
+ kb_id TEXT NOT NULL,
269
+ name TEXT NOT NULL,
270
+ normalized_name TEXT NOT NULL,
271
+ kind TEXT NOT NULL,
272
+ file_path TEXT NOT NULL,
273
+ file_type TEXT NOT NULL DEFAULT 'text',
274
+ start_line INTEGER NOT NULL DEFAULT 0,
275
+ end_line INTEGER NOT NULL DEFAULT 0,
276
+ container_name TEXT,
277
+ signature TEXT,
278
+ text TEXT NOT NULL DEFAULT '',
279
+ metadata_json TEXT NOT NULL DEFAULT '{}',
280
+ indexed_at INTEGER NOT NULL,
281
+ FOREIGN KEY (kb_id) REFERENCES knowledge_bases(id) ON DELETE CASCADE
282
+ );
283
+ CREATE INDEX IF NOT EXISTS idx_symbols_kb_name ON symbols(kb_id, normalized_name);
284
+ CREATE INDEX IF NOT EXISTS idx_symbols_kb_kind ON symbols(kb_id, kind);
285
+ CREATE INDEX IF NOT EXISTS idx_symbols_file ON symbols(kb_id, file_path);
286
+ `);
287
+ continue;
288
+ }
242
289
  if (migrations[v])
243
290
  db.exec(migrations[v]);
244
291
  }
@@ -265,6 +312,7 @@ export function listKBs(db) {
265
312
  }
266
313
  export function deleteKB(db, id) {
267
314
  db.prepare("DELETE FROM indexing_jobs WHERE kb_id = ?").run(id);
315
+ db.prepare("DELETE FROM symbols WHERE kb_id = ?").run(id);
268
316
  db.prepare("DELETE FROM chunks WHERE kb_id = ?").run(id);
269
317
  db.prepare("DELETE FROM knowledge_bases WHERE id = ?").run(id);
270
318
  }
@@ -332,15 +380,19 @@ export function getIndexingJob(db, kbId) {
332
380
  }
333
381
  // --- CRUD: Chunks ---
334
382
  export function insertChunks(db, kbId, chunks) {
383
+ const ids = [];
335
384
  const stmt = db.prepare(`INSERT INTO chunks (id, kb_id, content_hash, content, content_tokenized, file_path, file_type, start_line, end_line, metadata_json, indexed_at)
336
385
  VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
337
386
  const now = Date.now();
338
387
  const insertMany = db.transaction((items) => {
339
388
  for (const c of items) {
340
- stmt.run(randomUUID(), kbId, c.content_hash, c.content, c.content_tokenized, c.file_path, c.file_type, c.start_line, c.end_line, c.metadata_json, now);
389
+ const id = randomUUID();
390
+ ids.push(id);
391
+ stmt.run(id, kbId, c.content_hash, c.content, c.content_tokenized, c.file_path, c.file_type, c.start_line, c.end_line, c.metadata_json, now);
341
392
  }
342
393
  });
343
394
  insertMany(chunks);
395
+ return ids;
344
396
  }
345
397
  export function getChunksByKB(db, kbId) {
346
398
  return db.prepare("SELECT * FROM chunks WHERE kb_id = ? ORDER BY rowid").all(kbId);
@@ -447,3 +499,108 @@ export function getFileCount(db, kbId) {
447
499
  const row = db.prepare("SELECT COUNT(DISTINCT file_path) as count FROM chunks WHERE kb_id = ?").get(kbId);
448
500
  return row.count;
449
501
  }
502
+ // --- CRUD: Symbols ---
503
+ function normalizeSymbolName(name) {
504
+ return name.trim().toLowerCase();
505
+ }
506
+ function escapeLike(value) {
507
+ return value.replaceAll("\\", "\\\\").replaceAll("%", "\\%").replaceAll("_", "\\_");
508
+ }
509
+ export function insertSymbols(db, kbId, symbols) {
510
+ if (symbols.length === 0)
511
+ return;
512
+ const stmt = db.prepare(`INSERT INTO symbols (
513
+ id, kb_id, name, normalized_name, kind, file_path, file_type, start_line, end_line,
514
+ container_name, signature, text, metadata_json, indexed_at
515
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
516
+ const now = Date.now();
517
+ const insertMany = db.transaction((items) => {
518
+ for (const symbol of items) {
519
+ stmt.run(randomUUID(), kbId, symbol.name, normalizeSymbolName(symbol.name), symbol.kind, symbol.file_path, symbol.file_type, symbol.start_line, symbol.end_line, symbol.container_name ?? null, symbol.signature ?? null, symbol.text, symbol.metadata_json, now);
520
+ }
521
+ });
522
+ insertMany(symbols);
523
+ }
524
+ export function deleteSymbolsByKB(db, kbId) {
525
+ db.prepare("DELETE FROM symbols WHERE kb_id = ?").run(kbId);
526
+ }
527
+ export function deleteSymbolsByFile(db, kbId, filePath) {
528
+ db.prepare("DELETE FROM symbols WHERE kb_id = ? AND file_path = ?").run(kbId, filePath);
529
+ }
530
+ export function searchSymbols(db, query, options = {}) {
531
+ const clauses = ["kb.status IN ('ready', 'stale')"];
532
+ const params = [];
533
+ const normalizedQuery = normalizeSymbolName(query);
534
+ if (options.kbId) {
535
+ clauses.push("s.kb_id = ?");
536
+ params.push(options.kbId);
537
+ }
538
+ if (options.kind) {
539
+ clauses.push("s.kind = ?");
540
+ params.push(options.kind);
541
+ }
542
+ if (options.filePattern) {
543
+ clauses.push("s.file_path LIKE ? ESCAPE '\\'");
544
+ params.push(`%${escapeLike(options.filePattern)}%`);
545
+ }
546
+ if (normalizedQuery) {
547
+ if (options.exact) {
548
+ clauses.push("s.normalized_name = ?");
549
+ params.push(normalizedQuery);
550
+ }
551
+ else {
552
+ clauses.push("(s.normalized_name LIKE ? ESCAPE '\\' OR s.file_path LIKE ? ESCAPE '\\' OR s.text LIKE ? ESCAPE '\\')");
553
+ const pattern = `%${escapeLike(normalizedQuery)}%`;
554
+ params.push(pattern, pattern, pattern);
555
+ }
556
+ }
557
+ const where = `WHERE ${clauses.join(" AND ")}`;
558
+ const limit = Math.max(1, Math.min(200, options.limit ?? 20));
559
+ const offset = Math.max(0, options.offset ?? 0);
560
+ return db
561
+ .prepare(`SELECT s.* FROM symbols s
562
+ JOIN knowledge_bases kb ON kb.id = s.kb_id
563
+ ${where}
564
+ ORDER BY
565
+ CASE WHEN s.normalized_name = ? THEN 0 WHEN s.normalized_name LIKE ? ESCAPE '\\' THEN 1 ELSE 2 END,
566
+ s.file_path, s.start_line
567
+ LIMIT ? OFFSET ?`)
568
+ .all(...params, normalizedQuery, `${escapeLike(normalizedQuery)}%`, limit, offset);
569
+ }
570
+ export function countSymbols(db, query, options = {}) {
571
+ const clauses = ["kb.status IN ('ready', 'stale')"];
572
+ const params = [];
573
+ const normalizedQuery = normalizeSymbolName(query);
574
+ if (options.kbId) {
575
+ clauses.push("s.kb_id = ?");
576
+ params.push(options.kbId);
577
+ }
578
+ if (options.kind) {
579
+ clauses.push("s.kind = ?");
580
+ params.push(options.kind);
581
+ }
582
+ if (options.filePattern) {
583
+ clauses.push("s.file_path LIKE ? ESCAPE '\\'");
584
+ params.push(`%${escapeLike(options.filePattern)}%`);
585
+ }
586
+ if (normalizedQuery) {
587
+ if (options.exact) {
588
+ clauses.push("s.normalized_name = ?");
589
+ params.push(normalizedQuery);
590
+ }
591
+ else {
592
+ clauses.push("(s.normalized_name LIKE ? ESCAPE '\\' OR s.file_path LIKE ? ESCAPE '\\' OR s.text LIKE ? ESCAPE '\\')");
593
+ const pattern = `%${escapeLike(normalizedQuery)}%`;
594
+ params.push(pattern, pattern, pattern);
595
+ }
596
+ }
597
+ const where = `WHERE ${clauses.join(" AND ")}`;
598
+ const row = db
599
+ .prepare(`SELECT COUNT(*) as count FROM symbols s JOIN knowledge_bases kb ON kb.id = s.kb_id ${where}`)
600
+ .get(...params);
601
+ return row.count;
602
+ }
603
+ export function getSymbolCount(db, kbId) {
604
+ const row = db.prepare("SELECT COUNT(*) as count FROM symbols WHERE kb_id = ?").get(kbId);
605
+ return row.count;
606
+ }
@@ -1,4 +1,5 @@
1
- export declare function startWatcher(kbId: string, dirPath: string, onUpdate: (kbId: string) => void): void;
1
+ import { type ScanOptions } from "../indexer/chunker.ts";
2
+ export declare function startWatcher(kbId: string, dirPath: string, onUpdate: (kbId: string) => void, options?: ScanOptions): void;
2
3
  export declare function stopWatcher(kbId: string): void;
3
4
  export declare function stopAllWatchers(): void;
4
5
  export declare function getActiveWatcherCount(): number;
@@ -1,5 +1,5 @@
1
- import { existsSync, readdirSync, statSync, watch } from "node:fs";
2
- import { join } from "node:path";
1
+ import { existsSync, statSync, watch } from "node:fs";
2
+ import { createSkippedScanStats, iterateScannableFiles } from "../indexer/chunker.js";
3
3
  const watchers = new Map();
4
4
  const pollers = new Map();
5
5
  const snapshots = new Map();
@@ -15,34 +15,20 @@ function scheduleUpdate(kbId, onUpdate) {
15
15
  onUpdate(kbId);
16
16
  }, DEBOUNCE_MS));
17
17
  }
18
- function scanSnapshot(dirPath) {
18
+ function scanSnapshot(dirPath, options = {}) {
19
19
  const snapshot = new Map();
20
- function scan(dir) {
21
- let entries;
20
+ if (!existsSync(dirPath))
21
+ return snapshot;
22
+ const skipped = createSkippedScanStats();
23
+ for (const file of iterateScannableFiles(dirPath, skipped, options)) {
22
24
  try {
23
- entries = readdirSync(dir, { withFileTypes: true });
25
+ const stat = statSync(file.path);
26
+ snapshot.set(file.path, `${stat.mtimeMs}:${stat.size}`);
24
27
  }
25
28
  catch {
26
- return;
27
- }
28
- for (const entry of entries) {
29
- const fullPath = join(dir, entry.name);
30
- try {
31
- const stat = statSync(fullPath);
32
- if (entry.isDirectory()) {
33
- scan(fullPath);
34
- }
35
- else if (entry.isFile()) {
36
- snapshot.set(fullPath, `${stat.mtimeMs}:${stat.size}`);
37
- }
38
- }
39
- catch {
40
- /* file disappeared or is unreadable */
41
- }
29
+ /* file disappeared or is unreadable */
42
30
  }
43
31
  }
44
- if (existsSync(dirPath))
45
- scan(dirPath);
46
32
  return snapshot;
47
33
  }
48
34
  function snapshotsDiffer(a, b) {
@@ -54,23 +40,27 @@ function snapshotsDiffer(a, b) {
54
40
  }
55
41
  return false;
56
42
  }
57
- function startPoller(kbId, dirPath, onUpdate) {
58
- snapshots.set(kbId, scanSnapshot(dirPath));
43
+ function startPoller(kbId, dirPath, onUpdate, options = {}) {
44
+ snapshots.set(kbId, scanSnapshot(dirPath, options));
59
45
  pollers.set(kbId, setInterval(() => {
60
46
  const previous = snapshots.get(kbId) ?? new Map();
61
- const next = scanSnapshot(dirPath);
47
+ const next = scanSnapshot(dirPath, options);
62
48
  if (snapshotsDiffer(previous, next)) {
63
49
  snapshots.set(kbId, next);
64
50
  scheduleUpdate(kbId, onUpdate);
65
51
  }
66
52
  }, POLL_MS));
67
53
  }
68
- export function startWatcher(kbId, dirPath, onUpdate) {
54
+ export function startWatcher(kbId, dirPath, onUpdate, options = {}) {
69
55
  stopWatcher(kbId);
70
- startPoller(kbId, dirPath, onUpdate);
56
+ startPoller(kbId, dirPath, onUpdate, options);
71
57
  try {
72
58
  const watcher = watch(dirPath, { recursive: true }, () => {
73
- snapshots.set(kbId, scanSnapshot(dirPath));
59
+ const previous = snapshots.get(kbId) ?? new Map();
60
+ const next = scanSnapshot(dirPath, options);
61
+ if (!snapshotsDiffer(previous, next))
62
+ return;
63
+ snapshots.set(kbId, next);
74
64
  scheduleUpdate(kbId, onUpdate);
75
65
  });
76
66
  watcher.on("error", () => {
@@ -26,8 +26,8 @@ Default data storage is `~/.pi/knowledge` under Pi and `~/.omp/knowledge` under
26
26
  | `OPENAI_BASE_URL` | `https://api.openai.com/v1` | Common OpenAI-compatible API root fallback. |
27
27
  | `PI_KNOWLEDGE_EMBEDDING_MAX_CHARS` | `20000` | Final per-input API embedding safety cap for OpenAI-compatible servers with smaller context windows. This does not replace chunker bounds. |
28
28
  | `PI_KNOWLEDGE_EMBEDDING_API_FALLBACK` | unset | Set to `local` to explicitly fall back to local embeddings after API failures. Without this, API failures are surfaced. |
29
- | `PI_KNOWLEDGE_ENABLE_NATIVE_IDLE_DISPOSE` | unset | Set to `true` to opt into native ONNX idle disposal. Disabled by default for stable shutdown. |
30
- | `PI_KNOWLEDGE_EMBEDDING_IDLE_MS` | `30000` | Idle-dispose timer used only when native idle disposal is enabled. Mainly for lifecycle stress tests. |
29
+ | `PI_KNOWLEDGE_ENABLE_NATIVE_IDLE_DISPOSE` | unset | Enables idle timers for embedding/reranker run coordination. It does not guarantee immediate worker memory release; model-worker shutdown remains tied to engine/session shutdown. Disabled by default for stable shutdown. |
30
+ | `PI_KNOWLEDGE_EMBEDDING_IDLE_MS` | `30000` | Idle timer used only when native idle disposal coordination is enabled. Mainly for lifecycle stress tests. |
31
31
  | `PI_KNOWLEDGE_OFFLINE` | unset | Use with a pre-populated model cache for offline local model operation. See `docs/offline-mode.md`. |
32
32
 
33
33
  API embedding failures intentionally surface by default. Silent fallback can hide bad API keys, wrong base URLs, unsupported model names, or context-window errors and can produce a KB with a different embedding model than intended.
@@ -40,6 +40,14 @@ API embedding failures intentionally surface by default. Silent fallback can hid
40
40
  | `PI_KNOWLEDGE_AUTO_INJECT` | unset | Set to `true` to auto-search KB context before model calls. This is opt-in. |
41
41
  | `PI_KNOWLEDGE_STALE_INDEXING_MS` | built-in stale threshold | Override the stale indexing threshold used by diagnostics and `knowledge_doctor`. |
42
42
 
43
+ Search diagnostics include result provenance in both text output and structured `details`: chunk id, chunk hash, match reason, indexed timestamp, stale flag, and source mtime when the original source file is available. This provenance is stored in SQLite and derived from indexed source metadata; it does not create or modify files in the indexed project.
44
+
45
+ `knowledge_search` supports `file_type` and `path_pattern` filters. `hybrid` mode is lexical-anchored to prevent low-evidence semantic false positives; use `semantic` when the query is conceptual and exact terms may differ from indexed wording.
46
+
47
+ `knowledge_doctor` emits both human-readable issues and machine-readable actions in structured `details`. Action codes include `run_update`, `rebuild_kb`, `wait_for_indexing`, `review_skipped_scope`, `rebuild_vectors`, `check_source`, and `none`. Agents should prefer these action codes over parsing the English action text.
48
+
49
+ `knowledge_symbol_search` uses a lightweight declaration-pattern index stored in `knowledge.db`. It is rebuilt during `knowledge_add`, `knowledge_update`, and `knowledge_import`, and can find common code symbols, route-like handlers, Markdown headings, config keys, and environment variables. It is not a full LSP or code graph; for methods, uncommon syntax, or empty important lookups, use `knowledge_search` mode `fast`/`adaptive` and check `knowledge_doctor` for stale or missing symbol metadata. Older file/directory/URL KBs without symbol metadata should be updated before relying on symbol lookup. Imported portable KBs and inline text KBs do not retain an active source path; rebuild their symbols by re-importing or re-adding the content.
50
+
43
51
  ## Test and Release Fixtures
44
52
 
45
53
  | Variable | Default | Purpose |
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "pi-knowledge",
3
- "version": "0.4.7",
3
+ "version": "0.5.1",
4
4
  "description": "Local-first RAG knowledge base for Pi and OMP coding agents with hybrid code/doc search, reranking, diagnostics, and large-project indexing",
5
5
  "type": "module",
6
6
  "main": "dist/index.js",