pi-knowledge 0.4.6 → 0.4.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/README.md +10 -6
- package/dist/src/engine.d.ts +2 -0
- package/dist/src/engine.js +20 -0
- package/dist/src/storage/sqlite.js +54 -6
- package/package.json +4 -2
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,15 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [0.4.7] - 2026-07-04
|
|
4
|
+
|
|
5
|
+
### Fixed
|
|
6
|
+
- Avoided live SQLite statement iterators in chunk row scans so search, diagnostics, and vector rebuild paths cannot leave the database connection busy before `knowledge_update`.
|
|
7
|
+
- Coalesced overlapping `knowledge_update` calls per knowledge base so watcher-triggered updates, manual updates, retries, and shutdown do not re-enter the same update flow.
|
|
8
|
+
|
|
9
|
+
### Changed
|
|
10
|
+
- Increased SQLite busy timeout to tolerate short-lived external contention during long indexing/update work.
|
|
11
|
+
- Updated npm-facing README and package metadata to show Pi and OMP (`omp.sh`) coding-agent support.
|
|
12
|
+
|
|
3
13
|
## [0.4.6] - 2026-06-26
|
|
4
14
|
|
|
5
15
|
### Fixed
|
package/README.md
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
# pi-knowledge
|
|
2
2
|
|
|
3
|
-
Local-first RAG knowledge base for Pi agents.
|
|
3
|
+
Local-first RAG knowledge base for Pi and OMP coding agents.
|
|
4
4
|
|
|
5
|
-
Index your codebase, docs, PDFs, URLs, and notes into persistent knowledge bases that Pi can search across sessions. `pi-knowledge` combines semantic embeddings, BM25 keyword search, code-aware chunking, reranking, diagnostics, and stable large-project indexing so agents can answer from your actual project knowledge instead of guessing.
|
|
5
|
+
Index your codebase, docs, PDFs, URLs, and notes into persistent knowledge bases that Pi and [OMP](https://omp.sh/) can search across sessions. `pi-knowledge` combines semantic embeddings, BM25 keyword search, code-aware chunking, reranking, diagnostics, and stable large-project indexing so agents can answer from your actual project knowledge instead of guessing.
|
|
6
6
|
|
|
7
|
-
Built as a native [Pi extension](https://pi.dev/docs/latest/extensions)
|
|
7
|
+
Built as a native [Pi extension](https://pi.dev/docs/latest/extensions) with verified [OMP](https://omp.sh/) compatibility through the packaged extension entry. Designed for local-first project memory, agentic code search, and retrieval-augmented development workflows.
|
|
8
8
|
|
|
9
9
|
## Why It Matters
|
|
10
10
|
|
|
11
|
-
Agents lose project context between sessions and cannot fit large repositories into one prompt. `pi-knowledge` gives Pi durable, searchable project memory:
|
|
11
|
+
Agents lose project context between sessions and cannot fit large repositories into one prompt. `pi-knowledge` gives Pi and OMP durable, searchable project memory:
|
|
12
12
|
|
|
13
13
|
- Search code and documentation by meaning, exact symbols, or both.
|
|
14
14
|
- Keep private source data local by default.
|
|
@@ -74,11 +74,15 @@ In project-level dogfood, these changes improved a real codebase evaluation from
|
|
|
74
74
|
## Quick Start
|
|
75
75
|
|
|
76
76
|
```bash
|
|
77
|
-
# Install
|
|
77
|
+
# Install for Pi
|
|
78
78
|
pi install npm:pi-knowledge
|
|
79
79
|
|
|
80
|
+
# Install for OMP
|
|
81
|
+
omp install npm:pi-knowledge
|
|
82
|
+
|
|
80
83
|
# Or from source
|
|
81
84
|
pi install ./pi-knowledge
|
|
85
|
+
omp install ./pi-knowledge
|
|
82
86
|
|
|
83
87
|
# Index a directory
|
|
84
88
|
# (agent will call knowledge_add automatically, or you can ask it)
|
|
@@ -170,7 +174,7 @@ Full configuration details are in [docs/configuration.md](docs/configuration.md)
|
|
|
170
174
|
|
|
171
175
|
## Pi and OMP Support
|
|
172
176
|
|
|
173
|
-
`pi-knowledge` supports Pi and OMP
|
|
177
|
+
`pi-knowledge` supports Pi and [OMP](https://omp.sh/) extension loading through the packaged `extension.js` entry shim. The entry stays startup-light: install-time validation can inspect the extension without resolving native runtime dependencies, and runtime modules load lazily only when tools or lifecycle hooks need them.
|
|
174
178
|
|
|
175
179
|
Default storage is `~/.pi/knowledge` for Pi and `~/.omp/knowledge` for OMP. Explicit overrides are available with `PI_KNOWLEDGE_DIR` and `OMP_KNOWLEDGE_DIR`. Under the default home OMP root, existing legacy `~/.pi/knowledge` data remains visible when `~/.omp/knowledge` has not been created yet.
|
|
176
180
|
|
package/dist/src/engine.d.ts
CHANGED
|
@@ -62,6 +62,7 @@ export interface IndexPlan {
|
|
|
62
62
|
export declare class KnowledgeEngine {
|
|
63
63
|
private db;
|
|
64
64
|
private knowledgeDir;
|
|
65
|
+
private activeUpdates;
|
|
65
66
|
initialize(knowledgeDir: string): Promise<void>;
|
|
66
67
|
plan(source: string, options?: AddOptions): IndexPlan;
|
|
67
68
|
add(source: string, name: string, onProgress?: ProgressCallback, signal?: AbortSignal, options?: AddOptions): Promise<{
|
|
@@ -73,6 +74,7 @@ export declare class KnowledgeEngine {
|
|
|
73
74
|
removed: number;
|
|
74
75
|
unchanged: number;
|
|
75
76
|
}>;
|
|
77
|
+
private runUpdate;
|
|
76
78
|
search(query: string, options?: SearchOptions): Promise<SearchResponse>;
|
|
77
79
|
remove(nameOrId: string): boolean;
|
|
78
80
|
list(): KnowledgeBase[];
|
package/dist/src/engine.js
CHANGED
|
@@ -350,6 +350,7 @@ function normalizeExtractedText(text) {
|
|
|
350
350
|
export class KnowledgeEngine {
|
|
351
351
|
db = null;
|
|
352
352
|
knowledgeDir = "";
|
|
353
|
+
activeUpdates = new Map();
|
|
353
354
|
async initialize(knowledgeDir) {
|
|
354
355
|
this.knowledgeDir = knowledgeDir;
|
|
355
356
|
this.db = openDatabase(knowledgeDir);
|
|
@@ -614,6 +615,22 @@ export class KnowledgeEngine {
|
|
|
614
615
|
if (!kb.source_path || (kb.source_type !== "url" && !existsSync(kb.source_path))) {
|
|
615
616
|
throw new Error(`Source path not available or missing: ${kb.source_path}`);
|
|
616
617
|
}
|
|
618
|
+
const updateableKB = { ...kb, source_path: kb.source_path };
|
|
619
|
+
const activeUpdate = this.activeUpdates.get(updateableKB.id);
|
|
620
|
+
if (activeUpdate) {
|
|
621
|
+
onProgress?.(`Update already running for "${updateableKB.name}"; waiting for the active update.`);
|
|
622
|
+
return activeUpdate;
|
|
623
|
+
}
|
|
624
|
+
const updateRun = this.runUpdate(updateableKB, onProgress, signal).finally(() => {
|
|
625
|
+
if (this.activeUpdates.get(updateableKB.id) === updateRun)
|
|
626
|
+
this.activeUpdates.delete(updateableKB.id);
|
|
627
|
+
});
|
|
628
|
+
this.activeUpdates.set(updateableKB.id, updateRun);
|
|
629
|
+
return updateRun;
|
|
630
|
+
}
|
|
631
|
+
async runUpdate(kb, onProgress, signal) {
|
|
632
|
+
if (!this.db)
|
|
633
|
+
throw new Error("Engine not initialized");
|
|
617
634
|
updateKBStatus(this.db, kb.id, "indexing");
|
|
618
635
|
startIndexingJob(this.db, kb.id, "update", `Starting update for "${kb.name}"`);
|
|
619
636
|
const scanOptions = toScanOptions(parseAddOptions(kb.source_options));
|
|
@@ -1354,6 +1371,9 @@ export class KnowledgeEngine {
|
|
|
1354
1371
|
}
|
|
1355
1372
|
}
|
|
1356
1373
|
async dispose(options = {}) {
|
|
1374
|
+
const activeUpdates = [...this.activeUpdates.values()];
|
|
1375
|
+
if (activeUpdates.length > 0)
|
|
1376
|
+
await Promise.allSettled(activeUpdates);
|
|
1357
1377
|
const disposeModels = options.disposeModels ?? true;
|
|
1358
1378
|
if (disposeModels) {
|
|
1359
1379
|
await disposeEmbedding();
|
|
@@ -5,6 +5,7 @@ import { homedir } from "node:os";
|
|
|
5
5
|
import { basename, dirname, join } from "node:path";
|
|
6
6
|
import { fileURLToPath } from "node:url";
|
|
7
7
|
const SCHEMA_VERSION = 3;
|
|
8
|
+
const ITERATION_BATCH_SIZE = 500;
|
|
8
9
|
const SCHEMA_SQL = `
|
|
9
10
|
CREATE TABLE IF NOT EXISTS schema_version (
|
|
10
11
|
version INTEGER NOT NULL
|
|
@@ -189,7 +190,7 @@ export function openDatabase(knowledgeDir) {
|
|
|
189
190
|
const nativeBinding = isBunRuntime() ? undefined : findBetterSqlite3NativeBinding();
|
|
190
191
|
const db = nativeBinding ? new Database(dbPath, { nativeBinding }) : new Database(dbPath);
|
|
191
192
|
applyPragma(db, "journal_mode = WAL");
|
|
192
|
-
applyPragma(db, "busy_timeout =
|
|
193
|
+
applyPragma(db, "busy_timeout = 60000");
|
|
193
194
|
applyPragma(db, "foreign_keys = ON");
|
|
194
195
|
const hasVersion = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='schema_version'").get();
|
|
195
196
|
if (!hasVersion) {
|
|
@@ -345,14 +346,14 @@ export function getChunksByKB(db, kbId) {
|
|
|
345
346
|
return db.prepare("SELECT * FROM chunks WHERE kb_id = ? ORDER BY rowid").all(kbId);
|
|
346
347
|
}
|
|
347
348
|
export function iterateChunksByKB(db, kbId) {
|
|
348
|
-
return db
|
|
349
|
+
return iterateChunkRowsByKB(db, kbId);
|
|
349
350
|
}
|
|
350
351
|
export function getChunkIdsByKB(db, kbId) {
|
|
351
352
|
const rows = db.prepare("SELECT id FROM chunks WHERE kb_id = ? ORDER BY rowid").all(kbId);
|
|
352
353
|
return rows.map((r) => r.id);
|
|
353
354
|
}
|
|
354
355
|
export function iterateChunkIdsByKB(db, kbId) {
|
|
355
|
-
return db
|
|
356
|
+
return iterateChunkIdRowsByKB(db, kbId);
|
|
356
357
|
}
|
|
357
358
|
export function getChunkById(db, id) {
|
|
358
359
|
return db.prepare("SELECT * FROM chunks WHERE id = ?").get(id);
|
|
@@ -387,9 +388,56 @@ export function getChunkHashesByKB(db, kbId) {
|
|
|
387
388
|
return new Map(rows.map((r) => [r.content_hash, r.id]));
|
|
388
389
|
}
|
|
389
390
|
export function iterateChunkHashesByKB(db, kbId) {
|
|
390
|
-
return db
|
|
391
|
-
|
|
392
|
-
|
|
391
|
+
return iterateChunkHashRowsByKB(db, kbId);
|
|
392
|
+
}
|
|
393
|
+
function* iterateChunkRowsByKB(db, kbId) {
|
|
394
|
+
let lastRowid = 0;
|
|
395
|
+
const statement = db.prepare(`SELECT rowid as rowid, * FROM chunks
|
|
396
|
+
WHERE kb_id = ? AND rowid > ?
|
|
397
|
+
ORDER BY rowid
|
|
398
|
+
LIMIT ?`);
|
|
399
|
+
while (true) {
|
|
400
|
+
const rows = statement.all(kbId, lastRowid, ITERATION_BATCH_SIZE);
|
|
401
|
+
if (rows.length === 0)
|
|
402
|
+
return;
|
|
403
|
+
for (const row of rows) {
|
|
404
|
+
lastRowid = row.rowid;
|
|
405
|
+
const { rowid: _rowid, ...chunk } = row;
|
|
406
|
+
yield chunk;
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
function* iterateChunkIdRowsByKB(db, kbId) {
|
|
411
|
+
let lastRowid = 0;
|
|
412
|
+
const statement = db.prepare(`SELECT rowid, id FROM chunks
|
|
413
|
+
WHERE kb_id = ? AND rowid > ?
|
|
414
|
+
ORDER BY rowid
|
|
415
|
+
LIMIT ?`);
|
|
416
|
+
while (true) {
|
|
417
|
+
const rows = statement.all(kbId, lastRowid, ITERATION_BATCH_SIZE);
|
|
418
|
+
if (rows.length === 0)
|
|
419
|
+
return;
|
|
420
|
+
for (const row of rows) {
|
|
421
|
+
lastRowid = row.rowid;
|
|
422
|
+
yield { id: row.id };
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
function* iterateChunkHashRowsByKB(db, kbId) {
|
|
427
|
+
let lastRowid = 0;
|
|
428
|
+
const statement = db.prepare(`SELECT rowid, id, content_hash FROM chunks
|
|
429
|
+
WHERE kb_id = ? AND rowid > ?
|
|
430
|
+
ORDER BY rowid
|
|
431
|
+
LIMIT ?`);
|
|
432
|
+
while (true) {
|
|
433
|
+
const rows = statement.all(kbId, lastRowid, ITERATION_BATCH_SIZE);
|
|
434
|
+
if (rows.length === 0)
|
|
435
|
+
return;
|
|
436
|
+
for (const row of rows) {
|
|
437
|
+
lastRowid = row.rowid;
|
|
438
|
+
yield { id: row.id, content_hash: row.content_hash };
|
|
439
|
+
}
|
|
440
|
+
}
|
|
393
441
|
}
|
|
394
442
|
export function getChunkCount(db, kbId) {
|
|
395
443
|
const row = db.prepare("SELECT COUNT(*) as count FROM chunks WHERE kb_id = ?").get(kbId);
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-knowledge",
|
|
3
|
-
"version": "0.4.
|
|
4
|
-
"description": "Local-first RAG knowledge base for Pi agents with hybrid code/doc search, reranking, diagnostics, and large-project indexing",
|
|
3
|
+
"version": "0.4.7",
|
|
4
|
+
"description": "Local-first RAG knowledge base for Pi and OMP coding agents with hybrid code/doc search, reranking, diagnostics, and large-project indexing",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
7
7
|
"types": "dist/index.d.ts",
|
|
@@ -24,6 +24,8 @@
|
|
|
24
24
|
"pi-package",
|
|
25
25
|
"pi",
|
|
26
26
|
"pi-extension",
|
|
27
|
+
"omp",
|
|
28
|
+
"omp-sh",
|
|
27
29
|
"ai-agent",
|
|
28
30
|
"coding-agent",
|
|
29
31
|
"knowledge-base",
|