pi-mega-compact 0.4.21 → 0.4.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/extensions/dashboard-server.js +3 -3
- package/dist/extensions/mega-compact-driver.js +79 -0
- package/dist/extensions/mega-compact.test.js +54 -18
- package/dist/extensions/mega-config.js +10 -0
- package/dist/extensions/mega-events.js +45 -23
- package/dist/extensions/mega-pipeline.js +77 -3
- package/dist/src/config/dedup.js +4 -1
- package/dist/src/config.js +21 -0
- package/dist/src/dedup/raptor/index.js +28 -6
- package/dist/src/dedup/raptor/promote.test.js +69 -0
- package/dist/src/engine.js +1 -0
- package/dist/src/recall.js +30 -4
- package/dist/src/recall.test.js +28 -0
- package/dist/src/store/backfill.js +5 -6
- package/dist/src/store/compression.js +47 -7
- package/dist/src/store/compression.test.js +48 -0
- package/dist/src/store/sqlite.js +64 -41
- package/dist/src/store.test.js +19 -0
- package/dist/src/vectorStore.js +56 -1
- package/extensions/DASHBOARD.md +3 -3
- package/extensions/dashboard-server.ts +4 -4
- package/extensions/mega-compact-driver.ts +105 -0
- package/extensions/mega-compact.test.ts +65 -18
- package/extensions/mega-config.ts +25 -0
- package/extensions/mega-events.ts +43 -24
- package/extensions/mega-pipeline.ts +83 -4
- package/package.json +6 -7
- package/src/config/dedup.ts +4 -1
- package/src/config.ts +26 -0
- package/src/dedup/raptor/index.ts +42 -7
- package/src/dedup/raptor/promote.test.ts +82 -0
- package/src/engine.ts +5 -0
- package/src/recall.test.ts +44 -0
- package/src/recall.ts +43 -4
- package/src/store/backfill.ts +10 -11
- package/src/store/compression.test.ts +58 -0
- package/src/store/compression.ts +48 -7
- package/src/store/sqlite.ts +72 -49
- package/src/store.test.ts +22 -0
- package/src/vectorStore.ts +63 -1
- package/dist/extensions/openclaw-mega-compact.js +0 -291
- package/dist/src/minilm.js +0 -92
- package/dist/src/wordpiece.js +0 -129
package/src/store/sqlite.ts
CHANGED
|
@@ -2,11 +2,13 @@
|
|
|
2
2
|
* sqlite.ts — Sprint 8 storage backbone (the "one store").
|
|
3
3
|
*
|
|
4
4
|
* Replaces the per-session gzipped-JSON checkpoint files with a single local
|
|
5
|
-
* SQLite database (
|
|
6
|
-
* honors PREVENT-PI-004).
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
5
|
+
* SQLite database (node:sqlite — the Node built-in, in-process, FS-backed,
|
|
6
|
+
* ZERO network calls — honors PREVENT-PI-004). No native build and no install
|
|
7
|
+
* scripts, so it survives pi's npm blocked-install-scripts gate (better-sqlite3's
|
|
8
|
+
* native binary could not be built under pi, which crashed every `pi update
|
|
9
|
+
* --extensions`). node:sqlite is synchronous, so every VectorStore signature
|
|
10
|
+
* stays sync. PGlite + pgvector (async) is layered on in vectorIndex.ts for
|
|
11
|
+
* real HNSW indexing (Slice 2 of the dual-backend plan).
|
|
10
12
|
*
|
|
11
13
|
* FTS5 `trigram` tokenizer is created for the Sprint 9+ dedup tiers (MinHash/LSH
|
|
12
14
|
* / pg_trgm-equivalent verification). The default cosine path stays a linear
|
|
@@ -15,7 +17,7 @@
|
|
|
15
17
|
* All queries are parameterized (PREVENT-002) — never string-concatenated.
|
|
16
18
|
*/
|
|
17
19
|
|
|
18
|
-
import
|
|
20
|
+
import { DatabaseSync } from "node:sqlite";
|
|
19
21
|
import { existsSync, mkdirSync } from "node:fs";
|
|
20
22
|
import { homedir, tmpdir } from "node:os";
|
|
21
23
|
import { join } from "node:path";
|
|
@@ -31,12 +33,15 @@ function encodeEmbedding(v: number[]): Buffer {
|
|
|
31
33
|
for (let i = 0; i < v.length; i++) buf.writeFloatLE(v[i] ?? 0, i * 4);
|
|
32
34
|
return buf;
|
|
33
35
|
}
|
|
34
|
-
/** Decode a Float32 BLOB back to a number[].
|
|
35
|
-
|
|
36
|
+
/** Decode a Float32 BLOB back to a number[]. node:sqlite returns BLOBs as
|
|
37
|
+
* Uint8Array, so decode via DataView (Buffer is a Uint8Array subclass — both
|
|
38
|
+
* work). */
|
|
39
|
+
function decodeEmbedding(buf: Uint8Array | null | undefined): number[] {
|
|
36
40
|
if (!buf || buf.length === 0) return [];
|
|
41
|
+
const dv = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
|
|
37
42
|
const n = buf.length / 4;
|
|
38
43
|
const out = new Array<number>(n);
|
|
39
|
-
for (let i = 0; i < n; i++) out[i] =
|
|
44
|
+
for (let i = 0; i < n; i++) out[i] = dv.getFloat32(i * 4, true);
|
|
40
45
|
return out;
|
|
41
46
|
}
|
|
42
47
|
|
|
@@ -47,17 +52,17 @@ function jsonText(v: unknown): string {
|
|
|
47
52
|
// In-process cache so the same stateDir reuses one connection (and so a fresh
|
|
48
53
|
// VectorStore over the same dir shares the open DB). Cross-process durability
|
|
49
54
|
// comes from reopening the same file path — proven by the integration test.
|
|
50
|
-
const cache = new Map<string,
|
|
55
|
+
const cache = new Map<string, DatabaseSync>();
|
|
51
56
|
|
|
52
57
|
/** Open (or reuse) the SQLite store for a state dir. */
|
|
53
|
-
export function openStore(stateDir: string = getStateDir()):
|
|
58
|
+
export function openStore(stateDir: string = getStateDir()): DatabaseSync {
|
|
54
59
|
const existing = cache.get(stateDir);
|
|
55
60
|
if (existing) return existing;
|
|
56
61
|
|
|
57
62
|
if (!existsSync(stateDir)) mkdirSync(stateDir, { recursive: true });
|
|
58
|
-
const db = new
|
|
59
|
-
db.
|
|
60
|
-
db.
|
|
63
|
+
const db = new DatabaseSync(join(stateDir, "sqlite.db"));
|
|
64
|
+
db.exec("PRAGMA journal_mode = WAL");
|
|
65
|
+
db.exec("PRAGMA foreign_keys = ON");
|
|
61
66
|
initSchema(db);
|
|
62
67
|
cache.set(stateDir, db);
|
|
63
68
|
return db;
|
|
@@ -87,17 +92,17 @@ export function getIndexDir(): string {
|
|
|
87
92
|
}
|
|
88
93
|
}
|
|
89
94
|
|
|
90
|
-
let indexCache:
|
|
95
|
+
let indexCache: DatabaseSync | undefined;
|
|
91
96
|
let indexCacheDir: string | undefined;
|
|
92
97
|
|
|
93
98
|
/** Open (or reuse) the machine-wide index DB. WAL for concurrent writers. */
|
|
94
|
-
export function openIndexStore(indexDir: string = getIndexDir()):
|
|
99
|
+
export function openIndexStore(indexDir: string = getIndexDir()): DatabaseSync {
|
|
95
100
|
if (indexCache && indexCacheDir === indexDir) return indexCache;
|
|
96
101
|
if (!existsSync(indexDir)) mkdirSync(indexDir, { recursive: true });
|
|
97
|
-
const
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
102
|
+
const iddb = new DatabaseSync(join(indexDir, "index.sqlite"));
|
|
103
|
+
iddb.exec("PRAGMA journal_mode = WAL");
|
|
104
|
+
iddb.exec("PRAGMA busy_timeout = 3000"); // tolerate brief cross-process write contention
|
|
105
|
+
iddb.exec(`
|
|
101
106
|
CREATE TABLE IF NOT EXISTS repo_registry (
|
|
102
107
|
repo_root TEXT PRIMARY KEY,
|
|
103
108
|
display_name TEXT,
|
|
@@ -117,9 +122,9 @@ export function openIndexStore(indexDir: string = getIndexDir()): Database.Datab
|
|
|
117
122
|
);
|
|
118
123
|
CREATE INDEX IF NOT EXISTS idx_registry_last_seen ON repo_registry(last_seen DESC);
|
|
119
124
|
`);
|
|
120
|
-
indexCache =
|
|
125
|
+
indexCache = iddb;
|
|
121
126
|
indexCacheDir = indexDir;
|
|
122
|
-
return
|
|
127
|
+
return iddb;
|
|
123
128
|
}
|
|
124
129
|
|
|
125
130
|
/** One row of the global repo registry (multi-repo dashboard source). */
|
|
@@ -276,7 +281,7 @@ export function closeIndexStore(): void {
|
|
|
276
281
|
}
|
|
277
282
|
}
|
|
278
283
|
|
|
279
|
-
function initSchema(db:
|
|
284
|
+
function initSchema(db: DatabaseSync): void {
|
|
280
285
|
db.exec(`
|
|
281
286
|
CREATE TABLE IF NOT EXISTS context_chunks (
|
|
282
287
|
id TEXT NOT NULL,
|
|
@@ -457,7 +462,7 @@ function initSchema(db: Database.Database): void {
|
|
|
457
462
|
* input), so the unavoidable identifier interpolation here does not violate
|
|
458
463
|
* PREVENT-002 (no external data reaches this SQL).
|
|
459
464
|
*/
|
|
460
|
-
function ensureColumn(db:
|
|
465
|
+
function ensureColumn(db: DatabaseSync, table: string, column: string, decl: string): void {
|
|
461
466
|
const cols = db.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>;
|
|
462
467
|
if (cols.some((c) => c.name === column)) return;
|
|
463
468
|
db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${decl}`);
|
|
@@ -666,6 +671,24 @@ function mapMemoryRow(row: any): MemoryRecord {
|
|
|
666
671
|
};
|
|
667
672
|
}
|
|
668
673
|
|
|
674
|
+
/**
|
|
675
|
+
* Run `fn` atomically. Uses SAVEPOINT so it nests safely under an outer
|
|
676
|
+
* transaction (unlike `BEGIN`, which SQLite rejects when one is already open).
|
|
677
|
+
* Mirrors better-sqlite3's `db.transaction(fn)` semantics — callers that wrap a
|
|
678
|
+
* batch in withTx (e.g. backfill) can still call helpers that also use withTx.
|
|
679
|
+
*/
|
|
680
|
+
export function withTx(db: DatabaseSync, fn: () => void): void {
|
|
681
|
+
db.exec("SAVEPOINT mc_tx");
|
|
682
|
+
try {
|
|
683
|
+
fn();
|
|
684
|
+
db.exec("RELEASE mc_tx");
|
|
685
|
+
} catch (e) {
|
|
686
|
+
db.exec("ROLLBACK TO mc_tx");
|
|
687
|
+
db.exec("RELEASE mc_tx");
|
|
688
|
+
throw e;
|
|
689
|
+
}
|
|
690
|
+
}
|
|
691
|
+
|
|
669
692
|
/** Map a DB row to the public StoredCheckpoint shape. */
|
|
670
693
|
function rowToCheckpoint(row: any): StoredCheckpoint {
|
|
671
694
|
return {
|
|
@@ -684,7 +707,9 @@ function rowToCheckpoint(row: any): StoredCheckpoint {
|
|
|
684
707
|
contentHash2: row.content_hash2 ?? undefined,
|
|
685
708
|
contentHashVersion: row.content_hash_version ?? undefined,
|
|
686
709
|
normalizedText: row.normalized_text ?? undefined,
|
|
687
|
-
|
|
710
|
+
// node:sqlite returns BLOBs as Uint8Array; normalize to Buffer so callers
|
|
711
|
+
// (e.g. decompressSmart → Buffer.toString) behave as under better-sqlite3.
|
|
712
|
+
compressedOriginal: row.compressed_original ? Buffer.from(row.compressed_original) : undefined,
|
|
688
713
|
embedding: decodeEmbedding(row.embedding_blob),
|
|
689
714
|
timestamp: Number(row.timestamp ?? 0),
|
|
690
715
|
dedupStatus: row.dedup_status ?? undefined,
|
|
@@ -695,7 +720,7 @@ function rowToCheckpoint(row: any): StoredCheckpoint {
|
|
|
695
720
|
export function upsertCheckpoint(cp: StoredCheckpoint, stateDir: string = getStateDir()): void {
|
|
696
721
|
const db = openStore(stateDir);
|
|
697
722
|
const sid = normalizeSessionId(cp.sessionId);
|
|
698
|
-
|
|
723
|
+
withTx(db, () => {
|
|
699
724
|
db.prepare(
|
|
700
725
|
`INSERT INTO context_chunks
|
|
701
726
|
(id, session_id, region_hash, content_hash, content_hash2, content_hash_version,
|
|
@@ -720,25 +745,25 @@ export function upsertCheckpoint(cp: StoredCheckpoint, stateDir: string = getSta
|
|
|
720
745
|
dedup_status=excluded.dedup_status,
|
|
721
746
|
compressed_original=excluded.compressed_original`,
|
|
722
747
|
).run({
|
|
723
|
-
id: cp.checkpointId,
|
|
724
|
-
sid,
|
|
725
|
-
region_hash: cp.regionHash ?? null,
|
|
726
|
-
content_hash: cp.contentHash ?? null,
|
|
727
|
-
content_hash2: cp.contentHash2 ?? null,
|
|
728
|
-
content_hash_version: cp.contentHashVersion ?? null,
|
|
729
|
-
normalized_text: cp.normalizedText ?? null,
|
|
730
|
-
summary: cp.summary ?? "",
|
|
731
|
-
topic_summary: cp.topicSummary ?? null,
|
|
732
|
-
summary_hash: cp.summaryHash ?? null,
|
|
733
|
-
key_decisions: jsonText(cp.keyDecisions),
|
|
734
|
-
next_steps: jsonText(cp.nextSteps),
|
|
735
|
-
files_modified: jsonText(cp.filesModified),
|
|
736
|
-
embedding_blob: encodeEmbedding(cp.embedding ?? []),
|
|
737
|
-
token_estimate: cp.tokenEstimate ?? 0,
|
|
738
|
-
original_token_estimate: cp.originalTokenEstimate ?? null,
|
|
739
|
-
timestamp: cp.timestamp ?? 0,
|
|
740
|
-
dedup_status: "active",
|
|
741
|
-
compressed_original: cp.compressedOriginal ?? null,
|
|
748
|
+
"@id": cp.checkpointId,
|
|
749
|
+
"@sid": sid,
|
|
750
|
+
"@region_hash": cp.regionHash ?? null,
|
|
751
|
+
"@content_hash": cp.contentHash ?? null,
|
|
752
|
+
"@content_hash2": cp.contentHash2 ?? null,
|
|
753
|
+
"@content_hash_version": cp.contentHashVersion ?? null,
|
|
754
|
+
"@normalized_text": cp.normalizedText ?? null,
|
|
755
|
+
"@summary": cp.summary ?? "",
|
|
756
|
+
"@topic_summary": cp.topicSummary ?? null,
|
|
757
|
+
"@summary_hash": cp.summaryHash ?? null,
|
|
758
|
+
"@key_decisions": jsonText(cp.keyDecisions),
|
|
759
|
+
"@next_steps": jsonText(cp.nextSteps),
|
|
760
|
+
"@files_modified": jsonText(cp.filesModified),
|
|
761
|
+
"@embedding_blob": encodeEmbedding(cp.embedding ?? []),
|
|
762
|
+
"@token_estimate": cp.tokenEstimate ?? 0,
|
|
763
|
+
"@original_token_estimate": cp.originalTokenEstimate ?? null,
|
|
764
|
+
"@timestamp": cp.timestamp ?? 0,
|
|
765
|
+
"@dedup_status": "active",
|
|
766
|
+
"@compressed_original": cp.compressedOriginal ?? null,
|
|
742
767
|
});
|
|
743
768
|
|
|
744
769
|
// FTS5 virtual tables don't support UPSERT — delete any prior row, reinsert.
|
|
@@ -749,7 +774,6 @@ export function upsertCheckpoint(cp: StoredCheckpoint, stateDir: string = getSta
|
|
|
749
774
|
"INSERT INTO context_chunks_trgm(id, normalized_text) VALUES(?, ?)",
|
|
750
775
|
).run(cp.checkpointId, cp.normalizedText ?? cp.summary ?? "");
|
|
751
776
|
});
|
|
752
|
-
tx();
|
|
753
777
|
}
|
|
754
778
|
|
|
755
779
|
// --- Sprint 11: MinHash signatures + LSH buckets --------------------------
|
|
@@ -786,11 +810,10 @@ export function insertLshBuckets(
|
|
|
786
810
|
const ins = db.prepare(
|
|
787
811
|
"INSERT OR IGNORE INTO dedup_lsh_buckets(bucket_key, chunk_id, session_id, signature_version) VALUES(?, ?, ?, ?)",
|
|
788
812
|
);
|
|
789
|
-
|
|
813
|
+
withTx(db, () => {
|
|
790
814
|
del.run(chunkId);
|
|
791
815
|
for (const key of bucketKeys) ins.run(key, chunkId, sid, signatureVersion);
|
|
792
816
|
});
|
|
793
|
-
tx();
|
|
794
817
|
}
|
|
795
818
|
|
|
796
819
|
/**
|
|
@@ -864,7 +887,7 @@ export function setDedupStatus(
|
|
|
864
887
|
|
|
865
888
|
// --- Session state (injection tracking) ------------------------------------
|
|
866
889
|
|
|
867
|
-
function loadSessionStateRow(sid: string, db:
|
|
890
|
+
function loadSessionStateRow(sid: string, db: DatabaseSync): SessionState {
|
|
868
891
|
const row = db.prepare("SELECT * FROM session_state WHERE session_id = ?").get(sid) as any;
|
|
869
892
|
if (!row) {
|
|
870
893
|
return { injectedCheckpointIds: [], storedRegionHashes: [] };
|
package/src/store.test.ts
CHANGED
|
@@ -130,6 +130,28 @@ test("compression tier: GZIP-1 and GZIP-6 tiers produce valid, smaller output",
|
|
|
130
130
|
assert.deepEqual(decompressSmart(big), Buffer.from("compress me ".repeat(1800)));
|
|
131
131
|
});
|
|
132
132
|
|
|
133
|
+
test("Fix E: pressure escalates gzip tier strength (sync, no zstd)", () => {
|
|
134
|
+
const small = Buffer.from("compress me ".repeat(200)); // 512B–4KB band
|
|
135
|
+
const medium = Buffer.from("compress me ".repeat(1800)); // 4KB–32KB band
|
|
136
|
+
|
|
137
|
+
// Low pressure → cheap levels (gzip-1 / gzip-6).
|
|
138
|
+
const lowSmall = compressSmart(small, 0);
|
|
139
|
+
const lowMedium = compressSmart(medium, 0);
|
|
140
|
+
assert.equal(lowSmall[3], 0x01, "small tier tag");
|
|
141
|
+
assert.equal(lowMedium[3], 0x02, "medium tier tag");
|
|
142
|
+
assert.ok(decompressSmart(lowSmall).equals(small), "low-pressure small roundtrips");
|
|
143
|
+
assert.ok(decompressSmart(lowMedium).equals(medium), "low-pressure medium roundtrips");
|
|
144
|
+
|
|
145
|
+
// High pressure → stronger levels (gzip-9 / gzip-9); tag unchanged, output
|
|
146
|
+
// must still decode to the exact original (versioned header preserved).
|
|
147
|
+
const highSmall = compressSmart(small, 1);
|
|
148
|
+
const highMedium = compressSmart(medium, 1);
|
|
149
|
+
assert.equal(highSmall[3], 0x01, "tag unchanged under pressure");
|
|
150
|
+
assert.equal(highMedium[3], 0x02, "tag unchanged under pressure");
|
|
151
|
+
assert.ok(decompressSmart(highSmall).equals(small), "high-pressure small roundtrips");
|
|
152
|
+
assert.ok(decompressSmart(highMedium).equals(medium), "high-pressure medium roundtrips");
|
|
153
|
+
});
|
|
154
|
+
|
|
133
155
|
// ---------------------------------------------------------------------------
|
|
134
156
|
// normalizeSessionId
|
|
135
157
|
// ---------------------------------------------------------------------------
|
package/src/vectorStore.ts
CHANGED
|
@@ -38,6 +38,8 @@ import {
|
|
|
38
38
|
repoStats as repoStatsFromStore,
|
|
39
39
|
dataInvariantStats,
|
|
40
40
|
} from "./store/sqlite.js";
|
|
41
|
+
import { rehydrateRaptorTree } from "./dedup/raptor/index.js";
|
|
42
|
+
import { stagedExpansion } from "./dedup/raptor/retrieval.js";
|
|
41
43
|
import { migrateJsonToSqlite } from "./store/migrate.js";
|
|
42
44
|
|
|
43
45
|
export interface SearchHit {
|
|
@@ -66,6 +68,9 @@ export interface AddInput {
|
|
|
66
68
|
* Lets the UI render live per-tier progress during compaction. Never awaited;
|
|
67
69
|
* must be cheap. Optional for back-compat. */
|
|
68
70
|
onTier?: (ev: { tier: "L0" | "L1" | "L2" | "new"; status: "scanning" | "deduped" | "passed" | "stored"; detail?: string }) => void;
|
|
71
|
+
/** Context-window pressure (0–1) — escalates the stored checkpoint's sync
|
|
72
|
+
* compression strength (Fix E). Optional; defaults to 0 (brotli-4). */
|
|
73
|
+
compressionPressure?: number;
|
|
69
74
|
}
|
|
70
75
|
|
|
71
76
|
/** Default L2 semantic-dedup enable flag (trigram embedder is local, zero-network). */
|
|
@@ -318,7 +323,10 @@ export class VectorStore {
|
|
|
318
323
|
contentHash2: digest.contentHash2,
|
|
319
324
|
contentHashVersion: digest.contentHashVersion,
|
|
320
325
|
normalizedText: digest.normalizedText,
|
|
321
|
-
compressedOriginal: compressSmart(
|
|
326
|
+
compressedOriginal: compressSmart(
|
|
327
|
+
Buffer.from(input.regionText, "utf-8"),
|
|
328
|
+
input.compressionPressure,
|
|
329
|
+
),
|
|
322
330
|
embedding,
|
|
323
331
|
timestamp: input.timestamp,
|
|
324
332
|
};
|
|
@@ -432,6 +440,29 @@ export class VectorStore {
|
|
|
432
440
|
// MMR (QA #10) is part of the L2 semantic tier: skip it when L2 is disabled
|
|
433
441
|
// (Sprint 14 flag), returning the plain relevance-ranked window instead.
|
|
434
442
|
if (!this.cfg.L2_ENABLED) return window.slice(0, k);
|
|
443
|
+
|
|
444
|
+
// Fix D: when RAPTOR is promoted, ALSO recall high-level tree summaries and
|
|
445
|
+
// merge them with the flat hits via MMR so RAPTOR + flat don't double-cover.
|
|
446
|
+
// RAPTOR returns fewer, broader hits (O(log n) high-level nodes) than the
|
|
447
|
+
// O(n) flat leaves, tightening the block at read time.
|
|
448
|
+
if (this.cfg.RAPTOR_ENABLED) {
|
|
449
|
+
const raptorHits = this.raptorSearchHits(sid, query, k);
|
|
450
|
+
if (raptorHits.length > 0) {
|
|
451
|
+
const merged: SearchHit[] = [...window];
|
|
452
|
+
for (const rh of raptorHits) {
|
|
453
|
+
if (!merged.some((m) => m.checkpoint.checkpointId === rh.checkpoint.checkpointId)) {
|
|
454
|
+
merged.push(rh);
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
const mmrItems: MmrItem<SearchHit>[] = merged.map((h) => ({
|
|
458
|
+
item: h,
|
|
459
|
+
vector: h.checkpoint.embedding,
|
|
460
|
+
relevance: h.score,
|
|
461
|
+
}));
|
|
462
|
+
return mmrRerank(mmrItems, k, this.cfg.MMR_LAMBDA);
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
|
|
435
466
|
const mmrItems: MmrItem<SearchHit>[] = window.map((h) => ({
|
|
436
467
|
item: h,
|
|
437
468
|
vector: h.checkpoint.embedding,
|
|
@@ -441,6 +472,37 @@ export class VectorStore {
|
|
|
441
472
|
return ranked;
|
|
442
473
|
}
|
|
443
474
|
|
|
475
|
+
/**
|
|
476
|
+
* Serve the RAPTOR tree for a query (Fix D): rehydrate the persisted tree and
|
|
477
|
+
* return its staged-expansion leaf hits as SearchHits. Returns [] when no tree
|
|
478
|
+
* exists (small sessions — flat search remains the path). Best-effort/non-fatal.
|
|
479
|
+
*/
|
|
480
|
+
private raptorSearchHits(sid: string, query: string, k: number): SearchHit[] {
|
|
481
|
+
try {
|
|
482
|
+
const tree = rehydrateRaptorTree(sid, this.stateDir);
|
|
483
|
+
if (!tree || !tree.rootId) return [];
|
|
484
|
+
const leafIds = stagedExpansion(query, tree, {
|
|
485
|
+
embedder: this.embedder,
|
|
486
|
+
k,
|
|
487
|
+
topM: this.cfg.RAPTOR_CLUSTERS_PER_LEVEL,
|
|
488
|
+
mmrLambda: this.cfg.MMR_LAMBDA,
|
|
489
|
+
});
|
|
490
|
+
if (leafIds.length === 0) return [];
|
|
491
|
+
const all = listCheckpoints(sid, this.stateDir).filter(
|
|
492
|
+
(cp) => cp.dedupStatus !== "removed",
|
|
493
|
+
);
|
|
494
|
+
const qv = this.embedder.embed(query);
|
|
495
|
+
const hits: SearchHit[] = [];
|
|
496
|
+
for (const id of leafIds) {
|
|
497
|
+
const cp = all.find((c) => c.checkpointId === id);
|
|
498
|
+
if (cp) hits.push({ checkpoint: cp, score: cosineSimilarity(qv, cp.embedding) });
|
|
499
|
+
}
|
|
500
|
+
return hits;
|
|
501
|
+
} catch {
|
|
502
|
+
return [];
|
|
503
|
+
}
|
|
504
|
+
}
|
|
505
|
+
|
|
444
506
|
/**
|
|
445
507
|
* SemDeDup offline cleanup (Sprint 12, QA #17): within a session, mark the
|
|
446
508
|
* lower-quality row of any pair scoring cosine > `threshold` as
|
|
@@ -1,291 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* openclaw-mega-compact — OpenClaw plugin adapter for the pi-mega-compact engine.
|
|
3
|
-
*
|
|
4
|
-
* Wires the pi-agnostic Trident engine (src/) into OpenClaw's plugin lifecycle:
|
|
5
|
-
* - Registers a CompactionProvider that replaces the built-in summarizeInStages.
|
|
6
|
-
* - Exposes `mega_status` and `mega_recall` tools for on-demand inspection.
|
|
7
|
-
* - Hooks into `before_compaction` / `after_compaction` for diagnostics.
|
|
8
|
-
*
|
|
9
|
-
* Design constraints:
|
|
10
|
-
* - NO imports from `@earendil-works/pi-coding-agent` or pi-agent-core.
|
|
11
|
-
* - The engine core (src/) is pi-agnostic; this file is the sole OpenClaw boundary.
|
|
12
|
-
* - No network at runtime — everything is local (stores + extractive summarizer).
|
|
13
|
-
*/
|
|
14
|
-
import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
|
|
15
|
-
import { compactSession, setDefaultStore, } from "../src/engine.js";
|
|
16
|
-
import { recallAndInline } from "../src/recall.js";
|
|
17
|
-
import { VectorStore } from "../src/vectorStore.js";
|
|
18
|
-
// ---------------------------------------------------------------------------
|
|
19
|
-
// Constants
|
|
20
|
-
// ---------------------------------------------------------------------------
|
|
21
|
-
const PLUGIN_ID = "mega-compact";
|
|
22
|
-
const PLUGIN_LABEL = "Mega Compact (Trident)";
|
|
23
|
-
/** Default state directory for vector store persistence. */
|
|
24
|
-
const STATE_DIR = process.env.MEGA_COMPACT_STATE_DIR ?? undefined;
|
|
25
|
-
/** Minimum messages before we bother compacting. */
|
|
26
|
-
const MIN_MESSAGES_FOR_COMPACT = 6;
|
|
27
|
-
// ---------------------------------------------------------------------------
|
|
28
|
-
// Message conversion — OpenClaw unknown[] → EngineMessage[]
|
|
29
|
-
// ---------------------------------------------------------------------------
|
|
30
|
-
/**
|
|
31
|
-
* Best-effort conversion from OpenClaw's opaque message array to our
|
|
32
|
-
* EngineMessage shape. OpenClaw messages are typed as `unknown[]` so we
|
|
33
|
-
* handle whatever shape comes through gracefully.
|
|
34
|
-
*/
|
|
35
|
-
function toEngineMessages(messages) {
|
|
36
|
-
return messages.map((msg) => {
|
|
37
|
-
if (!msg || typeof msg !== "object") {
|
|
38
|
-
// Primitive fallback — treat as custom text.
|
|
39
|
-
return {
|
|
40
|
-
role: "custom",
|
|
41
|
-
text: String(msg ?? ""),
|
|
42
|
-
};
|
|
43
|
-
}
|
|
44
|
-
const m = msg;
|
|
45
|
-
const role = typeof m.role === "string" ? m.role : "custom";
|
|
46
|
-
// Normalize role to one of our four engine roles.
|
|
47
|
-
let engineRole;
|
|
48
|
-
switch (role) {
|
|
49
|
-
case "user":
|
|
50
|
-
engineRole = "user";
|
|
51
|
-
break;
|
|
52
|
-
case "assistant":
|
|
53
|
-
engineRole = "assistant";
|
|
54
|
-
break;
|
|
55
|
-
case "tool":
|
|
56
|
-
case "function":
|
|
57
|
-
engineRole = "tool";
|
|
58
|
-
break;
|
|
59
|
-
default:
|
|
60
|
-
engineRole = "custom";
|
|
61
|
-
break;
|
|
62
|
-
}
|
|
63
|
-
// Extract text content from common message shapes.
|
|
64
|
-
const text = typeof m.content === "string"
|
|
65
|
-
? m.content
|
|
66
|
-
: typeof m.text === "string"
|
|
67
|
-
? m.text
|
|
68
|
-
: Array.isArray(m.content)
|
|
69
|
-
? m.content
|
|
70
|
-
.filter((part) => part.type === "text" && typeof part.text === "string")
|
|
71
|
-
.map((part) => part.text)
|
|
72
|
-
.join("\n")
|
|
73
|
-
: "";
|
|
74
|
-
// Preserve tool metadata when present.
|
|
75
|
-
const toolName = typeof m.name === "string"
|
|
76
|
-
? m.name
|
|
77
|
-
: typeof m.toolName === "string"
|
|
78
|
-
? m.toolName
|
|
79
|
-
: undefined;
|
|
80
|
-
const input = typeof m.input === "string"
|
|
81
|
-
? m.input
|
|
82
|
-
: typeof m.arguments === "string"
|
|
83
|
-
? m.arguments
|
|
84
|
-
: m.arguments !== undefined
|
|
85
|
-
? JSON.stringify(m.arguments)
|
|
86
|
-
: undefined;
|
|
87
|
-
const output = typeof m.output === "string"
|
|
88
|
-
? m.output
|
|
89
|
-
: engineRole === "tool" && typeof m.content === "string"
|
|
90
|
-
? m.content
|
|
91
|
-
: undefined;
|
|
92
|
-
return { role: engineRole, text, toolName, input, output };
|
|
93
|
-
});
|
|
94
|
-
}
|
|
95
|
-
// ---------------------------------------------------------------------------
|
|
96
|
-
// Compaction provider
|
|
97
|
-
// ---------------------------------------------------------------------------
|
|
98
|
-
function createCompactionProvider(store) {
|
|
99
|
-
return {
|
|
100
|
-
id: PLUGIN_ID,
|
|
101
|
-
label: PLUGIN_LABEL,
|
|
102
|
-
async summarize({ messages, signal, compressionRatio, }) {
|
|
103
|
-
// Abort check — bail early if the caller cancelled.
|
|
104
|
-
if (signal?.aborted) {
|
|
105
|
-
throw new DOMException("Aborted", "AbortError");
|
|
106
|
-
}
|
|
107
|
-
const engineMessages = toEngineMessages(messages);
|
|
108
|
-
// Nothing meaningful to compact.
|
|
109
|
-
if (engineMessages.length < MIN_MESSAGES_FOR_COMPACT) {
|
|
110
|
-
return "";
|
|
111
|
-
}
|
|
112
|
-
// Map compression ratio → keepFrom boundary.
|
|
113
|
-
// compressionRatio=0.5 means "compact the oldest 50%".
|
|
114
|
-
// Default to compacting the oldest half if not specified.
|
|
115
|
-
const ratio = compressionRatio ?? 0.5;
|
|
116
|
-
const keepFrom = Math.max(MIN_MESSAGES_FOR_COMPACT, Math.floor(engineMessages.length * (1 - ratio)));
|
|
117
|
-
// Abort check after conversion (conversion is cheap but check anyway).
|
|
118
|
-
if (signal?.aborted) {
|
|
119
|
-
throw new DOMException("Aborted", "AbortError");
|
|
120
|
-
}
|
|
121
|
-
const sessionId = `openclaw-${Date.now()}`;
|
|
122
|
-
const input = {
|
|
123
|
-
sessionId,
|
|
124
|
-
messages: engineMessages,
|
|
125
|
-
keepFrom,
|
|
126
|
-
};
|
|
127
|
-
const result = compactSession(input, store);
|
|
128
|
-
if (result.skipped) {
|
|
129
|
-
return "";
|
|
130
|
-
}
|
|
131
|
-
return result.summary;
|
|
132
|
-
},
|
|
133
|
-
};
|
|
134
|
-
}
|
|
135
|
-
// ---------------------------------------------------------------------------
|
|
136
|
-
// Plugin entry
|
|
137
|
-
// ---------------------------------------------------------------------------
|
|
138
|
-
export default definePluginEntry({
|
|
139
|
-
id: PLUGIN_ID,
|
|
140
|
-
name: "Mega Compact",
|
|
141
|
-
description: "Layered, local, vector-backed context compressor (Trident engine) for OpenClaw compaction.",
|
|
142
|
-
register(api) {
|
|
143
|
-
const logger = api.logger;
|
|
144
|
-
// Resolve state directory — prefer plugin config override.
|
|
145
|
-
const pluginCfg = (api.pluginConfig ?? {});
|
|
146
|
-
const stateDir = typeof pluginCfg.stateDir === "string" && pluginCfg.stateDir.length > 0
|
|
147
|
-
? pluginCfg.stateDir
|
|
148
|
-
: STATE_DIR;
|
|
149
|
-
// Initialize vector store.
|
|
150
|
-
let store;
|
|
151
|
-
try {
|
|
152
|
-
store = new VectorStore({ stateDir });
|
|
153
|
-
setDefaultStore(store);
|
|
154
|
-
logger.info?.(`${PLUGIN_ID}: vector store initialized (stateDir=${stateDir ?? "default"})`);
|
|
155
|
-
}
|
|
156
|
-
catch (err) {
|
|
157
|
-
logger.error?.(`${PLUGIN_ID}: failed to init vector store:`, err);
|
|
158
|
-
return; // Hard bail — no point registering if store is broken.
|
|
159
|
-
}
|
|
160
|
-
// -----------------------------------------------------------------------
|
|
161
|
-
// Register compaction provider
|
|
162
|
-
// -----------------------------------------------------------------------
|
|
163
|
-
const provider = createCompactionProvider(store);
|
|
164
|
-
api.registerCompactionProvider(provider);
|
|
165
|
-
logger.info?.(`${PLUGIN_ID}: registered compaction provider "${provider.id}"`);
|
|
166
|
-
// -----------------------------------------------------------------------
|
|
167
|
-
// Hooks — before / after compaction diagnostics
|
|
168
|
-
// -----------------------------------------------------------------------
|
|
169
|
-
api.registerHook({
|
|
170
|
-
event: "before_compaction",
|
|
171
|
-
handler: async (ctx) => {
|
|
172
|
-
const msgCount = Array.isArray(ctx?.messages) ? ctx.messages.length : 0;
|
|
173
|
-
logger.info?.(`${PLUGIN_ID}: before_compaction — ${msgCount} messages in scope`);
|
|
174
|
-
},
|
|
175
|
-
});
|
|
176
|
-
api.registerHook({
|
|
177
|
-
event: "after_compaction",
|
|
178
|
-
handler: async (ctx) => {
|
|
179
|
-
const summaryLen = typeof ctx?.summary === "string" ? ctx.summary.length : 0;
|
|
180
|
-
logger.info?.(`${PLUGIN_ID}: after_compaction — summary ${summaryLen} chars`);
|
|
181
|
-
},
|
|
182
|
-
});
|
|
183
|
-
// -----------------------------------------------------------------------
|
|
184
|
-
// Tool: mega_status
|
|
185
|
-
// -----------------------------------------------------------------------
|
|
186
|
-
api.registerTool({
|
|
187
|
-
name: "mega_status",
|
|
188
|
-
description: "Show the current status of the mega-compact engine: vector store stats, checkpoint count, and recent compaction activity.",
|
|
189
|
-
parameters: {
|
|
190
|
-
type: "object",
|
|
191
|
-
properties: {
|
|
192
|
-
sessionId: {
|
|
193
|
-
type: "string",
|
|
194
|
-
description: "Optional session ID to scope stats to.",
|
|
195
|
-
},
|
|
196
|
-
},
|
|
197
|
-
additionalProperties: false,
|
|
198
|
-
},
|
|
199
|
-
handler: async (args) => {
|
|
200
|
-
const sessionId = args?.sessionId ?? "global";
|
|
201
|
-
try {
|
|
202
|
-
const stats = store.stats(sessionId);
|
|
203
|
-
const parts = [
|
|
204
|
-
`**Mega Compact Status**`,
|
|
205
|
-
`Session: ${sessionId}`,
|
|
206
|
-
`Checkpoints: ${stats.checkpointCount}`,
|
|
207
|
-
`Total tokens saved: ${stats.totalTokenEstimate}`,
|
|
208
|
-
`Last checkpoint: ${stats.lastCheckpointId ?? "—"}`,
|
|
209
|
-
`Injected count: ${stats.injectedCount}`,
|
|
210
|
-
`Dedup hit rate: ${(stats.dedupHitRate * 100).toFixed(0)}%`,
|
|
211
|
-
];
|
|
212
|
-
if (stats.lastSummary) {
|
|
213
|
-
parts.push(`\nLast summary (truncated):\n ${stats.lastSummary.slice(0, 120).replace(/\n/g, " ")}…`);
|
|
214
|
-
}
|
|
215
|
-
return { content: [{ type: "text", text: parts.join("\n") }] };
|
|
216
|
-
}
|
|
217
|
-
catch (err) {
|
|
218
|
-
return {
|
|
219
|
-
content: [{ type: "text", text: `Error reading mega-compact status: ${err}` }],
|
|
220
|
-
isError: true,
|
|
221
|
-
};
|
|
222
|
-
}
|
|
223
|
-
},
|
|
224
|
-
});
|
|
225
|
-
// -----------------------------------------------------------------------
|
|
226
|
-
// Tool: mega_recall
|
|
227
|
-
// -----------------------------------------------------------------------
|
|
228
|
-
api.registerTool({
|
|
229
|
-
name: "mega_recall",
|
|
230
|
-
description: "Recall and inline relevant context from the mega-compact vector store for the current session.",
|
|
231
|
-
parameters: {
|
|
232
|
-
type: "object",
|
|
233
|
-
properties: {
|
|
234
|
-
sessionId: {
|
|
235
|
-
type: "string",
|
|
236
|
-
description: "Session ID to recall context for.",
|
|
237
|
-
},
|
|
238
|
-
query: {
|
|
239
|
-
type: "string",
|
|
240
|
-
description: "Natural language query for relevant context.",
|
|
241
|
-
},
|
|
242
|
-
limit: {
|
|
243
|
-
type: "number",
|
|
244
|
-
description: "Max checkpoints to recall (default 3).",
|
|
245
|
-
},
|
|
246
|
-
},
|
|
247
|
-
required: ["sessionId", "query"],
|
|
248
|
-
additionalProperties: false,
|
|
249
|
-
},
|
|
250
|
-
handler: async (args) => {
|
|
251
|
-
const { sessionId, query, limit } = args;
|
|
252
|
-
if (!sessionId || !query) {
|
|
253
|
-
return {
|
|
254
|
-
content: [{ type: "text", text: "Both `sessionId` and `query` are required." }],
|
|
255
|
-
isError: true,
|
|
256
|
-
};
|
|
257
|
-
}
|
|
258
|
-
try {
|
|
259
|
-
const result = recallAndInline({ sessionId, query, limit: limit ?? 3, source: "command", skipInjected: false }, store);
|
|
260
|
-
if (result.toInject.length === 0) {
|
|
261
|
-
return {
|
|
262
|
-
content: [{ type: "text", text: "No relevant context found in the mega-compact store." }],
|
|
263
|
-
};
|
|
264
|
-
}
|
|
265
|
-
const parts = [
|
|
266
|
-
`**Recalled ${result.toInject.length} checkpoint(s):**`,
|
|
267
|
-
...result.report,
|
|
268
|
-
"",
|
|
269
|
-
"---",
|
|
270
|
-
result.block,
|
|
271
|
-
];
|
|
272
|
-
return { content: [{ type: "text", text: parts.join("\n") }] };
|
|
273
|
-
}
|
|
274
|
-
catch (err) {
|
|
275
|
-
return {
|
|
276
|
-
content: [{ type: "text", text: `Error during mega-recall: ${err}` }],
|
|
277
|
-
isError: true,
|
|
278
|
-
};
|
|
279
|
-
}
|
|
280
|
-
},
|
|
281
|
-
});
|
|
282
|
-
// -----------------------------------------------------------------------
|
|
283
|
-
// Cleanup on shutdown
|
|
284
|
-
// -----------------------------------------------------------------------
|
|
285
|
-
api.on("shutdown", () => {
|
|
286
|
-
logger.info?.(`${PLUGIN_ID}: shutting down — clearing default store`);
|
|
287
|
-
setDefaultStore(undefined);
|
|
288
|
-
});
|
|
289
|
-
logger.info?.(`${PLUGIN_ID}: plugin registered (tools: mega_status, mega_recall)`);
|
|
290
|
-
},
|
|
291
|
-
});
|