pi-mega-compact 0.4.21 → 0.4.23

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/dist/extensions/dashboard-server.js +3 -3
  2. package/dist/extensions/mega-compact-driver.js +79 -0
  3. package/dist/extensions/mega-compact.test.js +54 -18
  4. package/dist/extensions/mega-config.js +10 -0
  5. package/dist/extensions/mega-events.js +45 -23
  6. package/dist/extensions/mega-pipeline.js +77 -3
  7. package/dist/src/config/dedup.js +4 -1
  8. package/dist/src/config.js +21 -0
  9. package/dist/src/dedup/raptor/index.js +28 -6
  10. package/dist/src/dedup/raptor/promote.test.js +69 -0
  11. package/dist/src/engine.js +1 -0
  12. package/dist/src/recall.js +30 -4
  13. package/dist/src/recall.test.js +28 -0
  14. package/dist/src/store/backfill.js +5 -6
  15. package/dist/src/store/compression.js +47 -7
  16. package/dist/src/store/compression.test.js +48 -0
  17. package/dist/src/store/sqlite.js +64 -41
  18. package/dist/src/store.test.js +19 -0
  19. package/dist/src/vectorStore.js +56 -1
  20. package/extensions/DASHBOARD.md +3 -3
  21. package/extensions/dashboard-server.ts +4 -4
  22. package/extensions/mega-compact-driver.ts +105 -0
  23. package/extensions/mega-compact.test.ts +65 -18
  24. package/extensions/mega-config.ts +25 -0
  25. package/extensions/mega-events.ts +43 -24
  26. package/extensions/mega-pipeline.ts +83 -4
  27. package/package.json +6 -7
  28. package/src/config/dedup.ts +4 -1
  29. package/src/config.ts +26 -0
  30. package/src/dedup/raptor/index.ts +42 -7
  31. package/src/dedup/raptor/promote.test.ts +82 -0
  32. package/src/engine.ts +5 -0
  33. package/src/recall.test.ts +44 -0
  34. package/src/recall.ts +43 -4
  35. package/src/store/backfill.ts +10 -11
  36. package/src/store/compression.test.ts +58 -0
  37. package/src/store/compression.ts +48 -7
  38. package/src/store/sqlite.ts +72 -49
  39. package/src/store.test.ts +22 -0
  40. package/src/vectorStore.ts +63 -1
  41. package/dist/extensions/openclaw-mega-compact.js +0 -291
  42. package/dist/src/minilm.js +0 -92
  43. package/dist/src/wordpiece.js +0 -129
@@ -2,11 +2,13 @@
2
2
  * sqlite.ts — Sprint 8 storage backbone (the "one store").
3
3
  *
4
4
  * Replaces the per-session gzipped-JSON checkpoint files with a single local
5
- * SQLite database (better-sqlite3, in-process, FS-backed, ZERO network calls —
6
- * honors PREVENT-PI-004). Chosen over PGlite because PGlite is async-only in
7
- * every published version, and VectorStore (engine.ts / recall.ts / the
8
- * extension) is fully synchronous adopting PGlite would have cascaded async
9
- * through the whole call chain. SQLite keeps every VectorStore signature sync.
5
+ * SQLite database (node:sqlite — the Node built-in, in-process, FS-backed,
6
+ * ZERO network calls — honors PREVENT-PI-004). No native build and no install
7
+ * scripts, so it survives pi's npm blocked-install-scripts gate (better-sqlite3's
8
+ * native binary could not be built under pi, which crashed every `pi update
9
+ * --extensions`). node:sqlite is synchronous, so every VectorStore signature
10
+ * stays sync. PGlite + pgvector (async) is layered on in vectorIndex.ts for
11
+ * real HNSW indexing (Slice 2 of the dual-backend plan).
10
12
  *
11
13
  * FTS5 `trigram` tokenizer is created for the Sprint 9+ dedup tiers (MinHash/LSH
12
14
  * / pg_trgm-equivalent verification). The default cosine path stays a linear
@@ -15,7 +17,7 @@
15
17
  * All queries are parameterized (PREVENT-002) — never string-concatenated.
16
18
  */
17
19
 
18
- import Database from "better-sqlite3";
20
+ import { DatabaseSync } from "node:sqlite";
19
21
  import { existsSync, mkdirSync } from "node:fs";
20
22
  import { homedir, tmpdir } from "node:os";
21
23
  import { join } from "node:path";
@@ -31,12 +33,15 @@ function encodeEmbedding(v: number[]): Buffer {
31
33
  for (let i = 0; i < v.length; i++) buf.writeFloatLE(v[i] ?? 0, i * 4);
32
34
  return buf;
33
35
  }
34
- /** Decode a Float32 BLOB back to a number[]. */
35
- function decodeEmbedding(buf: Buffer | null | undefined): number[] {
36
+ /** Decode a Float32 BLOB back to a number[]. node:sqlite returns BLOBs as
37
+ * Uint8Array, so decode via DataView (Buffer is a Uint8Array subclass both
38
+ * work). */
39
+ function decodeEmbedding(buf: Uint8Array | null | undefined): number[] {
36
40
  if (!buf || buf.length === 0) return [];
41
+ const dv = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
37
42
  const n = buf.length / 4;
38
43
  const out = new Array<number>(n);
39
- for (let i = 0; i < n; i++) out[i] = buf.readFloatLE(i * 4);
44
+ for (let i = 0; i < n; i++) out[i] = dv.getFloat32(i * 4, true);
40
45
  return out;
41
46
  }
42
47
 
@@ -47,17 +52,17 @@ function jsonText(v: unknown): string {
47
52
  // In-process cache so the same stateDir reuses one connection (and so a fresh
48
53
  // VectorStore over the same dir shares the open DB). Cross-process durability
49
54
  // comes from reopening the same file path — proven by the integration test.
50
- const cache = new Map<string, Database.Database>();
55
+ const cache = new Map<string, DatabaseSync>();
51
56
 
52
57
  /** Open (or reuse) the SQLite store for a state dir. */
53
- export function openStore(stateDir: string = getStateDir()): Database.Database {
58
+ export function openStore(stateDir: string = getStateDir()): DatabaseSync {
54
59
  const existing = cache.get(stateDir);
55
60
  if (existing) return existing;
56
61
 
57
62
  if (!existsSync(stateDir)) mkdirSync(stateDir, { recursive: true });
58
- const db = new Database(join(stateDir, "sqlite.db"));
59
- db.pragma("journal_mode = WAL");
60
- db.pragma("foreign_keys = ON");
63
+ const db = new DatabaseSync(join(stateDir, "sqlite.db"));
64
+ db.exec("PRAGMA journal_mode = WAL");
65
+ db.exec("PRAGMA foreign_keys = ON");
61
66
  initSchema(db);
62
67
  cache.set(stateDir, db);
63
68
  return db;
@@ -87,17 +92,17 @@ export function getIndexDir(): string {
87
92
  }
88
93
  }
89
94
 
90
- let indexCache: Database.Database | undefined;
95
+ let indexCache: DatabaseSync | undefined;
91
96
  let indexCacheDir: string | undefined;
92
97
 
93
98
  /** Open (or reuse) the machine-wide index DB. WAL for concurrent writers. */
94
- export function openIndexStore(indexDir: string = getIndexDir()): Database.Database {
99
+ export function openIndexStore(indexDir: string = getIndexDir()): DatabaseSync {
95
100
  if (indexCache && indexCacheDir === indexDir) return indexCache;
96
101
  if (!existsSync(indexDir)) mkdirSync(indexDir, { recursive: true });
97
- const db = new Database(join(indexDir, "index.sqlite"));
98
- db.pragma("journal_mode = WAL");
99
- db.pragma("busy_timeout = 3000"); // tolerate brief cross-process write contention
100
- db.exec(`
102
+ const iddb = new DatabaseSync(join(indexDir, "index.sqlite"));
103
+ iddb.exec("PRAGMA journal_mode = WAL");
104
+ iddb.exec("PRAGMA busy_timeout = 3000"); // tolerate brief cross-process write contention
105
+ iddb.exec(`
101
106
  CREATE TABLE IF NOT EXISTS repo_registry (
102
107
  repo_root TEXT PRIMARY KEY,
103
108
  display_name TEXT,
@@ -117,9 +122,9 @@ export function openIndexStore(indexDir: string = getIndexDir()): Database.Datab
117
122
  );
118
123
  CREATE INDEX IF NOT EXISTS idx_registry_last_seen ON repo_registry(last_seen DESC);
119
124
  `);
120
- indexCache = db;
125
+ indexCache = iddb;
121
126
  indexCacheDir = indexDir;
122
- return db;
127
+ return iddb;
123
128
  }
124
129
 
125
130
  /** One row of the global repo registry (multi-repo dashboard source). */
@@ -276,7 +281,7 @@ export function closeIndexStore(): void {
276
281
  }
277
282
  }
278
283
 
279
- function initSchema(db: Database.Database): void {
284
+ function initSchema(db: DatabaseSync): void {
280
285
  db.exec(`
281
286
  CREATE TABLE IF NOT EXISTS context_chunks (
282
287
  id TEXT NOT NULL,
@@ -457,7 +462,7 @@ function initSchema(db: Database.Database): void {
457
462
  * input), so the unavoidable identifier interpolation here does not violate
458
463
  * PREVENT-002 (no external data reaches this SQL).
459
464
  */
460
- function ensureColumn(db: Database.Database, table: string, column: string, decl: string): void {
465
+ function ensureColumn(db: DatabaseSync, table: string, column: string, decl: string): void {
461
466
  const cols = db.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>;
462
467
  if (cols.some((c) => c.name === column)) return;
463
468
  db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${decl}`);
@@ -666,6 +671,24 @@ function mapMemoryRow(row: any): MemoryRecord {
666
671
  };
667
672
  }
668
673
 
674
+ /**
675
+ * Run `fn` atomically. Uses SAVEPOINT so it nests safely under an outer
676
+ * transaction (unlike `BEGIN`, which SQLite rejects when one is already open).
677
+ * Mirrors better-sqlite3's `db.transaction(fn)` semantics — callers that wrap a
678
+ * batch in withTx (e.g. backfill) can still call helpers that also use withTx.
679
+ */
680
+ export function withTx(db: DatabaseSync, fn: () => void): void {
681
+ db.exec("SAVEPOINT mc_tx");
682
+ try {
683
+ fn();
684
+ db.exec("RELEASE mc_tx");
685
+ } catch (e) {
686
+ db.exec("ROLLBACK TO mc_tx");
687
+ db.exec("RELEASE mc_tx");
688
+ throw e;
689
+ }
690
+ }
691
+
669
692
  /** Map a DB row to the public StoredCheckpoint shape. */
670
693
  function rowToCheckpoint(row: any): StoredCheckpoint {
671
694
  return {
@@ -684,7 +707,9 @@ function rowToCheckpoint(row: any): StoredCheckpoint {
684
707
  contentHash2: row.content_hash2 ?? undefined,
685
708
  contentHashVersion: row.content_hash_version ?? undefined,
686
709
  normalizedText: row.normalized_text ?? undefined,
687
- compressedOriginal: row.compressed_original ?? undefined,
710
+ // node:sqlite returns BLOBs as Uint8Array; normalize to Buffer so callers
711
+ // (e.g. decompressSmart → Buffer.toString) behave as under better-sqlite3.
712
+ compressedOriginal: row.compressed_original ? Buffer.from(row.compressed_original) : undefined,
688
713
  embedding: decodeEmbedding(row.embedding_blob),
689
714
  timestamp: Number(row.timestamp ?? 0),
690
715
  dedupStatus: row.dedup_status ?? undefined,
@@ -695,7 +720,7 @@ function rowToCheckpoint(row: any): StoredCheckpoint {
695
720
  export function upsertCheckpoint(cp: StoredCheckpoint, stateDir: string = getStateDir()): void {
696
721
  const db = openStore(stateDir);
697
722
  const sid = normalizeSessionId(cp.sessionId);
698
- const tx = db.transaction(() => {
723
+ withTx(db, () => {
699
724
  db.prepare(
700
725
  `INSERT INTO context_chunks
701
726
  (id, session_id, region_hash, content_hash, content_hash2, content_hash_version,
@@ -720,25 +745,25 @@ export function upsertCheckpoint(cp: StoredCheckpoint, stateDir: string = getSta
720
745
  dedup_status=excluded.dedup_status,
721
746
  compressed_original=excluded.compressed_original`,
722
747
  ).run({
723
- id: cp.checkpointId,
724
- sid,
725
- region_hash: cp.regionHash ?? null,
726
- content_hash: cp.contentHash ?? null,
727
- content_hash2: cp.contentHash2 ?? null,
728
- content_hash_version: cp.contentHashVersion ?? null,
729
- normalized_text: cp.normalizedText ?? null,
730
- summary: cp.summary ?? "",
731
- topic_summary: cp.topicSummary ?? null,
732
- summary_hash: cp.summaryHash ?? null,
733
- key_decisions: jsonText(cp.keyDecisions),
734
- next_steps: jsonText(cp.nextSteps),
735
- files_modified: jsonText(cp.filesModified),
736
- embedding_blob: encodeEmbedding(cp.embedding ?? []),
737
- token_estimate: cp.tokenEstimate ?? 0,
738
- original_token_estimate: cp.originalTokenEstimate ?? null,
739
- timestamp: cp.timestamp ?? 0,
740
- dedup_status: "active",
741
- compressed_original: cp.compressedOriginal ?? null,
748
+ "@id": cp.checkpointId,
749
+ "@sid": sid,
750
+ "@region_hash": cp.regionHash ?? null,
751
+ "@content_hash": cp.contentHash ?? null,
752
+ "@content_hash2": cp.contentHash2 ?? null,
753
+ "@content_hash_version": cp.contentHashVersion ?? null,
754
+ "@normalized_text": cp.normalizedText ?? null,
755
+ "@summary": cp.summary ?? "",
756
+ "@topic_summary": cp.topicSummary ?? null,
757
+ "@summary_hash": cp.summaryHash ?? null,
758
+ "@key_decisions": jsonText(cp.keyDecisions),
759
+ "@next_steps": jsonText(cp.nextSteps),
760
+ "@files_modified": jsonText(cp.filesModified),
761
+ "@embedding_blob": encodeEmbedding(cp.embedding ?? []),
762
+ "@token_estimate": cp.tokenEstimate ?? 0,
763
+ "@original_token_estimate": cp.originalTokenEstimate ?? null,
764
+ "@timestamp": cp.timestamp ?? 0,
765
+ "@dedup_status": "active",
766
+ "@compressed_original": cp.compressedOriginal ?? null,
742
767
  });
743
768
 
744
769
  // FTS5 virtual tables don't support UPSERT — delete any prior row, reinsert.
@@ -749,7 +774,6 @@ export function upsertCheckpoint(cp: StoredCheckpoint, stateDir: string = getSta
749
774
  "INSERT INTO context_chunks_trgm(id, normalized_text) VALUES(?, ?)",
750
775
  ).run(cp.checkpointId, cp.normalizedText ?? cp.summary ?? "");
751
776
  });
752
- tx();
753
777
  }
754
778
 
755
779
  // --- Sprint 11: MinHash signatures + LSH buckets --------------------------
@@ -786,11 +810,10 @@ export function insertLshBuckets(
786
810
  const ins = db.prepare(
787
811
  "INSERT OR IGNORE INTO dedup_lsh_buckets(bucket_key, chunk_id, session_id, signature_version) VALUES(?, ?, ?, ?)",
788
812
  );
789
- const tx = db.transaction(() => {
813
+ withTx(db, () => {
790
814
  del.run(chunkId);
791
815
  for (const key of bucketKeys) ins.run(key, chunkId, sid, signatureVersion);
792
816
  });
793
- tx();
794
817
  }
795
818
 
796
819
  /**
@@ -864,7 +887,7 @@ export function setDedupStatus(
864
887
 
865
888
  // --- Session state (injection tracking) ------------------------------------
866
889
 
867
- function loadSessionStateRow(sid: string, db: Database.Database): SessionState {
890
+ function loadSessionStateRow(sid: string, db: DatabaseSync): SessionState {
868
891
  const row = db.prepare("SELECT * FROM session_state WHERE session_id = ?").get(sid) as any;
869
892
  if (!row) {
870
893
  return { injectedCheckpointIds: [], storedRegionHashes: [] };
package/src/store.test.ts CHANGED
@@ -130,6 +130,28 @@ test("compression tier: GZIP-1 and GZIP-6 tiers produce valid, smaller output",
130
130
  assert.deepEqual(decompressSmart(big), Buffer.from("compress me ".repeat(1800)));
131
131
  });
132
132
 
133
+ test("Fix E: pressure escalates gzip tier strength (sync, no zstd)", () => {
134
+ const small = Buffer.from("compress me ".repeat(200)); // 512B–4KB band
135
+ const medium = Buffer.from("compress me ".repeat(1800)); // 4KB–32KB band
136
+
137
+ // Low pressure → cheap levels (gzip-1 / gzip-6).
138
+ const lowSmall = compressSmart(small, 0);
139
+ const lowMedium = compressSmart(medium, 0);
140
+ assert.equal(lowSmall[3], 0x01, "small tier tag");
141
+ assert.equal(lowMedium[3], 0x02, "medium tier tag");
142
+ assert.ok(decompressSmart(lowSmall).equals(small), "low-pressure small roundtrips");
143
+ assert.ok(decompressSmart(lowMedium).equals(medium), "low-pressure medium roundtrips");
144
+
145
+ // High pressure → stronger levels (gzip-9 / gzip-9); tag unchanged, output
146
+ // must still decode to the exact original (versioned header preserved).
147
+ const highSmall = compressSmart(small, 1);
148
+ const highMedium = compressSmart(medium, 1);
149
+ assert.equal(highSmall[3], 0x01, "tag unchanged under pressure");
150
+ assert.equal(highMedium[3], 0x02, "tag unchanged under pressure");
151
+ assert.ok(decompressSmart(highSmall).equals(small), "high-pressure small roundtrips");
152
+ assert.ok(decompressSmart(highMedium).equals(medium), "high-pressure medium roundtrips");
153
+ });
154
+
133
155
  // ---------------------------------------------------------------------------
134
156
  // normalizeSessionId
135
157
  // ---------------------------------------------------------------------------
@@ -38,6 +38,8 @@ import {
38
38
  repoStats as repoStatsFromStore,
39
39
  dataInvariantStats,
40
40
  } from "./store/sqlite.js";
41
+ import { rehydrateRaptorTree } from "./dedup/raptor/index.js";
42
+ import { stagedExpansion } from "./dedup/raptor/retrieval.js";
41
43
  import { migrateJsonToSqlite } from "./store/migrate.js";
42
44
 
43
45
  export interface SearchHit {
@@ -66,6 +68,9 @@ export interface AddInput {
66
68
  * Lets the UI render live per-tier progress during compaction. Never awaited;
67
69
  * must be cheap. Optional for back-compat. */
68
70
  onTier?: (ev: { tier: "L0" | "L1" | "L2" | "new"; status: "scanning" | "deduped" | "passed" | "stored"; detail?: string }) => void;
71
+ /** Context-window pressure (0–1) — escalates the stored checkpoint's sync
72
+ * compression strength (Fix E). Optional; defaults to 0 (brotli-4). */
73
+ compressionPressure?: number;
69
74
  }
70
75
 
71
76
  /** Default L2 semantic-dedup enable flag (trigram embedder is local, zero-network). */
@@ -318,7 +323,10 @@ export class VectorStore {
318
323
  contentHash2: digest.contentHash2,
319
324
  contentHashVersion: digest.contentHashVersion,
320
325
  normalizedText: digest.normalizedText,
321
- compressedOriginal: compressSmart(Buffer.from(input.regionText, "utf-8")),
326
+ compressedOriginal: compressSmart(
327
+ Buffer.from(input.regionText, "utf-8"),
328
+ input.compressionPressure,
329
+ ),
322
330
  embedding,
323
331
  timestamp: input.timestamp,
324
332
  };
@@ -432,6 +440,29 @@ export class VectorStore {
432
440
  // MMR (QA #10) is part of the L2 semantic tier: skip it when L2 is disabled
433
441
  // (Sprint 14 flag), returning the plain relevance-ranked window instead.
434
442
  if (!this.cfg.L2_ENABLED) return window.slice(0, k);
443
+
444
+ // Fix D: when RAPTOR is promoted, ALSO recall high-level tree summaries and
445
+ // merge them with the flat hits via MMR so RAPTOR + flat don't double-cover.
446
+ // RAPTOR returns fewer, broader hits (O(log n) high-level nodes) than the
447
+ // O(n) flat leaves, tightening the block at read time.
448
+ if (this.cfg.RAPTOR_ENABLED) {
449
+ const raptorHits = this.raptorSearchHits(sid, query, k);
450
+ if (raptorHits.length > 0) {
451
+ const merged: SearchHit[] = [...window];
452
+ for (const rh of raptorHits) {
453
+ if (!merged.some((m) => m.checkpoint.checkpointId === rh.checkpoint.checkpointId)) {
454
+ merged.push(rh);
455
+ }
456
+ }
457
+ const mmrItems: MmrItem<SearchHit>[] = merged.map((h) => ({
458
+ item: h,
459
+ vector: h.checkpoint.embedding,
460
+ relevance: h.score,
461
+ }));
462
+ return mmrRerank(mmrItems, k, this.cfg.MMR_LAMBDA);
463
+ }
464
+ }
465
+
435
466
  const mmrItems: MmrItem<SearchHit>[] = window.map((h) => ({
436
467
  item: h,
437
468
  vector: h.checkpoint.embedding,
@@ -441,6 +472,37 @@ export class VectorStore {
441
472
  return ranked;
442
473
  }
443
474
 
475
+ /**
476
+ * Serve the RAPTOR tree for a query (Fix D): rehydrate the persisted tree and
477
+ * return its staged-expansion leaf hits as SearchHits. Returns [] when no tree
478
+ * exists (small sessions — flat search remains the path). Best-effort/non-fatal.
479
+ */
480
+ private raptorSearchHits(sid: string, query: string, k: number): SearchHit[] {
481
+ try {
482
+ const tree = rehydrateRaptorTree(sid, this.stateDir);
483
+ if (!tree || !tree.rootId) return [];
484
+ const leafIds = stagedExpansion(query, tree, {
485
+ embedder: this.embedder,
486
+ k,
487
+ topM: this.cfg.RAPTOR_CLUSTERS_PER_LEVEL,
488
+ mmrLambda: this.cfg.MMR_LAMBDA,
489
+ });
490
+ if (leafIds.length === 0) return [];
491
+ const all = listCheckpoints(sid, this.stateDir).filter(
492
+ (cp) => cp.dedupStatus !== "removed",
493
+ );
494
+ const qv = this.embedder.embed(query);
495
+ const hits: SearchHit[] = [];
496
+ for (const id of leafIds) {
497
+ const cp = all.find((c) => c.checkpointId === id);
498
+ if (cp) hits.push({ checkpoint: cp, score: cosineSimilarity(qv, cp.embedding) });
499
+ }
500
+ return hits;
501
+ } catch {
502
+ return [];
503
+ }
504
+ }
505
+
444
506
  /**
445
507
  * SemDeDup offline cleanup (Sprint 12, QA #17): within a session, mark the
446
508
  * lower-quality row of any pair scoring cosine > `threshold` as
@@ -1,291 +0,0 @@
1
- /**
2
- * openclaw-mega-compact — OpenClaw plugin adapter for the pi-mega-compact engine.
3
- *
4
- * Wires the pi-agnostic Trident engine (src/) into OpenClaw's plugin lifecycle:
5
- * - Registers a CompactionProvider that replaces the built-in summarizeInStages.
6
- * - Exposes `mega_status` and `mega_recall` tools for on-demand inspection.
7
- * - Hooks into `before_compaction` / `after_compaction` for diagnostics.
8
- *
9
- * Design constraints:
10
- * - NO imports from `@earendil-works/pi-coding-agent` or pi-agent-core.
11
- * - The engine core (src/) is pi-agnostic; this file is the sole OpenClaw boundary.
12
- * - No network at runtime — everything is local (stores + extractive summarizer).
13
- */
14
- import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
15
- import { compactSession, setDefaultStore, } from "../src/engine.js";
16
- import { recallAndInline } from "../src/recall.js";
17
- import { VectorStore } from "../src/vectorStore.js";
18
- // ---------------------------------------------------------------------------
19
- // Constants
20
- // ---------------------------------------------------------------------------
21
- const PLUGIN_ID = "mega-compact";
22
- const PLUGIN_LABEL = "Mega Compact (Trident)";
23
- /** Default state directory for vector store persistence. */
24
- const STATE_DIR = process.env.MEGA_COMPACT_STATE_DIR ?? undefined;
25
- /** Minimum messages before we bother compacting. */
26
- const MIN_MESSAGES_FOR_COMPACT = 6;
27
- // ---------------------------------------------------------------------------
28
- // Message conversion — OpenClaw unknown[] → EngineMessage[]
29
- // ---------------------------------------------------------------------------
30
- /**
31
- * Best-effort conversion from OpenClaw's opaque message array to our
32
- * EngineMessage shape. OpenClaw messages are typed as `unknown[]` so we
33
- * handle whatever shape comes through gracefully.
34
- */
35
- function toEngineMessages(messages) {
36
- return messages.map((msg) => {
37
- if (!msg || typeof msg !== "object") {
38
- // Primitive fallback — treat as custom text.
39
- return {
40
- role: "custom",
41
- text: String(msg ?? ""),
42
- };
43
- }
44
- const m = msg;
45
- const role = typeof m.role === "string" ? m.role : "custom";
46
- // Normalize role to one of our four engine roles.
47
- let engineRole;
48
- switch (role) {
49
- case "user":
50
- engineRole = "user";
51
- break;
52
- case "assistant":
53
- engineRole = "assistant";
54
- break;
55
- case "tool":
56
- case "function":
57
- engineRole = "tool";
58
- break;
59
- default:
60
- engineRole = "custom";
61
- break;
62
- }
63
- // Extract text content from common message shapes.
64
- const text = typeof m.content === "string"
65
- ? m.content
66
- : typeof m.text === "string"
67
- ? m.text
68
- : Array.isArray(m.content)
69
- ? m.content
70
- .filter((part) => part.type === "text" && typeof part.text === "string")
71
- .map((part) => part.text)
72
- .join("\n")
73
- : "";
74
- // Preserve tool metadata when present.
75
- const toolName = typeof m.name === "string"
76
- ? m.name
77
- : typeof m.toolName === "string"
78
- ? m.toolName
79
- : undefined;
80
- const input = typeof m.input === "string"
81
- ? m.input
82
- : typeof m.arguments === "string"
83
- ? m.arguments
84
- : m.arguments !== undefined
85
- ? JSON.stringify(m.arguments)
86
- : undefined;
87
- const output = typeof m.output === "string"
88
- ? m.output
89
- : engineRole === "tool" && typeof m.content === "string"
90
- ? m.content
91
- : undefined;
92
- return { role: engineRole, text, toolName, input, output };
93
- });
94
- }
95
- // ---------------------------------------------------------------------------
96
- // Compaction provider
97
- // ---------------------------------------------------------------------------
98
- function createCompactionProvider(store) {
99
- return {
100
- id: PLUGIN_ID,
101
- label: PLUGIN_LABEL,
102
- async summarize({ messages, signal, compressionRatio, }) {
103
- // Abort check — bail early if the caller cancelled.
104
- if (signal?.aborted) {
105
- throw new DOMException("Aborted", "AbortError");
106
- }
107
- const engineMessages = toEngineMessages(messages);
108
- // Nothing meaningful to compact.
109
- if (engineMessages.length < MIN_MESSAGES_FOR_COMPACT) {
110
- return "";
111
- }
112
- // Map compression ratio → keepFrom boundary.
113
- // compressionRatio=0.5 means "compact the oldest 50%".
114
- // Default to compacting the oldest half if not specified.
115
- const ratio = compressionRatio ?? 0.5;
116
- const keepFrom = Math.max(MIN_MESSAGES_FOR_COMPACT, Math.floor(engineMessages.length * (1 - ratio)));
117
- // Abort check after conversion (conversion is cheap but check anyway).
118
- if (signal?.aborted) {
119
- throw new DOMException("Aborted", "AbortError");
120
- }
121
- const sessionId = `openclaw-${Date.now()}`;
122
- const input = {
123
- sessionId,
124
- messages: engineMessages,
125
- keepFrom,
126
- };
127
- const result = compactSession(input, store);
128
- if (result.skipped) {
129
- return "";
130
- }
131
- return result.summary;
132
- },
133
- };
134
- }
135
- // ---------------------------------------------------------------------------
136
- // Plugin entry
137
- // ---------------------------------------------------------------------------
138
- export default definePluginEntry({
139
- id: PLUGIN_ID,
140
- name: "Mega Compact",
141
- description: "Layered, local, vector-backed context compressor (Trident engine) for OpenClaw compaction.",
142
- register(api) {
143
- const logger = api.logger;
144
- // Resolve state directory — prefer plugin config override.
145
- const pluginCfg = (api.pluginConfig ?? {});
146
- const stateDir = typeof pluginCfg.stateDir === "string" && pluginCfg.stateDir.length > 0
147
- ? pluginCfg.stateDir
148
- : STATE_DIR;
149
- // Initialize vector store.
150
- let store;
151
- try {
152
- store = new VectorStore({ stateDir });
153
- setDefaultStore(store);
154
- logger.info?.(`${PLUGIN_ID}: vector store initialized (stateDir=${stateDir ?? "default"})`);
155
- }
156
- catch (err) {
157
- logger.error?.(`${PLUGIN_ID}: failed to init vector store:`, err);
158
- return; // Hard bail — no point registering if store is broken.
159
- }
160
- // -----------------------------------------------------------------------
161
- // Register compaction provider
162
- // -----------------------------------------------------------------------
163
- const provider = createCompactionProvider(store);
164
- api.registerCompactionProvider(provider);
165
- logger.info?.(`${PLUGIN_ID}: registered compaction provider "${provider.id}"`);
166
- // -----------------------------------------------------------------------
167
- // Hooks — before / after compaction diagnostics
168
- // -----------------------------------------------------------------------
169
- api.registerHook({
170
- event: "before_compaction",
171
- handler: async (ctx) => {
172
- const msgCount = Array.isArray(ctx?.messages) ? ctx.messages.length : 0;
173
- logger.info?.(`${PLUGIN_ID}: before_compaction — ${msgCount} messages in scope`);
174
- },
175
- });
176
- api.registerHook({
177
- event: "after_compaction",
178
- handler: async (ctx) => {
179
- const summaryLen = typeof ctx?.summary === "string" ? ctx.summary.length : 0;
180
- logger.info?.(`${PLUGIN_ID}: after_compaction — summary ${summaryLen} chars`);
181
- },
182
- });
183
- // -----------------------------------------------------------------------
184
- // Tool: mega_status
185
- // -----------------------------------------------------------------------
186
- api.registerTool({
187
- name: "mega_status",
188
- description: "Show the current status of the mega-compact engine: vector store stats, checkpoint count, and recent compaction activity.",
189
- parameters: {
190
- type: "object",
191
- properties: {
192
- sessionId: {
193
- type: "string",
194
- description: "Optional session ID to scope stats to.",
195
- },
196
- },
197
- additionalProperties: false,
198
- },
199
- handler: async (args) => {
200
- const sessionId = args?.sessionId ?? "global";
201
- try {
202
- const stats = store.stats(sessionId);
203
- const parts = [
204
- `**Mega Compact Status**`,
205
- `Session: ${sessionId}`,
206
- `Checkpoints: ${stats.checkpointCount}`,
207
- `Total tokens saved: ${stats.totalTokenEstimate}`,
208
- `Last checkpoint: ${stats.lastCheckpointId ?? "—"}`,
209
- `Injected count: ${stats.injectedCount}`,
210
- `Dedup hit rate: ${(stats.dedupHitRate * 100).toFixed(0)}%`,
211
- ];
212
- if (stats.lastSummary) {
213
- parts.push(`\nLast summary (truncated):\n ${stats.lastSummary.slice(0, 120).replace(/\n/g, " ")}…`);
214
- }
215
- return { content: [{ type: "text", text: parts.join("\n") }] };
216
- }
217
- catch (err) {
218
- return {
219
- content: [{ type: "text", text: `Error reading mega-compact status: ${err}` }],
220
- isError: true,
221
- };
222
- }
223
- },
224
- });
225
- // -----------------------------------------------------------------------
226
- // Tool: mega_recall
227
- // -----------------------------------------------------------------------
228
- api.registerTool({
229
- name: "mega_recall",
230
- description: "Recall and inline relevant context from the mega-compact vector store for the current session.",
231
- parameters: {
232
- type: "object",
233
- properties: {
234
- sessionId: {
235
- type: "string",
236
- description: "Session ID to recall context for.",
237
- },
238
- query: {
239
- type: "string",
240
- description: "Natural language query for relevant context.",
241
- },
242
- limit: {
243
- type: "number",
244
- description: "Max checkpoints to recall (default 3).",
245
- },
246
- },
247
- required: ["sessionId", "query"],
248
- additionalProperties: false,
249
- },
250
- handler: async (args) => {
251
- const { sessionId, query, limit } = args;
252
- if (!sessionId || !query) {
253
- return {
254
- content: [{ type: "text", text: "Both `sessionId` and `query` are required." }],
255
- isError: true,
256
- };
257
- }
258
- try {
259
- const result = recallAndInline({ sessionId, query, limit: limit ?? 3, source: "command", skipInjected: false }, store);
260
- if (result.toInject.length === 0) {
261
- return {
262
- content: [{ type: "text", text: "No relevant context found in the mega-compact store." }],
263
- };
264
- }
265
- const parts = [
266
- `**Recalled ${result.toInject.length} checkpoint(s):**`,
267
- ...result.report,
268
- "",
269
- "---",
270
- result.block,
271
- ];
272
- return { content: [{ type: "text", text: parts.join("\n") }] };
273
- }
274
- catch (err) {
275
- return {
276
- content: [{ type: "text", text: `Error during mega-recall: ${err}` }],
277
- isError: true,
278
- };
279
- }
280
- },
281
- });
282
- // -----------------------------------------------------------------------
283
- // Cleanup on shutdown
284
- // -----------------------------------------------------------------------
285
- api.on("shutdown", () => {
286
- logger.info?.(`${PLUGIN_ID}: shutting down — clearing default store`);
287
- setDefaultStore(undefined);
288
- });
289
- logger.info?.(`${PLUGIN_ID}: plugin registered (tools: mega_status, mega_recall)`);
290
- },
291
- });