pi-mega-compact 0.4.27 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +47 -2
- package/dist/extensions/dashboard-server.js +58 -2
- package/dist/extensions/dashboard-server.test.js +95 -3
- package/dist/extensions/mega-commands.js +25 -9
- package/dist/extensions/mega-compact.test.js +161 -29
- package/dist/extensions/mega-config.js +5 -0
- package/dist/extensions/mega-conflict-cmds.js +79 -0
- package/dist/extensions/mega-dashboard-cmds.js +6 -4
- package/dist/extensions/mega-events.js +145 -20
- package/dist/extensions/mega-pipeline.js +179 -1
- package/dist/extensions/mega-runtime.js +14 -0
- package/dist/extensions/mega-trim.js +48 -0
- package/dist/extensions/mega-trim.test.js +58 -0
- package/dist/src/config/dedup.js +1 -0
- package/dist/src/driftDetection.js +103 -0
- package/dist/src/driftDetection.test.js +87 -0
- package/dist/src/memory.js +147 -0
- package/dist/src/memory.test.js +41 -0
- package/dist/src/memoryConsolidate.test.js +38 -0
- package/dist/src/memoryOps.js +58 -0
- package/dist/src/memoryOps.test.js +41 -0
- package/dist/src/memoryRecall.js +60 -0
- package/dist/src/memoryRecall.test.js +92 -0
- package/dist/src/recall.js +70 -1
- package/dist/src/recall.test.js +69 -1
- package/dist/src/store/sqlite.js +127 -11
- package/dist/src/vectorStore.js +6 -1
- package/extensions/dashboard-server.test.ts +115 -3
- package/extensions/dashboard-server.ts +63 -2
- package/extensions/mega-commands.ts +24 -9
- package/extensions/mega-compact.test.ts +162 -29
- package/extensions/mega-config.ts +22 -0
- package/extensions/mega-conflict-cmds.ts +81 -0
- package/extensions/mega-dashboard-cmds.ts +6 -4
- package/extensions/mega-events.ts +139 -20
- package/extensions/mega-pipeline.ts +179 -1
- package/extensions/mega-runtime.ts +15 -0
- package/extensions/mega-trim.test.ts +64 -0
- package/extensions/mega-trim.ts +75 -0
- package/extensions/openclaw-mega-compact.ts +24 -9
- package/package.json +2 -2
- package/src/config/dedup.ts +2 -0
- package/src/driftDetection.test.ts +100 -0
- package/src/driftDetection.ts +136 -0
- package/src/memory.test.ts +46 -0
- package/src/memory.ts +164 -0
- package/src/memoryConsolidate.test.ts +47 -0
- package/src/memoryOps.test.ts +53 -0
- package/src/memoryOps.ts +75 -0
- package/src/memoryRecall.test.ts +100 -0
- package/src/memoryRecall.ts +83 -0
- package/src/recall.test.ts +77 -1
- package/src/recall.ts +94 -1
- package/src/store/sqlite.ts +188 -11
- package/src/store.ts +3 -0
- package/src/vectorStore.ts +10 -1
- package/dist/extensions/openclaw-mega-compact.js +0 -291
- package/dist/src/minilm.js +0 -92
- package/dist/src/wordpiece.js +0 -129
package/src/store/sqlite.ts
CHANGED
|
@@ -57,7 +57,17 @@ const cache = new Map<string, DatabaseSync>();
|
|
|
57
57
|
/** Open (or reuse) the SQLite store for a state dir. */
|
|
58
58
|
export function openStore(stateDir: string = getStateDir()): DatabaseSync {
|
|
59
59
|
const existing = cache.get(stateDir);
|
|
60
|
-
if (existing)
|
|
60
|
+
if (existing) {
|
|
61
|
+
// A closed handle in the cache (e.g. a test calling db.close() directly
|
|
62
|
+
// instead of closeStore) would surface as "database is not open" on the
|
|
63
|
+
// next reuse. Detect and evict so callers never see a dead handle.
|
|
64
|
+
try {
|
|
65
|
+
existing.prepare("SELECT 1");
|
|
66
|
+
return existing;
|
|
67
|
+
} catch {
|
|
68
|
+
cache.delete(stateDir);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
61
71
|
|
|
62
72
|
if (!existsSync(stateDir)) mkdirSync(stateDir, { recursive: true });
|
|
63
73
|
const db = new DatabaseSync(join(stateDir, "sqlite.db"));
|
|
@@ -121,6 +131,19 @@ export function openIndexStore(indexDir: string = getIndexDir()): DatabaseSync {
|
|
|
121
131
|
model_captured_at INTEGER
|
|
122
132
|
);
|
|
123
133
|
CREATE INDEX IF NOT EXISTS idx_registry_last_seen ON repo_registry(last_seen DESC);
|
|
134
|
+
-- S18: machine-wide injected-set. A foreign checkpoint injected in repo A is
|
|
135
|
+
-- recorded here so repo B's recall never re-injects it. Keyed by checkpoint
|
|
136
|
+
-- + session (a checkpoint may be injected once per session); repo_id is the
|
|
137
|
+
-- source repo (the foreign repo's stateDir) for tracking/source labels.
|
|
138
|
+
-- PRAMETERIZED queries (PREVENT-002); local node:sqlite (PREVENT-PI-004).
|
|
139
|
+
CREATE TABLE IF NOT EXISTS injected_global (
|
|
140
|
+
checkpoint_id TEXT NOT NULL,
|
|
141
|
+
repo_id TEXT NOT NULL,
|
|
142
|
+
session_id TEXT NOT NULL,
|
|
143
|
+
injected_at INTEGER NOT NULL,
|
|
144
|
+
PRIMARY KEY (checkpoint_id, session_id)
|
|
145
|
+
);
|
|
146
|
+
CREATE INDEX IF NOT EXISTS idx_injected_global_cid ON injected_global(checkpoint_id);
|
|
124
147
|
`);
|
|
125
148
|
indexCache = iddb;
|
|
126
149
|
indexCacheDir = indexDir;
|
|
@@ -160,6 +183,19 @@ export function upsertRepoRegistry(
|
|
|
160
183
|
tokensSaved: number;
|
|
161
184
|
compressedOriginalBytes: number;
|
|
162
185
|
lastCompactedAt?: number | null;
|
|
186
|
+
// The fields below are optional passthroughs so test fixtures and the
|
|
187
|
+
// /api/repos active-window filter can seed them directly. They're also
|
|
188
|
+
// written by other paths (recordRepoModel, registry refresh) — passing
|
|
189
|
+
// them here is harmless because the ON CONFLICT clause keeps first_seen
|
|
190
|
+
// and the model columns from being clobbered.
|
|
191
|
+
firstSeen?: number;
|
|
192
|
+
lastSeen?: number;
|
|
193
|
+
provider?: string | null;
|
|
194
|
+
providerName?: string | null;
|
|
195
|
+
modelName?: string | null;
|
|
196
|
+
inputRate?: number | null;
|
|
197
|
+
outputRate?: number | null;
|
|
198
|
+
modelCapturedAt?: number | null;
|
|
163
199
|
},
|
|
164
200
|
indexDir: string = getIndexDir(),
|
|
165
201
|
): void {
|
|
@@ -168,26 +204,42 @@ export function upsertRepoRegistry(
|
|
|
168
204
|
db.prepare(
|
|
169
205
|
`INSERT INTO repo_registry
|
|
170
206
|
(repo_root, display_name, state_dir, first_seen, last_seen, last_compacted_at,
|
|
171
|
-
checkpoint_count, tokens_saved, compressed_original_bytes
|
|
172
|
-
|
|
173
|
-
|
|
207
|
+
checkpoint_count, tokens_saved, compressed_original_bytes,
|
|
208
|
+
provider, provider_name, model_name, input_rate, output_rate, model_captured_at)
|
|
209
|
+
VALUES (@repo_root, @display_name, @state_dir, @first_seen, @last_seen, @last_compacted_at,
|
|
210
|
+
@checkpoint_count, @tokens_saved, @compressed_original_bytes,
|
|
211
|
+
@provider, @provider_name, @model_name, @input_rate, @output_rate, @model_captured_at)
|
|
174
212
|
ON CONFLICT(repo_root) DO UPDATE SET
|
|
175
213
|
display_name = excluded.display_name,
|
|
176
214
|
state_dir = excluded.state_dir,
|
|
177
|
-
last_seen = excluded.last_seen,
|
|
215
|
+
last_seen = COALESCE(excluded.last_seen, @now),
|
|
178
216
|
last_compacted_at = COALESCE(excluded.last_compacted_at, repo_registry.last_compacted_at),
|
|
179
217
|
checkpoint_count = excluded.checkpoint_count,
|
|
180
218
|
tokens_saved = excluded.tokens_saved,
|
|
181
|
-
compressed_original_bytes = excluded.compressed_original_bytes
|
|
219
|
+
compressed_original_bytes = excluded.compressed_original_bytes,
|
|
220
|
+
provider = COALESCE(excluded.provider, repo_registry.provider),
|
|
221
|
+
provider_name = COALESCE(excluded.provider_name, repo_registry.provider_name),
|
|
222
|
+
model_name = COALESCE(excluded.model_name, repo_registry.model_name),
|
|
223
|
+
input_rate = COALESCE(excluded.input_rate, repo_registry.input_rate),
|
|
224
|
+
output_rate = COALESCE(excluded.output_rate, repo_registry.output_rate),
|
|
225
|
+
model_captured_at = COALESCE(excluded.model_captured_at, repo_registry.model_captured_at)`,
|
|
182
226
|
).run({
|
|
183
227
|
repo_root: row.repoRoot,
|
|
184
228
|
display_name: row.displayName,
|
|
185
229
|
state_dir: row.stateDir,
|
|
186
230
|
now,
|
|
231
|
+
first_seen: row.firstSeen ?? null,
|
|
232
|
+
last_seen: row.lastSeen ?? null,
|
|
187
233
|
last_compacted_at: row.lastCompactedAt ?? null,
|
|
188
234
|
checkpoint_count: row.checkpointCount,
|
|
189
235
|
tokens_saved: row.tokensSaved,
|
|
190
236
|
compressed_original_bytes: row.compressedOriginalBytes,
|
|
237
|
+
provider: row.provider ?? null,
|
|
238
|
+
provider_name: row.providerName ?? null,
|
|
239
|
+
model_name: row.modelName ?? null,
|
|
240
|
+
input_rate: row.inputRate ?? null,
|
|
241
|
+
output_rate: row.outputRate ?? null,
|
|
242
|
+
model_captured_at: row.modelCapturedAt ?? null,
|
|
191
243
|
});
|
|
192
244
|
}
|
|
193
245
|
|
|
@@ -281,6 +333,51 @@ export function closeIndexStore(): void {
|
|
|
281
333
|
}
|
|
282
334
|
}
|
|
283
335
|
|
|
336
|
+
// ---------------------------------------------------------------------------
|
|
337
|
+
// S18: machine-wide injected-set (cross-repo dedup markers)
|
|
338
|
+
//
|
|
339
|
+
// A foreign checkpoint injected in repo A is recorded here so repo B's recall
|
|
340
|
+
// never re-injects it (a stronger, machine-wide version of the per-session
|
|
341
|
+
// injected-set in the local store). Keyed by (checkpoint_id, session_id); the
|
|
342
|
+
// session_id here is the RECEIVING session, so the same foreign checkpoint can
|
|
343
|
+
// be injected into different sessions but never twice into the same one.
|
|
344
|
+
// PRAMETERIZED queries (PREVENT-002); local node:sqlite + WAL (PREVENT-PI-004),
|
|
345
|
+
// multi-process safe.
|
|
346
|
+
// ---------------------------------------------------------------------------
|
|
347
|
+
|
|
348
|
+
/** Record that a (foreign) checkpoint was injected into `sessionId`. Idempotent. */
|
|
349
|
+
export function markInjectedGlobal(
|
|
350
|
+
checkpointId: string,
|
|
351
|
+
repoId: string,
|
|
352
|
+
sessionId: string,
|
|
353
|
+
indexDir: string = getIndexDir(),
|
|
354
|
+
): void {
|
|
355
|
+
const db = openIndexStore(indexDir);
|
|
356
|
+
db.prepare(
|
|
357
|
+
"INSERT OR IGNORE INTO injected_global (checkpoint_id, repo_id, session_id, injected_at) VALUES ($cid, $rid, $sid, $ts)",
|
|
358
|
+
).run({ $cid: checkpointId, $rid: repoId, $sid: sessionId, $ts: Date.now() });
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
/** True when a checkpoint was already injected into `sessionId` (machine-wide). */
|
|
362
|
+
export function wasInjectedGlobal(
|
|
363
|
+
checkpointId: string,
|
|
364
|
+
sessionId: string,
|
|
365
|
+
indexDir: string = getIndexDir(),
|
|
366
|
+
): boolean {
|
|
367
|
+
const db = openIndexStore(indexDir);
|
|
368
|
+
const row = db.prepare(
|
|
369
|
+
"SELECT 1 FROM injected_global WHERE checkpoint_id = $cid AND session_id = $sid LIMIT 1",
|
|
370
|
+
).get({ $cid: checkpointId, $sid: sessionId }) as { "1": number } | undefined;
|
|
371
|
+
return row !== undefined;
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
/** Count of cross-repo injections recorded (for /mega-status stats). */
|
|
375
|
+
export function countInjectedGlobal(indexDir: string = getIndexDir()): number {
|
|
376
|
+
const db = openIndexStore(indexDir);
|
|
377
|
+
const row = db.prepare("SELECT COUNT(*) AS n FROM injected_global").get() as { n: number } | undefined;
|
|
378
|
+
return row?.n ?? 0;
|
|
379
|
+
}
|
|
380
|
+
|
|
284
381
|
function initSchema(db: DatabaseSync): void {
|
|
285
382
|
db.exec(`
|
|
286
383
|
CREATE TABLE IF NOT EXISTS context_chunks (
|
|
@@ -430,7 +527,12 @@ function initSchema(db: DatabaseSync): void {
|
|
|
430
527
|
content TEXT NOT NULL,
|
|
431
528
|
tags TEXT, -- JSON array of strings
|
|
432
529
|
created_at INTEGER,
|
|
433
|
-
last_recalled_at INTEGER
|
|
530
|
+
last_recalled_at INTEGER,
|
|
531
|
+
-- S20 memory-RAG extension (auto-review add/replace/remove ops).
|
|
532
|
+
category TEXT, -- typed bucket, e.g. decision | fact | preference
|
|
533
|
+
target TEXT, -- optional subject/scope this memory targets
|
|
534
|
+
last_referenced INTEGER, -- last time memory was referenced by recall (epoch s)
|
|
535
|
+
source_turn INTEGER -- conversation turn that produced this memory
|
|
434
536
|
);
|
|
435
537
|
CREATE INDEX IF NOT EXISTS idx_memories_repo ON memories(repo);
|
|
436
538
|
|
|
@@ -447,6 +549,12 @@ function initSchema(db: DatabaseSync): void {
|
|
|
447
549
|
// databases created by an older version — otherwise repoStats()/upsert crash
|
|
448
550
|
// with "no such column" and the extension fails to load. Additive only.
|
|
449
551
|
ensureColumn(db, "context_chunks", "original_token_estimate", "INTEGER");
|
|
552
|
+
// S20 memory-RAG extension: additive columns for auto-review ops. Idempotent —
|
|
553
|
+
// only alters DBs created by an older version that lack these columns.
|
|
554
|
+
ensureColumn(db, "memories", "category", "TEXT");
|
|
555
|
+
ensureColumn(db, "memories", "target", "TEXT");
|
|
556
|
+
ensureColumn(db, "memories", "last_referenced", "INTEGER");
|
|
557
|
+
ensureColumn(db, "memories", "source_turn", "INTEGER");
|
|
450
558
|
const v = db.prepare("SELECT value FROM meta WHERE key='schema_version'").get() as
|
|
451
559
|
| { value: string }
|
|
452
560
|
| undefined;
|
|
@@ -613,11 +721,15 @@ export interface MemoryRecord {
|
|
|
613
721
|
tags: string[];
|
|
614
722
|
createdAt: number;
|
|
615
723
|
lastRecalledAt: number | null;
|
|
724
|
+
category: string | null;
|
|
725
|
+
target: string | null;
|
|
726
|
+
lastReferenced: number | null;
|
|
727
|
+
sourceTurn: number | null;
|
|
616
728
|
}
|
|
617
729
|
|
|
618
730
|
/** Save a memory to the current repo's store. Returns the new row id. */
|
|
619
731
|
export function addMemory(
|
|
620
|
-
memory: { kind?: string; content: string; tags?: string[] },
|
|
732
|
+
memory: { kind?: string; content: string; tags?: string[]; category?: string; target?: string; sourceTurn?: number },
|
|
621
733
|
repo: string | null,
|
|
622
734
|
stateDir: string = getStateDir(),
|
|
623
735
|
): number {
|
|
@@ -625,10 +737,19 @@ export function addMemory(
|
|
|
625
737
|
const now = Math.floor(Date.now() / 1000);
|
|
626
738
|
const res = db
|
|
627
739
|
.prepare(
|
|
628
|
-
`INSERT INTO memories(repo, kind, content, tags, created_at, last_recalled_at)
|
|
629
|
-
VALUES(?, ?, ?, ?, ?, NULL)`,
|
|
740
|
+
`INSERT INTO memories(repo, kind, content, tags, created_at, last_recalled_at, category, target, source_turn)
|
|
741
|
+
VALUES(?, ?, ?, ?, ?, NULL, ?, ?, ?)`,
|
|
630
742
|
)
|
|
631
|
-
.run(
|
|
743
|
+
.run(
|
|
744
|
+
repo ?? null,
|
|
745
|
+
memory.kind ?? "note",
|
|
746
|
+
memory.content,
|
|
747
|
+
JSON.stringify(memory.tags ?? []),
|
|
748
|
+
now,
|
|
749
|
+
memory.category ?? null,
|
|
750
|
+
memory.target ?? null,
|
|
751
|
+
memory.sourceTurn ?? null,
|
|
752
|
+
);
|
|
632
753
|
return Number(res.lastInsertRowid);
|
|
633
754
|
}
|
|
634
755
|
|
|
@@ -659,6 +780,58 @@ export function recallMemory(id: number, stateDir: string = getStateDir()): bool
|
|
|
659
780
|
return res.changes > 0;
|
|
660
781
|
}
|
|
661
782
|
|
|
783
|
+
/** Mark a memory as referenced (updates last_referenced). Returns true if found. */
|
|
784
|
+
export function referenceMemory(id: number, stateDir: string = getStateDir()): boolean {
|
|
785
|
+
const db = openStore(stateDir);
|
|
786
|
+
const now = Math.floor(Date.now() / 1000);
|
|
787
|
+
const res = db.prepare("UPDATE memories SET last_referenced = ? WHERE id = ?").run(now, id);
|
|
788
|
+
return res.changes > 0;
|
|
789
|
+
}
|
|
790
|
+
|
|
791
|
+
/** Replace a memory's mutable fields by id. Returns true if a row was updated. */
|
|
792
|
+
export function replaceMemory(
|
|
793
|
+
id: number,
|
|
794
|
+
patch: { kind?: string; content?: string; tags?: string[]; category?: string; target?: string; sourceTurn?: number },
|
|
795
|
+
stateDir: string = getStateDir(),
|
|
796
|
+
): boolean {
|
|
797
|
+
const db = openStore(stateDir);
|
|
798
|
+
const res = db
|
|
799
|
+
.prepare(
|
|
800
|
+
`UPDATE memories
|
|
801
|
+
SET kind = COALESCE(?, kind),
|
|
802
|
+
content = COALESCE(?, content),
|
|
803
|
+
tags = COALESCE(?, tags),
|
|
804
|
+
category = COALESCE(?, category),
|
|
805
|
+
target = COALESCE(?, target),
|
|
806
|
+
source_turn = COALESCE(?, source_turn)
|
|
807
|
+
WHERE id = ?`,
|
|
808
|
+
)
|
|
809
|
+
.run(
|
|
810
|
+
patch.kind ?? null,
|
|
811
|
+
patch.content ?? null,
|
|
812
|
+
patch.tags ? JSON.stringify(patch.tags) : null,
|
|
813
|
+
"category" in patch ? (patch.category ?? null) : null,
|
|
814
|
+
"target" in patch ? (patch.target ?? null) : null,
|
|
815
|
+
"sourceTurn" in patch ? (patch.sourceTurn ?? null) : null,
|
|
816
|
+
id,
|
|
817
|
+
);
|
|
818
|
+
return res.changes > 0;
|
|
819
|
+
}
|
|
820
|
+
|
|
821
|
+
/** Remove a memory by id. Returns true if a row was deleted. */
|
|
822
|
+
export function removeMemory(id: number, stateDir: string = getStateDir()): boolean {
|
|
823
|
+
const db = openStore(stateDir);
|
|
824
|
+
const res = db.prepare("DELETE FROM memories WHERE id = ?").run(id);
|
|
825
|
+
return res.changes > 0;
|
|
826
|
+
}
|
|
827
|
+
|
|
828
|
+
/** Look up a single memory by id (or undefined). */
|
|
829
|
+
export function getMemory(id: number, stateDir: string = getStateDir()): MemoryRecord | undefined {
|
|
830
|
+
const db = openStore(stateDir);
|
|
831
|
+
const row = db.prepare("SELECT * FROM memories WHERE id = ?").get(id);
|
|
832
|
+
return row ? mapMemoryRow(row) : undefined;
|
|
833
|
+
}
|
|
834
|
+
|
|
662
835
|
function mapMemoryRow(row: any): MemoryRecord {
|
|
663
836
|
return {
|
|
664
837
|
id: row.id,
|
|
@@ -668,6 +841,10 @@ function mapMemoryRow(row: any): MemoryRecord {
|
|
|
668
841
|
tags: row.tags ? JSON.parse(row.tags) : [],
|
|
669
842
|
createdAt: row.created_at ?? 0,
|
|
670
843
|
lastRecalledAt: row.last_recalled_at ?? null,
|
|
844
|
+
category: row.category ?? null,
|
|
845
|
+
target: row.target ?? null,
|
|
846
|
+
lastReferenced: row.last_referenced ?? null,
|
|
847
|
+
sourceTurn: row.source_turn ?? null,
|
|
671
848
|
};
|
|
672
849
|
}
|
|
673
850
|
|
package/src/store.ts
CHANGED
|
@@ -49,6 +49,9 @@ export function normalizeSessionId(sessionId: string | undefined | null): string
|
|
|
49
49
|
export interface StoredCheckpoint {
|
|
50
50
|
checkpointId: string;
|
|
51
51
|
sessionId: string;
|
|
52
|
+
/** Source repo id (foreign stateDir) when this checkpoint came from a global
|
|
53
|
+
* cross-repo index entry. Undefined for same-repo checkpoints. Spec S17.1. */
|
|
54
|
+
repoId?: string;
|
|
52
55
|
summary: string;
|
|
53
56
|
/** Compressed topic summary (extractive, ~2K tokens vs ~70K raw). */
|
|
54
57
|
topicSummary?: string;
|
package/src/vectorStore.ts
CHANGED
|
@@ -51,6 +51,10 @@ import { migrateJsonToSqlite } from "./store/migrate.js";
|
|
|
51
51
|
export interface SearchHit {
|
|
52
52
|
checkpoint: StoredCheckpoint;
|
|
53
53
|
score: number;
|
|
54
|
+
/** Source repo id (the foreign repo's stateDir) for cross-repo hits, set by
|
|
55
|
+
* `searchAsync` so the recall block can label foreign checkpoints. Undefined
|
|
56
|
+
* for same-repo hits (the default path). */
|
|
57
|
+
repoId?: string;
|
|
54
58
|
}
|
|
55
59
|
|
|
56
60
|
export interface AddInput {
|
|
@@ -326,6 +330,7 @@ export class VectorStore {
|
|
|
326
330
|
const checkpoint: StoredCheckpoint = {
|
|
327
331
|
checkpointId,
|
|
328
332
|
sessionId,
|
|
333
|
+
repoId: this.repoId,
|
|
329
334
|
summary: input.summary,
|
|
330
335
|
topicSummary: input.topicSummary,
|
|
331
336
|
summaryHash,
|
|
@@ -525,11 +530,15 @@ export class VectorStore {
|
|
|
525
530
|
}
|
|
526
531
|
// Hydrate each index hit from the authoritative node:sqlite store. repoId is
|
|
527
532
|
// that repo's stateDir, so cross-repo hits resolve against their own store.
|
|
533
|
+
// Tag cross-repo hits with their source repoId so the recall block can label
|
|
534
|
+
// them ("from repo <name>"); same-repo hits stay unlabeled.
|
|
535
|
+
const selfRepo = this.repoId;
|
|
528
536
|
const hydrated: SearchHit[] = [];
|
|
529
537
|
for (const h of indexHits) {
|
|
530
538
|
const cp = getCheckpoint(h.sessionId, h.checkpointId, h.repoId);
|
|
531
539
|
if (cp && cp.dedupStatus !== "removed") {
|
|
532
|
-
|
|
540
|
+
const crossRepo = opts.crossRepo && selfRepo && h.repoId && h.repoId !== selfRepo;
|
|
541
|
+
hydrated.push({ checkpoint: cp, score: h.score, repoId: crossRepo ? h.repoId : undefined });
|
|
533
542
|
}
|
|
534
543
|
}
|
|
535
544
|
if (hydrated.length === 0) return this.search(sid, query, k);
|
|
@@ -1,291 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* openclaw-mega-compact — OpenClaw plugin adapter for the pi-mega-compact engine.
|
|
3
|
-
*
|
|
4
|
-
* Wires the pi-agnostic Trident engine (src/) into OpenClaw's plugin lifecycle:
|
|
5
|
-
* - Registers a CompactionProvider that replaces the built-in summarizeInStages.
|
|
6
|
-
* - Exposes `mega_status` and `mega_recall` tools for on-demand inspection.
|
|
7
|
-
* - Hooks into `before_compaction` / `after_compaction` for diagnostics.
|
|
8
|
-
*
|
|
9
|
-
* Design constraints:
|
|
10
|
-
* - NO imports from `@earendil-works/pi-coding-agent` or pi-agent-core.
|
|
11
|
-
* - The engine core (src/) is pi-agnostic; this file is the sole OpenClaw boundary.
|
|
12
|
-
* - No network at runtime — everything is local (stores + extractive summarizer).
|
|
13
|
-
*/
|
|
14
|
-
import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
|
|
15
|
-
import { compactSession, setDefaultStore, } from "../src/engine.js";
|
|
16
|
-
import { recallAndInline } from "../src/recall.js";
|
|
17
|
-
import { VectorStore } from "../src/vectorStore.js";
|
|
18
|
-
// ---------------------------------------------------------------------------
|
|
19
|
-
// Constants
|
|
20
|
-
// ---------------------------------------------------------------------------
|
|
21
|
-
const PLUGIN_ID = "mega-compact";
|
|
22
|
-
const PLUGIN_LABEL = "Mega Compact (Trident)";
|
|
23
|
-
/** Default state directory for vector store persistence. */
|
|
24
|
-
const STATE_DIR = process.env.MEGA_COMPACT_STATE_DIR ?? undefined;
|
|
25
|
-
/** Minimum messages before we bother compacting. */
|
|
26
|
-
const MIN_MESSAGES_FOR_COMPACT = 6;
|
|
27
|
-
// ---------------------------------------------------------------------------
|
|
28
|
-
// Message conversion — OpenClaw unknown[] → EngineMessage[]
|
|
29
|
-
// ---------------------------------------------------------------------------
|
|
30
|
-
/**
|
|
31
|
-
* Best-effort conversion from OpenClaw's opaque message array to our
|
|
32
|
-
* EngineMessage shape. OpenClaw messages are typed as `unknown[]` so we
|
|
33
|
-
* handle whatever shape comes through gracefully.
|
|
34
|
-
*/
|
|
35
|
-
function toEngineMessages(messages) {
|
|
36
|
-
return messages.map((msg) => {
|
|
37
|
-
if (!msg || typeof msg !== "object") {
|
|
38
|
-
// Primitive fallback — treat as custom text.
|
|
39
|
-
return {
|
|
40
|
-
role: "custom",
|
|
41
|
-
text: String(msg ?? ""),
|
|
42
|
-
};
|
|
43
|
-
}
|
|
44
|
-
const m = msg;
|
|
45
|
-
const role = typeof m.role === "string" ? m.role : "custom";
|
|
46
|
-
// Normalize role to one of our four engine roles.
|
|
47
|
-
let engineRole;
|
|
48
|
-
switch (role) {
|
|
49
|
-
case "user":
|
|
50
|
-
engineRole = "user";
|
|
51
|
-
break;
|
|
52
|
-
case "assistant":
|
|
53
|
-
engineRole = "assistant";
|
|
54
|
-
break;
|
|
55
|
-
case "tool":
|
|
56
|
-
case "function":
|
|
57
|
-
engineRole = "tool";
|
|
58
|
-
break;
|
|
59
|
-
default:
|
|
60
|
-
engineRole = "custom";
|
|
61
|
-
break;
|
|
62
|
-
}
|
|
63
|
-
// Extract text content from common message shapes.
|
|
64
|
-
const text = typeof m.content === "string"
|
|
65
|
-
? m.content
|
|
66
|
-
: typeof m.text === "string"
|
|
67
|
-
? m.text
|
|
68
|
-
: Array.isArray(m.content)
|
|
69
|
-
? m.content
|
|
70
|
-
.filter((part) => part.type === "text" && typeof part.text === "string")
|
|
71
|
-
.map((part) => part.text)
|
|
72
|
-
.join("\n")
|
|
73
|
-
: "";
|
|
74
|
-
// Preserve tool metadata when present.
|
|
75
|
-
const toolName = typeof m.name === "string"
|
|
76
|
-
? m.name
|
|
77
|
-
: typeof m.toolName === "string"
|
|
78
|
-
? m.toolName
|
|
79
|
-
: undefined;
|
|
80
|
-
const input = typeof m.input === "string"
|
|
81
|
-
? m.input
|
|
82
|
-
: typeof m.arguments === "string"
|
|
83
|
-
? m.arguments
|
|
84
|
-
: m.arguments !== undefined
|
|
85
|
-
? JSON.stringify(m.arguments)
|
|
86
|
-
: undefined;
|
|
87
|
-
const output = typeof m.output === "string"
|
|
88
|
-
? m.output
|
|
89
|
-
: engineRole === "tool" && typeof m.content === "string"
|
|
90
|
-
? m.content
|
|
91
|
-
: undefined;
|
|
92
|
-
return { role: engineRole, text, toolName, input, output };
|
|
93
|
-
});
|
|
94
|
-
}
|
|
95
|
-
// ---------------------------------------------------------------------------
|
|
96
|
-
// Compaction provider
|
|
97
|
-
// ---------------------------------------------------------------------------
|
|
98
|
-
function createCompactionProvider(store) {
|
|
99
|
-
return {
|
|
100
|
-
id: PLUGIN_ID,
|
|
101
|
-
label: PLUGIN_LABEL,
|
|
102
|
-
async summarize({ messages, signal, compressionRatio, }) {
|
|
103
|
-
// Abort check — bail early if the caller cancelled.
|
|
104
|
-
if (signal?.aborted) {
|
|
105
|
-
throw new DOMException("Aborted", "AbortError");
|
|
106
|
-
}
|
|
107
|
-
const engineMessages = toEngineMessages(messages);
|
|
108
|
-
// Nothing meaningful to compact.
|
|
109
|
-
if (engineMessages.length < MIN_MESSAGES_FOR_COMPACT) {
|
|
110
|
-
return "";
|
|
111
|
-
}
|
|
112
|
-
// Map compression ratio → keepFrom boundary.
|
|
113
|
-
// compressionRatio=0.5 means "compact the oldest 50%".
|
|
114
|
-
// Default to compacting the oldest half if not specified.
|
|
115
|
-
const ratio = compressionRatio ?? 0.5;
|
|
116
|
-
const keepFrom = Math.max(MIN_MESSAGES_FOR_COMPACT, Math.floor(engineMessages.length * (1 - ratio)));
|
|
117
|
-
// Abort check after conversion (conversion is cheap but check anyway).
|
|
118
|
-
if (signal?.aborted) {
|
|
119
|
-
throw new DOMException("Aborted", "AbortError");
|
|
120
|
-
}
|
|
121
|
-
const sessionId = `openclaw-${Date.now()}`;
|
|
122
|
-
const input = {
|
|
123
|
-
sessionId,
|
|
124
|
-
messages: engineMessages,
|
|
125
|
-
keepFrom,
|
|
126
|
-
};
|
|
127
|
-
const result = compactSession(input, store);
|
|
128
|
-
if (result.skipped) {
|
|
129
|
-
return "";
|
|
130
|
-
}
|
|
131
|
-
return result.summary;
|
|
132
|
-
},
|
|
133
|
-
};
|
|
134
|
-
}
|
|
135
|
-
// ---------------------------------------------------------------------------
|
|
136
|
-
// Plugin entry
|
|
137
|
-
// ---------------------------------------------------------------------------
|
|
138
|
-
export default definePluginEntry({
|
|
139
|
-
id: PLUGIN_ID,
|
|
140
|
-
name: "Mega Compact",
|
|
141
|
-
description: "Layered, local, vector-backed context compressor (Trident engine) for OpenClaw compaction.",
|
|
142
|
-
register(api) {
|
|
143
|
-
const logger = api.logger;
|
|
144
|
-
// Resolve state directory — prefer plugin config override.
|
|
145
|
-
const pluginCfg = (api.pluginConfig ?? {});
|
|
146
|
-
const stateDir = typeof pluginCfg.stateDir === "string" && pluginCfg.stateDir.length > 0
|
|
147
|
-
? pluginCfg.stateDir
|
|
148
|
-
: STATE_DIR;
|
|
149
|
-
// Initialize vector store.
|
|
150
|
-
let store;
|
|
151
|
-
try {
|
|
152
|
-
store = new VectorStore({ stateDir });
|
|
153
|
-
setDefaultStore(store);
|
|
154
|
-
logger.info?.(`${PLUGIN_ID}: vector store initialized (stateDir=${stateDir ?? "default"})`);
|
|
155
|
-
}
|
|
156
|
-
catch (err) {
|
|
157
|
-
logger.error?.(`${PLUGIN_ID}: failed to init vector store:`, err);
|
|
158
|
-
return; // Hard bail — no point registering if store is broken.
|
|
159
|
-
}
|
|
160
|
-
// -----------------------------------------------------------------------
|
|
161
|
-
// Register compaction provider
|
|
162
|
-
// -----------------------------------------------------------------------
|
|
163
|
-
const provider = createCompactionProvider(store);
|
|
164
|
-
api.registerCompactionProvider(provider);
|
|
165
|
-
logger.info?.(`${PLUGIN_ID}: registered compaction provider "${provider.id}"`);
|
|
166
|
-
// -----------------------------------------------------------------------
|
|
167
|
-
// Hooks — before / after compaction diagnostics
|
|
168
|
-
// -----------------------------------------------------------------------
|
|
169
|
-
api.registerHook({
|
|
170
|
-
event: "before_compaction",
|
|
171
|
-
handler: async (ctx) => {
|
|
172
|
-
const msgCount = Array.isArray(ctx?.messages) ? ctx.messages.length : 0;
|
|
173
|
-
logger.info?.(`${PLUGIN_ID}: before_compaction — ${msgCount} messages in scope`);
|
|
174
|
-
},
|
|
175
|
-
});
|
|
176
|
-
api.registerHook({
|
|
177
|
-
event: "after_compaction",
|
|
178
|
-
handler: async (ctx) => {
|
|
179
|
-
const summaryLen = typeof ctx?.summary === "string" ? ctx.summary.length : 0;
|
|
180
|
-
logger.info?.(`${PLUGIN_ID}: after_compaction — summary ${summaryLen} chars`);
|
|
181
|
-
},
|
|
182
|
-
});
|
|
183
|
-
// -----------------------------------------------------------------------
|
|
184
|
-
// Tool: mega_status
|
|
185
|
-
// -----------------------------------------------------------------------
|
|
186
|
-
api.registerTool({
|
|
187
|
-
name: "mega_status",
|
|
188
|
-
description: "Show the current status of the mega-compact engine: vector store stats, checkpoint count, and recent compaction activity.",
|
|
189
|
-
parameters: {
|
|
190
|
-
type: "object",
|
|
191
|
-
properties: {
|
|
192
|
-
sessionId: {
|
|
193
|
-
type: "string",
|
|
194
|
-
description: "Optional session ID to scope stats to.",
|
|
195
|
-
},
|
|
196
|
-
},
|
|
197
|
-
additionalProperties: false,
|
|
198
|
-
},
|
|
199
|
-
handler: async (args) => {
|
|
200
|
-
const sessionId = args?.sessionId ?? "global";
|
|
201
|
-
try {
|
|
202
|
-
const stats = store.stats(sessionId);
|
|
203
|
-
const parts = [
|
|
204
|
-
`**Mega Compact Status**`,
|
|
205
|
-
`Session: ${sessionId}`,
|
|
206
|
-
`Checkpoints: ${stats.checkpointCount}`,
|
|
207
|
-
`Total tokens saved: ${stats.totalTokenEstimate}`,
|
|
208
|
-
`Last checkpoint: ${stats.lastCheckpointId ?? "—"}`,
|
|
209
|
-
`Injected count: ${stats.injectedCount}`,
|
|
210
|
-
`Dedup hit rate: ${(stats.dedupHitRate * 100).toFixed(0)}%`,
|
|
211
|
-
];
|
|
212
|
-
if (stats.lastSummary) {
|
|
213
|
-
parts.push(`\nLast summary (truncated):\n ${stats.lastSummary.slice(0, 120).replace(/\n/g, " ")}…`);
|
|
214
|
-
}
|
|
215
|
-
return { content: [{ type: "text", text: parts.join("\n") }] };
|
|
216
|
-
}
|
|
217
|
-
catch (err) {
|
|
218
|
-
return {
|
|
219
|
-
content: [{ type: "text", text: `Error reading mega-compact status: ${err}` }],
|
|
220
|
-
isError: true,
|
|
221
|
-
};
|
|
222
|
-
}
|
|
223
|
-
},
|
|
224
|
-
});
|
|
225
|
-
// -----------------------------------------------------------------------
|
|
226
|
-
// Tool: mega_recall
|
|
227
|
-
// -----------------------------------------------------------------------
|
|
228
|
-
api.registerTool({
|
|
229
|
-
name: "mega_recall",
|
|
230
|
-
description: "Recall and inline relevant context from the mega-compact vector store for the current session.",
|
|
231
|
-
parameters: {
|
|
232
|
-
type: "object",
|
|
233
|
-
properties: {
|
|
234
|
-
sessionId: {
|
|
235
|
-
type: "string",
|
|
236
|
-
description: "Session ID to recall context for.",
|
|
237
|
-
},
|
|
238
|
-
query: {
|
|
239
|
-
type: "string",
|
|
240
|
-
description: "Natural language query for relevant context.",
|
|
241
|
-
},
|
|
242
|
-
limit: {
|
|
243
|
-
type: "number",
|
|
244
|
-
description: "Max checkpoints to recall (default 3).",
|
|
245
|
-
},
|
|
246
|
-
},
|
|
247
|
-
required: ["sessionId", "query"],
|
|
248
|
-
additionalProperties: false,
|
|
249
|
-
},
|
|
250
|
-
handler: async (args) => {
|
|
251
|
-
const { sessionId, query, limit } = args;
|
|
252
|
-
if (!sessionId || !query) {
|
|
253
|
-
return {
|
|
254
|
-
content: [{ type: "text", text: "Both `sessionId` and `query` are required." }],
|
|
255
|
-
isError: true,
|
|
256
|
-
};
|
|
257
|
-
}
|
|
258
|
-
try {
|
|
259
|
-
const result = recallAndInline({ sessionId, query, limit: limit ?? 3, source: "command", skipInjected: false }, store);
|
|
260
|
-
if (result.toInject.length === 0) {
|
|
261
|
-
return {
|
|
262
|
-
content: [{ type: "text", text: "No relevant context found in the mega-compact store." }],
|
|
263
|
-
};
|
|
264
|
-
}
|
|
265
|
-
const parts = [
|
|
266
|
-
`**Recalled ${result.toInject.length} checkpoint(s):**`,
|
|
267
|
-
...result.report,
|
|
268
|
-
"",
|
|
269
|
-
"---",
|
|
270
|
-
result.block,
|
|
271
|
-
];
|
|
272
|
-
return { content: [{ type: "text", text: parts.join("\n") }] };
|
|
273
|
-
}
|
|
274
|
-
catch (err) {
|
|
275
|
-
return {
|
|
276
|
-
content: [{ type: "text", text: `Error during mega-recall: ${err}` }],
|
|
277
|
-
isError: true,
|
|
278
|
-
};
|
|
279
|
-
}
|
|
280
|
-
},
|
|
281
|
-
});
|
|
282
|
-
// -----------------------------------------------------------------------
|
|
283
|
-
// Cleanup on shutdown
|
|
284
|
-
// -----------------------------------------------------------------------
|
|
285
|
-
api.on("shutdown", () => {
|
|
286
|
-
logger.info?.(`${PLUGIN_ID}: shutting down — clearing default store`);
|
|
287
|
-
setDefaultStore(undefined);
|
|
288
|
-
});
|
|
289
|
-
logger.info?.(`${PLUGIN_ID}: plugin registered (tools: mega_status, mega_recall)`);
|
|
290
|
-
},
|
|
291
|
-
});
|