pi-mega-compact 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +24 -0
- package/README.md +375 -0
- package/extensions/DASHBOARD.md +160 -0
- package/extensions/dashboard-server.test.ts +124 -0
- package/extensions/dashboard-server.ts +459 -0
- package/extensions/error-patterns.ts +175 -0
- package/extensions/mega-compact.test.ts +351 -0
- package/extensions/mega-compact.ts +846 -0
- package/extensions/openclaw-mega-compact.ts +370 -0
- package/package.json +61 -0
- package/src/adapt.ts +120 -0
- package/src/boundary.test.ts +61 -0
- package/src/boundary.ts +94 -0
- package/src/canary.ts +126 -0
- package/src/compact.test.ts +99 -0
- package/src/compact.ts +262 -0
- package/src/config/dedup.ts +120 -0
- package/src/config.ts +15 -0
- package/src/dedup/dedup.test.ts +46 -0
- package/src/dedup/digest.ts +40 -0
- package/src/dedup/l1-lsh.ts +67 -0
- package/src/dedup/l1-minhash.ts +90 -0
- package/src/dedup/l1-verify.ts +55 -0
- package/src/dedup/l1.test.ts +57 -0
- package/src/dedup/mmr.ts +54 -0
- package/src/dedup/normalize.ts +41 -0
- package/src/dedup/raptor/guardrails.ts +112 -0
- package/src/dedup/raptor/index.ts +118 -0
- package/src/dedup/raptor/kmeans.ts +156 -0
- package/src/dedup/raptor/raptor.test.ts +238 -0
- package/src/dedup/raptor/retrieval.ts +102 -0
- package/src/dedup/raptor/summarizer.ts +91 -0
- package/src/dedup/raptor/tree.ts +254 -0
- package/src/dedup/sprint12.test.ts +242 -0
- package/src/dedup/topk.ts +61 -0
- package/src/dedup-engine.test.ts +609 -0
- package/src/e2e.test.ts +843 -0
- package/src/embedder.ts +111 -0
- package/src/engine.test.ts +123 -0
- package/src/engine.ts +192 -0
- package/src/extractive.test.ts +156 -0
- package/src/extractive.ts +265 -0
- package/src/httpEmbedder.ts +154 -0
- package/src/log.test.ts +47 -0
- package/src/log.ts +60 -0
- package/src/monitoring.ts +171 -0
- package/src/ratio.bench.test.ts +1316 -0
- package/src/recall.integration.test.ts +96 -0
- package/src/recall.test.ts +59 -0
- package/src/recall.ts +100 -0
- package/src/sprint14.test.ts +245 -0
- package/src/store/backfill.ts +263 -0
- package/src/store/bloom.ts +122 -0
- package/src/store/compression.test.ts +83 -0
- package/src/store/compression.ts +203 -0
- package/src/store/integrity.ts +65 -0
- package/src/store/migrate.test.ts +158 -0
- package/src/store/migrate.ts +108 -0
- package/src/store/sprint10.test.ts +182 -0
- package/src/store/sqlite.ts +519 -0
- package/src/store.test.ts +169 -0
- package/src/store.ts +192 -0
- package/src/supersede.test.ts +42 -0
- package/src/supersede.ts +67 -0
- package/src/tokens.ts +35 -0
- package/src/types.test.ts +10 -0
- package/src/types.ts +49 -0
- package/src/vectorStore.test.ts +480 -0
- package/src/vectorStore.ts +544 -0
package/src/embedder.ts
ADDED
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* embedder.ts — pluggable text embedding for the local vector store.
|
|
3
|
+
*
|
|
4
|
+
* Default embedder is a zero-dependency, deterministic hashed n-gram bag
|
|
5
|
+
* encoder — no native build, no network, no external library, works offline.
|
|
6
|
+
* It is heuristic-strength (good enough to rank "which checkpoint is relevant
|
|
7
|
+
* to this query?"), not RAG-grade. A stronger LOCAL embedding backend (your own
|
|
8
|
+
* localhost ONNX/TEI/Ollama server) can be plugged in via MEGACOMPACT_EMBEDDING_URL
|
|
9
|
+
* — see httpEmbedder.ts. The `Embedder` interface is the seam both implement;
|
|
10
|
+
* this extension ships no model and makes no remote call (PREVENT-PI-004).
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { HttpEmbedder, embeddingConfigFromEnv } from "./httpEmbedder.js";
|
|
14
|
+
|
|
15
|
+
export type Vector = number[];
|
|
16
|
+
|
|
17
|
+
/** Common embedding contract. Implementations must be deterministic. */
|
|
18
|
+
export interface Embedder {
|
|
19
|
+
/** Dimensionality of vectors this embedder produces. */
|
|
20
|
+
readonly dim: number;
|
|
21
|
+
embed(text: string): Vector;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** Normalize a vector to unit length (cosine-sim safe). Returns a new array. */
|
|
25
|
+
export function l2Normalize(v: Vector): Vector {
|
|
26
|
+
let sumSq = 0;
|
|
27
|
+
for (const x of v) sumSq += x * x;
|
|
28
|
+
const norm = Math.sqrt(sumSq);
|
|
29
|
+
if (norm === 0) return v.map(() => 0);
|
|
30
|
+
return v.map((x) => x / norm);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** Cosine similarity in [-1, 1]. Assumes inputs are same dim. */
|
|
34
|
+
export function cosineSimilarity(a: Vector, b: Vector): number {
|
|
35
|
+
if (a.length !== b.length) return 0;
|
|
36
|
+
let dot = 0;
|
|
37
|
+
let na = 0;
|
|
38
|
+
let nb = 0;
|
|
39
|
+
for (let i = 0; i < a.length; i++) {
|
|
40
|
+
dot += a[i] * b[i];
|
|
41
|
+
na += a[i] * a[i];
|
|
42
|
+
nb += b[i] * b[i];
|
|
43
|
+
}
|
|
44
|
+
if (na === 0 || nb === 0) return 0;
|
|
45
|
+
return dot / (Math.sqrt(na) * Math.sqrt(nb));
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** Stable 32-bit string hash (FNV-1a). */
|
|
49
|
+
function fnv1a(str: string): number {
|
|
50
|
+
let h = 0x811c9dc5;
|
|
51
|
+
for (let i = 0; i < str.length; i++) {
|
|
52
|
+
h ^= str.charCodeAt(i);
|
|
53
|
+
h = Math.imul(h, 0x01000193);
|
|
54
|
+
}
|
|
55
|
+
return h >>> 0;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Default embedder: character 3-gram bag-of-counts, hashed into a fixed-dim
|
|
60
|
+
* vector, L2-normalized. Captures local lexical/structure overlap well enough
|
|
61
|
+
* for checkpoint relevance ranking.
|
|
62
|
+
*/
|
|
63
|
+
export class TrigramEmbedder implements Embedder {
|
|
64
|
+
readonly dim: number;
|
|
65
|
+
private readonly seed: number;
|
|
66
|
+
|
|
67
|
+
constructor(dim = 512, seed = 0x9e3779b9) {
|
|
68
|
+
this.dim = dim;
|
|
69
|
+
this.seed = seed >>> 0;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
embed(text: string): Vector {
|
|
73
|
+
const vec = new Array<number>(this.dim).fill(0);
|
|
74
|
+
const norm = text.toLowerCase().replace(/\s+/g, " ");
|
|
75
|
+
if (norm.length === 0) return l2Normalize(vec);
|
|
76
|
+
// Whole-string + word + char-trigram signals.
|
|
77
|
+
vec[fnv1a(norm) % this.dim] += 1;
|
|
78
|
+
for (const word of norm.split(" ")) {
|
|
79
|
+
if (word.length === 0) continue;
|
|
80
|
+
vec[fnv1a(word) % this.dim] += 1;
|
|
81
|
+
for (let i = 0; i + 3 <= word.length; i++) {
|
|
82
|
+
const gram = word.slice(i, i + 3);
|
|
83
|
+
const idx = (fnv1a(gram) ^ this.seed) % this.dim;
|
|
84
|
+
vec[idx] += 1;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
// Edge: very short tokens still get a slot.
|
|
88
|
+
if (norm.length < 3) vec[fnv1a(norm) % this.dim] += 1;
|
|
89
|
+
return l2Normalize(vec);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Select the default embedder used by VectorStore.
|
|
95
|
+
*
|
|
96
|
+
* - If MEGACOMPACT_EMBEDDING_URL points at a localhost server, use HttpEmbedder
|
|
97
|
+
* (your own local embedding backend — ONNX/TEI/Ollama/etc). This is the
|
|
98
|
+
* PREVENT-PI-004-sanctioned "bring your own" path: the endpoint is a
|
|
99
|
+
* user-spawned loopback server, so conversation content never leaves the box.
|
|
100
|
+
* - Otherwise the TrigramEmbedder is the shipped default: zero-dependency,
|
|
101
|
+
* deterministic, GPU-free, cross-platform, fully offline.
|
|
102
|
+
*
|
|
103
|
+
* The `Embedder` interface is the seam for any LOCAL embedder. Never point it
|
|
104
|
+
* at a remote provider — that would violate PREVENT-PI-004. A user wanting
|
|
105
|
+
* semantic-grade dedup should run a local embedding server and set the URL.
|
|
106
|
+
*/
|
|
107
|
+
export function defaultEmbedder(): Embedder {
|
|
108
|
+
const cfg = embeddingConfigFromEnv();
|
|
109
|
+
if (cfg) return new HttpEmbedder(cfg);
|
|
110
|
+
return new TrigramEmbedder();
|
|
111
|
+
}
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
import { test } from "node:test";
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
import { mkdtempSync, rmSync } from "node:fs";
|
|
4
|
+
import { tmpdir } from "node:os";
|
|
5
|
+
import { join } from "node:path";
|
|
6
|
+
import { VectorStore } from "./vectorStore.js";
|
|
7
|
+
import { compactSession, recall, mergeSummary, supersededCount } from "./engine.js";
|
|
8
|
+
import type { EngineMessage } from "./types.js";
|
|
9
|
+
|
|
10
|
+
const baseTmp = mkdtempSync(join(tmpdir(), "mc-engine-"));
|
|
11
|
+
let counter = 0;
|
|
12
|
+
function store() {
|
|
13
|
+
return new VectorStore({ dedupSim: 0.9, stateDir: join(baseTmp, `run-${counter++}`) });
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
const SESS = "sess_engine";
|
|
17
|
+
|
|
18
|
+
function msg(role: EngineMessage["role"], text: string, toolName?: string): EngineMessage {
|
|
19
|
+
return toolName ? { role, text, toolName, input: text, output: text } : { role, text };
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
test("compactSession supersedes then persists a checkpoint", () => {
|
|
23
|
+
const s = store();
|
|
24
|
+
const messages: EngineMessage[] = [
|
|
25
|
+
msg("user", "read src/server.ts"),
|
|
26
|
+
msg("assistant", "ok", "Read"),
|
|
27
|
+
msg("user", "edit src/server.ts"),
|
|
28
|
+
msg("assistant", "ok", "Edit"),
|
|
29
|
+
msg("user", "now fix the bug in src/server.ts"),
|
|
30
|
+
msg("assistant", "done", "Edit"),
|
|
31
|
+
];
|
|
32
|
+
const r = compactSession({ sessionId: SESS, messages, keepFrom: 4, timestamp: 1 }, s);
|
|
33
|
+
assert.equal(r.skipped, false);
|
|
34
|
+
assert.equal(r.deduped, false);
|
|
35
|
+
assert.match(r.checkpointId ?? "", /^chkpt_001$/);
|
|
36
|
+
assert.ok(r.summary.length > 0, "summary produced by COLLAPSE");
|
|
37
|
+
assert.ok(r.regionHash.length > 0);
|
|
38
|
+
// SUPERSEDE dropped the obsolete first read turn (user read @0 superseded by
|
|
39
|
+
// the edit @2) — so exactly one superseded message in the compacted slice.
|
|
40
|
+
assert.equal(supersededCount(messages.slice(0, 4)), 1);
|
|
41
|
+
assert.equal(r.compactedFrom, 4);
|
|
42
|
+
// The persisted checkpoint is searchable.
|
|
43
|
+
assert.equal(s.search(SESS, "bug src/server.ts", 5).length, 1);
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
test("compactSession is idempotent on identical region (dedup sentinel)", () => {
|
|
47
|
+
const s = store();
|
|
48
|
+
const messages: EngineMessage[] = [
|
|
49
|
+
msg("user", "alpha work on the parser"),
|
|
50
|
+
msg("assistant", "did it", "Edit"),
|
|
51
|
+
msg("user", "beta work on the renderer"),
|
|
52
|
+
msg("assistant", "done", "Edit"),
|
|
53
|
+
];
|
|
54
|
+
const r1 = compactSession({ sessionId: SESS, messages, keepFrom: 4, timestamp: 1 }, s);
|
|
55
|
+
const r2 = compactSession({ sessionId: SESS, messages, keepFrom: 4, timestamp: 2 }, s);
|
|
56
|
+
assert.equal(r1.deduped, false);
|
|
57
|
+
assert.equal(r2.deduped, true);
|
|
58
|
+
assert.equal(r1.checkpointId, r2.checkpointId);
|
|
59
|
+
assert.equal(s.search(SESS, "parser", 10).length, 1);
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
test("compactSession skipped when slice is empty", () => {
|
|
63
|
+
const s = store();
|
|
64
|
+
const r = compactSession({ sessionId: SESS, messages: [msg("user", "only tail")], keepFrom: 0 }, s);
|
|
65
|
+
assert.equal(r.skipped, true);
|
|
66
|
+
assert.equal(s.search(SESS, "x", 5).length, 0);
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
test("recall drops already-injected checkpoints", () => {
|
|
70
|
+
const s = store();
|
|
71
|
+
compactSession({ sessionId: SESS, messages: [msg("user", "investigated src/vectorStore.ts"), msg("assistant", "ok", "Edit")], keepFrom: 2, timestamp: 1 }, s);
|
|
72
|
+
const first = recall({ sessionId: SESS, query: "vectorStore", limit: 5, skipInjected: true }, s);
|
|
73
|
+
assert.equal(first.newHits.length, 1);
|
|
74
|
+
s.markInjected(SESS, first.hits[0].checkpoint.checkpointId);
|
|
75
|
+
const second = recall({ sessionId: SESS, query: "vectorStore", limit: 5, skipInjected: true }, s);
|
|
76
|
+
assert.equal(second.newHits.length, 0);
|
|
77
|
+
// Without the skip flag, both hits still surface.
|
|
78
|
+
assert.equal(second.hits.length, 1);
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
test("mergeSummary accumulates prior + new context", () => {
|
|
82
|
+
const prior = "<summary>Conversation summary:\n- Key files referenced: src/a.ts.\n</summary>";
|
|
83
|
+
const next = "<summary>Conversation summary:\n- Key files referenced: src/b.ts.\n</summary>";
|
|
84
|
+
const merged = mergeSummary(prior, next);
|
|
85
|
+
assert.ok(merged.includes("src/a.ts"));
|
|
86
|
+
assert.ok(merged.includes("src/b.ts"));
|
|
87
|
+
assert.ok(merged.includes("Newly compacted"));
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
test("supersededCount reports obsolete reads", () => {
|
|
91
|
+
const messages: EngineMessage[] = [
|
|
92
|
+
msg("user", "read src/x.ts"),
|
|
93
|
+
msg("assistant", "ok", "Read"),
|
|
94
|
+
msg("user", "write src/x.ts"),
|
|
95
|
+
msg("assistant", "ok", "Edit"),
|
|
96
|
+
];
|
|
97
|
+
assert.equal(supersededCount(messages), 1);
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
test("compactSession with useExtractive produces topicSummary on checkpoint", () => {
|
|
101
|
+
const s = store();
|
|
102
|
+
const messages: EngineMessage[] = [
|
|
103
|
+
msg("user", "let's refactor the auth module in src/auth.ts"),
|
|
104
|
+
msg("assistant", "I'll start by reading the current implementation", "Read"),
|
|
105
|
+
msg("user", "extract the login logic into a separate function"),
|
|
106
|
+
msg("assistant", "Extracted login() into src/auth.ts:45", "Edit"),
|
|
107
|
+
msg("user", "now add the session token generation"),
|
|
108
|
+
msg("assistant", "Added generateSessionToken in src/auth.ts:78", "Edit"),
|
|
109
|
+
];
|
|
110
|
+
const r = compactSession({ sessionId: "sess_extr", messages, keepFrom: 4, timestamp: 1, useExtractiveSummary: true }, s);
|
|
111
|
+
assert.equal(r.skipped, false);
|
|
112
|
+
assert.ok(r.checkpointId, "checkpoint created");
|
|
113
|
+
// The checkpoint should have topicSummary populated
|
|
114
|
+
const hits = s.search("sess_extr", "auth refactor", 5);
|
|
115
|
+
assert.ok(hits.length > 0);
|
|
116
|
+
// topicSummary should be present on the stored checkpoint (via extractive path)
|
|
117
|
+
assert.ok(hits[0].checkpoint.topicSummary, "topicSummary should be populated when useExtractive is true");
|
|
118
|
+
assert.ok(hits[0].checkpoint.topicSummary!.length > 0);
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
test("cleanup", () => {
|
|
122
|
+
rmSync(baseTmp, { recursive: true, force: true });
|
|
123
|
+
});
|
package/src/engine.ts
ADDED
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* engine.ts — Layer 4 (PERSIST / checkpoint) orchestration.
|
|
3
|
+
*
|
|
4
|
+
* Ties the Sprint 1–2 primitives into the compaction pipeline the extension
|
|
5
|
+
* calls. Pure of any pi runtime type: it consumes EngineMessage[] and talks to
|
|
6
|
+
* the on-disk VectorStore. The extension adapts pi messages -> EngineMessage
|
|
7
|
+
* (see adapt.ts) and reports status.
|
|
8
|
+
*
|
|
9
|
+
* Pipeline (mirrors the PLAN Trident stack):
|
|
10
|
+
* SUPERSEDE (drop obsolete file reads)
|
|
11
|
+
* -> COLLAPSE (summarize the compacted slice)
|
|
12
|
+
* -> CLUSTER (embed + persist a checkpoint to the vector store)
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import { findSuperseded, supersede } from "./supersede.js";
|
|
16
|
+
import { summarizeMessages, mergeCompactSummaries, formatCompactSummary } from "./compact.js";
|
|
17
|
+
import { extractiveSummarize } from "./extractive.js";
|
|
18
|
+
import { estimateSessionTokens } from "./tokens.js";
|
|
19
|
+
import { computeRegionHash, VectorStore, type SearchHit } from "./vectorStore.js";
|
|
20
|
+
import type { EngineMessage } from "./types.js";
|
|
21
|
+
|
|
22
|
+
export interface CompactInput {
|
|
23
|
+
sessionId: string;
|
|
24
|
+
messages: EngineMessage[];
|
|
25
|
+
/** Index (into `messages`) of the first message to keep verbatim. Everything
|
|
26
|
+
* before this is eligible to be compacted. Defaults to `preserveRecent`
|
|
27
|
+
* from the tail. */
|
|
28
|
+
keepFrom?: number;
|
|
29
|
+
/** Optional explicit summary; when omitted, COLLAPSE heuristics build one. */
|
|
30
|
+
summary?: string;
|
|
31
|
+
/** Region text the checkpoint is keyed on (for dedup). Defaults to the
|
|
32
|
+
* compacted slice's joined text. */
|
|
33
|
+
regionText?: string;
|
|
34
|
+
keyDecisions?: string[];
|
|
35
|
+
nextSteps?: string[];
|
|
36
|
+
filesModified?: string[];
|
|
37
|
+
tokenEstimate?: number;
|
|
38
|
+
timestamp?: number;
|
|
39
|
+
/** When true (default), use extractive summary instead of raw concatenation. */
|
|
40
|
+
useExtractiveSummary?: boolean;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export interface CompactResult {
|
|
44
|
+
/** True when nothing was compacted (slice empty / below floor). */
|
|
45
|
+
skipped: boolean;
|
|
46
|
+
/** True when the region was a duplicate of an already-stored checkpoint. */
|
|
47
|
+
deduped: boolean;
|
|
48
|
+
/** Which dedup tier matched: regionHash | summaryHash | contentSimilarity. */
|
|
49
|
+
dedupReason?: string;
|
|
50
|
+
checkpointId?: string;
|
|
51
|
+
summary: string;
|
|
52
|
+
regionHash: string;
|
|
53
|
+
tokenEstimate: number;
|
|
54
|
+
/** Index in `messages` where the compacted slice begins (for the caller to
|
|
55
|
+
* build a drop range). */
|
|
56
|
+
compactedFrom: number;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** Default store used by the convenience `compactSession`. */
|
|
60
|
+
let defaultStore: VectorStore | undefined;
|
|
61
|
+
export function getDefaultStore(stateDir?: string): VectorStore {
|
|
62
|
+
if (!defaultStore) defaultStore = new VectorStore({ stateDir });
|
|
63
|
+
return defaultStore;
|
|
64
|
+
}
|
|
65
|
+
/** Replace the default store (used by tests to inject a temp dir). */
|
|
66
|
+
export function setDefaultStore(store: VectorStore | undefined): void {
|
|
67
|
+
defaultStore = store;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Run the Trident pipeline over a message slice and persist a checkpoint.
|
|
72
|
+
*
|
|
73
|
+
* `messages` is the FULL session view; `keepFrom` marks where the verbatim tail
|
|
74
|
+
* starts, so indices stay absolute and the caller can map the drop range back
|
|
75
|
+
* onto the real (pi) message array via adapt.ts. Returns a `skipped` result
|
|
76
|
+
* when the compactable slice is empty.
|
|
77
|
+
*/
|
|
78
|
+
export function compactSession(input: CompactInput, store: VectorStore = getDefaultStore()): CompactResult {
|
|
79
|
+
const keepFrom = input.keepFrom ?? input.messages.length;
|
|
80
|
+
const compactable = input.messages.slice(0, keepFrom);
|
|
81
|
+
const compactedFrom = keepFrom;
|
|
82
|
+
|
|
83
|
+
if (compactable.length === 0) {
|
|
84
|
+
return {
|
|
85
|
+
skipped: true,
|
|
86
|
+
deduped: false,
|
|
87
|
+
summary: "",
|
|
88
|
+
regionHash: "",
|
|
89
|
+
tokenEstimate: 0,
|
|
90
|
+
compactedFrom,
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// LAYER 1 — SUPERSEDE: zero-cost factual pruning of obsolete file reads.
|
|
95
|
+
const supersededIdx = new Set(findSuperseded(compactable));
|
|
96
|
+
const keep = compactable.filter((_m, i) => !supersededIdx.has(i));
|
|
97
|
+
|
|
98
|
+
// LAYER 2 — COLLAPSE: build (or accept) the summary.
|
|
99
|
+
// When useExtractiveSummary is enabled (default), use the deterministic
|
|
100
|
+
// extractive engine that compresses ~70K tokens → ~2K tokens with structured
|
|
101
|
+
// fields populated. Falls back to legacy concatenation when disabled.
|
|
102
|
+
const useExtractive = input.useExtractiveSummary !== false;
|
|
103
|
+
let summary: string;
|
|
104
|
+
let topicSummary: string | undefined;
|
|
105
|
+
let keyDecisions: string[];
|
|
106
|
+
let nextSteps: string[];
|
|
107
|
+
let filesModified: string[];
|
|
108
|
+
let tokenEstimate: number;
|
|
109
|
+
|
|
110
|
+
if (useExtractive && !input.summary) {
|
|
111
|
+
const ext = extractiveSummarize(keep);
|
|
112
|
+
summary = ext.topicSummary;
|
|
113
|
+
topicSummary = ext.topicSummary;
|
|
114
|
+
keyDecisions = input.keyDecisions ?? ext.keyDecisions;
|
|
115
|
+
nextSteps = input.nextSteps ?? ext.nextSteps;
|
|
116
|
+
filesModified = input.filesModified ?? ext.filesModified;
|
|
117
|
+
tokenEstimate = input.tokenEstimate ?? ext.tokenEstimate;
|
|
118
|
+
} else {
|
|
119
|
+
const collapsed = input.summary ?? summarizeMessages(keep);
|
|
120
|
+
summary = formatCompactSummary(collapsed);
|
|
121
|
+
topicSummary = undefined;
|
|
122
|
+
keyDecisions = input.keyDecisions ?? [];
|
|
123
|
+
nextSteps = input.nextSteps ?? [];
|
|
124
|
+
filesModified = input.filesModified ?? [];
|
|
125
|
+
tokenEstimate = input.tokenEstimate ?? estimateSessionTokens(compactable);
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
// Region text = the compacted slice, used for dedup + embedding.
|
|
129
|
+
const regionText = input.regionText ?? keep.map((m) => m.text).join("\n");
|
|
130
|
+
const regionHash = computeRegionHash(regionText);
|
|
131
|
+
|
|
132
|
+
const add = store.add({
|
|
133
|
+
sessionId: input.sessionId,
|
|
134
|
+
summary,
|
|
135
|
+
topicSummary,
|
|
136
|
+
keyDecisions,
|
|
137
|
+
nextSteps,
|
|
138
|
+
filesModified,
|
|
139
|
+
regionText,
|
|
140
|
+
tokenEstimate,
|
|
141
|
+
timestamp: input.timestamp ?? 0,
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
return {
|
|
145
|
+
skipped: false,
|
|
146
|
+
deduped: add.deduped,
|
|
147
|
+
dedupReason: add.reason,
|
|
148
|
+
checkpointId: add.checkpoint.checkpointId,
|
|
149
|
+
summary,
|
|
150
|
+
regionHash,
|
|
151
|
+
tokenEstimate,
|
|
152
|
+
compactedFrom,
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
export interface RecallInput {
|
|
157
|
+
sessionId: string;
|
|
158
|
+
query: string;
|
|
159
|
+
limit?: number;
|
|
160
|
+
/** Skip checkpoints already injected this session (recall dedup). */
|
|
161
|
+
skipInjected?: boolean;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
export interface RecallResult {
|
|
165
|
+
hits: SearchHit[];
|
|
166
|
+
/** Indices into `hits` that were *not* already injected (ready to inline). */
|
|
167
|
+
newHits: SearchHit[];
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* Layer 5 (query side, shared by auto-inline + on-demand): search the store and
|
|
172
|
+
* drop any checkpoint already injected this session. The caller decides how to
|
|
173
|
+
* inject (Sprint 4 wires injection); this module only does the deduped search.
|
|
174
|
+
*/
|
|
175
|
+
export function recall(input: RecallInput, store: VectorStore = getDefaultStore()): RecallResult {
|
|
176
|
+
const hits = store.search(input.sessionId, input.query, input.limit ?? 3);
|
|
177
|
+
const newHits = input.skipInjected === false ? hits : hits.filter((h) => !store.wasInjected(input.sessionId, h.checkpoint.checkpointId));
|
|
178
|
+
return { hits, newHits };
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/** Merge a freshly compacted summary into the prior persisted summary text. */
|
|
182
|
+
export function mergeSummary(existing: string | undefined, next: string): string {
|
|
183
|
+
return mergeCompactSummaries(existing, next);
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/** Exposed for callers that want raw supersede stats (status reporting). */
|
|
187
|
+
export function supersededCount(messages: EngineMessage[]): number {
|
|
188
|
+
return new Set(findSuperseded(messages)).size;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/** Re-export so the extension has one import surface. */
|
|
192
|
+
export { supersede, summarizeMessages, formatCompactSummary };
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
import { test } from "node:test";
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
import { extractiveSummarize } from "./extractive.js";
|
|
4
|
+
import type { EngineMessage } from "./types.js";
|
|
5
|
+
|
|
6
|
+
function msg(role: EngineMessage["role"], text: string, toolName?: string): EngineMessage {
|
|
7
|
+
return toolName ? { role, text, toolName, input: text, output: text } : { role, text };
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
// ---- Determinism -----------------------------------------------------------
|
|
11
|
+
|
|
12
|
+
test("extractive summary is deterministic", () => {
|
|
13
|
+
const messages: EngineMessage[] = [
|
|
14
|
+
msg("user", "please write src/index.ts"),
|
|
15
|
+
msg("assistant", "I'll write src/index.ts now."),
|
|
16
|
+
msg("tool", '{"file_path":"src/index.ts"}', "write"),
|
|
17
|
+
msg("assistant", "Done."),
|
|
18
|
+
];
|
|
19
|
+
const s1 = extractiveSummarize(messages);
|
|
20
|
+
const s2 = extractiveSummarize(messages);
|
|
21
|
+
assert.deepStrictEqual(s1, s2);
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
// ---- Compression -----------------------------------------------------------
|
|
25
|
+
|
|
26
|
+
test("extractive summary produces small output", () => {
|
|
27
|
+
// Build 70 messages (simulating a session)
|
|
28
|
+
const messages: EngineMessage[] = [];
|
|
29
|
+
for (let i = 0; i < 70; i++) {
|
|
30
|
+
messages.push(msg("user", `request ${i}: please help with feature ${i}`));
|
|
31
|
+
messages.push(msg("assistant", `working on feature ${i} in src/file${i % 5}.ts`));
|
|
32
|
+
messages.push(msg("tool", `{"file_path":"src/file${i % 5}.ts","content":"..."}`, "write"));
|
|
33
|
+
messages.push(msg("assistant", `done with feature ${i}`));
|
|
34
|
+
}
|
|
35
|
+
const rawText = messages.map((m) => m.text).join("\n");
|
|
36
|
+
const rawTokens = Math.ceil(rawText.length / 4);
|
|
37
|
+
|
|
38
|
+
const summary = extractiveSummarize(messages);
|
|
39
|
+
const ratio = rawTokens / summary.tokenEstimate;
|
|
40
|
+
|
|
41
|
+
// Compression should be at least 5:1 (target is 35:1)
|
|
42
|
+
assert.ok(ratio >= 5, `compression ratio ${ratio.toFixed(1)}:1 is less than 5:1`);
|
|
43
|
+
assert.ok(summary.tokenEstimate < 5000, `summary is ${summary.tokenEstimate} tokens (expected < 5000)`);
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
// ---- Empty input ------------------------------------------------------------
|
|
47
|
+
|
|
48
|
+
test("empty messages returns minimal summary", () => {
|
|
49
|
+
const summary = extractiveSummarize([]);
|
|
50
|
+
assert.equal(summary.topicSummary, "(empty)");
|
|
51
|
+
assert.equal(summary.keyDecisions.length, 0);
|
|
52
|
+
assert.equal(summary.nextSteps.length, 0);
|
|
53
|
+
assert.equal(summary.filesModified.length, 0);
|
|
54
|
+
assert.equal(summary.tokenEstimate, 0);
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
// ---- Key decisions ----------------------------------------------------------
|
|
58
|
+
|
|
59
|
+
test("extracts decisions from assistant messages", () => {
|
|
60
|
+
const messages: EngineMessage[] = [
|
|
61
|
+
msg("user", "which database should we use?"),
|
|
62
|
+
msg("assistant", "I recommend using better-sqlite3 for the local vector store."),
|
|
63
|
+
msg("assistant", "Let's go with the Trident pipeline architecture."),
|
|
64
|
+
];
|
|
65
|
+
const summary = extractiveSummarize(messages);
|
|
66
|
+
assert.ok(summary.keyDecisions.length >= 1, "should extract at least 1 decision");
|
|
67
|
+
assert.ok(
|
|
68
|
+
summary.keyDecisions.some((d) => d.includes("better-sqlite3")),
|
|
69
|
+
`decisions: ${JSON.stringify(summary.keyDecisions)}`,
|
|
70
|
+
);
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
test("no decisions in tool messages", () => {
|
|
74
|
+
const messages: EngineMessage[] = [
|
|
75
|
+
msg("tool", "I recommend something", "bash"),
|
|
76
|
+
];
|
|
77
|
+
const summary = extractiveSummarize(messages);
|
|
78
|
+
// Tool messages should not be checked for decisions
|
|
79
|
+
assert.equal(summary.keyDecisions.length, 0);
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
// ---- Files modified ---------------------------------------------------------
|
|
83
|
+
|
|
84
|
+
test("extracts files from write/edit tool calls", () => {
|
|
85
|
+
const messages: EngineMessage[] = [
|
|
86
|
+
msg("tool", '{"file_path":"/home/user/project/src/index.ts","content":"..."}', "write"),
|
|
87
|
+
msg("tool", '{"file_path":"/home/user/project/README.md","content":"..."}', "edit"),
|
|
88
|
+
];
|
|
89
|
+
const summary = extractiveSummarize(messages);
|
|
90
|
+
assert.ok(summary.filesModified.includes("/home/user/project/src/index.ts"));
|
|
91
|
+
assert.ok(summary.filesModified.includes("/home/user/project/README.md"));
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
test("extracts files from git commands in bash", () => {
|
|
95
|
+
const messages: EngineMessage[] = [
|
|
96
|
+
msg("tool", "git add src/index.ts src/types.ts", "bash"),
|
|
97
|
+
];
|
|
98
|
+
const summary = extractiveSummarize(messages);
|
|
99
|
+
assert.ok(summary.filesModified.some((f) => f.includes("index.ts")));
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
// ---- Pending work -----------------------------------------------------------
|
|
103
|
+
|
|
104
|
+
test("extracts pending work markers", () => {
|
|
105
|
+
const messages: EngineMessage[] = [
|
|
106
|
+
msg("user", "run the tests"),
|
|
107
|
+
msg("assistant", "Tests pass. TODO: add integration tests for the dedup path."),
|
|
108
|
+
];
|
|
109
|
+
const summary = extractiveSummarize(messages);
|
|
110
|
+
assert.ok(summary.nextSteps.length >= 1, "should find TODO");
|
|
111
|
+
assert.ok(summary.nextSteps.some((s) => /integration tests/i.test(s)));
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
// ---- topicSummary structure -------------------------------------------------
|
|
115
|
+
|
|
116
|
+
test("topicSummary contains scope line", () => {
|
|
117
|
+
const messages: EngineMessage[] = [
|
|
118
|
+
msg("user", "hello"),
|
|
119
|
+
msg("assistant", "hi there"),
|
|
120
|
+
];
|
|
121
|
+
const summary = extractiveSummarize(messages);
|
|
122
|
+
assert.ok(summary.topicSummary.includes("Conversation: 2 messages"));
|
|
123
|
+
assert.ok(summary.topicSummary.includes("1 user"));
|
|
124
|
+
assert.ok(summary.topicSummary.includes("1 assistant"));
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
test("topicSummary includes tools when present", () => {
|
|
128
|
+
const messages: EngineMessage[] = [
|
|
129
|
+
msg("tool", "ok", "write"),
|
|
130
|
+
msg("tool", "ok", "bash"),
|
|
131
|
+
];
|
|
132
|
+
const summary = extractiveSummarize(messages);
|
|
133
|
+
assert.ok(summary.topicSummary.includes("Tools:"));
|
|
134
|
+
assert.ok(summary.topicSummary.includes("write"));
|
|
135
|
+
assert.ok(summary.topicSummary.includes("bash"));
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
// ---- Same messages produce same summary ------------------------------------
|
|
139
|
+
|
|
140
|
+
test("deterministic across invocations with complex input", () => {
|
|
141
|
+
const messages: EngineMessage[] = [
|
|
142
|
+
msg("user", "help me refactor the auth module"),
|
|
143
|
+
msg("assistant", "I'll look at the current auth implementation."),
|
|
144
|
+
msg("tool", '{"file_path":"src/auth.ts"}', "read"),
|
|
145
|
+
msg("assistant", "I recommend splitting auth.ts into separate files."),
|
|
146
|
+
msg("user", "sounds good, go ahead"),
|
|
147
|
+
msg("assistant", "I'll create src/auth/login.ts and src/auth/register.ts."),
|
|
148
|
+
msg("tool", '{"file_path":"src/auth/login.ts","content":"..."}', "write"),
|
|
149
|
+
msg("tool", '{"file_path":"src/auth/register.ts","content":"..."}', "write"),
|
|
150
|
+
msg("assistant", "Done. TODO: update the import paths in main.ts."),
|
|
151
|
+
];
|
|
152
|
+
const results = Array.from({ length: 5 }, () => extractiveSummarize(messages));
|
|
153
|
+
for (let i = 1; i < results.length; i++) {
|
|
154
|
+
assert.deepStrictEqual(results[i], results[0], `run ${i} differs from run 0`);
|
|
155
|
+
}
|
|
156
|
+
});
|