pi-mega-compact 0.8.21 → 0.8.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/extensions/dashboard-server/dashboard-client-core.js +201 -0
- package/dist/extensions/dashboard-server/dashboard-client-game.js +241 -0
- package/dist/extensions/dashboard-server/dashboard-client-repos.js +212 -0
- package/dist/extensions/dashboard-server/dashboard-client.js +19 -0
- package/dist/extensions/dashboard-server/html.js +2 -621
- package/dist/extensions/dashboard-server/routes-core.js +62 -0
- package/dist/extensions/dashboard-server/routes-game.js +323 -0
- package/dist/extensions/dashboard-server/routes-repo.js +170 -0
- package/dist/extensions/dashboard-server/routes-sessions.js +159 -0
- package/dist/extensions/dashboard-server/routes.js +10 -0
- package/dist/extensions/dashboard-server/server.js +26 -623
- package/dist/extensions/mega-commands.js +4 -3
- package/dist/extensions/mega-events/agent-handlers.js +2 -1
- package/dist/extensions/mega-events/compact-handlers.js +26 -0
- package/dist/extensions/mega-events/session-handlers.js +2 -1
- package/dist/extensions/mega-pipeline/compact.js +3 -2
- package/dist/extensions/mega-runtime/state.js +7 -7
- package/dist/src/dedup/raptor/promote.test.js +5 -5
- package/dist/src/dedup/sprint12.test.js +7 -7
- package/dist/src/dedup-engine.test.js +29 -29
- package/dist/src/e2e.test.js +38 -38
- package/dist/src/engine.js +3 -3
- package/dist/src/engine.test.js +6 -6
- package/dist/src/importance.js +197 -0
- package/dist/src/importance.test.js +372 -0
- package/dist/src/ratio.bench.test.js +18 -18
- package/dist/src/recall.js +6 -5
- package/dist/src/recall.test.js +85 -27
- package/dist/src/sprint14.test.js +2 -2
- package/dist/src/store/migrate.test.js +5 -5
- package/dist/src/store/sprint10.test.js +5 -5
- package/dist/src/store/sqlite/global-index.js +5 -174
- package/dist/src/store/sqlite/global-sessions.js +190 -0
- package/dist/src/vector-read.js +168 -0
- package/dist/src/vector-search.js +191 -0
- package/dist/src/vectorStore.js +10 -297
- package/dist/src/vectorStore.test.js +32 -32
- package/extensions/dashboard-server/dashboard-client-core.ts +202 -0
- package/extensions/dashboard-server/dashboard-client-game.ts +242 -0
- package/extensions/dashboard-server/dashboard-client-repos.ts +213 -0
- package/extensions/dashboard-server/dashboard-client.ts +21 -0
- package/extensions/dashboard-server/html.ts +2 -621
- package/extensions/dashboard-server/routes-core.ts +113 -0
- package/extensions/dashboard-server/routes-game.ts +386 -0
- package/extensions/dashboard-server/routes-repo.ts +212 -0
- package/extensions/dashboard-server/routes-sessions.ts +195 -0
- package/extensions/dashboard-server/routes.ts +13 -0
- package/extensions/dashboard-server/server.ts +37 -700
- package/extensions/mega-commands.ts +4 -3
- package/extensions/mega-events/agent-handlers.ts +2 -1
- package/extensions/mega-events/compact-handlers.ts +28 -0
- package/extensions/mega-events/session-handlers.ts +2 -1
- package/extensions/mega-pipeline/compact.ts +3 -2
- package/extensions/mega-runtime/state.ts +7 -7
- package/extensions/openclaw-mega-compact.ts +2 -2
- package/package.json +1 -1
- package/src/dedup/raptor/promote.test.ts +5 -5
- package/src/dedup/sprint12.test.ts +7 -7
- package/src/dedup-engine.test.ts +30 -30
- package/src/e2e.test.ts +38 -38
- package/src/engine.test.ts +6 -6
- package/src/engine.ts +3 -3
- package/src/importance.test.ts +538 -0
- package/src/importance.ts +312 -0
- package/src/ratio.bench.test.ts +18 -18
- package/src/recall.test.ts +101 -29
- package/src/recall.ts +9 -9
- package/src/sprint14.test.ts +2 -2
- package/src/store/migrate.test.ts +5 -5
- package/src/store/sprint10.test.ts +5 -5
- package/src/store/sqlite/global-index.ts +18 -290
- package/src/store/sqlite/global-sessions.ts +291 -0
- package/src/vector-read.ts +237 -0
- package/src/vector-search.ts +231 -0
- package/src/vectorStore.test.ts +32 -32
- package/src/vectorStore.ts +29 -356
|
@@ -0,0 +1,372 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* importance.test.ts — S40A unit tests for the importance scoring engine.
|
|
3
|
+
*
|
|
4
|
+
* Covers all exports: ContextItemType, DEFAULT_MULTIPLIERS, detectItemType,
|
|
5
|
+
* ageDecay, recencyBoost, retentionBoost, score, preservationCutoff,
|
|
6
|
+
* itemsToPreserve, scoreEngineMessages.
|
|
7
|
+
*
|
|
8
|
+
* Determinism is enforced — same inputs MUST produce same outputs.
|
|
9
|
+
*/
|
|
10
|
+
import { test } from "node:test";
|
|
11
|
+
import assert from "node:assert/strict";
|
|
12
|
+
import { ContextItemType, DEFAULT_MULTIPLIERS, detectItemType, ageDecay, recencyBoost, retentionBoost, score, preservationCutoff, itemsToPreserve, scoreEngineMessages, } from "./importance.js";
|
|
13
|
+
const NOW = 1_700_000_000_000; // fixed epoch ms for determinism
|
|
14
|
+
// ---- S40A-1: types and enums ----
|
|
15
|
+
test("ContextItemType enum has 8 variants with snake_case values", () => {
|
|
16
|
+
assert.equal(ContextItemType.UserMessage, "user_message");
|
|
17
|
+
assert.equal(ContextItemType.AssistantMessage, "assistant_message");
|
|
18
|
+
assert.equal(ContextItemType.SystemMessage, "system_message");
|
|
19
|
+
assert.equal(ContextItemType.CodeBlock, "code_block");
|
|
20
|
+
assert.equal(ContextItemType.Error, "error");
|
|
21
|
+
assert.equal(ContextItemType.Decision, "decision");
|
|
22
|
+
assert.equal(ContextItemType.FileModification, "file_modification");
|
|
23
|
+
assert.equal(ContextItemType.ToolExecution, "tool_execution");
|
|
24
|
+
});
|
|
25
|
+
// ---- S40A-2: DEFAULT_MULTIPLIERS ----
|
|
26
|
+
test("DEFAULT_MULTIPLIERS matches the Rust reference values", () => {
|
|
27
|
+
assert.equal(DEFAULT_MULTIPLIERS[ContextItemType.UserMessage], 1.5);
|
|
28
|
+
assert.equal(DEFAULT_MULTIPLIERS[ContextItemType.AssistantMessage], 1.0);
|
|
29
|
+
assert.equal(DEFAULT_MULTIPLIERS[ContextItemType.SystemMessage], 0.5);
|
|
30
|
+
assert.equal(DEFAULT_MULTIPLIERS[ContextItemType.CodeBlock], 1.2);
|
|
31
|
+
assert.equal(DEFAULT_MULTIPLIERS[ContextItemType.Error], 2.0);
|
|
32
|
+
assert.equal(DEFAULT_MULTIPLIERS[ContextItemType.Decision], 2.5);
|
|
33
|
+
assert.equal(DEFAULT_MULTIPLIERS[ContextItemType.FileModification], 1.8);
|
|
34
|
+
assert.equal(DEFAULT_MULTIPLIERS[ContextItemType.ToolExecution], 1.3);
|
|
35
|
+
});
|
|
36
|
+
test("DEFAULT_MULTIPLIERS has an entry for every ContextItemType variant", () => {
|
|
37
|
+
for (const t of Object.values(ContextItemType)) {
|
|
38
|
+
assert.equal(typeof DEFAULT_MULTIPLIERS[t], "number", `missing multiplier for ${t}`);
|
|
39
|
+
}
|
|
40
|
+
});
|
|
41
|
+
// ---- S40A-3: ageDecay ----
|
|
42
|
+
test("ageDecay: fresh (age=0) returns 0", () => {
|
|
43
|
+
assert.equal(ageDecay(0), 0);
|
|
44
|
+
});
|
|
45
|
+
test("ageDecay: 1 hour at 5%/hr returns 0.05", () => {
|
|
46
|
+
assert.equal(ageDecay(3_600_000), 0.05);
|
|
47
|
+
});
|
|
48
|
+
test("ageDecay: 14 hours caps at 0.7 (maxDecay)", () => {
|
|
49
|
+
assert.equal(ageDecay(14 * 3_600_000), 0.7);
|
|
50
|
+
});
|
|
51
|
+
test("ageDecay: beyond 14 hours stays capped at 0.7", () => {
|
|
52
|
+
assert.equal(ageDecay(100 * 3_600_000), 0.7);
|
|
53
|
+
});
|
|
54
|
+
test("ageDecay: negative age (future timestamp) returns 0", () => {
|
|
55
|
+
assert.equal(ageDecay(-5_000), 0);
|
|
56
|
+
});
|
|
57
|
+
test("ageDecay: custom rate and max override defaults", () => {
|
|
58
|
+
// 10h at 0.1/hr = 1.0 → capped at 0.8.
|
|
59
|
+
assert.equal(ageDecay(10 * 3_600_000, 0.1, 0.8), 0.8);
|
|
60
|
+
// 2h at 0.1/hr = 0.2.
|
|
61
|
+
assert.equal(ageDecay(2 * 3_600_000, 0.1, 0.8), 0.2);
|
|
62
|
+
});
|
|
63
|
+
// ---- S40A-4: recencyBoost + retentionBoost ----
|
|
64
|
+
test("recencyBoost: <5min returns 1.2", () => {
|
|
65
|
+
assert.equal(recencyBoost(299_000), 1.2);
|
|
66
|
+
});
|
|
67
|
+
test("recencyBoost: >5min returns 1.0", () => {
|
|
68
|
+
assert.equal(recencyBoost(301_000), 1.0);
|
|
69
|
+
});
|
|
70
|
+
test("recencyBoost: exactly 5min (300_000ms) returns 1.0 (boundary)", () => {
|
|
71
|
+
assert.equal(recencyBoost(300_000), 1.0);
|
|
72
|
+
});
|
|
73
|
+
test("recencyBoost: custom threshold respected", () => {
|
|
74
|
+
assert.equal(recencyBoost(5000, 10_000), 1.2);
|
|
75
|
+
assert.equal(recencyBoost(11_000, 10_000), 1.0);
|
|
76
|
+
});
|
|
77
|
+
test("retentionBoost: flagged returns 3.0", () => {
|
|
78
|
+
assert.equal(retentionBoost(true), 3.0);
|
|
79
|
+
});
|
|
80
|
+
test("retentionBoost: unflagged returns 1.0", () => {
|
|
81
|
+
assert.equal(retentionBoost(false), 1.0);
|
|
82
|
+
});
|
|
83
|
+
// ---- S40A-5: detectItemType ----
|
|
84
|
+
test("detectItemType: role=tool → ToolExecution (priority 1)", () => {
|
|
85
|
+
// Even if content looks like an error, tool role wins.
|
|
86
|
+
assert.equal(detectItemType("error: something failed", "tool"), ContextItemType.ToolExecution);
|
|
87
|
+
});
|
|
88
|
+
test("detectItemType: role=custom → SystemMessage (priority 2)", () => {
|
|
89
|
+
assert.equal(detectItemType("anything here", "custom"), ContextItemType.SystemMessage);
|
|
90
|
+
});
|
|
91
|
+
test("detectItemType: error content → Error", () => {
|
|
92
|
+
assert.equal(detectItemType("Error: ENOENT at src/config.ts:42", "assistant"), ContextItemType.Error);
|
|
93
|
+
assert.equal(detectItemType("Traceback (most recent call last)", "assistant"), ContextItemType.Error);
|
|
94
|
+
assert.equal(detectItemType("panic: runtime error", "assistant"), ContextItemType.Error);
|
|
95
|
+
assert.equal(detectItemType("E4030 connection refused", "assistant"), ContextItemType.Error);
|
|
96
|
+
});
|
|
97
|
+
test("detectItemType: decision content → Decision", () => {
|
|
98
|
+
assert.equal(detectItemType("we decided to use JWT auth", "assistant"), ContextItemType.Decision);
|
|
99
|
+
assert.equal(detectItemType("going with option B", "assistant"), ContextItemType.Decision);
|
|
100
|
+
assert.equal(detectItemType("let's go with the async approach", "assistant"), ContextItemType.Decision);
|
|
101
|
+
assert.equal(detectItemType("switching to a different database", "assistant"), ContextItemType.Decision);
|
|
102
|
+
});
|
|
103
|
+
test("detectItemType: fenced code block ≥20 chars → CodeBlock", () => {
|
|
104
|
+
const code = "```js\nconst x = 42;\nconst y = 'hello world';\n```";
|
|
105
|
+
assert.equal(detectItemType(code, "assistant"), ContextItemType.CodeBlock);
|
|
106
|
+
});
|
|
107
|
+
test("detectItemType: short fenced block (<20 chars) does NOT → CodeBlock", () => {
|
|
108
|
+
const short = "```\nshort\n```";
|
|
109
|
+
assert.notEqual(detectItemType(short, "assistant"), ContextItemType.CodeBlock);
|
|
110
|
+
});
|
|
111
|
+
test("detectItemType: file modification verbs → FileModification", () => {
|
|
112
|
+
assert.equal(detectItemType("edited src/config.ts to add a flag", "assistant"), ContextItemType.FileModification);
|
|
113
|
+
assert.equal(detectItemType("created README.md", "assistant"), ContextItemType.FileModification);
|
|
114
|
+
assert.equal(detectItemType("wrote tests/main.test.ts", "assistant"), ContextItemType.FileModification);
|
|
115
|
+
});
|
|
116
|
+
test("detectItemType: role=user → UserMessage", () => {
|
|
117
|
+
assert.equal(detectItemType("plain question", "user"), ContextItemType.UserMessage);
|
|
118
|
+
});
|
|
119
|
+
test("detectItemType: role=assistant → AssistantMessage", () => {
|
|
120
|
+
assert.equal(detectItemType("plain answer", "assistant"), ContextItemType.AssistantMessage);
|
|
121
|
+
});
|
|
122
|
+
test("detectItemType: fallback → AssistantMessage", () => {
|
|
123
|
+
// Unknown role with non-matching content falls back to AssistantMessage.
|
|
124
|
+
assert.equal(detectItemType("nothing matches here", "unknown"), ContextItemType.AssistantMessage);
|
|
125
|
+
});
|
|
126
|
+
test("detectItemType: empty string does not crash and falls back", () => {
|
|
127
|
+
assert.equal(detectItemType("", "user"), ContextItemType.UserMessage);
|
|
128
|
+
assert.equal(detectItemType("", "assistant"), ContextItemType.AssistantMessage);
|
|
129
|
+
});
|
|
130
|
+
test("detectItemType: priority — error before decision (first match wins)", () => {
|
|
131
|
+
// A message that contains BOTH "error" and "decided" — error wins (rule 3 before rule 4).
|
|
132
|
+
assert.equal(detectItemType("error: we decided to fail", "assistant"), ContextItemType.Error);
|
|
133
|
+
});
|
|
134
|
+
// ---- S40A-6: score ----
|
|
135
|
+
test("score: fresh decision gets full multiplier (no decay, no recency/retention boost expected unless young)", () => {
|
|
136
|
+
const result = score({
|
|
137
|
+
id: "0",
|
|
138
|
+
content: "we decided to use JWT",
|
|
139
|
+
role: "assistant",
|
|
140
|
+
timestamp: NOW, // age 0
|
|
141
|
+
}, NOW);
|
|
142
|
+
assert.equal(result.type, ContextItemType.Decision);
|
|
143
|
+
assert.equal(result.rawMultiplier, 2.5);
|
|
144
|
+
assert.equal(result.ageDecay, 0);
|
|
145
|
+
assert.equal(result.recencyBoost, 1.2); // age 0 < 5min
|
|
146
|
+
assert.equal(result.retentionBoost, 1.0); // not userFlagged
|
|
147
|
+
// 2.5 * (1 - 0) * 1.2 * 1.0 = 3.0
|
|
148
|
+
assert.equal(result.finalScore, 3.0);
|
|
149
|
+
});
|
|
150
|
+
test("score: old decision decays", () => {
|
|
151
|
+
const tenHoursAgo = NOW - 10 * 3_600_000;
|
|
152
|
+
const result = score({
|
|
153
|
+
id: "1",
|
|
154
|
+
content: "we decided to use JWT",
|
|
155
|
+
role: "assistant",
|
|
156
|
+
timestamp: tenHoursAgo,
|
|
157
|
+
}, NOW);
|
|
158
|
+
// decay at 10h = 0.5; recency 1.0 (>5min); retention 1.0
|
|
159
|
+
// 2.5 * (1 - 0.5) * 1.0 * 1.0 = 1.25
|
|
160
|
+
assert.equal(result.ageDecay, 0.5);
|
|
161
|
+
assert.equal(result.recencyBoost, 1.0);
|
|
162
|
+
assert.equal(result.finalScore, 1.25);
|
|
163
|
+
});
|
|
164
|
+
test("score: userFlagged triples the result via retentionBoost", () => {
|
|
165
|
+
const flagged = score({
|
|
166
|
+
id: "2",
|
|
167
|
+
content: "we decided to use JWT",
|
|
168
|
+
role: "assistant",
|
|
169
|
+
timestamp: NOW,
|
|
170
|
+
userFlagged: true,
|
|
171
|
+
}, NOW);
|
|
172
|
+
const unflagged = score({
|
|
173
|
+
id: "3",
|
|
174
|
+
content: "we decided to use JWT",
|
|
175
|
+
role: "assistant",
|
|
176
|
+
timestamp: NOW,
|
|
177
|
+
userFlagged: false,
|
|
178
|
+
}, NOW);
|
|
179
|
+
// 3.0 / 1.0 retention ratio.
|
|
180
|
+
assert.equal(flagged.finalScore / unflagged.finalScore, 3.0);
|
|
181
|
+
});
|
|
182
|
+
test("score: clamps finalScore to minimum 0.01", () => {
|
|
183
|
+
// A system message (0.5x) at 14h+ (max decay 0.7) → 0.5 * 0.3 * 1.0 * 1.0 = 0.15. Above 0.01.
|
|
184
|
+
// Force below 0.01 with a custom multiplier of 0 and max decay.
|
|
185
|
+
const result = score({
|
|
186
|
+
id: "4",
|
|
187
|
+
content: "system filler",
|
|
188
|
+
role: "custom",
|
|
189
|
+
timestamp: NOW - 100 * 3_600_000,
|
|
190
|
+
}, NOW, { [ContextItemType.SystemMessage]: 0.01 }, { maxDecay: 0.99 });
|
|
191
|
+
// 0.01 * (1 - 0.99) * 1.0 * 1.0 = 0.0001 → clamped to 0.01.
|
|
192
|
+
assert.equal(result.finalScore, 0.01);
|
|
193
|
+
});
|
|
194
|
+
test("score: custom multipliers override DEFAULT_MULTIPLIERS", () => {
|
|
195
|
+
const result = score({
|
|
196
|
+
id: "5",
|
|
197
|
+
content: "we decided to use JWT",
|
|
198
|
+
role: "assistant",
|
|
199
|
+
timestamp: NOW,
|
|
200
|
+
}, NOW, { [ContextItemType.Decision]: 10.0 });
|
|
201
|
+
assert.equal(result.rawMultiplier, 10.0);
|
|
202
|
+
assert.equal(result.finalScore, 10.0 * 1.2); // 10 * (1-0) * 1.2 * 1.0
|
|
203
|
+
});
|
|
204
|
+
test("score: unknown role narrows to assistant defensively", () => {
|
|
205
|
+
const result = score({
|
|
206
|
+
id: "6",
|
|
207
|
+
content: "plain text",
|
|
208
|
+
role: "weird-role",
|
|
209
|
+
timestamp: NOW,
|
|
210
|
+
}, NOW);
|
|
211
|
+
assert.equal(result.role, "assistant");
|
|
212
|
+
assert.equal(result.type, ContextItemType.AssistantMessage);
|
|
213
|
+
});
|
|
214
|
+
test("score: undefined/null content does not crash", () => {
|
|
215
|
+
const r1 = score({ id: "a", content: undefined, role: "user", timestamp: NOW }, NOW);
|
|
216
|
+
const r2 = score({ id: "b", content: null, role: "assistant", timestamp: NOW }, NOW);
|
|
217
|
+
assert.equal(r1.type, ContextItemType.UserMessage);
|
|
218
|
+
assert.equal(r2.type, ContextItemType.AssistantMessage);
|
|
219
|
+
assert.equal(r1.content, "");
|
|
220
|
+
assert.equal(r2.content, "");
|
|
221
|
+
});
|
|
222
|
+
test("score: determinism — same inputs → same output", () => {
|
|
223
|
+
const item = {
|
|
224
|
+
id: "d1",
|
|
225
|
+
content: "we decided to use JWT",
|
|
226
|
+
role: "assistant",
|
|
227
|
+
timestamp: NOW - 3_600_000,
|
|
228
|
+
};
|
|
229
|
+
const a = score(item, NOW);
|
|
230
|
+
const b = score(item, NOW);
|
|
231
|
+
assert.deepEqual(a, b);
|
|
232
|
+
});
|
|
233
|
+
// ---- S40A-7: preservationCutoff + itemsToPreserve ----
|
|
234
|
+
test("preservationCutoff: 10 items at ratio 0.3 → threshold is the 3rd-highest score", () => {
|
|
235
|
+
const items = Array.from({ length: 10 }, (_, i) => ({
|
|
236
|
+
id: String(i),
|
|
237
|
+
type: ContextItemType.AssistantMessage,
|
|
238
|
+
content: "",
|
|
239
|
+
role: "assistant",
|
|
240
|
+
timestamp: 0,
|
|
241
|
+
rawMultiplier: 1.0,
|
|
242
|
+
ageDecay: 0,
|
|
243
|
+
recencyBoost: 1.0,
|
|
244
|
+
retentionBoost: 1.0,
|
|
245
|
+
// scores 1.0, 2.0, ..., 10.0
|
|
246
|
+
finalScore: (i + 1) * 1.0,
|
|
247
|
+
}));
|
|
248
|
+
const threshold = preservationCutoff(items, 0.3);
|
|
249
|
+
// top 30% of 10 = 3 items; the 3rd highest is 8.0 (scores 10,9,8 preserved).
|
|
250
|
+
assert.equal(threshold, 8.0);
|
|
251
|
+
});
|
|
252
|
+
test("preservationCutoff: ratio=1.0 returns 0 (preserve all)", () => {
|
|
253
|
+
const items = [{ finalScore: 5, id: "x" }];
|
|
254
|
+
assert.equal(preservationCutoff(items, 1.0), 0);
|
|
255
|
+
});
|
|
256
|
+
test("preservationCutoff: ratio=0 returns Infinity (preserve none)", () => {
|
|
257
|
+
const items = [{ finalScore: 5, id: "x" }];
|
|
258
|
+
assert.equal(preservationCutoff(items, 0), Infinity);
|
|
259
|
+
});
|
|
260
|
+
test("preservationCutoff: empty input returns Infinity", () => {
|
|
261
|
+
assert.equal(preservationCutoff([], 0.5), Infinity);
|
|
262
|
+
});
|
|
263
|
+
test("itemsToPreserve: returns IDs at or above threshold", () => {
|
|
264
|
+
const items = Array.from({ length: 10 }, (_, i) => ({
|
|
265
|
+
id: String(i),
|
|
266
|
+
type: ContextItemType.AssistantMessage,
|
|
267
|
+
content: "",
|
|
268
|
+
role: "assistant",
|
|
269
|
+
timestamp: 0,
|
|
270
|
+
rawMultiplier: 1.0,
|
|
271
|
+
ageDecay: 0,
|
|
272
|
+
recencyBoost: 1.0,
|
|
273
|
+
retentionBoost: 1.0,
|
|
274
|
+
finalScore: (i + 1) * 1.0,
|
|
275
|
+
}));
|
|
276
|
+
const result = itemsToPreserve(items, 0.3);
|
|
277
|
+
assert.equal(result.totalScored, 10);
|
|
278
|
+
assert.equal(result.totalPreserved, 3);
|
|
279
|
+
assert.deepEqual([...result.preservedIds].sort(), ["7", "8", "9"]);
|
|
280
|
+
});
|
|
281
|
+
test("itemsToPreserve: ratio=1.0 preserves everything", () => {
|
|
282
|
+
const items = Array.from({ length: 5 }, (_, i) => ({
|
|
283
|
+
id: String(i),
|
|
284
|
+
type: ContextItemType.AssistantMessage,
|
|
285
|
+
content: "",
|
|
286
|
+
role: "assistant",
|
|
287
|
+
timestamp: 0,
|
|
288
|
+
rawMultiplier: 1.0,
|
|
289
|
+
ageDecay: 0,
|
|
290
|
+
recencyBoost: 1.0,
|
|
291
|
+
retentionBoost: 1.0,
|
|
292
|
+
finalScore: i + 1,
|
|
293
|
+
}));
|
|
294
|
+
const result = itemsToPreserve(items, 1.0);
|
|
295
|
+
assert.equal(result.totalPreserved, 5);
|
|
296
|
+
});
|
|
297
|
+
test("itemsToPreserve: ratio=0 preserves nothing", () => {
|
|
298
|
+
const items = Array.from({ length: 5 }, (_, i) => ({
|
|
299
|
+
id: String(i),
|
|
300
|
+
type: ContextItemType.AssistantMessage,
|
|
301
|
+
content: "",
|
|
302
|
+
role: "assistant",
|
|
303
|
+
timestamp: 0,
|
|
304
|
+
rawMultiplier: 1.0,
|
|
305
|
+
ageDecay: 0,
|
|
306
|
+
recencyBoost: 1.0,
|
|
307
|
+
retentionBoost: 1.0,
|
|
308
|
+
finalScore: i + 1,
|
|
309
|
+
}));
|
|
310
|
+
const result = itemsToPreserve(items, 0);
|
|
311
|
+
assert.equal(result.totalPreserved, 0);
|
|
312
|
+
assert.equal(result.threshold, Infinity);
|
|
313
|
+
});
|
|
314
|
+
test("itemsToPreserve: empty items returns empty set", () => {
|
|
315
|
+
const result = itemsToPreserve([], 0.5);
|
|
316
|
+
assert.equal(result.totalScored, 0);
|
|
317
|
+
assert.equal(result.totalPreserved, 0);
|
|
318
|
+
assert.equal(result.preservedIds.size, 0);
|
|
319
|
+
});
|
|
320
|
+
// ---- scoreEngineMessages: position-based age fallback ----
|
|
321
|
+
test("scoreEngineMessages: assigns position-based timestamps (oldest first)", () => {
|
|
322
|
+
const messages = [
|
|
323
|
+
{ role: "assistant", text: "we decided to use JWT" },
|
|
324
|
+
{ role: "user", text: "ok" },
|
|
325
|
+
{ role: "assistant", text: "edited src/config.ts" },
|
|
326
|
+
];
|
|
327
|
+
const scored = scoreEngineMessages(messages, NOW);
|
|
328
|
+
assert.equal(scored.length, 3);
|
|
329
|
+
// Oldest (i=0) is `messages.length` minutes ago = 3 min.
|
|
330
|
+
assert.equal(scored[0].timestamp, NOW - 3 * 60_000);
|
|
331
|
+
// Newest (i=2) is 1 min ago.
|
|
332
|
+
assert.equal(scored[2].timestamp, NOW - 1 * 60_000);
|
|
333
|
+
// All recency-boosted (< 5 min).
|
|
334
|
+
assert.equal(scored[0].recencyBoost, 1.2);
|
|
335
|
+
assert.equal(scored[2].recencyBoost, 1.2);
|
|
336
|
+
});
|
|
337
|
+
test("scoreEngineMessages: preserves input order and ids are stringified indices", () => {
|
|
338
|
+
const messages = [
|
|
339
|
+
{ role: "user", text: "first" },
|
|
340
|
+
{ role: "user", text: "second" },
|
|
341
|
+
];
|
|
342
|
+
const scored = scoreEngineMessages(messages, NOW);
|
|
343
|
+
assert.equal(scored[0].id, "0");
|
|
344
|
+
assert.equal(scored[1].id, "1");
|
|
345
|
+
});
|
|
346
|
+
test("scoreEngineMessages: empty input returns empty array", () => {
|
|
347
|
+
assert.deepEqual(scoreEngineMessages([], NOW), []);
|
|
348
|
+
});
|
|
349
|
+
test("scoreEngineMessages: determinism — same input → same output", () => {
|
|
350
|
+
const messages = [
|
|
351
|
+
{ role: "assistant", text: "we decided to use JWT" },
|
|
352
|
+
{ role: "user", text: "ok" },
|
|
353
|
+
];
|
|
354
|
+
const a = scoreEngineMessages(messages, NOW);
|
|
355
|
+
const b = scoreEngineMessages(messages, NOW);
|
|
356
|
+
assert.deepEqual(a, b);
|
|
357
|
+
});
|
|
358
|
+
// ---- integration: a decision outsurvives filler ----
|
|
359
|
+
test("integration: a decision is preserved over filler at the same age", () => {
|
|
360
|
+
// 10 messages, 1 decision (i=4) and 9 filler, all equally old.
|
|
361
|
+
const messages = Array.from({ length: 10 }, (_, i) => i === 4
|
|
362
|
+
? { role: "assistant", text: "we decided to use JWT auth" }
|
|
363
|
+
: { role: "assistant", text: "ok, sounds good" });
|
|
364
|
+
const scored = scoreEngineMessages(messages, NOW - 10 * 60_000); // 10 min old → no recency boost
|
|
365
|
+
// decision rawMultiplier 2.5 vs filler 1.0; same decay, same recency.
|
|
366
|
+
const decision = scored[4];
|
|
367
|
+
const filler = scored[0];
|
|
368
|
+
assert.ok(decision.finalScore > filler.finalScore);
|
|
369
|
+
// top 20% → 2 items; the decision must be in the preserved set.
|
|
370
|
+
const result = itemsToPreserve(scored, 0.2);
|
|
371
|
+
assert.ok(result.preservedIds.has("4"), "decision must be preserved");
|
|
372
|
+
});
|
|
@@ -17,7 +17,7 @@ import assert from "node:assert/strict";
|
|
|
17
17
|
import * as fs from "node:fs";
|
|
18
18
|
import * as path from "node:path";
|
|
19
19
|
import * as os from "node:os";
|
|
20
|
-
import { VectorStore, computeRegionHash } from "./vectorStore.js";
|
|
20
|
+
import { VectorStore, computeRegionHash, vectorStats, vectorWasInjected, vectorMarkInjected, vectorTopSimilar } from "./vectorStore.js";
|
|
21
21
|
import { findSuperseded } from "./supersede.js";
|
|
22
22
|
import { autoCompactCheck, isChatty } from "./compact.js";
|
|
23
23
|
import { extractiveSummarize } from "./extractive.js";
|
|
@@ -433,7 +433,7 @@ describe("Dedup Hit Rates by Similarity Level", () => {
|
|
|
433
433
|
}
|
|
434
434
|
// Check stats for each session
|
|
435
435
|
for (const sid of sessionIds) {
|
|
436
|
-
const stats = store
|
|
436
|
+
const stats = vectorStats(store, sid);
|
|
437
437
|
console.log(` ${sid}: ${stats.checkpointCount} checkpoint(s), dedup rate: ${(stats.dedupHitRate * 100).toFixed(0)}%`);
|
|
438
438
|
}
|
|
439
439
|
});
|
|
@@ -455,7 +455,7 @@ describe("Dedup Hit Rates by Similarity Level", () => {
|
|
|
455
455
|
// Check stats across sessions
|
|
456
456
|
let totalCheckpoints = 0;
|
|
457
457
|
for (let i = 0; i < variations.length; i++) {
|
|
458
|
-
const stats = store
|
|
458
|
+
const stats = vectorStats(store, `sess_l1_${i}`);
|
|
459
459
|
totalCheckpoints += stats.checkpointCount;
|
|
460
460
|
}
|
|
461
461
|
console.log(` Near-duplicates added: ${variations.length}`);
|
|
@@ -480,7 +480,7 @@ describe("Dedup Hit Rates by Similarity Level", () => {
|
|
|
480
480
|
}
|
|
481
481
|
let totalCheckpoints = 0;
|
|
482
482
|
for (let i = 0; i < variations.length; i++) {
|
|
483
|
-
const stats = store
|
|
483
|
+
const stats = vectorStats(store, `sess_l1neg_${i}`);
|
|
484
484
|
totalCheckpoints += stats.checkpointCount;
|
|
485
485
|
}
|
|
486
486
|
const collapseRate = 1 - totalCheckpoints / variations.length;
|
|
@@ -509,7 +509,7 @@ describe("Dedup Hit Rates by Similarity Level", () => {
|
|
|
509
509
|
}
|
|
510
510
|
let totalCheckpoints = 0;
|
|
511
511
|
for (let i = 0; i < paraphrases.length; i++) {
|
|
512
|
-
const stats = store
|
|
512
|
+
const stats = vectorStats(store, `sess_l2_${i}`);
|
|
513
513
|
totalCheckpoints += stats.checkpointCount;
|
|
514
514
|
}
|
|
515
515
|
console.log(` Paraphrases: ${paraphrases.length}`);
|
|
@@ -565,7 +565,7 @@ describe("Dedup Hit Rates by Similarity Level", () => {
|
|
|
565
565
|
timestamp: Date.now() + idx++ * 1000,
|
|
566
566
|
});
|
|
567
567
|
}
|
|
568
|
-
const stats = store
|
|
568
|
+
const stats = vectorStats(store, dedupSession);
|
|
569
569
|
const totalAdded = 3 + 4 + 3;
|
|
570
570
|
console.log(` Total added: ${totalAdded}`);
|
|
571
571
|
console.log(` Checkpoints stored: ${stats.checkpointCount}`);
|
|
@@ -769,7 +769,7 @@ describe("Store Stats & Dedup Metrics", () => {
|
|
|
769
769
|
timestamp: Date.now() + 100 + i,
|
|
770
770
|
});
|
|
771
771
|
}
|
|
772
|
-
const stats = store
|
|
772
|
+
const stats = vectorStats(store, session);
|
|
773
773
|
console.log(` Total added: 5`);
|
|
774
774
|
console.log(` Checkpoints stored: ${stats.checkpointCount}`);
|
|
775
775
|
console.log(` Dedup hit rate: ${(stats.dedupHitRate * 100).toFixed(0)}%`);
|
|
@@ -782,15 +782,15 @@ describe("Store Stats & Dedup Metrics", () => {
|
|
|
782
782
|
const id1 = "chkpt_inject_a";
|
|
783
783
|
const id2 = "chkpt_inject_b";
|
|
784
784
|
const sid = "sess_inject";
|
|
785
|
-
assert.equal(store
|
|
786
|
-
store
|
|
787
|
-
assert.equal(store
|
|
788
|
-
assert.equal(store
|
|
789
|
-
store
|
|
790
|
-
assert.equal(store
|
|
785
|
+
assert.equal(vectorWasInjected(store, sid, id1), false, "Should not be injected initially");
|
|
786
|
+
vectorMarkInjected(store, sid, id1);
|
|
787
|
+
assert.equal(vectorWasInjected(store, sid, id1), true, "Should be injected after mark");
|
|
788
|
+
assert.equal(vectorWasInjected(store, sid, id2), false, "Different ID should not be affected");
|
|
789
|
+
vectorMarkInjected(store, sid, id2);
|
|
790
|
+
assert.equal(vectorWasInjected(store, sid, id2), true, "Second ID should also be injected");
|
|
791
791
|
// Idempotent
|
|
792
|
-
store
|
|
793
|
-
assert.equal(store
|
|
792
|
+
vectorMarkInjected(store, sid, id1);
|
|
793
|
+
assert.equal(vectorWasInjected(store, sid, id1), true, "Re-mark should be idempotent");
|
|
794
794
|
});
|
|
795
795
|
it("recall skips already-injected checkpoints", () => {
|
|
796
796
|
const content = "Checkpoint for recall dedup sentinel test.";
|
|
@@ -813,7 +813,7 @@ describe("Store Stats & Dedup Metrics", () => {
|
|
|
813
813
|
}, store);
|
|
814
814
|
assert.ok(results1.hits.length >= 1, "First recall should return checkpoint");
|
|
815
815
|
// Mark first hit as injected
|
|
816
|
-
store
|
|
816
|
+
vectorMarkInjected(store, sid, results1.hits[0].checkpoint.checkpointId);
|
|
817
817
|
// Second recall should skip it
|
|
818
818
|
const results2 = recall({
|
|
819
819
|
sessionId: sid,
|
|
@@ -859,13 +859,13 @@ describe("topSimilar Edge Cases & Coverage", () => {
|
|
|
859
859
|
});
|
|
860
860
|
}
|
|
861
861
|
for (const n of [1, 3, 5, 10]) {
|
|
862
|
-
const results = store
|
|
862
|
+
const results = vectorTopSimilar(store, sid, n);
|
|
863
863
|
assert.ok(results.length <= n, `topSimilar(${n}) returned ${results.length} results (should be <= ${n})`);
|
|
864
864
|
console.log(` n=${n}: ${results.length} results`);
|
|
865
865
|
}
|
|
866
866
|
});
|
|
867
867
|
it("topSimilar returns empty for empty session", () => {
|
|
868
|
-
const results = store
|
|
868
|
+
const results = vectorTopSimilar(store, "sess_unknown_empty", 5);
|
|
869
869
|
assert.equal(results.length, 0, "Empty session should return no results");
|
|
870
870
|
});
|
|
871
871
|
});
|
package/dist/src/recall.js
CHANGED
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
* extension decides where it lands.
|
|
15
15
|
*/
|
|
16
16
|
import { recall as searchRecall } from "./engine.js";
|
|
17
|
+
import { vectorWasInjected, vectorMarkInjected, vectorSearchAsync } from "./vectorStore.js";
|
|
17
18
|
import { estimateBlockTokens } from "./tokens.js";
|
|
18
19
|
import { defaultEmbedder, cosineSimilarity } from "./embedder.js";
|
|
19
20
|
/** Wrap a recall block so the model reads it as restored compacted context. */
|
|
@@ -98,7 +99,7 @@ export function recallAndInline(opts, store) {
|
|
|
98
99
|
const parts = [];
|
|
99
100
|
let blockTokens = 0;
|
|
100
101
|
for (const h of hits) {
|
|
101
|
-
if (skip && store
|
|
102
|
+
if (skip && vectorWasInjected(store, opts.sessionId, h.checkpoint.checkpointId))
|
|
102
103
|
continue;
|
|
103
104
|
// Inline dedupe: skip a hit already resident in the live window (Fix C).
|
|
104
105
|
if (doWindowDedupe && liveEmbeddings.length > 0) {
|
|
@@ -114,7 +115,7 @@ export function recallAndInline(opts, store) {
|
|
|
114
115
|
parts.push(part);
|
|
115
116
|
toInject.push(h);
|
|
116
117
|
blockTokens += partTokens;
|
|
117
|
-
store
|
|
118
|
+
vectorMarkInjected(store, opts.sessionId, h.checkpoint.checkpointId);
|
|
118
119
|
}
|
|
119
120
|
const block = parts.join("\n");
|
|
120
121
|
const report = toInject.map((h) => ` • ${h.checkpoint.checkpointId} (${h.checkpoint.summary.slice(0, 60).replace(/\n/g, " ")}…)`);
|
|
@@ -210,7 +211,7 @@ export async function recallAndInlineAsync(opts, store) {
|
|
|
210
211
|
const dedupSim = opts.dedupSim ?? 0.9;
|
|
211
212
|
let hits = [];
|
|
212
213
|
try {
|
|
213
|
-
hits = await store
|
|
214
|
+
hits = await vectorSearchAsync(store, opts.sessionId, opts.query, limit, {
|
|
214
215
|
crossRepo: opts.crossRepo,
|
|
215
216
|
repoId: opts.repoId,
|
|
216
217
|
});
|
|
@@ -227,7 +228,7 @@ export async function recallAndInlineAsync(opts, store) {
|
|
|
227
228
|
const parts = [];
|
|
228
229
|
let blockTokens = 0;
|
|
229
230
|
for (const h of hits) {
|
|
230
|
-
if (skip && store
|
|
231
|
+
if (skip && vectorWasInjected(store, opts.sessionId, h.checkpoint.checkpointId))
|
|
231
232
|
continue;
|
|
232
233
|
// S18: machine-wide injected-set — a foreign checkpoint already injected
|
|
233
234
|
// (in any session) is never re-injected. Only applies to cross-repo hits
|
|
@@ -254,7 +255,7 @@ export async function recallAndInlineAsync(opts, store) {
|
|
|
254
255
|
parts.push(part);
|
|
255
256
|
toInject.push(h);
|
|
256
257
|
blockTokens += partTokens;
|
|
257
|
-
store
|
|
258
|
+
vectorMarkInjected(store, opts.sessionId, h.checkpoint.checkpointId);
|
|
258
259
|
// S18: record the cross-repo injection machine-wide so it's not re-injected
|
|
259
260
|
// by a later recall (same or different session).
|
|
260
261
|
if (opts.globalIndexDir && h.repoId) {
|