pi-mega-compact 0.8.21 → 0.8.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +6 -2
- package/README.md +1 -1
- package/dist/extensions/dashboard-server/dashboard-client-core.js +201 -0
- package/dist/extensions/dashboard-server/dashboard-client-game.js +241 -0
- package/dist/extensions/dashboard-server/dashboard-client-repos.js +212 -0
- package/dist/extensions/dashboard-server/dashboard-client.js +19 -0
- package/dist/extensions/dashboard-server/html.js +2 -621
- package/dist/extensions/dashboard-server/routes-core.js +62 -0
- package/dist/extensions/dashboard-server/routes-game.js +323 -0
- package/dist/extensions/dashboard-server/routes-repo.js +170 -0
- package/dist/extensions/dashboard-server/routes-sessions.js +159 -0
- package/dist/extensions/dashboard-server/routes.js +10 -0
- package/dist/extensions/dashboard-server/server.js +26 -623
- package/dist/extensions/mega-commands.js +4 -3
- package/dist/extensions/mega-events/agent-handlers.js +2 -1
- package/dist/extensions/mega-events/compact-handlers.js +26 -0
- package/dist/extensions/mega-events/session-handlers.js +2 -1
- package/dist/extensions/mega-pipeline/compact.js +3 -2
- package/dist/extensions/mega-runtime/state.js +7 -7
- package/dist/src/dedup/raptor/multilevel.js +172 -0
- package/dist/src/dedup/raptor/multilevel.test.js +203 -0
- package/dist/src/dedup/raptor/promote.test.js +5 -5
- package/dist/src/dedup/raptor/retrieval.js +1 -1
- package/dist/src/dedup/sprint12.test.js +7 -7
- package/dist/src/dedup-engine.test.js +29 -29
- package/dist/src/e2e.test.js +38 -38
- package/dist/src/engine.js +3 -3
- package/dist/src/engine.test.js +6 -6
- package/dist/src/importance.js +197 -0
- package/dist/src/importance.test.js +372 -0
- package/dist/src/ratio.bench.test.js +18 -18
- package/dist/src/recall.js +6 -5
- package/dist/src/recall.test.js +85 -27
- package/dist/src/sprint14.test.js +2 -2
- package/dist/src/store/migrate.test.js +5 -5
- package/dist/src/store/sprint10.test.js +5 -5
- package/dist/src/store/sqlite/global-index.js +5 -174
- package/dist/src/store/sqlite/global-sessions.js +190 -0
- package/dist/src/vector-read.js +168 -0
- package/dist/src/vector-search.js +191 -0
- package/dist/src/vectorStore.js +10 -297
- package/dist/src/vectorStore.test.js +32 -32
- package/extensions/dashboard-server/dashboard-client-core.ts +202 -0
- package/extensions/dashboard-server/dashboard-client-game.ts +242 -0
- package/extensions/dashboard-server/dashboard-client-repos.ts +213 -0
- package/extensions/dashboard-server/dashboard-client.ts +21 -0
- package/extensions/dashboard-server/html.ts +2 -621
- package/extensions/dashboard-server/routes-core.ts +113 -0
- package/extensions/dashboard-server/routes-game.ts +386 -0
- package/extensions/dashboard-server/routes-repo.ts +212 -0
- package/extensions/dashboard-server/routes-sessions.ts +195 -0
- package/extensions/dashboard-server/routes.ts +13 -0
- package/extensions/dashboard-server/server.ts +37 -700
- package/extensions/mega-commands.ts +4 -3
- package/extensions/mega-events/agent-handlers.ts +2 -1
- package/extensions/mega-events/compact-handlers.ts +28 -0
- package/extensions/mega-events/session-handlers.ts +2 -1
- package/extensions/mega-pipeline/compact.ts +3 -2
- package/extensions/mega-runtime/state.ts +7 -7
- package/extensions/openclaw-mega-compact.ts +2 -2
- package/package.json +2 -2
- package/src/dedup/raptor/multilevel.test.ts +278 -0
- package/src/dedup/raptor/multilevel.ts +246 -0
- package/src/dedup/raptor/promote.test.ts +5 -5
- package/src/dedup/raptor/retrieval.ts +1 -1
- package/src/dedup/sprint12.test.ts +7 -7
- package/src/dedup-engine.test.ts +30 -30
- package/src/e2e.test.ts +38 -38
- package/src/engine.test.ts +6 -6
- package/src/engine.ts +3 -3
- package/src/importance.test.ts +538 -0
- package/src/importance.ts +312 -0
- package/src/ratio.bench.test.ts +18 -18
- package/src/recall.test.ts +101 -29
- package/src/recall.ts +9 -9
- package/src/sprint14.test.ts +2 -2
- package/src/store/migrate.test.ts +5 -5
- package/src/store/sprint10.test.ts +5 -5
- package/src/store/sqlite/global-index.ts +18 -290
- package/src/store/sqlite/global-sessions.ts +291 -0
- package/src/vector-read.ts +237 -0
- package/src/vector-search.ts +231 -0
- package/src/vectorStore.test.ts +32 -32
- package/src/vectorStore.ts +29 -356
|
@@ -0,0 +1,312 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* importance.ts — S40 importance scoring for compaction.
|
|
3
|
+
*
|
|
4
|
+
* A standalone, deterministic, pi-agnostic scoring module with no side
|
|
5
|
+
* effects. Scores each context item by type, age decay, recency/retention
|
|
6
|
+
* boosts, and selects the top-N for verbatim preservation (bypassing
|
|
7
|
+
* summarization) so high-importance old messages (decisions, errors) are
|
|
8
|
+
* not lost to compaction.
|
|
9
|
+
*
|
|
10
|
+
* PREVENT-PI-001 (anchor floor): importance scoring AUGMENTS, never
|
|
11
|
+
* replaces, the boundary guard. `itemsToPreserve` only marks items the
|
|
12
|
+
* caller already considered compactable; the caller's `computeDropRange`
|
|
13
|
+
* still protects the most recent N user messages.
|
|
14
|
+
* PREVENT-PI-002 (tool pairs): callers preserve pairs; `score()` itself
|
|
15
|
+
* is pair-agnostic.
|
|
16
|
+
* PREVENT-PI-PI-003 (no system role): scored items are surfaced via the
|
|
17
|
+
* existing systemPrompt-prepend path, never as role:"system".
|
|
18
|
+
*
|
|
19
|
+
* @module
|
|
20
|
+
*/
|
|
21
|
+
import type { EngineMessage } from "./types.js";
|
|
22
|
+
|
|
23
|
+
/** The 8 content/item types that drive the type multiplier. */
|
|
24
|
+
export enum ContextItemType {
|
|
25
|
+
UserMessage = "user_message",
|
|
26
|
+
AssistantMessage = "assistant_message",
|
|
27
|
+
SystemMessage = "system_message",
|
|
28
|
+
CodeBlock = "code_block",
|
|
29
|
+
Error = "error",
|
|
30
|
+
Decision = "decision",
|
|
31
|
+
FileModification = "file_modification",
|
|
32
|
+
ToolExecution = "tool_execution",
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** A scored context item. */
|
|
36
|
+
export interface ScoredItem {
|
|
37
|
+
/** Message index or checkpoint ID (caller-supplied). */
|
|
38
|
+
id: string;
|
|
39
|
+
type: ContextItemType;
|
|
40
|
+
content: string;
|
|
41
|
+
/** Raw role from EngineMessage ("user" | "assistant" | "tool" | "custom"). */
|
|
42
|
+
role: string;
|
|
43
|
+
/** Epoch ms. */
|
|
44
|
+
timestamp: number;
|
|
45
|
+
/** Type-based multiplier, before decay/boost. */
|
|
46
|
+
rawMultiplier: number;
|
|
47
|
+
/** 0–0.7 fraction SUBTRACTED from the score. */
|
|
48
|
+
ageDecay: number;
|
|
49
|
+
/** 1.0 or 1.2. */
|
|
50
|
+
recencyBoost: number;
|
|
51
|
+
/** 1.0 or 3.0. */
|
|
52
|
+
retentionBoost: number;
|
|
53
|
+
/** Composite score. */
|
|
54
|
+
finalScore: number;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** Result of a preservation pass. */
|
|
58
|
+
export interface PreservationResult {
|
|
59
|
+
preservedIds: Set<string>;
|
|
60
|
+
/** Score cutoff used. */
|
|
61
|
+
threshold: number;
|
|
62
|
+
totalScored: number;
|
|
63
|
+
totalPreserved: number;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Default type multipliers (ported from the Rust reference
|
|
68
|
+
* `router/src/context/importance.rs`). Decisions (2.5x) and errors (2.0x)
|
|
69
|
+
* dominate; system/filler (0.5x) sinks.
|
|
70
|
+
*/
|
|
71
|
+
export const DEFAULT_MULTIPLIERS: Record<ContextItemType, number> = {
|
|
72
|
+
[ContextItemType.UserMessage]: 1.5,
|
|
73
|
+
[ContextItemType.AssistantMessage]: 1.0,
|
|
74
|
+
[ContextItemType.SystemMessage]: 0.5,
|
|
75
|
+
[ContextItemType.CodeBlock]: 1.2,
|
|
76
|
+
[ContextItemType.Error]: 2.0,
|
|
77
|
+
[ContextItemType.Decision]: 2.5,
|
|
78
|
+
[ContextItemType.FileModification]: 1.8,
|
|
79
|
+
[ContextItemType.ToolExecution]: 1.3,
|
|
80
|
+
};
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Age decay: fraction to SUBTRACT from the score.
|
|
84
|
+
* Formula: `min(maxDecay, (ageMs / 3_600_000) * decayRatePerHour)`.
|
|
85
|
+
* 0 = fresh, 0.7 = very old. At 14h with 0.05/hr: 14 * 0.05 = 0.70 (capped).
|
|
86
|
+
*/
|
|
87
|
+
export function ageDecay(
|
|
88
|
+
itemAgeMs: number,
|
|
89
|
+
decayRatePerHour: number = 0.05,
|
|
90
|
+
maxDecay: number = 0.7,
|
|
91
|
+
): number {
|
|
92
|
+
if (itemAgeMs <= 0) return 0;
|
|
93
|
+
const hours = itemAgeMs / 3_600_000;
|
|
94
|
+
return Math.min(maxDecay, hours * decayRatePerHour);
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/** Recency boost: 1.2 if `ageMs < thresholdMs` (default 5min), else 1.0. */
|
|
98
|
+
export function recencyBoost(
|
|
99
|
+
ageMs: number,
|
|
100
|
+
thresholdMs: number = 300_000,
|
|
101
|
+
): number {
|
|
102
|
+
return ageMs < thresholdMs ? 1.2 : 1.0;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* Retention boost: 3.0 if the user flagged the item, else 1.0.
|
|
107
|
+
* User-flagged items are also inferred by `detectItemType` returning
|
|
108
|
+
* Decision or Error.
|
|
109
|
+
*/
|
|
110
|
+
export function retentionBoost(userFlagged: boolean): number {
|
|
111
|
+
return userFlagged ? 3.0 : 1.0;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Classify a content blob into a `ContextItemType`.
|
|
116
|
+
*
|
|
117
|
+
* Rules are ordered by priority — first match wins:
|
|
118
|
+
* 1. role === "tool" → ToolExecution
|
|
119
|
+
* 2. role === "custom" → SystemMessage
|
|
120
|
+
* 3. error-ish content → Error
|
|
121
|
+
* 4. decision-ish content → Decision
|
|
122
|
+
* 5. fenced code block (≥20 chars) → CodeBlock
|
|
123
|
+
* 6. file-modification verbs → FileModification
|
|
124
|
+
* 7. role === "user" → UserMessage
|
|
125
|
+
* 8. role === "assistant" → AssistantMessage
|
|
126
|
+
* 9. Fallback → AssistantMessage
|
|
127
|
+
*/
|
|
128
|
+
export function detectItemType(
|
|
129
|
+
content: string,
|
|
130
|
+
role: "user" | "assistant" | "tool" | "custom",
|
|
131
|
+
): ContextItemType {
|
|
132
|
+
const c = content ?? "";
|
|
133
|
+
if (role === "tool") return ContextItemType.ToolExecution;
|
|
134
|
+
if (role === "custom") return ContextItemType.SystemMessage;
|
|
135
|
+
// (3) errors — traceback / panic / E#### / exception / failure / crash.
|
|
136
|
+
if (/error|exception|failure|crash|panic|traceback|E\d{4}/i.test(c)) {
|
|
137
|
+
return ContextItemType.Error;
|
|
138
|
+
}
|
|
139
|
+
// (4) decisions — "we decided", "going with", "switching to", etc.
|
|
140
|
+
if (
|
|
141
|
+
/decided|we chose|going with|switching to|using .* instead|final decision|let's go with/i.test(
|
|
142
|
+
c,
|
|
143
|
+
)
|
|
144
|
+
) {
|
|
145
|
+
return ContextItemType.Decision;
|
|
146
|
+
}
|
|
147
|
+
// (5) fenced code block ≥20 chars.
|
|
148
|
+
if (/```[\s\S]{20,}/.test(c)) {
|
|
149
|
+
return ContextItemType.CodeBlock;
|
|
150
|
+
}
|
|
151
|
+
// (6) file modification verbs.
|
|
152
|
+
if (/(wrote|edited|created|modified|updated|patched)\s+\S+\.\w+/i.test(c)) {
|
|
153
|
+
return ContextItemType.FileModification;
|
|
154
|
+
}
|
|
155
|
+
if (role === "user") return ContextItemType.UserMessage;
|
|
156
|
+
if (role === "assistant") return ContextItemType.AssistantMessage;
|
|
157
|
+
// (9) Fallback.
|
|
158
|
+
return ContextItemType.AssistantMessage;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* Composite score for a single item.
|
|
163
|
+
*
|
|
164
|
+
* Formula:
|
|
165
|
+
* type = detectItemType(content, role)
|
|
166
|
+
* rawMult = multipliers[type] ?? DEFAULT_MULTIPLIERS[type]
|
|
167
|
+
* decay = ageDecay(now - timestamp, decayRatePerHour, maxDecay)
|
|
168
|
+
* recency = recencyBoost(now - timestamp, recencyThresholdMs)
|
|
169
|
+
* retention = retentionBoost(userFlagged)
|
|
170
|
+
* finalScore = rawMult * (1 - decay) * recency * retention
|
|
171
|
+
*
|
|
172
|
+
* Clamps `finalScore` to a minimum of 0.01 (never zero, so an item is
|
|
173
|
+
* always a candidate unless explicitly filtered).
|
|
174
|
+
*/
|
|
175
|
+
export function score(
|
|
176
|
+
item: {
|
|
177
|
+
id: string;
|
|
178
|
+
content: string;
|
|
179
|
+
role: string;
|
|
180
|
+
timestamp: number;
|
|
181
|
+
userFlagged?: boolean;
|
|
182
|
+
},
|
|
183
|
+
now: number,
|
|
184
|
+
multipliers?: Partial<Record<ContextItemType, number>>,
|
|
185
|
+
opts?: {
|
|
186
|
+
decayRatePerHour?: number;
|
|
187
|
+
maxDecay?: number;
|
|
188
|
+
recencyThresholdMs?: number;
|
|
189
|
+
},
|
|
190
|
+
): ScoredItem {
|
|
191
|
+
const content = item.content ?? "";
|
|
192
|
+
// detectItemType expects the 4-role union; narrow defensively.
|
|
193
|
+
const role = (["user", "assistant", "tool", "custom"].includes(item.role)
|
|
194
|
+
? item.role
|
|
195
|
+
: "assistant") as "user" | "assistant" | "tool" | "custom";
|
|
196
|
+
const type = detectItemType(content, role);
|
|
197
|
+
const rawMultiplier =
|
|
198
|
+
multipliers?.[type] ?? DEFAULT_MULTIPLIERS[type];
|
|
199
|
+
const ageMs = Math.max(0, now - item.timestamp);
|
|
200
|
+
const decay = ageDecay(
|
|
201
|
+
ageMs,
|
|
202
|
+
opts?.decayRatePerHour,
|
|
203
|
+
opts?.maxDecay,
|
|
204
|
+
);
|
|
205
|
+
const recency = recencyBoost(ageMs, opts?.recencyThresholdMs);
|
|
206
|
+
const retention = retentionBoost(item.userFlagged === true);
|
|
207
|
+
const finalScore = Math.max(
|
|
208
|
+
0.01,
|
|
209
|
+
rawMultiplier * (1 - decay) * recency * retention,
|
|
210
|
+
);
|
|
211
|
+
return {
|
|
212
|
+
id: item.id,
|
|
213
|
+
type,
|
|
214
|
+
content,
|
|
215
|
+
role,
|
|
216
|
+
timestamp: item.timestamp,
|
|
217
|
+
rawMultiplier,
|
|
218
|
+
ageDecay: decay,
|
|
219
|
+
recencyBoost: recency,
|
|
220
|
+
retentionBoost: retention,
|
|
221
|
+
finalScore,
|
|
222
|
+
};
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
/**
|
|
226
|
+
* Score cutoff at the `preserveRatio` percentile boundary.
|
|
227
|
+
*
|
|
228
|
+
* Sorts items by `finalScore` descending; returns the score at the
|
|
229
|
+
* `preserveRatio` quantile. Items with `finalScore >= threshold` are
|
|
230
|
+
* preserved.
|
|
231
|
+
*
|
|
232
|
+
* - `ratio = 1.0` → return 0 (preserve all)
|
|
233
|
+
* - `ratio = 0` → return Infinity (preserve none)
|
|
234
|
+
* - empty input → return Infinity
|
|
235
|
+
*/
|
|
236
|
+
export function preservationCutoff(
|
|
237
|
+
items: ScoredItem[],
|
|
238
|
+
preserveRatio: number,
|
|
239
|
+
): number {
|
|
240
|
+
if (items.length === 0) return Infinity;
|
|
241
|
+
if (preserveRatio >= 1) return 0;
|
|
242
|
+
if (preserveRatio <= 0) return Infinity;
|
|
243
|
+
// Sort descending; the top `preserveRatio` fraction is preserved.
|
|
244
|
+
const sorted = [...items].sort((a, b) => b.finalScore - a.finalScore);
|
|
245
|
+
// Index of the lowest-scoring item in the preserved set.
|
|
246
|
+
// ceil(N * ratio) - 1, clamped to [0, N-1].
|
|
247
|
+
const cutoffIdx = Math.min(
|
|
248
|
+
sorted.length - 1,
|
|
249
|
+
Math.max(0, Math.ceil(sorted.length * preserveRatio) - 1),
|
|
250
|
+
);
|
|
251
|
+
return sorted[cutoffIdx].finalScore;
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
/**
|
|
255
|
+
* Select which items to preserve verbatim based on `preserveRatio`.
|
|
256
|
+
*
|
|
257
|
+
* Returns the IDs of items with `finalScore >= threshold`. The caller
|
|
258
|
+
* is responsible for excluding items inside the anchor window (the
|
|
259
|
+
* boundary guard already protects those).
|
|
260
|
+
*/
|
|
261
|
+
export function itemsToPreserve(
|
|
262
|
+
items: ScoredItem[],
|
|
263
|
+
preserveRatio: number,
|
|
264
|
+
): PreservationResult {
|
|
265
|
+
const threshold = preservationCutoff(items, preserveRatio);
|
|
266
|
+
const preservedIds = new Set(
|
|
267
|
+
items.filter((i) => i.finalScore >= threshold).map((i) => i.id),
|
|
268
|
+
);
|
|
269
|
+
return {
|
|
270
|
+
preservedIds,
|
|
271
|
+
threshold,
|
|
272
|
+
totalScored: items.length,
|
|
273
|
+
totalPreserved: preservedIds.size,
|
|
274
|
+
};
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
/**
|
|
278
|
+
* Convenience: score a list of `EngineMessage`s in position order using
|
|
279
|
+
* approximate ages from their index (1 minute per position, oldest first).
|
|
280
|
+
* Real timestamps should be threaded through the extension adapter when
|
|
281
|
+
* available; this is the position-based fallback documented in S40B-2.
|
|
282
|
+
*
|
|
283
|
+
* The returned `ScoredItem[]` is in the SAME ORDER as the input — callers
|
|
284
|
+
* index back into `messages` by `id` (which is the stringified index).
|
|
285
|
+
*/
|
|
286
|
+
export function scoreEngineMessages(
|
|
287
|
+
messages: EngineMessage[],
|
|
288
|
+
now: number,
|
|
289
|
+
multipliers?: Partial<Record<ContextItemType, number>>,
|
|
290
|
+
opts?: {
|
|
291
|
+
decayRatePerHour?: number;
|
|
292
|
+
maxDecay?: number;
|
|
293
|
+
recencyThresholdMs?: number;
|
|
294
|
+
},
|
|
295
|
+
): ScoredItem[] {
|
|
296
|
+
return messages.map((m, i) => {
|
|
297
|
+
// Oldest message (i=0) is `messages.length` minutes ago; newest is 1 min.
|
|
298
|
+
const timestamp = now - (messages.length - i) * 60_000;
|
|
299
|
+
return score(
|
|
300
|
+
{
|
|
301
|
+
id: String(i),
|
|
302
|
+
content: m.text ?? "",
|
|
303
|
+
role: m.role,
|
|
304
|
+
timestamp,
|
|
305
|
+
userFlagged: false,
|
|
306
|
+
},
|
|
307
|
+
now,
|
|
308
|
+
multipliers,
|
|
309
|
+
opts,
|
|
310
|
+
);
|
|
311
|
+
});
|
|
312
|
+
}
|
package/src/ratio.bench.test.ts
CHANGED
|
@@ -19,7 +19,7 @@ import * as fs from "node:fs";
|
|
|
19
19
|
import * as path from "node:path";
|
|
20
20
|
import * as os from "node:os";
|
|
21
21
|
|
|
22
|
-
import { VectorStore, computeRegionHash } from "./vectorStore.js";
|
|
22
|
+
import { VectorStore, computeRegionHash, vectorStats, vectorWasInjected, vectorMarkInjected, vectorTopSimilar } from "./vectorStore.js";
|
|
23
23
|
import type { SearchHit } from "./vectorStore.js";
|
|
24
24
|
import { findSuperseded } from "./supersede.js";
|
|
25
25
|
import { autoCompactCheck, isChatty } from "./compact.js";
|
|
@@ -635,7 +635,7 @@ describe("Dedup Hit Rates by Similarity Level", () => {
|
|
|
635
635
|
|
|
636
636
|
// Check stats for each session
|
|
637
637
|
for (const sid of sessionIds) {
|
|
638
|
-
const stats = store
|
|
638
|
+
const stats = vectorStats(store,sid);
|
|
639
639
|
console.log(
|
|
640
640
|
` ${sid}: ${stats.checkpointCount} checkpoint(s), dedup rate: ${(stats.dedupHitRate * 100).toFixed(0)}%`,
|
|
641
641
|
);
|
|
@@ -663,7 +663,7 @@ describe("Dedup Hit Rates by Similarity Level", () => {
|
|
|
663
663
|
// Check stats across sessions
|
|
664
664
|
let totalCheckpoints = 0;
|
|
665
665
|
for (let i = 0; i < variations.length; i++) {
|
|
666
|
-
const stats = store
|
|
666
|
+
const stats = vectorStats(store,`sess_l1_${i}`);
|
|
667
667
|
totalCheckpoints += stats.checkpointCount;
|
|
668
668
|
}
|
|
669
669
|
|
|
@@ -693,7 +693,7 @@ describe("Dedup Hit Rates by Similarity Level", () => {
|
|
|
693
693
|
|
|
694
694
|
let totalCheckpoints = 0;
|
|
695
695
|
for (let i = 0; i < variations.length; i++) {
|
|
696
|
-
const stats = store
|
|
696
|
+
const stats = vectorStats(store,`sess_l1neg_${i}`);
|
|
697
697
|
totalCheckpoints += stats.checkpointCount;
|
|
698
698
|
}
|
|
699
699
|
|
|
@@ -733,7 +733,7 @@ describe("Dedup Hit Rates by Similarity Level", () => {
|
|
|
733
733
|
|
|
734
734
|
let totalCheckpoints = 0;
|
|
735
735
|
for (let i = 0; i < paraphrases.length; i++) {
|
|
736
|
-
const stats = store
|
|
736
|
+
const stats = vectorStats(store,`sess_l2_${i}`);
|
|
737
737
|
totalCheckpoints += stats.checkpointCount;
|
|
738
738
|
}
|
|
739
739
|
|
|
@@ -798,7 +798,7 @@ describe("Dedup Hit Rates by Similarity Level", () => {
|
|
|
798
798
|
});
|
|
799
799
|
}
|
|
800
800
|
|
|
801
|
-
const stats = store
|
|
801
|
+
const stats = vectorStats(store,dedupSession);
|
|
802
802
|
const totalAdded = 3 + 4 + 3;
|
|
803
803
|
|
|
804
804
|
console.log(` Total added: ${totalAdded}`);
|
|
@@ -1113,7 +1113,7 @@ describe("Store Stats & Dedup Metrics", () => {
|
|
|
1113
1113
|
});
|
|
1114
1114
|
}
|
|
1115
1115
|
|
|
1116
|
-
const stats = store
|
|
1116
|
+
const stats = vectorStats(store,session);
|
|
1117
1117
|
|
|
1118
1118
|
console.log(` Total added: 5`);
|
|
1119
1119
|
console.log(` Checkpoints stored: ${stats.checkpointCount}`);
|
|
@@ -1136,34 +1136,34 @@ describe("Store Stats & Dedup Metrics", () => {
|
|
|
1136
1136
|
const sid = "sess_inject";
|
|
1137
1137
|
|
|
1138
1138
|
assert.equal(
|
|
1139
|
-
store
|
|
1139
|
+
vectorWasInjected(store,sid, id1),
|
|
1140
1140
|
false,
|
|
1141
1141
|
"Should not be injected initially",
|
|
1142
1142
|
);
|
|
1143
1143
|
|
|
1144
|
-
store
|
|
1144
|
+
vectorMarkInjected(store,sid, id1);
|
|
1145
1145
|
assert.equal(
|
|
1146
|
-
store
|
|
1146
|
+
vectorWasInjected(store,sid, id1),
|
|
1147
1147
|
true,
|
|
1148
1148
|
"Should be injected after mark",
|
|
1149
1149
|
);
|
|
1150
1150
|
assert.equal(
|
|
1151
|
-
store
|
|
1151
|
+
vectorWasInjected(store,sid, id2),
|
|
1152
1152
|
false,
|
|
1153
1153
|
"Different ID should not be affected",
|
|
1154
1154
|
);
|
|
1155
1155
|
|
|
1156
|
-
store
|
|
1156
|
+
vectorMarkInjected(store,sid, id2);
|
|
1157
1157
|
assert.equal(
|
|
1158
|
-
store
|
|
1158
|
+
vectorWasInjected(store,sid, id2),
|
|
1159
1159
|
true,
|
|
1160
1160
|
"Second ID should also be injected",
|
|
1161
1161
|
);
|
|
1162
1162
|
|
|
1163
1163
|
// Idempotent
|
|
1164
|
-
store
|
|
1164
|
+
vectorMarkInjected(store,sid, id1);
|
|
1165
1165
|
assert.equal(
|
|
1166
|
-
store
|
|
1166
|
+
vectorWasInjected(store,sid, id1),
|
|
1167
1167
|
true,
|
|
1168
1168
|
"Re-mark should be idempotent",
|
|
1169
1169
|
);
|
|
@@ -1200,7 +1200,7 @@ describe("Store Stats & Dedup Metrics", () => {
|
|
|
1200
1200
|
);
|
|
1201
1201
|
|
|
1202
1202
|
// Mark first hit as injected
|
|
1203
|
-
store
|
|
1203
|
+
vectorMarkInjected(store,
|
|
1204
1204
|
sid,
|
|
1205
1205
|
results1.hits[0].checkpoint.checkpointId,
|
|
1206
1206
|
);
|
|
@@ -1272,7 +1272,7 @@ describe("topSimilar Edge Cases & Coverage", () => {
|
|
|
1272
1272
|
}
|
|
1273
1273
|
|
|
1274
1274
|
for (const n of [1, 3, 5, 10]) {
|
|
1275
|
-
const results = store
|
|
1275
|
+
const results = vectorTopSimilar(store,sid, n);
|
|
1276
1276
|
assert.ok(
|
|
1277
1277
|
results.length <= n,
|
|
1278
1278
|
`topSimilar(${n}) returned ${results.length} results (should be <= ${n})`,
|
|
@@ -1282,7 +1282,7 @@ describe("topSimilar Edge Cases & Coverage", () => {
|
|
|
1282
1282
|
});
|
|
1283
1283
|
|
|
1284
1284
|
it("topSimilar returns empty for empty session", () => {
|
|
1285
|
-
const results = store
|
|
1285
|
+
const results = vectorTopSimilar(store,"sess_unknown_empty", 5);
|
|
1286
1286
|
assert.equal(results.length, 0, "Empty session should return no results");
|
|
1287
1287
|
});
|
|
1288
1288
|
});
|
package/src/recall.test.ts
CHANGED
|
@@ -6,7 +6,13 @@ import { join } from "node:path";
|
|
|
6
6
|
import { VectorStore } from "./vectorStore.js";
|
|
7
7
|
import { compactSession } from "./engine.js";
|
|
8
8
|
import { recallAndInline, recallAndInlineAsync, formatRecallBlock } from "./recall.js";
|
|
9
|
+
import { vectorList } from "./vectorStore.js";
|
|
9
10
|
import { markInjectedGlobal, wasInjectedGlobal, closeIndexStore } from "./store/sqlite.js";
|
|
11
|
+
import {
|
|
12
|
+
closeVectorIndex,
|
|
13
|
+
initVectorIndex,
|
|
14
|
+
rebuildFromSqlite,
|
|
15
|
+
} from "./store/vectorIndex.js";
|
|
10
16
|
import type { EngineMessage } from "./types.js";
|
|
11
17
|
|
|
12
18
|
const baseTmp = mkdtempSync(join(tmpdir(), "mc-recall-"));
|
|
@@ -119,58 +125,124 @@ test("Fix C: inline dedupe drops a hit already resident in the live window", ()
|
|
|
119
125
|
);
|
|
120
126
|
});
|
|
121
127
|
|
|
128
|
+
// ---- S18 cross-repo global injected-set (real-data, no mocks) ----------------
|
|
129
|
+
//
|
|
130
|
+
// Earlier versions of these tests used `as any` mock stores with canned
|
|
131
|
+
// `searchAsync` returns. That was mock data: it asserted recall's orchestration
|
|
132
|
+
// against a fake search result, not the real embed → HNSW → hydrate → inject
|
|
133
|
+
// path. Per the no-mock-data principle, both tests now seed a REAL foreign
|
|
134
|
+
// VectorStore (real checkpoint + real TrigramEmbedder embedding persisted to
|
|
135
|
+
// SQLite), rebuild the real PGlite index from it via `rebuildFromSqlite`, and
|
|
136
|
+
// run the full `recallAndInlineAsync` cross-repo path against a separate self
|
|
137
|
+
// store. The foreign checkpoint hydrates from the foreign store via its real
|
|
138
|
+
// repoId (== stateDir, per VectorStore's repoId convention).
|
|
139
|
+
|
|
140
|
+
async function seedForeignRepo(foreignStateDir: string): Promise<{ sessionId: string; checkpointId: string; summary: string }> {
|
|
141
|
+
// A real VectorStore at the foreign repo's stateDir. repoId == stateDir.
|
|
142
|
+
const foreign = new VectorStore({ stateDir: foreignStateDir, dedupSim: 0.9 });
|
|
143
|
+
const sess = "sess_foreign";
|
|
144
|
+
// Seed a real checkpoint via the real compactSession pipeline. The summary
|
|
145
|
+
// text determines the embedding; the query below must land near it.
|
|
146
|
+
const summary = "foreign repo authentication jwt token validation";
|
|
147
|
+
const result = compactSession(
|
|
148
|
+
{
|
|
149
|
+
sessionId: sess,
|
|
150
|
+
messages: [msg("user", summary), msg("assistant", "ok", "Edit")],
|
|
151
|
+
keepFrom: 2,
|
|
152
|
+
timestamp: 1,
|
|
153
|
+
},
|
|
154
|
+
foreign,
|
|
155
|
+
);
|
|
156
|
+
return { sessionId: sess, checkpointId: result.checkpointId ?? "", summary };
|
|
157
|
+
}
|
|
158
|
+
|
|
122
159
|
test("S18: global injected-set skips a foreign checkpoint already injected machine-wide", async () => {
|
|
160
|
+
if (process.env.MEGACOMPACT_PGLITE_DISABLED === "true") { return; } // skip when WASM index is off
|
|
123
161
|
const indexDir = mkdtempSync(join(tmpdir(), "mc-gi-"));
|
|
162
|
+
const foreignStateDir = mkdtempSync(join(tmpdir(), "mc-gi-foreign-"));
|
|
163
|
+
const selfStateDir = mkdtempSync(join(tmpdir(), "mc-gi-self-"));
|
|
164
|
+
process.env.MEGACOMPACT_VECTOR_INDEX_DIR = mkdtempSync(join(tmpdir(), "mc-gi-vidx-"));
|
|
124
165
|
try {
|
|
166
|
+
await closeVectorIndex(); // fresh singleton per this index dir
|
|
167
|
+
const seed = await seedForeignRepo(foreignStateDir);
|
|
168
|
+
// Populate the real PGlite index from the foreign store's real checkpoints.
|
|
169
|
+
await rebuildFromSqlite(
|
|
170
|
+
() => [{ repoId: foreignStateDir, stateDir: foreignStateDir }],
|
|
171
|
+
(sd) => {
|
|
172
|
+
// readCheckpoints: yield (sessionId, checkpointId, embedding) for every
|
|
173
|
+
// real checkpoint in this store. Mirrors the production enumerator.
|
|
174
|
+
const store = new VectorStore({ stateDir: sd, dedupSim: 0.9 });
|
|
175
|
+
return vectorList(store, seed.sessionId).map((cp) => ({
|
|
176
|
+
sessionId: seed.sessionId,
|
|
177
|
+
checkpointId: cp.checkpointId,
|
|
178
|
+
embedding: cp.embedding,
|
|
179
|
+
}));
|
|
180
|
+
},
|
|
181
|
+
);
|
|
182
|
+
const pg = await initVectorIndex();
|
|
183
|
+
assert.ok(pg, "PGlite index should initialize (WASM available)");
|
|
184
|
+
|
|
125
185
|
const sess = "sess_cross";
|
|
126
|
-
//
|
|
127
|
-
markInjectedGlobal(
|
|
128
|
-
assert.equal(wasInjectedGlobal(
|
|
129
|
-
|
|
130
|
-
// (
|
|
131
|
-
const
|
|
132
|
-
searchAsync: async () => [{
|
|
133
|
-
checkpoint: { checkpointId: "chkpt_foreign", summary: "foreign work", filesModified: [], dedupStatus: "active" },
|
|
134
|
-
score: 0.92,
|
|
135
|
-
repoId: "/repo/other",
|
|
136
|
-
}],
|
|
137
|
-
wasInjected: () => false,
|
|
138
|
-
markInjected: () => {},
|
|
139
|
-
} as any;
|
|
186
|
+
// Pre-mark the foreign checkpoint as already injected machine-wide.
|
|
187
|
+
markInjectedGlobal(seed.checkpointId, foreignStateDir, sess, indexDir);
|
|
188
|
+
assert.equal(wasInjectedGlobal(seed.checkpointId, sess, indexDir), true);
|
|
189
|
+
|
|
190
|
+
// Self store (different repo) — the cross-repo query runs against the index.
|
|
191
|
+
const selfStore = new VectorStore({ stateDir: selfStateDir, dedupSim: 0.9 });
|
|
140
192
|
const r = await recallAndInlineAsync(
|
|
141
|
-
{ sessionId: sess, query: "foreign", limit: 3, source: "command", crossRepo: true, globalIndexDir: indexDir },
|
|
142
|
-
|
|
193
|
+
{ sessionId: sess, query: "foreign repo authentication jwt", limit: 3, source: "command", crossRepo: true, globalIndexDir: indexDir },
|
|
194
|
+
selfStore,
|
|
143
195
|
);
|
|
144
196
|
assert.equal(r.toInject.length, 0, "globally-injected foreign checkpoint skipped");
|
|
145
197
|
} finally {
|
|
198
|
+
await closeVectorIndex();
|
|
146
199
|
closeIndexStore();
|
|
200
|
+
delete process.env.MEGACOMPACT_VECTOR_INDEX_DIR;
|
|
147
201
|
rmSync(indexDir, { recursive: true, force: true });
|
|
202
|
+
rmSync(foreignStateDir, { recursive: true, force: true });
|
|
203
|
+
rmSync(selfStateDir, { recursive: true, force: true });
|
|
148
204
|
}
|
|
149
205
|
});
|
|
150
206
|
|
|
151
207
|
test("S18: a fresh foreign checkpoint is injected AND recorded globally", async () => {
|
|
208
|
+
if (process.env.MEGACOMPACT_PGLITE_DISABLED === "true") { return; } // skip when WASM index is off
|
|
152
209
|
const indexDir = mkdtempSync(join(tmpdir(), "mc-gi2-"));
|
|
210
|
+
const foreignStateDir = mkdtempSync(join(tmpdir(), "mc-gi2-foreign-"));
|
|
211
|
+
const selfStateDir = mkdtempSync(join(tmpdir(), "mc-gi2-self-"));
|
|
212
|
+
process.env.MEGACOMPACT_VECTOR_INDEX_DIR = mkdtempSync(join(tmpdir(), "mc-gi2-vidx-"));
|
|
153
213
|
try {
|
|
214
|
+
await closeVectorIndex(); // fresh singleton per this index dir
|
|
215
|
+
const seed = await seedForeignRepo(foreignStateDir);
|
|
216
|
+
assert.equal(wasInjectedGlobal(seed.checkpointId, "sess_fresh", indexDir), false);
|
|
217
|
+
await rebuildFromSqlite(
|
|
218
|
+
() => [{ repoId: foreignStateDir, stateDir: foreignStateDir }],
|
|
219
|
+
(sd) => {
|
|
220
|
+
const store = new VectorStore({ stateDir: sd, dedupSim: 0.9 });
|
|
221
|
+
return vectorList(store, seed.sessionId).map((cp) => ({
|
|
222
|
+
sessionId: seed.sessionId,
|
|
223
|
+
checkpointId: cp.checkpointId,
|
|
224
|
+
embedding: cp.embedding,
|
|
225
|
+
}));
|
|
226
|
+
},
|
|
227
|
+
);
|
|
228
|
+
const pg = await initVectorIndex();
|
|
229
|
+
assert.ok(pg, "PGlite index should initialize (WASM available)");
|
|
230
|
+
|
|
154
231
|
const sess = "sess_fresh";
|
|
155
|
-
|
|
156
|
-
const mockStore = {
|
|
157
|
-
searchAsync: async () => [{
|
|
158
|
-
checkpoint: { checkpointId: "chkpt_new", summary: "brand new foreign work", filesModified: [], dedupStatus: "active" },
|
|
159
|
-
score: 0.93,
|
|
160
|
-
repoId: "/repo/alpha",
|
|
161
|
-
}],
|
|
162
|
-
wasInjected: () => false,
|
|
163
|
-
markInjected: () => {},
|
|
164
|
-
} as any;
|
|
232
|
+
const selfStore = new VectorStore({ stateDir: selfStateDir, dedupSim: 0.9 });
|
|
165
233
|
const r = await recallAndInlineAsync(
|
|
166
|
-
{ sessionId: sess, query: "foreign", limit: 3, source: "command", crossRepo: true, globalIndexDir: indexDir },
|
|
167
|
-
|
|
234
|
+
{ sessionId: sess, query: "foreign repo authentication jwt", limit: 3, source: "command", crossRepo: true, globalIndexDir: indexDir },
|
|
235
|
+
selfStore,
|
|
168
236
|
);
|
|
169
237
|
assert.equal(r.toInject.length, 1, "fresh foreign checkpoint injected");
|
|
170
|
-
assert.equal(wasInjectedGlobal(
|
|
238
|
+
assert.equal(wasInjectedGlobal(seed.checkpointId, sess, indexDir), true, "recorded machine-wide");
|
|
171
239
|
} finally {
|
|
240
|
+
await closeVectorIndex();
|
|
172
241
|
closeIndexStore();
|
|
242
|
+
delete process.env.MEGACOMPACT_VECTOR_INDEX_DIR;
|
|
173
243
|
rmSync(indexDir, { recursive: true, force: true });
|
|
244
|
+
rmSync(foreignStateDir, { recursive: true, force: true });
|
|
245
|
+
rmSync(selfStateDir, { recursive: true, force: true });
|
|
174
246
|
}
|
|
175
247
|
});
|
|
176
248
|
|
package/src/recall.ts
CHANGED
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
*/
|
|
16
16
|
|
|
17
17
|
import { recall as searchRecall } from "./engine.js";
|
|
18
|
-
import type
|
|
18
|
+
import { vectorWasInjected, vectorMarkInjected, type SearchHit, type VectorStore, vectorSearchAsync } from "./vectorStore.js";
|
|
19
19
|
import { estimateBlockTokens } from "./tokens.js";
|
|
20
20
|
import { defaultEmbedder, cosineSimilarity } from "./embedder.js";
|
|
21
21
|
|
|
@@ -107,7 +107,7 @@ export function formatRecallBlock(hits: SearchHit[]): string {
|
|
|
107
107
|
*/
|
|
108
108
|
export function recallAndInline(
|
|
109
109
|
opts: RecallInjectOptions,
|
|
110
|
-
store:
|
|
110
|
+
store: VectorStore,
|
|
111
111
|
): RecallInjectResult {
|
|
112
112
|
// ── S27 Recall Demotion ─────────────────────────────────────────────
|
|
113
113
|
//
|
|
@@ -134,7 +134,7 @@ export function recallAndInline(
|
|
|
134
134
|
|
|
135
135
|
const { hits } = searchRecall(
|
|
136
136
|
{ sessionId: opts.sessionId, query: opts.query, limit, skipInjected: false },
|
|
137
|
-
store
|
|
137
|
+
store,
|
|
138
138
|
);
|
|
139
139
|
|
|
140
140
|
// Precompute live-window embeddings once for inline dedupe (Fix C). Trigram
|
|
@@ -152,7 +152,7 @@ export function recallAndInline(
|
|
|
152
152
|
let blockTokens = 0;
|
|
153
153
|
|
|
154
154
|
for (const h of hits) {
|
|
155
|
-
if (skip && store
|
|
155
|
+
if (skip && vectorWasInjected(store, opts.sessionId, h.checkpoint.checkpointId)) continue;
|
|
156
156
|
|
|
157
157
|
// Inline dedupe: skip a hit already resident in the live window (Fix C).
|
|
158
158
|
if (doWindowDedupe && liveEmbeddings.length > 0) {
|
|
@@ -168,7 +168,7 @@ export function recallAndInline(
|
|
|
168
168
|
parts.push(part);
|
|
169
169
|
toInject.push(h);
|
|
170
170
|
blockTokens += partTokens;
|
|
171
|
-
store
|
|
171
|
+
vectorMarkInjected(store, opts.sessionId, h.checkpoint.checkpointId);
|
|
172
172
|
}
|
|
173
173
|
|
|
174
174
|
const block = parts.join("\n");
|
|
@@ -285,7 +285,7 @@ export async function recallMemoriesAndInline(
|
|
|
285
285
|
*/
|
|
286
286
|
export async function recallAndInlineAsync(
|
|
287
287
|
opts: RecallInjectOptions & { crossRepo?: boolean; repoId?: string },
|
|
288
|
-
store:
|
|
288
|
+
store: VectorStore,
|
|
289
289
|
): Promise<RecallInjectResult> {
|
|
290
290
|
const limit = opts.limit ?? 3;
|
|
291
291
|
const skip = opts.skipInjected ?? true;
|
|
@@ -295,7 +295,7 @@ export async function recallAndInlineAsync(
|
|
|
295
295
|
|
|
296
296
|
let hits: SearchHit[] = [];
|
|
297
297
|
try {
|
|
298
|
-
hits = await store
|
|
298
|
+
hits = await vectorSearchAsync(store, opts.sessionId, opts.query, limit, {
|
|
299
299
|
crossRepo: opts.crossRepo,
|
|
300
300
|
repoId: opts.repoId,
|
|
301
301
|
});
|
|
@@ -314,7 +314,7 @@ export async function recallAndInlineAsync(
|
|
|
314
314
|
let blockTokens = 0;
|
|
315
315
|
|
|
316
316
|
for (const h of hits) {
|
|
317
|
-
if (skip && store
|
|
317
|
+
if (skip && vectorWasInjected(store, opts.sessionId, h.checkpoint.checkpointId)) continue;
|
|
318
318
|
// S18: machine-wide injected-set — a foreign checkpoint already injected
|
|
319
319
|
// (in any session) is never re-injected. Only applies to cross-repo hits
|
|
320
320
|
// (same-repo hits have no repoId and are handled by the per-session set).
|
|
@@ -336,7 +336,7 @@ export async function recallAndInlineAsync(
|
|
|
336
336
|
parts.push(part);
|
|
337
337
|
toInject.push(h);
|
|
338
338
|
blockTokens += partTokens;
|
|
339
|
-
store
|
|
339
|
+
vectorMarkInjected(store, opts.sessionId, h.checkpoint.checkpointId);
|
|
340
340
|
// S18: record the cross-repo injection machine-wide so it's not re-injected
|
|
341
341
|
// by a later recall (same or different session).
|
|
342
342
|
if (opts.globalIndexDir && h.repoId) {
|