pi-mega-compact 0.20.88 → 0.21.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/extensions/dashboard-server/routes-rag-settings-helpers.js +7 -0
- package/dist/extensions/mega-events/context-handler/thrashGuard.js +60 -4
- package/dist/extensions/mega-events/context-handler/threeWayTelemetry.fixture.js +128 -0
- package/dist/extensions/mega-events/session-handlers.js +12 -0
- package/extensions/dashboard-server/routes-rag-settings-helpers.ts +17 -0
- package/extensions/mega-events/context-handler/thrashGuard.ts +63 -2
- package/extensions/mega-events/context-handler/threeWayTelemetry.fixture.ts +164 -0
- package/extensions/mega-events/session-handlers.ts +12 -0
- package/package.json +1 -1
|
@@ -134,6 +134,13 @@ export const SETTINGS = [
|
|
|
134
134
|
num("MEGACOMPACT_THRASH_REARM_PCT", "Thrash Re-arm %", "After an ineffective compaction (live window did not shrink), refuse to re-fire until the live window grows by this fraction of the effective threshold. Default 0.10 (10%)", 0.1, 0.01, 0.5),
|
|
135
135
|
],
|
|
136
136
|
},
|
|
137
|
+
{
|
|
138
|
+
name: "Three-Way Failback",
|
|
139
|
+
settings: [
|
|
140
|
+
boolDirect("MEGACOMPACT_THREE_WAY_FAILBACK", "Three-Way Failback", "Umbrella for the 3-way failback safety system: TriggerGuard (stages a recall block even when session_start never fires), the live-window ReductionValidator + persisted ThrashGuard (stops ineffective compaction re-fire loops), the 3-source read-only recall vote + same-repo relevance floor, and InjectionConfirm (asserts the staged block reached the message list pi sends). OFF = byte-identical pre-3WF behavior (v0.20.83). Runtime reads envBool(plain key), so this uses the plain-write convention (not _DISABLED).", true),
|
|
141
|
+
boolDirect("MEGACOMPACT_RECALL_TAIL_INJECT", "Recall Tail Inject", "Compose the staged recall block as a trailing user message on the context event (tail inject) instead of the legacy system-prompt prepend. Tail mode keeps the cache prefix stable and is the mode InjectionConfirm verifies against ContextEvent.messages; OFF falls back to the legacy prepend path (verified by string-contains). Runtime reads envBool(plain key), so this uses the plain-write convention (not _DISABLED).", true),
|
|
142
|
+
],
|
|
143
|
+
},
|
|
137
144
|
VECTOR_CORTEX_SETTINGS,
|
|
138
145
|
{
|
|
139
146
|
name: "Cost API",
|
|
@@ -35,6 +35,30 @@ export const ReductionValidator = {
|
|
|
35
35
|
return { effective, liveBefore, liveAfter };
|
|
36
36
|
},
|
|
37
37
|
};
|
|
38
|
+
/**
|
|
39
|
+
* Disarm the ThrashGuard — clears the persisted `blocked_until` so subsequent
|
|
40
|
+
* compactions are not refused. Called when a compaction is judged EFFECTIVE
|
|
41
|
+
* (the thrash condition is over) and on session start (so a stale guard from a
|
|
42
|
+
* prior session cannot suppress the next session's compactions).
|
|
43
|
+
*
|
|
44
|
+
* Best-effort: any failure is swallowed (never blocks on a store fault).
|
|
45
|
+
*/
|
|
46
|
+
export function disarmThrashGuard(stateDir, logger, emit) {
|
|
47
|
+
try {
|
|
48
|
+
setMetaNumber(THRASH_BLOCKED_KEY, 0, stateDir);
|
|
49
|
+
setMetaNumber(THRASH_BASELINE_KEY, 0, stateDir);
|
|
50
|
+
logger?.info("thrasguard_disarmed", { reason: "effective_or_reset" });
|
|
51
|
+
try {
|
|
52
|
+
emit?.("thrasguard_disarmed", { reason: "effective_or_reset" });
|
|
53
|
+
}
|
|
54
|
+
catch {
|
|
55
|
+
/* non-fatal */
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
catch {
|
|
59
|
+
/* non-fatal: best-effort meta write */
|
|
60
|
+
}
|
|
61
|
+
}
|
|
38
62
|
/**
|
|
39
63
|
* Arm the ThrashGuard after an ineffective compaction. Persists:
|
|
40
64
|
* - `thrasguard.baseline_tokens` = the live currentTokens at this (post-fire)
|
|
@@ -48,8 +72,17 @@ export const ReductionValidator = {
|
|
|
48
72
|
* Infinity/NaN into meta (getMetaNumber would read it back as 0). Skip arming
|
|
49
73
|
* + log instead; the next over-threshold event simply re-fires (pre-sprint
|
|
50
74
|
* behavior) rather than corrupting the guard.
|
|
75
|
+
*
|
|
76
|
+
* 3WF-5 telemetry: `logger` is the debug-gated runtime logger (mega-compact.log,
|
|
77
|
+
* silent unless config.debug) — which means the breadcrumb was invisible to the
|
|
78
|
+
* dashboard Events tab, whose SSE tail reads the repo's events.log. `emit` is
|
|
79
|
+
* the always-on events.log sink (MegaRuntime.appendEvent), so the armed
|
|
80
|
+
* breadcrumb lands in the same stream as the other 3WF events
|
|
81
|
+
* (three_way_guard_fired / three_way_floor_used / injection_confirmed /
|
|
82
|
+
* injection_recovered). Both sinks are optional + best-effort; passing neither
|
|
83
|
+
* keeps the pre-3WF-5 behavior.
|
|
51
84
|
*/
|
|
52
|
-
export function armThrashGuard(currentTokens, rearmPct, effectiveThreshold, stateDir, logger) {
|
|
85
|
+
export function armThrashGuard(currentTokens, rearmPct, effectiveThreshold, stateDir, logger, emit) {
|
|
53
86
|
if (!Number.isFinite(currentTokens) || currentTokens <= 0)
|
|
54
87
|
return;
|
|
55
88
|
if (!Number.isFinite(rearmPct) || rearmPct <= 0)
|
|
@@ -65,11 +98,18 @@ export function armThrashGuard(currentTokens, rearmPct, effectiveThreshold, stat
|
|
|
65
98
|
const n = Math.round(rearmPct * effectiveThreshold);
|
|
66
99
|
setMetaNumber(THRASH_BASELINE_KEY, Math.round(currentTokens), stateDir);
|
|
67
100
|
setMetaNumber(THRASH_BLOCKED_KEY, Math.round(currentTokens + n), stateDir);
|
|
68
|
-
|
|
101
|
+
const fields = {
|
|
69
102
|
baselineTokens: Math.round(currentTokens),
|
|
70
103
|
blockedUntilTokens: Math.round(currentTokens + n),
|
|
71
104
|
rearmTokens: n,
|
|
72
|
-
}
|
|
105
|
+
};
|
|
106
|
+
logger?.info("thrasguard_armed", fields);
|
|
107
|
+
try {
|
|
108
|
+
emit?.("thrasguard_armed", fields);
|
|
109
|
+
}
|
|
110
|
+
catch {
|
|
111
|
+
/* non-fatal: events.log sink must never break arming */
|
|
112
|
+
}
|
|
73
113
|
}
|
|
74
114
|
catch {
|
|
75
115
|
/* non-fatal: best-effort meta write */
|
|
@@ -177,7 +217,23 @@ export function evaluatePendingReduction(runtime, currentTokens, config) {
|
|
|
177
217
|
// Remember which event armed us so the consult later in THIS SAME event
|
|
178
218
|
// does not swallow it (see armedOnEvent).
|
|
179
219
|
armedOnEvent.set(runtime, currentTokens);
|
|
180
|
-
|
|
220
|
+
// 3WF-5: also emit the breadcrumb on the always-on events.log sink so
|
|
221
|
+
// the dashboard Events tab sees it (runtime.logger is debug-gated).
|
|
222
|
+
// Optional-chained: a thin runtime stub without appendEvent stays valid.
|
|
223
|
+
const emit = typeof runtime.appendEvent === "function"
|
|
224
|
+
? runtime.appendEvent.bind(runtime)
|
|
225
|
+
: undefined;
|
|
226
|
+
armThrashGuard(currentTokens, config.thrashRearmPct, runtime.effectiveThreshold, runtime.currentStateDir, runtime.logger, emit);
|
|
227
|
+
}
|
|
228
|
+
else {
|
|
229
|
+
// Effective compaction — the thrash condition is over. Disarm the guard
|
|
230
|
+
// so it does not block legitimate subsequent compactions (the bug: without
|
|
231
|
+
// this, `blocked_until` from the prior ineffective fire persists and blocks
|
|
232
|
+
// the now-smaller window from ever re-firing).
|
|
233
|
+
const emit = typeof runtime.appendEvent === "function"
|
|
234
|
+
? runtime.appendEvent.bind(runtime)
|
|
235
|
+
: undefined;
|
|
236
|
+
disarmThrashGuard(runtime.currentStateDir, runtime.logger, emit);
|
|
181
237
|
}
|
|
182
238
|
}
|
|
183
239
|
catch {
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* context-handler/threeWayTelemetry.fixture.ts — shared fixtures for the 3WF-5
|
|
3
|
+
* telemetry verification tests.
|
|
4
|
+
*
|
|
5
|
+
* Split out so the test file stays under the soft cap, mirroring
|
|
6
|
+
* src/recall/recall3wf.fixture.ts (3WF-3) and injectionConfirm.fixture.ts (3WF-4).
|
|
7
|
+
*
|
|
8
|
+
* These are REAL fixtures, not mocks/stubs. The critical difference from the
|
|
9
|
+
* 3WF-1..4 fixtures: those record `appendEvent` calls into an in-memory array,
|
|
10
|
+
* which proves the CALL happened but says nothing about the wire format or the
|
|
11
|
+
* file the dashboard actually tails. Here `appendEvent` is wired to the REAL
|
|
12
|
+
* `appendEventImpl` (extensions/mega-runtime/append-event.ts) against a temp
|
|
13
|
+
* stateDir, so each breadcrumb is serialized to a real events.log exactly as it
|
|
14
|
+
* is in production. The tests then read that file back and parse the JSON — the
|
|
15
|
+
* `ts` + `event` shape is observed, never simulated.
|
|
16
|
+
*/
|
|
17
|
+
import { mkdtempSync, readFileSync, existsSync } from "node:fs";
|
|
18
|
+
import { tmpdir } from "node:os";
|
|
19
|
+
import { join } from "node:path";
|
|
20
|
+
import { VectorStore } from "../../../src/vectorStore.js";
|
|
21
|
+
import { compactSession } from "../../../src/engine.js";
|
|
22
|
+
import { appendEventImpl } from "../../mega-runtime/append-event.js";
|
|
23
|
+
/** Real EngineMessage fixture. */
|
|
24
|
+
export function msg(role, text) {
|
|
25
|
+
return { role, text };
|
|
26
|
+
}
|
|
27
|
+
/** A user-role AgentMessage carrying `text` (the tail-block shape). */
|
|
28
|
+
export function userMsg(text) {
|
|
29
|
+
return { role: "user", content: text, timestamp: 1 };
|
|
30
|
+
}
|
|
31
|
+
/** Fresh isolated state dir per VectorStore. */
|
|
32
|
+
export function freshStore() {
|
|
33
|
+
const dir = mkdtempSync(join(tmpdir(), "mc-3wf5-"));
|
|
34
|
+
return { store: new VectorStore({ dedupSim: 0.9, stateDir: dir }), dir };
|
|
35
|
+
}
|
|
36
|
+
/** Persist N distinct checkpoints with ascending timestamps. */
|
|
37
|
+
export function seed(store, topics, sid) {
|
|
38
|
+
topics.forEach((t, i) => {
|
|
39
|
+
compactSession({
|
|
40
|
+
sessionId: sid,
|
|
41
|
+
messages: [msg("user", t), msg("assistant", "ok")],
|
|
42
|
+
keepFrom: 2,
|
|
43
|
+
timestamp: i + 1,
|
|
44
|
+
}, store);
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Read + JSON-parse every line of the REAL events.log written under `stateDir`.
|
|
49
|
+
* PREVENT-001: parse failures are surfaced as null and filtered, never thrown.
|
|
50
|
+
*/
|
|
51
|
+
export function readEventsLog(stateDir) {
|
|
52
|
+
const path = join(stateDir, "events.log");
|
|
53
|
+
if (!existsSync(path))
|
|
54
|
+
return [];
|
|
55
|
+
return readFileSync(path, "utf-8")
|
|
56
|
+
.split("\n")
|
|
57
|
+
.filter((l) => l.trim().length > 0)
|
|
58
|
+
.map((line) => {
|
|
59
|
+
try {
|
|
60
|
+
const parsed = JSON.parse(line);
|
|
61
|
+
if (parsed == null || typeof parsed !== "object")
|
|
62
|
+
return null;
|
|
63
|
+
return parsed;
|
|
64
|
+
}
|
|
65
|
+
catch {
|
|
66
|
+
return null;
|
|
67
|
+
}
|
|
68
|
+
})
|
|
69
|
+
.filter((e) => e != null);
|
|
70
|
+
}
|
|
71
|
+
/** Silent logger (the debug-gated sink is not under test here). */
|
|
72
|
+
export const noopLogger = {
|
|
73
|
+
info: () => { },
|
|
74
|
+
warn: () => { },
|
|
75
|
+
error: () => { },
|
|
76
|
+
};
|
|
77
|
+
/**
|
|
78
|
+
* MegaRuntime whose `appendEvent` is the REAL events.log writer bound to
|
|
79
|
+
* `stateDir`. Only the fields the 3WF guards touch are populated.
|
|
80
|
+
*/
|
|
81
|
+
export function realEventRuntime(store, stateDir, over = {}) {
|
|
82
|
+
const self = {
|
|
83
|
+
store,
|
|
84
|
+
currentStateDir: stateDir,
|
|
85
|
+
pendingRecallBlock: over.pendingRecallBlock,
|
|
86
|
+
pendingMemoryRecallBlock: over.pendingMemoryRecallBlock,
|
|
87
|
+
effectiveThreshold: over.effectiveThreshold ?? 100_000,
|
|
88
|
+
lastCtxWindow: 0,
|
|
89
|
+
logger: noopLogger,
|
|
90
|
+
perfTurnStart: undefined,
|
|
91
|
+
rt: { recallInjectedThisTurn: false, lastCheckpointId: null },
|
|
92
|
+
appendEvent(event, fields) {
|
|
93
|
+
appendEventImpl({ currentStateDir: stateDir }, event, fields);
|
|
94
|
+
},
|
|
95
|
+
};
|
|
96
|
+
return self;
|
|
97
|
+
}
|
|
98
|
+
/** ctx stub with a configurable session id + latest-user query. */
|
|
99
|
+
export function ctxStub(opts = {}) {
|
|
100
|
+
const sessionId = opts.sessionId ?? "sess_3wf5";
|
|
101
|
+
const query = opts.query ?? "dedupe race in store";
|
|
102
|
+
return {
|
|
103
|
+
cwd: "/tmp",
|
|
104
|
+
sessionManager: {
|
|
105
|
+
getSessionId: () => sessionId,
|
|
106
|
+
getEntries: () => [
|
|
107
|
+
{
|
|
108
|
+
type: "message",
|
|
109
|
+
id: "e1",
|
|
110
|
+
parentId: null,
|
|
111
|
+
timestamp: "1",
|
|
112
|
+
message: { role: "user", content: query },
|
|
113
|
+
},
|
|
114
|
+
],
|
|
115
|
+
},
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
/** Config stub: only the flags the 3WF guards read. */
|
|
119
|
+
export function configStub(over = {}) {
|
|
120
|
+
return {
|
|
121
|
+
threeWayFailback: over.threeWayFailback ?? true,
|
|
122
|
+
recallTailInject: over.recallTailInject ?? true,
|
|
123
|
+
thrashRearmPct: over.thrashRearmPct ?? 0.1,
|
|
124
|
+
autoInlineK: 3,
|
|
125
|
+
tier: "custom",
|
|
126
|
+
tierPct: null,
|
|
127
|
+
};
|
|
128
|
+
}
|
|
@@ -5,6 +5,7 @@ import { doRecall, doRecallAsync } from "../mega-pipeline.js";
|
|
|
5
5
|
import { recallMemoriesAndInline } from "../../src/recall.js";
|
|
6
6
|
import { vectorStats } from "../../src/vectorStore.js";
|
|
7
7
|
import { openTurnStore } from "../../src/store/turns/connection.js";
|
|
8
|
+
import { disarmThrashGuard } from "./context-handler/thrashGuard.js";
|
|
8
9
|
import { openIntentQueue } from "../../src/intent.js";
|
|
9
10
|
/** Register session lifecycle event handlers. */
|
|
10
11
|
export function registerSessionHandlers(pi, runtime, config) {
|
|
@@ -15,6 +16,17 @@ export function registerSessionHandlers(pi, runtime, config) {
|
|
|
15
16
|
});
|
|
16
17
|
pi.on("session_start", async (event, ctx) => {
|
|
17
18
|
runtime.resetRuntime(ctx.sessionManager.getSessionId());
|
|
19
|
+
// Disarm the ThrashGuard on session start so a stale `blocked_until` from a
|
|
20
|
+
// prior session cannot suppress this session's compactions. Bind FIRST:
|
|
21
|
+
// currentStateDir defaults to the global dir until bindRepo resolves the
|
|
22
|
+
// per-repo <repo>/.pi/mega-compact store, and the guard arms in the
|
|
23
|
+
// PER-REPO store — disarming before the bind would clear the wrong (global)
|
|
24
|
+
// dir after a process restart. Gated on the umbrella so flag-OFF stays
|
|
25
|
+
// byte-identical (no meta writes).
|
|
26
|
+
if (config.threeWayFailback) {
|
|
27
|
+
runtime.bindRepo(ctx.cwd);
|
|
28
|
+
disarmThrashGuard(runtime.currentStateDir);
|
|
29
|
+
}
|
|
18
30
|
runtime.captureModel(ctx); // best-effort: ctx.model may be set by session start
|
|
19
31
|
runtime.setStatus(ctx, config.auto ? "mega-compact: ready" : "mega-compact: manual only");
|
|
20
32
|
// S21: clear any stale memory block from a prior session.
|
|
@@ -296,6 +296,23 @@ export const SETTINGS: ReadonlyArray<SettingGroup> = [
|
|
|
296
296
|
),
|
|
297
297
|
],
|
|
298
298
|
},
|
|
299
|
+
{
|
|
300
|
+
name: "Three-Way Failback",
|
|
301
|
+
settings: [
|
|
302
|
+
boolDirect(
|
|
303
|
+
"MEGACOMPACT_THREE_WAY_FAILBACK",
|
|
304
|
+
"Three-Way Failback",
|
|
305
|
+
"Umbrella for the 3-way failback safety system: TriggerGuard (stages a recall block even when session_start never fires), the live-window ReductionValidator + persisted ThrashGuard (stops ineffective compaction re-fire loops), the 3-source read-only recall vote + same-repo relevance floor, and InjectionConfirm (asserts the staged block reached the message list pi sends). OFF = byte-identical pre-3WF behavior (v0.20.83). Runtime reads envBool(plain key), so this uses the plain-write convention (not _DISABLED).",
|
|
306
|
+
true,
|
|
307
|
+
),
|
|
308
|
+
boolDirect(
|
|
309
|
+
"MEGACOMPACT_RECALL_TAIL_INJECT",
|
|
310
|
+
"Recall Tail Inject",
|
|
311
|
+
"Compose the staged recall block as a trailing user message on the context event (tail inject) instead of the legacy system-prompt prepend. Tail mode keeps the cache prefix stable and is the mode InjectionConfirm verifies against ContextEvent.messages; OFF falls back to the legacy prepend path (verified by string-contains). Runtime reads envBool(plain key), so this uses the plain-write convention (not _DISABLED).",
|
|
312
|
+
true,
|
|
313
|
+
),
|
|
314
|
+
],
|
|
315
|
+
},
|
|
299
316
|
VECTOR_CORTEX_SETTINGS,
|
|
300
317
|
{
|
|
301
318
|
name: "Cost API",
|
|
@@ -62,6 +62,33 @@ export const ReductionValidator = {
|
|
|
62
62
|
},
|
|
63
63
|
};
|
|
64
64
|
|
|
65
|
+
/**
|
|
66
|
+
* Disarm the ThrashGuard — clears the persisted `blocked_until` so subsequent
|
|
67
|
+
* compactions are not refused. Called when a compaction is judged EFFECTIVE
|
|
68
|
+
* (the thrash condition is over) and on session start (so a stale guard from a
|
|
69
|
+
* prior session cannot suppress the next session's compactions).
|
|
70
|
+
*
|
|
71
|
+
* Best-effort: any failure is swallowed (never blocks on a store fault).
|
|
72
|
+
*/
|
|
73
|
+
export function disarmThrashGuard(
|
|
74
|
+
stateDir: string,
|
|
75
|
+
logger?: { info(event: string, fields?: Record<string, unknown>): void },
|
|
76
|
+
emit?: (event: string, fields: Record<string, unknown>) => void,
|
|
77
|
+
): void {
|
|
78
|
+
try {
|
|
79
|
+
setMetaNumber(THRASH_BLOCKED_KEY, 0, stateDir);
|
|
80
|
+
setMetaNumber(THRASH_BASELINE_KEY, 0, stateDir);
|
|
81
|
+
logger?.info("thrasguard_disarmed", { reason: "effective_or_reset" });
|
|
82
|
+
try {
|
|
83
|
+
emit?.("thrasguard_disarmed", { reason: "effective_or_reset" });
|
|
84
|
+
} catch {
|
|
85
|
+
/* non-fatal */
|
|
86
|
+
}
|
|
87
|
+
} catch {
|
|
88
|
+
/* non-fatal: best-effort meta write */
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
65
92
|
/**
|
|
66
93
|
* Arm the ThrashGuard after an ineffective compaction. Persists:
|
|
67
94
|
* - `thrasguard.baseline_tokens` = the live currentTokens at this (post-fire)
|
|
@@ -75,6 +102,15 @@ export const ReductionValidator = {
|
|
|
75
102
|
* Infinity/NaN into meta (getMetaNumber would read it back as 0). Skip arming
|
|
76
103
|
* + log instead; the next over-threshold event simply re-fires (pre-sprint
|
|
77
104
|
* behavior) rather than corrupting the guard.
|
|
105
|
+
*
|
|
106
|
+
* 3WF-5 telemetry: `logger` is the debug-gated runtime logger (mega-compact.log,
|
|
107
|
+
* silent unless config.debug) — which means the breadcrumb was invisible to the
|
|
108
|
+
* dashboard Events tab, whose SSE tail reads the repo's events.log. `emit` is
|
|
109
|
+
* the always-on events.log sink (MegaRuntime.appendEvent), so the armed
|
|
110
|
+
* breadcrumb lands in the same stream as the other 3WF events
|
|
111
|
+
* (three_way_guard_fired / three_way_floor_used / injection_confirmed /
|
|
112
|
+
* injection_recovered). Both sinks are optional + best-effort; passing neither
|
|
113
|
+
* keeps the pre-3WF-5 behavior.
|
|
78
114
|
*/
|
|
79
115
|
export function armThrashGuard(
|
|
80
116
|
currentTokens: number,
|
|
@@ -82,6 +118,7 @@ export function armThrashGuard(
|
|
|
82
118
|
effectiveThreshold: number,
|
|
83
119
|
stateDir: string,
|
|
84
120
|
logger?: { info(event: string, fields?: Record<string, unknown>): void },
|
|
121
|
+
emit?: (event: string, fields: Record<string, unknown>) => void,
|
|
85
122
|
): void {
|
|
86
123
|
if (!Number.isFinite(currentTokens) || currentTokens <= 0) return;
|
|
87
124
|
if (!Number.isFinite(rearmPct) || rearmPct <= 0) return;
|
|
@@ -96,11 +133,17 @@ export function armThrashGuard(
|
|
|
96
133
|
const n = Math.round(rearmPct * effectiveThreshold);
|
|
97
134
|
setMetaNumber(THRASH_BASELINE_KEY, Math.round(currentTokens), stateDir);
|
|
98
135
|
setMetaNumber(THRASH_BLOCKED_KEY, Math.round(currentTokens + n), stateDir);
|
|
99
|
-
|
|
136
|
+
const fields = {
|
|
100
137
|
baselineTokens: Math.round(currentTokens),
|
|
101
138
|
blockedUntilTokens: Math.round(currentTokens + n),
|
|
102
139
|
rearmTokens: n,
|
|
103
|
-
}
|
|
140
|
+
};
|
|
141
|
+
logger?.info("thrasguard_armed", fields);
|
|
142
|
+
try {
|
|
143
|
+
emit?.("thrasguard_armed", fields);
|
|
144
|
+
} catch {
|
|
145
|
+
/* non-fatal: events.log sink must never break arming */
|
|
146
|
+
}
|
|
104
147
|
} catch {
|
|
105
148
|
/* non-fatal: best-effort meta write */
|
|
106
149
|
}
|
|
@@ -214,13 +257,31 @@ export function evaluatePendingReduction(
|
|
|
214
257
|
// Remember which event armed us so the consult later in THIS SAME event
|
|
215
258
|
// does not swallow it (see armedOnEvent).
|
|
216
259
|
armedOnEvent.set(runtime, currentTokens);
|
|
260
|
+
// 3WF-5: also emit the breadcrumb on the always-on events.log sink so
|
|
261
|
+
// the dashboard Events tab sees it (runtime.logger is debug-gated).
|
|
262
|
+
// Optional-chained: a thin runtime stub without appendEvent stays valid.
|
|
263
|
+
const emit =
|
|
264
|
+
typeof runtime.appendEvent === "function"
|
|
265
|
+
? runtime.appendEvent.bind(runtime)
|
|
266
|
+
: undefined;
|
|
217
267
|
armThrashGuard(
|
|
218
268
|
currentTokens,
|
|
219
269
|
config.thrashRearmPct,
|
|
220
270
|
runtime.effectiveThreshold,
|
|
221
271
|
runtime.currentStateDir,
|
|
222
272
|
runtime.logger,
|
|
273
|
+
emit,
|
|
223
274
|
);
|
|
275
|
+
} else {
|
|
276
|
+
// Effective compaction — the thrash condition is over. Disarm the guard
|
|
277
|
+
// so it does not block legitimate subsequent compactions (the bug: without
|
|
278
|
+
// this, `blocked_until` from the prior ineffective fire persists and blocks
|
|
279
|
+
// the now-smaller window from ever re-firing).
|
|
280
|
+
const emit =
|
|
281
|
+
typeof runtime.appendEvent === "function"
|
|
282
|
+
? runtime.appendEvent.bind(runtime)
|
|
283
|
+
: undefined;
|
|
284
|
+
disarmThrashGuard(runtime.currentStateDir, runtime.logger, emit);
|
|
224
285
|
}
|
|
225
286
|
} catch {
|
|
226
287
|
/* non-fatal */
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* context-handler/threeWayTelemetry.fixture.ts — shared fixtures for the 3WF-5
|
|
3
|
+
* telemetry verification tests.
|
|
4
|
+
*
|
|
5
|
+
* Split out so the test file stays under the soft cap, mirroring
|
|
6
|
+
* src/recall/recall3wf.fixture.ts (3WF-3) and injectionConfirm.fixture.ts (3WF-4).
|
|
7
|
+
*
|
|
8
|
+
* These are REAL fixtures, not mocks/stubs. The critical difference from the
|
|
9
|
+
* 3WF-1..4 fixtures: those record `appendEvent` calls into an in-memory array,
|
|
10
|
+
* which proves the CALL happened but says nothing about the wire format or the
|
|
11
|
+
* file the dashboard actually tails. Here `appendEvent` is wired to the REAL
|
|
12
|
+
* `appendEventImpl` (extensions/mega-runtime/append-event.ts) against a temp
|
|
13
|
+
* stateDir, so each breadcrumb is serialized to a real events.log exactly as it
|
|
14
|
+
* is in production. The tests then read that file back and parse the JSON — the
|
|
15
|
+
* `ts` + `event` shape is observed, never simulated.
|
|
16
|
+
*/
|
|
17
|
+
import { mkdtempSync, readFileSync, existsSync } from "node:fs";
|
|
18
|
+
import { tmpdir } from "node:os";
|
|
19
|
+
import { join } from "node:path";
|
|
20
|
+
|
|
21
|
+
import type { AgentMessage } from "@earendil-works/pi-agent-core";
|
|
22
|
+
import { VectorStore } from "../../../src/vectorStore.js";
|
|
23
|
+
import { compactSession } from "../../../src/engine.js";
|
|
24
|
+
import { appendEventImpl } from "../../mega-runtime/append-event.js";
|
|
25
|
+
import type { MegaRuntime } from "../../mega-runtime.js";
|
|
26
|
+
import type { MegaConfig } from "../../mega-config.js";
|
|
27
|
+
import type { Logger } from "../../../src/log.js";
|
|
28
|
+
|
|
29
|
+
/** Real EngineMessage fixture. */
|
|
30
|
+
export function msg(role: "user" | "assistant", text: string): any {
|
|
31
|
+
return { role, text };
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** A user-role AgentMessage carrying `text` (the tail-block shape). */
|
|
35
|
+
export function userMsg(text: string): AgentMessage {
|
|
36
|
+
return { role: "user", content: text, timestamp: 1 } as unknown as AgentMessage;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** One parsed events.log line. `ts` + `event` are the contract the dashboard
|
|
40
|
+
* Events tab (server.ts SSE tail of stateDir/events.log) consumes. */
|
|
41
|
+
export interface LoggedEvent {
|
|
42
|
+
ts: unknown;
|
|
43
|
+
event: unknown;
|
|
44
|
+
[k: string]: unknown;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** Fresh isolated state dir per VectorStore. */
|
|
48
|
+
export function freshStore(): { store: VectorStore; dir: string } {
|
|
49
|
+
const dir = mkdtempSync(join(tmpdir(), "mc-3wf5-"));
|
|
50
|
+
return { store: new VectorStore({ dedupSim: 0.9, stateDir: dir }), dir };
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** Persist N distinct checkpoints with ascending timestamps. */
|
|
54
|
+
export function seed(store: VectorStore, topics: string[], sid: string): void {
|
|
55
|
+
topics.forEach((t, i) => {
|
|
56
|
+
compactSession(
|
|
57
|
+
{
|
|
58
|
+
sessionId: sid,
|
|
59
|
+
messages: [msg("user", t), msg("assistant", "ok")],
|
|
60
|
+
keepFrom: 2,
|
|
61
|
+
timestamp: i + 1,
|
|
62
|
+
},
|
|
63
|
+
store,
|
|
64
|
+
);
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Read + JSON-parse every line of the REAL events.log written under `stateDir`.
|
|
70
|
+
* PREVENT-001: parse failures are surfaced as null and filtered, never thrown.
|
|
71
|
+
*/
|
|
72
|
+
export function readEventsLog(stateDir: string): LoggedEvent[] {
|
|
73
|
+
const path = join(stateDir, "events.log");
|
|
74
|
+
if (!existsSync(path)) return [];
|
|
75
|
+
return readFileSync(path, "utf-8")
|
|
76
|
+
.split("\n")
|
|
77
|
+
.filter((l) => l.trim().length > 0)
|
|
78
|
+
.map((line): LoggedEvent | null => {
|
|
79
|
+
try {
|
|
80
|
+
const parsed: unknown = JSON.parse(line);
|
|
81
|
+
if (parsed == null || typeof parsed !== "object") return null;
|
|
82
|
+
return parsed as LoggedEvent;
|
|
83
|
+
} catch {
|
|
84
|
+
return null;
|
|
85
|
+
}
|
|
86
|
+
})
|
|
87
|
+
.filter((e): e is LoggedEvent => e != null);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** Silent logger (the debug-gated sink is not under test here). */
|
|
91
|
+
export const noopLogger: Logger = {
|
|
92
|
+
info: () => {},
|
|
93
|
+
warn: () => {},
|
|
94
|
+
error: () => {},
|
|
95
|
+
} as unknown as Logger;
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* MegaRuntime whose `appendEvent` is the REAL events.log writer bound to
|
|
99
|
+
* `stateDir`. Only the fields the 3WF guards touch are populated.
|
|
100
|
+
*/
|
|
101
|
+
export function realEventRuntime(
|
|
102
|
+
store: VectorStore,
|
|
103
|
+
stateDir: string,
|
|
104
|
+
over: Partial<{
|
|
105
|
+
pendingRecallBlock: string | undefined;
|
|
106
|
+
pendingMemoryRecallBlock: string | undefined;
|
|
107
|
+
effectiveThreshold: number;
|
|
108
|
+
}> = {},
|
|
109
|
+
): MegaRuntime {
|
|
110
|
+
const self = {
|
|
111
|
+
store,
|
|
112
|
+
currentStateDir: stateDir,
|
|
113
|
+
pendingRecallBlock: over.pendingRecallBlock,
|
|
114
|
+
pendingMemoryRecallBlock: over.pendingMemoryRecallBlock,
|
|
115
|
+
effectiveThreshold: over.effectiveThreshold ?? 100_000,
|
|
116
|
+
lastCtxWindow: 0,
|
|
117
|
+
logger: noopLogger,
|
|
118
|
+
perfTurnStart: undefined,
|
|
119
|
+
rt: { recallInjectedThisTurn: false, lastCheckpointId: null as string | null },
|
|
120
|
+
appendEvent(event: string, fields: Record<string, unknown>): void {
|
|
121
|
+
appendEventImpl({ currentStateDir: stateDir }, event, fields);
|
|
122
|
+
},
|
|
123
|
+
};
|
|
124
|
+
return self as unknown as MegaRuntime;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** ctx stub with a configurable session id + latest-user query. */
|
|
128
|
+
export function ctxStub(opts: { sessionId?: string; query?: string } = {}): any {
|
|
129
|
+
const sessionId = opts.sessionId ?? "sess_3wf5";
|
|
130
|
+
const query = opts.query ?? "dedupe race in store";
|
|
131
|
+
return {
|
|
132
|
+
cwd: "/tmp",
|
|
133
|
+
sessionManager: {
|
|
134
|
+
getSessionId: () => sessionId,
|
|
135
|
+
getEntries: () => [
|
|
136
|
+
{
|
|
137
|
+
type: "message",
|
|
138
|
+
id: "e1",
|
|
139
|
+
parentId: null,
|
|
140
|
+
timestamp: "1",
|
|
141
|
+
message: { role: "user", content: query },
|
|
142
|
+
},
|
|
143
|
+
],
|
|
144
|
+
},
|
|
145
|
+
};
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/** Config stub: only the flags the 3WF guards read. */
|
|
149
|
+
export function configStub(
|
|
150
|
+
over: Partial<{
|
|
151
|
+
threeWayFailback: boolean;
|
|
152
|
+
recallTailInject: boolean;
|
|
153
|
+
thrashRearmPct: number;
|
|
154
|
+
}> = {},
|
|
155
|
+
): MegaConfig {
|
|
156
|
+
return {
|
|
157
|
+
threeWayFailback: over.threeWayFailback ?? true,
|
|
158
|
+
recallTailInject: over.recallTailInject ?? true,
|
|
159
|
+
thrashRearmPct: over.thrashRearmPct ?? 0.1,
|
|
160
|
+
autoInlineK: 3,
|
|
161
|
+
tier: "custom",
|
|
162
|
+
tierPct: null,
|
|
163
|
+
} as unknown as MegaConfig;
|
|
164
|
+
}
|
|
@@ -16,6 +16,7 @@ import { doRecall, doRecallAsync } from "../mega-pipeline.js";
|
|
|
16
16
|
import { recallMemoriesAndInline } from "../../src/recall.js";
|
|
17
17
|
import { vectorStats } from "../../src/vectorStore.js";
|
|
18
18
|
import { openTurnStore } from "../../src/store/turns/connection.js";
|
|
19
|
+
import { disarmThrashGuard } from "./context-handler/thrashGuard.js";
|
|
19
20
|
import { openIntentQueue } from "../../src/intent.js";
|
|
20
21
|
import type { MegaConfig } from "../mega-config.js";
|
|
21
22
|
|
|
@@ -33,6 +34,17 @@ export function registerSessionHandlers(
|
|
|
33
34
|
|
|
34
35
|
pi.on("session_start", async (event, ctx) => {
|
|
35
36
|
runtime.resetRuntime(ctx.sessionManager.getSessionId());
|
|
37
|
+
// Disarm the ThrashGuard on session start so a stale `blocked_until` from a
|
|
38
|
+
// prior session cannot suppress this session's compactions. Bind FIRST:
|
|
39
|
+
// currentStateDir defaults to the global dir until bindRepo resolves the
|
|
40
|
+
// per-repo <repo>/.pi/mega-compact store, and the guard arms in the
|
|
41
|
+
// PER-REPO store — disarming before the bind would clear the wrong (global)
|
|
42
|
+
// dir after a process restart. Gated on the umbrella so flag-OFF stays
|
|
43
|
+
// byte-identical (no meta writes).
|
|
44
|
+
if (config.threeWayFailback) {
|
|
45
|
+
runtime.bindRepo(ctx.cwd);
|
|
46
|
+
disarmThrashGuard(runtime.currentStateDir);
|
|
47
|
+
}
|
|
36
48
|
runtime.captureModel(ctx); // best-effort: ctx.model may be set by session start
|
|
37
49
|
runtime.setStatus(
|
|
38
50
|
ctx,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-mega-compact",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.21.1",
|
|
4
4
|
"description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "BSD-3-Clause",
|