pi-mega-compact 0.20.86 → 0.20.88
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config.js +9 -0
- package/dist/extensions/dashboard-server/routes-rag-settings-helpers.js +1 -0
- package/dist/extensions/mega-config.js +6 -0
- package/dist/extensions/mega-events/context-handler/injectionConfirm.fixture.js +63 -0
- package/dist/extensions/mega-events/context-handler/injectionConfirm.js +107 -0
- package/dist/extensions/mega-events/context-handler/triggerGuard.js +11 -17
- package/dist/extensions/mega-events/context-handler.js +17 -1
- package/dist/extensions/mega-pipeline/recall/impl.js +258 -0
- package/dist/extensions/mega-pipeline/recall.js +6 -253
- package/dist/src/config.js +9 -0
- package/dist/src/failback/floor.js +35 -0
- package/dist/src/recall/readonly.js +39 -0
- package/dist/src/recall/recall3wf.fixture.js +67 -0
- package/dist/src/recall/validator.js +99 -0
- package/dist/src/recall/vote.js +217 -0
- package/dist/src/store/sqlite/fts5-search.js +26 -0
- package/extensions/dashboard-server/routes-rag-settings-helpers.ts +1 -0
- package/extensions/mega-config-types.ts +5 -0
- package/extensions/mega-config.ts +6 -0
- package/extensions/mega-events/context-handler/injectionConfirm.fixture.ts +90 -0
- package/extensions/mega-events/context-handler/injectionConfirm.ts +168 -0
- package/extensions/mega-events/context-handler/triggerGuard.ts +14 -22
- package/extensions/mega-events/context-handler.ts +16 -1
- package/extensions/mega-pipeline/recall/impl.ts +312 -0
- package/extensions/mega-pipeline/recall.ts +10 -306
- package/package.json +1 -1
- package/src/config.ts +12 -0
- package/src/failback/floor.ts +71 -0
- package/src/failback/types.ts +44 -0
- package/src/recall/readonly.ts +57 -0
- package/src/recall/recall3wf.fixture.ts +87 -0
- package/src/recall/validator.ts +137 -0
- package/src/recall/vote.ts +240 -0
- package/src/store/sqlite/fts5-search.ts +40 -0
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* recall/vote.ts — the 3-independent-source recall vote (3WF-3).
|
|
3
|
+
*
|
|
4
|
+
* Three INDEPENDENT, read-only sources name candidate checkpoints:
|
|
5
|
+
* A vector — raw semantic hits (recall/readonly.ts), cosine 0..1 scale.
|
|
6
|
+
* B fts5 — BM25 trigram hits (hydrated to checkpointIds), FTS5 BM25 scale
|
|
7
|
+
* (negative score = better match; ranking order is what matters).
|
|
8
|
+
* C recency — the N freshest checkpoints by timestamp, query-INDEPENDENT.
|
|
9
|
+
* (NOT turn_recall / TurnReader — that would just echo already-
|
|
10
|
+
* injected content, not an independent signal.)
|
|
11
|
+
*
|
|
12
|
+
* Each source is a different score scale, so they are NOT directly comparable.
|
|
13
|
+
* We normalize each source's scores to a 0..1 scale (per-source min-max) BEFORE
|
|
14
|
+
* combining. Averaging raw cosine (0..1) with raw BM25 (arbitrary negative
|
|
15
|
+
* magnitude) or a recency rank would be meaningless — the largest-magnitude
|
|
16
|
+
* scale would always dominate. Normalization makes each source a peer voter.
|
|
17
|
+
*
|
|
18
|
+
* Overlap rule: a checkpoint named by >=2 of 3 distinct sources short-circuits
|
|
19
|
+
* as a winner. Fallback (no 2/3 majority): rank all candidates by the
|
|
20
|
+
* cross-source MEAN of their normalized scores.
|
|
21
|
+
*
|
|
22
|
+
* Non-fatal throughout. Pi-agnostic: no pi runtime imports.
|
|
23
|
+
*/
|
|
24
|
+
import { openStore } from "../store/sqlite/utils.js";
|
|
25
|
+
import { fts5SearchScoped, hydrateFts5Hits } from "../store/sqlite/fts5-search.js";
|
|
26
|
+
// The SQLite store is the source of truth (src/store.ts's same-named helper
|
|
27
|
+
// reads the LEGACY gzipped-JSON DR snapshot, which is empty for live sessions —
|
|
28
|
+
// importing it here would silently starve sources B and C). Mirrors the import
|
|
29
|
+
// in vector-search.ts + tieredRouter.ts.
|
|
30
|
+
import { listCheckpoints } from "../store/sqlite.js";
|
|
31
|
+
import { computeContentDigest } from "../dedup/digest.js";
|
|
32
|
+
import { recallRawHits } from "./readonly.js";
|
|
33
|
+
import { Logger } from "../log.js";
|
|
34
|
+
/** Per-source normalization: map raw scores to 0..1 via min-max within source. */
|
|
35
|
+
function normalizeScores(scores) {
|
|
36
|
+
const map = new Map();
|
|
37
|
+
if (scores.length === 0)
|
|
38
|
+
return map;
|
|
39
|
+
const min = Math.min(...scores);
|
|
40
|
+
const max = Math.max(...scores);
|
|
41
|
+
const span = max - min;
|
|
42
|
+
scores.forEach((s, i) => {
|
|
43
|
+
map.set(i, span === 0 ? 1 : (s - min) / span);
|
|
44
|
+
});
|
|
45
|
+
return map;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Run the three-source recall vote. Returns agreement winners + a per-id vote
|
|
49
|
+
* count + the names of sources that produced no winning candidate. Non-fatal:
|
|
50
|
+
* any search failure degrades to the remaining sources (empty winners allowed).
|
|
51
|
+
*/
|
|
52
|
+
export function voteRecall(opts, store) {
|
|
53
|
+
const logger = new Logger();
|
|
54
|
+
const limit = opts.limit ?? 3;
|
|
55
|
+
const recencyCount = opts.recencyCount ?? limit;
|
|
56
|
+
// ── Source A: vector (raw hits, cosine 0..1). ────────────────────────────
|
|
57
|
+
const vectorCands = recallRawHits({ sessionId: opts.sessionId, query: opts.query, limit }, store).map((h) => ({
|
|
58
|
+
checkpointId: h.checkpoint.checkpointId,
|
|
59
|
+
score: h.score,
|
|
60
|
+
source: "vector",
|
|
61
|
+
}));
|
|
62
|
+
// ── Source B: fts5 (BM25, hydrated to checkpointIds). ─────────────────────
|
|
63
|
+
// Dedup on L0 content digest so the SAME normalized text under two ids does
|
|
64
|
+
// not double-vote — collapse to one candidate per digest.
|
|
65
|
+
const fts5Cands = (() => {
|
|
66
|
+
try {
|
|
67
|
+
const reader = openStore(store.stateDir);
|
|
68
|
+
const hits = fts5SearchScoped(opts.query, reader, opts.sessionId, limit);
|
|
69
|
+
// FTS5 returns scores ordered best-first (bm25 asc); lower = better.
|
|
70
|
+
// We keep the raw score (negative-is-better) and flip in normalization.
|
|
71
|
+
const hydrated = hydrateFts5Hits(hits, opts.sessionId, store.stateDir);
|
|
72
|
+
// Dedup on checkpointId so the same checkpoint cannot double-count, AND
|
|
73
|
+
// on the L0 CONTENT digest so identical normalized text stored under two
|
|
74
|
+
// different ids collapses to one vote. The digest is taken over the
|
|
75
|
+
// joined `summary` (real content): hashing the id string would be a
|
|
76
|
+
// no-op tier, since ids are unique by definition.
|
|
77
|
+
const seenDigest = new Set();
|
|
78
|
+
const seenId = new Set();
|
|
79
|
+
const out = [];
|
|
80
|
+
for (const h of hydrated) {
|
|
81
|
+
if (seenId.has(h.checkpointId))
|
|
82
|
+
continue;
|
|
83
|
+
seenId.add(h.checkpointId);
|
|
84
|
+
const digest = computeContentDigest(h.summary).contentHash;
|
|
85
|
+
if (seenDigest.has(digest))
|
|
86
|
+
continue;
|
|
87
|
+
seenDigest.add(digest);
|
|
88
|
+
out.push({ checkpointId: h.checkpointId, score: h.score, source: "fts5" });
|
|
89
|
+
}
|
|
90
|
+
return out;
|
|
91
|
+
}
|
|
92
|
+
catch {
|
|
93
|
+
return [];
|
|
94
|
+
}
|
|
95
|
+
})();
|
|
96
|
+
// ── Source C: recency (N freshest checkpoints, query-independent). ────────
|
|
97
|
+
// Timestamp-ordered; the lower the index the fresher. Score = recency rank
|
|
98
|
+
// (fresh = high) so normalization treats newest as best.
|
|
99
|
+
const recencyCands = (() => {
|
|
100
|
+
try {
|
|
101
|
+
const cps = listCheckpoints(opts.sessionId, store.stateDir)
|
|
102
|
+
.filter((c) => c.dedupStatus !== "removed")
|
|
103
|
+
.sort((a, b) => (b.timestamp ?? 0) - (a.timestamp ?? 0))
|
|
104
|
+
.slice(0, recencyCount);
|
|
105
|
+
return cps.map((cp, i) => ({
|
|
106
|
+
checkpointId: cp.checkpointId,
|
|
107
|
+
// Fresher => higher raw score (recency rank). Normalized below.
|
|
108
|
+
score: cps.length - i,
|
|
109
|
+
source: "recency",
|
|
110
|
+
}));
|
|
111
|
+
}
|
|
112
|
+
catch {
|
|
113
|
+
return [];
|
|
114
|
+
}
|
|
115
|
+
})();
|
|
116
|
+
const sources = [
|
|
117
|
+
{ name: "vector", cands: vectorCands },
|
|
118
|
+
{ name: "fts5", cands: fts5Cands },
|
|
119
|
+
{ name: "recency", cands: recencyCands },
|
|
120
|
+
];
|
|
121
|
+
// Per-source normalization to 0..1 so the three scales are comparable.
|
|
122
|
+
const perSource = sources.map((s) => ({
|
|
123
|
+
name: s.name,
|
|
124
|
+
norm: normalizeScores(s.cands.map((c) => c.score)),
|
|
125
|
+
}));
|
|
126
|
+
// Aggregate: best normalized score per source per checkpointId + vote count.
|
|
127
|
+
const bestScoreBySource = new Map();
|
|
128
|
+
const seenIds = new Set();
|
|
129
|
+
for (const src of sources) {
|
|
130
|
+
const norm = perSource.find((p) => p.name === src.name).norm;
|
|
131
|
+
const bestByCp = new Map();
|
|
132
|
+
src.cands.forEach((c, i) => {
|
|
133
|
+
const n = norm.get(i) ?? 0;
|
|
134
|
+
const prev = bestByCp.get(c.checkpointId);
|
|
135
|
+
if (prev === undefined || n > prev)
|
|
136
|
+
bestByCp.set(c.checkpointId, n);
|
|
137
|
+
seenIds.add(c.checkpointId);
|
|
138
|
+
});
|
|
139
|
+
bestScoreBySource.set(src.name, bestByCp);
|
|
140
|
+
}
|
|
141
|
+
// Vote count = number of DISTINCT sources naming each checkpointId.
|
|
142
|
+
const votes = {};
|
|
143
|
+
const sumByCp = new Map();
|
|
144
|
+
for (const id of seenIds) {
|
|
145
|
+
let count = 0;
|
|
146
|
+
let sum = 0;
|
|
147
|
+
for (const src of sources) {
|
|
148
|
+
const m = bestScoreBySource.get(src.name);
|
|
149
|
+
if (m.has(id)) {
|
|
150
|
+
count++;
|
|
151
|
+
sum += m.get(id);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
votes[id] = count;
|
|
155
|
+
sumByCp.set(id, sum);
|
|
156
|
+
}
|
|
157
|
+
/** Mean normalized score across the sources that named `id`. */
|
|
158
|
+
const meanScore = (id) => (sumByCp.get(id) ?? 0) / (votes[id] ?? 1);
|
|
159
|
+
// Short-circuit: >=2 of 3 distinct sources => winner (agreement). Ranked by
|
|
160
|
+
// vote count first (stronger agreement wins), then by mean normalized score —
|
|
161
|
+
// the validator consumes this list in order and takes the first that passes,
|
|
162
|
+
// so the ordering IS the ranking and must not be Set-insertion order.
|
|
163
|
+
const winners = [];
|
|
164
|
+
const divergent = new Set(sources.map((s) => s.name));
|
|
165
|
+
const agreed = [...seenIds]
|
|
166
|
+
.filter((id) => (votes[id] ?? 0) >= 2)
|
|
167
|
+
.sort((a, b) => (votes[b] ?? 0) - (votes[a] ?? 0) || meanScore(b) - meanScore(a));
|
|
168
|
+
for (const id of agreed) {
|
|
169
|
+
const cand = (() => {
|
|
170
|
+
for (const src of sources) {
|
|
171
|
+
const c = src.cands.find((x) => x.checkpointId === id);
|
|
172
|
+
if (c)
|
|
173
|
+
return c;
|
|
174
|
+
}
|
|
175
|
+
return null;
|
|
176
|
+
})();
|
|
177
|
+
if (cand)
|
|
178
|
+
winners.push(cand);
|
|
179
|
+
}
|
|
180
|
+
if (winners.length === 0) {
|
|
181
|
+
// Fallback (no 2-of-3 agreement): rank by cross-source MEAN normalized score.
|
|
182
|
+
const ranked = [...seenIds].sort((a, b) => meanScore(b) - meanScore(a));
|
|
183
|
+
for (const id of ranked) {
|
|
184
|
+
const cand = (() => {
|
|
185
|
+
for (const src of sources) {
|
|
186
|
+
const c = src.cands.find((x) => x.checkpointId === id);
|
|
187
|
+
if (c)
|
|
188
|
+
return c;
|
|
189
|
+
}
|
|
190
|
+
return null;
|
|
191
|
+
})();
|
|
192
|
+
if (cand)
|
|
193
|
+
winners.push(cand);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
// Divergence = a source that named NONE of the winning checkpoints. This must
|
|
197
|
+
// be computed per SOURCE against the winning ID SET, not from `winner.source`
|
|
198
|
+
// (a winner is one candidate object carrying a single source label, so an id
|
|
199
|
+
// agreed on by all three sources would still credit only one of them).
|
|
200
|
+
const winningIds = new Set(winners.map((w) => w.checkpointId));
|
|
201
|
+
for (const src of sources) {
|
|
202
|
+
if (src.cands.some((c) => winningIds.has(c.checkpointId))) {
|
|
203
|
+
divergent.delete(src.name);
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
if (divergent.size > 0) {
|
|
207
|
+
logger.info("recall_vote_divergence", {
|
|
208
|
+
divergentSources: [...divergent],
|
|
209
|
+
winnerCount: winners.length,
|
|
210
|
+
});
|
|
211
|
+
}
|
|
212
|
+
return {
|
|
213
|
+
winners,
|
|
214
|
+
votes,
|
|
215
|
+
divergentSources: [...divergent],
|
|
216
|
+
};
|
|
217
|
+
}
|
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
* time automatically — no manual n-gram splitting needed. The BM25 rank from
|
|
12
12
|
* fts5 ranks results by trigram-overlap density.
|
|
13
13
|
*/
|
|
14
|
+
import { listCheckpoints } from "./checkpoints.js";
|
|
14
15
|
/**
|
|
15
16
|
* Search the context_chunks_trgm FTS5 table for checkpoints whose
|
|
16
17
|
* `normalized_text` matches `query` via trigram similarity.
|
|
@@ -66,3 +67,28 @@ export function fts5SearchScoped(query, reader, sessionId, limit = 10) {
|
|
|
66
67
|
// No session filter — plain FTS5 search across all sessions.
|
|
67
68
|
return fts5Search(query, reader, limit);
|
|
68
69
|
}
|
|
70
|
+
/**
|
|
71
|
+
* Hydrate FTS5 hits (which only carry checkpoint `id` + BM25 `score`, with NO
|
|
72
|
+
* checkpoint object) by joining against the real stored checkpoints. Mirrors the
|
|
73
|
+
* private `hydrateHits` in tieredRouter.ts (listCheckpoints → Map on
|
|
74
|
+
* checkpointId → join, dropping ids with no checkpoint).
|
|
75
|
+
*
|
|
76
|
+
* The joined `summary` is returned alongside the id because the 3WF-3 voter
|
|
77
|
+
* dedups candidates on the L0 CONTENT digest — hashing the id string instead
|
|
78
|
+
* would be a no-op tier (ids are already unique), silently disabling the check.
|
|
79
|
+
*
|
|
80
|
+
* @param hits Raw Fts5Hit rows (id + BM25 score).
|
|
81
|
+
* @param sessionId Session scope used to fetch the checkpoint set.
|
|
82
|
+
* @param stateDir On-disk state dir for the sync store.
|
|
83
|
+
*/
|
|
84
|
+
export function hydrateFts5Hits(hits, sessionId, stateDir) {
|
|
85
|
+
const cps = listCheckpoints(sessionId, stateDir);
|
|
86
|
+
const cpMap = new Map(cps.map((cp) => [cp.checkpointId, cp]));
|
|
87
|
+
const out = [];
|
|
88
|
+
for (const h of hits) {
|
|
89
|
+
const cp = cpMap.get(h.id);
|
|
90
|
+
if (cp)
|
|
91
|
+
out.push({ checkpointId: h.id, score: h.score, summary: cp.summary });
|
|
92
|
+
}
|
|
93
|
+
return out;
|
|
94
|
+
}
|
|
@@ -241,6 +241,7 @@ export const SETTINGS: ReadonlyArray<SettingGroup> = [
|
|
|
241
241
|
num("MEGACOMPACT_L2_THRESHOLD", "L2 Cosine Threshold", "L2 semantic dedup firing point", 0.85, 0, 1),
|
|
242
242
|
num("MEGACOMPACT_L1_JACCARD", "L1 Jaccard Threshold", "L1 MinHash near-dup threshold", 0.8, 0, 1),
|
|
243
243
|
num("MEGACOMPACT_DEDUP_SIM", "Dedup Similarity", "Legacy content-similarity fallback", 0.9, 0, 1),
|
|
244
|
+
num("MEGACOMPACT_RECALL_MIN_COSINE", "Recall Min Cosine (same-repo)", "3WF-3 same-repo floor the 3-source validator applies to the top winner (cross-repo 0.90 stays separate)", 0.12, 0, 1),
|
|
244
245
|
num("MEGACOMPACT_MMR_LAMBDA", "MMR Lambda", "Maximal Marginal Relevance diversity", 0.5, 0, 1),
|
|
245
246
|
num("MEGACOMPACT_SEMDEDUP_COSINE", "SemDeDup Cosine", "Offline SemDeDup pair threshold", 0.95, 0, 1),
|
|
246
247
|
num("MEGACOMPACT_CONSOLIDATE_COSINE", "Consolidate Cosine", "Memory consolidation merge threshold", 0.7, 0, 1),
|
|
@@ -119,6 +119,11 @@ export interface MegaConfig {
|
|
|
119
119
|
* tighter than same-repo so only genuinely-relevant cross-repo context is
|
|
120
120
|
* injected. */
|
|
121
121
|
crossRepoCosine: number;
|
|
122
|
+
/** Same-repo recall cosine floor (3WF-3). Default 0.12. SEPARATE from
|
|
123
|
+
* `crossRepoCosine` (S17, default 0.90 — stricter, cross-repo only). The
|
|
124
|
+
* 3-source validator applies this to the top winner; hits below it are
|
|
125
|
+
* rejected in favor of the next-ranked candidate or the provenance floor. */
|
|
126
|
+
recallMinCosine: number;
|
|
122
127
|
/** Memory-RAG auto-review enabled (S20). Every memoryReviewInterval turns the
|
|
123
128
|
* conversation is auto-reviewed into durable add/replace/remove memories. */
|
|
124
129
|
memoryAutoReview: boolean;
|
|
@@ -217,6 +217,12 @@ export function loadConfig(): MegaConfig {
|
|
|
217
217
|
autoWikiEnabled: envBool("MEGACOMPACT_AUTO_WIKI", true),
|
|
218
218
|
crossRepoEnabled: envBool("MEGACOMPACT_CROSSREPO_ENABLED", true),
|
|
219
219
|
crossRepoCosine: Number(process.env.MEGACOMPACT_CROSSREPO_COSINE ?? "0.90"),
|
|
220
|
+
// 3WF-3: SAME-repo recall cosine floor applied by the 3-source validator to
|
|
221
|
+
// the top winner. SEPARATE from crossRepoCosine (S17, default 0.90, stricter
|
|
222
|
+
// and cross-repo only). This same-repo floor is permissive by default (0.12)
|
|
223
|
+
// so recall still surfaces loosely-relevant within-repo context while
|
|
224
|
+
// rejecting effectively-unrelated hits. Mirrors src/config.ts RECALL_MIN_COSINE.
|
|
225
|
+
recallMinCosine: Number(process.env.MEGACOMPACT_RECALL_MIN_COSINE ?? "0.12"),
|
|
220
226
|
memoryAutoReview: envBool("MEGACOMPACT_MEMORY_AUTO_REVIEW", true),
|
|
221
227
|
memoryReviewInterval: envFlag("MEGACOMPACT_MEMORY_REVIEW_INTERVAL", 10),
|
|
222
228
|
recallMaxTokens: envFlag("MEGACOMPACT_RECALL_MAX_TOKENS", 1500),
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* context-handler/injectionConfirm.fixture.ts — shared fixtures for the 3WF-4
|
|
3
|
+
* InjectionConfirm tests.
|
|
4
|
+
*
|
|
5
|
+
* Split out so each test file stays under the extensions/300-soft-cap the way
|
|
6
|
+
* src/recall/recall3wf.fixture.ts does for 3WF-3. These are REAL fixtures, not
|
|
7
|
+
* mocks/stubs: a REAL VectorStore over a temp stateDir with REAL checkpoints
|
|
8
|
+
* persisted via compactSession; the MegaRuntime is a minimal typed stub exposing
|
|
9
|
+
* only the fields confirmInjection touches (store, pendingRecallBlock,
|
|
10
|
+
* pendingMemoryRecallBlock, appendEvent), matching the triggerGuard/thrashGuard
|
|
11
|
+
* test conventions.
|
|
12
|
+
*/
|
|
13
|
+
import { mkdtempSync } from "node:fs";
|
|
14
|
+
import { tmpdir } from "node:os";
|
|
15
|
+
import { join } from "node:path";
|
|
16
|
+
|
|
17
|
+
import type { AgentMessage } from "@earendil-works/pi-agent-core";
|
|
18
|
+
import { VectorStore } from "../../../src/vectorStore.js";
|
|
19
|
+
import { compactSession } from "../../../src/engine.js";
|
|
20
|
+
import type { MegaRuntime } from "../../mega-runtime.js";
|
|
21
|
+
import type { MegaConfig } from "../../mega-config.js";
|
|
22
|
+
|
|
23
|
+
/** Real EngineMessage fixture. */
|
|
24
|
+
export function msg(role: "user" | "assistant", text: string): any {
|
|
25
|
+
return { role, text };
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** A user-role AgentMessage carrying `text` (the tail-block shape). */
|
|
29
|
+
export function userMsg(text: string): AgentMessage {
|
|
30
|
+
return { role: "user", content: text, timestamp: 1 } as unknown as AgentMessage;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** Recorded appendEvent calls, for telemetry assertions. */
|
|
34
|
+
export interface RecordedEvent {
|
|
35
|
+
name: string;
|
|
36
|
+
payload: Record<string, unknown>;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** Fresh isolated state dir per VectorStore. */
|
|
40
|
+
export function freshStore(): { store: VectorStore; dir: string } {
|
|
41
|
+
const dir = mkdtempSync(join(tmpdir(), "mc-inject-"));
|
|
42
|
+
return { store: new VectorStore({ dedupSim: 0.9, stateDir: dir }), dir };
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** Persist N distinct checkpoints with ascending timestamps. */
|
|
46
|
+
export function seed(store: VectorStore, topics: string[], sid = "sess_inject"): void {
|
|
47
|
+
topics.forEach((t, i) => {
|
|
48
|
+
compactSession(
|
|
49
|
+
{
|
|
50
|
+
sessionId: sid,
|
|
51
|
+
messages: [msg("user", t), msg("assistant", "ok")],
|
|
52
|
+
keepFrom: 2,
|
|
53
|
+
timestamp: i + 1,
|
|
54
|
+
},
|
|
55
|
+
store,
|
|
56
|
+
);
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** Minimal MegaRuntime stub exposing only the confirmInjection touch-points. */
|
|
61
|
+
export function runtimeStub(
|
|
62
|
+
store: VectorStore,
|
|
63
|
+
over: Partial<{
|
|
64
|
+
pendingRecallBlock: string | undefined;
|
|
65
|
+
pendingMemoryRecallBlock: string | undefined;
|
|
66
|
+
}> = {},
|
|
67
|
+
): { runtime: MegaRuntime; events: RecordedEvent[] } {
|
|
68
|
+
const events: RecordedEvent[] = [];
|
|
69
|
+
const runtime = {
|
|
70
|
+
store,
|
|
71
|
+
pendingRecallBlock: over.pendingRecallBlock,
|
|
72
|
+
pendingMemoryRecallBlock: over.pendingMemoryRecallBlock,
|
|
73
|
+
perfTurnStart: undefined,
|
|
74
|
+
rt: { recallInjectedThisTurn: false },
|
|
75
|
+
appendEvent: (name: string, payload: Record<string, unknown>) => {
|
|
76
|
+
events.push({ name, payload });
|
|
77
|
+
},
|
|
78
|
+
} as unknown as MegaRuntime;
|
|
79
|
+
return { runtime, events };
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** Config stub: only the flags confirmInjection reads. */
|
|
83
|
+
export function configStub(
|
|
84
|
+
over: Partial<{ threeWayFailback: boolean; recallTailInject: boolean }> = {},
|
|
85
|
+
): MegaConfig {
|
|
86
|
+
return {
|
|
87
|
+
threeWayFailback: over.threeWayFailback ?? true,
|
|
88
|
+
recallTailInject: over.recallTailInject ?? true,
|
|
89
|
+
} as unknown as MegaConfig;
|
|
90
|
+
}
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* context-handler/injectionConfirm.ts — 3WF-4 InjectionConfirm.
|
|
3
|
+
*
|
|
4
|
+
* QA amendment A3 (binding): pi exposes NO prompt readback API, so the only
|
|
5
|
+
* verifiable proxy for what the provider will actually receive is the pre-LLM
|
|
6
|
+
* message list. In DEFAULT tail mode (`recallTailInject` ON) the staged recall
|
|
7
|
+
* block rides in as a user-role tail message, so we assert the block's marker
|
|
8
|
+
* text is present in the message list we are about to return. In LEGACY prepend
|
|
9
|
+
* mode (`recallTailInject` OFF) the block never enters the message list at all,
|
|
10
|
+
* so the guard degrades to a string-contains check over our own composed return
|
|
11
|
+
* value and never reports a false miss.
|
|
12
|
+
*
|
|
13
|
+
* Recovery ladder when the marker is absent (tail mode only):
|
|
14
|
+
* 1. recomposed — rebuild the view from the runtime's pending blocks via the
|
|
15
|
+
* SAME `buildTailResult` composition the handler uses (self-repair on this
|
|
16
|
+
* event; the user sees nothing).
|
|
17
|
+
* 2. floor — nothing pending either: append the shared provenance floor text
|
|
18
|
+
* (src/failback/floor.ts) as a user-role tail message so the model is never
|
|
19
|
+
* silently left with no compacted-context provenance at all.
|
|
20
|
+
*
|
|
21
|
+
* Stack position: wraps the `tailResult` closure returned by buildTailResult, so
|
|
22
|
+
* EVERY return point of the context handler (gate / replay / debounce /
|
|
23
|
+
* thrash-guard / pipeline / live-trim) is verified with one wiring point.
|
|
24
|
+
*
|
|
25
|
+
* Non-fatal everywhere: any throw degrades to the unverified view (pre-sprint
|
|
26
|
+
* behavior). Flag OFF => the wrapper is never installed (byte-identical).
|
|
27
|
+
*/
|
|
28
|
+
import type { AgentMessage } from "@earendil-works/pi-agent-core";
|
|
29
|
+
import type { MegaRuntime } from "../../mega-runtime.js";
|
|
30
|
+
import type { MegaConfig } from "../../mega-config.js";
|
|
31
|
+
import type { InjectionVerdict } from "../../../src/failback/types.js";
|
|
32
|
+
import { vectorList } from "../../../src/vectorStore.js";
|
|
33
|
+
import { normalizeSessionId } from "../../../src/store.js";
|
|
34
|
+
import { buildFloorBlock } from "../../../src/failback/floor.js";
|
|
35
|
+
import { withRecallTail } from "../recall-tail.js";
|
|
36
|
+
import { messageContentText } from "./messageText.js";
|
|
37
|
+
|
|
38
|
+
/** A composed context view (the shape every handler return point produces). */
|
|
39
|
+
export interface TailView {
|
|
40
|
+
messages: AgentMessage[];
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/** The staged blocks + mode this pass verifies (pure inputs, no pi runtime). */
|
|
44
|
+
export interface ConfirmInput {
|
|
45
|
+
/** The staged block text expected to have landed (null => nothing to verify). */
|
|
46
|
+
staged: string | null;
|
|
47
|
+
/** False in legacy prepend mode: verify the return string, not the list. */
|
|
48
|
+
tailMode: boolean;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* The marker substring used to locate a staged block inside a message. The
|
|
53
|
+
* block's first non-empty line, capped, so prompt reshapes (cache striping /
|
|
54
|
+
* message separation) that regroup messages cannot defeat the match, while a
|
|
55
|
+
* genuinely dropped block still fails it.
|
|
56
|
+
*/
|
|
57
|
+
export function blockMarker(block: string): string {
|
|
58
|
+
const line = block
|
|
59
|
+
.split("\n")
|
|
60
|
+
.map((l) => l.trim())
|
|
61
|
+
.find((l) => l.length > 0);
|
|
62
|
+
return (line ?? "").slice(0, 80);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* PURE decision function: did the staged block land in this view, and if not,
|
|
67
|
+
* which rung should repair it? Takes the already-extracted message texts so it
|
|
68
|
+
* stays free of pi types and is directly unit-testable.
|
|
69
|
+
*/
|
|
70
|
+
export function decideInjection(
|
|
71
|
+
input: ConfirmInput,
|
|
72
|
+
messageTexts: readonly string[],
|
|
73
|
+
hasPendingBlocks: boolean,
|
|
74
|
+
): InjectionVerdict {
|
|
75
|
+
const marker = input.staged ? blockMarker(input.staged) : "";
|
|
76
|
+
// Nothing staged => nothing to assert; NOT a miss. Injecting a floor here
|
|
77
|
+
// would push provenance text into sessions that never had recall to lose.
|
|
78
|
+
if (!marker) return { landed: true, recovered: "none" };
|
|
79
|
+
const landed = messageTexts.some((t) => t.includes(marker));
|
|
80
|
+
if (landed) return { landed: true, recovered: "none" };
|
|
81
|
+
// Absent: recompose when the runtime still holds pending blocks, else floor.
|
|
82
|
+
return { landed: false, recovered: hasPendingBlocks ? "recomposed" : "floor" };
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/** Append `text` as a user-role tail message (same shape as recall-tail.ts). */
|
|
86
|
+
function withFloorTail(view: TailView, text: string): TailView {
|
|
87
|
+
const tailMsg = {
|
|
88
|
+
role: "user" as const,
|
|
89
|
+
content: text,
|
|
90
|
+
timestamp: Date.now(),
|
|
91
|
+
} as unknown as AgentMessage;
|
|
92
|
+
return { messages: [...view.messages, tailMsg] };
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Thin caller: verify (and if needed repair) one composed view. Returns the view
|
|
97
|
+
* to actually return from the handler. `sessionId` sources the floor checkpoints.
|
|
98
|
+
*
|
|
99
|
+
* The recompose rung deliberately re-appends via `withRecallTail` onto the
|
|
100
|
+
* ALREADY-COMPOSED view rather than re-running `buildTailResult`: the reshape
|
|
101
|
+
* stages (cache striping / message separation) are the realistic way a tail
|
|
102
|
+
* message gets regrouped away, and re-running the same composition would
|
|
103
|
+
* reproduce the same loss. Appending after the reshape is the actual repair, and
|
|
104
|
+
* it keeps the PREVENT-PI-001/002 tail-append invariant (a single user-role
|
|
105
|
+
* message after a complete prefix can never split a toolCall/toolResult pair).
|
|
106
|
+
*/
|
|
107
|
+
export function confirmInjection(
|
|
108
|
+
runtime: MegaRuntime,
|
|
109
|
+
config: MegaConfig,
|
|
110
|
+
view: TailView,
|
|
111
|
+
sessionId: string,
|
|
112
|
+
): TailView {
|
|
113
|
+
try {
|
|
114
|
+
if (!config.threeWayFailback) return view;
|
|
115
|
+
// What the tail composition was supposed to inject. BOTH staged blocks
|
|
116
|
+
// count: withRecallTail joins recall + memory blocks into one tail
|
|
117
|
+
// message, so either one going missing is a real injection failure.
|
|
118
|
+
const staged =
|
|
119
|
+
runtime.pendingRecallBlock ?? runtime.pendingMemoryRecallBlock ?? null;
|
|
120
|
+
// Can the recompose rung actually re-append? Only when a block is still
|
|
121
|
+
// staged on the runtime. When it is not (blocks consumed between
|
|
122
|
+
// composition and this check), or when withRecallTail declines to append,
|
|
123
|
+
// the ladder falls through to the floor rung below.
|
|
124
|
+
const hasPending =
|
|
125
|
+
runtime.pendingRecallBlock != null ||
|
|
126
|
+
runtime.pendingMemoryRecallBlock != null;
|
|
127
|
+
// Legacy prepend mode: the block is not expected in the message list —
|
|
128
|
+
// verify our composed return value contains it instead (A3 degrade path).
|
|
129
|
+
if (!config.recallTailInject) {
|
|
130
|
+
const composed = view.messages.map(messageContentText).join("\n");
|
|
131
|
+
const marker = staged ? blockMarker(staged) : "";
|
|
132
|
+
runtime.appendEvent("injection_confirmed", {
|
|
133
|
+
mode: "prepend",
|
|
134
|
+
landed: marker ? composed.includes(marker) : true,
|
|
135
|
+
});
|
|
136
|
+
return view;
|
|
137
|
+
}
|
|
138
|
+
const verdict = decideInjection(
|
|
139
|
+
{ staged, tailMode: true },
|
|
140
|
+
view.messages.map(messageContentText),
|
|
141
|
+
hasPending,
|
|
142
|
+
);
|
|
143
|
+
if (verdict.landed) {
|
|
144
|
+
runtime.appendEvent("injection_confirmed", { mode: "tail", landed: true });
|
|
145
|
+
return view;
|
|
146
|
+
}
|
|
147
|
+
if (verdict.recovered === "recomposed") {
|
|
148
|
+
const rebuilt = withRecallTail(view.messages, runtime, config);
|
|
149
|
+
// withRecallTail returns the input array unchanged on failure; only
|
|
150
|
+
// treat a genuine append as a recovery.
|
|
151
|
+
if (rebuilt.length > view.messages.length) {
|
|
152
|
+
runtime.appendEvent("injection_recovered", { via: "recomposed" });
|
|
153
|
+
return { messages: rebuilt };
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
const floor = buildFloorBlock(
|
|
157
|
+
vectorList(runtime.store, normalizeSessionId(sessionId)),
|
|
158
|
+
);
|
|
159
|
+
runtime.appendEvent("injection_recovered", {
|
|
160
|
+
via: "floor",
|
|
161
|
+
basis: floor.basis,
|
|
162
|
+
});
|
|
163
|
+
return withFloorTail(view, floor.text);
|
|
164
|
+
} catch {
|
|
165
|
+
// Non-fatal: return the unverified view (pre-sprint behavior).
|
|
166
|
+
return view;
|
|
167
|
+
}
|
|
168
|
+
}
|
|
@@ -30,6 +30,10 @@ import { normalizeSessionId } from "../../../src/store.js";
|
|
|
30
30
|
import { recall } from "../../../src/engine.js";
|
|
31
31
|
import { formatRecallBlock } from "../../../src/recall.js";
|
|
32
32
|
import { vectorStats, vectorList } from "../../../src/vectorStore.js";
|
|
33
|
+
import {
|
|
34
|
+
buildFloorBlock as sharedFloorBlock,
|
|
35
|
+
FLOOR_UNAVAILABLE_TEXT,
|
|
36
|
+
} from "../../../src/failback/floor.js";
|
|
33
37
|
import { recentUserQuery } from "../../mega-runtime.js";
|
|
34
38
|
|
|
35
39
|
/** One-shot completion marker per MegaRuntime (dies with the runtime). */
|
|
@@ -108,30 +112,18 @@ export function runTriggerGuard(
|
|
|
108
112
|
}
|
|
109
113
|
}
|
|
110
114
|
|
|
111
|
-
/**
|
|
115
|
+
/**
|
|
116
|
+
* Build the provenance floor string from the session's newest checkpoint.
|
|
117
|
+
*
|
|
118
|
+
* 3WF-4: the text construction moved to the SHARED pure builder
|
|
119
|
+
* (src/failback/floor.ts) — this keeps the store read (`vectorList`, unfiltered,
|
|
120
|
+
* exactly as 3WF-1 shipped) and the string return type, so output is
|
|
121
|
+
* byte-identical to the pre-refactor version.
|
|
122
|
+
*/
|
|
112
123
|
function buildFloorBlock(runtime: MegaRuntime, sid: string): string {
|
|
113
124
|
try {
|
|
114
|
-
|
|
115
|
-
let newest = cps[0];
|
|
116
|
-
for (const cp of cps) {
|
|
117
|
-
if (!newest || (cp.timestamp ?? 0) > (newest.timestamp ?? 0)) newest = cp;
|
|
118
|
-
}
|
|
119
|
-
const summary = newest?.summary?.trim();
|
|
120
|
-
if (summary) {
|
|
121
|
-
return (
|
|
122
|
-
"The following compacted context is the most recent checkpoint from " +
|
|
123
|
-
"this session (recall found no query-relevant match):\n\n" + summary
|
|
124
|
-
);
|
|
125
|
-
}
|
|
126
|
-
return (
|
|
127
|
-
"This session has compacted context but recall could not surface a " +
|
|
128
|
-
"checkpoint relevant to the current request; the most recent checkpoint " +
|
|
129
|
-
"summary is unavailable."
|
|
130
|
-
);
|
|
125
|
+
return sharedFloorBlock(vectorList(runtime.store, sid)).text;
|
|
131
126
|
} catch {
|
|
132
|
-
return
|
|
133
|
-
"This session has compacted context but recall could not surface a " +
|
|
134
|
-
"checkpoint relevant to the current request."
|
|
135
|
-
);
|
|
127
|
+
return FLOOR_UNAVAILABLE_TEXT;
|
|
136
128
|
}
|
|
137
129
|
}
|
|
@@ -24,6 +24,7 @@ import { piCompactWouldNoop } from "../mega-pipeline.js";
|
|
|
24
24
|
import type { MegaConfig } from "../mega-config.js";
|
|
25
25
|
import { buildTailResult } from "./context-handler/tailResult.js";
|
|
26
26
|
import { runTriggerGuard } from "./context-handler/triggerGuard.js";
|
|
27
|
+
import { confirmInjection } from "./context-handler/injectionConfirm.js";
|
|
27
28
|
import { persistEpochAndMaintain } from "./context-handler/afterCompact.js";
|
|
28
29
|
import { appendMirrorAndLedger } from "./context-handler/dbMirrorAppend.js";
|
|
29
30
|
import { evaluateGate, thrashGuardBlocks } from "./context-handler/gateCheck.js";
|
|
@@ -72,7 +73,21 @@ export function registerContextHandler(
|
|
|
72
73
|
// tail message at any view-return point. Returns undefined when nothing
|
|
73
74
|
// is staged (or the flag is OFF) so the caller falls through to its
|
|
74
75
|
// normal return.
|
|
75
|
-
const
|
|
76
|
+
const composeTail = buildTailResult(runtime, config, messages);
|
|
77
|
+
// 3WF-4 InjectionConfirm: wrap the tail factory so EVERY return point of
|
|
78
|
+
// this handler (gate / replay / debounce / thrash-guard / pipeline /
|
|
79
|
+
// live-trim) is verified — the staged block's marker must be present in
|
|
80
|
+
// the message list pi will send (tail mode), else we re-compose from the
|
|
81
|
+
// runtime's pending blocks and finally fall back to the shared floor.
|
|
82
|
+
// A composition that yields nothing staged (undefined) is passed through
|
|
83
|
+
// untouched, so flag-OFF and no-recall paths are byte-identical.
|
|
84
|
+
const tailResult: typeof composeTail = config.threeWayFailback
|
|
85
|
+
? (msgs) => {
|
|
86
|
+
const view = composeTail(msgs);
|
|
87
|
+
if (!view) return view;
|
|
88
|
+
return confirmInjection(runtime, config, view, ctx.sessionManager.getSessionId());
|
|
89
|
+
}
|
|
90
|
+
: composeTail;
|
|
76
91
|
// Always track context for the dashboard/widget, even when auto is off.
|
|
77
92
|
// (v0.8 regression: !config.auto gate sat above this, leaving ctx stats
|
|
78
93
|
// null -> widget '?% / ?/?' when auto disabled. Track first, THEN gate.)
|