pi-mega-compact 0.21.1 → 0.21.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config.js +13 -2
- package/dist/dedup/l1-minhash.js +5 -4
- package/dist/extensions/dashboard-server/routes-rag-settings-helpers.js +1 -0
- package/dist/extensions/mega-compact-child.js +126 -0
- package/dist/extensions/mega-compact.js +10 -0
- package/dist/extensions/mega-config.js +11 -3
- package/dist/src/bridge/factory.js +181 -0
- package/dist/src/bridge/types.js +10 -0
- package/dist/src/bridge.js +1 -0
- package/dist/src/config.js +13 -2
- package/dist/src/dedup/l1-minhash.js +5 -4
- package/dist/src/recall/validator.js +8 -1
- package/dist/src/recall/vote.js +34 -9
- package/dist/src/store/sqlite/fts5-search.js +6 -1
- package/dist/src/vector-read.js +6 -2
- package/dist/src/vectorStore/add.js +5 -1
- package/extensions/dashboard-server/routes-rag-settings-helpers.ts +6 -0
- package/extensions/mega-compact-child.ts +129 -0
- package/extensions/mega-compact.ts +10 -0
- package/extensions/mega-config-types.ts +5 -0
- package/extensions/mega-config.ts +11 -3
- package/package.json +1 -1
- package/src/bridge/factory.ts +231 -0
- package/src/bridge/types.ts +138 -0
- package/src/bridge.ts +24 -0
- package/src/config.ts +12 -3
- package/src/dedup/l1-minhash.ts +5 -4
- package/src/recall/validator.ts +8 -1
- package/src/recall/vote.ts +29 -8
- package/src/store/sqlite/fts5-search.ts +7 -1
- package/src/vector-read.ts +9 -2
- package/src/vectorStore/add.ts +7 -1
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* mega-compact-child.ts — minimal extension loaded ONLY into dispatched child
|
|
3
|
+
* pi subprocesses (spawned by ithacus with a second `-e` flag).
|
|
4
|
+
*
|
|
5
|
+
* Design: a child is a FRESH pi process started with `--no-extensions -e <this
|
|
6
|
+
* file>` (see ithacus-spawn.ts). It does NOT receive the parent's MegaConfig and
|
|
7
|
+
* is a separate process, so it reads its two control env vars directly and owns
|
|
8
|
+
* its own bridge. It gives children recall-at-start + compaction-on-shutdown via
|
|
9
|
+
* the mega-compact bridge, with NO tools and NO console output, so it never
|
|
10
|
+
* pollutes the child's `--mode json` JSONL stdout that ithacus-spawn parses.
|
|
11
|
+
*
|
|
12
|
+
* Per the teammate brief this mirrors ithacus-child-mailbox.ts (default export,
|
|
13
|
+
* no console, dispose on session_shutdown) but registers ZERO tools — registering
|
|
14
|
+
* any tool risks a pi duplicate-tool-name hard-fail and children need none.
|
|
15
|
+
*
|
|
16
|
+
* PREVENT-PI-004: no network. The bridge is a same-repo relative import over a
|
|
17
|
+
* local sqlite store; the only I/O is the read-only `git rev-parse` inside
|
|
18
|
+
* repoStateDir. Nothing to flag.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
|
|
22
|
+
import { sessionEntryToContextMessages } from "@earendil-works/pi-coding-agent";
|
|
23
|
+
import { createMegaBridge } from "../src/bridge.js";
|
|
24
|
+
import type {
|
|
25
|
+
MegaBridge,
|
|
26
|
+
BridgeMessage,
|
|
27
|
+
} from "../src/bridge.js";
|
|
28
|
+
import { repoStateDir } from "./mega-config.js";
|
|
29
|
+
import { STATE_DIR_DEFAULT } from "../src/config.js";
|
|
30
|
+
|
|
31
|
+
/** Default-ON env bool: only `=false`/`=0` disables (matches mega-config envBool). */
|
|
32
|
+
function envBool(name: string, fallback: boolean): boolean {
|
|
33
|
+
const v = process.env[name];
|
|
34
|
+
if (v == null || v === "") return fallback;
|
|
35
|
+
return v === "true" || v === "1";
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** Extract a recall query from a single AgentMessage (string or content blocks). */
|
|
39
|
+
function messageToText(m: { content: unknown }): string {
|
|
40
|
+
const c = (m as { content: unknown }).content;
|
|
41
|
+
if (typeof c === "string") return c;
|
|
42
|
+
if (Array.isArray(c)) return c.map((b: { text?: string }) => b.text ?? "").join(" ");
|
|
43
|
+
return "";
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** Convert a session's AgentMessages into the bridge's lightweight shape. */
|
|
47
|
+
function toBridgeMessages(ctx: ExtensionContext): BridgeMessage[] {
|
|
48
|
+
const out: BridgeMessage[] = [];
|
|
49
|
+
try {
|
|
50
|
+
for (const entry of ctx.sessionManager.getEntries()) {
|
|
51
|
+
for (const m of sessionEntryToContextMessages(entry as never)) {
|
|
52
|
+
if (m.role === "user" || m.role === "assistant") {
|
|
53
|
+
out.push({ role: m.role, text: messageToText(m) });
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
} catch {
|
|
58
|
+
/* non-fatal: a child without a session manager yields no messages */
|
|
59
|
+
}
|
|
60
|
+
return out;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export default function (pi: ExtensionAPI): void {
|
|
64
|
+
// Flag read at LOAD time: a flag-OFF child registers nothing and a flag-ON
|
|
65
|
+
// child that never fires a hook pays zero cost (bridge is built lazily).
|
|
66
|
+
if (!envBool("MEGACOMPACT_ITHACUS_BRIDGE", true)) return;
|
|
67
|
+
|
|
68
|
+
let bridge: MegaBridge | undefined;
|
|
69
|
+
|
|
70
|
+
// Build the bridge lazily on first hook fire so cost is opt-in by usage.
|
|
71
|
+
const getBridge = (): MegaBridge => {
|
|
72
|
+
if (!bridge) {
|
|
73
|
+
bridge = createMegaBridge({
|
|
74
|
+
stateDir: repoStateDir(process.cwd(), STATE_DIR_DEFAULT),
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
return bridge;
|
|
78
|
+
};
|
|
79
|
+
|
|
80
|
+
// S52-style recall injection: prepend staged checkpoints + durable memories
|
|
81
|
+
// to the system prompt, mirroring the main entry's before_agent_start path.
|
|
82
|
+
// 4th-layer stability guard: an unset/empty sessionId makes recall silently
|
|
83
|
+
// useless (the openclaw Date.now() gotcha), so skip outright.
|
|
84
|
+
pi.on("before_agent_start", async (event) => {
|
|
85
|
+
try {
|
|
86
|
+
const sessionId = process.env.ITHACUS_MEGA_SESSION_ID;
|
|
87
|
+
if (!sessionId || sessionId === "") return undefined;
|
|
88
|
+
|
|
89
|
+
// Prefer the event's raw prompt; fall back to a generic query.
|
|
90
|
+
const query = event.prompt && event.prompt.trim() ? event.prompt.trim() : "";
|
|
91
|
+
if (query === "") return undefined;
|
|
92
|
+
|
|
93
|
+
const b = getBridge();
|
|
94
|
+
const cp = b.recallCheckpoints({ sessionId, query, limit: 3 });
|
|
95
|
+
const mem = await b.recallMemories({ query, limit: 5 });
|
|
96
|
+
|
|
97
|
+
const blocks: string[] = [];
|
|
98
|
+
if (!cp.empty && cp.block) blocks.push(cp.block);
|
|
99
|
+
if (!mem.empty && mem.block) blocks.push(mem.block);
|
|
100
|
+
if (blocks.length === 0) return undefined;
|
|
101
|
+
|
|
102
|
+
return { systemPrompt: `${event.systemPrompt ?? ""}\n\n${blocks.join("\n\n")}` };
|
|
103
|
+
} catch {
|
|
104
|
+
// layer b: non-fatal — never break the agent loop. No injection.
|
|
105
|
+
return undefined;
|
|
106
|
+
}
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
// Compaction on shutdown: persist the session's messages as a checkpoint.
|
|
110
|
+
// Best-effort; non-fatal. Releases the sqlite handle via close().
|
|
111
|
+
// The bridge is constructed lazily here too: a child that only compacts (no
|
|
112
|
+
// recall fired) still persists its session. Best-effort; non-fatal.
|
|
113
|
+
pi.on("session_shutdown", async (_event, ctx) => {
|
|
114
|
+
try {
|
|
115
|
+
const sessionId = process.env.ITHACUS_MEGA_SESSION_ID;
|
|
116
|
+
if (!sessionId || sessionId === "") return;
|
|
117
|
+
const messages = toBridgeMessages(ctx);
|
|
118
|
+
if (messages.length === 0) return;
|
|
119
|
+
await getBridge().compact({ sessionId, messages });
|
|
120
|
+
} catch {
|
|
121
|
+
/* non-fatal */
|
|
122
|
+
} finally {
|
|
123
|
+
if (bridge) {
|
|
124
|
+
bridge.close();
|
|
125
|
+
bridge = undefined;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
});
|
|
129
|
+
}
|
|
@@ -98,6 +98,16 @@ export default function (pi: ExtensionAPI) {
|
|
|
98
98
|
);
|
|
99
99
|
config.poisonedContextRepeatThreshold = 3;
|
|
100
100
|
}
|
|
101
|
+
// E1: validate similarity thresholds — NaN or out of range silently disables
|
|
102
|
+
// recall dedup (anything >= NaN is false). envFlag guards NaN; clamp the rest.
|
|
103
|
+
if (!(config.dedupSim > 0 && config.dedupSim <= 1)) {
|
|
104
|
+
console.warn("[mega-compact] MEGACOMPACT_DEDUP_SIM must be in (0,1]; using default 0.9");
|
|
105
|
+
config.dedupSim = 0.9;
|
|
106
|
+
}
|
|
107
|
+
if (!(config.crossRepoCosine >= 0 && config.crossRepoCosine <= 1)) {
|
|
108
|
+
console.warn("[mega-compact] MEGACOMPACT_CROSSREPO_COSINE must be in [0,1]; using default 0.9");
|
|
109
|
+
config.crossRepoCosine = 0.9;
|
|
110
|
+
}
|
|
101
111
|
const runtime = new MegaRuntime(config);
|
|
102
112
|
registerEventHandlers(pi, runtime, config);
|
|
103
113
|
registerCommands(pi, runtime, config);
|
|
@@ -159,6 +159,11 @@ export interface MegaConfig {
|
|
|
159
159
|
* to the pre-change OFF state. The single gate lives at the call site
|
|
160
160
|
* (tailResult.ts, config.messageSeparation), not inside buildSeparatedPrompt. */
|
|
161
161
|
messageSeparation: boolean;
|
|
162
|
+
/** Sprint A: Mega↔ithacus bridge — gate the child extension + bridge usage
|
|
163
|
+
* that tie this extension to ithacus's durable compaction. Positive sprint
|
|
164
|
+
* flag, default ON; flag-OFF (=0/`=false`) is byte-identical to pre-bridge
|
|
165
|
+
* behavior (the bridge is only consulted when this is ON). */
|
|
166
|
+
ithacusBridge: boolean;
|
|
162
167
|
/** P3: Cache-aware striping (PLAN_V2 Phase 3). Inserts stability-ordered
|
|
163
168
|
* cache stripes between summaries and thread. Default OFF. */
|
|
164
169
|
cacheStriping: boolean;
|
|
@@ -207,7 +207,7 @@ export function loadConfig(): MegaConfig {
|
|
|
207
207
|
advisoryChannel: envBool("MEGACOMPACT_ADVISORY_CHANNEL", true),
|
|
208
208
|
autoPctTrigger,
|
|
209
209
|
autoInlineK: envFlag("MEGACOMPACT_AUTO_INLINE_K", 3),
|
|
210
|
-
dedupSim:
|
|
210
|
+
dedupSim: envFlag("MEGACOMPACT_DEDUP_SIM", 0.9),
|
|
211
211
|
raptorEnabled: envBool("MEGACOMPACT_RAPTOR_ENABLED", true),
|
|
212
212
|
legacyDurableTrim: envBool("MEGACOMPACT_LEGACY_DURABLE_TRIM", false),
|
|
213
213
|
dbMirror: envBool("MEGACOMPACT_DB_MIRROR", false),
|
|
@@ -216,13 +216,16 @@ export function loadConfig(): MegaConfig {
|
|
|
216
216
|
turnsDbEnabled: envBool("MEGACOMPACT_TURNS_DB", true),
|
|
217
217
|
autoWikiEnabled: envBool("MEGACOMPACT_AUTO_WIKI", true),
|
|
218
218
|
crossRepoEnabled: envBool("MEGACOMPACT_CROSSREPO_ENABLED", true),
|
|
219
|
-
crossRepoCosine:
|
|
219
|
+
crossRepoCosine: envFlag("MEGACOMPACT_CROSSREPO_COSINE", 0.9),
|
|
220
220
|
// 3WF-3: SAME-repo recall cosine floor applied by the 3-source validator to
|
|
221
221
|
// the top winner. SEPARATE from crossRepoCosine (S17, default 0.90, stricter
|
|
222
222
|
// and cross-repo only). This same-repo floor is permissive by default (0.12)
|
|
223
223
|
// so recall still surfaces loosely-relevant within-repo context while
|
|
224
224
|
// rejecting effectively-unrelated hits. Mirrors src/config.ts RECALL_MIN_COSINE.
|
|
225
|
-
|
|
225
|
+
// E1 follow-up (PR #18 review): envFlag (Number.isFinite-guarded) like the
|
|
226
|
+
// dedupSim/crossRepoCosine fix in PR #18 — a typo'd env var must fall back
|
|
227
|
+
// to 0.12, not yield NaN.
|
|
228
|
+
recallMinCosine: envFlag("MEGACOMPACT_RECALL_MIN_COSINE", 0.12),
|
|
226
229
|
memoryAutoReview: envBool("MEGACOMPACT_MEMORY_AUTO_REVIEW", true),
|
|
227
230
|
memoryReviewInterval: envFlag("MEGACOMPACT_MEMORY_REVIEW_INTERVAL", 10),
|
|
228
231
|
recallMaxTokens: envFlag("MEGACOMPACT_RECALL_MAX_TOKENS", 1500),
|
|
@@ -231,6 +234,11 @@ export function loadConfig(): MegaConfig {
|
|
|
231
234
|
// 3WF-1: TriggerGuard — guarantee a staged recall block on every context
|
|
232
235
|
// event even when session_start never fires. Default ON; OFF = byte-identical.
|
|
233
236
|
threeWayFailback: envBool("MEGACOMPACT_THREE_WAY_FAILBACK", true),
|
|
237
|
+
// Sprint A: Mega↔ithacus bridge — gate the child extension + bridge usage.
|
|
238
|
+
// Default ON; OFF (=0/`=false`) = byte-identical pre-bridge behavior.
|
|
239
|
+
// Runtime reads envBool(plain key), mirroring threeWayFailback (plain-write
|
|
240
|
+
// convention, not _DISABLED).
|
|
241
|
+
ithacusBridge: envBool("MEGACOMPACT_ITHACUS_BRIDGE", true),
|
|
234
242
|
// 3WF-2: ThrashGuard re-arm budget as a fraction of effectiveThreshold.
|
|
235
243
|
// 0.10 default (10% of the effective threshold) — see mega-config-types.
|
|
236
244
|
// Clamped to [0.01, 0.5]: below 1% the guard is almost never armed (any
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-mega-compact",
|
|
3
|
-
"version": "0.21.
|
|
3
|
+
"version": "0.21.3",
|
|
4
4
|
"description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "BSD-3-Clause",
|
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* bridge/factory.ts — `createMegaBridge(opts)` implementation.
|
|
3
|
+
*
|
|
4
|
+
* A thin, pi-agnostic wrapper over the engine's compaction / recall / memory /
|
|
5
|
+
* fork / vector APIs. Stores are constructed lazily on first use (a consumer
|
|
6
|
+
* that only calls recallMemories pays no VectorStore cost). Exceptions
|
|
7
|
+
* propagate from every method except `fork` (catches ForkError by design) and
|
|
8
|
+
* `close` (swallows best-effort cleanup), so failures surface in tests.
|
|
9
|
+
*/
|
|
10
|
+
import { compactSession } from "../engine.js";
|
|
11
|
+
import {
|
|
12
|
+
recallAndInline,
|
|
13
|
+
recallAndInlineAsync,
|
|
14
|
+
recallMemoriesAndInline,
|
|
15
|
+
} from "../recall.js";
|
|
16
|
+
import type {
|
|
17
|
+
RecallInjectOptions,
|
|
18
|
+
RecallInjectResult,
|
|
19
|
+
MemoryRecallInjectOptions,
|
|
20
|
+
} from "../recall/types.js";
|
|
21
|
+
import { forkFromConversation, ForkError } from "../fork.js";
|
|
22
|
+
import { createTurnStore } from "../store/turns/index.js";
|
|
23
|
+
import type { TurnStore, TurnEntry } from "../store/turns/types.js";
|
|
24
|
+
import { addMemory } from "../store/sqlite/memories.js";
|
|
25
|
+
import { VectorStore, vectorSearch } from "../vectorStore.js";
|
|
26
|
+
import type { SearchHit } from "../vectorStore.js";
|
|
27
|
+
import { repoKey } from "../store/repoKey.js";
|
|
28
|
+
import type {
|
|
29
|
+
BridgeOptions,
|
|
30
|
+
BridgeCompactInput,
|
|
31
|
+
BridgeCompactResult,
|
|
32
|
+
BridgeRecallOptions,
|
|
33
|
+
BridgeRecallResult,
|
|
34
|
+
BridgeMemoryRecallOptions,
|
|
35
|
+
BridgeMemoryRecallResult,
|
|
36
|
+
BridgeForkOptions,
|
|
37
|
+
BridgeForkResult,
|
|
38
|
+
BridgeCortexOptions,
|
|
39
|
+
BridgeCortexResult,
|
|
40
|
+
BridgeAddMemoryInput,
|
|
41
|
+
BridgeRecordTurnInput,
|
|
42
|
+
MegaBridge,
|
|
43
|
+
} from "./types.js";
|
|
44
|
+
|
|
45
|
+
/** Map a RecallInjectResult to the bridge's slimmer result contract. */
|
|
46
|
+
function mapRecallResult(r: RecallInjectResult): BridgeRecallResult {
|
|
47
|
+
return {
|
|
48
|
+
block: r.block,
|
|
49
|
+
report: r.report,
|
|
50
|
+
hitCount: r.toInject.length,
|
|
51
|
+
empty: r.empty,
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** Map the memoryRecallAndInline tuple result to the bridge contract. */
|
|
56
|
+
function mapMemoryResult(
|
|
57
|
+
r: { empty: boolean; block: string; report: string[] },
|
|
58
|
+
): BridgeMemoryRecallResult {
|
|
59
|
+
return {
|
|
60
|
+
block: r.block,
|
|
61
|
+
report: r.report,
|
|
62
|
+
hitCount: r.report.length,
|
|
63
|
+
empty: r.empty,
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** Map vectorSearch hits to the cortex result contract. */
|
|
68
|
+
function mapCortexHits(hits: SearchHit[], limit: number): BridgeCortexResult {
|
|
69
|
+
const top = hits.slice(0, limit);
|
|
70
|
+
return {
|
|
71
|
+
results: top.map((h) => ({
|
|
72
|
+
checkpointId: h.checkpoint.checkpointId,
|
|
73
|
+
score: h.score,
|
|
74
|
+
summary: h.checkpoint.summary,
|
|
75
|
+
})),
|
|
76
|
+
hitCount: top.length,
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Create a MegaBridge over a single stateDir.
|
|
82
|
+
*
|
|
83
|
+
* The VectorStore and TurnStore are lazy: constructed on first use and cached
|
|
84
|
+
* in closures. The stateDir is retained for memory recall, which needs it
|
|
85
|
+
* directly.
|
|
86
|
+
*/
|
|
87
|
+
export function createMegaBridge(opts: BridgeOptions): MegaBridge {
|
|
88
|
+
const stateDir = opts.stateDir;
|
|
89
|
+
let vectorStore: VectorStore | undefined;
|
|
90
|
+
let turnStore: TurnStore | undefined;
|
|
91
|
+
|
|
92
|
+
const getVectorStore = (): VectorStore => {
|
|
93
|
+
if (!vectorStore) vectorStore = new VectorStore({ stateDir });
|
|
94
|
+
return vectorStore;
|
|
95
|
+
};
|
|
96
|
+
const getTurnStore = (): TurnStore => {
|
|
97
|
+
if (!turnStore) turnStore = createTurnStore({ stateDir });
|
|
98
|
+
return turnStore;
|
|
99
|
+
};
|
|
100
|
+
|
|
101
|
+
return {
|
|
102
|
+
compact(input: BridgeCompactInput): BridgeCompactResult {
|
|
103
|
+
const result = compactSession(
|
|
104
|
+
{
|
|
105
|
+
sessionId: input.sessionId,
|
|
106
|
+
messages: input.messages,
|
|
107
|
+
keepFrom: input.keepFrom,
|
|
108
|
+
summary: input.summary,
|
|
109
|
+
keyDecisions: input.keyDecisions,
|
|
110
|
+
nextSteps: input.nextSteps,
|
|
111
|
+
filesModified: input.filesModified,
|
|
112
|
+
compressionPressure: input.compressionPressure,
|
|
113
|
+
},
|
|
114
|
+
getVectorStore(),
|
|
115
|
+
);
|
|
116
|
+
return {
|
|
117
|
+
skipped: result.skipped,
|
|
118
|
+
deduped: result.deduped,
|
|
119
|
+
summary: result.summary,
|
|
120
|
+
checkpointId: result.checkpointId,
|
|
121
|
+
tokenEstimate: result.tokenEstimate,
|
|
122
|
+
originalTokenEstimate: result.originalTokenEstimate,
|
|
123
|
+
compactedFrom: result.compactedFrom,
|
|
124
|
+
};
|
|
125
|
+
},
|
|
126
|
+
|
|
127
|
+
recallCheckpoints(opts: BridgeRecallOptions): BridgeRecallResult {
|
|
128
|
+
const recallOpts: RecallInjectOptions = {
|
|
129
|
+
sessionId: opts.sessionId,
|
|
130
|
+
query: opts.query,
|
|
131
|
+
limit: opts.limit ?? 3,
|
|
132
|
+
source: "command",
|
|
133
|
+
skipInjected: opts.skipInjected,
|
|
134
|
+
recallMaxTokens: opts.recallMaxTokens,
|
|
135
|
+
};
|
|
136
|
+
return mapRecallResult(recallAndInline(recallOpts, getVectorStore()));
|
|
137
|
+
},
|
|
138
|
+
|
|
139
|
+
async recallMemories(opts: BridgeMemoryRecallOptions): Promise<BridgeMemoryRecallResult> {
|
|
140
|
+
const memOpts: MemoryRecallInjectOptions = {
|
|
141
|
+
query: opts.query,
|
|
142
|
+
stateDir,
|
|
143
|
+
limit: opts.limit,
|
|
144
|
+
minSimilarity: opts.minSimilarity,
|
|
145
|
+
crossRepo: opts.crossRepo,
|
|
146
|
+
crossRepoCosine: opts.crossRepoCosine,
|
|
147
|
+
recallMaxTokens: opts.recallMaxTokens,
|
|
148
|
+
};
|
|
149
|
+
const r = await recallMemoriesAndInline(memOpts);
|
|
150
|
+
return mapMemoryResult(r);
|
|
151
|
+
},
|
|
152
|
+
|
|
153
|
+
async recallAndInlineAsync(opts: BridgeRecallOptions): Promise<BridgeRecallResult> {
|
|
154
|
+
const recallOpts: RecallInjectOptions = {
|
|
155
|
+
sessionId: opts.sessionId,
|
|
156
|
+
query: opts.query,
|
|
157
|
+
limit: opts.limit ?? 3,
|
|
158
|
+
source: "command",
|
|
159
|
+
skipInjected: opts.skipInjected,
|
|
160
|
+
recallMaxTokens: opts.recallMaxTokens,
|
|
161
|
+
};
|
|
162
|
+
const r = await recallAndInlineAsync(recallOpts, getVectorStore());
|
|
163
|
+
return mapRecallResult(r);
|
|
164
|
+
},
|
|
165
|
+
|
|
166
|
+
fork(opts: BridgeForkOptions): BridgeForkResult {
|
|
167
|
+
try {
|
|
168
|
+
const outcome = forkFromConversation(
|
|
169
|
+
getTurnStore(),
|
|
170
|
+
opts.parentConversationId,
|
|
171
|
+
opts.turnIndex,
|
|
172
|
+
);
|
|
173
|
+
return {
|
|
174
|
+
childConversationId: outcome.childConversationId,
|
|
175
|
+
checkpointIds: outcome.checkpointIds,
|
|
176
|
+
forkTurnIndex: opts.turnIndex,
|
|
177
|
+
};
|
|
178
|
+
} catch (e) {
|
|
179
|
+
if (e instanceof ForkError) {
|
|
180
|
+
return { error: e.code };
|
|
181
|
+
}
|
|
182
|
+
throw e;
|
|
183
|
+
}
|
|
184
|
+
},
|
|
185
|
+
|
|
186
|
+
cortexQuery(opts: BridgeCortexOptions): BridgeCortexResult {
|
|
187
|
+
const limit = opts.limit ?? 3;
|
|
188
|
+
const scope = opts.repo ?? repoKey(stateDir);
|
|
189
|
+
const hits = vectorSearch(getVectorStore(), scope, opts.query, limit);
|
|
190
|
+
return mapCortexHits(hits, limit);
|
|
191
|
+
},
|
|
192
|
+
|
|
193
|
+
addMemory(input: BridgeAddMemoryInput): number | void {
|
|
194
|
+
// repo === null ⇒ stateDir-scoped durable memory (matches recallMemories).
|
|
195
|
+
return addMemory(
|
|
196
|
+
{
|
|
197
|
+
kind: input.kind,
|
|
198
|
+
content: input.content,
|
|
199
|
+
tags: input.tags,
|
|
200
|
+
category: input.category,
|
|
201
|
+
},
|
|
202
|
+
null,
|
|
203
|
+
stateDir,
|
|
204
|
+
);
|
|
205
|
+
},
|
|
206
|
+
|
|
207
|
+
recordTurn(input: BridgeRecordTurnInput): void {
|
|
208
|
+
const turn: TurnEntry = {
|
|
209
|
+
conversationId: input.conversationId,
|
|
210
|
+
sessionId: input.sessionId,
|
|
211
|
+
turnIndex: input.turnIndex,
|
|
212
|
+
role: (input.role as TurnEntry["role"]) ?? "assistant",
|
|
213
|
+
endedAt: input.endedAt ?? Date.now(),
|
|
214
|
+
ctxTokens: input.ctxTokens,
|
|
215
|
+
ctxPercent: input.ctxPercent,
|
|
216
|
+
model: input.model,
|
|
217
|
+
};
|
|
218
|
+
getTurnStore().asWriter().appendTurn(turn);
|
|
219
|
+
},
|
|
220
|
+
|
|
221
|
+
close(): void {
|
|
222
|
+
if (turnStore) {
|
|
223
|
+
try {
|
|
224
|
+
turnStore.close();
|
|
225
|
+
} catch {
|
|
226
|
+
/* best-effort */
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
},
|
|
230
|
+
};
|
|
231
|
+
}
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* bridge/types.ts — contract types for the bidirectional mega-compact bridge.
|
|
3
|
+
*
|
|
4
|
+
* A pi-agnostic, unit-testable adapter surface that wraps the engine's
|
|
5
|
+
* compaction / recall / memory / fork / vector APIs behind one factory so an
|
|
6
|
+
* external host (ithacus) can drive them without importing pi-runtime types.
|
|
7
|
+
* Every type here mirrors a real engine signature (see factory.ts for the
|
|
8
|
+
* wiring).
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import type { EngineMessage } from "../types.js";
|
|
12
|
+
|
|
13
|
+
/** Same shape as the engine's internal message. Identity re-export. */
|
|
14
|
+
export type BridgeMessage = EngineMessage;
|
|
15
|
+
|
|
16
|
+
/** Bridge construction options. */
|
|
17
|
+
export interface BridgeOptions {
|
|
18
|
+
stateDir: string;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/** Input to compact a message slice into a checkpoint. */
|
|
22
|
+
export interface BridgeCompactInput {
|
|
23
|
+
sessionId: string;
|
|
24
|
+
messages: BridgeMessage[];
|
|
25
|
+
keepFrom?: number;
|
|
26
|
+
summary?: string;
|
|
27
|
+
keyDecisions?: string[];
|
|
28
|
+
nextSteps?: string[];
|
|
29
|
+
filesModified?: string[];
|
|
30
|
+
compressionPressure?: number;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** Useful subset of CompactResult. */
|
|
34
|
+
export interface BridgeCompactResult {
|
|
35
|
+
skipped: boolean;
|
|
36
|
+
deduped: boolean;
|
|
37
|
+
summary: string;
|
|
38
|
+
checkpointId?: string;
|
|
39
|
+
tokenEstimate: number;
|
|
40
|
+
originalTokenEstimate?: number;
|
|
41
|
+
compactedFrom?: number;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** Options for checkpoint recall (per-session). */
|
|
45
|
+
export interface BridgeRecallOptions {
|
|
46
|
+
sessionId: string;
|
|
47
|
+
query: string;
|
|
48
|
+
limit?: number;
|
|
49
|
+
recallMaxTokens?: number;
|
|
50
|
+
skipInjected?: boolean;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** Mapped from RecallInjectResult. */
|
|
54
|
+
export interface BridgeRecallResult {
|
|
55
|
+
block: string;
|
|
56
|
+
report: string[];
|
|
57
|
+
hitCount: number;
|
|
58
|
+
empty: boolean;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** Options for durable memory recall (stateDir-scoped, no sessionId). */
|
|
62
|
+
export interface BridgeMemoryRecallOptions {
|
|
63
|
+
query: string;
|
|
64
|
+
limit?: number;
|
|
65
|
+
minSimilarity?: number;
|
|
66
|
+
crossRepo?: boolean;
|
|
67
|
+
crossRepoCosine?: number;
|
|
68
|
+
recallMaxTokens?: number;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export interface BridgeMemoryRecallResult {
|
|
72
|
+
block: string;
|
|
73
|
+
report: string[];
|
|
74
|
+
hitCount: number;
|
|
75
|
+
empty: boolean;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** Options to fork a child conversation off a parent turn. */
|
|
79
|
+
export interface BridgeForkOptions {
|
|
80
|
+
parentConversationId: string;
|
|
81
|
+
turnIndex: number;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** Fork result: success variant OR a graceful error variant. */
|
|
85
|
+
export interface BridgeForkSuccess {
|
|
86
|
+
childConversationId: string;
|
|
87
|
+
checkpointIds: string[];
|
|
88
|
+
forkTurnIndex: number;
|
|
89
|
+
}
|
|
90
|
+
export interface BridgeForkError {
|
|
91
|
+
error: "TURN_NOT_FOUND" | "NO_RECALL";
|
|
92
|
+
}
|
|
93
|
+
export type BridgeForkResult = BridgeForkSuccess | BridgeForkError;
|
|
94
|
+
|
|
95
|
+
/** Options for a top-k corpus / vector query. */
|
|
96
|
+
export interface BridgeCortexOptions {
|
|
97
|
+
query: string;
|
|
98
|
+
limit?: number;
|
|
99
|
+
repo?: string;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
export interface BridgeCortexResult {
|
|
103
|
+
results: Array<{ checkpointId: string; score: number; summary?: string }>;
|
|
104
|
+
hitCount: number;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** Input to persist a durable memory. */
|
|
108
|
+
export interface BridgeAddMemoryInput {
|
|
109
|
+
content: string;
|
|
110
|
+
kind?: string;
|
|
111
|
+
tags?: string[];
|
|
112
|
+
category?: string;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/** Input to record a turn fact. */
|
|
116
|
+
export interface BridgeRecordTurnInput {
|
|
117
|
+
conversationId: string;
|
|
118
|
+
sessionId: string;
|
|
119
|
+
turnIndex: number;
|
|
120
|
+
role?: string;
|
|
121
|
+
endedAt?: number;
|
|
122
|
+
ctxTokens?: number;
|
|
123
|
+
ctxPercent?: number;
|
|
124
|
+
model?: string;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** The bridge surface exposed to the host. */
|
|
128
|
+
export interface MegaBridge {
|
|
129
|
+
compact(input: BridgeCompactInput): BridgeCompactResult;
|
|
130
|
+
recallCheckpoints(opts: BridgeRecallOptions): BridgeRecallResult;
|
|
131
|
+
recallMemories(opts: BridgeMemoryRecallOptions): Promise<BridgeMemoryRecallResult>;
|
|
132
|
+
recallAndInlineAsync(opts: BridgeRecallOptions): Promise<BridgeRecallResult>;
|
|
133
|
+
fork(opts: BridgeForkOptions): BridgeForkResult;
|
|
134
|
+
cortexQuery(opts: BridgeCortexOptions): BridgeCortexResult;
|
|
135
|
+
addMemory(input: BridgeAddMemoryInput): number | void;
|
|
136
|
+
recordTurn(input: BridgeRecordTurnInput): void;
|
|
137
|
+
close(): void;
|
|
138
|
+
}
|
package/src/bridge.ts
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* bridge.ts — public barrel for the mega-compact bidirectional bridge.
|
|
3
|
+
*
|
|
4
|
+
* Hosts import only this file. The factory and all contracts live under
|
|
5
|
+
* src/bridge/ (kept thin per the delegate-shell pattern).
|
|
6
|
+
*/
|
|
7
|
+
export type {
|
|
8
|
+
MegaBridge,
|
|
9
|
+
BridgeOptions,
|
|
10
|
+
BridgeMessage,
|
|
11
|
+
BridgeCompactInput,
|
|
12
|
+
BridgeCompactResult,
|
|
13
|
+
BridgeRecallOptions,
|
|
14
|
+
BridgeRecallResult,
|
|
15
|
+
BridgeMemoryRecallOptions,
|
|
16
|
+
BridgeMemoryRecallResult,
|
|
17
|
+
BridgeForkOptions,
|
|
18
|
+
BridgeForkResult,
|
|
19
|
+
BridgeCortexOptions,
|
|
20
|
+
BridgeCortexResult,
|
|
21
|
+
BridgeAddMemoryInput,
|
|
22
|
+
BridgeRecordTurnInput,
|
|
23
|
+
} from "./bridge/types.js";
|
|
24
|
+
export { createMegaBridge } from "./bridge/factory.js";
|
package/src/config.ts
CHANGED
|
@@ -152,9 +152,18 @@ export const NEW_UI = (): boolean => ragEnabled("MEGACOMPACT_NEW_UI");
|
|
|
152
152
|
// effectively-unrelated hits. Call-time read so tests can set the env per-test.
|
|
153
153
|
// ---------------------------------------------------------------------------
|
|
154
154
|
|
|
155
|
-
/**
|
|
156
|
-
|
|
157
|
-
|
|
155
|
+
/**
|
|
156
|
+
* Same-repo recall cosine floor: top winner must be >= this to be injected.
|
|
157
|
+
*
|
|
158
|
+
* E1 follow-up (PR #18 review): NaN-safe + clamped to [0,1]. A typo'd env var
|
|
159
|
+
* yielded NaN before; `cosine < NaN` is false, which disabled gate 1 entirely
|
|
160
|
+
* (every candidate passed). Non-finite falls back to 0.12; out-of-range clamps.
|
|
161
|
+
*/
|
|
162
|
+
export const RECALL_MIN_COSINE = (): number => {
|
|
163
|
+
const n = Number(process.env.MEGACOMPACT_RECALL_MIN_COSINE ?? "0.12");
|
|
164
|
+
if (!Number.isFinite(n)) return 0.12;
|
|
165
|
+
return Math.min(1, Math.max(0, n));
|
|
166
|
+
};
|
|
158
167
|
|
|
159
168
|
// ---------------------------------------------------------------------------
|
|
160
169
|
// Vector-cortex flags + breaker constants (VC0A+). Positive sprint flags,
|
package/src/dedup/l1-minhash.ts
CHANGED
|
@@ -21,6 +21,7 @@ export const SHINGLE_SIZE = 5; // char 5-grams
|
|
|
21
21
|
const MAX_SHINGLES = 50_000; // QA #7/#15 complexity cap
|
|
22
22
|
const SEED = 0xdeadbeef;
|
|
23
23
|
const P = 2147483647; // 2^31 - 1, Mersenne prime
|
|
24
|
+
const PBigInt = 2147483647n; // BigInt twin for overflow-safe modular reduction
|
|
24
25
|
|
|
25
26
|
/** Per-index universal-hashing coefficients, derived deterministically from SEED. */
|
|
26
27
|
function coeffA(i: number): number {
|
|
@@ -69,10 +70,10 @@ export function minhashSignature(text: string): number[] {
|
|
|
69
70
|
const b = coeffB(i);
|
|
70
71
|
let min = P;
|
|
71
72
|
for (const x of grams) {
|
|
72
|
-
// (a*x + b) mod p
|
|
73
|
-
//
|
|
74
|
-
|
|
75
|
-
const h = (
|
|
73
|
+
// (a*x + b) mod p. a, x < 2^31 so a*x < 2^62 — EXCEEDS 2^53. The naive
|
|
74
|
+
// (a*(x%P))%P loses precision (verified: a=x=p-1 → lossy 2147483644 vs
|
|
75
|
+
// exact 1). BigInt is correct + cheap (~5ms per signature).
|
|
76
|
+
const h = Number((BigInt(a) * BigInt(x % P) + BigInt(b)) % PBigInt);
|
|
76
77
|
if (h < min) min = h;
|
|
77
78
|
}
|
|
78
79
|
sig[i] = min;
|
package/src/recall/validator.ts
CHANGED
|
@@ -113,7 +113,14 @@ export function validateRecall(
|
|
|
113
113
|
// No comparable cosine available => cannot clear a cosine gate.
|
|
114
114
|
continue;
|
|
115
115
|
}
|
|
116
|
-
|
|
116
|
+
// E1 follow-up (PR #18 review): NaN/Infinity must NEVER clear the floor.
|
|
117
|
+
// `NaN < floor` is false, so an unguarded comparison lets a NaN cosine
|
|
118
|
+
// PASS gate 1 and inject — one NaN source poisons the whole 3WF-3
|
|
119
|
+
// quorum. Reject non-finite scores explicitly; the candidate is skipped
|
|
120
|
+
// and, if all fail, the provenance floor ("no recall") is returned —
|
|
121
|
+
// never a zero-score injection. The default TrigramEmbedder cannot
|
|
122
|
+
// produce NaN (zero-norm guard), but a BYO localhost embedder can.
|
|
123
|
+
if (!Number.isFinite(cosine) || cosine < floor) continue;
|
|
117
124
|
|
|
118
125
|
// Gate 2: not already resident in the live window.
|
|
119
126
|
if (liveVecs.length > 0) {
|