pi-mega-compact 0.13.0 → 0.13.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/extensions/dashboard-server/api-contracts/embedder-health.js +11 -0
- package/dist/extensions/dashboard-server/api-contracts/endpoints/registry.js +225 -0
- package/dist/extensions/dashboard-server/api-contracts/endpoints/types.js +1 -0
- package/dist/extensions/dashboard-server/api-contracts/endpoints.js +5 -224
- package/dist/extensions/dashboard-server/api-contracts/raptor.js +8 -0
- package/dist/extensions/dashboard-server/api-contracts.test/_helpers.js +32 -0
- package/dist/extensions/dashboard-server/routes-cache.test/_helpers.js +48 -0
- package/dist/extensions/dashboard-server/routes-embedder-health.js +67 -0
- package/dist/extensions/dashboard-server/routes-raptor.js +88 -0
- package/dist/extensions/dashboard-server/routes-setup.js +1 -1
- package/dist/extensions/dashboard-server/routes.js +2 -0
- package/dist/extensions/dashboard-server/server.js +5 -1
- package/dist/extensions/mega-cache-replay.test/_helpers.js +175 -0
- package/dist/extensions/mega-commands/dataCommands.js +151 -0
- package/dist/extensions/mega-commands/helpers.js +61 -0
- package/dist/extensions/mega-commands/historyCommands.js +97 -0
- package/dist/extensions/mega-commands/setupCommand.js +200 -0
- package/dist/extensions/mega-commands.js +14 -499
- package/dist/extensions/mega-compact-s38.test/_helpers.js +194 -0
- package/dist/extensions/mega-compact.js +12 -2
- package/dist/extensions/mega-compact.test/_helpers.js +225 -0
- package/dist/extensions/mega-events/agent-handlers/agentEndHandler.js +211 -0
- package/dist/extensions/mega-events/agent-handlers/turnEndHandler/cacheStripe.js +49 -0
- package/dist/extensions/mega-events/agent-handlers/turnEndHandler/contextHealth.js +33 -0
- package/dist/extensions/mega-events/agent-handlers/turnEndHandler/errorRetry.js +387 -0
- package/dist/extensions/mega-events/agent-handlers/turnEndHandler/event.js +7 -0
- package/dist/extensions/mega-events/agent-handlers/turnEndHandler/gameScoring.js +55 -0
- package/dist/extensions/mega-events/agent-handlers/turnEndHandler/lengthStop.js +12 -0
- package/dist/extensions/mega-events/agent-handlers/turnEndHandler/memoryReview.js +22 -0
- package/dist/extensions/mega-events/agent-handlers/turnEndHandler/recordTurnRow.js +38 -0
- package/dist/extensions/mega-events/agent-handlers/turnEndHandler.js +36 -0
- package/dist/extensions/mega-events/agent-handlers.js +4 -799
- package/dist/extensions/mega-events/context-handler/afterCompact.js +163 -0
- package/dist/extensions/mega-events/context-handler/messageText.js +12 -0
- package/dist/extensions/mega-events/context-handler/tailResult.js +46 -0
- package/dist/extensions/mega-events/context-handler.js +11 -198
- package/dist/extensions/mega-events/separated-prompt.test/_helpers.js +25 -0
- package/dist/src/config.js +2 -0
- package/dist/src/dedup-engine.test/_helpers.js +55 -0
- package/dist/src/e2e.test/_helpers.js +52 -0
- package/dist/src/embedder.js +1 -0
- package/dist/src/httpEmbedder.js +6 -0
- package/dist/src/hyde.js +146 -0
- package/dist/src/importance.test/_helpers.js +5 -0
- package/dist/src/memoryGraph/sources/checkpoints.js +78 -0
- package/dist/src/memoryGraph/sources/edges.js +17 -0
- package/dist/src/memoryGraph/sources/flags.js +22 -0
- package/dist/src/memoryGraph/sources/helpers.js +14 -0
- package/dist/src/memoryGraph/sources/memories.js +88 -0
- package/dist/src/memoryGraph/sources/raptor.js +26 -0
- package/dist/src/memoryGraph/sources/turns.js +176 -0
- package/dist/src/memoryGraph/sources.js +15 -441
- package/dist/src/memoryGraph-gates.test/_helpers.js +120 -0
- package/dist/src/ratio.bench.test/_helpers.js +198 -0
- package/dist/src/recall/async.js +112 -0
- package/dist/src/recall/format.js +149 -0
- package/dist/src/recall/memory.js +62 -0
- package/dist/src/recall/reformulate.js +82 -0
- package/dist/src/recall/sync.js +169 -0
- package/dist/src/recall/types.js +1 -0
- package/dist/src/recall.js +13 -530
- package/dist/src/store/sqlite/schema/core.js +232 -0
- package/dist/src/store/sqlite/schema/game.js +65 -0
- package/dist/src/store/sqlite/schema/plan-v2.js +107 -0
- package/dist/src/store/sqlite/schema/turns.js +59 -0
- package/dist/src/store/sqlite/schema.js +8 -441
- package/dist/src/store/turns/contract-compliance.test/_helpers.js +21 -0
- package/dist/src/store/turns/sqlite-store/admin.js +171 -0
- package/dist/src/store/turns/sqlite-store/ctx.js +1 -0
- package/dist/src/store/turns/sqlite-store/read.js +111 -0
- package/dist/src/store/turns/sqlite-store/rows.js +42 -0
- package/dist/src/store/turns/sqlite-store/write.js +102 -0
- package/dist/src/store/turns/sqlite-store.js +31 -367
- package/dist/src/vectorStore/class.js +356 -0
- package/dist/src/vectorStore/hash.js +10 -0
- package/dist/src/vectorStore/types.js +2 -0
- package/dist/src/vectorStore.js +8 -359
- package/dist/src/vectorStore.test/_helpers.js +17 -0
- package/extensions/dashboard-client/dist/assets/{CacheTab-BZNnKhh5.js → CacheTab-DPRHjYZ0.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{CacheTab-BZNnKhh5.js.map → CacheTab-DPRHjYZ0.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{EventsTab-CzF5L42U.js → EventsTab-CgwYq-Qf.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{EventsTab-CzF5L42U.js.map → EventsTab-CgwYq-Qf.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{HealthTab-BLpidMrX.js → HealthTab-BgOCjVzR.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{HealthTab-BLpidMrX.js.map → HealthTab-BgOCjVzR.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/MaintenanceTab-Yw1yFpsy.js +2 -0
- package/extensions/dashboard-client/dist/assets/MaintenanceTab-Yw1yFpsy.js.map +1 -0
- package/extensions/dashboard-client/dist/assets/MemoryMapTab-yHUX7Yzi.js +2 -0
- package/extensions/dashboard-client/dist/assets/MemoryMapTab-yHUX7Yzi.js.map +1 -0
- package/extensions/dashboard-client/dist/assets/{MetricsTab-B0PCysqs.js → MetricsTab-CcH9fP8P.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{MetricsTab-B0PCysqs.js.map → MetricsTab-CcH9fP8P.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{OverviewTab-DsOhqFu4.js → OverviewTab-BOs8ZNev.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{OverviewTab-DsOhqFu4.js.map → OverviewTab-BOs8ZNev.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{ReposTab-B3YKYuOf.js → ReposTab-B-MfSPvg.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{ReposTab-B3YKYuOf.js.map → ReposTab-B-MfSPvg.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{SessionsTab-B96FWcrC.js → SessionsTab-BmeN649Q.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{SessionsTab-B96FWcrC.js.map → SessionsTab-BmeN649Q.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/SetupTab-nOknoph5.js +2 -0
- package/extensions/dashboard-client/dist/assets/SetupTab-nOknoph5.js.map +1 -0
- package/extensions/dashboard-client/dist/assets/{TimeSavedCard-BfkUVTvA.js → TimeSavedCard-DnPg0L0z.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{TimeSavedCard-BfkUVTvA.js.map → TimeSavedCard-DnPg0L0z.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{TopicsTab-DLRsZmXJ.js → TopicsTab-BODHOg_b.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{TopicsTab-DLRsZmXJ.js.map → TopicsTab-BODHOg_b.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{TurnsTab-DKGU2Bf6.js → TurnsTab-BK0I7hKr.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{TurnsTab-DKGU2Bf6.js.map → TurnsTab-BK0I7hKr.js.map} +1 -1
- package/extensions/dashboard-client/dist/assets/{index-D-lPLRWB.js → index-mOlljTFI.js} +10 -10
- package/extensions/dashboard-client/dist/assets/index-mOlljTFI.js.map +1 -0
- package/extensions/dashboard-client/dist/assets/{useSSE-ClbbcgIp.js → useSSE-BYOtv1-3.js} +2 -2
- package/extensions/dashboard-client/dist/assets/{useSSE-ClbbcgIp.js.map → useSSE-BYOtv1-3.js.map} +1 -1
- package/extensions/dashboard-client/dist/index.html +1 -1
- package/extensions/dashboard-client/src/App.tsx +0 -4
- package/extensions/dashboard-client/src/api/client.ts +16 -0
- package/extensions/dashboard-client/src/tabs/MaintenanceTab/ActionsCard.tsx +178 -0
- package/extensions/dashboard-client/src/tabs/MaintenanceTab/DbStatsCard.tsx +140 -0
- package/extensions/dashboard-client/src/tabs/MaintenanceTab/DebugBundleCard.tsx +109 -0
- package/extensions/dashboard-client/src/tabs/MaintenanceTab/HealthMitigationCard.tsx +98 -0
- package/extensions/dashboard-client/src/tabs/MaintenanceTab/SchemaHealthCard.tsx +108 -0
- package/extensions/dashboard-client/src/tabs/MaintenanceTab.tsx +12 -598
- package/extensions/dashboard-client/src/tabs/MemoryMapTab/MemoryMapView.tsx +476 -0
- package/extensions/dashboard-client/src/tabs/MemoryMapTab/RaptorTreeView.tsx +163 -0
- package/extensions/dashboard-client/src/tabs/MemoryMapTab.tsx +56 -466
- package/extensions/dashboard-client/src/tabs/SetupTab/EmbedderHealthCard.tsx +109 -0
- package/extensions/dashboard-client/src/tabs/SetupTab/EmbedderSetup.tsx +468 -0
- package/extensions/dashboard-client/src/tabs/SetupTab.tsx +23 -484
- package/extensions/dashboard-server/api-contracts/embedder-health.ts +32 -0
- package/extensions/dashboard-server/api-contracts/endpoints/registry.ts +360 -0
- package/extensions/dashboard-server/api-contracts/endpoints/types.ts +350 -0
- package/extensions/dashboard-server/api-contracts/endpoints.ts +27 -648
- package/extensions/dashboard-server/api-contracts/index.ts +6 -0
- package/extensions/dashboard-server/api-contracts/raptor.ts +49 -0
- package/extensions/dashboard-server/api-contracts.test/_helpers.ts +47 -0
- package/extensions/dashboard-server/routes-cache.test/_helpers.ts +94 -0
- package/extensions/dashboard-server/routes-embedder-health.ts +77 -0
- package/extensions/dashboard-server/routes-raptor.ts +128 -0
- package/extensions/dashboard-server/routes-setup.ts +1 -1
- package/extensions/dashboard-server/routes.ts +2 -0
- package/extensions/dashboard-server/server.ts +4 -0
- package/extensions/mega-cache-replay.test/_helpers.ts +188 -0
- package/extensions/mega-commands/dataCommands.ts +169 -0
- package/extensions/mega-commands/helpers.ts +62 -0
- package/extensions/mega-commands/historyCommands.ts +117 -0
- package/extensions/mega-commands/setupCommand.ts +235 -0
- package/extensions/mega-commands.ts +25 -538
- package/extensions/mega-compact-s38.test/_helpers.ts +201 -0
- package/extensions/mega-compact.test/_helpers.ts +237 -0
- package/extensions/mega-compact.ts +12 -2
- package/extensions/mega-events/agent-handlers/agentEndHandler.ts +238 -0
- package/extensions/mega-events/agent-handlers/turnEndHandler/cacheStripe.ts +74 -0
- package/extensions/mega-events/agent-handlers/turnEndHandler/contextHealth.ts +50 -0
- package/extensions/mega-events/agent-handlers/turnEndHandler/errorRetry.ts +421 -0
- package/extensions/mega-events/agent-handlers/turnEndHandler/event.ts +16 -0
- package/extensions/mega-events/agent-handlers/turnEndHandler/gameScoring.ts +73 -0
- package/extensions/mega-events/agent-handlers/turnEndHandler/lengthStop.ts +29 -0
- package/extensions/mega-events/agent-handlers/turnEndHandler/memoryReview.ts +42 -0
- package/extensions/mega-events/agent-handlers/turnEndHandler/recordTurnRow.ts +61 -0
- package/extensions/mega-events/agent-handlers/turnEndHandler.ts +64 -0
- package/extensions/mega-events/agent-handlers.ts +15 -861
- package/extensions/mega-events/context-handler/afterCompact.ts +213 -0
- package/extensions/mega-events/context-handler/messageText.ts +20 -0
- package/extensions/mega-events/context-handler/tailResult.ts +63 -0
- package/extensions/mega-events/context-handler.ts +17 -235
- package/extensions/mega-events/separated-prompt.test/_helpers.ts +35 -0
- package/package.json +1 -1
- package/src/config.ts +4 -0
- package/src/dedup-engine.test/_helpers.ts +79 -0
- package/src/e2e.test/_helpers.ts +59 -0
- package/src/embedder.ts +4 -0
- package/src/httpEmbedder.ts +7 -0
- package/src/hyde.ts +162 -0
- package/src/importance.test/_helpers.ts +5 -0
- package/src/memoryGraph/sources/checkpoints.ts +99 -0
- package/src/memoryGraph/sources/edges.ts +23 -0
- package/src/memoryGraph/sources/flags.ts +26 -0
- package/src/memoryGraph/sources/helpers.ts +13 -0
- package/src/memoryGraph/sources/memories.ts +108 -0
- package/src/memoryGraph/sources/raptor.ts +37 -0
- package/src/memoryGraph/sources/turns.ts +226 -0
- package/src/memoryGraph/sources.ts +19 -544
- package/src/memoryGraph-gates.test/_helpers.ts +164 -0
- package/src/ratio.bench.test/_helpers.ts +268 -0
- package/src/recall/async.ts +142 -0
- package/src/recall/format.ts +173 -0
- package/src/recall/memory.ts +90 -0
- package/src/recall/reformulate.ts +111 -0
- package/src/recall/sync.ts +196 -0
- package/src/recall/types.ts +63 -0
- package/src/recall.ts +25 -712
- package/src/store/sqlite/schema/core.ts +233 -0
- package/src/store/sqlite/schema/game.ts +66 -0
- package/src/store/sqlite/schema/plan-v2.ts +108 -0
- package/src/store/sqlite/schema/turns.ts +60 -0
- package/src/store/sqlite/schema.ts +13 -441
- package/src/store/turns/contract-compliance.test/_helpers.ts +38 -0
- package/src/store/turns/sqlite-store/admin.ts +249 -0
- package/src/store/turns/sqlite-store/ctx.ts +14 -0
- package/src/store/turns/sqlite-store/read.ts +174 -0
- package/src/store/turns/sqlite-store/rows.ts +55 -0
- package/src/store/turns/sqlite-store/write.ts +193 -0
- package/src/store/turns/sqlite-store.ts +33 -514
- package/src/vectorStore/class.ts +433 -0
- package/src/vectorStore/hash.ts +11 -0
- package/src/vectorStore/types.ts +58 -0
- package/src/vectorStore.test/_helpers.ts +20 -0
- package/src/vectorStore.ts +13 -489
- package/extensions/dashboard-client/dist/assets/ConfigTab-DG0WkuuG.js +0 -2
- package/extensions/dashboard-client/dist/assets/ConfigTab-DG0WkuuG.js.map +0 -1
- package/extensions/dashboard-client/dist/assets/MaintenanceTab-BQF7W7xX.js +0 -2
- package/extensions/dashboard-client/dist/assets/MaintenanceTab-BQF7W7xX.js.map +0 -1
- package/extensions/dashboard-client/dist/assets/MemoryMapTab-CPtHLv0A.js +0 -2
- package/extensions/dashboard-client/dist/assets/MemoryMapTab-CPtHLv0A.js.map +0 -1
- package/extensions/dashboard-client/dist/assets/SetupTab-hbzQ_NZv.js +0 -2
- package/extensions/dashboard-client/dist/assets/SetupTab-hbzQ_NZv.js.map +0 -1
- package/extensions/dashboard-client/dist/assets/index-D-lPLRWB.js.map +0 -1
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* context-handler/afterCompact.ts — post-compaction DB persistence + maintenance.
|
|
3
|
+
*
|
|
4
|
+
* Extracted from context-handler.ts (delegate-shell split). Runs after a live
|
|
5
|
+
* compaction: writes the checkpoint_epoch row (deterministic nonce), stamps turn
|
|
6
|
+
* epochs, rebuilds the auto-categorizing wiki, seeds the topic model from
|
|
7
|
+
* raw_transcript, and fire-and-forgets the dedup pipeline. All best-effort +
|
|
8
|
+
* non-fatal — a failure never breaks the agent loop.
|
|
9
|
+
*/
|
|
10
|
+
import {
|
|
11
|
+
openStore,
|
|
12
|
+
writeCheckpointEpoch,
|
|
13
|
+
type CheckpointEpoch,
|
|
14
|
+
} from "../../../src/store/sqlite.js";
|
|
15
|
+
import { epochIdFor } from "../../../src/mirror/epoch.js";
|
|
16
|
+
import type { MegaRuntime } from "../../mega-runtime.js";
|
|
17
|
+
import { stampTurnsEpochFor } from "../../mega-turn-store.js";
|
|
18
|
+
import { TurnsConfig } from "../../../src/config/turns.js";
|
|
19
|
+
import { openTurnStore } from "../../../src/store/turns/connection.js";
|
|
20
|
+
import {
|
|
21
|
+
buildTopicModel,
|
|
22
|
+
createTopicStore,
|
|
23
|
+
bumpWikiCompactCounter,
|
|
24
|
+
} from "../../../src/topics/index.js";
|
|
25
|
+
import { TrigramEmbedder } from "../../../src/embedder.js";
|
|
26
|
+
import type { EmbeddedChunk } from "../../../src/topics/types.js";
|
|
27
|
+
import type { MegaConfig } from "../../mega-config.js";
|
|
28
|
+
|
|
29
|
+
/** Shape of the compact result consumed by the epoch/maintenance writes. */
|
|
30
|
+
interface CompactResult {
|
|
31
|
+
checkpointId?: string;
|
|
32
|
+
compactedFrom: number;
|
|
33
|
+
summary: string;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Persist the checkpoint_epoch, stamp turns, rebuild the auto-wiki, seed the
|
|
38
|
+
* topic model, and dedup — gated on dbMirror. Non-fatal end-to-end.
|
|
39
|
+
*/
|
|
40
|
+
export async function persistEpochAndMaintain(
|
|
41
|
+
runtime: MegaRuntime,
|
|
42
|
+
config: MegaConfig,
|
|
43
|
+
ran: { result: CompactResult },
|
|
44
|
+
): Promise<void> {
|
|
45
|
+
// S27 DB-mirror: write checkpoint_epoch with deterministic nonce.
|
|
46
|
+
// This makes the cache key stable across identical compactions.
|
|
47
|
+
if (config.dbMirror) {
|
|
48
|
+
try {
|
|
49
|
+
const db = openStore(runtime.currentStateDir);
|
|
50
|
+
const cpId = ran.result.checkpointId ?? `epoch-${Date.now()}`;
|
|
51
|
+
const epoch: CheckpointEpoch = {
|
|
52
|
+
epochId: epochIdFor(cpId),
|
|
53
|
+
sessionId: runtime.rt.sessionId,
|
|
54
|
+
startedSeq: 0,
|
|
55
|
+
committedSeq: ran.result.compactedFrom,
|
|
56
|
+
checkpointId: cpId,
|
|
57
|
+
cutIndex: ran.result.compactedFrom,
|
|
58
|
+
summaryMessageText: ran.result.summary,
|
|
59
|
+
createdAt: Date.now(),
|
|
60
|
+
};
|
|
61
|
+
writeCheckpointEpoch(db, epoch);
|
|
62
|
+
// S50B: link this session's turns to the epoch that just compacted
|
|
63
|
+
// them (compression-by-conversation-epoch metrics). Isolated-store
|
|
64
|
+
// only; best-effort + non-fatal.
|
|
65
|
+
try {
|
|
66
|
+
stampTurnsEpochFor(
|
|
67
|
+
config,
|
|
68
|
+
runtime.rt.sessionId,
|
|
69
|
+
epoch.epochId,
|
|
70
|
+
runtime.currentStateDir,
|
|
71
|
+
);
|
|
72
|
+
} catch {
|
|
73
|
+
/* non-fatal: epoch stamping never breaks compaction */
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
// S51B: auto-categorizing wiki rebuild — every Nth compaction, derived
|
|
77
|
+
// from real context_chunks embeddings. Isolated-store only, gated on
|
|
78
|
+
// AUTO_WIKI_ENABLED; best-effort + non-fatal (never breaks compaction).
|
|
79
|
+
// Fire regardless of dbMirror — context_chunks is the isolated store.
|
|
80
|
+
// Uses the already-open db when inside the dbMirror block; opens its own
|
|
81
|
+
// connection otherwise.
|
|
82
|
+
// from real context_chunks embeddings. Isolated-store only, gated on
|
|
83
|
+
// AUTO_WIKI_ENABLED; best-effort + non-fatal (never breaks compaction).
|
|
84
|
+
try {
|
|
85
|
+
if (config.autoWikiEnabled && config.turnsDbEnabled) {
|
|
86
|
+
const every = Math.max(
|
|
87
|
+
1,
|
|
88
|
+
TurnsConfig.WIKI_REBUILD_EVERY_N_COMPACTS,
|
|
89
|
+
);
|
|
90
|
+
const tdb = openTurnStore(runtime.currentStateDir);
|
|
91
|
+
const n = bumpWikiCompactCounter(tdb);
|
|
92
|
+
if (n % every === 0) {
|
|
93
|
+
const model = buildTopicModel(db, {
|
|
94
|
+
kRange: [TurnsConfig.WIKI_K_MIN, TurnsConfig.WIKI_K_MAX],
|
|
95
|
+
labelTopTerms: TurnsConfig.WIKI_LABEL_TOP_TERMS,
|
|
96
|
+
restarts: 5,
|
|
97
|
+
seed: 0x9e3779b9,
|
|
98
|
+
});
|
|
99
|
+
createTopicStore(runtime.currentStateDir).replaceTopicModel(
|
|
100
|
+
model,
|
|
101
|
+
);
|
|
102
|
+
runtime.logger.info("wiki_rebuild", {
|
|
103
|
+
clusterCount: model.k,
|
|
104
|
+
totalChunks: model.totalChunks,
|
|
105
|
+
method: "kmeans+tfidf",
|
|
106
|
+
criterion: model.criterion,
|
|
107
|
+
silhouetteScore: model.silhouetteScore,
|
|
108
|
+
uncalibrated: false,
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
} catch (wikiErr) {
|
|
113
|
+
runtime.logger.warn("wiki_rebuild_failed", {
|
|
114
|
+
error: String(wikiErr),
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
// D1: seed the topic model from raw_transcript when context_chunks is
|
|
119
|
+
// thin (pre-compaction). Gated on WIKI_SEED_FROM_TURNS; non-fatal.
|
|
120
|
+
// Seeds buildTopicModel with on-the-fly trigram embeddings from
|
|
121
|
+
// recent raw_transcript rows for the current session.
|
|
122
|
+
try {
|
|
123
|
+
if (
|
|
124
|
+
config.autoWikiEnabled &&
|
|
125
|
+
config.turnsDbEnabled &&
|
|
126
|
+
TurnsConfig.WIKI_SEED_FROM_TURNS
|
|
127
|
+
) {
|
|
128
|
+
const floor = 50;
|
|
129
|
+
const countRow = db
|
|
130
|
+
.prepare(
|
|
131
|
+
`SELECT COUNT(*) AS cnt FROM context_chunks WHERE session_id = ?`,
|
|
132
|
+
)
|
|
133
|
+
.get(runtime.rt.sessionId) as { cnt: number } | undefined;
|
|
134
|
+
const chunkCount = countRow?.cnt ?? 0;
|
|
135
|
+
if (chunkCount < floor) {
|
|
136
|
+
const transcriptRows = db
|
|
137
|
+
.prepare(
|
|
138
|
+
`SELECT DISTINCT content_bytes
|
|
139
|
+
FROM raw_transcript
|
|
140
|
+
WHERE session_id = ?
|
|
141
|
+
AND length(content_bytes) > 10
|
|
142
|
+
ORDER BY seq ASC
|
|
143
|
+
LIMIT 200`,
|
|
144
|
+
)
|
|
145
|
+
.all(runtime.rt.sessionId) as Array<{
|
|
146
|
+
content_bytes: string;
|
|
147
|
+
}>;
|
|
148
|
+
if (transcriptRows.length > 0) {
|
|
149
|
+
const embedder = new TrigramEmbedder();
|
|
150
|
+
const seedChunks: EmbeddedChunk[] = [];
|
|
151
|
+
for (let i = 0; i < transcriptRows.length; i++) {
|
|
152
|
+
const text = transcriptRows[i].content_bytes.trim();
|
|
153
|
+
if (text.length === 0) continue;
|
|
154
|
+
const vec = embedder.embed(text);
|
|
155
|
+
seedChunks.push({
|
|
156
|
+
chunkId: `seed_transcript_${i}`,
|
|
157
|
+
sessionId: runtime.rt.sessionId,
|
|
158
|
+
vec,
|
|
159
|
+
text,
|
|
160
|
+
});
|
|
161
|
+
}
|
|
162
|
+
if (seedChunks.length > 0) {
|
|
163
|
+
const model = buildTopicModel(
|
|
164
|
+
db,
|
|
165
|
+
{
|
|
166
|
+
kRange: [
|
|
167
|
+
TurnsConfig.WIKI_K_MIN,
|
|
168
|
+
TurnsConfig.WIKI_K_MAX,
|
|
169
|
+
],
|
|
170
|
+
labelTopTerms:
|
|
171
|
+
TurnsConfig.WIKI_LABEL_TOP_TERMS,
|
|
172
|
+
restarts: 5,
|
|
173
|
+
seed: 0x9e3779b9,
|
|
174
|
+
},
|
|
175
|
+
seedChunks,
|
|
176
|
+
);
|
|
177
|
+
createTopicStore(
|
|
178
|
+
runtime.currentStateDir,
|
|
179
|
+
).replaceTopicModel(model);
|
|
180
|
+
runtime.logger.info("wiki_seed", {
|
|
181
|
+
clusterCount: model.k,
|
|
182
|
+
sourceChunks: seedChunks.length,
|
|
183
|
+
totalChunks: model.totalChunks,
|
|
184
|
+
method: "kmeans+tfidf",
|
|
185
|
+
});
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
} catch (seedErr) {
|
|
191
|
+
runtime.logger.warn("wiki_seed_failed", {
|
|
192
|
+
error: String(seedErr),
|
|
193
|
+
});
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
// S27 Task 6: Fire-and-forget dedup pipeline.
|
|
197
|
+
// Deduplicates raw_transcript rows for the compacted range.
|
|
198
|
+
try {
|
|
199
|
+
const { dedupTranscript } = await import("../../../src/mirror/dedup.js");
|
|
200
|
+
dedupTranscript(
|
|
201
|
+
db,
|
|
202
|
+
runtime.rt.sessionId,
|
|
203
|
+
0,
|
|
204
|
+
ran.result.compactedFrom,
|
|
205
|
+
);
|
|
206
|
+
} catch (_dedupErr) {
|
|
207
|
+
// Fire-and-forget: dedup failure is non-fatal
|
|
208
|
+
}
|
|
209
|
+
} catch (e) {
|
|
210
|
+
runtime.logger.warn("db-mirror-epoch-fail", { error: String(e) });
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* context-handler/messageText.ts — best-effort text extraction helpers.
|
|
3
|
+
*
|
|
4
|
+
* Extracted from context-handler.ts (delegate-shell split). AgentMessage is a
|
|
5
|
+
* discriminated union; .content exists only on some variants
|
|
6
|
+
* (user/assistant/toolResult/custom). Narrow by role, never assume.
|
|
7
|
+
*/
|
|
8
|
+
import type { AgentMessage } from "@earendil-works/pi-agent-core";
|
|
9
|
+
|
|
10
|
+
/** Best-effort text extraction from an AgentMessage for analytics/logging. */
|
|
11
|
+
export function messageContentText(m: AgentMessage): string {
|
|
12
|
+
const c = (m as { content?: unknown }).content;
|
|
13
|
+
if (typeof c === "string") return c;
|
|
14
|
+
if (Array.isArray(c)) {
|
|
15
|
+
return c
|
|
16
|
+
.map((b) => (b && typeof b === "object" && "text" in b ? String((b as { text?: string }).text ?? "") : ""))
|
|
17
|
+
.join(" ");
|
|
18
|
+
}
|
|
19
|
+
return "";
|
|
20
|
+
}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* context-handler/tailResult.ts — buildTailResult(factory).
|
|
3
|
+
*
|
|
4
|
+
* Extracted from context-handler.ts (delegate-shell split). Constructs the
|
|
5
|
+
* closure that injects the staged recall/memory block as a user-role tail
|
|
6
|
+
* message at any context view-return point, plus optional cache-striping /
|
|
7
|
+
* message-separation prompt reshapes and prefix-stability logging.
|
|
8
|
+
*/
|
|
9
|
+
import type { AgentMessage } from "@earendil-works/pi-agent-core";
|
|
10
|
+
import type { MegaRuntime } from "../../mega-runtime.js";
|
|
11
|
+
import type { MegaConfig } from "../../mega-config.js";
|
|
12
|
+
import { stagedForTail, withRecallTail } from "../recall-tail.js";
|
|
13
|
+
import { buildSeparatedPrompt, buildCacheOptimizedPrompt } from "../separated-prompt.js";
|
|
14
|
+
import { messageContentText } from "./messageText.js";
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Build the tail injection closure. Returns undefined when nothing is staged
|
|
18
|
+
* (or the flags are OFF) so the caller falls through to its normal return.
|
|
19
|
+
*/
|
|
20
|
+
export function buildTailResult(
|
|
21
|
+
runtime: MegaRuntime,
|
|
22
|
+
config: MegaConfig,
|
|
23
|
+
messages: readonly AgentMessage[],
|
|
24
|
+
): ((msgs?: readonly AgentMessage[]) => { messages: AgentMessage[] } | undefined) {
|
|
25
|
+
// S53: helper to inject the staged recall/memory block as a user-role
|
|
26
|
+
// tail message at any view-return point. Returns undefined when nothing
|
|
27
|
+
// is staged (or the flag is OFF) so the caller falls through to its
|
|
28
|
+
// normal return. The F3 mirror append (above the gates) runs on the
|
|
29
|
+
// REAL transcript before any view is built, so the injected tail never
|
|
30
|
+
// reaches raw_transcript (PREVENT-PI: append-only, view-only).
|
|
31
|
+
const tailResult = (msgs?: readonly AgentMessage[]) => {
|
|
32
|
+
const base = msgs ?? messages;
|
|
33
|
+
if (!stagedForTail(runtime, config) && !config.messageSeparation && !config.cacheStriping) return undefined;
|
|
34
|
+
let result: AgentMessage[] = stagedForTail(runtime, config)
|
|
35
|
+
? withRecallTail(base, runtime, config)
|
|
36
|
+
: (base as AgentMessage[]);
|
|
37
|
+
if (config.cacheStriping) {
|
|
38
|
+
result = buildCacheOptimizedPrompt(result);
|
|
39
|
+
} else if (config.messageSeparation) {
|
|
40
|
+
result = buildSeparatedPrompt(result);
|
|
41
|
+
}
|
|
42
|
+
// P2.5: log prefix stability (fire-and-forget, non-fatal).
|
|
43
|
+
// tailResult is sync, so use .then().catch() on the dynamic import.
|
|
44
|
+
if (result.length > 1) {
|
|
45
|
+
import("../../../src/cache-stripe-impl.js").then(({ computeStabilityScore }) => {
|
|
46
|
+
const stableScore = computeStabilityScore(
|
|
47
|
+
{ content: messageContentText(result[0] ?? result[0]), chunkId: "prefix", accessCount: 0, lastAccessedAt: 0 },
|
|
48
|
+
result.slice(0, 2).map((m) => ({ content: messageContentText(m), chunkId: "prefix", accessCount: 0, lastAccessedAt: 0 })),
|
|
49
|
+
);
|
|
50
|
+
runtime.logger.info("prefix_stability", {
|
|
51
|
+
stableScore: Number.isFinite(stableScore) ? stableScore : 0,
|
|
52
|
+
prefixMessages: result.length,
|
|
53
|
+
separation: config.messageSeparation ? "v2" : "off",
|
|
54
|
+
striping: config.cacheStriping ? "v3" : "off",
|
|
55
|
+
});
|
|
56
|
+
}).catch(() => {
|
|
57
|
+
// Non-fatal: stability logging is best-effort.
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
return { messages: result };
|
|
61
|
+
};
|
|
62
|
+
return tailResult;
|
|
63
|
+
}
|
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* mega-events/context-handler.ts — the context event handler (auto-trigger).
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
* threshold check, pipeline invocation,
|
|
6
|
-
* the live-trim message reconstruction that feeds pi's
|
|
4
|
+
* Delegate-shell (extensions split): handles the live-trim compaction pipeline —
|
|
5
|
+
* DB-mirror append, fast-gate threshold check, pipeline invocation, legacy
|
|
6
|
+
* durable compact, and the live-trim message reconstruction that feeds pi's
|
|
7
|
+
* transformContext. The extracted pieces live in ./context-handler/:
|
|
8
|
+
* - messageText.ts (messageContentText — best-effort text extraction)
|
|
9
|
+
* - tailResult.ts (buildTailResult — recall-tail injection factory)
|
|
10
|
+
* - afterCompact.ts (persistEpochAndMaintain — epoch/wiki/seed/dedup writes)
|
|
7
11
|
*/
|
|
8
12
|
import type {
|
|
9
13
|
ExtensionAPI,
|
|
@@ -13,47 +17,22 @@ import type {
|
|
|
13
17
|
import type { AgentMessage } from "@earendil-works/pi-agent-core";
|
|
14
18
|
import {
|
|
15
19
|
openStore,
|
|
16
|
-
writeCheckpointEpoch,
|
|
17
|
-
type CheckpointEpoch,
|
|
18
20
|
} from "../../src/store/sqlite.js";
|
|
19
|
-
import { epochIdFor } from "../../src/mirror/epoch.js";
|
|
20
21
|
import { autoCompactCheck } from "../../src/compact.js";
|
|
21
22
|
import { estimateSessionTokens } from "../../src/tokens.js";
|
|
22
23
|
import type { MegaRuntime } from "../mega-runtime.js";
|
|
23
24
|
import { runCompact, piCompactWouldNoop } from "../mega-pipeline.js";
|
|
24
|
-
import { stampTurnsEpochFor } from "../mega-turn-store.js";
|
|
25
|
-
import { TurnsConfig } from "../../src/config/turns.js";
|
|
26
|
-
import { openTurnStore } from "../../src/store/turns/connection.js";
|
|
27
|
-
import {
|
|
28
|
-
buildTopicModel,
|
|
29
|
-
createTopicStore,
|
|
30
|
-
bumpWikiCompactCounter,
|
|
31
|
-
} from "../../src/topics/index.js";
|
|
32
|
-
import { TrigramEmbedder } from "../../src/embedder.js";
|
|
33
|
-
import type { EmbeddedChunk } from "../../src/topics/types.js";
|
|
34
|
-
import { computeLiveTrimCut, liveTrimSummaryMessage } from "../mega-trim.js";
|
|
35
25
|
import {
|
|
36
26
|
pressureFromPct,
|
|
37
27
|
pressureRatio,
|
|
38
28
|
type MegaConfig,
|
|
39
29
|
} from "../mega-config.js";
|
|
40
30
|
import { appendMirrorMessages } from "./mirror-append.js";
|
|
41
|
-
import {
|
|
42
|
-
import {
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
* variants (user/assistant/toolResult/custom). Narrow by role, never assume. */
|
|
47
|
-
function messageContentText(m: AgentMessage): string {
|
|
48
|
-
const c = (m as { content?: unknown }).content;
|
|
49
|
-
if (typeof c === "string") return c;
|
|
50
|
-
if (Array.isArray(c)) {
|
|
51
|
-
return c
|
|
52
|
-
.map((b) => (b && typeof b === "object" && "text" in b ? String((b as { text?: string }).text ?? "") : ""))
|
|
53
|
-
.join(" ");
|
|
54
|
-
}
|
|
55
|
-
return "";
|
|
56
|
-
}
|
|
31
|
+
import { epochIdFor } from "../../src/mirror/epoch.js";
|
|
32
|
+
import { computeLiveTrimCut, liveTrimSummaryMessage } from "../mega-trim.js";
|
|
33
|
+
import { messageContentText } from "./context-handler/messageText.js";
|
|
34
|
+
import { buildTailResult } from "./context-handler/tailResult.js";
|
|
35
|
+
import { persistEpochAndMaintain } from "./context-handler/afterCompact.js";
|
|
57
36
|
|
|
58
37
|
/** Register the context event handler (live-trim auto-trigger). */
|
|
59
38
|
export function registerContextHandler(
|
|
@@ -82,40 +61,8 @@ export function registerContextHandler(
|
|
|
82
61
|
// S53: helper to inject the staged recall/memory block as a user-role
|
|
83
62
|
// tail message at any view-return point. Returns undefined when nothing
|
|
84
63
|
// is staged (or the flag is OFF) so the caller falls through to its
|
|
85
|
-
// normal return.
|
|
86
|
-
|
|
87
|
-
// reaches raw_transcript (PREVENT-PI: append-only, view-only).
|
|
88
|
-
const tailResult = (msgs?: readonly AgentMessage[]) => {
|
|
89
|
-
const base = msgs ?? messages;
|
|
90
|
-
if (!stagedForTail(runtime, config) && !config.messageSeparation && !config.cacheStriping) return undefined;
|
|
91
|
-
let result: AgentMessage[] = stagedForTail(runtime, config)
|
|
92
|
-
? withRecallTail(base, runtime, config)
|
|
93
|
-
: (base as AgentMessage[]);
|
|
94
|
-
if (config.cacheStriping) {
|
|
95
|
-
result = buildCacheOptimizedPrompt(result);
|
|
96
|
-
} else if (config.messageSeparation) {
|
|
97
|
-
result = buildSeparatedPrompt(result);
|
|
98
|
-
}
|
|
99
|
-
// P2.5: log prefix stability (fire-and-forget, non-fatal).
|
|
100
|
-
// tailResult is sync, so use .then().catch() on the dynamic import.
|
|
101
|
-
if (result.length > 1) {
|
|
102
|
-
import("../../src/cache-stripe-impl.js").then(({ computeStabilityScore }) => {
|
|
103
|
-
const stableScore = computeStabilityScore(
|
|
104
|
-
{ content: messageContentText(result[0] ?? result[0]), chunkId: "prefix", accessCount: 0, lastAccessedAt: 0 },
|
|
105
|
-
result.slice(0, 2).map((m) => ({ content: messageContentText(m), chunkId: "prefix", accessCount: 0, lastAccessedAt: 0 })),
|
|
106
|
-
);
|
|
107
|
-
runtime.logger.info("prefix_stability", {
|
|
108
|
-
stableScore: Number.isFinite(stableScore) ? stableScore : 0,
|
|
109
|
-
prefixMessages: result.length,
|
|
110
|
-
separation: config.messageSeparation ? "v2" : "off",
|
|
111
|
-
striping: config.cacheStriping ? "v3" : "off",
|
|
112
|
-
});
|
|
113
|
-
}).catch(() => {
|
|
114
|
-
// Non-fatal: stability logging is best-effort.
|
|
115
|
-
});
|
|
116
|
-
}
|
|
117
|
-
return { messages: result };
|
|
118
|
-
};
|
|
64
|
+
// normal return.
|
|
65
|
+
const tailResult = buildTailResult(runtime, config, messages);
|
|
119
66
|
// Always track context for the dashboard/widget, even when auto is off.
|
|
120
67
|
// (v0.8 regression: !config.auto gate sat above this, leaving ctx stats
|
|
121
68
|
// null -> widget '?% / ?/?' when auto disabled. Track first, THEN gate.)
|
|
@@ -303,174 +250,9 @@ export function registerContextHandler(
|
|
|
303
250
|
return tailResult() ?? undefined;
|
|
304
251
|
}
|
|
305
252
|
|
|
306
|
-
// S27 DB-mirror: write checkpoint_epoch
|
|
307
|
-
//
|
|
308
|
-
|
|
309
|
-
try {
|
|
310
|
-
const db = openStore(runtime.currentStateDir);
|
|
311
|
-
const cpId = ran.result.checkpointId ?? `epoch-${Date.now()}`;
|
|
312
|
-
const epoch: CheckpointEpoch = {
|
|
313
|
-
epochId: epochIdFor(cpId),
|
|
314
|
-
sessionId: runtime.rt.sessionId,
|
|
315
|
-
startedSeq: 0,
|
|
316
|
-
committedSeq: ran.result.compactedFrom,
|
|
317
|
-
checkpointId: cpId,
|
|
318
|
-
cutIndex: ran.result.compactedFrom,
|
|
319
|
-
summaryMessageText: ran.result.summary,
|
|
320
|
-
createdAt: Date.now(),
|
|
321
|
-
};
|
|
322
|
-
writeCheckpointEpoch(db, epoch);
|
|
323
|
-
// S50B: link this session's turns to the epoch that just compacted
|
|
324
|
-
// them (compression-by-conversation-epoch metrics). Isolated-store
|
|
325
|
-
// only; best-effort + non-fatal.
|
|
326
|
-
try {
|
|
327
|
-
stampTurnsEpochFor(
|
|
328
|
-
config,
|
|
329
|
-
runtime.rt.sessionId,
|
|
330
|
-
epoch.epochId,
|
|
331
|
-
runtime.currentStateDir,
|
|
332
|
-
);
|
|
333
|
-
} catch {
|
|
334
|
-
/* non-fatal: epoch stamping never breaks compaction */
|
|
335
|
-
}
|
|
336
|
-
|
|
337
|
-
// S51B: auto-categorizing wiki rebuild — every Nth compaction, derived
|
|
338
|
-
// from real context_chunks embeddings. Isolated-store only, gated on
|
|
339
|
-
// AUTO_WIKI_ENABLED; best-effort + non-fatal (never breaks compaction).
|
|
340
|
-
// Fire regardless of dbMirror — context_chunks is the isolated store.
|
|
341
|
-
// Uses the already-open db when inside the dbMirror block; opens its own
|
|
342
|
-
// connection otherwise.
|
|
343
|
-
// from real context_chunks embeddings. Isolated-store only, gated on
|
|
344
|
-
// AUTO_WIKI_ENABLED; best-effort + non-fatal (never breaks compaction).
|
|
345
|
-
try {
|
|
346
|
-
if (config.autoWikiEnabled && config.turnsDbEnabled) {
|
|
347
|
-
const every = Math.max(
|
|
348
|
-
1,
|
|
349
|
-
TurnsConfig.WIKI_REBUILD_EVERY_N_COMPACTS,
|
|
350
|
-
);
|
|
351
|
-
const tdb = openTurnStore(runtime.currentStateDir);
|
|
352
|
-
const n = bumpWikiCompactCounter(tdb);
|
|
353
|
-
if (n % every === 0) {
|
|
354
|
-
const model = buildTopicModel(db, {
|
|
355
|
-
kRange: [TurnsConfig.WIKI_K_MIN, TurnsConfig.WIKI_K_MAX],
|
|
356
|
-
labelTopTerms: TurnsConfig.WIKI_LABEL_TOP_TERMS,
|
|
357
|
-
restarts: 5,
|
|
358
|
-
seed: 0x9e3779b9,
|
|
359
|
-
});
|
|
360
|
-
createTopicStore(runtime.currentStateDir).replaceTopicModel(
|
|
361
|
-
model,
|
|
362
|
-
);
|
|
363
|
-
runtime.logger.info("wiki_rebuild", {
|
|
364
|
-
clusterCount: model.k,
|
|
365
|
-
totalChunks: model.totalChunks,
|
|
366
|
-
method: "kmeans+tfidf",
|
|
367
|
-
criterion: model.criterion,
|
|
368
|
-
silhouetteScore: model.silhouetteScore,
|
|
369
|
-
uncalibrated: false,
|
|
370
|
-
});
|
|
371
|
-
}
|
|
372
|
-
}
|
|
373
|
-
} catch (wikiErr) {
|
|
374
|
-
runtime.logger.warn("wiki_rebuild_failed", {
|
|
375
|
-
error: String(wikiErr),
|
|
376
|
-
});
|
|
377
|
-
}
|
|
378
|
-
|
|
379
|
-
// D1: seed the topic model from raw_transcript when context_chunks is
|
|
380
|
-
// thin (pre-compaction). Gated on WIKI_SEED_FROM_TURNS; non-fatal.
|
|
381
|
-
// Seeds buildTopicModel with on-the-fly trigram embeddings from
|
|
382
|
-
// recent raw_transcript rows for the current session.
|
|
383
|
-
try {
|
|
384
|
-
if (
|
|
385
|
-
config.autoWikiEnabled &&
|
|
386
|
-
config.turnsDbEnabled &&
|
|
387
|
-
TurnsConfig.WIKI_SEED_FROM_TURNS
|
|
388
|
-
) {
|
|
389
|
-
const floor = 50;
|
|
390
|
-
const countRow = db
|
|
391
|
-
.prepare(
|
|
392
|
-
`SELECT COUNT(*) AS cnt FROM context_chunks WHERE session_id = ?`,
|
|
393
|
-
)
|
|
394
|
-
.get(runtime.rt.sessionId) as { cnt: number } | undefined;
|
|
395
|
-
const chunkCount = countRow?.cnt ?? 0;
|
|
396
|
-
if (chunkCount < floor) {
|
|
397
|
-
const transcriptRows = db
|
|
398
|
-
.prepare(
|
|
399
|
-
`SELECT DISTINCT content_bytes
|
|
400
|
-
FROM raw_transcript
|
|
401
|
-
WHERE session_id = ?
|
|
402
|
-
AND length(content_bytes) > 10
|
|
403
|
-
ORDER BY seq ASC
|
|
404
|
-
LIMIT 200`,
|
|
405
|
-
)
|
|
406
|
-
.all(runtime.rt.sessionId) as Array<{
|
|
407
|
-
content_bytes: string;
|
|
408
|
-
}>;
|
|
409
|
-
if (transcriptRows.length > 0) {
|
|
410
|
-
const embedder = new TrigramEmbedder();
|
|
411
|
-
const seedChunks: EmbeddedChunk[] = [];
|
|
412
|
-
for (let i = 0; i < transcriptRows.length; i++) {
|
|
413
|
-
const text = transcriptRows[i].content_bytes.trim();
|
|
414
|
-
if (text.length === 0) continue;
|
|
415
|
-
const vec = embedder.embed(text);
|
|
416
|
-
seedChunks.push({
|
|
417
|
-
chunkId: `seed_transcript_${i}`,
|
|
418
|
-
sessionId: runtime.rt.sessionId,
|
|
419
|
-
vec,
|
|
420
|
-
text,
|
|
421
|
-
});
|
|
422
|
-
}
|
|
423
|
-
if (seedChunks.length > 0) {
|
|
424
|
-
const model = buildTopicModel(
|
|
425
|
-
db,
|
|
426
|
-
{
|
|
427
|
-
kRange: [
|
|
428
|
-
TurnsConfig.WIKI_K_MIN,
|
|
429
|
-
TurnsConfig.WIKI_K_MAX,
|
|
430
|
-
],
|
|
431
|
-
labelTopTerms:
|
|
432
|
-
TurnsConfig.WIKI_LABEL_TOP_TERMS,
|
|
433
|
-
restarts: 5,
|
|
434
|
-
seed: 0x9e3779b9,
|
|
435
|
-
},
|
|
436
|
-
seedChunks,
|
|
437
|
-
);
|
|
438
|
-
createTopicStore(
|
|
439
|
-
runtime.currentStateDir,
|
|
440
|
-
).replaceTopicModel(model);
|
|
441
|
-
runtime.logger.info("wiki_seed", {
|
|
442
|
-
clusterCount: model.k,
|
|
443
|
-
sourceChunks: seedChunks.length,
|
|
444
|
-
totalChunks: model.totalChunks,
|
|
445
|
-
method: "kmeans+tfidf",
|
|
446
|
-
});
|
|
447
|
-
}
|
|
448
|
-
}
|
|
449
|
-
}
|
|
450
|
-
}
|
|
451
|
-
} catch (seedErr) {
|
|
452
|
-
runtime.logger.warn("wiki_seed_failed", {
|
|
453
|
-
error: String(seedErr),
|
|
454
|
-
});
|
|
455
|
-
}
|
|
456
|
-
|
|
457
|
-
// S27 Task 6: Fire-and-forget dedup pipeline.
|
|
458
|
-
// Deduplicates raw_transcript rows for the compacted range.
|
|
459
|
-
try {
|
|
460
|
-
const { dedupTranscript } = await import("../../src/mirror/dedup.js");
|
|
461
|
-
dedupTranscript(
|
|
462
|
-
db,
|
|
463
|
-
runtime.rt.sessionId,
|
|
464
|
-
0,
|
|
465
|
-
ran.result.compactedFrom,
|
|
466
|
-
);
|
|
467
|
-
} catch (_dedupErr) {
|
|
468
|
-
// Fire-and-forget: dedup failure is non-fatal
|
|
469
|
-
}
|
|
470
|
-
} catch (e) {
|
|
471
|
-
runtime.logger.warn("db-mirror-epoch-fail", { error: String(e) });
|
|
472
|
-
}
|
|
473
|
-
}
|
|
253
|
+
// S27 DB-mirror: write checkpoint_epoch + stamp turn epochs + auto-wiki +
|
|
254
|
+
// topic seed + fire-and-forget dedup. Best-effort + non-fatal.
|
|
255
|
+
await persistEpochAndMaintain(runtime, config, ran);
|
|
474
256
|
|
|
475
257
|
// LEGACY path (rollback): v0.4.28 ctx.compact() + the no-op gate. The
|
|
476
258
|
// manual compact path aborts the in-flight turn — only used behind the flag.
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared helpers for the separated-prompt.test split files.
|
|
3
|
+
* Extracted from extensions/mega-events/separated-prompt.test.ts.
|
|
4
|
+
*/
|
|
5
|
+
import { mkdtempSync } from "node:fs";
|
|
6
|
+
import { join } from "node:path";
|
|
7
|
+
import { tmpdir } from "node:os";
|
|
8
|
+
import { DatabaseSync } from "node:sqlite";
|
|
9
|
+
import { initSchema } from "../../../src/store/sqlite/schema.js";
|
|
10
|
+
|
|
11
|
+
/** Cast output to a plain Record array for role/content access in assertions. */
|
|
12
|
+
export type R = Record<string, unknown>;
|
|
13
|
+
|
|
14
|
+
export function asR(arr: unknown[]): R[] {
|
|
15
|
+
return arr as R[];
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/** Create a temp DB with schema initialized, returns db + dir. */
|
|
19
|
+
export function createTestDb(): { db: DatabaseSync; dir: string } {
|
|
20
|
+
const dir = mkdtempSync(join(tmpdir(), "separated-prompt-test-"));
|
|
21
|
+
const db = new DatabaseSync(join(dir, "sqlite.db"));
|
|
22
|
+
db.exec("PRAGMA journal_mode=WAL");
|
|
23
|
+
db.exec("PRAGMA foreign_keys=ON");
|
|
24
|
+
initSchema(db);
|
|
25
|
+
return { db, dir };
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** Helper: create an AgentMessage-like object. */
|
|
29
|
+
export function msg(
|
|
30
|
+
role: string,
|
|
31
|
+
content: unknown,
|
|
32
|
+
extra?: Record<string, unknown>,
|
|
33
|
+
): Record<string, unknown> {
|
|
34
|
+
return { role, content, ...extra };
|
|
35
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-mega-compact",
|
|
3
|
-
"version": "0.13.
|
|
3
|
+
"version": "0.13.2",
|
|
4
4
|
"description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "BSD-3-Clause",
|
package/src/config.ts
CHANGED
|
@@ -128,3 +128,7 @@ export const RAG_RECALL_METRICS = (): boolean =>
|
|
|
128
128
|
/** B4: Memory graph traversal (dashboard-oriented). */
|
|
129
129
|
export const RAG_MEMORY_GRAPH = (): boolean =>
|
|
130
130
|
ragFlag("MEGACOMPACT_MEMORY_GRAPH");
|
|
131
|
+
|
|
132
|
+
/** B5: HyDE — generate a hypothetical answer doc via LLM (HttpEmbedder only). Default OFF. */
|
|
133
|
+
export const RAG_HYDE_ENABLED = (): boolean =>
|
|
134
|
+
ragFlag("MEGACOMPACT_HYDE_ENABLED");
|