pi-mega-compact 0.9.2 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. package/README.md +3 -2
  2. package/dist/extensions/dashboard-server/api-contracts/endpoints.js +95 -47
  3. package/dist/extensions/dashboard-server/api-contracts/index.js +1 -1
  4. package/dist/extensions/dashboard-server/api-contracts/turns.js +10 -0
  5. package/dist/extensions/dashboard-server/api-contracts.test.js +3 -2
  6. package/dist/extensions/dashboard-server/routes-topics.js +87 -0
  7. package/dist/extensions/dashboard-server/routes-turns.js +264 -0
  8. package/dist/extensions/dashboard-server/routes.js +3 -1
  9. package/dist/extensions/dashboard-server/server.js +6 -2
  10. package/dist/extensions/mega-compact.js +10 -6
  11. package/dist/extensions/mega-config.js +4 -0
  12. package/dist/extensions/mega-events/agent-handlers.js +62 -49
  13. package/dist/extensions/mega-events/context-handler.js +48 -2
  14. package/dist/extensions/mega-events/session-handlers.js +25 -1
  15. package/dist/extensions/mega-metrics-cmds.js +90 -0
  16. package/dist/extensions/mega-pipeline/recall.js +47 -17
  17. package/dist/extensions/mega-topics-cmds.js +111 -0
  18. package/dist/extensions/mega-turn-store.js +110 -0
  19. package/dist/extensions/mega-turn-store.test.js +68 -0
  20. package/dist/src/config/turns.js +48 -0
  21. package/dist/src/fork.js +33 -0
  22. package/dist/src/fork.test.js +81 -0
  23. package/dist/src/intent.js +51 -0
  24. package/dist/src/intent.test.js +78 -0
  25. package/dist/src/metrics/index.js +4 -0
  26. package/dist/src/metrics/turns.js +118 -0
  27. package/dist/src/metrics/turns.test.js +152 -0
  28. package/dist/src/store/sqlite/raw-transcript.js +5 -3
  29. package/dist/src/store/sqlite.dbmirror.test.js +9 -0
  30. package/dist/src/store/turns/connection.js +123 -0
  31. package/dist/src/store/turns/contract-compliance.test.js +428 -0
  32. package/dist/src/store/turns/index.js +31 -0
  33. package/dist/src/store/turns/memory-store.js +357 -0
  34. package/dist/src/store/turns/memory-store.test.js +9 -0
  35. package/dist/src/store/turns/migrations.js +178 -0
  36. package/dist/src/store/turns/migrations.test.js +138 -0
  37. package/dist/src/store/turns/schema.js +115 -0
  38. package/dist/src/store/turns/sqlite-store.js +455 -0
  39. package/dist/src/store/turns/sqlite-store.test.js +144 -0
  40. package/dist/src/store/turns/types.js +16 -0
  41. package/dist/src/topics/cluster.js +141 -0
  42. package/dist/src/topics/cluster.test.js +150 -0
  43. package/dist/src/topics/index.js +6 -0
  44. package/dist/src/topics/kselection.js +142 -0
  45. package/dist/src/topics/labels.js +56 -0
  46. package/dist/src/topics/store.js +119 -0
  47. package/dist/src/topics/store.test.js +117 -0
  48. package/dist/src/topics/types.js +8 -0
  49. package/dist/src/wiki.js +133 -0
  50. package/dist/src/wiki.test.js +171 -0
  51. package/extensions/dashboard-client/dist/assets/{AchievementTiles-QPo46Pwg.js → AchievementTiles-DZp_xQOW.js} +2 -2
  52. package/extensions/dashboard-client/dist/assets/{AchievementTiles-QPo46Pwg.js.map → AchievementTiles-DZp_xQOW.js.map} +1 -1
  53. package/extensions/dashboard-client/dist/assets/{AchievementsTab-BDJxRXRY.js → AchievementsTab-BzFHpAHM.js} +2 -2
  54. package/extensions/dashboard-client/dist/assets/{AchievementsTab-BDJxRXRY.js.map → AchievementsTab-BzFHpAHM.js.map} +1 -1
  55. package/extensions/dashboard-client/dist/assets/{CacheTab-CUMqXQjd.js → CacheTab-DEQbTq2T.js} +2 -2
  56. package/extensions/dashboard-client/dist/assets/{CacheTab-CUMqXQjd.js.map → CacheTab-DEQbTq2T.js.map} +1 -1
  57. package/extensions/dashboard-client/dist/assets/{ConfigTab-j1dRzurG.js → ConfigTab-DQ4FG33J.js} +2 -2
  58. package/extensions/dashboard-client/dist/assets/{ConfigTab-j1dRzurG.js.map → ConfigTab-DQ4FG33J.js.map} +1 -1
  59. package/extensions/dashboard-client/dist/assets/{EventsTab-CMglXK4p.js → EventsTab-D5HoEVYo.js} +2 -2
  60. package/extensions/dashboard-client/dist/assets/{EventsTab-CMglXK4p.js.map → EventsTab-D5HoEVYo.js.map} +1 -1
  61. package/extensions/dashboard-client/dist/assets/{GameTab-fpaDoyxi.js → GameTab-Jzsc4yeY.js} +2 -2
  62. package/extensions/dashboard-client/dist/assets/{GameTab-fpaDoyxi.js.map → GameTab-Jzsc4yeY.js.map} +1 -1
  63. package/extensions/dashboard-client/dist/assets/{MetricsTab-CWG2crG_.js → MetricsTab-D_XnsEYX.js} +2 -2
  64. package/extensions/dashboard-client/dist/assets/{MetricsTab-CWG2crG_.js.map → MetricsTab-D_XnsEYX.js.map} +1 -1
  65. package/extensions/dashboard-client/dist/assets/{OverviewTab-BaV2D2nf.js → OverviewTab-BMREvAth.js} +2 -2
  66. package/extensions/dashboard-client/dist/assets/{OverviewTab-BaV2D2nf.js.map → OverviewTab-BMREvAth.js.map} +1 -1
  67. package/extensions/dashboard-client/dist/assets/{ReposTab-C-0kuZ_0.js → ReposTab-DuyhOktG.js} +2 -2
  68. package/extensions/dashboard-client/dist/assets/{ReposTab-C-0kuZ_0.js.map → ReposTab-DuyhOktG.js.map} +1 -1
  69. package/extensions/dashboard-client/dist/assets/{SessionsTab-CSYhddCA.js → SessionsTab-mz_vz8i2.js} +2 -2
  70. package/extensions/dashboard-client/dist/assets/{SessionsTab-CSYhddCA.js.map → SessionsTab-mz_vz8i2.js.map} +1 -1
  71. package/extensions/dashboard-client/dist/assets/{TimeSavedCard-Dy4KRjSM.js → TimeSavedCard-B8AIjNTY.js} +2 -2
  72. package/extensions/dashboard-client/dist/assets/{TimeSavedCard-Dy4KRjSM.js.map → TimeSavedCard-B8AIjNTY.js.map} +1 -1
  73. package/extensions/dashboard-client/dist/assets/TopicsTab-DKuct4oD.js +2 -0
  74. package/extensions/dashboard-client/dist/assets/TopicsTab-DKuct4oD.js.map +1 -0
  75. package/extensions/dashboard-client/dist/assets/TurnsTab-11MGLxY8.js +2 -0
  76. package/extensions/dashboard-client/dist/assets/TurnsTab-11MGLxY8.js.map +1 -0
  77. package/extensions/dashboard-client/dist/assets/{index-D_WtU2TV.js → index-CqLN7STl.js} +10 -10
  78. package/extensions/dashboard-client/dist/assets/index-CqLN7STl.js.map +1 -0
  79. package/extensions/dashboard-client/dist/assets/{index-og3ghxyY.css → index-gn66lYD2.css} +1 -1
  80. package/extensions/dashboard-client/dist/assets/{useSSE-C9yVJlGP.js → useSSE-CXrQSbVY.js} +2 -2
  81. package/extensions/dashboard-client/dist/assets/{useSSE-C9yVJlGP.js.map → useSSE-CXrQSbVY.js.map} +1 -1
  82. package/extensions/dashboard-client/dist/index.html +2 -2
  83. package/extensions/dashboard-client/src/App.tsx +10 -2
  84. package/extensions/dashboard-client/src/api/client.ts +90 -1
  85. package/extensions/dashboard-client/src/main.tsx +1 -0
  86. package/extensions/dashboard-client/src/styles/turns.css +142 -0
  87. package/extensions/dashboard-client/src/tabs/TopicsTab.tsx +186 -0
  88. package/extensions/dashboard-client/src/tabs/TurnsTab.tsx +336 -0
  89. package/extensions/dashboard-server/api-contracts/endpoints.ts +394 -291
  90. package/extensions/dashboard-server/api-contracts/game-types.ts +141 -88
  91. package/extensions/dashboard-server/api-contracts/index.ts +153 -133
  92. package/extensions/dashboard-server/api-contracts/turns.ts +114 -0
  93. package/extensions/dashboard-server/api-contracts.test.ts +920 -919
  94. package/extensions/dashboard-server/routes-topics.ts +119 -0
  95. package/extensions/dashboard-server/routes-turns.ts +305 -0
  96. package/extensions/dashboard-server/routes.ts +8 -1
  97. package/extensions/dashboard-server/server.ts +6 -2
  98. package/extensions/mega-compact.ts +75 -59
  99. package/extensions/mega-config.ts +232 -220
  100. package/extensions/mega-events/agent-handlers.ts +194 -151
  101. package/extensions/mega-events/context-handler.ts +50 -1
  102. package/extensions/mega-events/session-handlers.ts +26 -5
  103. package/extensions/mega-metrics-cmds.ts +114 -0
  104. package/extensions/mega-pipeline/recall.ts +215 -152
  105. package/extensions/mega-topics-cmds.ts +151 -0
  106. package/extensions/mega-turn-store.test.ts +108 -0
  107. package/extensions/mega-turn-store.ts +169 -0
  108. package/package.json +3 -1
  109. package/src/config/turns.ts +69 -0
  110. package/src/fork.test.ts +94 -0
  111. package/src/fork.ts +78 -0
  112. package/src/intent.test.ts +87 -0
  113. package/src/intent.ts +115 -0
  114. package/src/metrics/index.ts +9 -0
  115. package/src/metrics/turns.test.ts +165 -0
  116. package/src/metrics/turns.ts +201 -0
  117. package/src/store/sqlite/raw-transcript.ts +9 -3
  118. package/src/store/sqlite.dbmirror.test.ts +10 -0
  119. package/src/store/turns/connection.ts +137 -0
  120. package/src/store/turns/contract-compliance.test.ts +562 -0
  121. package/src/store/turns/index.ts +65 -0
  122. package/src/store/turns/memory-store.test.ts +15 -0
  123. package/src/store/turns/memory-store.ts +469 -0
  124. package/src/store/turns/migrations.test.ts +161 -0
  125. package/src/store/turns/migrations.ts +203 -0
  126. package/src/store/turns/schema.ts +158 -0
  127. package/src/store/turns/sqlite-store.test.ts +161 -0
  128. package/src/store/turns/sqlite-store.ts +656 -0
  129. package/src/store/turns/types.ts +181 -0
  130. package/src/topics/cluster.test.ts +176 -0
  131. package/src/topics/cluster.ts +191 -0
  132. package/src/topics/index.ts +18 -0
  133. package/src/topics/kselection.ts +182 -0
  134. package/src/topics/labels.ts +67 -0
  135. package/src/topics/store.test.ts +132 -0
  136. package/src/topics/store.ts +208 -0
  137. package/src/topics/types.ts +64 -0
  138. package/src/wiki.test.ts +217 -0
  139. package/src/wiki.ts +198 -0
  140. package/dist/extensions/openclaw-mega-compact.js +0 -291
  141. package/dist/src/minilm.js +0 -92
  142. package/dist/src/wordpiece.js +0 -129
  143. package/extensions/dashboard-client/dist/assets/index-D_WtU2TV.js.map +0 -1
@@ -0,0 +1,151 @@
1
+ /**
2
+ * mega-topics-cmds.ts — S51C /mega-topics + /mega-topic commands.
3
+ *
4
+ * Lists auto-categorized wiki topics and renders wiki pages via ctx.ui.notify.
5
+ * Read-only (PREVENT-PI-001/002); local SQLite only (PREVENT-PI-004);
6
+ * parameterized queries (PREVENT-002). Requires turnsDbEnabled; no-ops with
7
+ * guidance when OFF or when AUTO_WIKI_ENABLED is false.
8
+ */
9
+
10
+ import type {
11
+ ExtensionAPI,
12
+ ExtensionContext,
13
+ } from "@earendil-works/pi-coding-agent";
14
+ import type { MegaRuntime } from "./mega-runtime.js";
15
+ import type { MegaConfig } from "./mega-config.js";
16
+ import { createTopicStore } from "../src/topics/store.js";
17
+ import { buildTopicModel } from "../src/topics/cluster.js";
18
+ import { openStore } from "../src/store/sqlite/utils.js";
19
+
20
+ /** Register the /mega-topics + /mega-topic commands. */
21
+ export function registerTopicsCommands(
22
+ pi: ExtensionAPI,
23
+ runtime: MegaRuntime,
24
+ config: MegaConfig,
25
+ ): void {
26
+ pi.registerCommand("mega-topics", {
27
+ description:
28
+ "List auto-categorized wiki topics (k-means + TF-IDF over real embeddings). Usage: /mega-topics",
29
+ handler: async (_args: string, ctx: ExtensionContext) => {
30
+ try {
31
+ if (!config.turnsDbEnabled || !config.autoWikiEnabled) {
32
+ ctx.ui.notify(
33
+ "[mega-compact] /mega-topics requires turnsDbEnabled + autoWikiEnabled.",
34
+ );
35
+ return;
36
+ }
37
+ runtime.bindRepo(ctx.cwd);
38
+ const stateDir = runtime.currentStateDir;
39
+ const topicStore = createTopicStore(stateDir);
40
+ const topics = topicStore.getTopics();
41
+ const stats = topicStore.getTopicStats();
42
+
43
+ if (topics.length === 0) {
44
+ ctx.ui.notify(
45
+ "[mega-compact] No topics yet. Topics are auto-generated after every 10th compaction from real memory embeddings (k-means + TF-IDF).",
46
+ );
47
+ return;
48
+ }
49
+
50
+ ctx.ui.notify(
51
+ `[mega-compact] ${stats.totalTopics} topic(s) · ${stats.totalAssigned} assigned memories` +
52
+ (stats.lastRebuildAt
53
+ ? ` · last rebuild ${new Date(stats.lastRebuildAt).toLocaleString()}`
54
+ : ""),
55
+ );
56
+ for (const t of topics) {
57
+ const terms = t.termScores
58
+ .slice(0, 5)
59
+ .map((s) => s.term)
60
+ .join(", ");
61
+ ctx.ui.notify(
62
+ ` ${t.id}: ${t.label} (${t.memoryCount} memories) — terms: ${terms}`,
63
+ );
64
+ }
65
+ } catch (e) {
66
+ ctx.ui.notify(`[mega-compact] /mega-topics failed: ${String(e)}`);
67
+ }
68
+ },
69
+ });
70
+
71
+ pi.registerCommand("mega-topic", {
72
+ description: "Show a wiki page for one topic. Usage: /mega-topic <topicId>",
73
+ handler: async (args: string, ctx: ExtensionContext) => {
74
+ try {
75
+ if (!config.turnsDbEnabled || !config.autoWikiEnabled) {
76
+ ctx.ui.notify(
77
+ "[mega-compact] /mega-topic requires turnsDbEnabled + autoWikiEnabled.",
78
+ );
79
+ return;
80
+ }
81
+ const topicId = args.trim();
82
+ if (!topicId) {
83
+ ctx.ui.notify(
84
+ "[mega-compact] /mega-topic needs a topic id, e.g. /mega-topic topic_0",
85
+ );
86
+ return;
87
+ }
88
+ runtime.bindRepo(ctx.cwd);
89
+ const stateDir = runtime.currentStateDir;
90
+ const topicStore = createTopicStore(stateDir);
91
+ const topics = topicStore.getTopics();
92
+ const topic = topics.find((t) => t.id === topicId);
93
+ if (!topic) {
94
+ ctx.ui.notify(
95
+ `[mega-compact] topic ${topicId} not found. Use /mega-topics to list.`,
96
+ );
97
+ return;
98
+ }
99
+
100
+ // Show topic details.
101
+ const assignments = topicStore.getMemoriesForTopic(topicId);
102
+ ctx.ui.notify(
103
+ `[mega-compact] ${topic.id}: ${topic.label} (${topic.memoryCount} memories)`,
104
+ );
105
+ const terms = topic.termScores
106
+ .slice(0, 8)
107
+ .map((s) => `${s.term} (${s.score.toFixed(2)})`)
108
+ .join(", ");
109
+ ctx.ui.notify(` terms: ${terms}`);
110
+ ctx.ui.notify(` assignments: ${assignments.length} memories`);
111
+ for (const a of assignments.slice(0, 10)) {
112
+ ctx.ui.notify(
113
+ ` ${a.memoryId} — confidence ${(a.confidence * 100).toFixed(1)}%`,
114
+ );
115
+ }
116
+ if (assignments.length > 10) {
117
+ ctx.ui.notify(` ... and ${assignments.length - 10} more`);
118
+ }
119
+ } catch (e) {
120
+ ctx.ui.notify(`[mega-compact] /mega-topic failed: ${String(e)}`);
121
+ }
122
+ },
123
+ });
124
+
125
+ pi.registerCommand("mega-topics-rebuild", {
126
+ description:
127
+ "Force-rebuild the topic model now (k-means + TF-IDF over real embeddings). Usage: /mega-topics-rebuild",
128
+ handler: async (_args: string, ctx: ExtensionContext) => {
129
+ try {
130
+ if (!config.turnsDbEnabled || !config.autoWikiEnabled) {
131
+ ctx.ui.notify(
132
+ "[mega-compact] /mega-topics-rebuild requires turnsDbEnabled + autoWikiEnabled.",
133
+ );
134
+ return;
135
+ }
136
+ runtime.bindRepo(ctx.cwd);
137
+ const stateDir = runtime.currentStateDir;
138
+ const db = openStore(stateDir);
139
+ const model = buildTopicModel(db);
140
+ createTopicStore(stateDir).replaceTopicModel(model);
141
+ ctx.ui.notify(
142
+ `[mega-compact] wiki rebuilt: ${model.k} topics from ${model.totalChunks} chunks (${model.criterion}, silhouette=${model.silhouetteScore?.toFixed(3) ?? "n/a"})`,
143
+ );
144
+ } catch (e) {
145
+ ctx.ui.notify(
146
+ `[mega-compact] /mega-topics-rebuild failed: ${String(e)}`,
147
+ );
148
+ }
149
+ },
150
+ });
151
+ }
@@ -0,0 +1,108 @@
1
+ /**
2
+ * mega-turn-store.test.ts — S49C adapter routing tests.
3
+ *
4
+ * Proves the adapter writes to the isolated turns.db when turnsDbEnabled is
5
+ * true and to the legacy main-db helpers when false. No network; temp dirs.
6
+ */
7
+
8
+ import { test, beforeEach, afterEach } from "node:test";
9
+ import assert from "node:assert/strict";
10
+ import { mkdtempSync, rmSync, existsSync } from "node:fs";
11
+ import { tmpdir } from "node:os";
12
+ import { join } from "node:path";
13
+ import { DatabaseSync } from "node:sqlite";
14
+ import {
15
+ ensureConversationIdFor,
16
+ recordTurnWrite,
17
+ recordRecallWrite,
18
+ } from "./mega-turn-store.js";
19
+ import { closeTurnStore, turnDbPath } from "../src/store/turns/index.js";
20
+ import type { MegaConfig } from "./mega-config.js";
21
+
22
+ let tmpDir: string;
23
+ let counter = 0;
24
+
25
+ beforeEach(() => {
26
+ tmpDir = mkdtempSync(join(tmpdir(), "mc-adapter-"));
27
+ });
28
+
29
+ afterEach(() => {
30
+ rmSync(tmpDir, { recursive: true, force: true });
31
+ });
32
+
33
+ function stateDir(): string {
34
+ return join(tmpDir, `run-${counter++}`);
35
+ }
36
+
37
+ /** Minimal MegaConfig stub — only the flag the adapter reads. */
38
+ function cfg(turnsDbEnabled: boolean): MegaConfig {
39
+ return { turnsDbEnabled } as unknown as MegaConfig;
40
+ }
41
+
42
+ test("flag ON → writes land in turns.db (not main sqlite.db)", () => {
43
+ const dir = stateDir();
44
+ const c = cfg(true);
45
+ const conv = ensureConversationIdFor(c, "sess_on", dir);
46
+ const turnId = recordTurnWrite(
47
+ c,
48
+ {
49
+ conversationId: conv,
50
+ sessionId: "sess_on",
51
+ turnIndex: 0,
52
+ role: "assistant",
53
+ endedAt: Date.now(),
54
+ },
55
+ dir,
56
+ );
57
+ recordRecallWrite(
58
+ c,
59
+ turnId,
60
+ [{ checkpointId: "cp_on", score: 0.5, source: "flat" }],
61
+ dir,
62
+ );
63
+ closeTurnStore(dir);
64
+ assert.ok(existsSync(turnDbPath(dir)), "turns.db should exist");
65
+ assert.ok(
66
+ !existsSync(join(dir, "sqlite.db")),
67
+ "main sqlite.db must NOT be created on the flag-ON path",
68
+ );
69
+ });
70
+
71
+ test("flag OFF → writes land in main sqlite.db (legacy S48 path)", () => {
72
+ const dir = stateDir();
73
+ const c = cfg(false);
74
+ const conv = ensureConversationIdFor(c, "sess_off", dir);
75
+ const turnId = recordTurnWrite(
76
+ c,
77
+ {
78
+ conversationId: conv,
79
+ sessionId: "sess_off",
80
+ turnIndex: 0,
81
+ role: "assistant",
82
+ endedAt: Date.now(),
83
+ },
84
+ dir,
85
+ );
86
+ recordRecallWrite(
87
+ c,
88
+ turnId,
89
+ [{ checkpointId: "cp_off", score: 0.5, source: "flat" }],
90
+ dir,
91
+ );
92
+ // Legacy helpers open the main sqlite.db.
93
+ assert.ok(
94
+ existsSync(join(dir, "sqlite.db")),
95
+ "main sqlite.db should exist on the flag-OFF path",
96
+ );
97
+ assert.ok(
98
+ !existsSync(turnDbPath(dir)),
99
+ "turns.db must NOT be created on the flag-OFF path",
100
+ );
101
+ // And the row is actually in the main db's turns table.
102
+ const main = new DatabaseSync(join(dir, "sqlite.db"));
103
+ const rows = main
104
+ .prepare("SELECT * FROM turns WHERE session_id = ?")
105
+ .all("sess_off") as unknown[];
106
+ assert.equal(rows.length, 1);
107
+ main.close();
108
+ });
@@ -0,0 +1,169 @@
1
+ /**
2
+ * mega-turn-store.ts — S49 adapter: route per-turn writes to the isolated
3
+ * turns.db (flag ON) or the legacy main-db helpers (flag OFF).
4
+ *
5
+ * RECONCILIATION: maps the extension's per-turn writes onto the contract-first
6
+ * `TurnStore` (master's design): `appendTurn` / `appendRecall` /
7
+ * `ensureConversationId` / `asAdmin().stampTurnsEpoch` / `forkConversation`.
8
+ * src/store/turns/ stays pi-agnostic; this module is the ONLY pi-coupled seam for
9
+ * turn writes. Both paths are best-effort at the call site (try/catch in the
10
+ * handlers), so a failure never breaks the agent loop or the recall path.
11
+ */
12
+ import {
13
+ createTurnStore,
14
+ type TurnStore,
15
+ type TurnEntry,
16
+ type TurnRecallEntry,
17
+ } from "../src/store/turns/index.js";
18
+ import {
19
+ recordTurn as legacyRecordTurn,
20
+ recordTurnRecall as legacyRecordTurnRecall,
21
+ ensureConversationId as legacyEnsureConversationId,
22
+ } from "../src/store/sqlite/turns.js";
23
+ import type { MegaConfig } from "./mega-config.js";
24
+ import type { DatabaseSync } from "node:sqlite";
25
+ import { openStore } from "../src/store/sqlite/utils.js";
26
+
27
+ // Per-stateDir cache of turn stores so the extension doesn't reopen the file
28
+ // on every turn. Keyed by stateDir; the underlying connection is also cached
29
+ // in src/store/turns/connection.ts, so this is a thin factory memo.
30
+ const stores = new Map<string, TurnStore>();
31
+
32
+ function storeFor(stateDir: string): TurnStore {
33
+ let s = stores.get(stateDir);
34
+ if (!s) {
35
+ s = createTurnStore({ stateDir });
36
+ stores.set(stateDir, s);
37
+ }
38
+ return s;
39
+ }
40
+
41
+ /** Map a legacy RecallSource onto the contract `TurnRecallEntry.source` enum. */
42
+ function mapSource(s: string): TurnRecallEntry["source"] {
43
+ switch (s) {
44
+ case "flat":
45
+ case "checkpoint":
46
+ return "checkpoint";
47
+ case "raptor":
48
+ case "cluster_summary":
49
+ return "cluster_summary";
50
+ default:
51
+ return "memory";
52
+ }
53
+ }
54
+
55
+ /** Resolve (or generate) a session's conversation id on the active backend. */
56
+ export function ensureConversationIdFor(
57
+ config: MegaConfig,
58
+ sessionId: string,
59
+ stateDir: string,
60
+ ): string {
61
+ return config.turnsDbEnabled
62
+ ? storeFor(stateDir).ensureConversationId(sessionId)
63
+ : legacyEnsureConversationId(sessionId, stateDir);
64
+ }
65
+
66
+ /** Record one turn row (turn_end). Returns the contract TurnId (string). */
67
+ export function recordTurnWrite(
68
+ config: MegaConfig,
69
+ input: {
70
+ conversationId: string;
71
+ sessionId: string;
72
+ turnIndex: number;
73
+ role: string;
74
+ startedAt?: number;
75
+ endedAt?: number;
76
+ ctxTokens?: number;
77
+ ctxPercent?: number;
78
+ pressureBand?: string;
79
+ modelId?: string;
80
+ epochId?: string;
81
+ },
82
+ stateDir: string,
83
+ ): string {
84
+ if (!config.turnsDbEnabled) {
85
+ legacyRecordTurn(input as never, stateDir);
86
+ return "";
87
+ }
88
+ const entry: TurnEntry = {
89
+ conversationId: input.conversationId,
90
+ sessionId: input.sessionId,
91
+ turnIndex: input.turnIndex,
92
+ role: input.role as TurnEntry["role"],
93
+ endedAt: input.endedAt ?? input.startedAt ?? Date.now(),
94
+ ctxTokens: input.ctxTokens,
95
+ ctxPercent: input.ctxPercent,
96
+ pressureBand: input.pressureBand as TurnEntry["pressureBand"],
97
+ model: input.modelId,
98
+ epochId: input.epochId,
99
+ };
100
+ return storeFor(stateDir).appendTurn(entry);
101
+ }
102
+
103
+ /** Record recall provenance for a turn (one hit at a time — contract is append-only). */
104
+ export function recordRecallWrite(
105
+ config: MegaConfig,
106
+ turnId: string | number,
107
+ hits: Array<{
108
+ checkpointId: string;
109
+ score: number;
110
+ source: string;
111
+ raptorLevel?: number;
112
+ }>,
113
+ stateDir: string,
114
+ ): void {
115
+ if (!config.turnsDbEnabled) {
116
+ legacyRecordTurnRecall(Number(turnId), hits as never, stateDir);
117
+ return;
118
+ }
119
+ const writer = storeFor(stateDir).asWriter();
120
+ for (const h of hits) {
121
+ const entry: TurnRecallEntry = {
122
+ turnId: String(turnId),
123
+ checkpointId: h.checkpointId,
124
+ score: h.score,
125
+ source: mapSource(h.source),
126
+ raptorLevel: h.raptorLevel,
127
+ };
128
+ writer.appendRecall(entry);
129
+ }
130
+ }
131
+
132
+ /** S50B: stamp `epoch_id` on a session's unstamped turns (compact-commit).
133
+ * Isolated-store only — the legacy main-db turn path is being retired, so it
134
+ * is a no-op when turnsDbEnabled is false. Returns the number stamped. */
135
+ export function stampTurnsEpochFor(
136
+ config: MegaConfig,
137
+ sessionId: string,
138
+ epochId: string,
139
+ stateDir: string,
140
+ ): number {
141
+ return config.turnsDbEnabled
142
+ ? storeFor(stateDir).asAdmin().stampTurnsEpoch(sessionId, epochId)
143
+ : 0;
144
+ }
145
+
146
+ /** S50C: resolve the active turn store for metrics/fork commands.
147
+ * Returns null when turnsDbEnabled is OFF (legacy main-db turn path — the
148
+ * per-turn metrics/fork commands are isolated-store only). */
149
+ export function turnStoreFor(
150
+ config: MegaConfig,
151
+ stateDir: string,
152
+ ): TurnStore | null {
153
+ return config.turnsDbEnabled ? storeFor(stateDir) : null;
154
+ }
155
+
156
+ /** S50C: open (cached) the main db handle for metrics read-queries
157
+ * (raw_transcript + checkpoint_epochs). Callers must NOT close it. */
158
+ export function mainDbFor(stateDir: string): DatabaseSync {
159
+ return openStore(stateDir);
160
+ }
161
+
162
+ /** Test/teardown: close all cached turn stores for a state dir. */
163
+ export function closeTurnStoreFor(stateDir: string): void {
164
+ const s = stores.get(stateDir);
165
+ if (s) {
166
+ s.close();
167
+ stores.delete(stateDir);
168
+ }
169
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "pi-mega-compact",
3
- "version": "0.9.2",
3
+ "version": "0.11.0",
4
4
  "description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
5
5
  "type": "module",
6
6
  "license": "BSD-3-Clause",
@@ -41,6 +41,8 @@
41
41
  "compactionProvider": "mega-compact"
42
42
  },
43
43
  "scripts": {
44
+ "clean": "rm -rf dist",
45
+ "prebuild": "npm run clean",
44
46
  "build": "tsc -p tsconfig.json",
45
47
  "build:dashboard": "cd extensions/dashboard-client && npx vite build",
46
48
  "lint": "tsc --noEmit && node scripts/guardrails-scan.mjs && node scripts/semantic-scan.mjs",
@@ -0,0 +1,69 @@
1
+ /**
2
+ * config/turns.ts — SINGLE SOURCE OF TRUTH for the S49 turn-store flags.
3
+ *
4
+ * Feature-flag semantics (docs/specs/s49-turn-db-foundation.md):
5
+ * - TURNS_DB_ENABLED defaults ON (isolated turns.db). OFF = byte-identical S48
6
+ * behavior (turn helpers use the main sqlite.db). Opt-out, not opt-in.
7
+ * - All values read from MEGACOMPACT_* env at load, with the defaults below as
8
+ * fallback. Booleans/numbers only — no calibration.
9
+ *
10
+ * PREVENT-PI-004: pure config, no network.
11
+ */
12
+
13
+ function envBool(name: string, def: boolean): boolean {
14
+ const v = process.env[name];
15
+ if (v === undefined) return def;
16
+ return v === "true" || v === "1";
17
+ }
18
+
19
+ function envNum(name: string, def: number): number {
20
+ const v = process.env[name];
21
+ if (v === undefined) return def;
22
+ const n = Number(v);
23
+ return Number.isFinite(n) ? n : def;
24
+ }
25
+
26
+ export interface TurnsConfigShape {
27
+ /** Isolated turns.db store enabled (default true). OFF = legacy main-db path. */
28
+ TURNS_DB_ENABLED: boolean;
29
+ /** Per-turn provenance retention window, days (default 30). */
30
+ TURNS_RETENTION_DAYS: number;
31
+ /** Minimum turns always kept per conversation during prune (default 5). */
32
+ TURNS_KEEP_MIN_PER_CONVERSATION: number;
33
+ /** S51: auto-categorizing wiki (k-means + TF-IDF over real embeddings). Default ON. */
34
+ AUTO_WIKI_ENABLED: boolean;
35
+ /** S51: [minK, maxK] cluster-count search space (default 3..15). */
36
+ WIKI_K_MIN: number;
37
+ WIKI_K_MAX: number;
38
+ /** S51: how many TF-IDF terms form a topic label (default 5). */
39
+ WIKI_LABEL_TOP_TERMS: number;
40
+ /** S51: rebuild the topic model every Nth compaction (default 10). */
41
+ WIKI_REBUILD_EVERY_N_COMPACTS: number;
42
+ }
43
+
44
+ function envKRange(defMin: number, defMax: number): [number, number] {
45
+ const v = process.env.MEGACOMPACT_WIKI_K_RANGE;
46
+ if (v) {
47
+ const parts = v.split(",").map((s) => Number(s.trim()));
48
+ if (parts.length === 2 && parts.every((n) => Number.isFinite(n) && n >= 1)) {
49
+ return [Math.floor(parts[0]), Math.floor(parts[1])];
50
+ }
51
+ }
52
+ return [defMin, defMax];
53
+ }
54
+
55
+ export function loadTurnsConfig(): TurnsConfigShape {
56
+ const [kMin, kMax] = envKRange(3, 15);
57
+ return {
58
+ TURNS_DB_ENABLED: envBool("MEGACOMPACT_TURNS_DB", true),
59
+ TURNS_RETENTION_DAYS: envNum("MEGACOMPACT_TURNS_RETENTION_DAYS", 30),
60
+ TURNS_KEEP_MIN_PER_CONVERSATION: envNum("MEGACOMPACT_TURNS_KEEP_MIN", 5),
61
+ AUTO_WIKI_ENABLED: envBool("MEGACOMPACT_AUTO_WIKI", true),
62
+ WIKI_K_MIN: kMin,
63
+ WIKI_K_MAX: kMax,
64
+ WIKI_LABEL_TOP_TERMS: envNum("MEGACOMPACT_WIKI_LABEL_TOP_TERMS", 5),
65
+ WIKI_REBUILD_EVERY_N_COMPACTS: envNum("MEGACOMPACT_WIKI_REBUILD_EVERY", 10),
66
+ };
67
+ }
68
+
69
+ export const TurnsConfig: TurnsConfigShape = loadTurnsConfig();
@@ -0,0 +1,94 @@
1
+ /**
2
+ * fork.test.ts — S50C fork primitive tests. No network; temp dirs.
3
+ *
4
+ * Uses the contract-first TurnStore (createTurnStore({ stateDir })).
5
+ */
6
+ import { test, beforeEach, afterEach } from "node:test";
7
+ import assert from "node:assert/strict";
8
+ import { mkdtempSync, rmSync } from "node:fs";
9
+ import { tmpdir } from "node:os";
10
+ import { join } from "node:path";
11
+ import { createTurnStore, closeAllTurnDbs } from "./store/turns/index.js";
12
+ import { forkFromConversation, ForkError } from "./fork.js";
13
+
14
+ let tmpDir: string;
15
+ let counter = 0;
16
+
17
+ beforeEach(() => {
18
+ tmpDir = mkdtempSync(join(tmpdir(), "mc-fork-"));
19
+ });
20
+
21
+ afterEach(() => {
22
+ closeAllTurnDbs();
23
+ rmSync(tmpDir, { recursive: true, force: true });
24
+ });
25
+
26
+ function stateDir(): string {
27
+ return join(tmpDir, `run-${counter++}`);
28
+ }
29
+
30
+ test("fork resolves the turn + returns the recall set to rehydrate", () => {
31
+ const dir = stateDir();
32
+ const store = createTurnStore({ stateDir: dir });
33
+ const conv = store.ensureConversationId("sess_f");
34
+ const turnId = store.appendTurn({
35
+ conversationId: conv,
36
+ sessionId: "sess_f",
37
+ turnIndex: 2,
38
+ role: "assistant",
39
+ endedAt: Date.now(),
40
+ });
41
+ store.appendRecall({
42
+ turnId: turnId,
43
+ checkpointId: "cp_a",
44
+ score: 0.9,
45
+ source: "checkpoint",
46
+ });
47
+ store.appendRecall({
48
+ turnId: turnId,
49
+ checkpointId: "cp_b",
50
+ score: 0.7,
51
+ source: "cluster_summary",
52
+ raptorLevel: 1,
53
+ });
54
+ const out = forkFromConversation(store, conv, 2);
55
+ assert.ok(out.childConversationId.startsWith("conv_"));
56
+ assert.notEqual(out.childConversationId, conv);
57
+ assert.equal(out.forkTurn.turnIndex, 2);
58
+ assert.deepEqual(out.checkpointIds.sort(), ["cp_a", "cp_b"]);
59
+ assert.equal(out.recalled.length, 2);
60
+ store.close();
61
+ });
62
+
63
+ test("unknown turn → ForkError TURN_NOT_FOUND", () => {
64
+ const dir = stateDir();
65
+ const store = createTurnStore({ stateDir: dir });
66
+ const conv = store.ensureConversationId("sess_g");
67
+ assert.throws(
68
+ () => forkFromConversation(store, conv, 99),
69
+ (e: unknown) => {
70
+ return e instanceof ForkError && e.code === "TURN_NOT_FOUND";
71
+ },
72
+ );
73
+ store.close();
74
+ });
75
+
76
+ test("turn with no recall set → ForkError NO_RECALL", () => {
77
+ const dir = stateDir();
78
+ const store = createTurnStore({ stateDir: dir });
79
+ const conv = store.ensureConversationId("sess_h");
80
+ store.appendTurn({
81
+ conversationId: conv,
82
+ sessionId: "sess_h",
83
+ turnIndex: 0,
84
+ role: "assistant",
85
+ endedAt: Date.now(),
86
+ });
87
+ assert.throws(
88
+ () => forkFromConversation(store, conv, 0),
89
+ (e: unknown) => {
90
+ return e instanceof ForkError && e.code === "NO_RECALL";
91
+ },
92
+ );
93
+ store.close();
94
+ });
package/src/fork.ts ADDED
@@ -0,0 +1,78 @@
1
+ /**
2
+ * fork.ts — S50C host-agnostic conversation-fork primitive.
3
+ *
4
+ * Pi-agnostic: consumes only the S49 `TurnStore` contract. A "fork" branches a
5
+ * new child conversation off a parent's turn N and inherits that turn's
6
+ * injected-checkpoint set as its starting recall state (recall-to-point — NOT a
7
+ * live-window replay, which stays an S48 non-goal). The host (pi command,
8
+ * dashboard intent, own TUI, API gateway) supplies the store + identifiers and
9
+ * applies the returned recall set to its own session state.
10
+ *
11
+ * PREVENT-PI-001/002: fork writes to conversation_forks only; it never mutates
12
+ * memory, drop ranges, or the parent's turns. No network (PREVENT-PI-004).
13
+ */
14
+ import type {
15
+ TurnStore,
16
+ TurnEntry,
17
+ TurnRecallEntry,
18
+ } from "./store/turns/types.js";
19
+
20
+ /** Result of a fork: the new child conversation + the recall set to rehydrate. */
21
+ export interface ForkOutcome {
22
+ /** Newly-created child conversation id. */
23
+ childConversationId: string;
24
+ /** The parent turn the fork branched from (contract TurnEntry). */
25
+ forkTurn: TurnEntry;
26
+ /** The parent's injected checkpoints at the fork turn (the replay set). */
27
+ recalled: TurnRecallEntry[];
28
+ /** Distinct checkpoint ids to seed into the child's injected-set. */
29
+ checkpointIds: string[];
30
+ }
31
+
32
+ /** Typed error so hosts can distinguish "unknown turn" from storage failures. */
33
+ export class ForkError extends Error {
34
+ readonly code: "TURN_NOT_FOUND" | "NO_RECALL";
35
+ constructor(code: ForkError["code"], message: string) {
36
+ super(message);
37
+ this.name = "ForkError";
38
+ this.code = code;
39
+ }
40
+ }
41
+
42
+ /**
43
+ * Fork `parentConversationId` at `turnIndex`. Resolves the turn entry (by
44
+ * conversation + index via `getTurnByIndex`), creates the child conversation via
45
+ * `store.forkConversation` (returns the child ConversationId), and returns the
46
+ * parent turn's recorded recall set (`listRecallByIndex`) to rehydrate.
47
+ *
48
+ * @throws ForkError TURN_NOT_FOUND when the (conversation, turnIndex) row is absent.
49
+ * @throws ForkError NO_RECALL when the fork turn has no injected checkpoints to replay.
50
+ */
51
+ export function forkFromConversation(
52
+ store: TurnStore,
53
+ parentConversationId: string,
54
+ turnIndex: number,
55
+ ): ForkOutcome {
56
+ const turn = store.getTurnByIndex(parentConversationId, turnIndex);
57
+ if (!turn) {
58
+ throw new ForkError(
59
+ "TURN_NOT_FOUND",
60
+ `no turn ${turnIndex} in conversation ${parentConversationId}`,
61
+ );
62
+ }
63
+ const childConversationId = store.forkConversation(
64
+ parentConversationId,
65
+ turnIndex,
66
+ );
67
+ const recalled = store.listRecallByIndex(parentConversationId, turnIndex);
68
+ const checkpointIds = [...new Set(recalled.map((r) => r.checkpointId))];
69
+ if (checkpointIds.length === 0) {
70
+ // The child conversation was still created (lineage is recorded), but a fork
71
+ // with nothing to rehydrate is almost always a caller mistake — surface it.
72
+ throw new ForkError(
73
+ "NO_RECALL",
74
+ `turn ${turnIndex} in ${parentConversationId} has no injected checkpoints to fork from`,
75
+ );
76
+ }
77
+ return { childConversationId, forkTurn: turn, recalled, checkpointIds };
78
+ }