pi-mega-compact 0.9.2 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. package/README.md +3 -2
  2. package/dist/extensions/dashboard-server/api-contracts/endpoints.js +95 -47
  3. package/dist/extensions/dashboard-server/api-contracts/index.js +1 -1
  4. package/dist/extensions/dashboard-server/api-contracts/turns.js +10 -0
  5. package/dist/extensions/dashboard-server/api-contracts.test.js +3 -2
  6. package/dist/extensions/dashboard-server/routes-topics.js +87 -0
  7. package/dist/extensions/dashboard-server/routes-turns.js +264 -0
  8. package/dist/extensions/dashboard-server/routes.js +3 -1
  9. package/dist/extensions/dashboard-server/server.js +6 -2
  10. package/dist/extensions/mega-compact.js +10 -6
  11. package/dist/extensions/mega-config.js +4 -0
  12. package/dist/extensions/mega-events/agent-handlers.js +62 -49
  13. package/dist/extensions/mega-events/context-handler.js +48 -2
  14. package/dist/extensions/mega-events/session-handlers.js +25 -1
  15. package/dist/extensions/mega-metrics-cmds.js +90 -0
  16. package/dist/extensions/mega-pipeline/recall.js +47 -17
  17. package/dist/extensions/mega-topics-cmds.js +111 -0
  18. package/dist/extensions/mega-turn-store.js +110 -0
  19. package/dist/extensions/mega-turn-store.test.js +68 -0
  20. package/dist/src/config/turns.js +48 -0
  21. package/dist/src/fork.js +33 -0
  22. package/dist/src/fork.test.js +81 -0
  23. package/dist/src/intent.js +51 -0
  24. package/dist/src/intent.test.js +78 -0
  25. package/dist/src/metrics/index.js +4 -0
  26. package/dist/src/metrics/turns.js +118 -0
  27. package/dist/src/metrics/turns.test.js +152 -0
  28. package/dist/src/store/sqlite/raw-transcript.js +5 -3
  29. package/dist/src/store/sqlite.dbmirror.test.js +9 -0
  30. package/dist/src/store/turns/connection.js +123 -0
  31. package/dist/src/store/turns/contract-compliance.test.js +428 -0
  32. package/dist/src/store/turns/index.js +31 -0
  33. package/dist/src/store/turns/memory-store.js +357 -0
  34. package/dist/src/store/turns/memory-store.test.js +9 -0
  35. package/dist/src/store/turns/migrations.js +178 -0
  36. package/dist/src/store/turns/migrations.test.js +138 -0
  37. package/dist/src/store/turns/schema.js +115 -0
  38. package/dist/src/store/turns/sqlite-store.js +455 -0
  39. package/dist/src/store/turns/sqlite-store.test.js +144 -0
  40. package/dist/src/store/turns/types.js +16 -0
  41. package/dist/src/topics/cluster.js +141 -0
  42. package/dist/src/topics/cluster.test.js +150 -0
  43. package/dist/src/topics/index.js +6 -0
  44. package/dist/src/topics/kselection.js +142 -0
  45. package/dist/src/topics/labels.js +56 -0
  46. package/dist/src/topics/store.js +119 -0
  47. package/dist/src/topics/store.test.js +117 -0
  48. package/dist/src/topics/types.js +8 -0
  49. package/dist/src/wiki.js +133 -0
  50. package/dist/src/wiki.test.js +171 -0
  51. package/extensions/dashboard-client/dist/assets/{AchievementTiles-QPo46Pwg.js → AchievementTiles-DZp_xQOW.js} +2 -2
  52. package/extensions/dashboard-client/dist/assets/{AchievementTiles-QPo46Pwg.js.map → AchievementTiles-DZp_xQOW.js.map} +1 -1
  53. package/extensions/dashboard-client/dist/assets/{AchievementsTab-BDJxRXRY.js → AchievementsTab-BzFHpAHM.js} +2 -2
  54. package/extensions/dashboard-client/dist/assets/{AchievementsTab-BDJxRXRY.js.map → AchievementsTab-BzFHpAHM.js.map} +1 -1
  55. package/extensions/dashboard-client/dist/assets/{CacheTab-CUMqXQjd.js → CacheTab-DEQbTq2T.js} +2 -2
  56. package/extensions/dashboard-client/dist/assets/{CacheTab-CUMqXQjd.js.map → CacheTab-DEQbTq2T.js.map} +1 -1
  57. package/extensions/dashboard-client/dist/assets/{ConfigTab-j1dRzurG.js → ConfigTab-DQ4FG33J.js} +2 -2
  58. package/extensions/dashboard-client/dist/assets/{ConfigTab-j1dRzurG.js.map → ConfigTab-DQ4FG33J.js.map} +1 -1
  59. package/extensions/dashboard-client/dist/assets/{EventsTab-CMglXK4p.js → EventsTab-D5HoEVYo.js} +2 -2
  60. package/extensions/dashboard-client/dist/assets/{EventsTab-CMglXK4p.js.map → EventsTab-D5HoEVYo.js.map} +1 -1
  61. package/extensions/dashboard-client/dist/assets/{GameTab-fpaDoyxi.js → GameTab-Jzsc4yeY.js} +2 -2
  62. package/extensions/dashboard-client/dist/assets/{GameTab-fpaDoyxi.js.map → GameTab-Jzsc4yeY.js.map} +1 -1
  63. package/extensions/dashboard-client/dist/assets/{MetricsTab-CWG2crG_.js → MetricsTab-D_XnsEYX.js} +2 -2
  64. package/extensions/dashboard-client/dist/assets/{MetricsTab-CWG2crG_.js.map → MetricsTab-D_XnsEYX.js.map} +1 -1
  65. package/extensions/dashboard-client/dist/assets/{OverviewTab-BaV2D2nf.js → OverviewTab-BMREvAth.js} +2 -2
  66. package/extensions/dashboard-client/dist/assets/{OverviewTab-BaV2D2nf.js.map → OverviewTab-BMREvAth.js.map} +1 -1
  67. package/extensions/dashboard-client/dist/assets/{ReposTab-C-0kuZ_0.js → ReposTab-DuyhOktG.js} +2 -2
  68. package/extensions/dashboard-client/dist/assets/{ReposTab-C-0kuZ_0.js.map → ReposTab-DuyhOktG.js.map} +1 -1
  69. package/extensions/dashboard-client/dist/assets/{SessionsTab-CSYhddCA.js → SessionsTab-mz_vz8i2.js} +2 -2
  70. package/extensions/dashboard-client/dist/assets/{SessionsTab-CSYhddCA.js.map → SessionsTab-mz_vz8i2.js.map} +1 -1
  71. package/extensions/dashboard-client/dist/assets/{TimeSavedCard-Dy4KRjSM.js → TimeSavedCard-B8AIjNTY.js} +2 -2
  72. package/extensions/dashboard-client/dist/assets/{TimeSavedCard-Dy4KRjSM.js.map → TimeSavedCard-B8AIjNTY.js.map} +1 -1
  73. package/extensions/dashboard-client/dist/assets/TopicsTab-DKuct4oD.js +2 -0
  74. package/extensions/dashboard-client/dist/assets/TopicsTab-DKuct4oD.js.map +1 -0
  75. package/extensions/dashboard-client/dist/assets/TurnsTab-11MGLxY8.js +2 -0
  76. package/extensions/dashboard-client/dist/assets/TurnsTab-11MGLxY8.js.map +1 -0
  77. package/extensions/dashboard-client/dist/assets/{index-D_WtU2TV.js → index-CqLN7STl.js} +10 -10
  78. package/extensions/dashboard-client/dist/assets/index-CqLN7STl.js.map +1 -0
  79. package/extensions/dashboard-client/dist/assets/{index-og3ghxyY.css → index-gn66lYD2.css} +1 -1
  80. package/extensions/dashboard-client/dist/assets/{useSSE-C9yVJlGP.js → useSSE-CXrQSbVY.js} +2 -2
  81. package/extensions/dashboard-client/dist/assets/{useSSE-C9yVJlGP.js.map → useSSE-CXrQSbVY.js.map} +1 -1
  82. package/extensions/dashboard-client/dist/index.html +2 -2
  83. package/extensions/dashboard-client/src/App.tsx +10 -2
  84. package/extensions/dashboard-client/src/api/client.ts +90 -1
  85. package/extensions/dashboard-client/src/main.tsx +1 -0
  86. package/extensions/dashboard-client/src/styles/turns.css +142 -0
  87. package/extensions/dashboard-client/src/tabs/TopicsTab.tsx +186 -0
  88. package/extensions/dashboard-client/src/tabs/TurnsTab.tsx +336 -0
  89. package/extensions/dashboard-server/api-contracts/endpoints.ts +394 -291
  90. package/extensions/dashboard-server/api-contracts/game-types.ts +141 -88
  91. package/extensions/dashboard-server/api-contracts/index.ts +153 -133
  92. package/extensions/dashboard-server/api-contracts/turns.ts +114 -0
  93. package/extensions/dashboard-server/api-contracts.test.ts +920 -919
  94. package/extensions/dashboard-server/routes-topics.ts +119 -0
  95. package/extensions/dashboard-server/routes-turns.ts +305 -0
  96. package/extensions/dashboard-server/routes.ts +8 -1
  97. package/extensions/dashboard-server/server.ts +6 -2
  98. package/extensions/mega-compact.ts +75 -59
  99. package/extensions/mega-config.ts +232 -220
  100. package/extensions/mega-events/agent-handlers.ts +194 -151
  101. package/extensions/mega-events/context-handler.ts +50 -1
  102. package/extensions/mega-events/session-handlers.ts +26 -5
  103. package/extensions/mega-metrics-cmds.ts +114 -0
  104. package/extensions/mega-pipeline/recall.ts +215 -152
  105. package/extensions/mega-topics-cmds.ts +151 -0
  106. package/extensions/mega-turn-store.test.ts +108 -0
  107. package/extensions/mega-turn-store.ts +169 -0
  108. package/package.json +3 -1
  109. package/src/config/turns.ts +69 -0
  110. package/src/fork.test.ts +94 -0
  111. package/src/fork.ts +78 -0
  112. package/src/intent.test.ts +87 -0
  113. package/src/intent.ts +115 -0
  114. package/src/metrics/index.ts +9 -0
  115. package/src/metrics/turns.test.ts +165 -0
  116. package/src/metrics/turns.ts +201 -0
  117. package/src/store/sqlite/raw-transcript.ts +9 -3
  118. package/src/store/sqlite.dbmirror.test.ts +10 -0
  119. package/src/store/turns/connection.ts +137 -0
  120. package/src/store/turns/contract-compliance.test.ts +562 -0
  121. package/src/store/turns/index.ts +65 -0
  122. package/src/store/turns/memory-store.test.ts +15 -0
  123. package/src/store/turns/memory-store.ts +469 -0
  124. package/src/store/turns/migrations.test.ts +161 -0
  125. package/src/store/turns/migrations.ts +203 -0
  126. package/src/store/turns/schema.ts +158 -0
  127. package/src/store/turns/sqlite-store.test.ts +161 -0
  128. package/src/store/turns/sqlite-store.ts +656 -0
  129. package/src/store/turns/types.ts +181 -0
  130. package/src/topics/cluster.test.ts +176 -0
  131. package/src/topics/cluster.ts +191 -0
  132. package/src/topics/index.ts +18 -0
  133. package/src/topics/kselection.ts +182 -0
  134. package/src/topics/labels.ts +67 -0
  135. package/src/topics/store.test.ts +132 -0
  136. package/src/topics/store.ts +208 -0
  137. package/src/topics/types.ts +64 -0
  138. package/src/wiki.test.ts +217 -0
  139. package/src/wiki.ts +198 -0
  140. package/dist/extensions/openclaw-mega-compact.js +0 -291
  141. package/dist/src/minilm.js +0 -92
  142. package/dist/src/wordpiece.js +0 -129
  143. package/extensions/dashboard-client/dist/assets/index-D_WtU2TV.js.map +0 -1
@@ -0,0 +1,87 @@
1
+ /**
2
+ * intent.test.ts — S52A rewind-intent queue tests. No network; temp dirs.
3
+ */
4
+ import { test, beforeEach, afterEach } from "node:test";
5
+ import assert from "node:assert/strict";
6
+ import { mkdtempSync, rmSync } from "node:fs";
7
+ import { tmpdir } from "node:os";
8
+ import { join } from "node:path";
9
+ import { openTurnStore, closeAllTurnDbs } from "./store/turns/connection.js";
10
+ import { openIntentQueue } from "./intent.js";
11
+
12
+ let tmpDir: string;
13
+ let counter = 0;
14
+
15
+ beforeEach(() => {
16
+ tmpDir = mkdtempSync(join(tmpdir(), "mc-intent-"));
17
+ });
18
+
19
+ afterEach(() => {
20
+ closeAllTurnDbs();
21
+ rmSync(tmpDir, { recursive: true, force: true });
22
+ });
23
+
24
+ function stateDir(): string {
25
+ return join(tmpDir, `run-${counter++}`);
26
+ }
27
+
28
+ test("postIntent creates a pending intent", () => {
29
+ const dir = stateDir();
30
+ const db = openTurnStore(dir);
31
+ const q = openIntentQueue(db);
32
+ const intent = q.postIntent({ conversationId: "conv_a", targetTurnIndex: 3 });
33
+ assert.equal(intent.conversationId, "conv_a");
34
+ assert.equal(intent.targetTurnIndex, 3);
35
+ assert.equal(intent.status, "pending");
36
+ assert.ok(intent.id.length > 0);
37
+ assert.ok(intent.createdAt > 0);
38
+ });
39
+
40
+ test("pendingIntents lists only unconsumed", () => {
41
+ const dir = stateDir();
42
+ const db = openTurnStore(dir);
43
+ const q = openIntentQueue(db);
44
+ q.postIntent({ conversationId: "c1", targetTurnIndex: 1 });
45
+ const i2 = q.postIntent({ conversationId: "c2", targetTurnIndex: 2 });
46
+ assert.equal(q.pendingIntents().length, 2);
47
+ q.consumeIntent(i2.id);
48
+ assert.equal(q.pendingIntents().length, 1);
49
+ assert.equal(q.pendingIntents()[0].conversationId, "c1");
50
+ });
51
+
52
+ test("consumeIntent marks status consumed (not deleted)", () => {
53
+ const dir = stateDir();
54
+ const db = openTurnStore(dir);
55
+ const q = openIntentQueue(db);
56
+ const intent = q.postIntent({ conversationId: "c", targetTurnIndex: 0 });
57
+ q.consumeIntent(intent.id);
58
+ assert.equal(q.pendingIntents().length, 0);
59
+ const all = q.allIntents();
60
+ assert.equal(all.length, 1);
61
+ assert.equal(all[0].status, "consumed");
62
+ assert.equal(all[0].id, intent.id);
63
+ });
64
+
65
+ test("abandonIntent deletes the intent", () => {
66
+ const dir = stateDir();
67
+ const db = openTurnStore(dir);
68
+ const q = openIntentQueue(db);
69
+ const intent = q.postIntent({ conversationId: "c", targetTurnIndex: 0 });
70
+ q.abandonIntent(intent.id);
71
+ assert.equal(q.pendingIntents().length, 0);
72
+ assert.equal(q.allIntents().length, 0);
73
+ });
74
+
75
+ test("allIntents returns newest-first + respects limit", () => {
76
+ const dir = stateDir();
77
+ const db = openTurnStore(dir);
78
+ const q = openIntentQueue(db);
79
+ q.postIntent({ conversationId: "a", targetTurnIndex: 0 });
80
+ q.postIntent({ conversationId: "b", targetTurnIndex: 0 });
81
+ q.postIntent({ conversationId: "c", targetTurnIndex: 0 });
82
+ const all = q.allIntents();
83
+ assert.equal(all.length, 3);
84
+ // newest-first: c was posted last
85
+ assert.equal(all[0].conversationId, "c");
86
+ assert.equal(q.allIntents(2).length, 2);
87
+ });
package/src/intent.ts ADDED
@@ -0,0 +1,115 @@
1
+ /**
2
+ * intent.ts — S52A host-agnostic rewind-intent queue.
3
+ *
4
+ * A thin queue over the S49-pre-created `pending_fork` table. The dashboard
5
+ * (or any external surface) POSTS a "rewind to turn N" intent; the host polls
6
+ * pending intents at `before_agent_start` and CONSUMEs them to apply the rewind.
7
+ * The store never calls back into the host (ledger protocol — the host PUSHES
8
+ * facts and PULLS views; the store is passive).
9
+ *
10
+ * State is encoded on `consumed_at` (NULL = pending, non-NULL = consumed) +
11
+ * row absence (abandoned = deleted). No schema change to `pending_fork`.
12
+ *
13
+ * Pi-agnostic (PREVENT-PI-004: pure node:sqlite). Parameterized (PREVENT-002).
14
+ */
15
+ import type { DatabaseSync } from "node:sqlite";
16
+
17
+ /** A rewind intent — "rewind conversation X to turn N". */
18
+ export interface RewindIntent {
19
+ /** Stable string id (random 16-hex). */
20
+ id: string;
21
+ /** Conversation to rewind. */
22
+ conversationId: string;
23
+ /** The turn index to rewind to (the parent turn a fork branches from). */
24
+ targetTurnIndex: number;
25
+ /** epoch ms the intent was posted. */
26
+ createdAt: number;
27
+ /** pending (not yet acted on) | consumed (host applied it). */
28
+ status: "pending" | "consumed";
29
+ }
30
+
31
+ /** Input to postIntent (id/createdAt/status are derived). */
32
+ export interface PostIntentInput {
33
+ conversationId: string;
34
+ targetTurnIndex: number;
35
+ }
36
+
37
+ /** Append-only writer for rewind intents (dashboard / external surface). */
38
+ export interface IntentWriter {
39
+ postIntent(input: PostIntentInput): RewindIntent;
40
+ }
41
+
42
+ /** Reader for rewind intents (host polls; dashboard inspects). */
43
+ export interface IntentReader {
44
+ pendingIntents(): RewindIntent[];
45
+ allIntents(limit?: number): RewindIntent[];
46
+ consumeIntent(id: string): void;
47
+ /** Abandon = delete (the intent is withdrawn, never to be applied). */
48
+ abandonIntent(id: string): void;
49
+ }
50
+
51
+ /** Open an intent queue over a turns.db handle (from openTurnStore/openTurnDb). */
52
+ export function openIntentQueue(db: DatabaseSync): IntentWriter & IntentReader {
53
+ function rowToIntent(r: Record<string, unknown>): RewindIntent {
54
+ return {
55
+ id: String(r.id),
56
+ conversationId: String(r.target_conversation_id),
57
+ targetTurnIndex: Number(r.target_turn_id),
58
+ createdAt: Number(r.requested_at),
59
+ status: r.consumed_at == null ? "pending" : "consumed",
60
+ };
61
+ }
62
+
63
+ return {
64
+ postIntent(input: PostIntentInput): RewindIntent {
65
+ const createdAt = Date.now();
66
+ const res = db
67
+ .prepare(
68
+ `INSERT INTO pending_fork
69
+ (target_conversation_id, target_turn_id, requested_at, consumed_at)
70
+ VALUES (?, ?, ?, NULL)`,
71
+ )
72
+ .run(input.conversationId, input.targetTurnIndex, createdAt);
73
+ const id = String(Number(res.lastInsertRowid));
74
+ return {
75
+ id,
76
+ conversationId: input.conversationId,
77
+ targetTurnIndex: input.targetTurnIndex,
78
+ createdAt,
79
+ status: "pending",
80
+ };
81
+ },
82
+
83
+ pendingIntents(): RewindIntent[] {
84
+ const rows = db
85
+ .prepare(
86
+ `SELECT id, target_conversation_id, target_turn_id, requested_at, consumed_at
87
+ FROM pending_fork WHERE consumed_at IS NULL
88
+ ORDER BY requested_at ASC`,
89
+ )
90
+ .all() as Array<Record<string, unknown>>;
91
+ return rows.map(rowToIntent);
92
+ },
93
+
94
+ allIntents(limit = 100): RewindIntent[] {
95
+ const rows = db
96
+ .prepare(
97
+ `SELECT id, target_conversation_id, target_turn_id, requested_at, consumed_at
98
+ FROM pending_fork ORDER BY requested_at DESC, id DESC LIMIT ?`,
99
+ )
100
+ .all(limit) as Array<Record<string, unknown>>;
101
+ return rows.map(rowToIntent);
102
+ },
103
+
104
+ consumeIntent(id: string): void {
105
+ db.prepare("UPDATE pending_fork SET consumed_at = ? WHERE id = ?").run(
106
+ Date.now(),
107
+ id,
108
+ );
109
+ },
110
+
111
+ abandonIntent(id: string): void {
112
+ db.prepare("DELETE FROM pending_fork WHERE id = ?").run(id);
113
+ },
114
+ };
115
+ }
@@ -0,0 +1,9 @@
1
+ /**
2
+ * index.ts — S50 metrics barrel. Re-export only.
3
+ */
4
+ export {
5
+ turnMetrics,
6
+ conversationMetrics,
7
+ type TurnMetrics,
8
+ type ConversationMetrics,
9
+ } from "./turns.js";
@@ -0,0 +1,165 @@
1
+ /**
2
+ * turns.test.ts — S50C per-turn + per-conversation metrics tests. No network.
3
+ * Seeds a turns.db (S49 contract store) and a main db (raw_transcript +
4
+ * checkpoint_epochs) and asserts the rollups match seeded truth.
5
+ */
6
+ import { test, beforeEach, afterEach } from "node:test";
7
+ import assert from "node:assert/strict";
8
+ import { mkdtempSync, rmSync } from "node:fs";
9
+ import { tmpdir } from "node:os";
10
+ import { join } from "node:path";
11
+ import { DatabaseSync } from "node:sqlite";
12
+ import {
13
+ createTurnStore,
14
+ closeAllTurnDbs,
15
+ type TurnStore,
16
+ } from "../store/turns/index.js";
17
+ import { openStore } from "../store/sqlite/utils.js";
18
+ import { appendRawTranscript } from "../store/sqlite/raw-transcript.js";
19
+ import { writeCheckpointEpoch } from "../store/sqlite/raw-transcript.js";
20
+ import { turnMetrics, conversationMetrics } from "./turns.js";
21
+
22
+ let tmpDir: string;
23
+ let counter = 0;
24
+
25
+ beforeEach(() => {
26
+ tmpDir = mkdtempSync(join(tmpdir(), "mc-metrics-"));
27
+ });
28
+
29
+ afterEach(() => {
30
+ closeAllTurnDbs();
31
+ rmSync(tmpDir, { recursive: true, force: true });
32
+ });
33
+
34
+ function stateDir(): string {
35
+ return join(tmpDir, `run-${counter++}`);
36
+ }
37
+
38
+ /** Seed one conversation: 2 turns, recall on turn 0, raw messages, one epoch. */
39
+ function seed(dir: string): { store: TurnStore; conv: string } {
40
+ const store = createTurnStore({ stateDir: dir });
41
+ const conv = store.ensureConversationId("sess_m");
42
+ const t0 = store.appendTurn({
43
+ conversationId: conv,
44
+ sessionId: "sess_m",
45
+ turnIndex: 0,
46
+ role: "assistant",
47
+ endedAt: Date.now(),
48
+ ctxTokens: 800,
49
+ ctxPercent: 0.4,
50
+ });
51
+ store.appendRecall({
52
+ turnId: t0,
53
+ checkpointId: "cp_1",
54
+ score: 0.9,
55
+ source: "checkpoint",
56
+ });
57
+ store.appendRecall({
58
+ turnId: t0,
59
+ checkpointId: "cp_2",
60
+ score: 0.8,
61
+ source: "checkpoint",
62
+ });
63
+ store.appendTurn({
64
+ conversationId: conv,
65
+ sessionId: "sess_m",
66
+ turnIndex: 1,
67
+ role: "assistant",
68
+ endedAt: Date.now(),
69
+ ctxTokens: 1200,
70
+ });
71
+ store.asAdmin().stampTurnsEpoch("sess_m", "ep_1");
72
+
73
+ const main = openStore(dir);
74
+ // 5 raw messages in the committed range; ep_1's summary is much smaller
75
+ // (compression ratio < 1). turn_index is set on all of them.
76
+ const msgs = [
77
+ { hash: "h0", turn: 0, bytes: "aaaaaaaa" },
78
+ { hash: "h1", turn: 0, bytes: "bbbbbbbb" },
79
+ { hash: "h2", turn: 1, bytes: "cccccccc" },
80
+ { hash: "h3", turn: 1, bytes: "dddddddd" },
81
+ { hash: "h4", turn: 1, bytes: "eeeeeeee" },
82
+ ];
83
+ for (const m of msgs) {
84
+ appendRawTranscript(main, {
85
+ contentHash: m.hash,
86
+ sessionId: "sess_m",
87
+ seq: 0,
88
+ role: "user",
89
+ contentBytes: m.bytes,
90
+ toolName: null,
91
+ messageTimestamp: null,
92
+ checkpointEpoch: "ep_1",
93
+ turnIndex: m.turn,
94
+ });
95
+ }
96
+ writeCheckpointEpoch(main, {
97
+ epochId: "ep_1",
98
+ sessionId: "sess_m",
99
+ startedSeq: 0,
100
+ committedSeq: 3,
101
+ summaryMessageText: "sum",
102
+ cutIndex: 3,
103
+ checkpointId: "cp_x",
104
+ createdAt: Date.now(),
105
+ });
106
+ return { store, conv };
107
+ }
108
+
109
+ test("turnMetrics returns per-turn recall + dedup + compression", () => {
110
+ const dir = stateDir();
111
+ const { store, conv } = seed(dir);
112
+ const main = openStore(dir);
113
+ const rows = turnMetrics(store, main, conv);
114
+ assert.equal(rows.length, 2);
115
+ const t0 = rows.find((r) => r.turnIndex === 0)!;
116
+ assert.equal(t0.recallCount, 2);
117
+ assert.equal(t0.rawMessageCount, 2);
118
+ assert.equal(t0.dedupUniqueRatio, 1); // PK(content_hash,session_id) → all stored rows unique
119
+ const t1 = rows.find((r) => r.turnIndex === 1)!;
120
+ assert.equal(t1.rawMessageCount, 3);
121
+ assert.equal(t1.epochId, "ep_1");
122
+ assert.ok(t0.compressionRatio > 0 && t0.compressionRatio < 1); // summary smaller than raw
123
+ store.close();
124
+ });
125
+
126
+ test("conversationMetrics aggregates per-turn metrics", () => {
127
+ const dir = stateDir();
128
+ const { store, conv } = seed(dir);
129
+ const main = openStore(dir);
130
+ const m = conversationMetrics(store, main, conv);
131
+ assert.equal(m.turnCount, 2);
132
+ assert.equal(m.totalRecall, 2);
133
+ assert.equal(m.totalRawMessages, 5);
134
+ assert.equal(m.epochCount, 1);
135
+ assert.ok(m.avgDedupUniqueRatio > 0);
136
+ store.close();
137
+ });
138
+
139
+ test("metrics tolerate a main db without raw_transcript (reuse host)", () => {
140
+ const dir = stateDir();
141
+ const store = createTurnStore({ stateDir: dir });
142
+ const conv = store.ensureConversationId("sess_n");
143
+ const t = store.appendTurn({
144
+ conversationId: conv,
145
+ sessionId: "sess_n",
146
+ turnIndex: 0,
147
+ role: "assistant",
148
+ endedAt: Date.now(),
149
+ });
150
+ store.appendRecall({
151
+ turnId: t,
152
+ checkpointId: "cp",
153
+ score: 0.5,
154
+ source: "checkpoint",
155
+ });
156
+ // Bare in-memory db with NO raw_transcript / checkpoint_epochs tables.
157
+ const bare = new DatabaseSync(":memory:");
158
+ const rows = turnMetrics(store, bare, conv);
159
+ assert.equal(rows.length, 1);
160
+ assert.equal(rows[0].recallCount, 1);
161
+ assert.equal(rows[0].rawMessageCount, 0);
162
+ assert.equal(rows[0].dedupUniqueRatio, 0);
163
+ assert.equal(rows[0].compressionRatio, 0);
164
+ store.close();
165
+ });
@@ -0,0 +1,201 @@
1
+ /**
2
+ * turns.ts — S50C per-turn + per-conversation memory-quality rollups.
3
+ *
4
+ * Host-agnostic (pi-agnostic): consumes the S49 `TurnStore` interface plus a
5
+ * main-db `DatabaseSync` handle for raw_transcript / checkpoint_epochs. Pure
6
+ * read queries — this module NEVER mutates memory, drop ranges, or compaction
7
+ * (PREVENT-PI-001/002). Parameterized (PREVENT-002). No network (PREVENT-PI-004).
8
+ *
9
+ * These are the numbers the S52 dashboard Turns tab renders: per-turn cache-hit
10
+ * (recall reuse), dedup ratio, and compression ratio, rolled up per turn and
11
+ * per conversation.
12
+ */
13
+ import type { DatabaseSync } from "node:sqlite";
14
+ import type { TurnStore } from "../store/turns/types.js";
15
+
16
+ /** Per-turn memory-quality metrics. Ratios are 0 when their basis is absent. */
17
+ export interface TurnMetrics {
18
+ /** Per-conversation turn key (the contract TurnEntry is id-less; turnIndex
19
+ * is unique within a conversation). */
20
+ turnId: number;
21
+ conversationId: string;
22
+ sessionId: string;
23
+ turnIndex: number;
24
+ epochId: string | null;
25
+ ctxTokens: number | null;
26
+ ctxPercent: number | null;
27
+ /** Checkpoints/summaries injected this turn (turn_recall count). */
28
+ recallCount: number;
29
+ /** Raw messages mirrored for this turn (raw_transcript.turn_index count). 0 when dbMirror OFF. */
30
+ rawMessageCount: number;
31
+ /** Distinct content_hash / raw rows for this turn → 1 - ratio = dup fraction. */
32
+ dedupUniqueRatio: number;
33
+ /** Raw bytes → summary bytes compression for the epoch this turn closed (0 if none). */
34
+ compressionRatio: number;
35
+ }
36
+
37
+ /** Per-conversation aggregate of TurnMetrics. */
38
+ export interface ConversationMetrics {
39
+ conversationId: string;
40
+ turnCount: number;
41
+ totalRecall: number;
42
+ totalRawMessages: number;
43
+ avgDedupUniqueRatio: number;
44
+ avgCompressionRatio: number;
45
+ /** Distinct epochs that compacted this conversation's turns. */
46
+ epochCount: number;
47
+ }
48
+
49
+ interface RawAgg {
50
+ turn_index: number;
51
+ raw_count: number;
52
+ unique_count: number;
53
+ }
54
+
55
+ /** Approximate byte length of a text (utf8). */
56
+ function byteLen(s: string): number {
57
+ return Buffer.byteLength(s, "utf8");
58
+ }
59
+
60
+ /**
61
+ * Per-turn metrics for a conversation. Combines turns.db (turn spine + recall)
62
+ * with main-db raw_transcript (dedup) + checkpoint_epochs (compression).
63
+ *
64
+ * `mainDb` may be a store without raw_transcript/checkpoint_epochs (e.g. a
65
+ * reuse host that only mirrors turns); in that case dedup/compression = 0 and
66
+ * rawMessageCount = 0. Detected defensively per-query.
67
+ */
68
+ export function turnMetrics(
69
+ store: TurnStore,
70
+ mainDb: DatabaseSync,
71
+ conversationId: string,
72
+ ): TurnMetrics[] {
73
+ const turns = store.query({ conversationId, limit: 10000 });
74
+ if (turns.length === 0) return [];
75
+
76
+ const hasRaw = tableExists(mainDb, "raw_transcript");
77
+ const hasEpoch = tableExists(mainDb, "checkpoint_epochs");
78
+
79
+ // Aggregate raw_transcript per turn_index for this session (raw rows + unique
80
+ // content_hash). Grouped by turn_index; sessions may share turn_index values,
81
+ // so we scope by session_id below per turn.
82
+ const rawAggBySessionTurn = new Map<string, RawAgg>();
83
+ if (hasRaw) {
84
+ const sessionIds = [...new Set(turns.map((t) => t.sessionId))];
85
+ const placeholders = sessionIds.map(() => "?").join(",");
86
+ const rows = mainDb
87
+ .prepare(
88
+ `SELECT session_id, turn_index,
89
+ COUNT(*) AS raw_count,
90
+ COUNT(DISTINCT content_hash) AS unique_count
91
+ FROM raw_transcript
92
+ WHERE session_id IN (${placeholders}) AND turn_index IS NOT NULL
93
+ GROUP BY session_id, turn_index`,
94
+ )
95
+ .all(...sessionIds) as unknown as Array<RawAgg & { session_id: string }>;
96
+ for (const r of rows) {
97
+ rawAggBySessionTurn.set(`${r.session_id}::${r.turn_index}`, r);
98
+ }
99
+ }
100
+
101
+ // Compression ratio per epoch: committed raw bytes → summary bytes. Precomputed
102
+ // per epoch referenced by any turn.
103
+ const compressionByEpoch = new Map<string, number>();
104
+ if (hasEpoch) {
105
+ const epochIds = turns
106
+ .map((t) => t.epochId)
107
+ .filter((e): e is string => e != null);
108
+ for (const eid of new Set(epochIds)) {
109
+ const ratio = epochCompressionRatio(mainDb, eid);
110
+ if (ratio != null) compressionByEpoch.set(eid, ratio);
111
+ }
112
+ }
113
+
114
+ return turns.map((t) => {
115
+ const recall = store.listRecallByIndex(conversationId, t.turnIndex);
116
+ const agg = rawAggBySessionTurn.get(`${t.sessionId}::${t.turnIndex}`);
117
+ const rawCount = agg?.raw_count ?? 0;
118
+ const uniqueCount = agg?.unique_count ?? 0;
119
+ return {
120
+ turnId: t.turnIndex,
121
+ conversationId: t.conversationId,
122
+ sessionId: t.sessionId,
123
+ turnIndex: t.turnIndex,
124
+ epochId: t.epochId ?? null,
125
+ ctxTokens: t.ctxTokens ?? null,
126
+ ctxPercent: t.ctxPercent ?? null,
127
+ recallCount: recall.length,
128
+ rawMessageCount: rawCount,
129
+ dedupUniqueRatio: rawCount > 0 ? uniqueCount / rawCount : 0,
130
+ compressionRatio: t.epochId
131
+ ? (compressionByEpoch.get(t.epochId) ?? 0)
132
+ : 0,
133
+ };
134
+ });
135
+ }
136
+
137
+ /** Aggregate a conversation's per-turn metrics. */
138
+ export function conversationMetrics(
139
+ store: TurnStore,
140
+ mainDb: DatabaseSync,
141
+ conversationId: string,
142
+ ): ConversationMetrics {
143
+ const perTurn = turnMetrics(store, mainDb, conversationId);
144
+ const turnCount = perTurn.length;
145
+ const totalRecall = perTurn.reduce((s, t) => s + t.recallCount, 0);
146
+ const totalRaw = perTurn.reduce((s, t) => s + t.rawMessageCount, 0);
147
+ const dedupSum = perTurn.reduce((s, t) => s + t.dedupUniqueRatio, 0);
148
+ const compSum = perTurn.reduce((s, t) => s + t.compressionRatio, 0);
149
+ const epochs = new Set(perTurn.map((t) => t.epochId).filter(Boolean));
150
+ return {
151
+ conversationId,
152
+ turnCount,
153
+ totalRecall,
154
+ totalRawMessages: totalRaw,
155
+ avgDedupUniqueRatio: turnCount > 0 ? dedupSum / turnCount : 0,
156
+ avgCompressionRatio: turnCount > 0 ? compSum / turnCount : 0,
157
+ epochCount: epochs.size,
158
+ };
159
+ }
160
+
161
+ function tableExists(db: DatabaseSync, name: string): boolean {
162
+ return (
163
+ db
164
+ .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name=?")
165
+ .get(name) !== undefined
166
+ );
167
+ }
168
+
169
+ /**
170
+ * Compression ratio for an epoch: summaryMessageText bytes ÷ raw bytes of the
171
+ * committed range. Returns null when the epoch or its raw rows are absent.
172
+ * Ratio < 1 means compression shrank the content (lower = more compression).
173
+ */
174
+ function epochCompressionRatio(
175
+ mainDb: DatabaseSync,
176
+ epochId: string,
177
+ ): number | null {
178
+ const epoch = mainDb
179
+ .prepare(
180
+ "SELECT session_id, committed_seq, summary_message_text FROM checkpoint_epochs WHERE epoch_id = ?",
181
+ )
182
+ .get(epochId) as
183
+ | {
184
+ session_id: string;
185
+ committed_seq: number;
186
+ summary_message_text: string;
187
+ }
188
+ | undefined;
189
+ if (!epoch) return null;
190
+ const raw = mainDb
191
+ .prepare(
192
+ `SELECT COALESCE(SUM(LENGTH(content_bytes)), 0) AS bytes
193
+ FROM raw_transcript
194
+ WHERE session_id = ? AND seq <= ?`,
195
+ )
196
+ .get(epoch.session_id, epoch.committed_seq) as { bytes: number };
197
+ const rawBytes = raw?.bytes ?? 0;
198
+ if (rawBytes <= 0) return null;
199
+ const summaryBytes = byteLen(epoch.summary_message_text ?? "");
200
+ return summaryBytes / rawBytes;
201
+ }
@@ -22,6 +22,9 @@ export interface RawTranscriptRow {
22
22
  /** ORIGINAL message timestamp captured at append time (NOT a served ts). */
23
23
  messageTimestamp: number | null;
24
24
  checkpointEpoch: string;
25
+ /** S50: the per-session turn that produced this message (runtime.currentTurn).
26
+ * Null on rows written before S50 or when the writer omits it (back-compat). */
27
+ turnIndex?: number | null;
25
28
  }
26
29
 
27
30
  /** One checkpoint-epoch bookkeeping row (informational registry). */
@@ -46,6 +49,7 @@ interface RawTranscriptDBRow {
46
49
  tool_name: string | null;
47
50
  message_timestamp: number | null;
48
51
  checkpoint_epoch: string;
52
+ turn_index: number | null;
49
53
  }
50
54
 
51
55
  /** DB row shape for checkpoint_epochs (snake_case column names). */
@@ -71,6 +75,7 @@ function rowToRawTranscript(row: RawTranscriptDBRow): RawTranscriptRow {
71
75
  messageTimestamp:
72
76
  row.message_timestamp == null ? null : Number(row.message_timestamp),
73
77
  checkpointEpoch: row.checkpoint_epoch,
78
+ turnIndex: row.turn_index == null ? null : Number(row.turn_index),
74
79
  };
75
80
  }
76
81
 
@@ -99,11 +104,11 @@ export function appendRawTranscript(db: DatabaseSync, row: RawTranscriptRow): vo
99
104
  withTx(db, () => {
100
105
  db.prepare(
101
106
  `INSERT OR IGNORE INTO raw_transcript
102
- (content_hash, session_id, seq, role, content_bytes, tool_name, message_timestamp, checkpoint_epoch)
107
+ (content_hash, session_id, seq, role, content_bytes, tool_name, message_timestamp, checkpoint_epoch, turn_index)
103
108
  VALUES (
104
109
  @content_hash, @session_id,
105
110
  COALESCE((SELECT MAX(seq) FROM raw_transcript WHERE session_id = @session_id), 0) + 1,
106
- @role, @content_bytes, @tool_name, @message_timestamp, @checkpoint_epoch
111
+ @role, @content_bytes, @tool_name, @message_timestamp, @checkpoint_epoch, @turn_index
107
112
  )`,
108
113
  ).run({
109
114
  "@content_hash": row.contentHash,
@@ -113,6 +118,7 @@ export function appendRawTranscript(db: DatabaseSync, row: RawTranscriptRow): vo
113
118
  "@tool_name": row.toolName,
114
119
  "@message_timestamp": row.messageTimestamp,
115
120
  "@checkpoint_epoch": row.checkpointEpoch,
121
+ "@turn_index": row.turnIndex ?? null,
116
122
  });
117
123
  });
118
124
  }
@@ -129,7 +135,7 @@ export function listRawTranscriptRange(
129
135
  ): RawTranscriptRow[] {
130
136
  const rows = db
131
137
  .prepare(
132
- `SELECT content_hash, session_id, seq, role, content_bytes, tool_name, message_timestamp, checkpoint_epoch
138
+ `SELECT content_hash, session_id, seq, role, content_bytes, tool_name, message_timestamp, checkpoint_epoch, turn_index
133
139
  FROM raw_transcript
134
140
  WHERE session_id = @session_id AND seq >= @from_seq AND seq <= @to_seq
135
141
  ORDER BY seq ASC`,
@@ -107,6 +107,16 @@ describe("DB mirror", () => {
107
107
  assert.equal(rows[0].messageTimestamp, 1234567890);
108
108
  });
109
109
 
110
+ it("S50A: stores turn_index when provided; null when omitted (back-compat)", () => {
111
+ appendRawTranscript(db, makeRow({ contentHash: "with-turn", turnIndex: 7 }));
112
+ appendRawTranscript(db, makeRow({ contentHash: "no-turn" }));
113
+ const rows = listRawTranscriptRange(db, "sess-1", 0, 999999);
114
+ const withTurn = rows.find((r) => r.contentHash === "with-turn");
115
+ const noTurn = rows.find((r) => r.contentHash === "no-turn");
116
+ assert.equal(withTurn?.turnIndex, 7);
117
+ assert.equal(noTurn?.turnIndex, null);
118
+ });
119
+
110
120
  it("stores checkpoint_epoch", () => {
111
121
  appendRawTranscript(db, makeRow({ checkpointEpoch: "ep-42" }));
112
122
  const rows = listRawTranscriptRange(db, "sess-1", 0, 999999);