pi-mega-compact 0.8.7 → 0.8.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/dist/extensions/dashboard-server/api-contracts/core.js +7 -0
  2. package/dist/extensions/dashboard-server/api-contracts/endpoints.js +104 -0
  3. package/dist/extensions/dashboard-server/api-contracts/game.js +9 -0
  4. package/dist/extensions/dashboard-server/api-contracts/index.js +9 -0
  5. package/dist/extensions/dashboard-server/api-contracts/infrastructure.js +10 -0
  6. package/dist/extensions/dashboard-server/api-contracts/multi-repo.js +9 -0
  7. package/dist/extensions/dashboard-server/api-contracts/snapshot.js +8 -0
  8. package/dist/extensions/dashboard-server/api-contracts.js +8 -0
  9. package/dist/extensions/dashboard-server/api-contracts.test.js +869 -0
  10. package/dist/extensions/dashboard-server/auth.js +18 -0
  11. package/dist/extensions/dashboard-server/helpers.js +37 -0
  12. package/dist/extensions/dashboard-server/html/all-repos-tab.js +26 -0
  13. package/dist/extensions/dashboard-server/html/body-open.js +23 -0
  14. package/dist/extensions/dashboard-server/html/current-repo-tab.js +130 -0
  15. package/dist/extensions/dashboard-server/html/head-open.js +16 -0
  16. package/dist/extensions/dashboard-server/html/high-score-tab.js +25 -0
  17. package/dist/extensions/dashboard-server/html/repo-detail-modal.js +26 -0
  18. package/dist/extensions/dashboard-server/html/script.js +259 -0
  19. package/dist/extensions/dashboard-server/html/styles.js +103 -0
  20. package/dist/extensions/dashboard-server/html/summary-tab.js +19 -0
  21. package/dist/extensions/dashboard-server/html-template.js +41 -0
  22. package/dist/extensions/dashboard-server/html.js +46 -1
  23. package/dist/extensions/dashboard-server/perf-server.test.js +80 -0
  24. package/dist/extensions/dashboard-server/server.js +269 -30
  25. package/dist/extensions/dashboard-server/tailscale.js +8 -0
  26. package/dist/extensions/dashboard-server/types.js +4 -0
  27. package/dist/extensions/mega-dashboard.js +9 -0
  28. package/dist/extensions/mega-events/perf-handler.js +71 -0
  29. package/dist/extensions/mega-events/register.js +2 -0
  30. package/dist/extensions/mega-events.js +1 -0
  31. package/dist/extensions/mega-runtime/state.js +59 -1
  32. package/dist/src/store/sqlite/connection.js +35 -0
  33. package/dist/src/store/sqlite/index-store.js +167 -0
  34. package/dist/src/store/sqlite/memory.js +54 -0
  35. package/dist/src/store/sqlite/minhash-lsh.js +47 -0
  36. package/dist/src/store/sqlite/perf-samples.js +81 -0
  37. package/dist/src/store/sqlite/perf-samples.test.js +54 -0
  38. package/dist/src/store/sqlite/schema.js +14 -0
  39. package/dist/src/store/sqlite/sessions.js +39 -0
  40. package/dist/src/store/sqlite/transaction.js +19 -0
  41. package/dist/src/store/sqlite.js +1 -0
  42. package/dist/src/vectorStore/add.js +260 -0
  43. package/dist/src/vectorStore/dedup.js +52 -0
  44. package/dist/src/vectorStore/index.js +10 -0
  45. package/dist/src/vectorStore/queries.js +83 -0
  46. package/dist/src/vectorStore/search.js +95 -0
  47. package/dist/src/vectorStore/session.js +19 -0
  48. package/dist/src/vectorStore/store.js +105 -0
  49. package/dist/src/vectorStore/types.js +6 -0
  50. package/dist/src/vectorStore/utils.js +23 -0
  51. package/extensions/dashboard-client/package-lock.json +1771 -0
  52. package/extensions/dashboard-client/package.json +27 -0
  53. package/extensions/dashboard-client/src/App.tsx +82 -0
  54. package/extensions/dashboard-client/src/api/client.ts +145 -0
  55. package/extensions/dashboard-client/src/components/CacheStatusPerModel.tsx +44 -0
  56. package/extensions/dashboard-client/src/components/CompressionCard.tsx +61 -0
  57. package/extensions/dashboard-client/src/components/ContextGauge.tsx +59 -0
  58. package/extensions/dashboard-client/src/components/DataSafetyCard.tsx +49 -0
  59. package/extensions/dashboard-client/src/components/ErrorBoundary.tsx +53 -0
  60. package/extensions/dashboard-client/src/components/EventCategoryFilter.tsx +16 -0
  61. package/extensions/dashboard-client/src/components/EventStream.tsx +152 -0
  62. package/extensions/dashboard-client/src/components/LoadingSpinner.tsx +7 -0
  63. package/extensions/dashboard-client/src/components/MemoryStatusCard.tsx +24 -0
  64. package/extensions/dashboard-client/src/components/ModelBadge.tsx +41 -0
  65. package/extensions/dashboard-client/src/components/PerfChart.tsx +160 -0
  66. package/extensions/dashboard-client/src/components/RepoDetailModal.tsx +126 -0
  67. package/extensions/dashboard-client/src/components/RepoTable.tsx +150 -0
  68. package/extensions/dashboard-client/src/components/SessionInfo.tsx +65 -0
  69. package/extensions/dashboard-client/src/components/SummaryTiles.tsx +52 -0
  70. package/extensions/dashboard-client/src/components/TabBar.tsx +34 -0
  71. package/extensions/dashboard-client/src/components/TriggerStatus.tsx +62 -0
  72. package/extensions/dashboard-client/src/hooks/useApi.ts +77 -0
  73. package/extensions/dashboard-client/src/hooks/useSSE.ts +87 -0
  74. package/extensions/dashboard-client/src/index.html +12 -0
  75. package/extensions/dashboard-client/src/main.tsx +23 -0
  76. package/extensions/dashboard-client/src/styles/base.css +121 -0
  77. package/extensions/dashboard-client/src/styles/overview-events.css +318 -0
  78. package/extensions/dashboard-client/src/styles/repos-metrics.css +307 -0
  79. package/extensions/dashboard-client/src/tabs/ConfigTab.tsx +16 -0
  80. package/extensions/dashboard-client/src/tabs/EventsTab.tsx +37 -0
  81. package/extensions/dashboard-client/src/tabs/MetricsTab.tsx +49 -0
  82. package/extensions/dashboard-client/src/tabs/OverviewTab.tsx +80 -0
  83. package/extensions/dashboard-client/src/tabs/ReposTab.tsx +59 -0
  84. package/extensions/dashboard-client/tsconfig.json +28 -0
  85. package/extensions/dashboard-client/vite.config.ts +38 -0
  86. package/extensions/dashboard-server/api-contracts/core.ts +302 -0
  87. package/extensions/dashboard-server/api-contracts/endpoints.ts +476 -0
  88. package/extensions/dashboard-server/api-contracts/game.ts +145 -0
  89. package/extensions/dashboard-server/api-contracts/index.ts +159 -0
  90. package/extensions/dashboard-server/api-contracts/infrastructure.ts +465 -0
  91. package/extensions/dashboard-server/api-contracts/multi-repo.ts +328 -0
  92. package/extensions/dashboard-server/api-contracts/snapshot.ts +386 -0
  93. package/extensions/dashboard-server/api-contracts.test.ts +1050 -0
  94. package/extensions/dashboard-server/api-contracts.ts +96 -0
  95. package/extensions/dashboard-server/auth.ts +20 -0
  96. package/extensions/dashboard-server/html.ts +46 -1
  97. package/extensions/dashboard-server/perf-server.test.ts +101 -0
  98. package/extensions/dashboard-server/server.ts +862 -503
  99. package/extensions/dashboard-server/tailscale.ts +7 -0
  100. package/extensions/dashboard-server/types.ts +23 -114
  101. package/extensions/mega-dashboard.ts +17 -0
  102. package/extensions/mega-events/perf-handler.ts +113 -0
  103. package/extensions/mega-events/register.ts +2 -0
  104. package/extensions/mega-events.ts +1 -0
  105. package/extensions/mega-runtime/state.ts +57 -0
  106. package/package.json +2 -1
  107. package/src/store/sqlite/perf-samples.test.ts +65 -0
  108. package/src/store/sqlite/perf-samples.ts +125 -0
  109. package/src/store/sqlite/schema.ts +14 -0
  110. package/src/store/sqlite.ts +1 -0
@@ -15,7 +15,7 @@ import { VectorStore } from "../../src/vectorStore.js";
15
15
  import { toEngineMessages } from "../../src/adapt.js";
16
16
  import { normalizeSessionId } from "../../src/store.js";
17
17
  import { Logger } from "../../src/log.js";
18
- import { recordModelSnapshot, latestModelSnapshot, upsertRepoRegistry, recordRepoModel, getDedupStats, getCompactCount, getRecallInjected, getCacheHitTokensSaved, getGameState, } from "../../src/store/sqlite.js";
18
+ import { recordModelSnapshot, latestModelSnapshot, upsertRepoRegistry, recordRepoModel, getDedupStats, getCompactCount, getRecallInjected, getCacheHitTokensSaved, getGameState, recordPerfSample, } from "../../src/store/sqlite.js";
19
19
  import { detectCrossRepoDrift } from "../../src/driftDetection.js";
20
20
  import { repoStateDir, resolveRepoRoot, pressureRatio, pressureFromPct, pressureBand, effectiveThresholdTokens, } from "../mega-config.js";
21
21
  import { Dashboard } from "../mega-dashboard.js";
@@ -121,6 +121,12 @@ export class MegaRuntime {
121
121
  // Last explain-why line (dedup reason / anchor-kept / superseded), surfaced
122
122
  // while fresh.
123
123
  lastWhy = undefined;
124
+ // v0.8.8 Perf dashboard instrumentation: turn/provider start timestamps +
125
+ // the 5s cpu/mem interval handle (one per MegaRuntime, cleared in dispose()).
126
+ perfTurnStart = 0;
127
+ perfProviderStart = 0;
128
+ perfCpuInterval;
129
+ perfCpuBaseline;
124
130
  // Context tracking for the dashboard (updated in the context handler).
125
131
  lastCtxTokens = null;
126
132
  lastCtxPercent = null;
@@ -329,6 +335,7 @@ export class MegaRuntime {
329
335
  this.renderWidget(ctx);
330
336
  return;
331
337
  }
338
+ const perfT0 = performance.now();
332
339
  const st = this.store.stats(this.rt.sessionId);
333
340
  const repo = this.store.repoStats();
334
341
  const di = this.store.dataInvariant();
@@ -479,7 +486,13 @@ export class MegaRuntime {
479
486
  cacheHit: { sessionSec: sec(this.rt.cacheHitTokens), totalSec: sec(cacheHitsTotalTokens) },
480
487
  },
481
488
  model,
489
+ diag: {
490
+ ctxFastGate: this.diagCtxFastGate,
491
+ liveTrimFires: this.diagLiveTrimFires,
492
+ liveTrimReplays: this.diagLiveTrimReplays,
493
+ },
482
494
  });
495
+ const perfDiskMs = this.dashboard.lastWriteMs;
483
496
  // Live stats widget above the editor
484
497
  if (ctx) {
485
498
  // ── gather widget data (computed per snapshot, rendered per frame) ────
@@ -634,6 +647,13 @@ export class MegaRuntime {
634
647
  }
635
648
  // v0.8.5: record the material-change signature computed at the top so the
636
649
  // next snapshot() can skip this whole body when nothing material changed.
650
+ try {
651
+ recordPerfSample(this.currentStateDir, "db_recompute_ms", performance.now() - perfT0);
652
+ recordPerfSample(this.currentStateDir, "disk_write_ms", perfDiskMs);
653
+ }
654
+ catch {
655
+ /* non-fatal: perf instrumentation never blocks the agent */
656
+ }
637
657
  this.lastSnapshotSig = sig;
638
658
  }
639
659
  /** Register the above-editor widget as a width-aware factory so pi re-renders
@@ -892,6 +912,44 @@ export class MegaRuntime {
892
912
  this.gameStateWatcher = undefined;
893
913
  this.gameStateWatchDir = undefined;
894
914
  }
915
+ // v0.8.8: stop the cpu/mem sampling interval on teardown. Re-armed lazily
916
+ // by ensurePerfInterval() on the next turn_start.
917
+ if (this.perfCpuInterval) {
918
+ clearInterval(this.perfCpuInterval);
919
+ this.perfCpuInterval = undefined;
920
+ this.perfCpuBaseline = undefined;
921
+ }
922
+ }
923
+ /** v0.8.8: (re)start the 5s cpu/mem sampling interval (idempotent). One per
924
+ * MegaRuntime; cleared in dispose(). Samples process.cpuUsage() (user/sys
925
+ * delta vs the last tick → ms) + process.memoryUsage() (rss/heap → MB) and
926
+ * records them as perf_samples. unref'd so it never keeps the process alive
927
+ * on its own. Non-fatal: any failure is swallowed (instrumentation never
928
+ * blocks the agent). PREVENT-PI-004: local process stats + SQLite only. */
929
+ ensurePerfInterval() {
930
+ if (this.perfCpuInterval)
931
+ return;
932
+ this.perfCpuBaseline = undefined; // first tick sets the baseline (no delta)
933
+ this.perfCpuInterval = setInterval(() => {
934
+ try {
935
+ const dir = this.currentStateDir;
936
+ const cpu = process.cpuUsage();
937
+ const mem = process.memoryUsage();
938
+ if (this.perfCpuBaseline) {
939
+ const du = (cpu.user - this.perfCpuBaseline.user) / 1000; // μs → ms
940
+ const ds = (cpu.system - this.perfCpuBaseline.sys) / 1000;
941
+ recordPerfSample(dir, "cpu_user_ms", Math.max(0, du));
942
+ recordPerfSample(dir, "cpu_sys_ms", Math.max(0, ds));
943
+ }
944
+ this.perfCpuBaseline = { user: cpu.user, sys: cpu.system };
945
+ recordPerfSample(dir, "rss_mb", mem.rss / 1_000_000);
946
+ recordPerfSample(dir, "heap_mb", mem.heapUsed / 1_000_000);
947
+ }
948
+ catch {
949
+ /* non-fatal */
950
+ }
951
+ }, 5000);
952
+ this.perfCpuInterval.unref?.();
895
953
  }
896
954
  /** S31: the cached game-mode state (game_mode_on/theme/tui_display_mode).
897
955
  * Lazily read from the game_state SQLite row on the first call, then
@@ -0,0 +1,35 @@
1
+ /**
2
+ * Per-repo SQLite connection management (open / reuse / close).
3
+ *
4
+ * In-process cache so the same stateDir reuses one connection (and so a fresh
5
+ * VectorStore over the same dir shares the open DB). Cross-process durability
6
+ * comes from reopening the same file path — proven by the integration test.
7
+ */
8
+ import { DatabaseSync } from "node:sqlite";
9
+ import { existsSync, mkdirSync } from "node:fs";
10
+ import { join } from "node:path";
11
+ import { getStateDir } from "../../store.js";
12
+ import { initSchema } from "./schema.js";
13
+ const cache = new Map();
14
+ /** Open (or reuse) the SQLite store for a state dir. */
15
+ export function openStore(stateDir = getStateDir()) {
16
+ const existing = cache.get(stateDir);
17
+ if (existing)
18
+ return existing;
19
+ if (!existsSync(stateDir))
20
+ mkdirSync(stateDir, { recursive: true });
21
+ const db = new DatabaseSync(join(stateDir, "sqlite.db"));
22
+ db.exec("PRAGMA journal_mode = WAL");
23
+ db.exec("PRAGMA foreign_keys = ON");
24
+ initSchema(db);
25
+ cache.set(stateDir, db);
26
+ return db;
27
+ }
28
+ /** Close and evict a cached connection (test teardown only). */
29
+ export function closeStore(stateDir) {
30
+ const db = cache.get(stateDir);
31
+ if (db) {
32
+ db.close();
33
+ cache.delete(stateDir);
34
+ }
35
+ }
@@ -0,0 +1,167 @@
1
+ /**
2
+ * Global machine-wide index store (Phase 5b).
3
+ *
4
+ * A single SQLite DB, separate from every per-repo store, that aggregates one
5
+ * row per repo this machine has run on. The multi-repo dashboard (Summary /
6
+ * All-repos tabs) reads it so ONE dashboard can show every repo's checkpoints,
7
+ * tokens saved, and active model — instead of a per-repo dashboard that only
8
+ * ever sees the repo it was launched from.
9
+ *
10
+ * Written by every pi process on repo-switch (bindRepo) + model capture; read by
11
+ * the dashboard server. Concurrency across 10+ pi processes is handled by WAL +
12
+ * infrequent idempotent upserts (ON CONFLICT). Fully local (PREVENT-PI-004).
13
+ */
14
+ import { DatabaseSync } from "node:sqlite";
15
+ import { existsSync, mkdirSync } from "node:fs";
16
+ import { homedir, tmpdir } from "node:os";
17
+ import { join } from "node:path";
18
+ /** Resolve the machine-wide index directory (env-overridable). */
19
+ export function getIndexDir() {
20
+ const override = process.env.MEGACOMPACT_INDEX_DIR;
21
+ if (override && override.trim() !== "")
22
+ return override;
23
+ // homedir() can throw in exotic sandboxes; fall back to tmpdir.
24
+ try {
25
+ return join(homedir(), ".mega-compact-index");
26
+ }
27
+ catch {
28
+ return join(tmpdir(), ".mega-compact-index");
29
+ }
30
+ }
31
+ let indexCache;
32
+ let indexCacheDir;
33
+ /** Open (or reuse) the machine-wide index DB. WAL for concurrent writers. */
34
+ export function openIndexStore(indexDir = getIndexDir()) {
35
+ if (indexCache && indexCacheDir === indexDir)
36
+ return indexCache;
37
+ if (!existsSync(indexDir))
38
+ mkdirSync(indexDir, { recursive: true });
39
+ const iddb = new DatabaseSync(join(indexDir, "index.sqlite"));
40
+ iddb.exec("PRAGMA journal_mode = WAL");
41
+ iddb.exec("PRAGMA busy_timeout = 3000"); // tolerate brief cross-process write contention
42
+ iddb.exec(`
43
+ CREATE TABLE IF NOT EXISTS repo_registry (
44
+ repo_root TEXT PRIMARY KEY,
45
+ display_name TEXT,
46
+ state_dir TEXT NOT NULL,
47
+ first_seen INTEGER,
48
+ last_seen INTEGER,
49
+ last_compacted_at INTEGER,
50
+ checkpoint_count INTEGER DEFAULT 0,
51
+ tokens_saved INTEGER DEFAULT 0,
52
+ compressed_original_bytes INTEGER DEFAULT 0,
53
+ provider TEXT,
54
+ provider_name TEXT,
55
+ model_name TEXT,
56
+ input_rate REAL,
57
+ output_rate REAL,
58
+ model_captured_at INTEGER
59
+ );
60
+ CREATE INDEX IF NOT EXISTS idx_registry_last_seen ON repo_registry(last_seen DESC);
61
+ `);
62
+ indexCache = iddb;
63
+ indexCacheDir = indexDir;
64
+ return iddb;
65
+ }
66
+ /**
67
+ * Upsert a repo's aggregate stats into the global index. Called on repo-switch
68
+ * (infrequent). Preserves first_seen + the model columns on update (model is
69
+ * written separately by recordRepoModel so we never clobber it here with nulls).
70
+ */
71
+ export function upsertRepoRegistry(row, indexDir = getIndexDir()) {
72
+ const db = openIndexStore(indexDir);
73
+ const now = Date.now();
74
+ db.prepare(`INSERT INTO repo_registry
75
+ (repo_root, display_name, state_dir, first_seen, last_seen, last_compacted_at,
76
+ checkpoint_count, tokens_saved, compressed_original_bytes)
77
+ VALUES (@repo_root, @display_name, @state_dir, @now, @now, @last_compacted_at,
78
+ @checkpoint_count, @tokens_saved, @compressed_original_bytes)
79
+ ON CONFLICT(repo_root) DO UPDATE SET
80
+ display_name = excluded.display_name,
81
+ state_dir = excluded.state_dir,
82
+ last_seen = excluded.last_seen,
83
+ last_compacted_at = COALESCE(excluded.last_compacted_at, repo_registry.last_compacted_at),
84
+ checkpoint_count = excluded.checkpoint_count,
85
+ tokens_saved = excluded.tokens_saved,
86
+ compressed_original_bytes = excluded.compressed_original_bytes`).run({
87
+ repo_root: row.repoRoot,
88
+ display_name: row.displayName,
89
+ state_dir: row.stateDir,
90
+ now,
91
+ last_compacted_at: row.lastCompactedAt ?? null,
92
+ checkpoint_count: row.checkpointCount,
93
+ tokens_saved: row.tokensSaved,
94
+ compressed_original_bytes: row.compressedOriginalBytes,
95
+ });
96
+ }
97
+ /**
98
+ * Record the active model/provider for a repo in the global index (denormalized
99
+ * so the All-repos table shows model without opening each repo's DB). Upserts a
100
+ * bare registry row if the repo isn't registered yet.
101
+ */
102
+ export function recordRepoModel(repoRoot, model, indexDir = getIndexDir()) {
103
+ const db = openIndexStore(indexDir);
104
+ const now = Date.now();
105
+ db.prepare(`INSERT INTO repo_registry
106
+ (repo_root, display_name, state_dir, first_seen, last_seen,
107
+ provider, provider_name, model_name, input_rate, output_rate, model_captured_at)
108
+ VALUES (@repo_root, @display_name, @state_dir, @now, @now,
109
+ @provider, @provider_name, @model_name, @input_rate, @output_rate, @now)
110
+ ON CONFLICT(repo_root) DO UPDATE SET
111
+ last_seen = excluded.last_seen,
112
+ provider = excluded.provider,
113
+ provider_name = excluded.provider_name,
114
+ model_name = excluded.model_name,
115
+ input_rate = excluded.input_rate,
116
+ output_rate = excluded.output_rate,
117
+ model_captured_at = excluded.model_captured_at`).run({
118
+ repo_root: repoRoot,
119
+ display_name: model.displayName,
120
+ state_dir: model.stateDir,
121
+ now,
122
+ provider: model.provider,
123
+ provider_name: model.providerName,
124
+ model_name: model.modelName,
125
+ input_rate: model.inputRate,
126
+ output_rate: model.outputRate,
127
+ });
128
+ }
129
+ function mapRegistryRow(row) {
130
+ return {
131
+ repoRoot: row.repo_root,
132
+ displayName: row.display_name ?? "",
133
+ stateDir: row.state_dir,
134
+ firstSeen: row.first_seen ?? 0,
135
+ lastSeen: row.last_seen ?? 0,
136
+ lastCompactedAt: row.last_compacted_at ?? null,
137
+ checkpointCount: row.checkpoint_count ?? 0,
138
+ tokensSaved: row.tokens_saved ?? 0,
139
+ compressedOriginalBytes: row.compressed_original_bytes ?? 0,
140
+ provider: row.provider ?? null,
141
+ providerName: row.provider_name ?? null,
142
+ modelName: row.model_name ?? null,
143
+ inputRate: row.input_rate ?? null,
144
+ outputRate: row.output_rate ?? null,
145
+ modelCapturedAt: row.model_captured_at ?? null,
146
+ };
147
+ }
148
+ /** All registered repos, most-recently-seen first. */
149
+ export function listRepoRegistry(indexDir = getIndexDir()) {
150
+ const db = openIndexStore(indexDir);
151
+ const rows = db.prepare("SELECT * FROM repo_registry ORDER BY last_seen DESC").all();
152
+ return rows.map(mapRegistryRow);
153
+ }
154
+ /** A single repo's registry row, or undefined. */
155
+ export function getRepoRegistry(repoRoot, indexDir = getIndexDir()) {
156
+ const db = openIndexStore(indexDir);
157
+ const row = db.prepare("SELECT * FROM repo_registry WHERE repo_root = ?").get(repoRoot);
158
+ return row ? mapRegistryRow(row) : undefined;
159
+ }
160
+ /** Close the cached index connection (test teardown only). */
161
+ export function closeIndexStore() {
162
+ if (indexCache) {
163
+ indexCache.close();
164
+ indexCache = undefined;
165
+ indexCacheDir = undefined;
166
+ }
167
+ }
@@ -0,0 +1,54 @@
1
+ /**
2
+ * Durable "save to memory" store (taken over from memory extensions).
3
+ *
4
+ * One SQLite store for user-saved memories, scoped by repo. Mirrors the
5
+ * lessons/sessions pattern: all state lives in SQLite from day one. All queries
6
+ * are parameterized (PREVENT-002).
7
+ */
8
+ import { getStateDir } from "../../store.js";
9
+ import { openStore } from "./connection.js";
10
+ /** Save a memory to the current repo's store. Returns the new row id. */
11
+ export function addMemory(memory, repo, stateDir = getStateDir()) {
12
+ const db = openStore(stateDir);
13
+ const now = Math.floor(Date.now() / 1000);
14
+ const res = db
15
+ .prepare(`INSERT INTO memories(repo, kind, content, tags, created_at, last_recalled_at)
16
+ VALUES(?, ?, ?, ?, ?, NULL)`)
17
+ .run(repo ?? null, memory.kind ?? "note", memory.content, JSON.stringify(memory.tags ?? []), now);
18
+ return Number(res.lastInsertRowid);
19
+ }
20
+ /** List recent memories for a repo (or all repos when repo is null). */
21
+ export function listMemories(repo, limit = 50, stateDir = getStateDir()) {
22
+ const db = openStore(stateDir);
23
+ const rows = repo
24
+ ? db.prepare("SELECT * FROM memories WHERE repo = ? ORDER BY created_at DESC LIMIT ?").all(repo, limit)
25
+ : db.prepare("SELECT * FROM memories ORDER BY created_at DESC LIMIT ?").all(limit);
26
+ return rows.map(mapMemoryRow);
27
+ }
28
+ /** Substring search across content + tags. */
29
+ export function searchMemories(query, repo = null, limit = 50, stateDir = getStateDir()) {
30
+ const db = openStore(stateDir);
31
+ const like = `%${query}%`;
32
+ const rows = repo
33
+ ? db.prepare("SELECT * FROM memories WHERE repo = ? AND (content LIKE ? OR tags LIKE ?) ORDER BY created_at DESC LIMIT ?").all(repo, like, like, limit)
34
+ : db.prepare("SELECT * FROM memories WHERE content LIKE ? OR tags LIKE ? ORDER BY created_at DESC LIMIT ?").all(like, like, limit);
35
+ return rows.map(mapMemoryRow);
36
+ }
37
+ /** Mark a memory as recalled (updates last_recalled_at). Returns true if found. */
38
+ export function recallMemory(id, stateDir = getStateDir()) {
39
+ const db = openStore(stateDir);
40
+ const now = Math.floor(Date.now() / 1000);
41
+ const res = db.prepare("UPDATE memories SET last_recalled_at = ? WHERE id = ?").run(now, id);
42
+ return res.changes > 0;
43
+ }
44
+ function mapMemoryRow(row) {
45
+ return {
46
+ id: row.id,
47
+ repo: row.repo ?? null,
48
+ kind: row.kind ?? "note",
49
+ content: row.content ?? "",
50
+ tags: row.tags ? JSON.parse(row.tags) : [],
51
+ createdAt: row.created_at ?? 0,
52
+ lastRecalledAt: row.last_recalled_at ?? null,
53
+ };
54
+ }
@@ -0,0 +1,47 @@
1
+ /**
2
+ * Sprint 11: MinHash signatures + LSH bucket persistence and candidate lookup.
3
+ *
4
+ * All queries are parameterized (PREVENT-002) — never string-concatenated.
5
+ */
6
+ import { getStateDir, normalizeSessionId } from "../../store.js";
7
+ import { openStore } from "./connection.js";
8
+ import { withTx } from "./transaction.js";
9
+ /** Persist a checkpoint's MinHash signature (idempotent by chunk_id + version). */
10
+ export function upsertMinhashSignature(chunkId, sessionId, signatureVersion, signatures, stateDir = getStateDir()) {
11
+ const db = openStore(stateDir);
12
+ const sid = normalizeSessionId(sessionId);
13
+ db.prepare(`INSERT INTO minhash_signatures(chunk_id, session_id, signature_version, signatures)
14
+ VALUES(?, ?, ?, ?)
15
+ ON CONFLICT(chunk_id, signature_version) DO UPDATE SET
16
+ session_id=excluded.session_id, signatures=excluded.signatures`).run(chunkId, sid, signatureVersion, JSON.stringify(signatures));
17
+ }
18
+ /** Persist LSH bucket memberships for a chunk (one row per bucket key). */
19
+ export function insertLshBuckets(chunkId, sessionId, signatureVersion, bucketKeys, stateDir = getStateDir()) {
20
+ const db = openStore(stateDir);
21
+ const sid = normalizeSessionId(sessionId);
22
+ const del = db.prepare("DELETE FROM dedup_lsh_buckets WHERE chunk_id = ?");
23
+ const ins = db.prepare("INSERT OR IGNORE INTO dedup_lsh_buckets(bucket_key, chunk_id, session_id, signature_version) VALUES(?, ?, ?, ?)");
24
+ withTx(db, () => {
25
+ del.run(chunkId);
26
+ for (const key of bucketKeys)
27
+ ins.run(key, chunkId, sid, signatureVersion);
28
+ });
29
+ }
30
+ /**
31
+ * Candidate chunk_ids sharing any LSH bucket with `bucketKeys`, scoped to the
32
+ * session, capped at `limit`. Single query (no N loops) — QA #15 amplification
33
+ * guard. Returns DISTINCT chunk_ids excluding `excludeChunkId` (the new row).
34
+ */
35
+ export function lshCandidateChunks(bucketKeys, sessionId, excludeChunkId, stateDir = getStateDir(), limit = 100) {
36
+ if (bucketKeys.length === 0)
37
+ return [];
38
+ const db = openStore(stateDir);
39
+ const sid = normalizeSessionId(sessionId);
40
+ const placeholders = bucketKeys.map(() => "?").join(",");
41
+ const rows = db
42
+ .prepare(`SELECT DISTINCT chunk_id FROM dedup_lsh_buckets
43
+ WHERE bucket_key IN (${placeholders}) AND session_id = ? AND chunk_id != ?
44
+ LIMIT ?`)
45
+ .all(...bucketKeys, sid, excludeChunkId, limit);
46
+ return rows.map((r) => r.chunk_id);
47
+ }
@@ -0,0 +1,81 @@
1
+ /**
2
+ * perf-samples.ts — `perf_samples` table accessors (v0.8.8 Perf dashboard).
3
+ *
4
+ * Append-only local instrumentation store for the dashboard's Perf tab: model
5
+ * endpoint latency, TPS, cache hit %, CPU/mem, and the snapshot() recompute /
6
+ * disk-write cost. One row per sample; the dashboard server reads a rolling
7
+ * window and derives p50/p95 + latest values.
8
+ *
9
+ * PREVENT-PI-004: local SQLite only, zero network.
10
+ * PREVENT-002: all SQL parameterized (? placeholders). The optional `kind`
11
+ * filter is bound as a parameter (never string-concatenated); the only
12
+ * interpolated fragment is the code-controlled `AND kind = ?` clause toggle,
13
+ * never external input.
14
+ * Pi-agnostic: no pi runtime types (mirrors game-scores.ts / meta.ts).
15
+ */
16
+ import { getStateDir } from "../../store.js";
17
+ import { openStore } from "./utils.js";
18
+ /** Allow-list of valid perf sample kinds (mirrors the table's domain). */
19
+ export const PERF_KINDS = [
20
+ "turn_latency_ms",
21
+ "provider_latency_ms",
22
+ "tps",
23
+ "cache_hit_pct",
24
+ "rss_mb",
25
+ "heap_mb",
26
+ "cpu_user_ms",
27
+ "cpu_sys_ms",
28
+ "db_recompute_ms",
29
+ "disk_write_ms",
30
+ ];
31
+ function isPerfKind(k) {
32
+ return PERF_KINDS.includes(k);
33
+ }
34
+ /**
35
+ * Record one perf sample. `ts` is set to Date.now(). SQL is fully parameterized
36
+ * (PREVENT-002); the kind is validated against the fixed allow-list. Pi-agnostic.
37
+ * Never throws on an unknown kind or non-finite value (silently ignored) so
38
+ * instrumentation can never block the agent; a known kind + finite value always
39
+ * writes.
40
+ */
41
+ export function recordPerfSample(stateDir = getStateDir(), kind, value, meta) {
42
+ if (!isPerfKind(kind))
43
+ return;
44
+ if (!Number.isFinite(value))
45
+ return;
46
+ const db = openStore(stateDir);
47
+ db.prepare(`INSERT INTO perf_samples (ts, kind, value, meta)
48
+ VALUES (?, ?, ?, ?)`).run(Date.now(), kind, value, meta != null ? JSON.stringify(meta) : null);
49
+ }
50
+ /**
51
+ * Read perf samples since `sinceTs` (epoch ms), optionally filtered by kind.
52
+ * Returns rows ascending by ts. The optional kind filter is bound as a
53
+ * parameter (PREVENT-002). Pi-agnostic. `meta` is parsed defensively (null-safe:
54
+ * PREVENT-001 — assigned to a variable before any property access).
55
+ */
56
+ export function readPerfSamples(stateDir = getStateDir(), sinceTs = 0, kind) {
57
+ const db = openStore(stateDir);
58
+ const sql = kind
59
+ ? `SELECT id, ts, kind, value, meta FROM perf_samples
60
+ WHERE ts >= ? AND kind = ? ORDER BY ts ASC`
61
+ : `SELECT id, ts, kind, value, meta FROM perf_samples
62
+ WHERE ts >= ? ORDER BY ts ASC`;
63
+ const params = kind ? [sinceTs, kind] : [sinceTs];
64
+ const rows = db.prepare(sql).all(...params);
65
+ const out = [];
66
+ for (const r of rows) {
67
+ if (!isPerfKind(r.kind))
68
+ continue; // defensive: unknown kind row skipped
69
+ let meta = null;
70
+ if (r.meta != null) {
71
+ try {
72
+ meta = JSON.parse(r.meta);
73
+ }
74
+ catch {
75
+ meta = null;
76
+ }
77
+ }
78
+ out.push({ id: r.id, ts: r.ts, kind: r.kind, value: r.value, meta });
79
+ }
80
+ return out;
81
+ }
@@ -0,0 +1,54 @@
1
+ /**
2
+ * perf-samples.test.ts — v0.8.8 perf_samples table round-trip + filtering.
3
+ * Pi-agnostic. Uses an isolated state dir (never the real user dir — G7).
4
+ */
5
+ import { describe, it, before, after } from "node:test";
6
+ import assert from "node:assert/strict";
7
+ import { tmpdir } from "node:os";
8
+ import { join } from "node:path";
9
+ import { mkdtempSync, rmSync } from "node:fs";
10
+ import { closeStore } from "./utils.js";
11
+ import { recordPerfSample, readPerfSamples, PERF_KINDS, } from "./perf-samples.js";
12
+ describe("perf-samples (v0.8.8)", () => {
13
+ let dir;
14
+ before(() => {
15
+ dir = mkdtempSync(join(tmpdir(), "mc-perfsamples-"));
16
+ process.env.MEGACOMPACT_STATE_DIR = dir;
17
+ });
18
+ after(() => {
19
+ closeStore(dir);
20
+ delete process.env.MEGACOMPACT_STATE_DIR;
21
+ rmSync(dir, { recursive: true, force: true });
22
+ });
23
+ it("records + reads back a turn_latency_ms sample with parsed meta", () => {
24
+ recordPerfSample(dir, "turn_latency_ms", 123.4, { turnIndex: 2 });
25
+ const rows = readPerfSamples(dir, 0);
26
+ assert.equal(rows.length, 1);
27
+ assert.equal(rows[0].kind, "turn_latency_ms");
28
+ assert.equal(rows[0].value, 123.4);
29
+ assert.deepEqual(rows[0].meta, { turnIndex: 2 });
30
+ });
31
+ it("filters by kind and by sinceTs", () => {
32
+ recordPerfSample(dir, "tps", 50);
33
+ recordPerfSample(dir, "rss_mb", 256);
34
+ const tps = readPerfSamples(dir, 0, "tps");
35
+ assert.equal(tps.length, 1);
36
+ assert.equal(tps[0].kind, "tps");
37
+ assert.equal(tps[0].value, 50);
38
+ const future = readPerfSamples(dir, Date.now() + 10000, "tps");
39
+ assert.equal(future.length, 0);
40
+ });
41
+ it("ignores non-finite values + unknown kinds (never throws, nothing added)", () => {
42
+ const before = readPerfSamples(dir, 0).length;
43
+ recordPerfSample(dir, "tps", Number.NaN);
44
+ recordPerfSample(dir, "tps", Infinity);
45
+ assert.doesNotThrow(() => recordPerfSample(dir, "bogus", 1));
46
+ const after = readPerfSamples(dir, 0).length;
47
+ assert.equal(after, before);
48
+ });
49
+ it("PERF_KINDS lists the 10 instrumentation kinds", () => {
50
+ assert.equal(PERF_KINDS.length, 10);
51
+ assert.ok(PERF_KINDS.includes("db_recompute_ms"));
52
+ assert.ok(PERF_KINDS.includes("cache_hit_pct"));
53
+ });
54
+ });
@@ -261,6 +261,20 @@ export function initSchema(db) {
261
261
  icon TEXT,
262
262
  unlocked_at INTEGER NULL
263
263
  ) WITHOUT ROWID;
264
+
265
+ -- v0.8.8 Perf dashboard: append-only local instrumentation samples (one row
266
+ -- per turn / provider round-trip / 5s cpu-mem tick / snapshot-recompute).
267
+ -- Drives the dashboard Perf tab. Local SQLite (PREVENT-PI-004); parameterized
268
+ -- accessors in perf-samples.ts (PREVENT-002).
269
+ CREATE TABLE IF NOT EXISTS perf_samples (
270
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
271
+ ts INTEGER NOT NULL,
272
+ kind TEXT NOT NULL,
273
+ value REAL NOT NULL,
274
+ meta TEXT
275
+ );
276
+ CREATE INDEX IF NOT EXISTS idx_perf_samples_ts ON perf_samples(ts);
277
+ CREATE INDEX IF NOT EXISTS idx_perf_samples_kind_ts ON perf_samples(kind, ts);
264
278
  `);
265
279
  // Idempotent column migrations. `CREATE TABLE IF NOT EXISTS` is a no-op on a
266
280
  // pre-existing table, so new columns added to context_chunks after a store was
@@ -0,0 +1,39 @@
1
+ /**
2
+ * Future-feature foundation: resume sessions, daily activity log, lessons learned.
3
+ *
4
+ * Scaffolded tables + minimal helpers so all store data lives in SQLite from
5
+ * day one. Full UI/recall for these lands in later sprints. All queries are
6
+ * parameterized (PREVENT-002).
7
+ */
8
+ import { getStateDir, normalizeSessionId } from "../../store.js";
9
+ import { openStore } from "./connection.js";
10
+ /** Upsert a `sessions` row (resume + per-repo session history). */
11
+ export function touchSession(sessionId, repo, stateDir = getStateDir()) {
12
+ const db = openStore(stateDir);
13
+ const sid = normalizeSessionId(sessionId);
14
+ const existing = db
15
+ .prepare("SELECT started_at FROM sessions WHERE session_id = ?")
16
+ .get(sid);
17
+ const now = Math.floor(Date.now() / 1000);
18
+ if (!existing) {
19
+ db.prepare(`INSERT INTO sessions(session_id, repo, started_at, last_compacted_at, status)
20
+ VALUES(?, ?, ?, ?, 'active')`).run(sid, repo ?? null, now, now);
21
+ }
22
+ else {
23
+ db.prepare("UPDATE sessions SET last_compacted_at = ?, repo = COALESCE(?, repo), status = 'active' WHERE session_id = ?").run(now, repo ?? null, sid);
24
+ }
25
+ }
26
+ /** Append a `daily_log` entry (day = YYYY-MM-DD, local-naive from Date). */
27
+ export function logDaily(sessionId, event, detail, tokensSaved, stateDir = getStateDir()) {
28
+ const db = openStore(stateDir);
29
+ const day = new Date().toISOString().slice(0, 10);
30
+ const now = Math.floor(Date.now() / 1000);
31
+ db.prepare(`INSERT INTO daily_log(day, session_id, event, detail, tokens_saved, ts)
32
+ VALUES(?, ?, ?, ?, ?, ?)`).run(day, normalizeSessionId(sessionId), event, detail ?? null, tokensSaved, now);
33
+ }
34
+ /** Append a `lessons` entry (future lessons-learned browse/recall). */
35
+ export function addLesson(sessionId, repo, lesson, stateDir = getStateDir()) {
36
+ const db = openStore(stateDir);
37
+ const now = Math.floor(Date.now() / 1000);
38
+ db.prepare(`INSERT INTO lessons(session_id, repo, lesson, ts) VALUES(?, ?, ?, ?)`).run(normalizeSessionId(sessionId), repo ?? null, lesson, now);
39
+ }
@@ -0,0 +1,19 @@
1
+ /**
2
+ * Transaction wrapper using SAVEPOINT so it nests safely under an outer
3
+ * transaction (unlike `BEGIN`, which SQLite rejects when one is already open).
4
+ *
5
+ * Mirrors better-sqlite3's `db.transaction(fn)` semantics — callers that wrap a
6
+ * batch in withTx (e.g. backfill) can still call helpers that also use withTx.
7
+ */
8
+ export function withTx(db, fn) {
9
+ db.exec("SAVEPOINT mc_tx");
10
+ try {
11
+ fn();
12
+ db.exec("RELEASE mc_tx");
13
+ }
14
+ catch (e) {
15
+ db.exec("ROLLBACK TO mc_tx");
16
+ db.exec("RELEASE mc_tx");
17
+ throw e;
18
+ }
19
+ }
@@ -23,3 +23,4 @@ export * from "./sqlite/maintenance.js";
23
23
  export * from "./sqlite/game-state.js";
24
24
  export * from "./sqlite/game-scores.js";
25
25
  export * from "./sqlite/game-achievements.js";
26
+ export * from "./sqlite/perf-samples.js";