trajectories 0.0.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (204) hide show
  1. package/CHANGELOG.md +23 -0
  2. package/LICENSE +201 -0
  3. package/README.md +274 -2
  4. package/dist/cli.d.ts +2 -0
  5. package/dist/cli.js +8542 -0
  6. package/dist/convert.d.ts +31 -0
  7. package/dist/convert.js +431 -0
  8. package/dist/index.d.ts +7 -0
  9. package/dist/index.js +8402 -0
  10. package/dist/log-baseline.d.ts +26 -0
  11. package/dist/log-baseline.js +104 -0
  12. package/dist/log-content.d.ts +16 -0
  13. package/dist/log-content.js +80 -0
  14. package/dist/log-io.d.ts +10 -0
  15. package/dist/log-io.js +121 -0
  16. package/dist/log-read.d.ts +24 -0
  17. package/dist/log-read.js +117 -0
  18. package/dist/log-reconcile.d.ts +5 -0
  19. package/dist/log-reconcile.js +38 -0
  20. package/dist/log-records.d.ts +35 -0
  21. package/dist/log-records.js +85 -0
  22. package/dist/log-recovery.d.ts +18 -0
  23. package/dist/log-recovery.js +123 -0
  24. package/dist/log-stage.d.ts +14 -0
  25. package/dist/log-stage.js +138 -0
  26. package/dist/log-state.d.ts +89 -0
  27. package/dist/log-state.js +92 -0
  28. package/dist/log-switch.d.ts +3 -0
  29. package/dist/log-switch.js +98 -0
  30. package/dist/log-types.d.ts +129 -0
  31. package/dist/log-types.js +8 -0
  32. package/dist/log.d.ts +8 -0
  33. package/dist/log.js +62 -0
  34. package/dist/migrate.d.ts +29 -0
  35. package/dist/migrate.js +173 -0
  36. package/dist/normalization-runtime.d.ts +1 -0
  37. package/dist/normalization-runtime.js +3150 -0
  38. package/dist/portable.d.ts +9 -0
  39. package/dist/portable.js +26 -0
  40. package/dist/readers/canonical.d.ts +13 -0
  41. package/dist/readers/canonical.js +54 -0
  42. package/dist/readers/cline.d.ts +10 -0
  43. package/dist/readers/cline.js +107 -0
  44. package/dist/readers/cursor.d.ts +20 -0
  45. package/dist/readers/cursor.js +374 -0
  46. package/dist/readers/deepagents.d.ts +41 -0
  47. package/dist/readers/deepagents.js +163 -0
  48. package/dist/readers/fx.d.ts +11 -0
  49. package/dist/readers/fx.js +573 -0
  50. package/dist/readers/github-copilot.d.ts +8 -0
  51. package/dist/readers/github-copilot.js +216 -0
  52. package/dist/readers/grok-build.d.ts +12 -0
  53. package/dist/readers/grok-build.js +391 -0
  54. package/dist/readers/guards.d.ts +4 -0
  55. package/dist/readers/guards.js +121 -0
  56. package/dist/readers/pi-compaction.d.ts +7 -0
  57. package/dist/readers/pi-compaction.js +112 -0
  58. package/dist/readers/pi-context.d.ts +13 -0
  59. package/dist/readers/pi-context.js +64 -0
  60. package/dist/store.d.ts +25 -0
  61. package/dist/store.js +218 -0
  62. package/dist/types.d.ts +91 -0
  63. package/dist/types.js +1 -0
  64. package/dist/writers/claude-code.d.ts +5 -0
  65. package/dist/writers/claude-code.js +51 -0
  66. package/dist/writers/cline.d.ts +3 -0
  67. package/dist/writers/cline.js +77 -0
  68. package/dist/writers/codex.d.ts +3 -0
  69. package/dist/writers/codex.js +32 -0
  70. package/dist/writers/cursor.d.ts +3 -0
  71. package/dist/writers/cursor.js +197 -0
  72. package/dist/writers/deepagents.d.ts +3 -0
  73. package/dist/writers/deepagents.js +116 -0
  74. package/dist/writers/entropy.d.ts +5 -0
  75. package/dist/writers/entropy.js +33 -0
  76. package/dist/writers/fx.d.ts +3 -0
  77. package/dist/writers/fx.js +135 -0
  78. package/dist/writers/github-copilot.d.ts +3 -0
  79. package/dist/writers/github-copilot.js +71 -0
  80. package/dist/writers/grok-build.d.ts +3 -0
  81. package/dist/writers/grok-build.js +43 -0
  82. package/dist/writers/helpers.d.ts +17 -0
  83. package/dist/writers/helpers.js +75 -0
  84. package/dist/writers/omp.d.ts +3 -0
  85. package/dist/writers/omp.js +7 -0
  86. package/dist/writers/opencode.d.ts +3 -0
  87. package/dist/writers/opencode.js +65 -0
  88. package/dist/writers/pi.d.ts +3 -0
  89. package/dist/writers/pi.js +45 -0
  90. package/docs/cline-native-format.md +43 -0
  91. package/docs/copilot-native-format.md +78 -0
  92. package/docs/cursor-native-format.md +103 -0
  93. package/docs/deepagents-native-format.md +98 -0
  94. package/docs/evidence-api.md +170 -0
  95. package/docs/fx-native-format.md +87 -0
  96. package/docs/harness-parity-bootstrap.md +78 -0
  97. package/docs/harness-support-parity.md +107 -0
  98. package/docs/inspirations.md +103 -0
  99. package/docs/ledger-migration.md +141 -0
  100. package/docs/pi-compaction-context.md +75 -0
  101. package/docs/pi-provider-context.md +76 -0
  102. package/docs/research/2026-10-06-huihua-and-oar.md +176 -0
  103. package/docs/research/ledger/README.md +96 -0
  104. package/docs/research/ledger/confer/2026-09-13-round-10/README.md +61 -0
  105. package/docs/research/ledger/confer/2026-09-13-round-2/cass.md +72 -0
  106. package/docs/research/ledger/confer/2026-09-13-round-3/README.md +70 -0
  107. package/docs/research/ledger/confer/2026-09-13-round-5/README.md +61 -0
  108. package/docs/research/ledger/confer/2026-09-13-round-6/README.md +68 -0
  109. package/docs/research/ledger/confer/2026-09-13-round-7/README.md +64 -0
  110. package/docs/research/ledger/confer/2026-09-13-round-9/README.md +59 -0
  111. package/docs/research/ledger/confer/2026-09-14-wake/wake.md +146 -0
  112. package/docs/research/ledger/docs/inspirations.md +320 -0
  113. package/docs/research/ledger/docs/pr2-pr6-integration.md +53 -0
  114. package/docs/research/ledger/docs/research/2026-09-07-confer-inspirations.md +163 -0
  115. package/docs/research/ledger/docs/research/2026-09-07-letta-continuous-service.md +104 -0
  116. package/docs/research/ledger/docs/research/2026-09-08-claude-history-retrieval.md +117 -0
  117. package/docs/research/ledger/docs/research/2026-09-09-agentsview-native-discovery.md +87 -0
  118. package/docs/research/ledger/docs/research/2026-09-09-daemon-observation.md +231 -0
  119. package/docs/research/ledger/docs/research/2026-09-09-discovery-pagination.md +114 -0
  120. package/docs/research/ledger/docs/research/2026-09-09-harness-transcript-contract.md +145 -0
  121. package/docs/research/ledger/docs/research/2026-09-09-native-history-discovery.md +180 -0
  122. package/docs/research/ledger/docs/research/2026-09-09-native-history-implementation.md +137 -0
  123. package/docs/research/ledger/docs/research/2026-09-09-other-inspirations.md +279 -0
  124. package/docs/research/ledger/docs/research/2026-09-09-trajectory-and-letta-code.md +128 -0
  125. package/docs/research/ledger/docs/research/2026-09-13-diagnostics-coverage.md +116 -0
  126. package/docs/research/ledger/docs/research/2026-09-13-fts-rebuild.md +83 -0
  127. package/docs/research/ledger/docs/research/2026-09-13-read-path-split.md +74 -0
  128. package/docs/research/ledger/docs/research/2026-09-13-runtime-rollout.md +102 -0
  129. package/docs/research/ledger/docs/research/2026-09-13-scan-outcomes.md +80 -0
  130. package/docs/research/ledger/docs/research/2026-09-13-search-quality.md +105 -0
  131. package/docs/research/ledger/docs/research/2026-09-13-source-coverage.md +64 -0
  132. package/docs/research/ledger/docs/research/2026-09-13-sync-wall-budget.md +82 -0
  133. package/docs/research/ledger/docs/research/2026-09-13-typed-read-gate.md +81 -0
  134. package/docs/research/ledger/docs/research/2026-09-13-writer-lease.md +84 -0
  135. package/docs/research/ledger/docs/research/2026-09-16-c1-copy-fallback.md +227 -0
  136. package/docs/research/ledger/docs/research/2026-09-16-c10-kimi.md +414 -0
  137. package/docs/research/ledger/docs/research/2026-09-16-c2-derive-counter.md +216 -0
  138. package/docs/research/ledger/docs/research/2026-09-16-c3-echo-filter.md +305 -0
  139. package/docs/research/ledger/docs/research/2026-09-16-c4-title-index.md +349 -0
  140. package/docs/research/ledger/docs/research/2026-09-16-c5-trigram.md +268 -0
  141. package/docs/research/ledger/docs/research/2026-09-16-c6-recency-prior.md +438 -0
  142. package/docs/research/ledger/docs/research/2026-09-16-c7-tombstones.md +467 -0
  143. package/docs/research/ledger/docs/research/2026-09-16-c8-remote-truth.md +390 -0
  144. package/docs/research/ledger/docs/research/2026-09-16-c9-grok.md +380 -0
  145. package/docs/research/ledger/docs/research/2026-09-16-wake-port-program.md +38 -0
  146. package/docs/research/ledger/docs/research/2026-09-17-cursor-blobstore.md +157 -0
  147. package/docs/research/ledger/manifest.json +540 -0
  148. package/legacy/NOTICE.md +50 -0
  149. package/legacy/bin/ledger-core.ts +7 -0
  150. package/legacy/bin/ledger.ts +7 -0
  151. package/legacy/bin/traj-bridge.ts +7 -0
  152. package/legacy/scripts/import-claude-history.ts +130 -0
  153. package/legacy/scripts/native-history-smoke.ts +52 -0
  154. package/legacy/scripts/observe-daemon.ts +570 -0
  155. package/legacy/scripts/retitle-sessions.md +19 -0
  156. package/legacy/scripts/verify-runtime-rollout.ts +151 -0
  157. package/legacy/src/canonical.ts +108 -0
  158. package/legacy/src/cli.ts +1293 -0
  159. package/legacy/src/db.ts +2254 -0
  160. package/legacy/src/diagnostics.ts +87 -0
  161. package/legacy/src/discovery.ts +56 -0
  162. package/legacy/src/embeddings.ts +382 -0
  163. package/legacy/src/filter.ts +602 -0
  164. package/legacy/src/gemini_embeddings.ts +103 -0
  165. package/legacy/src/identity.ts +9 -0
  166. package/legacy/src/index.ts +10 -0
  167. package/legacy/src/ingest.ts +791 -0
  168. package/legacy/src/managed.ts +178 -0
  169. package/legacy/src/native/common.ts +68 -0
  170. package/legacy/src/native/cursor.ts +647 -0
  171. package/legacy/src/native/grok.ts +540 -0
  172. package/legacy/src/native/kimi.ts +706 -0
  173. package/legacy/src/native/opencode.ts +470 -0
  174. package/legacy/src/native/sqlite-ro.ts +281 -0
  175. package/legacy/src/native.ts +97 -0
  176. package/legacy/src/normalizer.ts +457 -0
  177. package/legacy/src/parentage.ts +562 -0
  178. package/legacy/src/pressure.ts +162 -0
  179. package/legacy/src/private-storage.ts +74 -0
  180. package/legacy/src/reader.ts +1255 -0
  181. package/legacy/src/scanner.ts +591 -0
  182. package/legacy/src/schema-gate.ts +263 -0
  183. package/legacy/src/schema.ts +829 -0
  184. package/legacy/src/semantic-top-k.ts +71 -0
  185. package/legacy/src/session-hash.ts +49 -0
  186. package/legacy/src/store-status.ts +218 -0
  187. package/legacy/src/sync-preparation.ts +45 -0
  188. package/legacy/src/sync-truth.ts +843 -0
  189. package/legacy/src/sync.ts +1219 -0
  190. package/legacy/src/types.ts +436 -0
  191. package/legacy/src/vocab.txt +30522 -0
  192. package/legacy/src/watch.ts +673 -0
  193. package/legacy/src/writer-lease.ts +310 -0
  194. package/ops/README.md +120 -0
  195. package/ops/install-cli.sh +63 -0
  196. package/ops/install-release.sh +49 -0
  197. package/ops/linux/install-systemd-user.sh +103 -0
  198. package/ops/macos/install-launchd.sh +72 -0
  199. package/ops/sync-settings.sh +37 -0
  200. package/package.json +19 -5
  201. package/skills/trajectories/SKILL.md +138 -0
  202. package/skills/trajectories/scripts/trajectories_recent.sh +26 -0
  203. package/skills/trajectories/scripts/trajectories_search.sh +26 -0
  204. package/index.js +0 -1
@@ -0,0 +1,2254 @@
1
+ import { Database } from "bun:sqlite";
2
+ import type { DiscoveryCheckpoint, DiscoverySummary } from "./discovery.js";
3
+ import type { ScanResult } from "./types.js";
4
+ import { ensurePrivateDatabase, ensurePrivateFile } from "./private-storage.js";
5
+ import { isMemoryStorePath } from "./writer-lease.js";
6
+ import {
7
+ CREATE_TABLES_SQL,
8
+ DERIVE_COUNTERS,
9
+ INDEXABLE_TITLE_SQL,
10
+ REPAIR_EMBEDDING_METADATA_TRIGGERS_SQL,
11
+ SCHEMA_VERSION,
12
+ TITLES_FTS_REBUILD_SQL,
13
+ V2_MIGRATION_SQL, V3_MIGRATION_SQL, V4_MIGRATION_SQL,
14
+ PARENTAGE_MIGRATION_SQL, INGEST_CURSORS_SQL, INGEST_DIAGNOSTICS_SQL,
15
+ FTS_REBUILD_MIGRATION_SQL,
16
+ } from "./schema.js";
17
+ import { ledgerId, nativeId } from "./identity.js";
18
+ import { echoCount, echoExcludedBatchIds, echoExcludedIds } from "./filter.js";
19
+ import { capDiagnosticSample, mergeIngestDiagnosticAggregates } from "./diagnostics.js";
20
+ import {
21
+ extendSessionHash,
22
+ hashSessionRecords,
23
+ SESSION_HASH_VERSION,
24
+ } from "./session-hash.js";
25
+ import {
26
+ type SessionParentage,
27
+ EMPTY_SESSION_PARENTAGE,
28
+ extractSessionParentage,
29
+ mergeParentage,
30
+ parentageEquals,
31
+ } from "./parentage.js";
32
+ import {
33
+ embed,
34
+ getEmbeddingProvider,
35
+ getRecordEmbeddingText,
36
+ serializeEmbedding,
37
+ } from "./embeddings.js";
38
+ import type {
39
+ CanonicalDbRecord,
40
+ DiagnosticsSummary,
41
+ DiscoveredSession,
42
+ IngestCursor,
43
+ IngestDiagnosticAggregate,
44
+ IngestDiagnosticRecord,
45
+ IngestQueueEntry,
46
+ QueryResult,
47
+ ScanOutcome,
48
+ SessionRecord,
49
+ SourceCoverageRow,
50
+ TombstoneMode,
51
+ TombstoneRecord,
52
+ } from "./types.js";
53
+ import {
54
+ DEFAULT_PRESSURE_STATE,
55
+ type AdaptivePressureState,
56
+ } from "./pressure.js";
57
+ import {
58
+ allSessions,
59
+ compatibleEmbeddingCount,
60
+ countRecords,
61
+ countSessions,
62
+ lastSessions,
63
+ queryHybrid,
64
+ queryLexical,
65
+ querySemantic,
66
+ sessionById,
67
+ sessionRecords,
68
+ } from "./reader.js";
69
+
70
+ /** A staged coverage payload that could not be vouched for. Names the stage
71
+ * identity so the operator can find the row; thrown inside the ingest
72
+ * transaction, so the pass aborts instead of fabricating coverage. */
73
+ export class StagedDiagnosticsParseError extends Error {
74
+ constructor(
75
+ readonly key: readonly [string, string, string],
76
+ readonly cause: unknown,
77
+ ) {
78
+ super(
79
+ `Staged diagnostics for ${key[0]}/${key[1]}/${key[2]} are not a valid ` +
80
+ `diagnostics aggregate${cause instanceof Error ? `: ${cause.message}` : ""}`,
81
+ );
82
+ this.name = "StagedDiagnosticsParseError";
83
+ }
84
+ }
85
+
86
+ /**
87
+ * The deletion verb was pointed at an id this store does not hold. A tombstone
88
+ * for a session that was never ingested would be an exclusion rule, not a
89
+ * deletion — a different mechanism with different consequences, and one this
90
+ * store deliberately does not have.
91
+ */
92
+ export class SessionNotFoundError extends Error {
93
+ readonly code = "SESSION_NOT_FOUND";
94
+ constructor(readonly sessionId: string) {
95
+ super(`No session ${sessionId} in this store; nothing to delete.`);
96
+ this.name = "SessionNotFoundError";
97
+ }
98
+ }
99
+
100
+ /**
101
+ * A writer open that was told not to initialize a store (`requireExisting`) and
102
+ * found nothing to open.
103
+ *
104
+ * `ledger delete` opens this way, because a deletion has nothing to act on when
105
+ * the path holds no Ledger store and an empty database conjured for it would be
106
+ * an artifact nobody asked for. The refusal happens before any PRAGMA or DDL
107
+ * touches the file, and the connection itself is opened with creation disabled,
108
+ * so a store that vanished between the verb's classification and this open is
109
+ * neither created nor adopted.
110
+ */
111
+ export class StoreNotInitializedError extends Error {
112
+ readonly code = "STORE_NOT_INITIALIZED";
113
+ constructor(readonly path: string) {
114
+ super(`No store at ${path}: the file is missing, empty, or not a Ledger store.`);
115
+ this.name = "StoreNotInitializedError";
116
+ }
117
+ }
118
+
119
+ /**
120
+ * A write tried to admit a session the user deleted. Thrown by the store's own
121
+ * write boundary (`saveSession`, `appendSessionRecords`, `commitStagedIngest`,
122
+ * `insertRecordEmbeddings`) rather than only by the ingest planner, so a racing
123
+ * writer, a library caller, or a staged promotion that outlived the deletion
124
+ * cannot put a tombstoned session back.
125
+ */
126
+ export class TombstonedSessionError extends Error {
127
+ readonly code = "SESSION_TOMBSTONED";
128
+ constructor(readonly sessionId: string) {
129
+ super(
130
+ `Session ${sessionId} was deleted by the user; this store does not re-admit it. ` +
131
+ `The tombstone is the record of that deletion (session_tombstones).`,
132
+ );
133
+ this.name = "TombstonedSessionError";
134
+ }
135
+ }
136
+
137
+ /** What one `tombstoneSession` call did, beyond the row it wrote or found. */
138
+ export interface TombstoneOutcome {
139
+ tombstone: TombstoneRecord;
140
+ /** True when this call wrote the tombstone; false when it already existed. */
141
+ created: boolean;
142
+ /** True when this call carried an existing `suppress` tombstone to `hard`. */
143
+ escalated: boolean;
144
+ /** Records removed from the store by this call: hard mode only, else 0. */
145
+ recordsRemoved: number;
146
+ }
147
+
148
+ /**
149
+ * What one ingest commit did: which write path it took (`inserted`/`updated` by
150
+ * whole-session replacement, `appended` by continuation), whether the pass only
151
+ * staged a replacement that a later chunk will promote, and whether it had to
152
+ * hand the session to a cold rebuild.
153
+ */
154
+ export interface IngestCommitOutcome {
155
+ inserted: boolean;
156
+ updated: boolean;
157
+ deduped: boolean;
158
+ appended: number;
159
+ staged?: boolean;
160
+ deferred?: boolean;
161
+ deferReason?: string;
162
+ historicalRecordsHashed: number;
163
+ }
164
+
165
+ function isDiagnosticAggregate(value: unknown): value is IngestDiagnosticAggregate {
166
+ if (typeof value !== "object" || value === null) return false;
167
+ const entry = value as Partial<IngestDiagnosticAggregate>;
168
+ return (
169
+ typeof entry.code === "string" &&
170
+ typeof entry.occurrenceCount === "number" &&
171
+ Number.isSafeInteger(entry.occurrenceCount) &&
172
+ entry.occurrenceCount >= 0 &&
173
+ (entry.sample === undefined || entry.sample === null || typeof entry.sample === "string") &&
174
+ (entry.firstSeenAt === undefined || typeof entry.firstSeenAt === "string")
175
+ );
176
+ }
177
+
178
+ export class TrajectoryDb {
179
+ private db: Database;
180
+ readonly path: string;
181
+
182
+ /**
183
+ * Open the store as a writer — creating and migrating it, as every writer verb
184
+ * does — unless `requireExisting` is set, which is `delete`'s open: a writer
185
+ * that must not create or adopt a store. That variant disables file creation on
186
+ * the connection and checks the shape before any PRAGMA or DDL runs, so a
187
+ * missing, empty, or foreign file is refused with `StoreNotInitializedError`
188
+ * and nothing is written into it.
189
+ *
190
+ * `initialize: false` is the parent handle after the bounded preparation child
191
+ * succeeded: never migrate or repair synchronously on the supervising event
192
+ * loop. Private-storage checks still run, but they must not create a missing
193
+ * primary file in either of those modes.
194
+ */
195
+ constructor(path: string = "ledger.sqlite", options: { initialize?: boolean; requireExisting?: boolean } = {}) {
196
+ this.path = path;
197
+ const requireExisting = options.requireExisting === true;
198
+ const skipInit = options.initialize === false;
199
+ if (!isMemoryStorePath(path)) {
200
+ if (requireExisting || skipInit) {
201
+ ensurePrivateFile(path, false);
202
+ for (const suffix of ["-wal", "-shm", "-journal"]) ensurePrivateFile(path + suffix, false);
203
+ } else {
204
+ ensurePrivateDatabase(path);
205
+ }
206
+ }
207
+ this.db = requireExisting
208
+ ? TrajectoryDb.openExisting(path)
209
+ : new Database(path, { readwrite: true, create: !skipInit });
210
+ try {
211
+ if (requireExisting) this.init(true);
212
+ else if (skipInit) {
213
+ this.db.run("PRAGMA busy_timeout = 0");
214
+ this.db.run("PRAGMA foreign_keys = ON");
215
+ if (this.getSchemaVersion() !== SCHEMA_VERSION) throw new Error("Prepared database schema version changed; retry sync");
216
+ } else this.init();
217
+ if (!isMemoryStorePath(path) && !requireExisting && !skipInit) ensurePrivateDatabase(path);
218
+ } catch (error) {
219
+ this.db.close();
220
+ throw error;
221
+ }
222
+ }
223
+
224
+ /**
225
+ * Open a writable connection that never creates the file. SQLite wants an
226
+ * explicit access mode for that (`create: false` alone is a misuse), and a
227
+ * missing file surfaces as the same typed refusal as an uninitialized one:
228
+ * from the caller's side both answers are "there is no store here".
229
+ */
230
+ private static openExisting(path: string): Database {
231
+ try {
232
+ return new Database(path, { readwrite: true, create: false });
233
+ } catch {
234
+ throw new StoreNotInitializedError(path);
235
+ }
236
+ }
237
+
238
+ private init(requireExisting = false): void {
239
+ // Read the shape first, before any connection setting or DDL: a store that
240
+ // must not be initialized has to be refused while the file is still
241
+ // untouched, and `PRAGMA journal_mode = WAL` alone already writes to it.
242
+ if (requireExisting) {
243
+ const version =
244
+ this.db.query<{ user_version: number }, []>("PRAGMA user_version;").get()?.user_version ?? 0;
245
+ const hasSessions = !!this.db
246
+ .query<{ name: string }, []>(
247
+ "SELECT name FROM sqlite_master WHERE type='table' AND name='sessions' LIMIT 1;",
248
+ )
249
+ .get();
250
+ if (version === 0 && !hasSessions) throw new StoreNotInitializedError(this.path);
251
+ }
252
+
253
+ this.db.run("PRAGMA busy_timeout = 15000;");
254
+ this.db.run("PRAGMA journal_mode = WAL;");
255
+ // Chunk transactions stay fully durable. Keep SQLite's conservative
256
+ // checkpoint window explicit; the scheduled maintenance child also
257
+ // performs a deadline-bound passive checkpoint before the one-shot exits.
258
+ this.db.run("PRAGMA wal_autocheckpoint = 1000;");
259
+ this.db.run("PRAGMA foreign_keys = ON;");
260
+
261
+ const userVersion =
262
+ this.db
263
+ .query<{ user_version: number }, []>("PRAGMA user_version;")
264
+ .get()?.user_version ?? 0;
265
+
266
+ const hasTables = !!this.db
267
+ .query<{ name: string }, []>(
268
+ "SELECT name FROM sqlite_master WHERE type='table' AND name='sessions' LIMIT 1;",
269
+ )
270
+ .get();
271
+
272
+ // The tables this store already holds, read before any DDL below creates
273
+ // the ones this build needs. A counter that names its rules is a claim
274
+ // about a derivation, and what that derivation produces has to be in the
275
+ // store already; creating the tables first would erase the evidence.
276
+ const heldTables = new Set(
277
+ this.db
278
+ .query<{ name: string }, []>("SELECT name FROM sqlite_master WHERE type = 'table';")
279
+ .all()
280
+ .map((row) => row.name),
281
+ );
282
+
283
+ if (userVersion === 0 && !hasTables) {
284
+ // A killed preparation child must leave an empty retryable store, never
285
+ // a half-created sessions table with user_version=0 that looks foreign.
286
+ this.db.transaction(() => {
287
+ this.db.run(CREATE_TABLES_SQL);
288
+ this.configureFtsAutomerge();
289
+ this.db.run(`PRAGMA user_version = ${SCHEMA_VERSION};`);
290
+ }).immediate();
291
+ } else if (userVersion < SCHEMA_VERSION) {
292
+ if (userVersion === 0 && hasTables) {
293
+ throw new Error(
294
+ `Database schema version ${userVersion} is older than required schema version ${SCHEMA_VERSION}. Database migrations are not yet implemented; please re-build the database.`,
295
+ );
296
+ }
297
+ this.migrate(userVersion);
298
+ } else if (userVersion > SCHEMA_VERSION) {
299
+ throw new Error(
300
+ `Database schema version ${userVersion} is newer than supported schema version ${SCHEMA_VERSION}.`,
301
+ );
302
+ } else {
303
+ this.db.run(CREATE_TABLES_SQL);
304
+ }
305
+
306
+ // Experimental v4 databases can contain obsolete trigger definitions.
307
+ // Repair only these owned triggers without replaying the identity migration.
308
+ this.db.transaction(() => {
309
+ this.db.run(REPAIR_EMBEDDING_METADATA_TRIGGERS_SQL);
310
+ })();
311
+
312
+ this.detectDeriveCounters(hasTables, heldTables);
313
+ }
314
+
315
+ /**
316
+ * Record this build's derive-rule counters, raising a re-derivation flag for
317
+ * every derived index the store holds under other rules.
318
+ *
319
+ * Open-time and unconditional, the shape Wake's `open_conn` uses
320
+ * (`db.rs#L176-L196`): a stored counter that contradicts this build's gets its
321
+ * flag, and the next pass that holds the writer lease re-derives it. Whether a
322
+ * store that records *nothing* — one written before the counter existed —
323
+ * also counts as pending is the counter's own call (`flagsWhenUnrecorded`: the
324
+ * FTS projection re-derives, the deferred vectors lane only records). A store
325
+ * that did not exist before this open is derived by the current rules by
326
+ * construction, so it records the counters and no flag; that is what keeps a
327
+ * fresh store from re-deriving what it has just written.
328
+ *
329
+ * A *newer* counter is stamped down rather than refused. The fence for a
330
+ * format this build does not understand is the read gate
331
+ * (`classifyReadStore`): a writer can restore an index to its own rules —
332
+ * `records` is canonical, so re-deriving is lossless in either direction —
333
+ * while a reader can only serve what it understands. An older build's pass
334
+ * over a newer store therefore costs one re-derivation, and the newer build's
335
+ * next pass costs one more; neither can lose canonical data.
336
+ *
337
+ * A counter that *matches* is checked against its outputs (the tables the
338
+ * rules derive): a store that records the current rule but is missing what
339
+ * the rule produces was not derived by it, so the flag is raised there too.
340
+ * The matching value is left in place — it is not the counter that is wrong,
341
+ * the derivation it claims is unfinished — and the next pass completes it.
342
+ */
343
+ private detectDeriveCounters(storeExisted: boolean, heldTables: ReadonlySet<string>): void {
344
+ const readFormat = this.db.query<{ value: string }, [string]>(
345
+ "SELECT value FROM schema_meta WHERE key = ?;",
346
+ );
347
+ const stamp = this.db.prepare(
348
+ "INSERT OR REPLACE INTO schema_meta (key, value) VALUES (?, ?)",
349
+ );
350
+ this.db.transaction(() => {
351
+ for (const counter of DERIVE_COUNTERS) {
352
+ const stored = readFormat.get(counter.format)?.value ?? null;
353
+ if (stored === counter.current) {
354
+ // The value claims a derivation under these rules; a matched counter
355
+ // whose output is gone is that claim without its evidence — the store
356
+ // cannot have been derived by these rules, whatever it records.
357
+ if (storeExisted && counter.outputs.some((table) => !heldTables.has(table))) {
358
+ stamp.run(counter.flag, "1");
359
+ }
360
+ continue;
361
+ }
362
+ if (storeExisted && (stored !== null || counter.flagsWhenUnrecorded)) {
363
+ stamp.run(counter.flag, "1");
364
+ }
365
+ stamp.run(counter.format, counter.current);
366
+ }
367
+ })();
368
+ }
369
+
370
+ private configureFtsAutomerge(): void {
371
+ // Pin SQLite's conservative default instead of inheriting an arbitrary
372
+ // database-level setting. Deeper catch-up compaction remains isolated in
373
+ // the deadline-bound maintenance child on idle scheduled passes.
374
+ this.db.run("INSERT INTO records_fts(records_fts, rank) VALUES ('automerge', 4)");
375
+ // The title index gets the same pinned policy rather than the table-level
376
+ // default it would otherwise inherit: one policy for both indexes, so a
377
+ // future change to it cannot reach one and miss the other.
378
+ this.db.run("INSERT INTO titles_fts(titles_fts, rank) VALUES ('automerge', 4)");
379
+ }
380
+
381
+ /**
382
+ * v10 → v11: rebuild `records_fts`. The index is derived — `records` is the
383
+ * authoritative side — so the rebuild discards the old table and repopulates
384
+ * it through the same projection ingest writes, and re-derives the index
385
+ * layout from that content.
386
+ *
387
+ * The projection is whole-table on purpose: `records` is what the echo filter
388
+ * reads, so the filter's set is recomputed here — the derivation is also what
389
+ * a *rule* change runs, and a set carried over from the previous rules would
390
+ * be exactly what a rule change must not keep. The fresh set then removes its
391
+ * rows from the freshly built index, which is what puts the rebuild on the
392
+ * same footing as ingest: there, the same decision wrote the set and skipped
393
+ * the FTS row; here, it rewrites the set and deletes what it names. The gate
394
+ * below checks the arithmetic either way.
395
+ */
396
+ private rebuildRecordsFts(): void {
397
+ this.db.run(FTS_REBUILD_MIGRATION_SQL);
398
+ const excluded = echoExcludedIds(this.db);
399
+ this.db.run("DELETE FROM records_echo;");
400
+ if (excluded.size > 0) {
401
+ // The ids are held as JSON and joined through the primary key: one probe
402
+ // per excluded record, never a scan of `records`.
403
+ this.db.run(
404
+ `INSERT INTO records_echo (record_id, session_id)
405
+ SELECT r.id, r.session_id FROM records r JOIN json_each(?) e ON r.id = e.value;`,
406
+ [JSON.stringify([...excluded])],
407
+ );
408
+ }
409
+ this.db.run("DELETE FROM records_fts WHERE record_id IN (SELECT record_id FROM records_echo);");
410
+ verifyRecordsFtsRebuild(this.db);
411
+ }
412
+
413
+ /**
414
+ * The title index's re-derivation: discard, recreate, repopulate from
415
+ * `sessions.title`, gate — the same increment as `records_fts`, for the same
416
+ * reason. The index is derived from a canonical column, so a rebuild is also
417
+ * the repair: a row the drift left behind comes back and a row no session
418
+ * backs goes away, where editing the table in place could do neither.
419
+ *
420
+ * The maintenance triggers live on `sessions` rather than on this table, so
421
+ * dropping and recreating it leaves them in place; a store written before the
422
+ * index existed gets both from `CREATE_TABLES_SQL` at writer open, and the
423
+ * population follows here on the same open's pass.
424
+ */
425
+ private rebuildTitlesFts(): void {
426
+ this.db.run(TITLES_FTS_REBUILD_SQL);
427
+ verifyTitlesFtsRebuild(this.db);
428
+ }
429
+
430
+ /** Whether the store is waiting for its one-shot FTS re-derivation. */
431
+ needsFtsReindex(): boolean {
432
+ return !!this.db
433
+ .query<{ key: string }, []>(
434
+ "SELECT key FROM schema_meta WHERE key = 'fts_reindex' LIMIT 1;",
435
+ )
436
+ .get();
437
+ }
438
+
439
+ /**
440
+ * Run the store's pending FTS re-derivation: the derive-rule counter's
441
+ * one-shot, taken by a pass that holds the writer lease. Returns whether
442
+ * there was work, so the caller can account for the wall time. Both derived
443
+ * indexes are this build's — `records_fts` (with the echo filter's set) and
444
+ * `titles_fts` — because the counter names the *rules*, and one rule version
445
+ * is one derivation.
446
+ *
447
+ * The rebuild is the v11 migration's increment exactly — drop, recreate,
448
+ * re-project from canonical `records` — which is the whole contrast with
449
+ * Wake: its `force_reindex` re-parses every session file
450
+ * (`scanner.rs#L179`, `#L281`, `#L439-L441`), because a file is the only place
451
+ * its units exist. Ledger keeps the canonical records in the store, so the
452
+ * re-derivation is one SQL statement over data that is already local: no
453
+ * source file is opened, no `ingest_cursors` row moves, and the queued or
454
+ * staged rebuild machinery (`ingest_queue`, `ingest_staged_*`, #49 D1's
455
+ * livelock path) is not entered at all.
456
+ *
457
+ * Vectors are untouched by construction, not by care: neither index has a
458
+ * foreign key into `records_embeddings`, and this transaction only ever names
459
+ * `records`, `records_fts`, `records_echo`, `titles_fts` and `sessions`.
460
+ *
461
+ * One transaction, flag clear included. An interrupted pass — a crash, or a
462
+ * failure in either index's own gates — rolls the whole increment back,
463
+ * leaving the previous indexes and the flag exactly as they were, so the next
464
+ * pass retries from a consistent state. Wake clears the flag after its scan
465
+ * (`scanner.rs#L439-L441`), which leaves a window where the flag is clear and
466
+ * the index is not what it names; here the two cannot disagree.
467
+ */
468
+ reindexFts(): boolean {
469
+ if (!this.needsFtsReindex()) return false;
470
+ this.db.transaction(() => {
471
+ this.rebuildRecordsFts();
472
+ this.rebuildTitlesFts();
473
+ this.configureFtsAutomerge();
474
+ this.db.run("DELETE FROM schema_meta WHERE key = 'fts_reindex';");
475
+ })();
476
+ return true;
477
+ }
478
+
479
+ private migrate(fromVersion: number): void {
480
+ const hasColumn = (table: string, name: string) =>
481
+ this.db.query<{ name: string }, []>(`PRAGMA table_info(${table})`).all().some((c) => c.name === name);
482
+ const hadParentage = hasColumn("sessions", "parent_session_id");
483
+ // The two unmerged branches used v4 differently. Shape, not an id prefix,
484
+ // distinguishes PR2's scoped v4 from PR6's native-id v4/v5 databases.
485
+ const needsIdentity = fromVersion < 4 || (fromVersion <= 5 && hadParentage);
486
+ this.db.run("PRAGMA foreign_keys = OFF;");
487
+ try {
488
+ this.db.transaction(() => {
489
+ if (fromVersion < 2) this.db.run(V2_MIGRATION_SQL);
490
+ if (fromVersion < 3) this.db.run(V3_MIGRATION_SQL);
491
+ if (!hadParentage) this.db.run(PARENTAGE_MIGRATION_SQL);
492
+ this.db.run(INGEST_CURSORS_SQL);
493
+ this.db.run(INGEST_DIAGNOSTICS_SQL);
494
+ if (!hasColumn("sessions", "content_hash_version")) {
495
+ this.db.run("ALTER TABLE sessions ADD COLUMN content_hash_version INTEGER NOT NULL DEFAULT 1");
496
+ }
497
+ if (needsIdentity) {
498
+ this.db.run("UPDATE sessions SET parent_session_id = source || ':' || parent_session_id WHERE parent_session_id IS NOT NULL");
499
+ this.db.run("UPDATE ingest_cursors SET session_id = source || ':' || session_id WHERE session_id IS NOT NULL");
500
+ this.db.run(V4_MIGRATION_SQL);
501
+ }
502
+ this.db.run("UPDATE sessions SET parent_session_id = NULL WHERE parent_session_id = id");
503
+ for (const [column, type] of [
504
+ ["file_identity", "TEXT"], ["prefix_hash", "TEXT"],
505
+ ["checkpoint_version", "INTEGER"], ["line_count", "INTEGER"],
506
+ ["tail_offset", "INTEGER"], ["tail_hash", "TEXT"],
507
+ ] as const) {
508
+ if (!hasColumn("ingest_cursors", column)) this.db.run(`ALTER TABLE ingest_cursors ADD COLUMN ${column} ${type}`);
509
+ }
510
+ // v13 renames the coverage status literal root_not_directory ->
511
+ // root_wrong_shape (file-kind roots make the old name a lie). SQLite
512
+ // bakes CHECK constraints into the table DDL, so stores created at v12
513
+ // reject the new literal; source_coverage is disposable per-pass state,
514
+ // so the migration drops and recreates it instead of rewriting rows.
515
+ if (fromVersion < 13) this.db.run("DROP TABLE IF EXISTS source_coverage");
516
+ // v14 reserved `source_discovery`; v15 adds `session_tombstones`. Both
517
+ // arrive from the same idempotent CREATE_TABLES_SQL rather than a
518
+ // numbered diff: a store at 13 (neither table) or 14 (discovery only)
519
+ // picks up whichever table it lacks and lands at 15. An already-v15
520
+ // store (tombstones present, discovery possibly missing) is accepted on
521
+ // the equal-version path, which also runs this DDL. Existing tombstones
522
+ // are never dropped; a table this statement does not know is left alone.
523
+ this.db.run(CREATE_TABLES_SQL);
524
+ if (!hasColumn("ingest_queue", "native_json")) {
525
+ this.db.run("ALTER TABLE ingest_queue ADD COLUMN native_json TEXT");
526
+ }
527
+ // Older releases predate the staged coverage column; the table itself
528
+ // may only exist once CREATE_TABLES_SQL has run.
529
+ if (!hasColumn("ingest_staged_sessions", "diagnostics_json")) {
530
+ this.db.run("ALTER TABLE ingest_staged_sessions ADD COLUMN diagnostics_json TEXT");
531
+ }
532
+ // v16 preserves per-record native lineage in managed conversations.
533
+ // Historical rows remain unknown; a session id or payload is not proof
534
+ // of the native origin of a record in a conversation that switched.
535
+ for (const column of ["native_session_id", "binding_id"]) {
536
+ if (!hasColumn("records", column)) this.db.run(`ALTER TABLE records ADD COLUMN ${column} TEXT`);
537
+ }
538
+ // v11 rebuilds the derived FTS index: drop it and repopulate from
539
+ // `records`, which is the side the read path trusts. The post-copy gates
540
+ // run inside this transaction, so a mismatch leaves the store at v10
541
+ // with its old index intact rather than half-rebuilt.
542
+ if (fromVersion < 11) this.rebuildRecordsFts();
543
+ // v12 adds `source_coverage` and nothing else, so the CREATE_TABLES_SQL
544
+ // above is its whole migration: the table arrives empty, and no row is
545
+ // fabricated for a source no pass has scanned yet.
546
+ this.configureFtsAutomerge();
547
+ if (this.db.query("PRAGMA foreign_key_check").all().length) {
548
+ throw new Error("Migration would leave dangling record or embedding references");
549
+ }
550
+ this.db.run(`PRAGMA user_version = ${SCHEMA_VERSION};`);
551
+ })();
552
+ } finally {
553
+ this.db.run("PRAGMA foreign_keys = ON;");
554
+ }
555
+ // Databases that already carry the parentage schema were backfilled by the
556
+ // migration that introduced it. Re-reading every historical transcript on
557
+ // an unrelated later schema upgrade defeats the one-shot wall boundary.
558
+ if (!hadParentage) this.backfillSessionParentage();
559
+ }
560
+
561
+ hasSession(id: string, contentHash?: string): boolean {
562
+ if (contentHash) {
563
+ const row = this.db
564
+ .query<{ id: string }, [string, string]>(
565
+ "SELECT id FROM sessions WHERE id = ? AND content_hash = ? LIMIT 1",
566
+ )
567
+ .get(id, contentHash);
568
+ return !!row;
569
+ }
570
+ const row = this.db
571
+ .query<{ id: string }, [string]>(
572
+ "SELECT id FROM sessions WHERE id = ? LIMIT 1",
573
+ )
574
+ .get(id);
575
+ return !!row;
576
+ }
577
+
578
+ getSessionByContentHash(contentHash: string): SessionRecord | null {
579
+ return (
580
+ this.db
581
+ .query<SessionRecord, [string]>(
582
+ "SELECT * FROM sessions WHERE content_hash = ? LIMIT 1",
583
+ )
584
+ .get(contentHash) ?? null
585
+ );
586
+ }
587
+
588
+ getSessionBySourcePath(sourcePath: string): SessionRecord | null {
589
+ return (
590
+ this.db
591
+ .query<SessionRecord, [string]>(
592
+ "SELECT * FROM sessions WHERE source_path = ? LIMIT 1",
593
+ )
594
+ .get(sourcePath) ?? null
595
+ );
596
+ }
597
+
598
+ getIngestCursor(
599
+ source: string,
600
+ listingId: string,
601
+ sourcePath: string,
602
+ ): IngestCursor | null {
603
+ return (
604
+ this.db
605
+ .query<IngestCursor, [string, string, string]>(
606
+ `SELECT * FROM ingest_cursors
607
+ WHERE source = ? AND listing_id = ? AND source_path = ? LIMIT 1`,
608
+ )
609
+ .get(source, listingId, sourcePath) ?? null
610
+ );
611
+ }
612
+
613
+ getIngestSession(cursor: IngestCursor): SessionRecord | null {
614
+ return this.getStagedIngestSession(cursor) ?? (cursor.session_id ? this.sessionRowById(cursor.session_id) : null);
615
+ }
616
+
617
+ private getStagedIngestSession(cursor: IngestCursor): SessionRecord | null {
618
+ const row = this.db.query<{ session_json: string }, [string, string, string]>(
619
+ "SELECT session_json FROM ingest_staged_sessions WHERE source = ? AND listing_id = ? AND source_path = ?",
620
+ ).get(cursor.source, cursor.listing_id, cursor.source_path);
621
+ return row ? JSON.parse(row.session_json) as SessionRecord : null;
622
+ }
623
+
624
+ upsertIngestCursor(cursor: IngestCursor): void {
625
+ this.db.run(
626
+ `INSERT INTO ingest_cursors (
627
+ source, listing_id, source_path, session_id, mtime, size,
628
+ bytes_ingested, record_count, status, updated_at, file_identity, prefix_hash,
629
+ checkpoint_version, line_count, tail_offset, tail_hash
630
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
631
+ ON CONFLICT(source, listing_id, source_path) DO UPDATE SET
632
+ session_id = excluded.session_id,
633
+ mtime = excluded.mtime,
634
+ size = excluded.size,
635
+ bytes_ingested = excluded.bytes_ingested,
636
+ record_count = excluded.record_count,
637
+ status = excluded.status,
638
+ updated_at = excluded.updated_at,
639
+ file_identity = excluded.file_identity,
640
+ prefix_hash = excluded.prefix_hash,
641
+ checkpoint_version = excluded.checkpoint_version,
642
+ line_count = excluded.line_count,
643
+ tail_offset = excluded.tail_offset,
644
+ tail_hash = excluded.tail_hash`,
645
+ [
646
+ cursor.source,
647
+ cursor.listing_id,
648
+ cursor.source_path,
649
+ cursor.session_id,
650
+ cursor.mtime,
651
+ cursor.size,
652
+ cursor.bytes_ingested,
653
+ cursor.record_count,
654
+ cursor.status,
655
+ cursor.updated_at,
656
+ cursor.file_identity ?? null,
657
+ cursor.prefix_hash ?? null,
658
+ cursor.checkpoint_version ?? null,
659
+ cursor.line_count ?? null,
660
+ cursor.tail_offset ?? null,
661
+ cursor.tail_hash ?? null,
662
+ ],
663
+ );
664
+ }
665
+
666
+ getDiscoveryCheckpoints(): Record<string, DiscoveryCheckpoint> {
667
+ return Object.fromEntries(this.db.query<DiscoveryCheckpoint, []>(
668
+ "SELECT * FROM source_discovery ORDER BY source",
669
+ ).all().map(row => [row.source, { ...row, degraded: Boolean(row.degraded) }]));
670
+ }
671
+
672
+ /** No checkpoint advances unless every item in that page was admitted. */
673
+ admitDiscovery(result: ScanResult, needsIngest: (item: DiscoveredSession) => boolean, stamp: Date): void {
674
+ this.db.transaction(() => {
675
+ this.recordSourceCoverage(result.outcomes, stamp);
676
+ for (const item of result.sessions) {
677
+ if (needsIngest(item)) this.enqueueIngest(item, stamp.toISOString());
678
+ else this.removeQueuedIngest(item);
679
+ }
680
+ const upsert = this.db.query(`INSERT INTO source_discovery
681
+ (source, root_key, cursor, status, degraded, last_completed_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)
682
+ ON CONFLICT(source) DO UPDATE SET root_key=excluded.root_key, cursor=excluded.cursor,
683
+ status=excluded.status, degraded=excluded.degraded,
684
+ last_completed_at=excluded.last_completed_at, updated_at=excluded.updated_at`);
685
+ for (const row of result.discovery ?? []) {
686
+ upsert.run(row.source, row.root_key, row.cursor, row.status, Number(row.degraded),
687
+ row.last_completed_at, stamp.toISOString());
688
+ }
689
+ }).immediate();
690
+ }
691
+
692
+ getDiscoverySummary(): DiscoverySummary | null { return readDiscoverySummary(this.db); }
693
+
694
+ enqueueIngest(item: Pick<IngestQueueEntry, "source" | "id" | "path" | "updatedAt" | "title" | "sizeBytes" | "native">, now = new Date().toISOString()): void {
695
+ this.db.run(
696
+ `INSERT INTO ingest_queue (
697
+ source, listing_id, source_path, source_updated_at, title, size_bytes, native_json, enqueued_at
698
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
699
+ ON CONFLICT(source, listing_id, source_path) DO UPDATE SET
700
+ source_updated_at = excluded.source_updated_at,
701
+ title = excluded.title,
702
+ size_bytes = excluded.size_bytes,
703
+ native_json = excluded.native_json,
704
+ attempt_count = CASE
705
+ WHEN ingest_queue.source_updated_at IS NOT excluded.source_updated_at
706
+ OR ingest_queue.size_bytes IS NOT excluded.size_bytes
707
+ OR ingest_queue.native_json IS NOT excluded.native_json THEN 0
708
+ ELSE ingest_queue.attempt_count END,
709
+ next_attempt_at = CASE
710
+ WHEN ingest_queue.source_updated_at IS NOT excluded.source_updated_at
711
+ OR ingest_queue.size_bytes IS NOT excluded.size_bytes
712
+ OR ingest_queue.native_json IS NOT excluded.native_json THEN NULL
713
+ ELSE ingest_queue.next_attempt_at END,
714
+ last_attempt_at = CASE
715
+ WHEN ingest_queue.source_updated_at IS NOT excluded.source_updated_at
716
+ OR ingest_queue.size_bytes IS NOT excluded.size_bytes
717
+ OR ingest_queue.native_json IS NOT excluded.native_json THEN NULL
718
+ ELSE ingest_queue.last_attempt_at END,
719
+ last_outcome = CASE
720
+ WHEN ingest_queue.source_updated_at IS NOT excluded.source_updated_at
721
+ OR ingest_queue.size_bytes IS NOT excluded.size_bytes
722
+ OR ingest_queue.native_json IS NOT excluded.native_json THEN NULL
723
+ ELSE ingest_queue.last_outcome END`,
724
+ [item.source, item.id, item.path, item.updatedAt ?? null, item.title ?? null, item.sizeBytes ?? null,
725
+ item.native ? JSON.stringify(item.native) : null, now],
726
+ );
727
+ }
728
+
729
+ getQueuedIngest(limit: number, now = new Date().toISOString()): IngestQueueEntry[] {
730
+ if (!Number.isSafeInteger(limit) || limit < 1) throw new Error("queue limit must be a positive safe integer");
731
+ return this.db.query<IngestQueueEntry & { native_json: string | null }, [string, number]>(
732
+ `SELECT source, listing_id AS id, source_path AS path,
733
+ source_updated_at AS updatedAt, title, size_bytes AS sizeBytes,
734
+ attempt_count, next_attempt_at, last_attempt_at, last_outcome, native_json
735
+ FROM ingest_queue
736
+ WHERE next_attempt_at IS NULL OR next_attempt_at <= ?
737
+ ORDER BY enqueued_at ASC, coalesce(source_updated_at, '') DESC, source, listing_id
738
+ LIMIT ?`,
739
+ ).all(now, limit).map(({ native_json, ...item }) => native_json === null
740
+ ? item : { ...item, native: JSON.parse(native_json) as DiscoveredSession["native"] });
741
+ }
742
+
743
+ getQueuedIngestLanes(
744
+ limit: number,
745
+ freshLimit: number,
746
+ freshCutoff: string,
747
+ now = new Date().toISOString(),
748
+ ): { items: IngestQueueEntry[]; fresh: number; backlog: number } {
749
+ if (!Number.isSafeInteger(limit) || limit < 1) throw new Error("queue limit must be a positive safe integer");
750
+ if (!Number.isSafeInteger(freshLimit) || freshLimit < 0 || freshLimit > limit) {
751
+ throw new Error("fresh queue limit must be between zero and the queue limit");
752
+ }
753
+ const columns = `source, listing_id AS id, source_path AS path,
754
+ source_updated_at AS updatedAt, title, size_bytes AS sizeBytes,
755
+ attempt_count, next_attempt_at, last_attempt_at, last_outcome, native_json`;
756
+ const fresh = this.db.query<IngestQueueEntry & { native_json: string | null }, [string, string, number]>(
757
+ `SELECT ${columns} FROM ingest_queue
758
+ WHERE (next_attempt_at IS NULL OR next_attempt_at <= ?) AND source_updated_at >= ?
759
+ ORDER BY source_updated_at DESC, enqueued_at ASC, source, listing_id
760
+ LIMIT ?`,
761
+ ).all(now, freshCutoff, limit);
762
+ const backlog = this.db.query<IngestQueueEntry & { native_json: string | null }, [string, string, number]>(
763
+ `SELECT ${columns} FROM ingest_queue
764
+ WHERE (next_attempt_at IS NULL OR next_attempt_at <= ?)
765
+ AND (source_updated_at IS NULL OR source_updated_at < ?)
766
+ ORDER BY enqueued_at ASC, coalesce(source_updated_at, '') DESC, source, listing_id
767
+ LIMIT ?`,
768
+ ).all(now, freshCutoff, limit);
769
+ const chosenFresh = fresh.slice(0, freshLimit);
770
+ const chosenBacklog = backlog.slice(0, limit - chosenFresh.length);
771
+ const spillFresh = fresh.slice(chosenFresh.length, chosenFresh.length + limit - chosenFresh.length - chosenBacklog.length);
772
+ const items = [...chosenFresh, ...chosenBacklog, ...spillFresh].map(({ native_json, ...item }) => native_json === null
773
+ ? item : { ...item, native: JSON.parse(native_json) as DiscoveredSession["native"] });
774
+ return { items, fresh: chosenFresh.length + spillFresh.length, backlog: chosenBacklog.length };
775
+ }
776
+
777
+ removeQueuedIngest(item: Pick<DiscoveredSession, "source" | "id" | "path">): void {
778
+ this.db.run(
779
+ "DELETE FROM ingest_queue WHERE source = ? AND listing_id = ? AND source_path = ?",
780
+ [item.source, item.id, item.path],
781
+ );
782
+ }
783
+
784
+ markQueuedIngestProgress(
785
+ item: Pick<DiscoveredSession, "source" | "id" | "path">,
786
+ outcome: string,
787
+ now = new Date().toISOString(),
788
+ ): void {
789
+ this.db.run(
790
+ `UPDATE ingest_queue SET attempt_count = 0, next_attempt_at = NULL,
791
+ last_attempt_at = ?, last_outcome = ?
792
+ WHERE source = ? AND listing_id = ? AND source_path = ?`,
793
+ [now, outcome, item.source, item.id, item.path],
794
+ );
795
+ }
796
+
797
+ markQueuedIngestRetry(
798
+ item: Pick<DiscoveredSession, "source" | "id" | "path">,
799
+ outcome: string,
800
+ nextAttemptAt: string,
801
+ now = new Date().toISOString(),
802
+ ): void {
803
+ this.db.run(
804
+ `UPDATE ingest_queue SET attempt_count = attempt_count + 1, next_attempt_at = ?,
805
+ last_attempt_at = ?, last_outcome = ?
806
+ WHERE source = ? AND listing_id = ? AND source_path = ?`,
807
+ [nextAttemptAt, now, outcome, item.source, item.id, item.path],
808
+ );
809
+ }
810
+
811
+ getIngestQueueSummary(now = new Date().toISOString()): { total: number; eligible: number; retrying: number } {
812
+ return this.db.query<{ total: number; eligible: number; retrying: number }, [string, string]>(
813
+ `SELECT count(*) AS total,
814
+ coalesce(sum(next_attempt_at IS NULL OR next_attempt_at <= ?), 0) AS eligible,
815
+ coalesce(sum(next_attempt_at > ?), 0) AS retrying
816
+ FROM ingest_queue`,
817
+ ).get(now, now)!;
818
+ }
819
+
820
+ getAdaptivePressureState(): AdaptivePressureState {
821
+ const state = this.db.query<AdaptivePressureState, []>(
822
+ "SELECT scale_permille, calm_runs, last_severity FROM ingest_runtime_state WHERE id = 1",
823
+ ).get();
824
+ return state && Number.isSafeInteger(state.scale_permille) && state.scale_permille >= 250 && state.scale_permille <= 1000 &&
825
+ Number.isSafeInteger(state.calm_runs) && state.calm_runs >= 0 &&
826
+ ["unavailable", "calm", "constrained", "critical"].includes(state.last_severity)
827
+ ? state : { ...DEFAULT_PRESSURE_STATE };
828
+ }
829
+
830
+ setAdaptivePressureState(state: AdaptivePressureState, now = new Date().toISOString()): void {
831
+ this.db.run(
832
+ `INSERT INTO ingest_runtime_state (id, scale_permille, calm_runs, last_severity, updated_at)
833
+ VALUES (1, ?, ?, ?, ?)
834
+ ON CONFLICT(id) DO UPDATE SET scale_permille = excluded.scale_permille,
835
+ calm_runs = excluded.calm_runs, last_severity = excluded.last_severity,
836
+ updated_at = excluded.updated_at`,
837
+ [state.scale_permille, state.calm_runs, state.last_severity, now],
838
+ );
839
+ }
840
+
841
+ hasRecord(id: string): boolean {
842
+ const row = this.db
843
+ .query<{ id: string }, [string]>(
844
+ "SELECT id FROM records WHERE id = ? LIMIT 1",
845
+ )
846
+ .get(id);
847
+ return !!row;
848
+ }
849
+
850
+ getSession(id: string): SessionRecord | null {
851
+ return sessionById(this.db, id);
852
+ }
853
+
854
+ getAllSessions(): SessionRecord[] {
855
+ return allSessions(this.db);
856
+ }
857
+
858
+ getLastSessions(options?: {
859
+ limit?: number;
860
+ source?: string;
861
+ }): SessionRecord[] {
862
+ return lastSessions(this.db, options);
863
+ }
864
+
865
+ getChildSessions(parentSessionId: string): SessionRecord[] {
866
+ return this.db
867
+ .query<SessionRecord, [string]>(
868
+ `SELECT * FROM sessions s WHERE s.parent_session_id = ?
869
+ AND NOT EXISTS (SELECT 1 FROM session_tombstones t WHERE t.session_id = s.id)
870
+ ORDER BY started_at ASC, id ASC`,
871
+ )
872
+ .all(parentSessionId);
873
+ }
874
+
875
+ getSessionLineage(sessionId: string): SessionRecord[] {
876
+ const chain: SessionRecord[] = [];
877
+ const seen = new Set<string>();
878
+ let current = this.getSession(sessionId);
879
+ while (current && !seen.has(current.id)) {
880
+ seen.add(current.id);
881
+ chain.push(current);
882
+ if (!current.parent_session_id) break;
883
+ current = this.getSession(current.parent_session_id);
884
+ }
885
+ return chain;
886
+ }
887
+
888
+ updateSessionParentage(sessionId: string, parentage: SessionParentage): boolean {
889
+ const existing = this.getSession(sessionId);
890
+ if (!existing) return false;
891
+ // Fresh source-backed metadata can correct an earlier edge even on a hash hit.
892
+ const next = mergeParentage({ ...parentage }, parentageOf(existing));
893
+ if (next.parent_session_id === sessionId) next.parent_session_id = null;
894
+ if (parentageEquals(parentageOf(existing), next)) return false;
895
+ this.db.run(
896
+ `UPDATE sessions SET
897
+ parent_session_id = ?, parent_tool_use_id = ?,
898
+ agent_nickname = ?, thread_role = ?
899
+ WHERE id = ?`,
900
+ [
901
+ next.parent_session_id,
902
+ next.parent_tool_use_id,
903
+ next.agent_nickname,
904
+ next.thread_role,
905
+ sessionId,
906
+ ],
907
+ );
908
+ return true;
909
+ }
910
+
911
+ backfillSessionParentage(): { scanned: number; updated: number } {
912
+ const sessions = this.getAllSessions();
913
+ let updated = 0;
914
+ for (const session of sessions) {
915
+ const extracted = extractSessionParentage(session.source, session.source_path, {
916
+ sessionId: nativeId(session.source, session.id),
917
+ onDiagnostic: (diagnostic) => console.warn(JSON.stringify({ type: "parentage_warning", ...diagnostic })),
918
+ });
919
+ if (extracted.parent_session_id) extracted.parent_session_id = ledgerId(session.source, extracted.parent_session_id);
920
+ if (extracted.parent_session_id === session.id) extracted.parent_session_id = null;
921
+ if (parentageEquals(extracted, EMPTY_SESSION_PARENTAGE)) continue;
922
+ if (this.updateSessionParentage(session.id, extracted)) updated++;
923
+ }
924
+ return { scanned: sessions.length, updated };
925
+ }
926
+
927
+ getSessionCount(): number {
928
+ return countSessions(this.db);
929
+ }
930
+
931
+ getRecordCount(): number {
932
+ return countRecords(this.db);
933
+ }
934
+
935
+ /** Store-wide diagnostics coverage, ordered by occurrences then code. */
936
+ getDiagnosticsSummary(): DiagnosticsSummary {
937
+ return readDiagnosticsSummary(this.db);
938
+ }
939
+
940
+ /** Diagnostics stored for one ledger session, ordered by code. */
941
+ getSessionDiagnostics(sessionId: string): IngestDiagnosticRecord[] {
942
+ return this.db
943
+ .query<IngestDiagnosticRecord, [string]>(
944
+ `SELECT session_id, source, code, occurrence_count, sample, first_seen_at, last_seen_at
945
+ FROM ingest_diagnostics WHERE session_id = ? ORDER BY code ASC`,
946
+ )
947
+ .all(sessionId);
948
+ }
949
+
950
+ /**
951
+ * Store this pass's verdict for every source it scanned: one row per outcome,
952
+ * replacing whatever the previous pass wrote — including a source that stopped
953
+ * being listable, so a stored success never outlives the scan that saw it.
954
+ * `first_seen_at` is deliberately not rewritten: the row keeps the first time
955
+ * this store saw the source, which is what makes coverage age readable.
956
+ *
957
+ * Called by the lease-holding passes (`sync`, `build`, `watch`) once per scan,
958
+ * with the complete outcome set: the write is the pass's, not the scan's, so a
959
+ * budget-truncated pass still reports what it saw and `scan` persists nothing.
960
+ * One transaction for the whole set, so readers see a pass's rows together.
961
+ */
962
+ recordSourceCoverage(outcomes: readonly ScanOutcome[], now: Date = new Date()): void {
963
+ if (outcomes.length === 0) return;
964
+ const updatedAt = now.toISOString();
965
+ const upsert = this.db.prepare(
966
+ `INSERT INTO source_coverage (source, root, status, discovered, detail, first_seen_at, updated_at)
967
+ VALUES (?, ?, ?, ?, ?, ?, ?)
968
+ ON CONFLICT(source) DO UPDATE SET
969
+ root = excluded.root,
970
+ status = excluded.status,
971
+ discovered = excluded.discovered,
972
+ detail = excluded.detail,
973
+ updated_at = excluded.updated_at`,
974
+ );
975
+ this.db.transaction(() => {
976
+ for (const outcome of outcomes) {
977
+ upsert.run(
978
+ outcome.source,
979
+ outcome.root,
980
+ outcome.status,
981
+ outcome.discovered,
982
+ outcome.detail ?? null,
983
+ updatedAt,
984
+ updatedAt,
985
+ );
986
+ }
987
+ })();
988
+ }
989
+
990
+ /** Stored per-source coverage in source order; empty until a pass scans. */
991
+ /** Compact coverage signal for the sync summary; null pre-coverage table. */
992
+ getSourceCoverageSummary(): SourceCoverageSummary | null {
993
+ const hasTable = !!this.db
994
+ .query<{ name: string }, []>(
995
+ "SELECT name FROM sqlite_master WHERE type='table' AND name='source_coverage' LIMIT 1;",
996
+ )
997
+ .get();
998
+ return hasTable ? readSourceCoverageSummary(this.db) : null;
999
+ }
1000
+
1001
+ getSourceCoverage(): SourceCoverageRow[] {
1002
+ return readSourceCoverage(this.db);
1003
+ }
1004
+
1005
+ getSessionRecords(sessionId: string): CanonicalDbRecord[] {
1006
+ return sessionRecords(this.db, sessionId);
1007
+ }
1008
+
1009
+ /**
1010
+ * Delete one session at the user's explicit request, and record the deletion.
1011
+ *
1012
+ * This method is the whole mechanism's only write, and the only way a
1013
+ * tombstone can come into existence: no ingest path, scan, sync pass, or
1014
+ * absence-of-file condition reaches it. That is the load-bearing half of
1015
+ * "source truncation is not authorization to erase captured evidence"
1016
+ * (`AGENTS.md`) — a vanished or shortened source is evidence about a source,
1017
+ * never an instruction to delete, and the only thing that can produce a
1018
+ * tombstone is a user naming a session.
1019
+ *
1020
+ * `suppress` (the default) leaves the store canonical: not one canonical or
1021
+ * derived row is removed or rewritten. What changes is that every serving path
1022
+ * stops resolving the session (reader.ts) and every admission path refuses to
1023
+ * re-create it (ingest.ts, plus the store-boundary guards). The records stay
1024
+ * because they are the evidence this store captured; a deletion must not be
1025
+ * the one write in the system that destroys what an adapter already
1026
+ * attributed, and the audit row is more useful beside the rows it names than
1027
+ * instead of them. `hard` carries the same deletion through to the content:
1028
+ * records, vectors, echo markers, titles, diagnostics, and any pending staged
1029
+ * replacement go in this same transaction, and only the tombstone is left.
1030
+ *
1031
+ * One transaction either way. `suppress` writes exactly one row, so a crash
1032
+ * leaves it present or absent; `hard` removes the content and writes the row
1033
+ * together, so no reader can catch a store where the records are gone and the
1034
+ * source is still free to re-admit the session — Wake's ordering, whose
1035
+ * `remove_sessions` writes the tombstone in the same transaction as its
1036
+ * deletes (`crates/wake-core/src/db.rs#L537-L571`).
1037
+ *
1038
+ * Repeat calls are idempotent and never rewrite the audit: a second
1039
+ * `suppress`, or a `hard` on an already-`hard` row, reports the existing
1040
+ * deletion and changes nothing. A `hard` on a `suppress` row is the one
1041
+ * escalation — the same deletion carried further, so `deleted_at` and `reason`
1042
+ * keep naming the moment and the account the user first gave.
1043
+ */
1044
+ tombstoneSession(
1045
+ sessionId: string,
1046
+ options: { reason: string; hard?: boolean; now?: Date },
1047
+ ): TombstoneOutcome {
1048
+ const reason = options.reason.trim();
1049
+ if (!reason) {
1050
+ // The audit row is the mechanism's record of why; an unexplained deletion
1051
+ // is a hole in it. The CLI refuses earlier, with a usage error.
1052
+ throw new Error("a tombstone needs a reason: the deletion's account is what the row preserves");
1053
+ }
1054
+ return this.db.transaction((): TombstoneOutcome => {
1055
+ const existing = this.tombstoneRow(sessionId);
1056
+ if (existing) {
1057
+ if (existing.mode === "hard" || options.hard !== true) {
1058
+ return { tombstone: existing, created: false, escalated: false, recordsRemoved: 0 };
1059
+ }
1060
+ this.db.run("UPDATE session_tombstones SET mode = 'hard' WHERE session_id = ?", [sessionId]);
1061
+ this.clearAdmissionState(sessionId, existing.source, existing.native_id);
1062
+ return {
1063
+ tombstone: { ...existing, mode: "hard" },
1064
+ created: false,
1065
+ escalated: true,
1066
+ recordsRemoved: this.removeSessionContent(sessionId),
1067
+ };
1068
+ }
1069
+
1070
+ // A tombstone needs no session row to remain meaningful (a `hard` deletion
1071
+ // removes it), but it does need one to arise: see SessionNotFoundError.
1072
+ const session = this.sessionRowById(sessionId);
1073
+ if (!session) throw new SessionNotFoundError(sessionId);
1074
+ const nativeIdValue = nativeId(session.source, sessionId);
1075
+ const tombstone: TombstoneRecord = {
1076
+ session_id: sessionId,
1077
+ source: session.source,
1078
+ native_id: nativeIdValue,
1079
+ source_path: session.source_path ?? null,
1080
+ mode: options.hard === true ? "hard" : "suppress",
1081
+ reason,
1082
+ record_count: session.record_count,
1083
+ deleted_at: (options.now ?? new Date()).toISOString(),
1084
+ };
1085
+ this.db.run(
1086
+ `INSERT INTO session_tombstones (
1087
+ session_id, source, native_id, source_path, mode, reason, record_count, deleted_at
1088
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
1089
+ [tombstone.session_id, tombstone.source, tombstone.native_id, tombstone.source_path,
1090
+ tombstone.mode, tombstone.reason, tombstone.record_count, tombstone.deleted_at],
1091
+ );
1092
+ this.clearAdmissionState(sessionId, tombstone.source, nativeIdValue);
1093
+ return {
1094
+ tombstone,
1095
+ created: true,
1096
+ escalated: false,
1097
+ recordsRemoved: tombstone.mode === "hard" ? this.removeSessionContent(sessionId) : 0,
1098
+ };
1099
+ }).immediate();
1100
+ }
1101
+
1102
+ /**
1103
+ * The serving axis: a tombstone by the Ledger session id every read verb
1104
+ * resolves sessions with. Null for a session that was never deleted,
1105
+ * including one this store has never held.
1106
+ */
1107
+ getTombstone(sessionId: string): TombstoneRecord | null {
1108
+ return this.tombstoneRow(sessionId);
1109
+ }
1110
+
1111
+ /**
1112
+ * The admission axis: a tombstone a *listing* names before it has been read.
1113
+ *
1114
+ * Two lookups, because a listing id and a session's native id are not the same
1115
+ * string in this store. The first is the direct one — the listing id *is* the
1116
+ * session's native id (hermes and other listing-keyed sources, and any source
1117
+ * that names the session after its file). The second uses evidence the store
1118
+ * already keeps: a cursor that was ingested into a session now tombstoned is
1119
+ * that session's own record of the listing it came from, which is what makes
1120
+ * the axis hold for a source that keys rollouts by filename while the session
1121
+ * inside carries its own id — measured on the production store, 1,724 of 2,681
1122
+ * cursors name a listing id that differs from the session's native part, so the
1123
+ * direct lookup alone would miss most real listings.
1124
+ *
1125
+ * Ledger declines to derive the second axis from the *path* (Wake's
1126
+ * `file_path`): a path is not listing identity here (db.ts's
1127
+ * `getLiveIngestSession`), and blocking on it would refuse every other listing
1128
+ * a shared store keeps in one file. A cursor is listing identity, by
1129
+ * construction — it is keyed by `(source, listing_id, source_path)`.
1130
+ */
1131
+ tombstoneForListing(item: Pick<DiscoveredSession, "source" | "id">): TombstoneRecord | null {
1132
+ const direct = this.db
1133
+ .query<TombstoneRecord, [string, string]>(
1134
+ "SELECT * FROM session_tombstones WHERE source = ? AND native_id = ? LIMIT 1",
1135
+ )
1136
+ .get(item.source, item.id);
1137
+ if (direct) return direct;
1138
+ return (
1139
+ this.db
1140
+ .query<TombstoneRecord, [string, string]>(
1141
+ `SELECT t.* FROM session_tombstones t
1142
+ JOIN ingest_cursors c ON c.session_id = t.session_id
1143
+ WHERE c.source = ? AND c.listing_id = ? LIMIT 1`,
1144
+ )
1145
+ .get(item.source, item.id) ?? null
1146
+ );
1147
+ }
1148
+
1149
+ /** Every tombstone, newest deletion first — the audit readout, not a search. */
1150
+ listTombstones(): TombstoneRecord[] {
1151
+ return this.db
1152
+ .query<TombstoneRecord, []>(
1153
+ "SELECT * FROM session_tombstones ORDER BY deleted_at DESC, session_id ASC",
1154
+ )
1155
+ .all();
1156
+ }
1157
+
1158
+ private tombstoneRow(sessionId: string): TombstoneRecord | null {
1159
+ return (
1160
+ this.db
1161
+ .query<TombstoneRecord, [string]>("SELECT * FROM session_tombstones WHERE session_id = ? LIMIT 1")
1162
+ .get(sessionId) ?? null
1163
+ );
1164
+ }
1165
+
1166
+ /**
1167
+ * The writer's own view of a session row, tombstones included. The read path's
1168
+ * `getSession` hides a tombstoned session, which is exactly what serving
1169
+ * means; the writer instead has to see the row it is about to refuse, replace,
1170
+ * or remove — a hidden row would read as "absent" and send ingest down its
1171
+ * cold-start path against a session that is still there.
1172
+ */
1173
+ private sessionRowById(id: string): SessionRecord | null {
1174
+ return this.db.query<SessionRecord, [string]>("SELECT * FROM sessions WHERE id = ? LIMIT 1").get(id) ?? null;
1175
+ }
1176
+
1177
+ /** Refuse a write that would admit a session the user deleted. */
1178
+ private assertTombstoneFree(sessionId: string): void {
1179
+ if (this.tombstoneRow(sessionId)) throw new TombstonedSessionError(sessionId);
1180
+ }
1181
+
1182
+ /**
1183
+ * Close the admission surfaces a tombstone opens: queued admissions for the
1184
+ * listings that fed this session, and any pending staged replacement of it.
1185
+ *
1186
+ * None of them can be admitted now (the planner's listing check and the
1187
+ * store's guards both refuse), so they are removed rather than left for a
1188
+ * worker to refuse — and a staged replacement holds the session's records,
1189
+ * which a `hard` deletion must not leave behind. The queue is matched both by
1190
+ * the listing id the tombstone's native id names and by the listings its
1191
+ * cursors name, which is the same pair of axes admission uses
1192
+ * (`tombstoneForListing`); a source that holds several listings in one path
1193
+ * keeps the rows of the listings that did not feed this session.
1194
+ */
1195
+ private clearAdmissionState(sessionId: string, source: string, nativeIdValue: string): void {
1196
+ this.db.run("DELETE FROM ingest_staged_sessions WHERE replaced_session_id = ?", [sessionId]);
1197
+ this.db.run(
1198
+ `DELETE FROM ingest_queue
1199
+ WHERE (source = ? AND listing_id = ?)
1200
+ OR listing_id IN (SELECT listing_id FROM ingest_cursors WHERE session_id = ?)`,
1201
+ [source, nativeIdValue, sessionId],
1202
+ );
1203
+ }
1204
+
1205
+ /**
1206
+ * A `hard` deletion: the session's content, in this transaction, children
1207
+ * first. The explicit deletes are not redundant with the foreign keys on
1208
+ * `records` — the same reason `trg_sessions_diagnostics_delete` exists: a
1209
+ * writer that never enabled them must still leave nothing behind. The title
1210
+ * row goes with the session row through the trigger on `sessions`
1211
+ * (`TITLES_FTS_TRIGGERS_SQL`), which is also what keeps the title index's
1212
+ * rebuild gate's parity claim true. `ingest_cursors` is deliberately left
1213
+ * alone: it is the evidence the listing axis reads, and it holds no content.
1214
+ */
1215
+ private removeSessionContent(sessionId: string): number {
1216
+ this.db.run("DELETE FROM records_fts WHERE session_id = ?", [sessionId]);
1217
+ this.db.run("DELETE FROM records_echo WHERE session_id = ?", [sessionId]);
1218
+ this.db.run("DELETE FROM records_embeddings WHERE session_id = ?", [sessionId]);
1219
+ this.db.run("DELETE FROM ingest_diagnostics WHERE session_id = ?", [sessionId]);
1220
+ const removed = this.db.run("DELETE FROM records WHERE session_id = ?", [sessionId]).changes;
1221
+ this.db.run("DELETE FROM sessions WHERE id = ?", [sessionId]);
1222
+ return removed;
1223
+ }
1224
+
1225
+ /**
1226
+ * Save one full session normalization. `diagnostics` follows the same
1227
+ * replacement rule as {@link commitIngest}: when supplied, this session's
1228
+ * stored rows become exactly that aggregate, including the empty case.
1229
+ */
1230
+ saveSession(
1231
+ session: SessionRecord,
1232
+ records: CanonicalDbRecord[],
1233
+ diagnostics?: readonly IngestDiagnosticAggregate[],
1234
+ ): { inserted: boolean; updated: boolean; deduped: boolean } {
1235
+ // One immediate transaction for the whole write: the deletion check, the
1236
+ // dedupe read, and the rows it decides on. A tombstone committed by another
1237
+ // writer between a check outside the transaction and the write inside it
1238
+ // would otherwise land *under* the tombstone — the session's rows rewritten
1239
+ // while the deletion's `record_count` spoke of a generation that no longer
1240
+ // exists. Taking the write lock first is what makes the check the verdict
1241
+ // rather than a guess about a store that may already have changed; a nested
1242
+ // call (commitIngest's own transaction) joins it as a savepoint.
1243
+ return this.db.transaction((): { inserted: boolean; updated: boolean; deduped: boolean } => {
1244
+ this.assertTombstoneFree(session.id);
1245
+ const existing = this.getSession(session.id);
1246
+ if (existing) {
1247
+ if (existing.content_hash === session.content_hash) {
1248
+ // Content deduplication must not freeze a renamed or relocated session.
1249
+ this.db.run(
1250
+ `UPDATE sessions SET source_path = ?, title = ?, cwd = ?, model = ?,
1251
+ started_at = ?, ended_at = ?, updated_at = ? WHERE id = ?`,
1252
+ [session.source_path, session.title, session.cwd, session.model,
1253
+ session.started_at, session.ended_at, session.updated_at, session.id],
1254
+ );
1255
+ this.updateSessionParentage(session.id, parentageOf(session));
1256
+ // The records are unchanged, but a fresh normalization still owns this
1257
+ // session's coverage: a finding that stopped occurring disappears.
1258
+ if (diagnostics) this.writeIngestDiagnostics(session.id, session.source, diagnostics);
1259
+ return { inserted: false, updated: false, deduped: true };
1260
+ }
1261
+ // Content updated -> replace records
1262
+ this.db.run("DELETE FROM records_fts WHERE session_id = ?", [
1263
+ session.id,
1264
+ ]);
1265
+ this.db.run("DELETE FROM records WHERE session_id = ?", [session.id]);
1266
+ this.db.run(
1267
+ `UPDATE sessions SET
1268
+ source = ?, source_path = ?, content_hash = ?, content_hash_version = ?, title = ?, cwd = ?,
1269
+ model = ?, started_at = ?, ended_at = ?, record_count = ?, updated_at = ?,
1270
+ parent_session_id = ?, parent_tool_use_id = ?,
1271
+ agent_nickname = ?, thread_role = ?
1272
+ WHERE id = ?`,
1273
+ [
1274
+ session.source,
1275
+ session.source_path,
1276
+ session.content_hash,
1277
+ session.content_hash_version ?? 1,
1278
+ session.title,
1279
+ session.cwd,
1280
+ session.model,
1281
+ session.started_at,
1282
+ session.ended_at,
1283
+ records.length,
1284
+ session.updated_at,
1285
+ session.parent_session_id ?? null,
1286
+ session.parent_tool_use_id ?? null,
1287
+ session.agent_nickname ?? null,
1288
+ session.thread_role ?? null,
1289
+ session.id,
1290
+ ],
1291
+ );
1292
+ this.insertRecordsInternal(session.id, session.source, records);
1293
+ if (diagnostics) this.writeIngestDiagnostics(session.id, session.source, diagnostics);
1294
+ return { inserted: false, updated: true, deduped: false };
1295
+ }
1296
+
1297
+ // New session
1298
+ this.db.run(
1299
+ `INSERT INTO sessions (
1300
+ id, source, source_path, content_hash, content_hash_version, title, cwd, model,
1301
+ started_at, ended_at, record_count, created_at, updated_at,
1302
+ parent_session_id, parent_tool_use_id, agent_nickname, thread_role
1303
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
1304
+ [
1305
+ session.id,
1306
+ session.source,
1307
+ session.source_path,
1308
+ session.content_hash,
1309
+ session.content_hash_version ?? 1,
1310
+ session.title,
1311
+ session.cwd,
1312
+ session.model,
1313
+ session.started_at,
1314
+ session.ended_at,
1315
+ records.length,
1316
+ session.created_at,
1317
+ session.updated_at,
1318
+ session.parent_session_id ?? null,
1319
+ session.parent_tool_use_id ?? null,
1320
+ session.agent_nickname ?? null,
1321
+ session.thread_role ?? null,
1322
+ ],
1323
+ );
1324
+ this.insertRecordsInternal(session.id, session.source, records);
1325
+ if (diagnostics) this.writeIngestDiagnostics(session.id, session.source, diagnostics);
1326
+ return { inserted: true, updated: false, deduped: false };
1327
+ }).immediate();
1328
+ }
1329
+
1330
+ appendSessionRecords(
1331
+ sessionId: string,
1332
+ records: CanonicalDbRecord[],
1333
+ ): { appended: number; deferred: boolean; historicalRecordsHashed: number; reason?: string } {
1334
+ // The same one-transaction rule as `saveSession`: the deletion check, the
1335
+ // per-record dedupe reads, and the append they authorize are one immediate
1336
+ // transaction, so a tombstone cannot land between the verdict and the write
1337
+ // (a nested call — commitIngest's transaction — joins it as a savepoint).
1338
+ return this.db.transaction(
1339
+ (): { appended: number; deferred: boolean; historicalRecordsHashed: number; reason?: string } => {
1340
+ this.assertTombstoneFree(sessionId);
1341
+ const session = this.getSession(sessionId);
1342
+ if (!session) return { appended: 0, deferred: true, historicalRecordsHashed: 0, reason: "session_missing" };
1343
+
1344
+ const existingHash = this.db.query<{ record_hash: string }, [string]>(
1345
+ "SELECT record_hash FROM records WHERE id = ?",
1346
+ );
1347
+ const fresh: CanonicalDbRecord[] = [];
1348
+ for (const record of records) {
1349
+ const previous = existingHash.get(record.id);
1350
+ if (previous?.record_hash === record.record_hash) continue;
1351
+ // Correcting an already committed record invalidates an append-only hash
1352
+ // state. Preserve the live generation and rebuild it out of process.
1353
+ if (previous) {
1354
+ return { appended: 0, deferred: true, historicalRecordsHashed: 0, reason: "record_replacement" };
1355
+ }
1356
+ fresh.push(record);
1357
+ }
1358
+ if (fresh.length === 0) return { appended: 0, deferred: false, historicalRecordsHashed: 0 };
1359
+ if ((session.content_hash_version ?? 1) !== SESSION_HASH_VERSION) {
1360
+ return { appended: 0, deferred: true, historicalRecordsHashed: 0, reason: "legacy_session_hash" };
1361
+ }
1362
+
1363
+ const now = new Date().toISOString();
1364
+ let endedAt = session.ended_at;
1365
+ for (const record of fresh) {
1366
+ const ts = record.source_timestamp ?? record.record_timestamp;
1367
+ if (ts && (!endedAt || ts > endedAt)) endedAt = ts;
1368
+ }
1369
+ const contentHash = extendSessionHash(session.content_hash, fresh);
1370
+
1371
+ this.insertRecordsInternal(sessionId, session.source, fresh);
1372
+ this.db.run(
1373
+ `UPDATE sessions SET
1374
+ content_hash = ?, content_hash_version = ?, ended_at = ?, record_count = ?, updated_at = ?
1375
+ WHERE id = ?`,
1376
+ [contentHash, SESSION_HASH_VERSION, endedAt, session.record_count + fresh.length, now, sessionId],
1377
+ );
1378
+ return { appended: fresh.length, deferred: false, historicalRecordsHashed: 0 };
1379
+ },
1380
+ ).immediate();
1381
+ }
1382
+
1383
+ /**
1384
+ * Publish one normalization result. `opts.diagnostics` aggregates the library
1385
+ * diagnostics of the records being published: a commit that publishes a whole
1386
+ * session replaces the session's rows with it (an empty aggregate clears
1387
+ * them), while a partial stage merges it into the staged replacement and an
1388
+ * append leaves existing rows alone — coverage always reflects the most
1389
+ * recent whole-file normalization, and appended content is not re-counted.
1390
+ */
1391
+ commitIngest(
1392
+ session: SessionRecord,
1393
+ records: CanonicalDbRecord[],
1394
+ cursor: IngestCursor,
1395
+ append = false,
1396
+ complete = true,
1397
+ opts: { diagnostics?: readonly IngestDiagnosticAggregate[] } = {},
1398
+ ): IngestCommitOutcome {
1399
+ return this.db.transaction(() => {
1400
+ const staged = this.getStagedIngestSession(cursor);
1401
+ const live = !append ? this.getLiveIngestSession(session, cursor) : null;
1402
+ if (staged || (!append && live && (!complete || live.id !== session.id))) {
1403
+ return this.commitStagedIngest(session, records, cursor, append, complete, staged, opts);
1404
+ }
1405
+ const result = append
1406
+ ? { inserted: false, updated: false, deduped: false, ...this.appendSessionRecords(session.id, records) }
1407
+ : { ...this.saveSession(session, records), appended: 0, deferred: false, historicalRecordsHashed: 0 };
1408
+ if (result.deferred) {
1409
+ this.deferIngestRebuild({ ...cursor, session_id: session.id });
1410
+ return { ...result, deferReason: result.reason };
1411
+ }
1412
+ const stored = this.getSession(session.id);
1413
+ this.upsertIngestCursor({ ...cursor, session_id: session.id, record_count: stored?.record_count ?? 0 });
1414
+ // Only a pass that normalized the whole source describes the whole record
1415
+ // set; a bounded prefix pass would claim coverage it never observed.
1416
+ if (opts.diagnostics && !append && complete) {
1417
+ this.writeIngestDiagnostics(session.id, cursor.source, opts.diagnostics);
1418
+ }
1419
+ return result;
1420
+ }).immediate();
1421
+ }
1422
+
1423
+ /**
1424
+ * Replace one session's diagnostics rows with an observed aggregate. Counts
1425
+ * are rewritten, never summed, so re-importing identical content is
1426
+ * idempotent; a code that stops occurring disappears and `first_seen_at`
1427
+ * survives while it keeps occurring. An empty aggregate clears stale rows and
1428
+ * touches nothing else.
1429
+ */
1430
+ private writeIngestDiagnostics(
1431
+ sessionId: string,
1432
+ source: string,
1433
+ diagnostics: readonly IngestDiagnosticAggregate[],
1434
+ ): void {
1435
+ // Nothing observed: one statement clears stale rows, no read and no write.
1436
+ if (!diagnostics.length) {
1437
+ this.db.run("DELETE FROM ingest_diagnostics WHERE session_id = ?", [sessionId]);
1438
+ return;
1439
+ }
1440
+ const now = new Date().toISOString();
1441
+ const firstSeen = new Map(
1442
+ this.db
1443
+ .query<{ code: string; first_seen_at: string }, [string]>(
1444
+ "SELECT code, first_seen_at FROM ingest_diagnostics WHERE session_id = ?",
1445
+ )
1446
+ .all(sessionId)
1447
+ .map((row) => [row.code, row.first_seen_at] as const),
1448
+ );
1449
+ const insert = this.db.prepare(
1450
+ `INSERT INTO ingest_diagnostics (
1451
+ session_id, code, source, occurrence_count, sample, first_seen_at, last_seen_at
1452
+ ) VALUES (?, ?, ?, ?, ?, ?, ?)`,
1453
+ );
1454
+ this.db.transaction(() => {
1455
+ this.db.run("DELETE FROM ingest_diagnostics WHERE session_id = ?", [sessionId]);
1456
+ for (const diagnostic of diagnostics) {
1457
+ insert.run(
1458
+ sessionId,
1459
+ diagnostic.code,
1460
+ source,
1461
+ diagnostic.occurrenceCount,
1462
+ // Public callers can hand over an unbounded message; the stored sample
1463
+ // stays bounded and code-point safe here too.
1464
+ capDiagnosticSample(diagnostic.sample ?? ""),
1465
+ firstSeen.get(diagnostic.code) ?? diagnostic.firstSeenAt ?? now,
1466
+ now,
1467
+ );
1468
+ }
1469
+ })();
1470
+ }
1471
+
1472
+ /** Diagnostics carried by a staged replacement, if any chunk recorded them. */
1473
+ private getStagedDiagnostics(key: [string, string, string]): IngestDiagnosticAggregate[] | undefined {
1474
+ const row = this.db.query<{ diagnostics_json: string | null }, typeof key>(
1475
+ "SELECT diagnostics_json FROM ingest_staged_sessions WHERE source = ? AND listing_id = ? AND source_path = ?",
1476
+ ).get(...key);
1477
+ if (!row?.diagnostics_json) return undefined;
1478
+ let parsed: unknown;
1479
+ try {
1480
+ parsed = JSON.parse(row.diagnostics_json);
1481
+ } catch (cause) {
1482
+ // Fail closed with the stage identity attached: the caller is inside
1483
+ // the ingest transaction, so this aborts the pass rather than
1484
+ // fabricating coverage from a value nobody can vouch for.
1485
+ throw new StagedDiagnosticsParseError(key, cause);
1486
+ }
1487
+ if (!Array.isArray(parsed) || !parsed.every(isDiagnosticAggregate)) {
1488
+ throw new StagedDiagnosticsParseError(key, null);
1489
+ }
1490
+ return parsed as IngestDiagnosticAggregate[];
1491
+ }
1492
+
1493
+ deferIngestRebuild(cursor: IngestCursor): void {
1494
+ this.db.transaction(() => {
1495
+ this.db.run(
1496
+ "DELETE FROM ingest_staged_sessions WHERE source = ? AND listing_id = ? AND source_path = ?",
1497
+ [cursor.source, cursor.listing_id, cursor.source_path],
1498
+ );
1499
+ this.upsertIngestCursor({
1500
+ ...cursor,
1501
+ bytes_ingested: 0,
1502
+ record_count: 0,
1503
+ status: "pending",
1504
+ prefix_hash: null,
1505
+ line_count: 0,
1506
+ tail_offset: null,
1507
+ tail_hash: null,
1508
+ });
1509
+ })();
1510
+ }
1511
+
1512
+ // A source path is not listing identity: shared stores (e.g. hermes) hold many
1513
+ // distinct listings at ONE source_path, so a path match alone must never
1514
+ // authorize deleting another listing's session. Replacement identity is
1515
+ // resolved from evidence: the incoming id, then the cursor that already
1516
+ // claims this exact listing key, and only as a legacy fallback an unclaimed
1517
+ // row that is this listing's own native projection. `cursor` is the incoming
1518
+ // generation's cursor, so its own session_id is the NEW id and is deliberately
1519
+ // not treated as evidence of the prior committed owner.
1520
+ private getLiveIngestSession(session: SessionRecord, cursor: IngestCursor): SessionRecord | null {
1521
+ const direct = this.sessionRowById(session.id);
1522
+ if (direct) return direct;
1523
+ // A stored claim authorizes replacement only while the claimed session is
1524
+ // still this listing's own projection. Multi-file groups (claude-code) let
1525
+ // several listings share one canonical id; a claim resolving to a session
1526
+ // stored at another path names another listing's projection, not a
1527
+ // predecessor of this one.
1528
+ const owner = this.getIngestCursor(cursor.source, cursor.listing_id, cursor.source_path)?.session_id;
1529
+ if (owner && owner !== session.id) {
1530
+ const claimed = this.sessionRowById(owner);
1531
+ if (claimed && claimed.source === cursor.source && claimed.source_path === cursor.source_path) {
1532
+ return claimed;
1533
+ }
1534
+ }
1535
+ // Pre-cursor rows only. An unclaimed path peer counts as this listing's
1536
+ // prior projection only when its id is exactly the one this listing's
1537
+ // native id produces (listing-keyed stores such as hermes/deepagents key
1538
+ // the listing by the native session id). Any other row is absent evidence
1539
+ // and must not authorize deleting another listing, even when it is the
1540
+ // only unclaimed candidate at the path.
1541
+ return this.db.query<SessionRecord, [string, string, string]>(
1542
+ `SELECT s.* FROM sessions s
1543
+ WHERE s.source = ? AND s.source_path = ? AND s.id = ?
1544
+ AND NOT EXISTS (SELECT 1 FROM ingest_cursors c WHERE c.session_id = s.id)
1545
+ LIMIT 1`,
1546
+ ).get(cursor.source, cursor.source_path, ledgerId(cursor.source, cursor.listing_id)) ?? null;
1547
+ }
1548
+
1549
+ private commitStagedIngest(
1550
+ session: SessionRecord,
1551
+ records: CanonicalDbRecord[],
1552
+ cursor: IngestCursor,
1553
+ append: boolean,
1554
+ complete: boolean,
1555
+ previous: SessionRecord | null,
1556
+ opts: { diagnostics?: readonly IngestDiagnosticAggregate[] } = {},
1557
+ ): IngestCommitOutcome {
1558
+ // A staged replacement is still an admission: it writes the session row and
1559
+ // its records when it promotes, so the deletion's verdict is taken here too
1560
+ // — inside the same transaction that would have promoted it.
1561
+ this.assertTombstoneFree(session.id);
1562
+ const key: [string, string, string] = [cursor.source, cursor.listing_id, cursor.source_path];
1563
+ const live = this.getLiveIngestSession(session, cursor);
1564
+ // A changed source generation invalidates only the pending replacement, so
1565
+ // its staged chunks never contribute coverage to the new generation.
1566
+ const carried = append ? this.getStagedDiagnostics(key) : undefined;
1567
+ if (!append) {
1568
+ this.db.run("DELETE FROM ingest_staged_sessions WHERE source = ? AND listing_id = ? AND source_path = ?", key);
1569
+ previous = null;
1570
+ }
1571
+ const fallback = previous ?? live;
1572
+ const staged: SessionRecord = {
1573
+ ...session,
1574
+ created_at: live?.created_at ?? session.created_at,
1575
+ ...mergeParentage(parentageOf(session), fallback ? parentageOf(fallback) : EMPTY_SESSION_PARENTAGE),
1576
+ };
1577
+ if (staged.parent_session_id === staged.id) staged.parent_session_id = null;
1578
+ for (const record of records) {
1579
+ const ts = record.source_timestamp ?? record.record_timestamp;
1580
+ if (ts && (!staged.started_at || ts < staged.started_at)) staged.started_at = ts;
1581
+ if (ts && (!staged.ended_at || ts > staged.ended_at)) staged.ended_at = ts;
1582
+ }
1583
+ this.db.run(
1584
+ `INSERT INTO ingest_staged_sessions (source, listing_id, source_path, replaced_session_id, session_json) VALUES (?, ?, ?, ?, ?)
1585
+ ON CONFLICT(source, listing_id, source_path) DO UPDATE SET session_json = excluded.session_json`,
1586
+ [...key, live?.id ?? session.id, JSON.stringify(staged)],
1587
+ );
1588
+ const insert = this.db.prepare(
1589
+ `INSERT INTO ingest_staged_records (
1590
+ source, listing_id, source_path, id, record_type, source_order_id, component_index,
1591
+ content_hash, record_hash, canonical_json
1592
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
1593
+ ON CONFLICT(source, listing_id, source_path, id) DO UPDATE SET
1594
+ record_type = excluded.record_type, source_order_id = excluded.source_order_id,
1595
+ component_index = excluded.component_index, content_hash = excluded.content_hash,
1596
+ record_hash = excluded.record_hash, canonical_json = excluded.canonical_json
1597
+ WHERE record_hash != excluded.record_hash`,
1598
+ );
1599
+ let appended = 0;
1600
+ for (const record of records) {
1601
+ appended += insert.run(...key, record.id, record.record_type, record.source_order_id,
1602
+ record.component_index, record.content_hash, record.record_hash, JSON.stringify(record)).changes;
1603
+ }
1604
+ staged.record_count = this.db.query<{ count: number }, typeof key>(
1605
+ "SELECT COUNT(*) AS count FROM ingest_staged_records WHERE source = ? AND listing_id = ? AND source_path = ?",
1606
+ ).get(...key)!.count;
1607
+ staged.updated_at = cursor.updated_at;
1608
+
1609
+ let historicalRecordsHashed = 0;
1610
+ if (complete) {
1611
+ let hash = hashSessionRecords(staged.source, staged.id, []);
1612
+ const rows = this.db.query<Pick<CanonicalDbRecord, "id" | "record_type" | "source_order_id" | "component_index" | "content_hash">, typeof key>(
1613
+ `SELECT id, record_type, source_order_id, component_index, content_hash FROM ingest_staged_records
1614
+ WHERE source = ? AND listing_id = ? AND source_path = ? ORDER BY source_order_id, component_index, id`,
1615
+ );
1616
+ for (const record of rows.iterate(...key)) {
1617
+ hash = extendSessionHash(hash, [record]);
1618
+ historicalRecordsHashed++;
1619
+ }
1620
+ staged.content_hash = hash;
1621
+ staged.content_hash_version = SESSION_HASH_VERSION;
1622
+ this.promoteStagedIngest(staged, key);
1623
+ this.db.run("DELETE FROM ingest_staged_sessions WHERE source = ? AND listing_id = ? AND source_path = ?", key);
1624
+ // The promotion publishes a complete replacement. Coverage is the union of
1625
+ // every chunk that produced a stage entry, or the fresh whole-file
1626
+ // aggregate when this pass normalized the source itself; with neither, the
1627
+ // promotion clears the predecessor's stale rows rather than keeping them.
1628
+ const coverage = mergeIngestDiagnosticAggregates(carried, opts.diagnostics);
1629
+ if (coverage) this.writeIngestDiagnostics(staged.id, key[0], coverage);
1630
+ } else {
1631
+ const pending = mergeIngestDiagnosticAggregates(carried, opts.diagnostics);
1632
+ this.db.run(
1633
+ `UPDATE ingest_staged_sessions SET session_json = ?, diagnostics_json = ?
1634
+ WHERE source = ? AND listing_id = ? AND source_path = ?`,
1635
+ [JSON.stringify(staged), pending ? JSON.stringify(pending) : null, ...key],
1636
+ );
1637
+ }
1638
+ // While a replacement is only staged, the cursor keeps claiming the still
1639
+ // live owner: writing the staged id here would transiently unclaim it and
1640
+ // destroy prior-owner evidence if staging were cancelled or restarted.
1641
+ this.upsertIngestCursor({
1642
+ ...cursor,
1643
+ session_id: complete ? staged.id : (live?.id ?? staged.id),
1644
+ record_count: staged.record_count,
1645
+ });
1646
+ return { inserted: complete && !live, updated: complete && !!live, deduped: false, appended,
1647
+ staged: !complete, deferred: false, historicalRecordsHashed };
1648
+ }
1649
+
1650
+ private promoteStagedIngest(session: SessionRecord, key: [string, string, string]): void {
1651
+ const previousId = this.db.query<{ replaced_session_id: string }, typeof key>(
1652
+ "SELECT replaced_session_id FROM ingest_staged_sessions WHERE source = ? AND listing_id = ? AND source_path = ?",
1653
+ ).get(...key)!.replaced_session_id;
1654
+ // Ownership can change after a stage is created, including across restart.
1655
+ // Recheck the current projection and other listing claims in this transaction
1656
+ // before deleting the predecessor captured by the stage.
1657
+ const previous = this.sessionRowById(previousId);
1658
+ // Legacy stages were selected by path and their cursors pointed at the
1659
+ // incoming generation. Require surviving listing evidence, not that old
1660
+ // stage's predecessor field, before deleting a possibly unrelated row.
1661
+ const owner = this.getIngestCursor(...key)?.session_id;
1662
+ const ownsPrevious = owner === previousId || previousId === ledgerId(key[0], key[1]);
1663
+ const otherClaim = this.db.query<{ present: number }, [string, string, string, string]>(
1664
+ `SELECT 1 AS present FROM ingest_cursors WHERE session_id = ?
1665
+ AND NOT (source = ? AND listing_id = ? AND source_path = ?) LIMIT 1`,
1666
+ ).get(previousId, ...key);
1667
+ if (previousId !== session.id && ownsPrevious && previous?.source === key[0] &&
1668
+ previous.source_path === key[2] && !otherClaim) {
1669
+ this.db.run("DELETE FROM records_fts WHERE session_id = ?", [previousId]);
1670
+ this.db.run("DELETE FROM ingest_diagnostics WHERE session_id = ?", [previousId]);
1671
+ this.db.run("DELETE FROM sessions WHERE id = ?", [previousId]);
1672
+ }
1673
+ this.db.run("DELETE FROM records_fts WHERE session_id = ?", [session.id]);
1674
+ this.db.run("DELETE FROM records WHERE session_id = ?", [session.id]);
1675
+ // The rows described records that this promotion replaces; they are stale
1676
+ // until the promotion writes the coverage of the new generation.
1677
+ this.db.run("DELETE FROM ingest_diagnostics WHERE session_id = ?", [session.id]);
1678
+ this.db.run(
1679
+ `INSERT INTO sessions (
1680
+ id, source, source_path, content_hash, content_hash_version, title, cwd, model, started_at, ended_at,
1681
+ record_count, created_at, updated_at, parent_session_id, parent_tool_use_id, agent_nickname, thread_role
1682
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
1683
+ ON CONFLICT(id) DO UPDATE SET
1684
+ source = excluded.source, source_path = excluded.source_path, content_hash = excluded.content_hash,
1685
+ content_hash_version = excluded.content_hash_version,
1686
+ title = excluded.title, cwd = excluded.cwd, model = excluded.model, started_at = excluded.started_at,
1687
+ ended_at = excluded.ended_at, record_count = excluded.record_count, updated_at = excluded.updated_at,
1688
+ parent_session_id = excluded.parent_session_id, parent_tool_use_id = excluded.parent_tool_use_id,
1689
+ agent_nickname = excluded.agent_nickname, thread_role = excluded.thread_role`,
1690
+ [session.id, session.source, session.source_path, session.content_hash, session.content_hash_version ?? 1,
1691
+ session.title, session.cwd,
1692
+ session.model, session.started_at, session.ended_at, session.record_count, session.created_at,
1693
+ session.updated_at, session.parent_session_id ?? null, session.parent_tool_use_id ?? null,
1694
+ session.agent_nickname ?? null, session.thread_role ?? null],
1695
+ );
1696
+ // Iterate the durable replacement; never materialize a session's full payload history.
1697
+ const rows = this.db.query<{ canonical_json: string }, typeof key>(
1698
+ `SELECT canonical_json FROM ingest_staged_records WHERE source = ? AND listing_id = ? AND source_path = ?
1699
+ ORDER BY source_order_id, component_index`,
1700
+ );
1701
+ let batch: CanonicalDbRecord[] = [];
1702
+ for (const row of rows.iterate(...key)) {
1703
+ batch.push(JSON.parse(row.canonical_json) as CanonicalDbRecord);
1704
+ if (batch.length === 256) {
1705
+ this.insertRecordsInternal(session.id, session.source, batch);
1706
+ batch = [];
1707
+ }
1708
+ }
1709
+ if (batch.length) this.insertRecordsInternal(session.id, session.source, batch);
1710
+ }
1711
+
1712
+ private insertRecordsInternal(
1713
+ sessionId: string,
1714
+ sourceType: string,
1715
+ records: CanonicalDbRecord[],
1716
+ ): void {
1717
+ const insertRecordStmt = this.db.prepare(
1718
+ `INSERT OR REPLACE INTO records (
1719
+ id, session_id, source_type, native_session_id, binding_id, record_type, stable_source_record_id,
1720
+ source_identity_kind, source_order_id, component_index, record_timestamp,
1721
+ source_timestamp, content, tool_call_id, tool_name, tool_arguments_json,
1722
+ tool_result_json, tool_result_ok, record_json, content_hash, record_hash
1723
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
1724
+ );
1725
+
1726
+ const insertFtsStmt = this.db.prepare(
1727
+ `INSERT INTO records_fts (
1728
+ record_id, session_id, source_type, record_type, content,
1729
+ tool_name, tool_arguments, tool_result
1730
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
1731
+ );
1732
+
1733
+ const insertEchoStmt = this.db.prepare(
1734
+ "INSERT OR REPLACE INTO records_echo (record_id, session_id) VALUES (?, ?)",
1735
+ );
1736
+
1737
+ // Canonical first, whole batch: the echo filter classifies a tool *result*
1738
+ // through its call, and that call may be a row the batch itself carries, so
1739
+ // the batch has to be readable in `records` before the index is asked.
1740
+ for (const r of records) {
1741
+ insertRecordStmt.run(
1742
+ r.id,
1743
+ sessionId,
1744
+ sourceType === "managed" ? r.source_type : sourceType,
1745
+ r.native_session_id ?? null,
1746
+ r.binding_id ?? null,
1747
+ r.record_type,
1748
+ r.stable_source_record_id,
1749
+ r.source_identity_kind,
1750
+ r.source_order_id,
1751
+ r.component_index,
1752
+ r.record_timestamp,
1753
+ r.source_timestamp,
1754
+ r.content,
1755
+ r.tool_call_id,
1756
+ r.tool_name,
1757
+ r.tool_arguments_json,
1758
+ r.tool_result_json,
1759
+ r.tool_result_ok,
1760
+ r.record_json,
1761
+ r.content_hash,
1762
+ r.record_hash,
1763
+ );
1764
+ }
1765
+
1766
+ // The self-referential echo filter (filter.ts): a lookup of Ledger itself
1767
+ // is canonical data but not searchable material, so it never reaches the
1768
+ // index. Same rule as the re-derivation's — and for a result whose call
1769
+ // landed in an earlier batch, the same store lookup the re-derivation
1770
+ // performs, so the two derivations cannot disagree. The id lands in the
1771
+ // filter's derived set instead, which is what keeps the index and the set
1772
+ // one decision: the status count and the parity gate both read it back.
1773
+ const { excluded, callIds } = echoExcludedBatchIds(this.db, sessionId, records);
1774
+
1775
+ if (callIds.size > 0) {
1776
+ // A result can be committed before its call (the source order is the
1777
+ // source's, and the batch that carried the result had no lookup to pair
1778
+ // it with), which left its index row behind. The call arriving now takes
1779
+ // it back in the same transaction: the committed rows pairing these call
1780
+ // ids leave the index and join the derived set. Rows of this batch are
1781
+ // touched too — the delete matches nothing yet and the insert is
1782
+ // idempotent with the loop below — so arrival order cannot decide what
1783
+ // the pair derives to.
1784
+ const ids = JSON.stringify([...callIds]);
1785
+ const pairedResults = `SELECT r.id FROM records r
1786
+ WHERE r.session_id = ? AND r.record_type = 'tool'
1787
+ AND r.tool_call_id IN (SELECT value FROM json_each(?))`;
1788
+ this.db.run(`DELETE FROM records_fts WHERE record_id IN (${pairedResults})`, [sessionId, ids]);
1789
+ this.db.run(
1790
+ `INSERT OR REPLACE INTO records_echo (record_id, session_id)
1791
+ SELECT r.id, r.session_id FROM records r
1792
+ WHERE r.session_id = ? AND r.record_type = 'tool'
1793
+ AND r.tool_call_id IN (SELECT value FROM json_each(?))`,
1794
+ [sessionId, ids],
1795
+ );
1796
+ }
1797
+
1798
+ for (const r of records) {
1799
+ if (excluded.has(r.id)) {
1800
+ insertEchoStmt.run(r.id, sessionId);
1801
+ continue;
1802
+ }
1803
+
1804
+ // Extract tool arguments / tool result text for FTS indexing
1805
+ const toolArgs = r.tool_arguments_json ?? "";
1806
+ const toolRes = r.tool_result_json ?? (r.record_type === "tool" ? r.content : "");
1807
+
1808
+ insertFtsStmt.run(
1809
+ r.id,
1810
+ sessionId,
1811
+ sourceType === "managed" ? r.source_type : sourceType,
1812
+ r.record_type,
1813
+ r.content ?? "",
1814
+ r.tool_name ?? "",
1815
+ toolArgs,
1816
+ toolRes ?? "",
1817
+ );
1818
+ }
1819
+ }
1820
+
1821
+ query(
1822
+ queryText: string,
1823
+ options?: { limit?: number; source?: string; mode?: "lexical" },
1824
+ ): QueryResult;
1825
+ query(
1826
+ queryText: string,
1827
+ options: { limit?: number; source?: string; mode: "semantic" | "hybrid" },
1828
+ ): Promise<QueryResult>;
1829
+ query(
1830
+ queryText: string,
1831
+ options?: { limit?: number; source?: string; mode?: "lexical" | "semantic" | "hybrid" },
1832
+ ): QueryResult | Promise<QueryResult>;
1833
+ query(
1834
+ queryText: string,
1835
+ options: { limit?: number; source?: string; mode?: "lexical" | "semantic" | "hybrid" } = {},
1836
+ ): QueryResult | Promise<QueryResult> {
1837
+ if (options.mode === "semantic") {
1838
+ return this.querySemantic(queryText, options);
1839
+ }
1840
+ if (options.mode === "hybrid") {
1841
+ return this.queryHybrid(queryText, options);
1842
+ }
1843
+ return this.queryLexical(queryText, options);
1844
+ }
1845
+
1846
+ queryLexical(
1847
+ queryText: string,
1848
+ options: { limit?: number; source?: string } = {},
1849
+ ): QueryResult {
1850
+ return queryLexical(this.db, queryText, options);
1851
+ }
1852
+
1853
+ /**
1854
+ * The writer handle may embed what the store is missing before it asks: a
1855
+ * library caller querying this store expects the vectors it can afford to
1856
+ * compute, while the read path (TrajectoryReader) never writes and reports
1857
+ * `fallback_reason: "embeddings_unavailable"` instead.
1858
+ */
1859
+ private async ensureQueryEmbeddings(): Promise<void> {
1860
+ if (compatibleEmbeddingCount(this.db) === 0 && this.getRecordCount() > 0) {
1861
+ await this.backfillEmbeddings();
1862
+ }
1863
+ }
1864
+
1865
+ async querySemantic(
1866
+ queryText: string,
1867
+ options: { limit?: number; source?: string } = {},
1868
+ ): Promise<QueryResult> {
1869
+ await this.ensureQueryEmbeddings();
1870
+ return querySemantic(this.db, queryText, options);
1871
+ }
1872
+
1873
+ async queryHybrid(
1874
+ queryText: string,
1875
+ options: { limit?: number; source?: string } = {},
1876
+ ): Promise<QueryResult> {
1877
+ await this.ensureQueryEmbeddings();
1878
+ return queryHybrid(this.db, queryText, options);
1879
+ }
1880
+
1881
+ insertRecordEmbeddings(
1882
+ records: Array<{
1883
+ recordId: string;
1884
+ sessionId: string;
1885
+ embedding: Float32Array;
1886
+ createdAt?: string;
1887
+ provider?: string;
1888
+ dim?: number;
1889
+ }>,
1890
+ ): void {
1891
+ if (records.length === 0) return;
1892
+ const now = new Date().toISOString();
1893
+ const insertStmt = this.db.prepare(
1894
+ `INSERT OR REPLACE INTO records_embeddings (record_id, session_id, embedding, provider, dim, created_at)
1895
+ VALUES (?, ?, ?, ?, ?, ?)`,
1896
+ );
1897
+
1898
+ // The vector lane's own admission check, inside the transaction that writes
1899
+ // it: a deleted session gets no new vectors (and, in `suppress` mode, keeps
1900
+ // none of its own either — serving filters them out through their session).
1901
+ // The batch may span sessions, so the refusal names the one that was deleted
1902
+ // rather than dropping the rows silently; the backfill below never asks for a
1903
+ // tombstoned session.
1904
+ this.db.transaction(() => {
1905
+ const sessionIds = [...new Set(records.map((record) => record.sessionId))];
1906
+ const tombstoned = this.db
1907
+ .query<TombstoneRecord, [string]>(
1908
+ `SELECT * FROM session_tombstones WHERE session_id IN (SELECT value FROM json_each(?)) LIMIT 1`,
1909
+ )
1910
+ .get(JSON.stringify(sessionIds));
1911
+ if (tombstoned) throw new TombstonedSessionError(tombstoned.session_id);
1912
+ for (const r of records) {
1913
+ insertStmt.run(
1914
+ r.recordId,
1915
+ r.sessionId,
1916
+ serializeEmbedding(r.embedding),
1917
+ r.provider ?? "all-MiniLM-L6-v2",
1918
+ r.dim ?? 384,
1919
+ r.createdAt ?? now,
1920
+ );
1921
+ }
1922
+ }).immediate();
1923
+ }
1924
+
1925
+ async embedSessionRecords(
1926
+ sessionId: string,
1927
+ records: CanonicalDbRecord[],
1928
+ ): Promise<number> {
1929
+ const toEmbed: Array<{ recordId: string; text: string }> = [];
1930
+ for (const r of records) {
1931
+ const text = getRecordEmbeddingText(r);
1932
+ if (text) {
1933
+ toEmbed.push({ recordId: r.id, text });
1934
+ }
1935
+ }
1936
+
1937
+ if (toEmbed.length === 0) return 0;
1938
+
1939
+ const texts = toEmbed.map((item) => item.text);
1940
+ const vectors = await embed(texts);
1941
+ const provider = getEmbeddingProvider();
1942
+
1943
+ const now = new Date().toISOString();
1944
+ const payload = toEmbed.map((item, i) => ({
1945
+ recordId: item.recordId,
1946
+ sessionId,
1947
+ embedding: vectors[i],
1948
+ createdAt: now,
1949
+ provider: provider.name,
1950
+ dim: provider.dimension,
1951
+ }));
1952
+
1953
+ this.insertRecordEmbeddings(payload);
1954
+ return payload.length;
1955
+ }
1956
+
1957
+ async backfillEmbeddings(
1958
+ options: { limit?: number; batchSize?: number; verbose?: boolean } = {},
1959
+ ): Promise<{ processed: number; embedded: number }> {
1960
+ const provider = getEmbeddingProvider();
1961
+ const batchSize = options.batchSize ?? 32;
1962
+ const limitClause = options.limit ? `LIMIT ${options.limit}` : "";
1963
+
1964
+ interface UnembeddedRow {
1965
+ id: string;
1966
+ session_id: string;
1967
+ record_type: string;
1968
+ content: string | null;
1969
+ tool_name: string | null;
1970
+ tool_arguments_json: string | null;
1971
+ tool_result_json: string | null;
1972
+ }
1973
+
1974
+ const rows = this.db
1975
+ .query<UnembeddedRow, [string, number]>(
1976
+ `SELECT r.id, r.session_id, r.record_type, r.content, r.tool_name, r.tool_arguments_json, r.tool_result_json
1977
+ FROM records r
1978
+ LEFT JOIN records_embeddings e
1979
+ ON e.record_id = r.id AND e.provider = ? AND e.dim = ?
1980
+ WHERE e.record_id IS NULL
1981
+ AND NOT EXISTS (SELECT 1 FROM session_tombstones t WHERE t.session_id = r.session_id)
1982
+ ORDER BY r.rowid ASC
1983
+ ${limitClause}`,
1984
+ )
1985
+ .all(provider.name, provider.dimension);
1986
+
1987
+ if (rows.length === 0) {
1988
+ return { processed: 0, embedded: 0 };
1989
+ }
1990
+
1991
+ let embeddedCount = 0;
1992
+ for (let i = 0; i < rows.length; i += batchSize) {
1993
+ const chunk = rows.slice(i, i + batchSize);
1994
+ const toEmbed: Array<{ recordId: string; sessionId: string; text: string }> = [];
1995
+ for (const r of chunk) {
1996
+ const text = getRecordEmbeddingText(r);
1997
+ if (text) {
1998
+ toEmbed.push({ recordId: r.id, sessionId: r.session_id, text });
1999
+ }
2000
+ }
2001
+
2002
+ if (toEmbed.length > 0) {
2003
+ const texts = toEmbed.map((item) => item.text);
2004
+ const vectors = await provider.embed(texts);
2005
+ const now = new Date().toISOString();
2006
+ const payload = toEmbed.map((item, idx) => ({
2007
+ recordId: item.recordId,
2008
+ sessionId: item.sessionId,
2009
+ embedding: vectors[idx],
2010
+ createdAt: now,
2011
+ provider: provider.name,
2012
+ dim: provider.dimension,
2013
+ }));
2014
+ this.insertRecordEmbeddings(payload);
2015
+ embeddedCount += payload.length;
2016
+ }
2017
+ }
2018
+
2019
+ return { processed: rows.length, embedded: embeddedCount };
2020
+ }
2021
+
2022
+ getEmbeddingCount(): number {
2023
+ return (
2024
+ this.db
2025
+ .query<{ count: number }, []>(
2026
+ "SELECT count(*) AS count FROM records_embeddings;",
2027
+ )
2028
+ .get()?.count ?? 0
2029
+ );
2030
+ }
2031
+
2032
+ /** Stored vectors of any provider; observational and never loads a model. */
2033
+ hasStoredSessionEmbeddings(sessionId: string): boolean {
2034
+ return !!this.db.query<{ present: number }, [string]>(
2035
+ "SELECT EXISTS(SELECT 1 FROM records_embeddings WHERE session_id = ?) AS present",
2036
+ ).get(sessionId)?.present;
2037
+ }
2038
+
2039
+ hasSessionEmbeddings(sessionId: string): boolean {
2040
+ const provider = getEmbeddingProvider();
2041
+ const row = this.db
2042
+ .query<{ count: number }, [string, string, number]>(
2043
+ "SELECT count(*) AS count FROM records_embeddings WHERE session_id = ? AND provider = ? AND dim = ? LIMIT 1;",
2044
+ )
2045
+ .get(sessionId, provider.name, provider.dimension);
2046
+ return (row?.count ?? 0) > 0;
2047
+ }
2048
+
2049
+ close(): void {
2050
+ this.db.close();
2051
+ }
2052
+
2053
+ getSchemaVersion(): number {
2054
+ return (
2055
+ this.db
2056
+ .query<{ user_version: number }, []>("PRAGMA user_version;")
2057
+ .get()?.user_version ?? 0
2058
+ );
2059
+ }
2060
+ }
2061
+
2062
+ function parentageOf(session: SessionRecord): SessionParentage {
2063
+ return {
2064
+ parent_session_id: session.parent_session_id ?? null,
2065
+ parent_tool_use_id: session.parent_tool_use_id ?? null,
2066
+ agent_nickname: session.agent_nickname ?? null,
2067
+ thread_role: session.thread_role ?? null,
2068
+ };
2069
+ }
2070
+
2071
+ /**
2072
+ * The v11 rebuild's post-copy gates, both fatal inside the migration
2073
+ * transaction: a failure aborts the migration and leaves the store at v10 with
2074
+ * its previous index rather than committing a half-rebuilt one.
2075
+ *
2076
+ * The parity count cannot see a table that answers `MATCH` while its internal
2077
+ * structures disagree with the content it holds, so FTS5's own
2078
+ * `integrity-check` follows it. The count is deliberately count-only: the
2079
+ * projection is one row per record the echo filter keeps, so a rowid-set
2080
+ * comparison would test the same fact more expensively — it is the recorded
2081
+ * tighten if the projection ever stops being a whole-table repopulation.
2082
+ *
2083
+ * The echo set is the third term because the projection is no longer one row
2084
+ * per record: `records - excluded` is what the index must hold. A store that
2085
+ * predates the filter's table excludes nothing (the honest reading: no
2086
+ * derivation under these rules has run), and an excluded id that no longer
2087
+ * backs a record would make the count a lie — the foreign key already forbids
2088
+ * one, so the check is what catches a writer that ran without them on. Module
2089
+ * scope rather than a method so the gates can be driven directly against a
2090
+ * doctored store: that same property leaves both branches unreachable through a
2091
+ * store that was merely written, healthy or not.
2092
+ */
2093
+ export function verifyRecordsFtsRebuild(db: Database): void {
2094
+ const parity = db
2095
+ .query<{ records: number; indexed: number }, []>(
2096
+ `SELECT (SELECT COUNT(*) FROM records) AS records,
2097
+ (SELECT COUNT(*) FROM records_fts) AS indexed;`,
2098
+ )
2099
+ .get();
2100
+ const excluded = echoCount(db) ?? 0;
2101
+ if (!parity || parity.records - excluded !== parity.indexed) {
2102
+ throw new Error(
2103
+ `records_fts rebuild parity check failed: records=${parity?.records ?? 0} excluded=${excluded} indexed=${parity?.indexed ?? 0}`,
2104
+ );
2105
+ }
2106
+ const orphans = db
2107
+ .query<{ count: number }, []>(
2108
+ `SELECT COUNT(*) AS count FROM records_echo e
2109
+ WHERE NOT EXISTS (SELECT 1 FROM records r WHERE r.id = e.record_id);`,
2110
+ )
2111
+ .get();
2112
+ if (orphans && orphans.count > 0) {
2113
+ throw new Error(
2114
+ `records_echo integrity check failed: ${orphans.count} excluded ids no longer back a record`,
2115
+ );
2116
+ }
2117
+ try {
2118
+ db.run("INSERT INTO records_fts(records_fts) VALUES('integrity-check');");
2119
+ } catch (error) {
2120
+ throw new Error(
2121
+ `records_fts rebuild integrity check failed: ${
2122
+ error instanceof Error ? error.message : String(error)
2123
+ }`,
2124
+ );
2125
+ }
2126
+ }
2127
+
2128
+ /**
2129
+ * The title index's rebuild gate, fatal inside the same transaction the
2130
+ * records index's gate guards.
2131
+ *
2132
+ * Parity here is exact rather than subtracted: the index holds one row per
2133
+ * session the store has a title for, so the count against
2134
+ * `sessions WHERE title != ''` (the repopulation's own predicate) is the whole
2135
+ * claim. The orphan check is the one thing a virtual table cannot enforce for
2136
+ * itself — no foreign key reaches into `titles_fts` — so a row whose session is
2137
+ * gone is caught here; the maintenance trigger is what keeps a healthy store
2138
+ * from ever producing one. FTS5's `integrity-check` follows for the same reason
2139
+ * it follows `records_fts`'s count: a table can answer `MATCH` while its
2140
+ * internal structures disagree with the content it holds.
2141
+ *
2142
+ * Module scope rather than a method so the gate can be driven directly against
2143
+ * a doctored store: that property leaves its failure branches unreachable
2144
+ * through a store that was merely written, healthy or not.
2145
+ */
2146
+ export function verifyTitlesFtsRebuild(db: Database): void {
2147
+ const parity = db
2148
+ .query<{ indexable: number; indexed: number }, []>(
2149
+ `SELECT (SELECT COUNT(*) FROM sessions WHERE ${INDEXABLE_TITLE_SQL}) AS indexable,
2150
+ (SELECT COUNT(*) FROM titles_fts) AS indexed;`,
2151
+ )
2152
+ .get();
2153
+ if (!parity || parity.indexable !== parity.indexed) {
2154
+ throw new Error(
2155
+ `titles_fts rebuild parity check failed: sessions with a title=${parity?.indexable ?? 0} indexed=${parity?.indexed ?? 0}`,
2156
+ );
2157
+ }
2158
+ const orphans = db
2159
+ .query<{ count: number }, []>(
2160
+ `SELECT COUNT(*) AS count FROM titles_fts t
2161
+ WHERE NOT EXISTS (SELECT 1 FROM sessions s WHERE s.id = t.session_id);`,
2162
+ )
2163
+ .get();
2164
+ if (orphans && orphans.count > 0) {
2165
+ throw new Error(
2166
+ `titles_fts integrity check failed: ${orphans.count} indexed titles no longer back a session`,
2167
+ );
2168
+ }
2169
+ try {
2170
+ db.run("INSERT INTO titles_fts(titles_fts) VALUES('integrity-check');");
2171
+ } catch (error) {
2172
+ throw new Error(
2173
+ `titles_fts rebuild integrity check failed: ${
2174
+ error instanceof Error ? error.message : String(error)
2175
+ }`,
2176
+ );
2177
+ }
2178
+ }
2179
+
2180
+ /**
2181
+ * Summarize stored diagnostics through any handle, including the read-only
2182
+ * connection a status probe uses. Callers that may face an unmigrated store
2183
+ * must confirm the table exists first.
2184
+ */
2185
+ export function readDiagnosticsSummary(db: Database): DiagnosticsSummary {
2186
+ const total = db
2187
+ .query<{ sessions: number; occurrences: number }, []>(
2188
+ `SELECT COUNT(DISTINCT session_id) AS sessions, COALESCE(SUM(occurrence_count), 0) AS occurrences
2189
+ FROM ingest_diagnostics`,
2190
+ )
2191
+ .get()!;
2192
+ const byCode = db
2193
+ .query<DiagnosticsSummary["byCode"][number], []>(
2194
+ `SELECT code, COUNT(DISTINCT session_id) AS sessions, COALESCE(SUM(occurrence_count), 0) AS occurrences
2195
+ FROM ingest_diagnostics GROUP BY code ORDER BY occurrences DESC, code ASC`,
2196
+ )
2197
+ .all();
2198
+ return { totalSessions: total.sessions, totalOccurrences: total.occurrences, byCode };
2199
+ }
2200
+
2201
+ /**
2202
+ * Read stored per-source coverage in source order through any handle, including
2203
+ * the read-only connection a status probe uses. Callers that may face an
2204
+ * unmigrated store must confirm the table exists first.
2205
+ */
2206
+ export function readSourceCoverage(db: Database): SourceCoverageRow[] {
2207
+ return db
2208
+ .query<SourceCoverageRow, []>(
2209
+ `SELECT source, root, status, discovered, detail, first_seen_at, updated_at
2210
+ FROM source_coverage ORDER BY source ASC`,
2211
+ )
2212
+ .all();
2213
+ }
2214
+
2215
+ export interface SourceCoverageSummary {
2216
+ /** Sources whose last scan verdict is anything other than `listable`. */
2217
+ degraded: number;
2218
+ /** Rows present — one per source the recorded passes have scanned. */
2219
+ total: number;
2220
+ /** Oldest `updated_at` across rows: the least-recently-rescanned source. */
2221
+ oldest_updated_at: string;
2222
+ }
2223
+
2224
+ /**
2225
+ * Compact coverage signal for the search/sync robot envelopes: how many
2226
+ * sources are degraded at their last scan, over how many recorded, plus the
2227
+ * staleness floor. Null when no pass has recorded coverage yet (or the store
2228
+ * predates the table), so the envelopes simply omit the field.
2229
+ */
2230
+ export function readSourceCoverageSummary(db: Database): SourceCoverageSummary | null {
2231
+ const row = db
2232
+ .query<{ degraded: number; total: number; oldest_updated_at: string }, []>(
2233
+ `SELECT
2234
+ SUM(CASE WHEN status <> 'listable' THEN 1 ELSE 0 END) AS degraded,
2235
+ COUNT(*) AS total,
2236
+ MIN(updated_at) AS oldest_updated_at
2237
+ FROM source_coverage`,
2238
+ )
2239
+ .get();
2240
+ if (!row || row.total === 0) return null;
2241
+ return { degraded: row.degraded ?? 0, total: row.total, oldest_updated_at: row.oldest_updated_at };
2242
+ }
2243
+
2244
+ /** Read-only and version-tolerant: pre-v14 or never-scanned stores are unknown. */
2245
+ export function readDiscoverySummary(db: Database): DiscoverySummary | null {
2246
+ if (!db.query("SELECT 1 FROM sqlite_master WHERE type='table' AND name='source_discovery'").get()) return null;
2247
+ const row = db.query<DiscoverySummary, []>(`SELECT COUNT(*) AS total_sources,
2248
+ COALESCE(SUM(cursor IS NOT NULL), 0) AS pending_sources,
2249
+ COALESCE(SUM(status = 'unknown' OR degraded = 1), 0) AS unknown_sources,
2250
+ COALESCE(SUM(status = 'exhausted'), 0) AS exhausted_sources,
2251
+ CASE WHEN COUNT(last_completed_at) = COUNT(*) THEN MIN(last_completed_at) ELSE NULL END AS oldest_completed_at
2252
+ FROM source_discovery`).get();
2253
+ return row?.total_sources ? row : null;
2254
+ }