trajectories 0.0.0 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (205) hide show
  1. package/CHANGELOG.md +36 -0
  2. package/LICENSE +201 -0
  3. package/README.md +288 -2
  4. package/dist/cli.js +30210 -0
  5. package/dist/convert.d.ts +31 -0
  6. package/dist/convert.js +431 -0
  7. package/dist/deepagents_checkpoint.py +362 -0
  8. package/dist/index.d.ts +7 -0
  9. package/dist/index.js +8402 -0
  10. package/dist/log-baseline.d.ts +26 -0
  11. package/dist/log-baseline.js +104 -0
  12. package/dist/log-content.d.ts +16 -0
  13. package/dist/log-content.js +80 -0
  14. package/dist/log-io.d.ts +10 -0
  15. package/dist/log-io.js +121 -0
  16. package/dist/log-read.d.ts +24 -0
  17. package/dist/log-read.js +117 -0
  18. package/dist/log-reconcile.d.ts +5 -0
  19. package/dist/log-reconcile.js +38 -0
  20. package/dist/log-records.d.ts +35 -0
  21. package/dist/log-records.js +85 -0
  22. package/dist/log-recovery.d.ts +18 -0
  23. package/dist/log-recovery.js +123 -0
  24. package/dist/log-stage.d.ts +14 -0
  25. package/dist/log-stage.js +138 -0
  26. package/dist/log-state.d.ts +89 -0
  27. package/dist/log-state.js +92 -0
  28. package/dist/log-switch.d.ts +3 -0
  29. package/dist/log-switch.js +98 -0
  30. package/dist/log-types.d.ts +129 -0
  31. package/dist/log-types.js +8 -0
  32. package/dist/log.d.ts +8 -0
  33. package/dist/log.js +62 -0
  34. package/dist/migrate.d.ts +29 -0
  35. package/dist/migrate.js +173 -0
  36. package/dist/normalization-runtime.d.ts +1 -0
  37. package/dist/normalization-runtime.js +3150 -0
  38. package/dist/portable.d.ts +9 -0
  39. package/dist/portable.js +26 -0
  40. package/dist/readers/canonical.d.ts +13 -0
  41. package/dist/readers/canonical.js +54 -0
  42. package/dist/readers/cline.d.ts +10 -0
  43. package/dist/readers/cline.js +107 -0
  44. package/dist/readers/cursor.d.ts +20 -0
  45. package/dist/readers/cursor.js +374 -0
  46. package/dist/readers/deepagents.d.ts +41 -0
  47. package/dist/readers/deepagents.js +163 -0
  48. package/dist/readers/fx.d.ts +11 -0
  49. package/dist/readers/fx.js +573 -0
  50. package/dist/readers/github-copilot.d.ts +8 -0
  51. package/dist/readers/github-copilot.js +216 -0
  52. package/dist/readers/grok-build.d.ts +12 -0
  53. package/dist/readers/grok-build.js +391 -0
  54. package/dist/readers/guards.d.ts +4 -0
  55. package/dist/readers/guards.js +121 -0
  56. package/dist/readers/pi-compaction.d.ts +7 -0
  57. package/dist/readers/pi-compaction.js +112 -0
  58. package/dist/readers/pi-context.d.ts +13 -0
  59. package/dist/readers/pi-context.js +64 -0
  60. package/dist/store.d.ts +25 -0
  61. package/dist/store.js +218 -0
  62. package/dist/types.d.ts +91 -0
  63. package/dist/types.js +1 -0
  64. package/dist/writers/claude-code.d.ts +5 -0
  65. package/dist/writers/claude-code.js +51 -0
  66. package/dist/writers/cline.d.ts +3 -0
  67. package/dist/writers/cline.js +77 -0
  68. package/dist/writers/codex.d.ts +3 -0
  69. package/dist/writers/codex.js +32 -0
  70. package/dist/writers/cursor.d.ts +3 -0
  71. package/dist/writers/cursor.js +197 -0
  72. package/dist/writers/deepagents.d.ts +3 -0
  73. package/dist/writers/deepagents.js +116 -0
  74. package/dist/writers/entropy.d.ts +5 -0
  75. package/dist/writers/entropy.js +33 -0
  76. package/dist/writers/fx.d.ts +3 -0
  77. package/dist/writers/fx.js +135 -0
  78. package/dist/writers/github-copilot.d.ts +3 -0
  79. package/dist/writers/github-copilot.js +71 -0
  80. package/dist/writers/grok-build.d.ts +3 -0
  81. package/dist/writers/grok-build.js +43 -0
  82. package/dist/writers/helpers.d.ts +17 -0
  83. package/dist/writers/helpers.js +75 -0
  84. package/dist/writers/omp.d.ts +3 -0
  85. package/dist/writers/omp.js +7 -0
  86. package/dist/writers/opencode.d.ts +3 -0
  87. package/dist/writers/opencode.js +65 -0
  88. package/dist/writers/pi.d.ts +3 -0
  89. package/dist/writers/pi.js +45 -0
  90. package/docs/cline-native-format.md +43 -0
  91. package/docs/copilot-native-format.md +78 -0
  92. package/docs/cursor-native-format.md +103 -0
  93. package/docs/deepagents-native-format.md +98 -0
  94. package/docs/evidence-api.md +170 -0
  95. package/docs/fx-native-format.md +87 -0
  96. package/docs/harness-parity-bootstrap.md +78 -0
  97. package/docs/harness-support-parity.md +107 -0
  98. package/docs/inspirations.md +103 -0
  99. package/docs/ledger-migration.md +141 -0
  100. package/docs/pi-compaction-context.md +75 -0
  101. package/docs/pi-provider-context.md +76 -0
  102. package/docs/research/2026-10-06-huihua-and-oar.md +176 -0
  103. package/docs/research/ledger/README.md +96 -0
  104. package/docs/research/ledger/confer/2026-09-13-round-10/README.md +61 -0
  105. package/docs/research/ledger/confer/2026-09-13-round-2/cass.md +72 -0
  106. package/docs/research/ledger/confer/2026-09-13-round-3/README.md +70 -0
  107. package/docs/research/ledger/confer/2026-09-13-round-5/README.md +61 -0
  108. package/docs/research/ledger/confer/2026-09-13-round-6/README.md +68 -0
  109. package/docs/research/ledger/confer/2026-09-13-round-7/README.md +64 -0
  110. package/docs/research/ledger/confer/2026-09-13-round-9/README.md +59 -0
  111. package/docs/research/ledger/confer/2026-09-14-wake/wake.md +146 -0
  112. package/docs/research/ledger/docs/inspirations.md +320 -0
  113. package/docs/research/ledger/docs/pr2-pr6-integration.md +53 -0
  114. package/docs/research/ledger/docs/research/2026-09-07-confer-inspirations.md +163 -0
  115. package/docs/research/ledger/docs/research/2026-09-07-letta-continuous-service.md +104 -0
  116. package/docs/research/ledger/docs/research/2026-09-08-claude-history-retrieval.md +117 -0
  117. package/docs/research/ledger/docs/research/2026-09-09-agentsview-native-discovery.md +87 -0
  118. package/docs/research/ledger/docs/research/2026-09-09-daemon-observation.md +231 -0
  119. package/docs/research/ledger/docs/research/2026-09-09-discovery-pagination.md +114 -0
  120. package/docs/research/ledger/docs/research/2026-09-09-harness-transcript-contract.md +145 -0
  121. package/docs/research/ledger/docs/research/2026-09-09-native-history-discovery.md +180 -0
  122. package/docs/research/ledger/docs/research/2026-09-09-native-history-implementation.md +137 -0
  123. package/docs/research/ledger/docs/research/2026-09-09-other-inspirations.md +279 -0
  124. package/docs/research/ledger/docs/research/2026-09-09-trajectory-and-letta-code.md +128 -0
  125. package/docs/research/ledger/docs/research/2026-09-13-diagnostics-coverage.md +116 -0
  126. package/docs/research/ledger/docs/research/2026-09-13-fts-rebuild.md +83 -0
  127. package/docs/research/ledger/docs/research/2026-09-13-read-path-split.md +74 -0
  128. package/docs/research/ledger/docs/research/2026-09-13-runtime-rollout.md +102 -0
  129. package/docs/research/ledger/docs/research/2026-09-13-scan-outcomes.md +80 -0
  130. package/docs/research/ledger/docs/research/2026-09-13-search-quality.md +105 -0
  131. package/docs/research/ledger/docs/research/2026-09-13-source-coverage.md +64 -0
  132. package/docs/research/ledger/docs/research/2026-09-13-sync-wall-budget.md +82 -0
  133. package/docs/research/ledger/docs/research/2026-09-13-typed-read-gate.md +81 -0
  134. package/docs/research/ledger/docs/research/2026-09-13-writer-lease.md +84 -0
  135. package/docs/research/ledger/docs/research/2026-09-16-c1-copy-fallback.md +227 -0
  136. package/docs/research/ledger/docs/research/2026-09-16-c10-kimi.md +414 -0
  137. package/docs/research/ledger/docs/research/2026-09-16-c2-derive-counter.md +216 -0
  138. package/docs/research/ledger/docs/research/2026-09-16-c3-echo-filter.md +305 -0
  139. package/docs/research/ledger/docs/research/2026-09-16-c4-title-index.md +349 -0
  140. package/docs/research/ledger/docs/research/2026-09-16-c5-trigram.md +268 -0
  141. package/docs/research/ledger/docs/research/2026-09-16-c6-recency-prior.md +438 -0
  142. package/docs/research/ledger/docs/research/2026-09-16-c7-tombstones.md +467 -0
  143. package/docs/research/ledger/docs/research/2026-09-16-c8-remote-truth.md +390 -0
  144. package/docs/research/ledger/docs/research/2026-09-16-c9-grok.md +380 -0
  145. package/docs/research/ledger/docs/research/2026-09-16-wake-port-program.md +38 -0
  146. package/docs/research/ledger/docs/research/2026-09-17-cursor-blobstore.md +157 -0
  147. package/docs/research/ledger/manifest.json +540 -0
  148. package/legacy/NOTICE.md +50 -0
  149. package/legacy/bin/ledger-core.ts +7 -0
  150. package/legacy/bin/ledger.ts +7 -0
  151. package/legacy/bin/traj-bridge.ts +7 -0
  152. package/legacy/scripts/import-claude-history.ts +130 -0
  153. package/legacy/scripts/native-history-smoke.ts +52 -0
  154. package/legacy/scripts/observe-daemon.ts +570 -0
  155. package/legacy/scripts/retitle-sessions.md +19 -0
  156. package/legacy/scripts/verify-runtime-rollout.ts +151 -0
  157. package/legacy/src/canonical.ts +108 -0
  158. package/legacy/src/cli.ts +1293 -0
  159. package/legacy/src/db.ts +2254 -0
  160. package/legacy/src/diagnostics.ts +87 -0
  161. package/legacy/src/discovery.ts +56 -0
  162. package/legacy/src/embeddings.ts +382 -0
  163. package/legacy/src/filter.ts +602 -0
  164. package/legacy/src/gemini_embeddings.ts +103 -0
  165. package/legacy/src/identity.ts +9 -0
  166. package/legacy/src/index.ts +10 -0
  167. package/legacy/src/ingest.ts +791 -0
  168. package/legacy/src/managed.ts +178 -0
  169. package/legacy/src/native/common.ts +68 -0
  170. package/legacy/src/native/cursor.ts +647 -0
  171. package/legacy/src/native/grok.ts +540 -0
  172. package/legacy/src/native/kimi.ts +706 -0
  173. package/legacy/src/native/opencode.ts +470 -0
  174. package/legacy/src/native/sqlite-ro.ts +281 -0
  175. package/legacy/src/native.ts +97 -0
  176. package/legacy/src/normalizer.ts +457 -0
  177. package/legacy/src/parentage.ts +562 -0
  178. package/legacy/src/pressure.ts +162 -0
  179. package/legacy/src/private-storage.ts +74 -0
  180. package/legacy/src/reader.ts +1255 -0
  181. package/legacy/src/scanner.ts +591 -0
  182. package/legacy/src/schema-gate.ts +263 -0
  183. package/legacy/src/schema.ts +829 -0
  184. package/legacy/src/semantic-top-k.ts +71 -0
  185. package/legacy/src/session-hash.ts +49 -0
  186. package/legacy/src/store-status.ts +218 -0
  187. package/legacy/src/sync-preparation.ts +45 -0
  188. package/legacy/src/sync-truth.ts +843 -0
  189. package/legacy/src/sync.ts +1233 -0
  190. package/legacy/src/types.ts +436 -0
  191. package/legacy/src/vocab.txt +30522 -0
  192. package/legacy/src/watch.ts +673 -0
  193. package/legacy/src/writer-lease.ts +310 -0
  194. package/ops/README.md +150 -0
  195. package/ops/install-cli.sh +63 -0
  196. package/ops/install-native.sh +35 -0
  197. package/ops/install-release.sh +49 -0
  198. package/ops/linux/install-systemd-user.sh +103 -0
  199. package/ops/macos/install-launchd.sh +72 -0
  200. package/ops/sync-settings.sh +37 -0
  201. package/package.json +21 -5
  202. package/skills/trajectories/SKILL.md +138 -0
  203. package/skills/trajectories/scripts/trajectories_recent.sh +26 -0
  204. package/skills/trajectories/scripts/trajectories_search.sh +26 -0
  205. package/index.js +0 -1
@@ -0,0 +1,1255 @@
1
+ import { closeSync, existsSync, openSync, readSync } from "node:fs";
2
+ import { Database } from "bun:sqlite";
3
+ import { readDiscoverySummary, readSourceCoverageSummary, type SourceCoverageSummary } from "./db.js";
4
+ import type { DiscoverySummary } from "./discovery.js";
5
+ import { SemanticTopK } from "./semantic-top-k.js";
6
+ import {
7
+ cosineSimilarity,
8
+ deserializeEmbedding,
9
+ embed,
10
+ getEmbeddingProvider,
11
+ type EmbeddingProvider,
12
+ } from "./embeddings.js";
13
+ import type { GeminiEmbeddingProvider } from "./gemini_embeddings.js";
14
+ import { SCHEMA_VERSION } from "./schema.js";
15
+ import { classifyReadStore, formatMismatchMessage, incompleteStoreMessage } from "./schema-gate.js";
16
+ import type {
17
+ CanonicalDbRecord,
18
+ QueryResult,
19
+ RecordMatchType,
20
+ RecordSearchHit,
21
+ SearchHit,
22
+ SearchMode,
23
+ SessionRecord,
24
+ TitleSearchHit,
25
+ WideningReport,
26
+ WideningSkipReason,
27
+ } from "./types.js";
28
+
29
+ /**
30
+ * The read path of one store, against any connection handle.
31
+ *
32
+ * Everything here is pure query work: it never initializes, migrates, repairs a
33
+ * trigger, configures FTS, or writes an embedding. `TrajectoryReader` owns the
34
+ * read-only connection lifecycle; `TrajectoryDb` reuses the same functions for
35
+ * the read methods its writer handle already offers, so lexical widening,
36
+ * envelope accounting, match provenance, and fusion cannot drift between the
37
+ * two.
38
+ */
39
+
40
+ // Zero-hit lexical widening. Every gate below runs only after the primary query
41
+ // returned nothing, so widening never touches the cost or the output of a
42
+ // normal search. LEDGER_WIDEN_MAX_RECORDS=0 disables widening outright.
43
+ //
44
+ // The default is the bound schema v11's rebuilt FTS index earns: the production
45
+ // store (156,801 records) sat above the old 10,000-record ceiling, so every
46
+ // zero-hit query there refused widening as `corpus_over_ceiling`. 250,000 admits
47
+ // it with room for growth — measured on a copy of that store, widened queries
48
+ // over ordinary term pairs ran in 19-106 ms after the rebuild where the pre-v11
49
+ // index measured 178-890 ms, with identical hits (both load-dependent; a warmer,
50
+ // loaded run measured 16-40 ms: docs/research/2026-09-13-fts-rebuild.md). The
51
+ // threshold is a record count, not a measured latency, so a store still at v10 —
52
+ // its index has not been rebuilt — widens on the old, slower plan until a writer
53
+ // open migrates it, and a widened query whose terms match a large share of the
54
+ // store stays expensive either way.
55
+ const DEFAULT_WIDEN_MAX_RECORDS = 250_000;
56
+ const MAX_WIDEN_TOKEN_CHARS = 16;
57
+ const MIN_WIDEN_TOKEN_ALNUM = 3;
58
+ const MAX_WIDEN_TOKENS = 16;
59
+ const BOOLEAN_OPERATOR_RE = /\b(AND|OR|NOT|NEAR)\b/i;
60
+ export const FTS_SYNTAX_FALLBACK_REASON = "fts_syntax_error";
61
+ /**
62
+ * Reported when a semantic or hybrid query had no vectors to search: either the
63
+ * store holds no compatible embeddings (a read path never backfills) or the
64
+ * active provider could not embed the query text.
65
+ */
66
+ export const EMBEDDINGS_UNAVAILABLE_REASON = "embeddings_unavailable";
67
+
68
+ /**
69
+ * The recency prior both lexical lanes order by. bm25 is negative and smaller
70
+ * means more relevant, so multiplying it by `1 + 1/(1 + age/H)` makes a newer
71
+ * session's hits "more negative": a session from today is ×2, one 30 days old
72
+ * ×1.5, one a year old ×1.08. Equal-relevance hits go to the newer session; a
73
+ * hit whose text relevance is more than the factor apart still wins on text.
74
+ * Wake multiplies its message lane and its title lane by exactly this
75
+ * (`crates/wake-core/src/db.rs#L1546-L1560`, tag v0.6.5, MIT), keyed on the
76
+ * *session's* timestamp rather than the record's, so the relative order of hits
77
+ * within one session is untouched — which is why Ledger's prior can read
78
+ * `started_at` and leave the lanes' `started_at DESC` tiebreak alone.
79
+ *
80
+ * Arithmetic, `max()` and one core date function only — no math extension — and
81
+ * one term in a lane's ORDER BY: no rescore pass. `unixepoch()` is the one way
82
+ * to an epoch from this store's ISO-8601 timestamps (no numeric time column
83
+ * exists), and it is a core function of the engine these queries run on —
84
+ * Bun's bundled SQLite 3.51.0, the same runtime C5 verified its SQLite feature
85
+ * set against. `COALESCE(…, 0)` puts an absent or unparsable timestamp at the
86
+ * epoch, the factor's neutral end (×1.0), where a NULL would sort such a hit
87
+ * first in both lanes. The clock is the caller's `Date.now()`, bound as the `?`
88
+ * Wake binds (`now_ms()`), evaluated once per query rather than once per
89
+ * matched row. H is spelled as a real (`${RECENCY_DECAY_MS}.0`), running text
90
+ * and all: SQLite divides integers with integer division, and a truncated
91
+ * divisor would silently make the factor ×2 for the whole corpus.
92
+ *
93
+ * `started_at` is the key, not `updated_at`: the latter is the ingest write
94
+ * time (`ingest.ts` stamps `new Date()`), so a backfilled store would read every
95
+ * ancient session as "today" (measured — all 2,613 sessions of the production
96
+ * store are stamped within four days). `ended_at`, the content analog of Wake's
97
+ * `updated_at`, moves the recency bucket of 5 of those 2,613 sessions, and the
98
+ * tiebreak already names the start; one timestamp per lane.
99
+ */
100
+ const RECENCY_DECAY_MS = 2_592_000_000;
101
+ const RECENCY_PRIOR_SQL =
102
+ `(1.0 + 1.0 / (1.0 + max(0, ? - COALESCE(unixepoch(s.started_at) * 1000, 0)) / ${RECENCY_DECAY_MS}.0))`;
103
+
104
+ const COUNT_REASON_LIMIT_BOUNDED = "limit_bounded";
105
+ const COUNT_REASON_EXACT = "all_matches_returned";
106
+ const COUNT_REASON_EMPTY_QUERY = "empty_query";
107
+
108
+ /**
109
+ * The tombstone filter every serving query carries: a session the user deleted
110
+ * is not served, whichever lane would have found it. Reads are where a deletion
111
+ * becomes visible — the store does not remove the rows a `suppress` tombstone
112
+ * covers (db.ts) — so the filter is one predicate applied to the session column
113
+ * of each lane rather than a rewrite of any lane.
114
+ *
115
+ * The predicate is resolved per call, and skipping it for a store that has no
116
+ * `session_tombstones` table is exact rather than defensive: a store without the
117
+ * table is one no build could have written a tombstone into. The table arrives
118
+ * with the migration that introduces the verb (db.ts, `migrate`), and a store
119
+ * written by this build is refused outright by an older build's read gate
120
+ * (`classifyReadStore`, user_version newer than supported), so a tombstoned
121
+ * store is never read unfiltered. Skipping the predicate there is the same
122
+ * answer as applying it to an empty set — and it is what lets this build keep
123
+ * serving a store written before the table existed, the same tolerance the title
124
+ * lane shows for a store whose format-`"3"` derivation has not run yet. The
125
+ * cost is one `sqlite_master` lookup per serving call; `1` keeps a caller that
126
+ * splices the predicate into an existing `AND` chain from having to branch.
127
+ */
128
+ function tombstonePredicate(db: Database, sessionColumn: string): string {
129
+ const present = !!db
130
+ .query<{ name: string }, []>(
131
+ "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'session_tombstones' LIMIT 1;",
132
+ )
133
+ .get();
134
+ return present
135
+ ? `NOT EXISTS (SELECT 1 FROM session_tombstones t WHERE t.session_id = ${sessionColumn})`
136
+ : "1";
137
+ }
138
+
139
+ /** Options shared by every read query. */
140
+ export interface ReadQueryOptions {
141
+ limit?: number;
142
+ source?: string;
143
+ }
144
+
145
+ /**
146
+ * Widening ceiling, read per query so operators and tests can flip it without
147
+ * restarting a long-lived process. Only a plain non-negative integer counts;
148
+ * anything else (empty, negative, `2.9`, `0x10`, `10k`, `1e3`) falls back to
149
+ * the default, and 0 disables widening.
150
+ */
151
+ function readWidenMaxRecords(): number {
152
+ const raw = process.env.LEDGER_WIDEN_MAX_RECORDS?.trim();
153
+ if (raw === undefined || !/^\d+$/.test(raw)) return DEFAULT_WIDEN_MAX_RECORDS;
154
+ return Number.parseInt(raw, 10);
155
+ }
156
+
157
+ function alphanumericCount(token: string): number {
158
+ let count = 0;
159
+ for (const ch of token) {
160
+ if (/[A-Za-z0-9]/.test(ch)) count += 1;
161
+ }
162
+ return count;
163
+ }
164
+
165
+ /**
166
+ * Tokens that may become prefix terms. Over-long tokens are dropped and the
167
+ * query widens with the remainder; tokens holding fewer than three
168
+ * alphanumerics are dropped too, because a quoted zero-token phrase followed by
169
+ * `*` is not a no-op in FTS5 (it can match unrelated records). Duplicates
170
+ * collapse case-insensitively before the cap is applied.
171
+ */
172
+ function widenableTokens(cleanQuery: string): string[] | { skipReason: WideningSkipReason } {
173
+ const sized = cleanQuery.split(/\s+/).filter((t) => t.length <= MAX_WIDEN_TOKEN_CHARS);
174
+ if (sized.length === 0) return { skipReason: "token_too_long" };
175
+ const rich = sized.filter((t) => alphanumericCount(t) >= MIN_WIDEN_TOKEN_ALNUM);
176
+ if (rich.length === 0) return { skipReason: "no_widenable_tokens" };
177
+ const seen = new Set<string>();
178
+ const unique: string[] = [];
179
+ for (const token of rich) {
180
+ const key = token.toLowerCase();
181
+ if (seen.has(key)) continue;
182
+ seen.add(key);
183
+ unique.push(token);
184
+ }
185
+ if (unique.length > MAX_WIDEN_TOKENS) return { skipReason: "query_too_long" };
186
+ return unique;
187
+ }
188
+
189
+ /** Engine output before the envelope is attached. */
190
+ interface RawFtsHits {
191
+ hits: RecordSearchHit[];
192
+ }
193
+
194
+ /**
195
+ * What the store *holds*, tombstones included — deliberately not what it
196
+ * serves. A `suppress` deletion keeps the session's rows (db.ts), so a count
197
+ * that subtracted them would claim the store holds less than it does and hide
198
+ * the very data the audit keeps; the deletion is disclosed beside these numbers
199
+ * instead (`ledger status`: `tombstones` and `tombstoned_records`). The other
200
+ * reader of this count is the widening ceiling, which bounds a query by the
201
+ * corpus it would scan rather than by the rows it may return.
202
+ */
203
+ export function countSessions(db: Database): number {
204
+ const row = db.query<{ count: number }, []>("SELECT COUNT(*) AS count FROM sessions").get();
205
+ return row?.count ?? 0;
206
+ }
207
+
208
+ /**
209
+ * Whether the store holds any record at all. `EXISTS` stops at the first row
210
+ * (an index seek); counting instead would scan the table on every semantic or
211
+ * hybrid query just to answer a yes/no question.
212
+ */
213
+ function hasRecords(db: Database, source?: string): boolean {
214
+ return !!db.query<{ present: number }, (string | number)[]>(
215
+ `SELECT EXISTS(SELECT 1 FROM records r JOIN sessions s ON s.id = r.session_id${source ? " WHERE s.source = ?" : ""}) AS present`,
216
+ ).get(...(source ? [source] : []))?.present;
217
+ }
218
+
219
+ /** Records held, tombstones included: the same holdings rule as `countSessions`. */
220
+ export function countRecords(db: Database): number {
221
+ const row = db.query<{ count: number }, []>("SELECT COUNT(*) AS count FROM records").get();
222
+ return row?.count ?? 0;
223
+ }
224
+
225
+ /**
226
+ * Vectors the active provider can actually score. Embeddings written by another
227
+ * provider or dimension are invisible to a semantic query rather than being
228
+ * compared across incompatible spaces.
229
+ */
230
+ export function compatibleEmbeddingCount(db: Database, source?: string): number {
231
+ const provider = getEmbeddingProvider();
232
+ return db.query<{ count: number }, (string | number)[]>(
233
+ `SELECT COUNT(*) AS count FROM records_embeddings e
234
+ JOIN records r ON r.id = e.record_id
235
+ JOIN sessions s ON s.id = e.session_id
236
+ WHERE e.provider = ? AND e.dim = ?${source ? " AND s.source = ?" : ""}`,
237
+ ).get(provider.name, provider.dimension, ...(source ? [source] : []))?.count ?? 0;
238
+ }
239
+
240
+ function buildFtsExpression(query: string): string {
241
+ // If the user already provided quotes, respect them
242
+ if (query.startsWith('"') && query.endsWith('"')) {
243
+ return query;
244
+ }
245
+ // Check if query has multiple words without boolean operators
246
+ const hasOperators = BOOLEAN_OPERATOR_RE.test(query);
247
+ if (!hasOperators && query.includes(" ")) {
248
+ // Match phrase or conjunct
249
+ const escaped = query.replace(/"/g, '""');
250
+ return `"${escaped}"`;
251
+ }
252
+ return query;
253
+ }
254
+
255
+ /**
256
+ * The escaped-term rewrite of a query whose own expression FTS5 rejected:
257
+ * every whitespace token as a quoted term, so punctuation cannot be read as
258
+ * syntax. Both lanes call this one implementation because they are handed the
259
+ * same expression — a query the fallback can still answer must keep whatever
260
+ * its titles matched, so the rewrite cannot drift between the two.
261
+ */
262
+ function escapedTermExpression(cleanQuery: string): string {
263
+ return cleanQuery
264
+ .split(/\s+/)
265
+ .filter(Boolean)
266
+ .map((t) => `"${t.replace(/"/g, '""')}"`)
267
+ .join(" ");
268
+ }
269
+
270
+ function executeFtsQuery(
271
+ db: Database,
272
+ ftsExpr: string,
273
+ limit: number,
274
+ now: number,
275
+ sourceFilter?: string,
276
+ matchType: RecordMatchType = "lexical",
277
+ ): RawFtsHits {
278
+ const sourceClause = sourceFilter ? "AND s.source = ?" : "";
279
+ // Parameter order follows the SQL text, and the recency prior's `?` sits in
280
+ // the SELECT list — ahead of the MATCH expression, the source filter and the
281
+ // limit.
282
+ const params: (string | number)[] = sourceFilter
283
+ ? [now, ftsExpr, sourceFilter, limit]
284
+ : [now, ftsExpr, limit];
285
+
286
+ const sql = `
287
+ SELECT
288
+ f.record_id,
289
+ f.session_id,
290
+ s.source,
291
+ s.title,
292
+ s.cwd,
293
+ s.started_at,
294
+ s.parent_session_id,
295
+ s.agent_nickname,
296
+ s.thread_role,
297
+ f.record_type,
298
+ f.content,
299
+ f.tool_name,
300
+ f.tool_arguments,
301
+ f.tool_result,
302
+ snippet(records_fts, 4, '«', '»', '...', 20) AS content_snippet,
303
+ snippet(records_fts, 6, '«', '»', '...', 20) AS args_snippet,
304
+ snippet(records_fts, 7, '«', '»', '...', 20) AS res_snippet,
305
+ bm25(records_fts) * ${RECENCY_PRIOR_SQL} AS rank
306
+ FROM records_fts f
307
+ JOIN sessions s ON s.id = f.session_id
308
+ WHERE records_fts MATCH ?
309
+ ${sourceClause}
310
+ AND ${tombstonePredicate(db, "s.id")}
311
+ ORDER BY rank ASC, s.started_at DESC
312
+ LIMIT ?
313
+ `;
314
+
315
+ interface RawHit {
316
+ record_id: string;
317
+ session_id: string;
318
+ source: string;
319
+ title: string | null;
320
+ cwd: string | null;
321
+ started_at: string | null;
322
+ parent_session_id: string | null;
323
+ agent_nickname: string | null;
324
+ thread_role: string | null;
325
+ record_type: string;
326
+ content: string | null;
327
+ tool_name: string | null;
328
+ tool_arguments: string | null;
329
+ tool_result: string | null;
330
+ content_snippet: string;
331
+ args_snippet: string;
332
+ res_snippet: string;
333
+ rank: number;
334
+ }
335
+
336
+ const rows = db.query<RawHit, (string | number)[]>(sql).all(...params);
337
+
338
+ const hits: RecordSearchHit[] = [];
339
+
340
+ for (const r of rows) {
341
+ let snippet = r.content_snippet;
342
+ if (!snippet || !snippet.includes("«")) {
343
+ if (r.args_snippet && r.args_snippet.includes("«")) {
344
+ snippet = r.args_snippet;
345
+ } else if (r.res_snippet && r.res_snippet.includes("«")) {
346
+ snippet = r.res_snippet;
347
+ } else {
348
+ snippet = (r.content ?? r.tool_arguments ?? r.tool_result ?? "").slice(0, 200);
349
+ }
350
+ }
351
+
352
+ hits.push({
353
+ session_id: r.session_id,
354
+ record_id: r.record_id,
355
+ source: r.source,
356
+ title: r.title,
357
+ cwd: r.cwd,
358
+ started_at: r.started_at,
359
+ record_type: r.record_type,
360
+ parent_session_id: r.parent_session_id,
361
+ agent_nickname: r.agent_nickname,
362
+ thread_role: r.thread_role,
363
+ content: r.content,
364
+ tool_name: r.tool_name,
365
+ tool_arguments: r.tool_arguments,
366
+ tool_result: r.tool_result,
367
+ snippet,
368
+ rank: r.rank,
369
+ match_type: matchType,
370
+ });
371
+ }
372
+
373
+ return { hits };
374
+ }
375
+
376
+ /**
377
+ * Prefix-OR widening for a lexical query that returned nothing. Gates: the
378
+ * query must not express phrase intent anywhere, must not use boolean
379
+ * operators or its own wildcard, must leave at least one widenable token, and
380
+ * the corpus must sit under the configured ceiling. Adoption requires
381
+ * strictly more hits than the primary query's zero.
382
+ */
383
+ function widenZeroHitQuery(
384
+ db: Database,
385
+ cleanQuery: string,
386
+ limit: number,
387
+ now: number,
388
+ sourceFilter?: string,
389
+ ): RawFtsHits | { skipReason: WideningSkipReason } {
390
+ const maxRecords = readWidenMaxRecords();
391
+ if (maxRecords === 0) return { skipReason: "disabled" };
392
+ // Any quote, not just a wrapping pair: `check "a b" field` is phrase intent.
393
+ if (cleanQuery.includes('"')) return { skipReason: "quoted_input" };
394
+ if (BOOLEAN_OPERATOR_RE.test(cleanQuery)) return { skipReason: "operators_present" };
395
+ if (cleanQuery.includes("*")) return { skipReason: "wildcard_present" };
396
+
397
+ const tokens = widenableTokens(cleanQuery);
398
+ if (!Array.isArray(tokens)) return tokens;
399
+ // The ceiling counts every record in the store, not only those matching
400
+ // `--source`, so a filtered search cannot dodge the corpus-size guard.
401
+ if (countRecords(db) > maxRecords) return { skipReason: "corpus_over_ceiling" };
402
+
403
+ // Quoted phrase-prefix terms: equivalent to `tok*` for plain words and
404
+ // safe for terms whose punctuation would otherwise be read as FTS5 syntax.
405
+ const widenedExpr = `(${tokens.map((t) => `"${t.replace(/"/g, '""')}"*`).join(" OR ")})`;
406
+ let widened: RawFtsHits;
407
+ try {
408
+ widened = executeFtsQuery(db, widenedExpr, limit, now, sourceFilter, "implicit_wildcard");
409
+ } catch {
410
+ // Defensive: the token gates make an unparsable widening expression
411
+ // unreachable today, and the catch keeps a future change from turning
412
+ // that into a thrown query.
413
+ return { skipReason: "widened_query_failed" };
414
+ }
415
+ return widened.hits.length > 0 ? widened : { skipReason: "no_improvement" };
416
+ }
417
+
418
+ /**
419
+ * The title lane: one hit per session whose title matches, best first. The
420
+ * title is a session's own metadata, so a hit here names the session and no
421
+ * record (see `TitleSearchHit`).
422
+ *
423
+ * Ordered by relevance under the same recency prior the record lane applies —
424
+ * `bm25(titles_fts) × (1 + 1/(1 + age/H))`, then `s.started_at DESC` — which is
425
+ * Wake's ordering for the same lane (`crates/wake-core/src/db.rs#L1676-L1700`)
426
+ * and the same term the record lane's `rank` carries.
427
+ */
428
+ function titlePrepass(
429
+ db: Database,
430
+ ftsExpr: string,
431
+ limit: number,
432
+ now: number,
433
+ sourceFilter?: string,
434
+ ): TitleSearchHit[] {
435
+ const sourceClause = sourceFilter ? "AND s.source = ?" : "";
436
+ const params: (string | number)[] = sourceFilter
437
+ ? [now, ftsExpr, sourceFilter, limit]
438
+ : [now, ftsExpr, limit];
439
+
440
+ const sql = `
441
+ SELECT
442
+ f.session_id,
443
+ s.source,
444
+ s.title,
445
+ s.cwd,
446
+ s.started_at,
447
+ s.parent_session_id,
448
+ s.agent_nickname,
449
+ s.thread_role,
450
+ highlight(titles_fts, 1, '«', '»') AS title_snippet,
451
+ bm25(titles_fts) * ${RECENCY_PRIOR_SQL} AS rank
452
+ FROM titles_fts f
453
+ JOIN sessions s ON s.id = f.session_id
454
+ WHERE titles_fts MATCH ?
455
+ ${sourceClause}
456
+ AND ${tombstonePredicate(db, "s.id")}
457
+ ORDER BY rank ASC, s.started_at DESC
458
+ LIMIT ?
459
+ `;
460
+
461
+ interface TitleRow {
462
+ session_id: string;
463
+ source: string;
464
+ title: string;
465
+ cwd: string | null;
466
+ started_at: string | null;
467
+ parent_session_id: string | null;
468
+ agent_nickname: string | null;
469
+ thread_role: string | null;
470
+ title_snippet: string;
471
+ rank: number;
472
+ }
473
+
474
+ // The whole title is the snippet — it is short by construction (the
475
+ // normalizer cuts a first-user-message title at 120 chars) and truncating it
476
+ // to a window around the match would hide the session's own name from the
477
+ // answer that matched it.
478
+ return db
479
+ .query<TitleRow, (string | number)[]>(sql)
480
+ .all(...params)
481
+ .map((row) => ({
482
+ session_id: row.session_id,
483
+ record_id: null,
484
+ source: row.source,
485
+ title: row.title,
486
+ cwd: row.cwd,
487
+ started_at: row.started_at,
488
+ record_type: "title",
489
+ content: null,
490
+ tool_name: null,
491
+ tool_arguments: null,
492
+ tool_result: null,
493
+ snippet: row.title_snippet.slice(0, 240),
494
+ rank: row.rank,
495
+ parent_session_id: row.parent_session_id,
496
+ agent_nickname: row.agent_nickname,
497
+ thread_role: row.thread_role,
498
+ match_type: "title",
499
+ }));
500
+ }
501
+
502
+ function buildQueryResult(
503
+ query: string,
504
+ hits: SearchHit[],
505
+ mode: SearchMode,
506
+ limit: number,
507
+ startedAt: number,
508
+ extras: {
509
+ fallbackReason?: string | null;
510
+ widened?: WideningReport | null;
511
+ countReason?: string;
512
+ /** The mode the caller asked for when `mode` is the engine that answered. */
513
+ requestedMode?: SearchMode;
514
+ } = {},
515
+ ): QueryResult {
516
+ const bounded = hits.length >= limit;
517
+ const sessions = new Set<string>();
518
+ for (const hit of hits) sessions.add(hit.session_id);
519
+ return {
520
+ query,
521
+ totalHits: hits.length,
522
+ sessionsCount: sessions.size,
523
+ hits,
524
+ meta: {
525
+ count_precision: bounded ? "lower_bound" : "exact",
526
+ count_reason: extras.countReason ?? (bounded ? COUNT_REASON_LIMIT_BOUNDED : COUNT_REASON_EXACT),
527
+ elapsed_ms: Math.round((performance.now() - startedAt) * 1000) / 1000,
528
+ search_mode: mode,
529
+ // Equal except when an engine fell back, which is exactly when an agent
530
+ // must not have to infer which one answered the query.
531
+ requested_search_mode: extras.requestedMode ?? mode,
532
+ fallback_reason: extras.fallbackReason ?? null,
533
+ widened: extras.widened ?? null,
534
+ },
535
+ };
536
+ }
537
+
538
+ export function queryLexical(
539
+ db: Database,
540
+ queryText: string,
541
+ options: ReadQueryOptions = {},
542
+ ): QueryResult {
543
+ const startedAt = performance.now();
544
+ const limit = options.limit ?? 50;
545
+ const cleanQuery = queryText.trim();
546
+ if (!cleanQuery) {
547
+ return buildQueryResult(queryText, [], "lexical", limit, startedAt, {
548
+ countReason: COUNT_REASON_EMPTY_QUERY,
549
+ });
550
+ }
551
+
552
+ const ftsExpr = buildFtsExpression(cleanQuery);
553
+
554
+ // One clock for both lanes: the recency prior weights a title hit and a
555
+ // record hit of the same session identically, and a query cannot read two
556
+ // different "now"s (Wake's `now_ms()`, `db.rs#L1553`).
557
+ const now = Date.now();
558
+
559
+ // The title lane runs first and its hits are emitted ahead of the record
560
+ // lane's: a word in a session's title is the strongest relevance signal a
561
+ // session-level search has, and the session's *name* is what a human is
562
+ // usually quoting (Wake orders the same way, db.rs#L1676-L1700). It costs
563
+ // nothing when the store has no title index — a store whose format-`"3"`
564
+ // derivation has not run yet — and a title that cannot be read must not turn
565
+ // a query the record lane can still answer into a thrown one.
566
+ const titlesIndexed = !!db
567
+ .query<{ name: string }, []>(
568
+ "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'titles_fts' LIMIT 1;",
569
+ )
570
+ .get();
571
+ const titleLane = (expr: string): TitleSearchHit[] => {
572
+ if (!titlesIndexed) return [];
573
+ try {
574
+ return titlePrepass(db, expr, limit, now, options.source);
575
+ } catch {
576
+ // The record lane decides whether this expression parses; whatever its
577
+ // answer is, the title lane gets no vote.
578
+ return [];
579
+ }
580
+ };
581
+
582
+ let titles = titleLane(ftsExpr);
583
+
584
+ // The result as a whole is bounded by the limit, so the title hits come out
585
+ // of the record lane's budget rather than on top of it: `totalHits` has to
586
+ // stay what `meta.count_precision` describes.
587
+ let raw: RawFtsHits;
588
+ let fallbackReason: string | null = null;
589
+ if (titles.length >= limit) {
590
+ raw = { hits: [] };
591
+ } else {
592
+ try {
593
+ raw = executeFtsQuery(db, ftsExpr, limit - titles.length, now, options.source);
594
+ } catch {
595
+ // Fallback: search with escaped individual terms — and re-ask the title
596
+ // lane with them, so both lanes answer one query rather than two.
597
+ fallbackReason = FTS_SYNTAX_FALLBACK_REASON;
598
+ const fallbackExpr = escapedTermExpression(cleanQuery);
599
+ titles = titleLane(fallbackExpr);
600
+ raw = executeFtsQuery(db, fallbackExpr, limit - titles.length, now, options.source);
601
+ }
602
+ }
603
+
604
+ let hits: SearchHit[] = [...titles, ...raw.hits];
605
+
606
+ // Widening is a zero-hit recovery of the *query*, so it runs only when both
607
+ // lanes found nothing: it neither costs nor changes anything on the happy
608
+ // path, which is why the corpus-size probe only runs below.
609
+ let widened: WideningReport | null = null;
610
+ if (hits.length === 0) {
611
+ const attempt = widenZeroHitQuery(db, cleanQuery, limit, now, options.source);
612
+ if ("hits" in attempt) {
613
+ hits = attempt.hits;
614
+ widened = { applied: true, original_hits: 0, widened_hits: attempt.hits.length };
615
+ } else {
616
+ widened = { applied: false, skip_reason: attempt.skipReason };
617
+ }
618
+ }
619
+
620
+ return buildQueryResult(cleanQuery, hits, "lexical", limit, startedAt, {
621
+ fallbackReason,
622
+ widened,
623
+ });
624
+ }
625
+
626
+ /**
627
+ * Rewrite a lexical result as the honest answer of a query whose vector leg
628
+ * could not run: the hits come from the lexical engine, so the envelope says
629
+ * that, names the mode that was requested, and reports the reason. The lexical
630
+ * answer includes its title lane — the fallback *is* that engine, so its
631
+ * session-level hits stay in it.
632
+ */
633
+ function lexicalFallback(
634
+ result: QueryResult,
635
+ limit: number,
636
+ requested: SearchMode,
637
+ startedAt: number,
638
+ ): QueryResult {
639
+ const hits = result.hits.slice(0, limit);
640
+ return buildQueryResult(result.query, hits, "lexical", limit, startedAt, {
641
+ requestedMode: requested,
642
+ fallbackReason: EMBEDDINGS_UNAVAILABLE_REASON,
643
+ widened: result.meta.widened,
644
+ });
645
+ }
646
+
647
+ /**
648
+ * Embed one query string with the given provider. Gemini needs its query-mode
649
+ * formatting, which is a per-call concern of this path rather than a rewritten
650
+ * provider: the flag is put back even when embedding fails.
651
+ */
652
+ async function embedQueryVector(
653
+ provider: EmbeddingProvider,
654
+ text: string,
655
+ ): Promise<Float32Array | undefined> {
656
+ const isGemini = "queryMode" in provider;
657
+ if (isGemini) {
658
+ (provider as GeminiEmbeddingProvider).queryMode = true;
659
+ }
660
+ try {
661
+ const [vector] = await embed([text]);
662
+ return vector;
663
+ } finally {
664
+ if (isGemini) {
665
+ (provider as GeminiEmbeddingProvider).queryMode = false;
666
+ }
667
+ }
668
+ }
669
+
670
+ /** The vector leg's answer before the envelope is attached. */
671
+ interface RawVectorHits {
672
+ hits: RecordSearchHit[];
673
+ }
674
+
675
+ /**
676
+ * Vector search over the embeddings already in the store, or null when the leg
677
+ * cannot run. A read path never backfills: with no compatible vectors, or a
678
+ * provider that cannot embed the query (no key, no model files, a failing
679
+ * endpoint), the caller answers lexically and says so instead of returning an
680
+ * empty result that looks like a miss.
681
+ */
682
+ async function vectorSearch(
683
+ db: Database,
684
+ cleanQuery: string,
685
+ options: { limit: number; source?: string },
686
+ ): Promise<RawVectorHits | null> {
687
+ const { limit } = options;
688
+ const provider = getEmbeddingProvider();
689
+ if (compatibleEmbeddingCount(db, options.source) === 0) return null;
690
+
691
+ let queryVec: Float32Array | undefined;
692
+ try {
693
+ queryVec = await embedQueryVector(provider, cleanQuery);
694
+ } catch {
695
+ return null;
696
+ }
697
+ // A provider that hands back a vector of a different length than it
698
+ // advertises cannot score the stored vectors: every comparison would read as
699
+ // zero similarity, which is a silent miss rather than an answer.
700
+ if (!queryVec || queryVec.length !== provider.dimension) return null;
701
+
702
+ const conditions: string[] = ["e.provider = ?", "e.dim = ?"];
703
+ const params: unknown[] = [provider.name, provider.dimension];
704
+ if (options.source) {
705
+ conditions.push("s.source = ?");
706
+ params.push(options.source);
707
+ }
708
+ // The vector lane carries the same tombstone filter as the lexical lanes: a
709
+ // retained vector of a deleted session is not an answer (and a store whose
710
+ // vectors are all tombstoned answers with no hits in this mode rather than
711
+ // falling back, so the envelope keeps naming the engine that ran).
712
+ conditions.push(tombstonePredicate(db, "s.id"));
713
+ const sourceClause = `WHERE ${conditions.join(" AND ")}`;
714
+
715
+ interface EmbeddingRow {
716
+ record_id: string;
717
+ session_id: string;
718
+ source: string;
719
+ title: string | null;
720
+ cwd: string | null;
721
+ started_at: string | null;
722
+ parent_session_id: string | null;
723
+ agent_nickname: string | null;
724
+ thread_role: string | null;
725
+ record_type: string;
726
+ content: string | null;
727
+ tool_name: string | null;
728
+ tool_arguments_json: string | null;
729
+ tool_result_json: string | null;
730
+ embedding: Uint8Array;
731
+ }
732
+
733
+ const sql = `
734
+ SELECT
735
+ e.record_id,
736
+ e.session_id,
737
+ s.source,
738
+ s.title,
739
+ s.cwd,
740
+ s.started_at,
741
+ s.parent_session_id,
742
+ s.agent_nickname,
743
+ s.thread_role,
744
+ r.record_type,
745
+ r.content,
746
+ r.tool_name,
747
+ r.tool_arguments_json,
748
+ r.tool_result_json,
749
+ e.embedding
750
+ FROM records_embeddings e
751
+ JOIN sessions s ON s.id = e.session_id
752
+ JOIN records r ON r.id = e.record_id
753
+ ${sourceClause}
754
+ `;
755
+
756
+ // Pure cosine, deliberately: the recency prior is the lexical lanes' (a
757
+ // multiplier on bm25, above). Cosine is a bounded, dense similarity whose
758
+ // whole top-50 span measures 1.8–2.6× on this store's own text (shipped
759
+ // MiniLM, 300 real records: r1/r50 = 2.19/1.77/2.60), so the same ×1–×2
760
+ // factor would order semantic hits by age nearly regardless of their text;
761
+ // and a fused hit already takes its recency from the lexical leg's position,
762
+ // which is what RRF reads. The lane's reported `rank` stays a cosine.
763
+ //
764
+ // A deferred snapshot keeps scoring and winner hydration coherent while a
765
+ // writer commits. Query embedding above awaits outside this transaction.
766
+ // Exact top-K distinct sessions: one hit per session (its best-scoring
767
+ // record). Without this, a single long session crowds out the whole top-N
768
+ // with near-identical records. Space is O(K); scoring remains exhaustive.
769
+ const topHits = db.transaction(() => {
770
+ const top = new SemanticTopK(limit);
771
+ let ordinal = 0;
772
+ const rows = db.query<{ record_id: string; session_id: string; embedding: Uint8Array }, (string | number)[]>(`
773
+ SELECT e.record_id, e.session_id, e.embedding
774
+ FROM records_embeddings e
775
+ JOIN sessions s ON s.id = e.session_id
776
+ JOIN records r ON r.id = e.record_id
777
+ ${sourceClause}
778
+ `).iterate(...(params as (string | number)[]));
779
+ for (const row of rows) {
780
+ const vector = deserializeEmbedding(row.embedding);
781
+ if (vector.length !== provider.dimension) continue;
782
+ top.offer({ record_id: row.record_id, session_id: row.session_id,
783
+ score: cosineSimilarity(queryVec!, vector), ordinal: ordinal++ });
784
+ }
785
+ // Only the winning K records bring transcript bodies into memory. Scoring
786
+ // remains exhaustive O(N * dimension); --limit bounds state, not CPU work.
787
+ const hydrate = db.query<EmbeddingRow, (string | number)[]>(sql + " AND e.record_id = ?");
788
+ return top.sorted().map(candidate => {
789
+ const row = hydrate.get(...(params as (string | number)[]), candidate.record_id);
790
+ if (!row) throw new Error("Semantic winner disappeared within its read snapshot");
791
+ return { row, score: candidate.score };
792
+ });
793
+ }).deferred();
794
+ const hits: RecordSearchHit[] = [];
795
+
796
+ for (const { row, score } of topHits) {
797
+ const snippet = (
798
+ row.content ??
799
+ row.tool_arguments_json ??
800
+ row.tool_result_json ??
801
+ ""
802
+ ).slice(0, 240);
803
+ hits.push({
804
+ session_id: row.session_id,
805
+ record_id: row.record_id,
806
+ source: row.source,
807
+ title: row.title,
808
+ cwd: row.cwd,
809
+ started_at: row.started_at,
810
+ record_type: row.record_type,
811
+ content: row.content,
812
+ tool_name: row.tool_name,
813
+ tool_arguments: row.tool_arguments_json,
814
+ tool_result: row.tool_result_json,
815
+ snippet,
816
+ rank: score,
817
+ parent_session_id: row.parent_session_id,
818
+ agent_nickname: row.agent_nickname,
819
+ thread_role: row.thread_role,
820
+ match_type: "semantic",
821
+ });
822
+ }
823
+
824
+ return { hits };
825
+ }
826
+
827
+ /**
828
+ * Semantic search: the vector leg when it can run, the lexical engine plus an
829
+ * `embeddings_unavailable` report when it cannot.
830
+ */
831
+ export async function querySemantic(
832
+ db: Database,
833
+ queryText: string,
834
+ options: ReadQueryOptions = {},
835
+ ): Promise<QueryResult> {
836
+ const startedAt = performance.now();
837
+ const limit = options.limit ?? 20;
838
+ const cleanQuery = queryText.trim();
839
+ if (!cleanQuery) {
840
+ return buildQueryResult(queryText, [], "semantic", limit, startedAt, {
841
+ countReason: COUNT_REASON_EMPTY_QUERY,
842
+ });
843
+ }
844
+
845
+ // Nothing stored means nothing fell back: the vector leg is not missing
846
+ // coverage, the corpus is empty.
847
+ if (!hasRecords(db, options.source)) {
848
+ return buildQueryResult(cleanQuery, [], "semantic", limit, startedAt);
849
+ }
850
+
851
+ const vector = await vectorSearch(db, cleanQuery, { limit, source: options.source });
852
+ if (!vector) {
853
+ return lexicalFallback(
854
+ queryLexical(db, cleanQuery, { limit, source: options.source }),
855
+ limit,
856
+ "semantic",
857
+ startedAt,
858
+ );
859
+ }
860
+ return buildQueryResult(cleanQuery, vector.hits, "semantic", limit, startedAt);
861
+ }
862
+
863
+ /**
864
+ * Reciprocal-Rank fusion of the lexical and vector legs. When the vector leg
865
+ * cannot run, fusion has nothing to add: the lexical answer is returned with
866
+ * the envelope naming the requested mode and the reason, so `--hybrid` never
867
+ * pretends two engines ran.
868
+ */
869
+ export async function queryHybrid(
870
+ db: Database,
871
+ queryText: string,
872
+ options: ReadQueryOptions = {},
873
+ ): Promise<QueryResult> {
874
+ const startedAt = performance.now();
875
+ const limit = options.limit ?? 20;
876
+ const cleanQuery = queryText.trim();
877
+ if (!cleanQuery) {
878
+ return buildQueryResult(queryText, [], "hybrid", limit, startedAt, {
879
+ countReason: COUNT_REASON_EMPTY_QUERY,
880
+ });
881
+ }
882
+
883
+ const candidateLimit = Math.max(limit * 3, 60);
884
+
885
+ // Nothing stored means nothing fell back: the vector leg is not missing
886
+ // coverage, the corpus is empty.
887
+ if (!hasRecords(db, options.source)) {
888
+ return buildQueryResult(cleanQuery, [], "hybrid", limit, startedAt);
889
+ }
890
+
891
+ const lexicalResult = queryLexical(db, cleanQuery, {
892
+ limit: candidateLimit,
893
+ source: options.source,
894
+ });
895
+
896
+ const vector = await vectorSearch(db, cleanQuery, {
897
+ limit: candidateLimit,
898
+ source: options.source,
899
+ });
900
+
901
+ if (!vector) {
902
+ return lexicalFallback(lexicalResult, limit, "hybrid", startedAt);
903
+ }
904
+
905
+ // The title lane precedes the fusion instead of entering it: a title hit is
906
+ // session-level and has no record id to be keyed by, and the lane it comes
907
+ // from is not an engine the fusion combines but the lexical engine's own
908
+ // first pass. The fused hits fill what the titles leave of the limit, so the
909
+ // result stays bounded by it.
910
+ const isTitleHit = (hit: SearchHit): hit is TitleSearchHit => hit.match_type === "title";
911
+ const isRecordHit = (hit: SearchHit): hit is RecordSearchHit => hit.match_type !== "title";
912
+ const titles = lexicalResult.hits.filter(isTitleHit).slice(0, limit);
913
+ const recordHits = lexicalResult.hits.filter(isRecordHit);
914
+
915
+ const RRF_K = 60;
916
+ const scoredMap = new Map<
917
+ string,
918
+ {
919
+ hit: RecordSearchHit;
920
+ rrfScore: number;
921
+ lexicalRank?: number;
922
+ semanticRank?: number;
923
+ }
924
+ >();
925
+
926
+ // The record leg keeps the ranks the lexical lane gave its records. The
927
+ // title hits it emitted first shift every one of them by the same offset, so
928
+ // the order within the leg — which is all RRF reads — is unchanged.
929
+ for (let i = 0; i < recordHits.length; i++) {
930
+ const hit = recordHits[i];
931
+ const rank = i + 1;
932
+ const rrf = 1 / (RRF_K + rank);
933
+ scoredMap.set(hit.record_id, {
934
+ hit,
935
+ rrfScore: rrf,
936
+ lexicalRank: rank,
937
+ });
938
+ }
939
+
940
+ for (let i = 0; i < vector.hits.length; i++) {
941
+ const hit = vector.hits[i];
942
+ const rank = i + 1;
943
+ const rrf = 1 / (RRF_K + rank);
944
+ const existing = scoredMap.get(hit.record_id);
945
+ if (existing) {
946
+ existing.rrfScore += rrf;
947
+ existing.semanticRank = rank;
948
+ } else {
949
+ scoredMap.set(hit.record_id, {
950
+ hit,
951
+ rrfScore: rrf,
952
+ semanticRank: rank,
953
+ });
954
+ }
955
+ }
956
+
957
+ const combined = Array.from(scoredMap.values());
958
+ combined.sort((a, b) => b.rrfScore - a.rrfScore);
959
+
960
+ const topCombined = combined.slice(0, limit - titles.length);
961
+ const hits: SearchHit[] = [...titles];
962
+
963
+ for (const item of topCombined) {
964
+ hits.push({
965
+ ...item.hit,
966
+ rank: item.rrfScore,
967
+ });
968
+ }
969
+
970
+ return buildQueryResult(cleanQuery, hits, "hybrid", limit, startedAt, {
971
+ // The lexical leg of the fusion may have widened a zero-hit candidate
972
+ // set or fallen back to escaped terms; report both rather than hiding
973
+ // them behind the hybrid label.
974
+ fallbackReason: lexicalResult.meta.fallback_reason,
975
+ widened: lexicalResult.meta.widened,
976
+ });
977
+ }
978
+
979
+ /**
980
+ * The session-level serving reads: a deleted session is not found, not listed,
981
+ * and offers no records, while its rows stay exactly where the store keeps them
982
+ * (`sessions`, `records` and every derived artifact — db.ts's `tombstoneSession`).
983
+ * The predicate is the lanes' own, so a session cannot be invisible to search
984
+ * while `ledger recent`, a parent-title lookup, or a record walk still shows it.
985
+ */
986
+ export function sessionById(db: Database, id: string): SessionRecord | null {
987
+ return (
988
+ db
989
+ .query<SessionRecord, [string]>(
990
+ `SELECT * FROM sessions WHERE id = ? AND ${tombstonePredicate(db, "sessions.id")} LIMIT 1`,
991
+ )
992
+ .get(id) ?? null
993
+ );
994
+ }
995
+
996
+ export function allSessions(db: Database): SessionRecord[] {
997
+ return db
998
+ .query<SessionRecord, []>(
999
+ `SELECT * FROM sessions WHERE ${tombstonePredicate(db, "sessions.id")}
1000
+ ORDER BY started_at DESC, id ASC`,
1001
+ )
1002
+ .all();
1003
+ }
1004
+
1005
+ export function lastSessions(
1006
+ db: Database,
1007
+ options: { limit?: number; source?: string } = {},
1008
+ ): SessionRecord[] {
1009
+ const limit = options.limit ?? 20;
1010
+ if (options.source) {
1011
+ return db
1012
+ .query<SessionRecord, [string, number]>(
1013
+ `SELECT * FROM sessions WHERE source = ? AND ${tombstonePredicate(db, "sessions.id")}
1014
+ ORDER BY updated_at DESC, started_at DESC, id ASC LIMIT ?`,
1015
+ )
1016
+ .all(options.source, limit);
1017
+ }
1018
+ return db
1019
+ .query<SessionRecord, [number]>(
1020
+ `SELECT * FROM sessions WHERE ${tombstonePredicate(db, "sessions.id")}
1021
+ ORDER BY updated_at DESC, started_at DESC, id ASC LIMIT ?`,
1022
+ )
1023
+ .all(limit);
1024
+ }
1025
+
1026
+ export function sessionRecords(db: Database, sessionId: string): CanonicalDbRecord[] {
1027
+ return db
1028
+ .query<CanonicalDbRecord, [string]>(
1029
+ `SELECT * FROM records WHERE session_id = ? AND ${tombstonePredicate(db, "records.session_id")}
1030
+ ORDER BY source_order_id ASC, component_index ASC`,
1031
+ )
1032
+ .all(sessionId);
1033
+ }
1034
+
1035
+ /**
1036
+ * Why a store could not be read: missing or empty, newer than this build
1037
+ * (schema or derived index), not a Ledger store, or missing the read schema.
1038
+ */
1039
+ export type ReaderErrorCode =
1040
+ | "STORE_NOT_INITIALIZED"
1041
+ | "SCHEMA_MISMATCH"
1042
+ | "FORMAT_MISMATCH"
1043
+ | "STORE_UNREADABLE"
1044
+ | "STORE_SCHEMA_INCOMPLETE";
1045
+
1046
+ /**
1047
+ * Typed read refusal. The message always names the path and the remedy: a read
1048
+ * verb never silently creates, upgrades, or repairs a store, and an operator
1049
+ * should not have to decode a raw SQLite error to learn that.
1050
+ */
1051
+ export class ReaderError extends Error {
1052
+ readonly code: ReaderErrorCode;
1053
+
1054
+ constructor(code: ReaderErrorCode, message: string) {
1055
+ super(message);
1056
+ this.name = "ReaderError";
1057
+ this.code = code;
1058
+ }
1059
+ }
1060
+
1061
+ /**
1062
+ * The SQLite header records the journal mode at bytes 18-19: write version 2 is
1063
+ * WAL. A WAL store needs its `-wal`/`-shm` sidecars before any connection may
1064
+ * open it read-only — unlike a writable open, a read-only one may not create
1065
+ * them — so the failure is diagnosed here instead of surfacing as a bare
1066
+ * "unable to open database file" or "disk I/O error".
1067
+ */
1068
+ function walStoreWithoutSidecars(dbPath: string): boolean {
1069
+ if (existsSync(`${dbPath}-wal`) && existsSync(`${dbPath}-shm`)) return false;
1070
+ let fd: number | undefined;
1071
+ try {
1072
+ fd = openSync(dbPath, "r");
1073
+ const header = Buffer.alloc(20);
1074
+ if (readSync(fd, header, 0, 20, 0) < 20) return false;
1075
+ return header.toString("latin1", 0, 15) === "SQLite format 3" && header[18] === 2;
1076
+ } catch {
1077
+ return false;
1078
+ } finally {
1079
+ if (fd !== undefined) closeSync(fd);
1080
+ }
1081
+ }
1082
+
1083
+ /**
1084
+ * The store-status wording, plus the one diagnosis that is specific to opening
1085
+ * a store read-only: a WAL database whose sidecars are gone cannot be opened
1086
+ * until a writer recreates them.
1087
+ */
1088
+ function unreadableMessage(dbPath: string, detail: string, phase: "open" | "read"): string {
1089
+ if (walStoreWithoutSidecars(dbPath)) {
1090
+ return `Cannot open database at ${dbPath}: ${detail}. The store is a WAL database missing its -wal/-shm sidecars;` +
1091
+ " a read-only reader cannot create them — run `trajectories sync` (or restore the sidecars).";
1092
+ }
1093
+ return phase === "open"
1094
+ ? `Cannot open database at ${dbPath}: ${detail}.`
1095
+ : `Cannot read database at ${dbPath}: ${detail}. The file may be corrupt or not a SQLite database.`;
1096
+ }
1097
+
1098
+ /**
1099
+ * Open `dbPath` read-only in the engine's own sense: the connection flag is
1100
+ * irreversible and enforced by SQLite, so no statement on it can write, create
1101
+ * a table, or migrate a schema. `PRAGMA query_only` is deliberately not used —
1102
+ * it is reversible per connection and still permits file creation.
1103
+ *
1104
+ * Every failure is typed, and every refusal comes from the shared read gate
1105
+ * (`classifyReadStore`), so this and `readStoreStatus` cannot disagree about
1106
+ * what a store is. A store whose schema is older than this build keeps serving
1107
+ * reads (`status` semantics) while it carries the read schema, and a store with
1108
+ * a pending index re-derivation — or one derived by an *older* rule set — is
1109
+ * read as written: the index is stale, not unreadable. A missing or empty
1110
+ * store, a store this build cannot read, a store written by a newer schema, a
1111
+ * store whose derived index a newer build's rules produced, and a store without
1112
+ * the read schema are refused.
1113
+ *
1114
+ * Exported for `TrajectoryReader`, for the package's own `delete` verb — which
1115
+ * must classify a store without initializing or adopting one — and for the
1116
+ * read-path tests, which pin the engine-level flag by writing through the handle
1117
+ * it returns. It is not part of the package's public surface (`index.ts` exports
1118
+ * the class, not this).
1119
+ */
1120
+ export function openReadOnlyStore(dbPath: string): Database {
1121
+ const store = classifyReadStore(dbPath);
1122
+ switch (store.state) {
1123
+ case "ok":
1124
+ return store.db;
1125
+ case "not_initialized":
1126
+ throw new ReaderError(
1127
+ "STORE_NOT_INITIALIZED",
1128
+ store.reason === "missing"
1129
+ ? `Database not found at ${dbPath}. Run \`trajectories sync\` or \`trajectories build\` first.`
1130
+ : `Database at ${dbPath} exists but is empty (zero bytes), so there is no store to read.` +
1131
+ " Run `trajectories sync` or `trajectories build` first.",
1132
+ );
1133
+ case "foreign":
1134
+ throw new ReaderError(
1135
+ "STORE_UNREADABLE",
1136
+ `Database at ${dbPath} exists but is not an initialized Ledger store (no sessions table).`,
1137
+ );
1138
+ case "unreadable":
1139
+ throw new ReaderError("STORE_UNREADABLE", unreadableMessage(dbPath, store.detail, store.phase));
1140
+ case "schema_mismatch":
1141
+ throw new ReaderError(
1142
+ "SCHEMA_MISMATCH",
1143
+ `Database schema version ${store.schemaVersion} at ${dbPath} is newer than supported schema version ${SCHEMA_VERSION}.`,
1144
+ );
1145
+ case "format_mismatch":
1146
+ throw new ReaderError("FORMAT_MISMATCH", formatMismatchMessage(dbPath, store));
1147
+ case "incomplete":
1148
+ throw new ReaderError("STORE_SCHEMA_INCOMPLETE", incompleteStoreMessage(dbPath, store.firstGap, store.schemaVersion));
1149
+ }
1150
+ }
1151
+
1152
+ /**
1153
+ * Read-only view of one Ledger store for one verb invocation.
1154
+ *
1155
+ * Short-lived by design: a reader is created per CLI verb and closed, so a
1156
+ * replaced store leaves no stranded handle or index behind. Reads never take
1157
+ * the writer lease, never create or upgrade the store, and never backfill
1158
+ * embeddings — `--semantic`/`--hybrid` answer lexically when the store holds no
1159
+ * vectors the active provider can use (see `SearchMeta.fallback_reason`).
1160
+ */
1161
+ export class TrajectoryReader {
1162
+ private db: Database;
1163
+ readonly path: string;
1164
+
1165
+ constructor(path: string = "ledger.sqlite") {
1166
+ this.path = path;
1167
+ this.db = openReadOnlyStore(path);
1168
+ }
1169
+
1170
+ query(
1171
+ queryText: string,
1172
+ options?: { limit?: number; source?: string; mode?: "lexical" },
1173
+ ): QueryResult;
1174
+ query(
1175
+ queryText: string,
1176
+ options: { limit?: number; source?: string; mode: "semantic" | "hybrid" },
1177
+ ): Promise<QueryResult>;
1178
+ query(
1179
+ queryText: string,
1180
+ options?: { limit?: number; source?: string; mode?: "lexical" | "semantic" | "hybrid" },
1181
+ ): QueryResult | Promise<QueryResult>;
1182
+ query(
1183
+ queryText: string,
1184
+ options: { limit?: number; source?: string; mode?: "lexical" | "semantic" | "hybrid" } = {},
1185
+ ): QueryResult | Promise<QueryResult> {
1186
+ if (options.mode === "semantic") {
1187
+ return this.querySemantic(queryText, options);
1188
+ }
1189
+ if (options.mode === "hybrid") {
1190
+ return this.queryHybrid(queryText, options);
1191
+ }
1192
+ return this.queryLexical(queryText, options);
1193
+ }
1194
+
1195
+ queryLexical(queryText: string, options: ReadQueryOptions = {}): QueryResult {
1196
+ return queryLexical(this.db, queryText, options);
1197
+ }
1198
+
1199
+ querySemantic(queryText: string, options: ReadQueryOptions = {}): Promise<QueryResult> {
1200
+ return querySemantic(this.db, queryText, options);
1201
+ }
1202
+
1203
+ queryHybrid(queryText: string, options: ReadQueryOptions = {}): Promise<QueryResult> {
1204
+ return queryHybrid(this.db, queryText, options);
1205
+ }
1206
+
1207
+ getSession(id: string): SessionRecord | null {
1208
+ return sessionById(this.db, id);
1209
+ }
1210
+
1211
+ getAllSessions(): SessionRecord[] {
1212
+ return allSessions(this.db);
1213
+ }
1214
+
1215
+ getLastSessions(options?: { limit?: number; source?: string }): SessionRecord[] {
1216
+ return lastSessions(this.db, options);
1217
+ }
1218
+
1219
+ getSessionRecords(sessionId: string): CanonicalDbRecord[] {
1220
+ return sessionRecords(this.db, sessionId);
1221
+ }
1222
+
1223
+ getSessionCount(): number {
1224
+ return countSessions(this.db);
1225
+ }
1226
+
1227
+ getRecordCount(): number {
1228
+ return countRecords(this.db);
1229
+ }
1230
+
1231
+ /** Compact durable coverage signal for robot envelopes; null when no pass
1232
+ * has recorded coverage (the field is then simply omitted). */
1233
+ getSourceCoverageSummary(): SourceCoverageSummary | null {
1234
+ const hasTable = !!this.db
1235
+ .query<{ name: string }, []>(
1236
+ "SELECT name FROM sqlite_master WHERE type='table' AND name='source_coverage' LIMIT 1;",
1237
+ )
1238
+ .get();
1239
+ return hasTable ? readSourceCoverageSummary(this.db) : null;
1240
+ }
1241
+
1242
+ getDiscoverySummary(): DiscoverySummary | null { return readDiscoverySummary(this.db); }
1243
+
1244
+ getSchemaVersion(): number {
1245
+ return (
1246
+ this.db
1247
+ .query<{ user_version: number }, []>("PRAGMA user_version;")
1248
+ .get()?.user_version ?? 0
1249
+ );
1250
+ }
1251
+
1252
+ close(): void {
1253
+ this.db.close();
1254
+ }
1255
+ }