trajectories 0.0.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +36 -0
- package/LICENSE +201 -0
- package/README.md +288 -2
- package/dist/cli.js +30210 -0
- package/dist/convert.d.ts +31 -0
- package/dist/convert.js +431 -0
- package/dist/deepagents_checkpoint.py +362 -0
- package/dist/index.d.ts +7 -0
- package/dist/index.js +8402 -0
- package/dist/log-baseline.d.ts +26 -0
- package/dist/log-baseline.js +104 -0
- package/dist/log-content.d.ts +16 -0
- package/dist/log-content.js +80 -0
- package/dist/log-io.d.ts +10 -0
- package/dist/log-io.js +121 -0
- package/dist/log-read.d.ts +24 -0
- package/dist/log-read.js +117 -0
- package/dist/log-reconcile.d.ts +5 -0
- package/dist/log-reconcile.js +38 -0
- package/dist/log-records.d.ts +35 -0
- package/dist/log-records.js +85 -0
- package/dist/log-recovery.d.ts +18 -0
- package/dist/log-recovery.js +123 -0
- package/dist/log-stage.d.ts +14 -0
- package/dist/log-stage.js +138 -0
- package/dist/log-state.d.ts +89 -0
- package/dist/log-state.js +92 -0
- package/dist/log-switch.d.ts +3 -0
- package/dist/log-switch.js +98 -0
- package/dist/log-types.d.ts +129 -0
- package/dist/log-types.js +8 -0
- package/dist/log.d.ts +8 -0
- package/dist/log.js +62 -0
- package/dist/migrate.d.ts +29 -0
- package/dist/migrate.js +173 -0
- package/dist/normalization-runtime.d.ts +1 -0
- package/dist/normalization-runtime.js +3150 -0
- package/dist/portable.d.ts +9 -0
- package/dist/portable.js +26 -0
- package/dist/readers/canonical.d.ts +13 -0
- package/dist/readers/canonical.js +54 -0
- package/dist/readers/cline.d.ts +10 -0
- package/dist/readers/cline.js +107 -0
- package/dist/readers/cursor.d.ts +20 -0
- package/dist/readers/cursor.js +374 -0
- package/dist/readers/deepagents.d.ts +41 -0
- package/dist/readers/deepagents.js +163 -0
- package/dist/readers/fx.d.ts +11 -0
- package/dist/readers/fx.js +573 -0
- package/dist/readers/github-copilot.d.ts +8 -0
- package/dist/readers/github-copilot.js +216 -0
- package/dist/readers/grok-build.d.ts +12 -0
- package/dist/readers/grok-build.js +391 -0
- package/dist/readers/guards.d.ts +4 -0
- package/dist/readers/guards.js +121 -0
- package/dist/readers/pi-compaction.d.ts +7 -0
- package/dist/readers/pi-compaction.js +112 -0
- package/dist/readers/pi-context.d.ts +13 -0
- package/dist/readers/pi-context.js +64 -0
- package/dist/store.d.ts +25 -0
- package/dist/store.js +218 -0
- package/dist/types.d.ts +91 -0
- package/dist/types.js +1 -0
- package/dist/writers/claude-code.d.ts +5 -0
- package/dist/writers/claude-code.js +51 -0
- package/dist/writers/cline.d.ts +3 -0
- package/dist/writers/cline.js +77 -0
- package/dist/writers/codex.d.ts +3 -0
- package/dist/writers/codex.js +32 -0
- package/dist/writers/cursor.d.ts +3 -0
- package/dist/writers/cursor.js +197 -0
- package/dist/writers/deepagents.d.ts +3 -0
- package/dist/writers/deepagents.js +116 -0
- package/dist/writers/entropy.d.ts +5 -0
- package/dist/writers/entropy.js +33 -0
- package/dist/writers/fx.d.ts +3 -0
- package/dist/writers/fx.js +135 -0
- package/dist/writers/github-copilot.d.ts +3 -0
- package/dist/writers/github-copilot.js +71 -0
- package/dist/writers/grok-build.d.ts +3 -0
- package/dist/writers/grok-build.js +43 -0
- package/dist/writers/helpers.d.ts +17 -0
- package/dist/writers/helpers.js +75 -0
- package/dist/writers/omp.d.ts +3 -0
- package/dist/writers/omp.js +7 -0
- package/dist/writers/opencode.d.ts +3 -0
- package/dist/writers/opencode.js +65 -0
- package/dist/writers/pi.d.ts +3 -0
- package/dist/writers/pi.js +45 -0
- package/docs/cline-native-format.md +43 -0
- package/docs/copilot-native-format.md +78 -0
- package/docs/cursor-native-format.md +103 -0
- package/docs/deepagents-native-format.md +98 -0
- package/docs/evidence-api.md +170 -0
- package/docs/fx-native-format.md +87 -0
- package/docs/harness-parity-bootstrap.md +78 -0
- package/docs/harness-support-parity.md +107 -0
- package/docs/inspirations.md +103 -0
- package/docs/ledger-migration.md +141 -0
- package/docs/pi-compaction-context.md +75 -0
- package/docs/pi-provider-context.md +76 -0
- package/docs/research/2026-10-06-huihua-and-oar.md +176 -0
- package/docs/research/ledger/README.md +96 -0
- package/docs/research/ledger/confer/2026-09-13-round-10/README.md +61 -0
- package/docs/research/ledger/confer/2026-09-13-round-2/cass.md +72 -0
- package/docs/research/ledger/confer/2026-09-13-round-3/README.md +70 -0
- package/docs/research/ledger/confer/2026-09-13-round-5/README.md +61 -0
- package/docs/research/ledger/confer/2026-09-13-round-6/README.md +68 -0
- package/docs/research/ledger/confer/2026-09-13-round-7/README.md +64 -0
- package/docs/research/ledger/confer/2026-09-13-round-9/README.md +59 -0
- package/docs/research/ledger/confer/2026-09-14-wake/wake.md +146 -0
- package/docs/research/ledger/docs/inspirations.md +320 -0
- package/docs/research/ledger/docs/pr2-pr6-integration.md +53 -0
- package/docs/research/ledger/docs/research/2026-09-07-confer-inspirations.md +163 -0
- package/docs/research/ledger/docs/research/2026-09-07-letta-continuous-service.md +104 -0
- package/docs/research/ledger/docs/research/2026-09-08-claude-history-retrieval.md +117 -0
- package/docs/research/ledger/docs/research/2026-09-09-agentsview-native-discovery.md +87 -0
- package/docs/research/ledger/docs/research/2026-09-09-daemon-observation.md +231 -0
- package/docs/research/ledger/docs/research/2026-09-09-discovery-pagination.md +114 -0
- package/docs/research/ledger/docs/research/2026-09-09-harness-transcript-contract.md +145 -0
- package/docs/research/ledger/docs/research/2026-09-09-native-history-discovery.md +180 -0
- package/docs/research/ledger/docs/research/2026-09-09-native-history-implementation.md +137 -0
- package/docs/research/ledger/docs/research/2026-09-09-other-inspirations.md +279 -0
- package/docs/research/ledger/docs/research/2026-09-09-trajectory-and-letta-code.md +128 -0
- package/docs/research/ledger/docs/research/2026-09-13-diagnostics-coverage.md +116 -0
- package/docs/research/ledger/docs/research/2026-09-13-fts-rebuild.md +83 -0
- package/docs/research/ledger/docs/research/2026-09-13-read-path-split.md +74 -0
- package/docs/research/ledger/docs/research/2026-09-13-runtime-rollout.md +102 -0
- package/docs/research/ledger/docs/research/2026-09-13-scan-outcomes.md +80 -0
- package/docs/research/ledger/docs/research/2026-09-13-search-quality.md +105 -0
- package/docs/research/ledger/docs/research/2026-09-13-source-coverage.md +64 -0
- package/docs/research/ledger/docs/research/2026-09-13-sync-wall-budget.md +82 -0
- package/docs/research/ledger/docs/research/2026-09-13-typed-read-gate.md +81 -0
- package/docs/research/ledger/docs/research/2026-09-13-writer-lease.md +84 -0
- package/docs/research/ledger/docs/research/2026-09-16-c1-copy-fallback.md +227 -0
- package/docs/research/ledger/docs/research/2026-09-16-c10-kimi.md +414 -0
- package/docs/research/ledger/docs/research/2026-09-16-c2-derive-counter.md +216 -0
- package/docs/research/ledger/docs/research/2026-09-16-c3-echo-filter.md +305 -0
- package/docs/research/ledger/docs/research/2026-09-16-c4-title-index.md +349 -0
- package/docs/research/ledger/docs/research/2026-09-16-c5-trigram.md +268 -0
- package/docs/research/ledger/docs/research/2026-09-16-c6-recency-prior.md +438 -0
- package/docs/research/ledger/docs/research/2026-09-16-c7-tombstones.md +467 -0
- package/docs/research/ledger/docs/research/2026-09-16-c8-remote-truth.md +390 -0
- package/docs/research/ledger/docs/research/2026-09-16-c9-grok.md +380 -0
- package/docs/research/ledger/docs/research/2026-09-16-wake-port-program.md +38 -0
- package/docs/research/ledger/docs/research/2026-09-17-cursor-blobstore.md +157 -0
- package/docs/research/ledger/manifest.json +540 -0
- package/legacy/NOTICE.md +50 -0
- package/legacy/bin/ledger-core.ts +7 -0
- package/legacy/bin/ledger.ts +7 -0
- package/legacy/bin/traj-bridge.ts +7 -0
- package/legacy/scripts/import-claude-history.ts +130 -0
- package/legacy/scripts/native-history-smoke.ts +52 -0
- package/legacy/scripts/observe-daemon.ts +570 -0
- package/legacy/scripts/retitle-sessions.md +19 -0
- package/legacy/scripts/verify-runtime-rollout.ts +151 -0
- package/legacy/src/canonical.ts +108 -0
- package/legacy/src/cli.ts +1293 -0
- package/legacy/src/db.ts +2254 -0
- package/legacy/src/diagnostics.ts +87 -0
- package/legacy/src/discovery.ts +56 -0
- package/legacy/src/embeddings.ts +382 -0
- package/legacy/src/filter.ts +602 -0
- package/legacy/src/gemini_embeddings.ts +103 -0
- package/legacy/src/identity.ts +9 -0
- package/legacy/src/index.ts +10 -0
- package/legacy/src/ingest.ts +791 -0
- package/legacy/src/managed.ts +178 -0
- package/legacy/src/native/common.ts +68 -0
- package/legacy/src/native/cursor.ts +647 -0
- package/legacy/src/native/grok.ts +540 -0
- package/legacy/src/native/kimi.ts +706 -0
- package/legacy/src/native/opencode.ts +470 -0
- package/legacy/src/native/sqlite-ro.ts +281 -0
- package/legacy/src/native.ts +97 -0
- package/legacy/src/normalizer.ts +457 -0
- package/legacy/src/parentage.ts +562 -0
- package/legacy/src/pressure.ts +162 -0
- package/legacy/src/private-storage.ts +74 -0
- package/legacy/src/reader.ts +1255 -0
- package/legacy/src/scanner.ts +591 -0
- package/legacy/src/schema-gate.ts +263 -0
- package/legacy/src/schema.ts +829 -0
- package/legacy/src/semantic-top-k.ts +71 -0
- package/legacy/src/session-hash.ts +49 -0
- package/legacy/src/store-status.ts +218 -0
- package/legacy/src/sync-preparation.ts +45 -0
- package/legacy/src/sync-truth.ts +843 -0
- package/legacy/src/sync.ts +1233 -0
- package/legacy/src/types.ts +436 -0
- package/legacy/src/vocab.txt +30522 -0
- package/legacy/src/watch.ts +673 -0
- package/legacy/src/writer-lease.ts +310 -0
- package/ops/README.md +150 -0
- package/ops/install-cli.sh +63 -0
- package/ops/install-native.sh +35 -0
- package/ops/install-release.sh +49 -0
- package/ops/linux/install-systemd-user.sh +103 -0
- package/ops/macos/install-launchd.sh +72 -0
- package/ops/sync-settings.sh +37 -0
- package/package.json +21 -5
- package/skills/trajectories/SKILL.md +138 -0
- package/skills/trajectories/scripts/trajectories_recent.sh +26 -0
- package/skills/trajectories/scripts/trajectories_search.sh +26 -0
- package/index.js +0 -1
|
@@ -0,0 +1,1255 @@
|
|
|
1
|
+
import { closeSync, existsSync, openSync, readSync } from "node:fs";
|
|
2
|
+
import { Database } from "bun:sqlite";
|
|
3
|
+
import { readDiscoverySummary, readSourceCoverageSummary, type SourceCoverageSummary } from "./db.js";
|
|
4
|
+
import type { DiscoverySummary } from "./discovery.js";
|
|
5
|
+
import { SemanticTopK } from "./semantic-top-k.js";
|
|
6
|
+
import {
|
|
7
|
+
cosineSimilarity,
|
|
8
|
+
deserializeEmbedding,
|
|
9
|
+
embed,
|
|
10
|
+
getEmbeddingProvider,
|
|
11
|
+
type EmbeddingProvider,
|
|
12
|
+
} from "./embeddings.js";
|
|
13
|
+
import type { GeminiEmbeddingProvider } from "./gemini_embeddings.js";
|
|
14
|
+
import { SCHEMA_VERSION } from "./schema.js";
|
|
15
|
+
import { classifyReadStore, formatMismatchMessage, incompleteStoreMessage } from "./schema-gate.js";
|
|
16
|
+
import type {
|
|
17
|
+
CanonicalDbRecord,
|
|
18
|
+
QueryResult,
|
|
19
|
+
RecordMatchType,
|
|
20
|
+
RecordSearchHit,
|
|
21
|
+
SearchHit,
|
|
22
|
+
SearchMode,
|
|
23
|
+
SessionRecord,
|
|
24
|
+
TitleSearchHit,
|
|
25
|
+
WideningReport,
|
|
26
|
+
WideningSkipReason,
|
|
27
|
+
} from "./types.js";
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* The read path of one store, against any connection handle.
|
|
31
|
+
*
|
|
32
|
+
* Everything here is pure query work: it never initializes, migrates, repairs a
|
|
33
|
+
* trigger, configures FTS, or writes an embedding. `TrajectoryReader` owns the
|
|
34
|
+
* read-only connection lifecycle; `TrajectoryDb` reuses the same functions for
|
|
35
|
+
* the read methods its writer handle already offers, so lexical widening,
|
|
36
|
+
* envelope accounting, match provenance, and fusion cannot drift between the
|
|
37
|
+
* two.
|
|
38
|
+
*/
|
|
39
|
+
|
|
40
|
+
// Zero-hit lexical widening. Every gate below runs only after the primary query
|
|
41
|
+
// returned nothing, so widening never touches the cost or the output of a
|
|
42
|
+
// normal search. LEDGER_WIDEN_MAX_RECORDS=0 disables widening outright.
|
|
43
|
+
//
|
|
44
|
+
// The default is the bound schema v11's rebuilt FTS index earns: the production
|
|
45
|
+
// store (156,801 records) sat above the old 10,000-record ceiling, so every
|
|
46
|
+
// zero-hit query there refused widening as `corpus_over_ceiling`. 250,000 admits
|
|
47
|
+
// it with room for growth — measured on a copy of that store, widened queries
|
|
48
|
+
// over ordinary term pairs ran in 19-106 ms after the rebuild where the pre-v11
|
|
49
|
+
// index measured 178-890 ms, with identical hits (both load-dependent; a warmer,
|
|
50
|
+
// loaded run measured 16-40 ms: docs/research/2026-09-13-fts-rebuild.md). The
|
|
51
|
+
// threshold is a record count, not a measured latency, so a store still at v10 —
|
|
52
|
+
// its index has not been rebuilt — widens on the old, slower plan until a writer
|
|
53
|
+
// open migrates it, and a widened query whose terms match a large share of the
|
|
54
|
+
// store stays expensive either way.
|
|
55
|
+
const DEFAULT_WIDEN_MAX_RECORDS = 250_000;
|
|
56
|
+
const MAX_WIDEN_TOKEN_CHARS = 16;
|
|
57
|
+
const MIN_WIDEN_TOKEN_ALNUM = 3;
|
|
58
|
+
const MAX_WIDEN_TOKENS = 16;
|
|
59
|
+
const BOOLEAN_OPERATOR_RE = /\b(AND|OR|NOT|NEAR)\b/i;
|
|
60
|
+
export const FTS_SYNTAX_FALLBACK_REASON = "fts_syntax_error";
|
|
61
|
+
/**
|
|
62
|
+
* Reported when a semantic or hybrid query had no vectors to search: either the
|
|
63
|
+
* store holds no compatible embeddings (a read path never backfills) or the
|
|
64
|
+
* active provider could not embed the query text.
|
|
65
|
+
*/
|
|
66
|
+
export const EMBEDDINGS_UNAVAILABLE_REASON = "embeddings_unavailable";
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* The recency prior both lexical lanes order by. bm25 is negative and smaller
|
|
70
|
+
* means more relevant, so multiplying it by `1 + 1/(1 + age/H)` makes a newer
|
|
71
|
+
* session's hits "more negative": a session from today is ×2, one 30 days old
|
|
72
|
+
* ×1.5, one a year old ×1.08. Equal-relevance hits go to the newer session; a
|
|
73
|
+
* hit whose text relevance is more than the factor apart still wins on text.
|
|
74
|
+
* Wake multiplies its message lane and its title lane by exactly this
|
|
75
|
+
* (`crates/wake-core/src/db.rs#L1546-L1560`, tag v0.6.5, MIT), keyed on the
|
|
76
|
+
* *session's* timestamp rather than the record's, so the relative order of hits
|
|
77
|
+
* within one session is untouched — which is why Ledger's prior can read
|
|
78
|
+
* `started_at` and leave the lanes' `started_at DESC` tiebreak alone.
|
|
79
|
+
*
|
|
80
|
+
* Arithmetic, `max()` and one core date function only — no math extension — and
|
|
81
|
+
* one term in a lane's ORDER BY: no rescore pass. `unixepoch()` is the one way
|
|
82
|
+
* to an epoch from this store's ISO-8601 timestamps (no numeric time column
|
|
83
|
+
* exists), and it is a core function of the engine these queries run on —
|
|
84
|
+
* Bun's bundled SQLite 3.51.0, the same runtime C5 verified its SQLite feature
|
|
85
|
+
* set against. `COALESCE(…, 0)` puts an absent or unparsable timestamp at the
|
|
86
|
+
* epoch, the factor's neutral end (×1.0), where a NULL would sort such a hit
|
|
87
|
+
* first in both lanes. The clock is the caller's `Date.now()`, bound as the `?`
|
|
88
|
+
* Wake binds (`now_ms()`), evaluated once per query rather than once per
|
|
89
|
+
* matched row. H is spelled as a real (`${RECENCY_DECAY_MS}.0`), running text
|
|
90
|
+
* and all: SQLite divides integers with integer division, and a truncated
|
|
91
|
+
* divisor would silently make the factor ×2 for the whole corpus.
|
|
92
|
+
*
|
|
93
|
+
* `started_at` is the key, not `updated_at`: the latter is the ingest write
|
|
94
|
+
* time (`ingest.ts` stamps `new Date()`), so a backfilled store would read every
|
|
95
|
+
* ancient session as "today" (measured — all 2,613 sessions of the production
|
|
96
|
+
* store are stamped within four days). `ended_at`, the content analog of Wake's
|
|
97
|
+
* `updated_at`, moves the recency bucket of 5 of those 2,613 sessions, and the
|
|
98
|
+
* tiebreak already names the start; one timestamp per lane.
|
|
99
|
+
*/
|
|
100
|
+
const RECENCY_DECAY_MS = 2_592_000_000;
|
|
101
|
+
const RECENCY_PRIOR_SQL =
|
|
102
|
+
`(1.0 + 1.0 / (1.0 + max(0, ? - COALESCE(unixepoch(s.started_at) * 1000, 0)) / ${RECENCY_DECAY_MS}.0))`;
|
|
103
|
+
|
|
104
|
+
const COUNT_REASON_LIMIT_BOUNDED = "limit_bounded";
|
|
105
|
+
const COUNT_REASON_EXACT = "all_matches_returned";
|
|
106
|
+
const COUNT_REASON_EMPTY_QUERY = "empty_query";
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* The tombstone filter every serving query carries: a session the user deleted
|
|
110
|
+
* is not served, whichever lane would have found it. Reads are where a deletion
|
|
111
|
+
* becomes visible — the store does not remove the rows a `suppress` tombstone
|
|
112
|
+
* covers (db.ts) — so the filter is one predicate applied to the session column
|
|
113
|
+
* of each lane rather than a rewrite of any lane.
|
|
114
|
+
*
|
|
115
|
+
* The predicate is resolved per call, and skipping it for a store that has no
|
|
116
|
+
* `session_tombstones` table is exact rather than defensive: a store without the
|
|
117
|
+
* table is one no build could have written a tombstone into. The table arrives
|
|
118
|
+
* with the migration that introduces the verb (db.ts, `migrate`), and a store
|
|
119
|
+
* written by this build is refused outright by an older build's read gate
|
|
120
|
+
* (`classifyReadStore`, user_version newer than supported), so a tombstoned
|
|
121
|
+
* store is never read unfiltered. Skipping the predicate there is the same
|
|
122
|
+
* answer as applying it to an empty set — and it is what lets this build keep
|
|
123
|
+
* serving a store written before the table existed, the same tolerance the title
|
|
124
|
+
* lane shows for a store whose format-`"3"` derivation has not run yet. The
|
|
125
|
+
* cost is one `sqlite_master` lookup per serving call; `1` keeps a caller that
|
|
126
|
+
* splices the predicate into an existing `AND` chain from having to branch.
|
|
127
|
+
*/
|
|
128
|
+
function tombstonePredicate(db: Database, sessionColumn: string): string {
|
|
129
|
+
const present = !!db
|
|
130
|
+
.query<{ name: string }, []>(
|
|
131
|
+
"SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'session_tombstones' LIMIT 1;",
|
|
132
|
+
)
|
|
133
|
+
.get();
|
|
134
|
+
return present
|
|
135
|
+
? `NOT EXISTS (SELECT 1 FROM session_tombstones t WHERE t.session_id = ${sessionColumn})`
|
|
136
|
+
: "1";
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/** Options shared by every read query. */
|
|
140
|
+
export interface ReadQueryOptions {
|
|
141
|
+
limit?: number;
|
|
142
|
+
source?: string;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* Widening ceiling, read per query so operators and tests can flip it without
|
|
147
|
+
* restarting a long-lived process. Only a plain non-negative integer counts;
|
|
148
|
+
* anything else (empty, negative, `2.9`, `0x10`, `10k`, `1e3`) falls back to
|
|
149
|
+
* the default, and 0 disables widening.
|
|
150
|
+
*/
|
|
151
|
+
function readWidenMaxRecords(): number {
|
|
152
|
+
const raw = process.env.LEDGER_WIDEN_MAX_RECORDS?.trim();
|
|
153
|
+
if (raw === undefined || !/^\d+$/.test(raw)) return DEFAULT_WIDEN_MAX_RECORDS;
|
|
154
|
+
return Number.parseInt(raw, 10);
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
function alphanumericCount(token: string): number {
|
|
158
|
+
let count = 0;
|
|
159
|
+
for (const ch of token) {
|
|
160
|
+
if (/[A-Za-z0-9]/.test(ch)) count += 1;
|
|
161
|
+
}
|
|
162
|
+
return count;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* Tokens that may become prefix terms. Over-long tokens are dropped and the
|
|
167
|
+
* query widens with the remainder; tokens holding fewer than three
|
|
168
|
+
* alphanumerics are dropped too, because a quoted zero-token phrase followed by
|
|
169
|
+
* `*` is not a no-op in FTS5 (it can match unrelated records). Duplicates
|
|
170
|
+
* collapse case-insensitively before the cap is applied.
|
|
171
|
+
*/
|
|
172
|
+
function widenableTokens(cleanQuery: string): string[] | { skipReason: WideningSkipReason } {
|
|
173
|
+
const sized = cleanQuery.split(/\s+/).filter((t) => t.length <= MAX_WIDEN_TOKEN_CHARS);
|
|
174
|
+
if (sized.length === 0) return { skipReason: "token_too_long" };
|
|
175
|
+
const rich = sized.filter((t) => alphanumericCount(t) >= MIN_WIDEN_TOKEN_ALNUM);
|
|
176
|
+
if (rich.length === 0) return { skipReason: "no_widenable_tokens" };
|
|
177
|
+
const seen = new Set<string>();
|
|
178
|
+
const unique: string[] = [];
|
|
179
|
+
for (const token of rich) {
|
|
180
|
+
const key = token.toLowerCase();
|
|
181
|
+
if (seen.has(key)) continue;
|
|
182
|
+
seen.add(key);
|
|
183
|
+
unique.push(token);
|
|
184
|
+
}
|
|
185
|
+
if (unique.length > MAX_WIDEN_TOKENS) return { skipReason: "query_too_long" };
|
|
186
|
+
return unique;
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/** Engine output before the envelope is attached. */
|
|
190
|
+
interface RawFtsHits {
|
|
191
|
+
hits: RecordSearchHit[];
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/**
|
|
195
|
+
* What the store *holds*, tombstones included — deliberately not what it
|
|
196
|
+
* serves. A `suppress` deletion keeps the session's rows (db.ts), so a count
|
|
197
|
+
* that subtracted them would claim the store holds less than it does and hide
|
|
198
|
+
* the very data the audit keeps; the deletion is disclosed beside these numbers
|
|
199
|
+
* instead (`ledger status`: `tombstones` and `tombstoned_records`). The other
|
|
200
|
+
* reader of this count is the widening ceiling, which bounds a query by the
|
|
201
|
+
* corpus it would scan rather than by the rows it may return.
|
|
202
|
+
*/
|
|
203
|
+
export function countSessions(db: Database): number {
|
|
204
|
+
const row = db.query<{ count: number }, []>("SELECT COUNT(*) AS count FROM sessions").get();
|
|
205
|
+
return row?.count ?? 0;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
/**
|
|
209
|
+
* Whether the store holds any record at all. `EXISTS` stops at the first row
|
|
210
|
+
* (an index seek); counting instead would scan the table on every semantic or
|
|
211
|
+
* hybrid query just to answer a yes/no question.
|
|
212
|
+
*/
|
|
213
|
+
function hasRecords(db: Database, source?: string): boolean {
|
|
214
|
+
return !!db.query<{ present: number }, (string | number)[]>(
|
|
215
|
+
`SELECT EXISTS(SELECT 1 FROM records r JOIN sessions s ON s.id = r.session_id${source ? " WHERE s.source = ?" : ""}) AS present`,
|
|
216
|
+
).get(...(source ? [source] : []))?.present;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/** Records held, tombstones included: the same holdings rule as `countSessions`. */
|
|
220
|
+
export function countRecords(db: Database): number {
|
|
221
|
+
const row = db.query<{ count: number }, []>("SELECT COUNT(*) AS count FROM records").get();
|
|
222
|
+
return row?.count ?? 0;
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
/**
|
|
226
|
+
* Vectors the active provider can actually score. Embeddings written by another
|
|
227
|
+
* provider or dimension are invisible to a semantic query rather than being
|
|
228
|
+
* compared across incompatible spaces.
|
|
229
|
+
*/
|
|
230
|
+
export function compatibleEmbeddingCount(db: Database, source?: string): number {
|
|
231
|
+
const provider = getEmbeddingProvider();
|
|
232
|
+
return db.query<{ count: number }, (string | number)[]>(
|
|
233
|
+
`SELECT COUNT(*) AS count FROM records_embeddings e
|
|
234
|
+
JOIN records r ON r.id = e.record_id
|
|
235
|
+
JOIN sessions s ON s.id = e.session_id
|
|
236
|
+
WHERE e.provider = ? AND e.dim = ?${source ? " AND s.source = ?" : ""}`,
|
|
237
|
+
).get(provider.name, provider.dimension, ...(source ? [source] : []))?.count ?? 0;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
function buildFtsExpression(query: string): string {
|
|
241
|
+
// If the user already provided quotes, respect them
|
|
242
|
+
if (query.startsWith('"') && query.endsWith('"')) {
|
|
243
|
+
return query;
|
|
244
|
+
}
|
|
245
|
+
// Check if query has multiple words without boolean operators
|
|
246
|
+
const hasOperators = BOOLEAN_OPERATOR_RE.test(query);
|
|
247
|
+
if (!hasOperators && query.includes(" ")) {
|
|
248
|
+
// Match phrase or conjunct
|
|
249
|
+
const escaped = query.replace(/"/g, '""');
|
|
250
|
+
return `"${escaped}"`;
|
|
251
|
+
}
|
|
252
|
+
return query;
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
/**
|
|
256
|
+
* The escaped-term rewrite of a query whose own expression FTS5 rejected:
|
|
257
|
+
* every whitespace token as a quoted term, so punctuation cannot be read as
|
|
258
|
+
* syntax. Both lanes call this one implementation because they are handed the
|
|
259
|
+
* same expression — a query the fallback can still answer must keep whatever
|
|
260
|
+
* its titles matched, so the rewrite cannot drift between the two.
|
|
261
|
+
*/
|
|
262
|
+
function escapedTermExpression(cleanQuery: string): string {
|
|
263
|
+
return cleanQuery
|
|
264
|
+
.split(/\s+/)
|
|
265
|
+
.filter(Boolean)
|
|
266
|
+
.map((t) => `"${t.replace(/"/g, '""')}"`)
|
|
267
|
+
.join(" ");
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
function executeFtsQuery(
|
|
271
|
+
db: Database,
|
|
272
|
+
ftsExpr: string,
|
|
273
|
+
limit: number,
|
|
274
|
+
now: number,
|
|
275
|
+
sourceFilter?: string,
|
|
276
|
+
matchType: RecordMatchType = "lexical",
|
|
277
|
+
): RawFtsHits {
|
|
278
|
+
const sourceClause = sourceFilter ? "AND s.source = ?" : "";
|
|
279
|
+
// Parameter order follows the SQL text, and the recency prior's `?` sits in
|
|
280
|
+
// the SELECT list — ahead of the MATCH expression, the source filter and the
|
|
281
|
+
// limit.
|
|
282
|
+
const params: (string | number)[] = sourceFilter
|
|
283
|
+
? [now, ftsExpr, sourceFilter, limit]
|
|
284
|
+
: [now, ftsExpr, limit];
|
|
285
|
+
|
|
286
|
+
const sql = `
|
|
287
|
+
SELECT
|
|
288
|
+
f.record_id,
|
|
289
|
+
f.session_id,
|
|
290
|
+
s.source,
|
|
291
|
+
s.title,
|
|
292
|
+
s.cwd,
|
|
293
|
+
s.started_at,
|
|
294
|
+
s.parent_session_id,
|
|
295
|
+
s.agent_nickname,
|
|
296
|
+
s.thread_role,
|
|
297
|
+
f.record_type,
|
|
298
|
+
f.content,
|
|
299
|
+
f.tool_name,
|
|
300
|
+
f.tool_arguments,
|
|
301
|
+
f.tool_result,
|
|
302
|
+
snippet(records_fts, 4, '«', '»', '...', 20) AS content_snippet,
|
|
303
|
+
snippet(records_fts, 6, '«', '»', '...', 20) AS args_snippet,
|
|
304
|
+
snippet(records_fts, 7, '«', '»', '...', 20) AS res_snippet,
|
|
305
|
+
bm25(records_fts) * ${RECENCY_PRIOR_SQL} AS rank
|
|
306
|
+
FROM records_fts f
|
|
307
|
+
JOIN sessions s ON s.id = f.session_id
|
|
308
|
+
WHERE records_fts MATCH ?
|
|
309
|
+
${sourceClause}
|
|
310
|
+
AND ${tombstonePredicate(db, "s.id")}
|
|
311
|
+
ORDER BY rank ASC, s.started_at DESC
|
|
312
|
+
LIMIT ?
|
|
313
|
+
`;
|
|
314
|
+
|
|
315
|
+
interface RawHit {
|
|
316
|
+
record_id: string;
|
|
317
|
+
session_id: string;
|
|
318
|
+
source: string;
|
|
319
|
+
title: string | null;
|
|
320
|
+
cwd: string | null;
|
|
321
|
+
started_at: string | null;
|
|
322
|
+
parent_session_id: string | null;
|
|
323
|
+
agent_nickname: string | null;
|
|
324
|
+
thread_role: string | null;
|
|
325
|
+
record_type: string;
|
|
326
|
+
content: string | null;
|
|
327
|
+
tool_name: string | null;
|
|
328
|
+
tool_arguments: string | null;
|
|
329
|
+
tool_result: string | null;
|
|
330
|
+
content_snippet: string;
|
|
331
|
+
args_snippet: string;
|
|
332
|
+
res_snippet: string;
|
|
333
|
+
rank: number;
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
const rows = db.query<RawHit, (string | number)[]>(sql).all(...params);
|
|
337
|
+
|
|
338
|
+
const hits: RecordSearchHit[] = [];
|
|
339
|
+
|
|
340
|
+
for (const r of rows) {
|
|
341
|
+
let snippet = r.content_snippet;
|
|
342
|
+
if (!snippet || !snippet.includes("«")) {
|
|
343
|
+
if (r.args_snippet && r.args_snippet.includes("«")) {
|
|
344
|
+
snippet = r.args_snippet;
|
|
345
|
+
} else if (r.res_snippet && r.res_snippet.includes("«")) {
|
|
346
|
+
snippet = r.res_snippet;
|
|
347
|
+
} else {
|
|
348
|
+
snippet = (r.content ?? r.tool_arguments ?? r.tool_result ?? "").slice(0, 200);
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
hits.push({
|
|
353
|
+
session_id: r.session_id,
|
|
354
|
+
record_id: r.record_id,
|
|
355
|
+
source: r.source,
|
|
356
|
+
title: r.title,
|
|
357
|
+
cwd: r.cwd,
|
|
358
|
+
started_at: r.started_at,
|
|
359
|
+
record_type: r.record_type,
|
|
360
|
+
parent_session_id: r.parent_session_id,
|
|
361
|
+
agent_nickname: r.agent_nickname,
|
|
362
|
+
thread_role: r.thread_role,
|
|
363
|
+
content: r.content,
|
|
364
|
+
tool_name: r.tool_name,
|
|
365
|
+
tool_arguments: r.tool_arguments,
|
|
366
|
+
tool_result: r.tool_result,
|
|
367
|
+
snippet,
|
|
368
|
+
rank: r.rank,
|
|
369
|
+
match_type: matchType,
|
|
370
|
+
});
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
return { hits };
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
/**
|
|
377
|
+
* Prefix-OR widening for a lexical query that returned nothing. Gates: the
|
|
378
|
+
* query must not express phrase intent anywhere, must not use boolean
|
|
379
|
+
* operators or its own wildcard, must leave at least one widenable token, and
|
|
380
|
+
* the corpus must sit under the configured ceiling. Adoption requires
|
|
381
|
+
* strictly more hits than the primary query's zero.
|
|
382
|
+
*/
|
|
383
|
+
function widenZeroHitQuery(
|
|
384
|
+
db: Database,
|
|
385
|
+
cleanQuery: string,
|
|
386
|
+
limit: number,
|
|
387
|
+
now: number,
|
|
388
|
+
sourceFilter?: string,
|
|
389
|
+
): RawFtsHits | { skipReason: WideningSkipReason } {
|
|
390
|
+
const maxRecords = readWidenMaxRecords();
|
|
391
|
+
if (maxRecords === 0) return { skipReason: "disabled" };
|
|
392
|
+
// Any quote, not just a wrapping pair: `check "a b" field` is phrase intent.
|
|
393
|
+
if (cleanQuery.includes('"')) return { skipReason: "quoted_input" };
|
|
394
|
+
if (BOOLEAN_OPERATOR_RE.test(cleanQuery)) return { skipReason: "operators_present" };
|
|
395
|
+
if (cleanQuery.includes("*")) return { skipReason: "wildcard_present" };
|
|
396
|
+
|
|
397
|
+
const tokens = widenableTokens(cleanQuery);
|
|
398
|
+
if (!Array.isArray(tokens)) return tokens;
|
|
399
|
+
// The ceiling counts every record in the store, not only those matching
|
|
400
|
+
// `--source`, so a filtered search cannot dodge the corpus-size guard.
|
|
401
|
+
if (countRecords(db) > maxRecords) return { skipReason: "corpus_over_ceiling" };
|
|
402
|
+
|
|
403
|
+
// Quoted phrase-prefix terms: equivalent to `tok*` for plain words and
|
|
404
|
+
// safe for terms whose punctuation would otherwise be read as FTS5 syntax.
|
|
405
|
+
const widenedExpr = `(${tokens.map((t) => `"${t.replace(/"/g, '""')}"*`).join(" OR ")})`;
|
|
406
|
+
let widened: RawFtsHits;
|
|
407
|
+
try {
|
|
408
|
+
widened = executeFtsQuery(db, widenedExpr, limit, now, sourceFilter, "implicit_wildcard");
|
|
409
|
+
} catch {
|
|
410
|
+
// Defensive: the token gates make an unparsable widening expression
|
|
411
|
+
// unreachable today, and the catch keeps a future change from turning
|
|
412
|
+
// that into a thrown query.
|
|
413
|
+
return { skipReason: "widened_query_failed" };
|
|
414
|
+
}
|
|
415
|
+
return widened.hits.length > 0 ? widened : { skipReason: "no_improvement" };
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
/**
|
|
419
|
+
* The title lane: one hit per session whose title matches, best first. The
|
|
420
|
+
* title is a session's own metadata, so a hit here names the session and no
|
|
421
|
+
* record (see `TitleSearchHit`).
|
|
422
|
+
*
|
|
423
|
+
* Ordered by relevance under the same recency prior the record lane applies —
|
|
424
|
+
* `bm25(titles_fts) × (1 + 1/(1 + age/H))`, then `s.started_at DESC` — which is
|
|
425
|
+
* Wake's ordering for the same lane (`crates/wake-core/src/db.rs#L1676-L1700`)
|
|
426
|
+
* and the same term the record lane's `rank` carries.
|
|
427
|
+
*/
|
|
428
|
+
function titlePrepass(
|
|
429
|
+
db: Database,
|
|
430
|
+
ftsExpr: string,
|
|
431
|
+
limit: number,
|
|
432
|
+
now: number,
|
|
433
|
+
sourceFilter?: string,
|
|
434
|
+
): TitleSearchHit[] {
|
|
435
|
+
const sourceClause = sourceFilter ? "AND s.source = ?" : "";
|
|
436
|
+
const params: (string | number)[] = sourceFilter
|
|
437
|
+
? [now, ftsExpr, sourceFilter, limit]
|
|
438
|
+
: [now, ftsExpr, limit];
|
|
439
|
+
|
|
440
|
+
const sql = `
|
|
441
|
+
SELECT
|
|
442
|
+
f.session_id,
|
|
443
|
+
s.source,
|
|
444
|
+
s.title,
|
|
445
|
+
s.cwd,
|
|
446
|
+
s.started_at,
|
|
447
|
+
s.parent_session_id,
|
|
448
|
+
s.agent_nickname,
|
|
449
|
+
s.thread_role,
|
|
450
|
+
highlight(titles_fts, 1, '«', '»') AS title_snippet,
|
|
451
|
+
bm25(titles_fts) * ${RECENCY_PRIOR_SQL} AS rank
|
|
452
|
+
FROM titles_fts f
|
|
453
|
+
JOIN sessions s ON s.id = f.session_id
|
|
454
|
+
WHERE titles_fts MATCH ?
|
|
455
|
+
${sourceClause}
|
|
456
|
+
AND ${tombstonePredicate(db, "s.id")}
|
|
457
|
+
ORDER BY rank ASC, s.started_at DESC
|
|
458
|
+
LIMIT ?
|
|
459
|
+
`;
|
|
460
|
+
|
|
461
|
+
interface TitleRow {
|
|
462
|
+
session_id: string;
|
|
463
|
+
source: string;
|
|
464
|
+
title: string;
|
|
465
|
+
cwd: string | null;
|
|
466
|
+
started_at: string | null;
|
|
467
|
+
parent_session_id: string | null;
|
|
468
|
+
agent_nickname: string | null;
|
|
469
|
+
thread_role: string | null;
|
|
470
|
+
title_snippet: string;
|
|
471
|
+
rank: number;
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
// The whole title is the snippet — it is short by construction (the
|
|
475
|
+
// normalizer cuts a first-user-message title at 120 chars) and truncating it
|
|
476
|
+
// to a window around the match would hide the session's own name from the
|
|
477
|
+
// answer that matched it.
|
|
478
|
+
return db
|
|
479
|
+
.query<TitleRow, (string | number)[]>(sql)
|
|
480
|
+
.all(...params)
|
|
481
|
+
.map((row) => ({
|
|
482
|
+
session_id: row.session_id,
|
|
483
|
+
record_id: null,
|
|
484
|
+
source: row.source,
|
|
485
|
+
title: row.title,
|
|
486
|
+
cwd: row.cwd,
|
|
487
|
+
started_at: row.started_at,
|
|
488
|
+
record_type: "title",
|
|
489
|
+
content: null,
|
|
490
|
+
tool_name: null,
|
|
491
|
+
tool_arguments: null,
|
|
492
|
+
tool_result: null,
|
|
493
|
+
snippet: row.title_snippet.slice(0, 240),
|
|
494
|
+
rank: row.rank,
|
|
495
|
+
parent_session_id: row.parent_session_id,
|
|
496
|
+
agent_nickname: row.agent_nickname,
|
|
497
|
+
thread_role: row.thread_role,
|
|
498
|
+
match_type: "title",
|
|
499
|
+
}));
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
function buildQueryResult(
|
|
503
|
+
query: string,
|
|
504
|
+
hits: SearchHit[],
|
|
505
|
+
mode: SearchMode,
|
|
506
|
+
limit: number,
|
|
507
|
+
startedAt: number,
|
|
508
|
+
extras: {
|
|
509
|
+
fallbackReason?: string | null;
|
|
510
|
+
widened?: WideningReport | null;
|
|
511
|
+
countReason?: string;
|
|
512
|
+
/** The mode the caller asked for when `mode` is the engine that answered. */
|
|
513
|
+
requestedMode?: SearchMode;
|
|
514
|
+
} = {},
|
|
515
|
+
): QueryResult {
|
|
516
|
+
const bounded = hits.length >= limit;
|
|
517
|
+
const sessions = new Set<string>();
|
|
518
|
+
for (const hit of hits) sessions.add(hit.session_id);
|
|
519
|
+
return {
|
|
520
|
+
query,
|
|
521
|
+
totalHits: hits.length,
|
|
522
|
+
sessionsCount: sessions.size,
|
|
523
|
+
hits,
|
|
524
|
+
meta: {
|
|
525
|
+
count_precision: bounded ? "lower_bound" : "exact",
|
|
526
|
+
count_reason: extras.countReason ?? (bounded ? COUNT_REASON_LIMIT_BOUNDED : COUNT_REASON_EXACT),
|
|
527
|
+
elapsed_ms: Math.round((performance.now() - startedAt) * 1000) / 1000,
|
|
528
|
+
search_mode: mode,
|
|
529
|
+
// Equal except when an engine fell back, which is exactly when an agent
|
|
530
|
+
// must not have to infer which one answered the query.
|
|
531
|
+
requested_search_mode: extras.requestedMode ?? mode,
|
|
532
|
+
fallback_reason: extras.fallbackReason ?? null,
|
|
533
|
+
widened: extras.widened ?? null,
|
|
534
|
+
},
|
|
535
|
+
};
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
export function queryLexical(
|
|
539
|
+
db: Database,
|
|
540
|
+
queryText: string,
|
|
541
|
+
options: ReadQueryOptions = {},
|
|
542
|
+
): QueryResult {
|
|
543
|
+
const startedAt = performance.now();
|
|
544
|
+
const limit = options.limit ?? 50;
|
|
545
|
+
const cleanQuery = queryText.trim();
|
|
546
|
+
if (!cleanQuery) {
|
|
547
|
+
return buildQueryResult(queryText, [], "lexical", limit, startedAt, {
|
|
548
|
+
countReason: COUNT_REASON_EMPTY_QUERY,
|
|
549
|
+
});
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
const ftsExpr = buildFtsExpression(cleanQuery);
|
|
553
|
+
|
|
554
|
+
// One clock for both lanes: the recency prior weights a title hit and a
|
|
555
|
+
// record hit of the same session identically, and a query cannot read two
|
|
556
|
+
// different "now"s (Wake's `now_ms()`, `db.rs#L1553`).
|
|
557
|
+
const now = Date.now();
|
|
558
|
+
|
|
559
|
+
// The title lane runs first and its hits are emitted ahead of the record
|
|
560
|
+
// lane's: a word in a session's title is the strongest relevance signal a
|
|
561
|
+
// session-level search has, and the session's *name* is what a human is
|
|
562
|
+
// usually quoting (Wake orders the same way, db.rs#L1676-L1700). It costs
|
|
563
|
+
// nothing when the store has no title index — a store whose format-`"3"`
|
|
564
|
+
// derivation has not run yet — and a title that cannot be read must not turn
|
|
565
|
+
// a query the record lane can still answer into a thrown one.
|
|
566
|
+
const titlesIndexed = !!db
|
|
567
|
+
.query<{ name: string }, []>(
|
|
568
|
+
"SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'titles_fts' LIMIT 1;",
|
|
569
|
+
)
|
|
570
|
+
.get();
|
|
571
|
+
const titleLane = (expr: string): TitleSearchHit[] => {
|
|
572
|
+
if (!titlesIndexed) return [];
|
|
573
|
+
try {
|
|
574
|
+
return titlePrepass(db, expr, limit, now, options.source);
|
|
575
|
+
} catch {
|
|
576
|
+
// The record lane decides whether this expression parses; whatever its
|
|
577
|
+
// answer is, the title lane gets no vote.
|
|
578
|
+
return [];
|
|
579
|
+
}
|
|
580
|
+
};
|
|
581
|
+
|
|
582
|
+
let titles = titleLane(ftsExpr);
|
|
583
|
+
|
|
584
|
+
// The result as a whole is bounded by the limit, so the title hits come out
|
|
585
|
+
// of the record lane's budget rather than on top of it: `totalHits` has to
|
|
586
|
+
// stay what `meta.count_precision` describes.
|
|
587
|
+
let raw: RawFtsHits;
|
|
588
|
+
let fallbackReason: string | null = null;
|
|
589
|
+
if (titles.length >= limit) {
|
|
590
|
+
raw = { hits: [] };
|
|
591
|
+
} else {
|
|
592
|
+
try {
|
|
593
|
+
raw = executeFtsQuery(db, ftsExpr, limit - titles.length, now, options.source);
|
|
594
|
+
} catch {
|
|
595
|
+
// Fallback: search with escaped individual terms — and re-ask the title
|
|
596
|
+
// lane with them, so both lanes answer one query rather than two.
|
|
597
|
+
fallbackReason = FTS_SYNTAX_FALLBACK_REASON;
|
|
598
|
+
const fallbackExpr = escapedTermExpression(cleanQuery);
|
|
599
|
+
titles = titleLane(fallbackExpr);
|
|
600
|
+
raw = executeFtsQuery(db, fallbackExpr, limit - titles.length, now, options.source);
|
|
601
|
+
}
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
let hits: SearchHit[] = [...titles, ...raw.hits];
|
|
605
|
+
|
|
606
|
+
// Widening is a zero-hit recovery of the *query*, so it runs only when both
|
|
607
|
+
// lanes found nothing: it neither costs nor changes anything on the happy
|
|
608
|
+
// path, which is why the corpus-size probe only runs below.
|
|
609
|
+
let widened: WideningReport | null = null;
|
|
610
|
+
if (hits.length === 0) {
|
|
611
|
+
const attempt = widenZeroHitQuery(db, cleanQuery, limit, now, options.source);
|
|
612
|
+
if ("hits" in attempt) {
|
|
613
|
+
hits = attempt.hits;
|
|
614
|
+
widened = { applied: true, original_hits: 0, widened_hits: attempt.hits.length };
|
|
615
|
+
} else {
|
|
616
|
+
widened = { applied: false, skip_reason: attempt.skipReason };
|
|
617
|
+
}
|
|
618
|
+
}
|
|
619
|
+
|
|
620
|
+
return buildQueryResult(cleanQuery, hits, "lexical", limit, startedAt, {
|
|
621
|
+
fallbackReason,
|
|
622
|
+
widened,
|
|
623
|
+
});
|
|
624
|
+
}
|
|
625
|
+
|
|
626
|
+
/**
|
|
627
|
+
* Rewrite a lexical result as the honest answer of a query whose vector leg
|
|
628
|
+
* could not run: the hits come from the lexical engine, so the envelope says
|
|
629
|
+
* that, names the mode that was requested, and reports the reason. The lexical
|
|
630
|
+
* answer includes its title lane — the fallback *is* that engine, so its
|
|
631
|
+
* session-level hits stay in it.
|
|
632
|
+
*/
|
|
633
|
+
function lexicalFallback(
|
|
634
|
+
result: QueryResult,
|
|
635
|
+
limit: number,
|
|
636
|
+
requested: SearchMode,
|
|
637
|
+
startedAt: number,
|
|
638
|
+
): QueryResult {
|
|
639
|
+
const hits = result.hits.slice(0, limit);
|
|
640
|
+
return buildQueryResult(result.query, hits, "lexical", limit, startedAt, {
|
|
641
|
+
requestedMode: requested,
|
|
642
|
+
fallbackReason: EMBEDDINGS_UNAVAILABLE_REASON,
|
|
643
|
+
widened: result.meta.widened,
|
|
644
|
+
});
|
|
645
|
+
}
|
|
646
|
+
|
|
647
|
+
/**
|
|
648
|
+
* Embed one query string with the given provider. Gemini needs its query-mode
|
|
649
|
+
* formatting, which is a per-call concern of this path rather than a rewritten
|
|
650
|
+
* provider: the flag is put back even when embedding fails.
|
|
651
|
+
*/
|
|
652
|
+
async function embedQueryVector(
|
|
653
|
+
provider: EmbeddingProvider,
|
|
654
|
+
text: string,
|
|
655
|
+
): Promise<Float32Array | undefined> {
|
|
656
|
+
const isGemini = "queryMode" in provider;
|
|
657
|
+
if (isGemini) {
|
|
658
|
+
(provider as GeminiEmbeddingProvider).queryMode = true;
|
|
659
|
+
}
|
|
660
|
+
try {
|
|
661
|
+
const [vector] = await embed([text]);
|
|
662
|
+
return vector;
|
|
663
|
+
} finally {
|
|
664
|
+
if (isGemini) {
|
|
665
|
+
(provider as GeminiEmbeddingProvider).queryMode = false;
|
|
666
|
+
}
|
|
667
|
+
}
|
|
668
|
+
}
|
|
669
|
+
|
|
670
|
+
/** The vector leg's answer before the envelope is attached. */
|
|
671
|
+
interface RawVectorHits {
|
|
672
|
+
hits: RecordSearchHit[];
|
|
673
|
+
}
|
|
674
|
+
|
|
675
|
+
/**
|
|
676
|
+
* Vector search over the embeddings already in the store, or null when the leg
|
|
677
|
+
* cannot run. A read path never backfills: with no compatible vectors, or a
|
|
678
|
+
* provider that cannot embed the query (no key, no model files, a failing
|
|
679
|
+
* endpoint), the caller answers lexically and says so instead of returning an
|
|
680
|
+
* empty result that looks like a miss.
|
|
681
|
+
*/
|
|
682
|
+
async function vectorSearch(
|
|
683
|
+
db: Database,
|
|
684
|
+
cleanQuery: string,
|
|
685
|
+
options: { limit: number; source?: string },
|
|
686
|
+
): Promise<RawVectorHits | null> {
|
|
687
|
+
const { limit } = options;
|
|
688
|
+
const provider = getEmbeddingProvider();
|
|
689
|
+
if (compatibleEmbeddingCount(db, options.source) === 0) return null;
|
|
690
|
+
|
|
691
|
+
let queryVec: Float32Array | undefined;
|
|
692
|
+
try {
|
|
693
|
+
queryVec = await embedQueryVector(provider, cleanQuery);
|
|
694
|
+
} catch {
|
|
695
|
+
return null;
|
|
696
|
+
}
|
|
697
|
+
// A provider that hands back a vector of a different length than it
|
|
698
|
+
// advertises cannot score the stored vectors: every comparison would read as
|
|
699
|
+
// zero similarity, which is a silent miss rather than an answer.
|
|
700
|
+
if (!queryVec || queryVec.length !== provider.dimension) return null;
|
|
701
|
+
|
|
702
|
+
const conditions: string[] = ["e.provider = ?", "e.dim = ?"];
|
|
703
|
+
const params: unknown[] = [provider.name, provider.dimension];
|
|
704
|
+
if (options.source) {
|
|
705
|
+
conditions.push("s.source = ?");
|
|
706
|
+
params.push(options.source);
|
|
707
|
+
}
|
|
708
|
+
// The vector lane carries the same tombstone filter as the lexical lanes: a
|
|
709
|
+
// retained vector of a deleted session is not an answer (and a store whose
|
|
710
|
+
// vectors are all tombstoned answers with no hits in this mode rather than
|
|
711
|
+
// falling back, so the envelope keeps naming the engine that ran).
|
|
712
|
+
conditions.push(tombstonePredicate(db, "s.id"));
|
|
713
|
+
const sourceClause = `WHERE ${conditions.join(" AND ")}`;
|
|
714
|
+
|
|
715
|
+
interface EmbeddingRow {
|
|
716
|
+
record_id: string;
|
|
717
|
+
session_id: string;
|
|
718
|
+
source: string;
|
|
719
|
+
title: string | null;
|
|
720
|
+
cwd: string | null;
|
|
721
|
+
started_at: string | null;
|
|
722
|
+
parent_session_id: string | null;
|
|
723
|
+
agent_nickname: string | null;
|
|
724
|
+
thread_role: string | null;
|
|
725
|
+
record_type: string;
|
|
726
|
+
content: string | null;
|
|
727
|
+
tool_name: string | null;
|
|
728
|
+
tool_arguments_json: string | null;
|
|
729
|
+
tool_result_json: string | null;
|
|
730
|
+
embedding: Uint8Array;
|
|
731
|
+
}
|
|
732
|
+
|
|
733
|
+
const sql = `
|
|
734
|
+
SELECT
|
|
735
|
+
e.record_id,
|
|
736
|
+
e.session_id,
|
|
737
|
+
s.source,
|
|
738
|
+
s.title,
|
|
739
|
+
s.cwd,
|
|
740
|
+
s.started_at,
|
|
741
|
+
s.parent_session_id,
|
|
742
|
+
s.agent_nickname,
|
|
743
|
+
s.thread_role,
|
|
744
|
+
r.record_type,
|
|
745
|
+
r.content,
|
|
746
|
+
r.tool_name,
|
|
747
|
+
r.tool_arguments_json,
|
|
748
|
+
r.tool_result_json,
|
|
749
|
+
e.embedding
|
|
750
|
+
FROM records_embeddings e
|
|
751
|
+
JOIN sessions s ON s.id = e.session_id
|
|
752
|
+
JOIN records r ON r.id = e.record_id
|
|
753
|
+
${sourceClause}
|
|
754
|
+
`;
|
|
755
|
+
|
|
756
|
+
// Pure cosine, deliberately: the recency prior is the lexical lanes' (a
|
|
757
|
+
// multiplier on bm25, above). Cosine is a bounded, dense similarity whose
|
|
758
|
+
// whole top-50 span measures 1.8–2.6× on this store's own text (shipped
|
|
759
|
+
// MiniLM, 300 real records: r1/r50 = 2.19/1.77/2.60), so the same ×1–×2
|
|
760
|
+
// factor would order semantic hits by age nearly regardless of their text;
|
|
761
|
+
// and a fused hit already takes its recency from the lexical leg's position,
|
|
762
|
+
// which is what RRF reads. The lane's reported `rank` stays a cosine.
|
|
763
|
+
//
|
|
764
|
+
// A deferred snapshot keeps scoring and winner hydration coherent while a
|
|
765
|
+
// writer commits. Query embedding above awaits outside this transaction.
|
|
766
|
+
// Exact top-K distinct sessions: one hit per session (its best-scoring
|
|
767
|
+
// record). Without this, a single long session crowds out the whole top-N
|
|
768
|
+
// with near-identical records. Space is O(K); scoring remains exhaustive.
|
|
769
|
+
const topHits = db.transaction(() => {
|
|
770
|
+
const top = new SemanticTopK(limit);
|
|
771
|
+
let ordinal = 0;
|
|
772
|
+
const rows = db.query<{ record_id: string; session_id: string; embedding: Uint8Array }, (string | number)[]>(`
|
|
773
|
+
SELECT e.record_id, e.session_id, e.embedding
|
|
774
|
+
FROM records_embeddings e
|
|
775
|
+
JOIN sessions s ON s.id = e.session_id
|
|
776
|
+
JOIN records r ON r.id = e.record_id
|
|
777
|
+
${sourceClause}
|
|
778
|
+
`).iterate(...(params as (string | number)[]));
|
|
779
|
+
for (const row of rows) {
|
|
780
|
+
const vector = deserializeEmbedding(row.embedding);
|
|
781
|
+
if (vector.length !== provider.dimension) continue;
|
|
782
|
+
top.offer({ record_id: row.record_id, session_id: row.session_id,
|
|
783
|
+
score: cosineSimilarity(queryVec!, vector), ordinal: ordinal++ });
|
|
784
|
+
}
|
|
785
|
+
// Only the winning K records bring transcript bodies into memory. Scoring
|
|
786
|
+
// remains exhaustive O(N * dimension); --limit bounds state, not CPU work.
|
|
787
|
+
const hydrate = db.query<EmbeddingRow, (string | number)[]>(sql + " AND e.record_id = ?");
|
|
788
|
+
return top.sorted().map(candidate => {
|
|
789
|
+
const row = hydrate.get(...(params as (string | number)[]), candidate.record_id);
|
|
790
|
+
if (!row) throw new Error("Semantic winner disappeared within its read snapshot");
|
|
791
|
+
return { row, score: candidate.score };
|
|
792
|
+
});
|
|
793
|
+
}).deferred();
|
|
794
|
+
const hits: RecordSearchHit[] = [];
|
|
795
|
+
|
|
796
|
+
for (const { row, score } of topHits) {
|
|
797
|
+
const snippet = (
|
|
798
|
+
row.content ??
|
|
799
|
+
row.tool_arguments_json ??
|
|
800
|
+
row.tool_result_json ??
|
|
801
|
+
""
|
|
802
|
+
).slice(0, 240);
|
|
803
|
+
hits.push({
|
|
804
|
+
session_id: row.session_id,
|
|
805
|
+
record_id: row.record_id,
|
|
806
|
+
source: row.source,
|
|
807
|
+
title: row.title,
|
|
808
|
+
cwd: row.cwd,
|
|
809
|
+
started_at: row.started_at,
|
|
810
|
+
record_type: row.record_type,
|
|
811
|
+
content: row.content,
|
|
812
|
+
tool_name: row.tool_name,
|
|
813
|
+
tool_arguments: row.tool_arguments_json,
|
|
814
|
+
tool_result: row.tool_result_json,
|
|
815
|
+
snippet,
|
|
816
|
+
rank: score,
|
|
817
|
+
parent_session_id: row.parent_session_id,
|
|
818
|
+
agent_nickname: row.agent_nickname,
|
|
819
|
+
thread_role: row.thread_role,
|
|
820
|
+
match_type: "semantic",
|
|
821
|
+
});
|
|
822
|
+
}
|
|
823
|
+
|
|
824
|
+
return { hits };
|
|
825
|
+
}
|
|
826
|
+
|
|
827
|
+
/**
|
|
828
|
+
* Semantic search: the vector leg when it can run, the lexical engine plus an
|
|
829
|
+
* `embeddings_unavailable` report when it cannot.
|
|
830
|
+
*/
|
|
831
|
+
export async function querySemantic(
|
|
832
|
+
db: Database,
|
|
833
|
+
queryText: string,
|
|
834
|
+
options: ReadQueryOptions = {},
|
|
835
|
+
): Promise<QueryResult> {
|
|
836
|
+
const startedAt = performance.now();
|
|
837
|
+
const limit = options.limit ?? 20;
|
|
838
|
+
const cleanQuery = queryText.trim();
|
|
839
|
+
if (!cleanQuery) {
|
|
840
|
+
return buildQueryResult(queryText, [], "semantic", limit, startedAt, {
|
|
841
|
+
countReason: COUNT_REASON_EMPTY_QUERY,
|
|
842
|
+
});
|
|
843
|
+
}
|
|
844
|
+
|
|
845
|
+
// Nothing stored means nothing fell back: the vector leg is not missing
|
|
846
|
+
// coverage, the corpus is empty.
|
|
847
|
+
if (!hasRecords(db, options.source)) {
|
|
848
|
+
return buildQueryResult(cleanQuery, [], "semantic", limit, startedAt);
|
|
849
|
+
}
|
|
850
|
+
|
|
851
|
+
const vector = await vectorSearch(db, cleanQuery, { limit, source: options.source });
|
|
852
|
+
if (!vector) {
|
|
853
|
+
return lexicalFallback(
|
|
854
|
+
queryLexical(db, cleanQuery, { limit, source: options.source }),
|
|
855
|
+
limit,
|
|
856
|
+
"semantic",
|
|
857
|
+
startedAt,
|
|
858
|
+
);
|
|
859
|
+
}
|
|
860
|
+
return buildQueryResult(cleanQuery, vector.hits, "semantic", limit, startedAt);
|
|
861
|
+
}
|
|
862
|
+
|
|
863
|
+
/**
|
|
864
|
+
* Reciprocal-Rank fusion of the lexical and vector legs. When the vector leg
|
|
865
|
+
* cannot run, fusion has nothing to add: the lexical answer is returned with
|
|
866
|
+
* the envelope naming the requested mode and the reason, so `--hybrid` never
|
|
867
|
+
* pretends two engines ran.
|
|
868
|
+
*/
|
|
869
|
+
export async function queryHybrid(
|
|
870
|
+
db: Database,
|
|
871
|
+
queryText: string,
|
|
872
|
+
options: ReadQueryOptions = {},
|
|
873
|
+
): Promise<QueryResult> {
|
|
874
|
+
const startedAt = performance.now();
|
|
875
|
+
const limit = options.limit ?? 20;
|
|
876
|
+
const cleanQuery = queryText.trim();
|
|
877
|
+
if (!cleanQuery) {
|
|
878
|
+
return buildQueryResult(queryText, [], "hybrid", limit, startedAt, {
|
|
879
|
+
countReason: COUNT_REASON_EMPTY_QUERY,
|
|
880
|
+
});
|
|
881
|
+
}
|
|
882
|
+
|
|
883
|
+
const candidateLimit = Math.max(limit * 3, 60);
|
|
884
|
+
|
|
885
|
+
// Nothing stored means nothing fell back: the vector leg is not missing
|
|
886
|
+
// coverage, the corpus is empty.
|
|
887
|
+
if (!hasRecords(db, options.source)) {
|
|
888
|
+
return buildQueryResult(cleanQuery, [], "hybrid", limit, startedAt);
|
|
889
|
+
}
|
|
890
|
+
|
|
891
|
+
const lexicalResult = queryLexical(db, cleanQuery, {
|
|
892
|
+
limit: candidateLimit,
|
|
893
|
+
source: options.source,
|
|
894
|
+
});
|
|
895
|
+
|
|
896
|
+
const vector = await vectorSearch(db, cleanQuery, {
|
|
897
|
+
limit: candidateLimit,
|
|
898
|
+
source: options.source,
|
|
899
|
+
});
|
|
900
|
+
|
|
901
|
+
if (!vector) {
|
|
902
|
+
return lexicalFallback(lexicalResult, limit, "hybrid", startedAt);
|
|
903
|
+
}
|
|
904
|
+
|
|
905
|
+
// The title lane precedes the fusion instead of entering it: a title hit is
|
|
906
|
+
// session-level and has no record id to be keyed by, and the lane it comes
|
|
907
|
+
// from is not an engine the fusion combines but the lexical engine's own
|
|
908
|
+
// first pass. The fused hits fill what the titles leave of the limit, so the
|
|
909
|
+
// result stays bounded by it.
|
|
910
|
+
const isTitleHit = (hit: SearchHit): hit is TitleSearchHit => hit.match_type === "title";
|
|
911
|
+
const isRecordHit = (hit: SearchHit): hit is RecordSearchHit => hit.match_type !== "title";
|
|
912
|
+
const titles = lexicalResult.hits.filter(isTitleHit).slice(0, limit);
|
|
913
|
+
const recordHits = lexicalResult.hits.filter(isRecordHit);
|
|
914
|
+
|
|
915
|
+
const RRF_K = 60;
|
|
916
|
+
const scoredMap = new Map<
|
|
917
|
+
string,
|
|
918
|
+
{
|
|
919
|
+
hit: RecordSearchHit;
|
|
920
|
+
rrfScore: number;
|
|
921
|
+
lexicalRank?: number;
|
|
922
|
+
semanticRank?: number;
|
|
923
|
+
}
|
|
924
|
+
>();
|
|
925
|
+
|
|
926
|
+
// The record leg keeps the ranks the lexical lane gave its records. The
|
|
927
|
+
// title hits it emitted first shift every one of them by the same offset, so
|
|
928
|
+
// the order within the leg — which is all RRF reads — is unchanged.
|
|
929
|
+
for (let i = 0; i < recordHits.length; i++) {
|
|
930
|
+
const hit = recordHits[i];
|
|
931
|
+
const rank = i + 1;
|
|
932
|
+
const rrf = 1 / (RRF_K + rank);
|
|
933
|
+
scoredMap.set(hit.record_id, {
|
|
934
|
+
hit,
|
|
935
|
+
rrfScore: rrf,
|
|
936
|
+
lexicalRank: rank,
|
|
937
|
+
});
|
|
938
|
+
}
|
|
939
|
+
|
|
940
|
+
for (let i = 0; i < vector.hits.length; i++) {
|
|
941
|
+
const hit = vector.hits[i];
|
|
942
|
+
const rank = i + 1;
|
|
943
|
+
const rrf = 1 / (RRF_K + rank);
|
|
944
|
+
const existing = scoredMap.get(hit.record_id);
|
|
945
|
+
if (existing) {
|
|
946
|
+
existing.rrfScore += rrf;
|
|
947
|
+
existing.semanticRank = rank;
|
|
948
|
+
} else {
|
|
949
|
+
scoredMap.set(hit.record_id, {
|
|
950
|
+
hit,
|
|
951
|
+
rrfScore: rrf,
|
|
952
|
+
semanticRank: rank,
|
|
953
|
+
});
|
|
954
|
+
}
|
|
955
|
+
}
|
|
956
|
+
|
|
957
|
+
const combined = Array.from(scoredMap.values());
|
|
958
|
+
combined.sort((a, b) => b.rrfScore - a.rrfScore);
|
|
959
|
+
|
|
960
|
+
const topCombined = combined.slice(0, limit - titles.length);
|
|
961
|
+
const hits: SearchHit[] = [...titles];
|
|
962
|
+
|
|
963
|
+
for (const item of topCombined) {
|
|
964
|
+
hits.push({
|
|
965
|
+
...item.hit,
|
|
966
|
+
rank: item.rrfScore,
|
|
967
|
+
});
|
|
968
|
+
}
|
|
969
|
+
|
|
970
|
+
return buildQueryResult(cleanQuery, hits, "hybrid", limit, startedAt, {
|
|
971
|
+
// The lexical leg of the fusion may have widened a zero-hit candidate
|
|
972
|
+
// set or fallen back to escaped terms; report both rather than hiding
|
|
973
|
+
// them behind the hybrid label.
|
|
974
|
+
fallbackReason: lexicalResult.meta.fallback_reason,
|
|
975
|
+
widened: lexicalResult.meta.widened,
|
|
976
|
+
});
|
|
977
|
+
}
|
|
978
|
+
|
|
979
|
+
/**
|
|
980
|
+
* The session-level serving reads: a deleted session is not found, not listed,
|
|
981
|
+
* and offers no records, while its rows stay exactly where the store keeps them
|
|
982
|
+
* (`sessions`, `records` and every derived artifact — db.ts's `tombstoneSession`).
|
|
983
|
+
* The predicate is the lanes' own, so a session cannot be invisible to search
|
|
984
|
+
* while `ledger recent`, a parent-title lookup, or a record walk still shows it.
|
|
985
|
+
*/
|
|
986
|
+
export function sessionById(db: Database, id: string): SessionRecord | null {
|
|
987
|
+
return (
|
|
988
|
+
db
|
|
989
|
+
.query<SessionRecord, [string]>(
|
|
990
|
+
`SELECT * FROM sessions WHERE id = ? AND ${tombstonePredicate(db, "sessions.id")} LIMIT 1`,
|
|
991
|
+
)
|
|
992
|
+
.get(id) ?? null
|
|
993
|
+
);
|
|
994
|
+
}
|
|
995
|
+
|
|
996
|
+
export function allSessions(db: Database): SessionRecord[] {
|
|
997
|
+
return db
|
|
998
|
+
.query<SessionRecord, []>(
|
|
999
|
+
`SELECT * FROM sessions WHERE ${tombstonePredicate(db, "sessions.id")}
|
|
1000
|
+
ORDER BY started_at DESC, id ASC`,
|
|
1001
|
+
)
|
|
1002
|
+
.all();
|
|
1003
|
+
}
|
|
1004
|
+
|
|
1005
|
+
export function lastSessions(
|
|
1006
|
+
db: Database,
|
|
1007
|
+
options: { limit?: number; source?: string } = {},
|
|
1008
|
+
): SessionRecord[] {
|
|
1009
|
+
const limit = options.limit ?? 20;
|
|
1010
|
+
if (options.source) {
|
|
1011
|
+
return db
|
|
1012
|
+
.query<SessionRecord, [string, number]>(
|
|
1013
|
+
`SELECT * FROM sessions WHERE source = ? AND ${tombstonePredicate(db, "sessions.id")}
|
|
1014
|
+
ORDER BY updated_at DESC, started_at DESC, id ASC LIMIT ?`,
|
|
1015
|
+
)
|
|
1016
|
+
.all(options.source, limit);
|
|
1017
|
+
}
|
|
1018
|
+
return db
|
|
1019
|
+
.query<SessionRecord, [number]>(
|
|
1020
|
+
`SELECT * FROM sessions WHERE ${tombstonePredicate(db, "sessions.id")}
|
|
1021
|
+
ORDER BY updated_at DESC, started_at DESC, id ASC LIMIT ?`,
|
|
1022
|
+
)
|
|
1023
|
+
.all(limit);
|
|
1024
|
+
}
|
|
1025
|
+
|
|
1026
|
+
export function sessionRecords(db: Database, sessionId: string): CanonicalDbRecord[] {
|
|
1027
|
+
return db
|
|
1028
|
+
.query<CanonicalDbRecord, [string]>(
|
|
1029
|
+
`SELECT * FROM records WHERE session_id = ? AND ${tombstonePredicate(db, "records.session_id")}
|
|
1030
|
+
ORDER BY source_order_id ASC, component_index ASC`,
|
|
1031
|
+
)
|
|
1032
|
+
.all(sessionId);
|
|
1033
|
+
}
|
|
1034
|
+
|
|
1035
|
+
/**
|
|
1036
|
+
* Why a store could not be read: missing or empty, newer than this build
|
|
1037
|
+
* (schema or derived index), not a Ledger store, or missing the read schema.
|
|
1038
|
+
*/
|
|
1039
|
+
export type ReaderErrorCode =
|
|
1040
|
+
| "STORE_NOT_INITIALIZED"
|
|
1041
|
+
| "SCHEMA_MISMATCH"
|
|
1042
|
+
| "FORMAT_MISMATCH"
|
|
1043
|
+
| "STORE_UNREADABLE"
|
|
1044
|
+
| "STORE_SCHEMA_INCOMPLETE";
|
|
1045
|
+
|
|
1046
|
+
/**
|
|
1047
|
+
* Typed read refusal. The message always names the path and the remedy: a read
|
|
1048
|
+
* verb never silently creates, upgrades, or repairs a store, and an operator
|
|
1049
|
+
* should not have to decode a raw SQLite error to learn that.
|
|
1050
|
+
*/
|
|
1051
|
+
export class ReaderError extends Error {
|
|
1052
|
+
readonly code: ReaderErrorCode;
|
|
1053
|
+
|
|
1054
|
+
constructor(code: ReaderErrorCode, message: string) {
|
|
1055
|
+
super(message);
|
|
1056
|
+
this.name = "ReaderError";
|
|
1057
|
+
this.code = code;
|
|
1058
|
+
}
|
|
1059
|
+
}
|
|
1060
|
+
|
|
1061
|
+
/**
|
|
1062
|
+
* The SQLite header records the journal mode at bytes 18-19: write version 2 is
|
|
1063
|
+
* WAL. A WAL store needs its `-wal`/`-shm` sidecars before any connection may
|
|
1064
|
+
* open it read-only — unlike a writable open, a read-only one may not create
|
|
1065
|
+
* them — so the failure is diagnosed here instead of surfacing as a bare
|
|
1066
|
+
* "unable to open database file" or "disk I/O error".
|
|
1067
|
+
*/
|
|
1068
|
+
function walStoreWithoutSidecars(dbPath: string): boolean {
|
|
1069
|
+
if (existsSync(`${dbPath}-wal`) && existsSync(`${dbPath}-shm`)) return false;
|
|
1070
|
+
let fd: number | undefined;
|
|
1071
|
+
try {
|
|
1072
|
+
fd = openSync(dbPath, "r");
|
|
1073
|
+
const header = Buffer.alloc(20);
|
|
1074
|
+
if (readSync(fd, header, 0, 20, 0) < 20) return false;
|
|
1075
|
+
return header.toString("latin1", 0, 15) === "SQLite format 3" && header[18] === 2;
|
|
1076
|
+
} catch {
|
|
1077
|
+
return false;
|
|
1078
|
+
} finally {
|
|
1079
|
+
if (fd !== undefined) closeSync(fd);
|
|
1080
|
+
}
|
|
1081
|
+
}
|
|
1082
|
+
|
|
1083
|
+
/**
|
|
1084
|
+
* The store-status wording, plus the one diagnosis that is specific to opening
|
|
1085
|
+
* a store read-only: a WAL database whose sidecars are gone cannot be opened
|
|
1086
|
+
* until a writer recreates them.
|
|
1087
|
+
*/
|
|
1088
|
+
function unreadableMessage(dbPath: string, detail: string, phase: "open" | "read"): string {
|
|
1089
|
+
if (walStoreWithoutSidecars(dbPath)) {
|
|
1090
|
+
return `Cannot open database at ${dbPath}: ${detail}. The store is a WAL database missing its -wal/-shm sidecars;` +
|
|
1091
|
+
" a read-only reader cannot create them — run `trajectories sync` (or restore the sidecars).";
|
|
1092
|
+
}
|
|
1093
|
+
return phase === "open"
|
|
1094
|
+
? `Cannot open database at ${dbPath}: ${detail}.`
|
|
1095
|
+
: `Cannot read database at ${dbPath}: ${detail}. The file may be corrupt or not a SQLite database.`;
|
|
1096
|
+
}
|
|
1097
|
+
|
|
1098
|
+
/**
|
|
1099
|
+
* Open `dbPath` read-only in the engine's own sense: the connection flag is
|
|
1100
|
+
* irreversible and enforced by SQLite, so no statement on it can write, create
|
|
1101
|
+
* a table, or migrate a schema. `PRAGMA query_only` is deliberately not used —
|
|
1102
|
+
* it is reversible per connection and still permits file creation.
|
|
1103
|
+
*
|
|
1104
|
+
* Every failure is typed, and every refusal comes from the shared read gate
|
|
1105
|
+
* (`classifyReadStore`), so this and `readStoreStatus` cannot disagree about
|
|
1106
|
+
* what a store is. A store whose schema is older than this build keeps serving
|
|
1107
|
+
* reads (`status` semantics) while it carries the read schema, and a store with
|
|
1108
|
+
* a pending index re-derivation — or one derived by an *older* rule set — is
|
|
1109
|
+
* read as written: the index is stale, not unreadable. A missing or empty
|
|
1110
|
+
* store, a store this build cannot read, a store written by a newer schema, a
|
|
1111
|
+
* store whose derived index a newer build's rules produced, and a store without
|
|
1112
|
+
* the read schema are refused.
|
|
1113
|
+
*
|
|
1114
|
+
* Exported for `TrajectoryReader`, for the package's own `delete` verb — which
|
|
1115
|
+
* must classify a store without initializing or adopting one — and for the
|
|
1116
|
+
* read-path tests, which pin the engine-level flag by writing through the handle
|
|
1117
|
+
* it returns. It is not part of the package's public surface (`index.ts` exports
|
|
1118
|
+
* the class, not this).
|
|
1119
|
+
*/
|
|
1120
|
+
export function openReadOnlyStore(dbPath: string): Database {
|
|
1121
|
+
const store = classifyReadStore(dbPath);
|
|
1122
|
+
switch (store.state) {
|
|
1123
|
+
case "ok":
|
|
1124
|
+
return store.db;
|
|
1125
|
+
case "not_initialized":
|
|
1126
|
+
throw new ReaderError(
|
|
1127
|
+
"STORE_NOT_INITIALIZED",
|
|
1128
|
+
store.reason === "missing"
|
|
1129
|
+
? `Database not found at ${dbPath}. Run \`trajectories sync\` or \`trajectories build\` first.`
|
|
1130
|
+
: `Database at ${dbPath} exists but is empty (zero bytes), so there is no store to read.` +
|
|
1131
|
+
" Run `trajectories sync` or `trajectories build` first.",
|
|
1132
|
+
);
|
|
1133
|
+
case "foreign":
|
|
1134
|
+
throw new ReaderError(
|
|
1135
|
+
"STORE_UNREADABLE",
|
|
1136
|
+
`Database at ${dbPath} exists but is not an initialized Ledger store (no sessions table).`,
|
|
1137
|
+
);
|
|
1138
|
+
case "unreadable":
|
|
1139
|
+
throw new ReaderError("STORE_UNREADABLE", unreadableMessage(dbPath, store.detail, store.phase));
|
|
1140
|
+
case "schema_mismatch":
|
|
1141
|
+
throw new ReaderError(
|
|
1142
|
+
"SCHEMA_MISMATCH",
|
|
1143
|
+
`Database schema version ${store.schemaVersion} at ${dbPath} is newer than supported schema version ${SCHEMA_VERSION}.`,
|
|
1144
|
+
);
|
|
1145
|
+
case "format_mismatch":
|
|
1146
|
+
throw new ReaderError("FORMAT_MISMATCH", formatMismatchMessage(dbPath, store));
|
|
1147
|
+
case "incomplete":
|
|
1148
|
+
throw new ReaderError("STORE_SCHEMA_INCOMPLETE", incompleteStoreMessage(dbPath, store.firstGap, store.schemaVersion));
|
|
1149
|
+
}
|
|
1150
|
+
}
|
|
1151
|
+
|
|
1152
|
+
/**
|
|
1153
|
+
* Read-only view of one Ledger store for one verb invocation.
|
|
1154
|
+
*
|
|
1155
|
+
* Short-lived by design: a reader is created per CLI verb and closed, so a
|
|
1156
|
+
* replaced store leaves no stranded handle or index behind. Reads never take
|
|
1157
|
+
* the writer lease, never create or upgrade the store, and never backfill
|
|
1158
|
+
* embeddings — `--semantic`/`--hybrid` answer lexically when the store holds no
|
|
1159
|
+
* vectors the active provider can use (see `SearchMeta.fallback_reason`).
|
|
1160
|
+
*/
|
|
1161
|
+
export class TrajectoryReader {
|
|
1162
|
+
private db: Database;
|
|
1163
|
+
readonly path: string;
|
|
1164
|
+
|
|
1165
|
+
constructor(path: string = "ledger.sqlite") {
|
|
1166
|
+
this.path = path;
|
|
1167
|
+
this.db = openReadOnlyStore(path);
|
|
1168
|
+
}
|
|
1169
|
+
|
|
1170
|
+
query(
|
|
1171
|
+
queryText: string,
|
|
1172
|
+
options?: { limit?: number; source?: string; mode?: "lexical" },
|
|
1173
|
+
): QueryResult;
|
|
1174
|
+
query(
|
|
1175
|
+
queryText: string,
|
|
1176
|
+
options: { limit?: number; source?: string; mode: "semantic" | "hybrid" },
|
|
1177
|
+
): Promise<QueryResult>;
|
|
1178
|
+
query(
|
|
1179
|
+
queryText: string,
|
|
1180
|
+
options?: { limit?: number; source?: string; mode?: "lexical" | "semantic" | "hybrid" },
|
|
1181
|
+
): QueryResult | Promise<QueryResult>;
|
|
1182
|
+
query(
|
|
1183
|
+
queryText: string,
|
|
1184
|
+
options: { limit?: number; source?: string; mode?: "lexical" | "semantic" | "hybrid" } = {},
|
|
1185
|
+
): QueryResult | Promise<QueryResult> {
|
|
1186
|
+
if (options.mode === "semantic") {
|
|
1187
|
+
return this.querySemantic(queryText, options);
|
|
1188
|
+
}
|
|
1189
|
+
if (options.mode === "hybrid") {
|
|
1190
|
+
return this.queryHybrid(queryText, options);
|
|
1191
|
+
}
|
|
1192
|
+
return this.queryLexical(queryText, options);
|
|
1193
|
+
}
|
|
1194
|
+
|
|
1195
|
+
queryLexical(queryText: string, options: ReadQueryOptions = {}): QueryResult {
|
|
1196
|
+
return queryLexical(this.db, queryText, options);
|
|
1197
|
+
}
|
|
1198
|
+
|
|
1199
|
+
querySemantic(queryText: string, options: ReadQueryOptions = {}): Promise<QueryResult> {
|
|
1200
|
+
return querySemantic(this.db, queryText, options);
|
|
1201
|
+
}
|
|
1202
|
+
|
|
1203
|
+
queryHybrid(queryText: string, options: ReadQueryOptions = {}): Promise<QueryResult> {
|
|
1204
|
+
return queryHybrid(this.db, queryText, options);
|
|
1205
|
+
}
|
|
1206
|
+
|
|
1207
|
+
getSession(id: string): SessionRecord | null {
|
|
1208
|
+
return sessionById(this.db, id);
|
|
1209
|
+
}
|
|
1210
|
+
|
|
1211
|
+
getAllSessions(): SessionRecord[] {
|
|
1212
|
+
return allSessions(this.db);
|
|
1213
|
+
}
|
|
1214
|
+
|
|
1215
|
+
getLastSessions(options?: { limit?: number; source?: string }): SessionRecord[] {
|
|
1216
|
+
return lastSessions(this.db, options);
|
|
1217
|
+
}
|
|
1218
|
+
|
|
1219
|
+
getSessionRecords(sessionId: string): CanonicalDbRecord[] {
|
|
1220
|
+
return sessionRecords(this.db, sessionId);
|
|
1221
|
+
}
|
|
1222
|
+
|
|
1223
|
+
getSessionCount(): number {
|
|
1224
|
+
return countSessions(this.db);
|
|
1225
|
+
}
|
|
1226
|
+
|
|
1227
|
+
getRecordCount(): number {
|
|
1228
|
+
return countRecords(this.db);
|
|
1229
|
+
}
|
|
1230
|
+
|
|
1231
|
+
/** Compact durable coverage signal for robot envelopes; null when no pass
|
|
1232
|
+
* has recorded coverage (the field is then simply omitted). */
|
|
1233
|
+
getSourceCoverageSummary(): SourceCoverageSummary | null {
|
|
1234
|
+
const hasTable = !!this.db
|
|
1235
|
+
.query<{ name: string }, []>(
|
|
1236
|
+
"SELECT name FROM sqlite_master WHERE type='table' AND name='source_coverage' LIMIT 1;",
|
|
1237
|
+
)
|
|
1238
|
+
.get();
|
|
1239
|
+
return hasTable ? readSourceCoverageSummary(this.db) : null;
|
|
1240
|
+
}
|
|
1241
|
+
|
|
1242
|
+
getDiscoverySummary(): DiscoverySummary | null { return readDiscoverySummary(this.db); }
|
|
1243
|
+
|
|
1244
|
+
getSchemaVersion(): number {
|
|
1245
|
+
return (
|
|
1246
|
+
this.db
|
|
1247
|
+
.query<{ user_version: number }, []>("PRAGMA user_version;")
|
|
1248
|
+
.get()?.user_version ?? 0
|
|
1249
|
+
);
|
|
1250
|
+
}
|
|
1251
|
+
|
|
1252
|
+
close(): void {
|
|
1253
|
+
this.db.close();
|
|
1254
|
+
}
|
|
1255
|
+
}
|