sumo-agents 0.6.1 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/search.mjs ADDED
@@ -0,0 +1,48 @@
1
+ import { LOOSE_BELOW, markHits, recordMiss, resolveProject, search } from './memory.mjs';
2
+ import { searchJournal } from './sessions.mjs';
3
+ import { words } from './text.mjs';
4
+
5
+ /** How many lines of the conversations follow memories that answered: the memories are the answer, these are where it was said. */
6
+ const SAID_AFTER_MEMORIES = 3;
7
+ const NONE = { results: [], hits: [], elsewhere: {}, more: 0, best: 0 };
8
+
9
+ /**
10
+ * The one search: memories first, then the conversations themselves — what the user typed and what sumo answered.
11
+ * Only extracted facts used to be searched, and a topic talked through at length and never made a memory of was a
12
+ * miss. All the words are tried first, across both; only when neither holds them all in scope is any word tried.
13
+ *
14
+ * When memories answer, a few lines of the conversations follow them; when none does, the conversations are the
15
+ * answer and the search is not a miss. A miss is kept when nothing is shown, or when what is shown is `loose`: the
16
+ * search had several words, fell back to any of them, and no line holds two. Memories shown count as used; rows of
17
+ * the conversations carry no such count.
18
+ *
19
+ * `turns` leaves memories out (and keeps no miss: nothing was asked of memory); `type` leaves the conversations out,
20
+ * a type being a kind of memory. `record: false` is a search nobody asked — a replay, a check: nothing shown counts
21
+ * as used and no miss is kept. The rest — `project`, `everywhere`, `all`, `since`, `before`, `limit` — is as
22
+ * `memory.search` has it, and holds for both. Returns `{ memories, conversations, more, moreSaid, elsewhere, loose }`:
23
+ * `more` and `moreSaid` count what matched in scope and was not shown, `elsewhere` what other projects hold, by scope.
24
+ */
25
+ export function searchAll(db, query, opts = {}) {
26
+ const slug = opts.project ? resolveProject(db, opts.project) : null;
27
+ const now = opts.now ?? new Date().toISOString();
28
+ const limit = opts.limit ?? 8;
29
+ const { everywhere, since, before } = opts;
30
+ let memories = NONE;
31
+ let said = NONE;
32
+ let fellBack = false;
33
+ for (const terms of new Set(words(query)).size > 1 ? ['all', 'any'] : ['all']) {
34
+ memories = opts.turns ? NONE : search(db, query, { project: opts.project, type: opts.type, all: opts.all, everywhere, since, before, limit, now, terms, record: false });
35
+ said = opts.type ? NONE : searchJournal(db, query, { terms, project: slug, everywhere, since, before, limit: memories.results.length > 0 ? Math.min(limit, SAID_AFTER_MEMORIES) : limit });
36
+ fellBack = terms === 'any';
37
+ if (memories.results.length + said.hits.length > 0) break;
38
+ }
39
+ const shown = memories.results.length + said.hits.length;
40
+ const loose = fellBack && shown > 0 && Math.max(memories.best, said.best) < LOOSE_BELOW;
41
+ if (!opts.turns && opts.record !== false) {
42
+ markHits(db, memories.results.map((m) => m.id), now);
43
+ if (shown === 0 || loose) recordMiss(db, query, everywhere ? 'everywhere' : slug ? `project:${slug}` : 'global', now);
44
+ }
45
+ const elsewhere = { ...memories.elsewhere };
46
+ for (const [scope, n] of Object.entries(said.elsewhere)) elsewhere[scope] = (elsewhere[scope] ?? 0) + n;
47
+ return { memories: memories.results, conversations: said.hits, more: memories.more, moreSaid: said.more, elsewhere, loose };
48
+ }
package/src/sessions.mjs CHANGED
@@ -1,3 +1,4 @@
1
+ import { createHash } from 'node:crypto';
1
2
  import { closeSync, mkdirSync, openSync, readdirSync, readFileSync, readSync, renameSync, statSync, unlinkSync, writeFileSync } from 'node:fs';
2
3
  import { join } from 'node:path';
3
4
  import { ago } from './card.mjs';
@@ -5,10 +6,18 @@ import { heldBy } from './lock.mjs';
5
6
  import { Refusal, UsageError } from './memory.mjs';
6
7
  import { paths } from './paths.mjs';
7
8
  import { redact } from './redact.mjs';
8
- import { cjkTerms, clip, containing, head } from './text.mjs';
9
+ import { cjkTerms, clip, containing, ftsAllQuery, ftsQuery, head, stem, termsHeld, words } from './text.mjs';
9
10
  import { readBackwards } from './transcript.mjs';
10
11
 
11
- const TURN_MAX_CHARS = 4000;
12
+ /**
13
+ * How much of one turn or reply the journal keeps: whole, in practice. The journal is what memory is rebuilt from and
14
+ * what search falls back on, and at 4,000 characters the pasted error or the long explanation a later question was
15
+ * about had been cut away. Past this it is a pasted file, and its head is kept.
16
+ */
17
+ const JOURNAL_MAX_CHARS = 64_000;
18
+ const CUT = ' …[cut]';
19
+ /** A conversation nobody has spoken in for this long is over, as far as reading it for memory goes. */
20
+ const FINISHED_AFTER_MS = 30 * 60_000;
12
21
 
13
22
  /**
14
23
  * Where a session starts costing quality. The numbers are absolute tokens, not
@@ -33,25 +42,185 @@ export function ensureSession(db, { id, harness = 'claude', cwd = null, transcri
33
42
  ).run(id, harness, cwd, transcriptPath, now);
34
43
  }
35
44
 
45
+ /** A slash command, a worker's brief and an empty line are not something the user said. A command's name may carry a dash or a plugin prefix (`/code-review`, `/codex:rescue`); a path (`/usr/bin`) still has a slash after its first word. */
46
+ const notSaid = (trimmed) => !trimmed || /^\/[\w:-]+(\s|$)/.test(trimmed) || trimmed.startsWith('JOB:');
47
+
36
48
  /**
37
- * Stores what the user typed, word for word — the ground truth every derived
38
- * memory points back to. Returns the turn id, or null when there was nothing
39
- * worth keeping (a slash command, a worker's brief, an empty line).
49
+ * Words as the journal keeps them: secrets out first, then the cut — half a token left at the edge of a cut would
50
+ * still be a leak. `fullLen` is how long they were, secrets out, before any cut. Null for nothing at all.
40
51
  */
41
- export function recordTurn(db, { sessionId, text, now }) {
42
- const trimmed = text.trim();
43
- // A command's name may carry a dash or a plugin prefix (`/code-review`, `/codex:rescue`); a path (`/usr/bin`) still has a slash after its first word.
44
- if (!trimmed || /^\/[\w:-]+(\s|$)/.test(trimmed) || trimmed.startsWith('JOB:')) return null;
45
-
52
+ function journalText(raw) {
53
+ const trimmed = String(raw ?? '').trim();
54
+ if (!trimmed) return null;
46
55
  const cleaned = redact(trimmed);
47
- const kept = cleaned.text.length > TURN_MAX_CHARS ? `${head(cleaned.text, TURN_MAX_CHARS)} …[cut]` : cleaned.text;
48
- const { lastInsertRowid } = db
49
- .prepare('INSERT INTO user_turns (session_id, ts, text, redacted) VALUES (?, ?, ?, ?)')
50
- .run(sessionId, now, kept, cleaned.count > 0 ? 1 : 0);
51
- db.prepare("UPDATE sessions SET last_turn_at = ?, dream_state = 'pending' WHERE id = ?").run(now, sessionId);
56
+ const cut = cleaned.text.length > JOURNAL_MAX_CHARS;
57
+ return { text: cut ? `${head(cleaned.text, JOURNAL_MAX_CHARS)}${CUT}` : cleaned.text, fullLen: cleaned.text.length, redacted: cleaned.count > 0 };
58
+ }
59
+
60
+ /**
61
+ * What the user typed, as the journal would keep it — `{ text, fullLen, redacted }` — or null when it is nothing
62
+ * the journal records (a slash command, a `JOB:` brief, an empty line). For whoever must keep a turn somewhere
63
+ * else first (the spool): the words leave here already redacted and cut.
64
+ */
65
+ export const turnText = (raw) => (notSaid(String(raw ?? '').trim()) ? null : journalText(raw));
66
+ /** The same for the assistant's reply; null only when it said nothing. */
67
+ export const replyText = (raw) => journalText(raw);
68
+
69
+ /**
70
+ * Words kept somewhere else first, on their way into the journal. They were redacted and cut when they were kept;
71
+ * they are redacted again here, since a file is not the hand that wrote it, and never cut a second time — a cut of a
72
+ * cut would be other words, and another digest.
73
+ */
74
+ function fromKept(kept) {
75
+ if (typeof kept?.text !== 'string' || !kept.text.trim()) return null;
76
+ const cleaned = redact(kept.text.trim());
77
+ const text = cleaned.text.length > JOURNAL_MAX_CHARS + CUT.length ? `${head(cleaned.text, JOURNAL_MAX_CHARS)}${CUT}` : cleaned.text;
78
+ return { text, fullLen: Number.isInteger(kept.fullLen) && kept.fullLen >= 0 ? kept.fullLen : text.length, redacted: kept.redacted === true || cleaned.count > 0 };
79
+ }
80
+
81
+ /** One journal row's digest: its session, its time and its words as kept. The same event written twice is the same row. */
82
+ const shaOf = (sessionId, ts, text) => createHash('sha256').update(`${sessionId}\n${ts}\n${text}`).digest('hex');
83
+
84
+ /** The project a session was about at a moment: the one whose card had gone in most recently by then. */
85
+ function projectAsOf(db, sessionId, at) {
86
+ return db.prepare(`SELECT slug FROM session_injections WHERE session_id = ? AND slug NOT LIKE '%:%' AND ts <= ? ORDER BY ts DESC LIMIT 1`).get(sessionId, at)?.slug ?? null;
87
+ }
88
+
89
+ /**
90
+ * Stores what the user typed, word for word and whole (the first 64,000 characters of a longer paste, the cut
91
+ * marked, `full_len` saying how long it was) — the ground truth every derived memory points back to. It is stamped
92
+ * with the project the session was about, and with a digest of its session, time and words: the same turn written
93
+ * again — an event replayed from the spool — inserts nothing and returns the row already there. Returns the turn id,
94
+ * or null when there was nothing worth keeping (a slash command, a worker's brief, an empty line).
95
+ *
96
+ * `kept` is for a replay: `{ text, fullLen, redacted }` as `turnText` made them when the event was set aside.
97
+ */
98
+ export function recordTurn(db, { sessionId, text, now, kept = null }) {
99
+ const said = kept ? fromKept(kept) : turnText(text);
100
+ if (!said || notSaid(said.text)) return null;
101
+ const sha = shaOf(sessionId, now, said.text);
102
+ const { changes, lastInsertRowid } = db
103
+ .prepare('INSERT OR IGNORE INTO user_turns (session_id, ts, text, redacted, project, full_len, sha) VALUES (?, ?, ?, ?, ?, ?, ?)')
104
+ .run(sessionId, now, said.text, said.redacted ? 1 : 0, projectAsOf(db, sessionId, now), said.fullLen, sha);
105
+ if (Number(changes) === 0) return db.prepare('SELECT id FROM user_turns WHERE sha = ?').get(sha).id;
106
+ // A turn replayed late is older than what the session has heard since: the newest time stays.
107
+ db.prepare("UPDATE sessions SET last_turn_at = MAX(COALESCE(last_turn_at, ''), ?), dream_state = 'pending' WHERE id = ?").run(now, sessionId);
52
108
  return Number(lastInsertRowid);
53
109
  }
54
110
 
111
+ /**
112
+ * Stores the assistant's final reply of a turn in the journal, beside the user's turns: redacted like a turn, whole
113
+ * up to 64,000 characters, stamped with the session's project, and with the same kind of digest, so a replay inserts
114
+ * nothing twice. It is what search falls back on and what the memory pass reads for context — never itself a memory:
115
+ * only the user's words are. The session's row must exist (`ensureSession`). Returns the reply id, or null when
116
+ * nothing was said.
117
+ */
118
+ export function recordReply(db, { sessionId, text, now, kept = null }) {
119
+ const said = kept ? fromKept(kept) : replyText(text);
120
+ if (!said) return null;
121
+ const sha = shaOf(sessionId, now, said.text);
122
+ const { changes, lastInsertRowid } = db
123
+ .prepare('INSERT OR IGNORE INTO replies (session_id, ts, text, project, full_len, sha) VALUES (?, ?, ?, ?, ?, ?)')
124
+ .run(sessionId, now, said.text, projectAsOf(db, sessionId, now), said.fullLen, sha);
125
+ return Number(changes) === 0 ? db.prepare('SELECT id FROM replies WHERE sha = ?').get(sha).id : Number(lastInsertRowid);
126
+ }
127
+
128
+ /**
129
+ * A session heard of late: an event that could not be written when it happened, replayed with the time it happened.
130
+ * Unlike `ensureSession` it never reopens a session that has ended since — the event is from before that — and an
131
+ * older start replaces a newer one.
132
+ */
133
+ export function heardSession(db, { id, harness = 'claude', cwd = null, transcriptPath = null, now }) {
134
+ db.prepare(
135
+ `INSERT INTO sessions (id, harness, cwd, transcript_path, started_at) VALUES (?, ?, ?, ?, ?)
136
+ ON CONFLICT(id) DO UPDATE SET
137
+ cwd = COALESCE(cwd, excluded.cwd),
138
+ transcript_path = COALESCE(transcript_path, excluded.transcript_path),
139
+ started_at = MIN(started_at, excluded.started_at)`,
140
+ ).run(id, harness, cwd, transcriptPath, now);
141
+ }
142
+
143
+ /** A goodbye replayed late: the session ended then, unless it has been heard from since or has ended already. */
144
+ export function endedThen(db, id, at) {
145
+ db.prepare('UPDATE sessions SET ended_at = ? WHERE id = ? AND ended_at IS NULL AND COALESCE(last_turn_at, started_at) <= ?').run(at, id, at);
146
+ }
147
+
148
+ /**
149
+ * One conversation as the journal holds it, in the order it was said: the user's turns and the assistant's final
150
+ * replies of a session, each `{ kind: 'turn' | 'reply', id, session_id, ts, text, project, full_len }`. `id` is the
151
+ * row's own in `user_turns` or `replies` (the two count separately). Ordered by time, not by id: a turn replayed
152
+ * from the spool has a late id and its own time; at the same instant the turn comes before the reply.
153
+ */
154
+ export function sessionJournal(db, sessionId) {
155
+ return db
156
+ .prepare(
157
+ `SELECT 'turn' AS kind, id, session_id, ts, text, project, full_len FROM user_turns WHERE session_id = ?1
158
+ UNION ALL
159
+ SELECT 'reply' AS kind, id, session_id, ts, text, project, full_len FROM replies WHERE session_id = ?1
160
+ ORDER BY ts, kind DESC, id`,
161
+ )
162
+ .all(sessionId);
163
+ }
164
+
165
+ const COVERED = (versions) => `EXISTS (SELECT 1 FROM extractions x WHERE x.session_id = t.session_id AND x.status IN ('ok', 'quarantined')
166
+ ${versions.length > 0 ? `AND x.extractor_version IN (${versions.map(() => '?').join(', ')}) ` : ''}AND t.id BETWEEN x.first_turn AND x.last_turn)`;
167
+ /** No version named is every version: a turn any extractor read counts as read. */
168
+ const listOf = (version) => (version === undefined || version === null ? [] : Array.isArray(version) ? version : [version]);
169
+
170
+ /**
171
+ * The finished conversations the memory pass has yet to read: sessions rows, oldest first, that are over — ended, or
172
+ * nobody has spoken or been answered in them for 30 minutes (a closed terminal never says goodbye) — and hold a turn
173
+ * no `ok` or `quarantined` extraction covers (its id within the row's `first_turn`…`last_turn`). A `failed` or
174
+ * `running` row covers nothing: the turn is still owed.
175
+ *
176
+ * With no `version` an extraction under any extractor covers, which is what the pass asks: a changed prompt, model
177
+ * or rule must not make every conversation owed again and have them all read anew by themselves — reading again under
178
+ * the current extractor is `sumo memory rebuild`, which shows its difference first. `version`, one or several,
179
+ * counts only those extractors' rows. `includeOpen` is for a pass the user asked for by hand, which waits for nothing.
180
+ */
181
+ export function sessionsAwaitingExtraction(db, { version, now, limit = 5, includeOpen = false }) {
182
+ const versions = listOf(version);
183
+ const cutoff = new Date(Date.parse(now) - FINISHED_AFTER_MS).toISOString();
184
+ return db
185
+ .prepare(
186
+ `SELECT s.* FROM sessions s
187
+ WHERE (? = 1 OR s.ended_at IS NOT NULL
188
+ OR MAX(COALESCE(s.last_turn_at, s.started_at), COALESCE((SELECT MAX(r.ts) FROM replies r WHERE r.session_id = s.id), '')) < ?)
189
+ AND EXISTS (SELECT 1 FROM user_turns t WHERE t.session_id = s.id AND NOT ${COVERED(versions)})
190
+ ORDER BY s.started_at, s.id LIMIT ?`,
191
+ )
192
+ .all(includeOpen ? 1 : 0, cutoff, ...versions, limit);
193
+ }
194
+
195
+ /** The turns of one session that no `ok` or `quarantined` extraction covers — of any extractor, or of `version` (one, or several) when given — oldest first: what a pass still owes it. */
196
+ export function turnsAwaitingExtraction(db, sessionId, version) {
197
+ const versions = listOf(version);
198
+ return db.prepare(`SELECT t.* FROM user_turns t WHERE t.session_id = ? AND NOT ${COVERED(versions)} ORDER BY t.ts, t.id`).all(sessionId, ...versions);
199
+ }
200
+
201
+ /**
202
+ * When the user was last heard: the time of the newest turn in the journal — or of a later prompt that left no turn
203
+ * (`noteHeard`) — or null when there is none. With a
204
+ * `harness` — 'claude' for what came in through `sumo hook`, 'sumo' for sumo's own chat — only that one's sessions:
205
+ * a harness whose hook lines are gone goes quiet while the other keeps talking.
206
+ */
207
+ export function lastHeard(db, harness = null) {
208
+ const said = db.prepare('SELECT MAX(t.ts) AS at FROM user_turns t JOIN sessions s ON s.id = t.session_id WHERE (? IS NULL OR s.harness = ?)').get(harness, harness).at ?? null;
209
+ const stamped = db.prepare(`SELECT MAX(value) AS at FROM meta WHERE key LIKE ? ESCAPE '\\'`).get(`${HEARD_AT}${harness ?? '%'}`).at ?? null;
210
+ return said !== null && stamped !== null ? (said > stamped ? said : stamped) : (said ?? stamped);
211
+ }
212
+
213
+ /** Where the time of a harness's last prompt that left no turn is kept: `heard_at.claude`, `heard_at.sumo`. */
214
+ const HEARD_AT = 'heard\\_at.';
215
+
216
+ /**
217
+ * A prompt that is nothing the journal keeps — a slash command, a brief — is still the harness being heard: its time
218
+ * is kept, so days of `/compact` and `/clear` are not read as hook lines gone missing.
219
+ */
220
+ export function noteHeard(db, harness, now) {
221
+ db.prepare('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = MAX(value, excluded.value)').run(`heard_at.${harness}`, now);
222
+ }
223
+
55
224
  /**
56
225
  * A call that reached past the machine — a web search or fetch, an MCP server — written on its session, each tool once:
57
226
  * what the conversation says after it may be what a page told it, so the memory pass holds what it would file from the
@@ -111,46 +280,128 @@ export function addCheckpoint(db, { sessionId = null, project, done, next = null
111
280
  db.prepare('UPDATE projects SET last_touched_at = ? WHERE slug = ?').run(now, project);
112
281
  }
113
282
 
283
+ /** The two halves of the journal, and the letter each one's ids are typed with: `t481` is a turn, `r3` a reply. */
284
+ const JOURNAL = { turn: { table: 'user_turns', fts: 'user_turns_fts', mark: 't' }, reply: { table: 'replies', fts: 'replies_fts', mark: 'r' } };
285
+ /** How much of a turn or a reply one line of a search shows: the part around the words asked for. */
286
+ const EXCERPT_CHARS = 200;
287
+ const EXCERPT_LEAD = 40;
288
+
114
289
  /**
115
- * Sessions the consolidation pass has not read yet. A session counts as over
116
- * when it said so, or when it has been silent for two hours — a closed terminal
117
- * or a crash never sends the goodbye. `includeOpen` is for a pass the user asked
118
- * for by hand, which should not wait for anything.
290
+ * The stretch of a text that holds the most of the words searched for, on one line: a reply can be pages long, and
291
+ * its first words are seldom where the answer is. A word is looked for by its stem, so "deploys" shows for "deploy".
119
292
  */
120
- export function sessionsAwaitingDream(db, now, limit = 5, { includeOpen = false } = {}) {
121
- const cutoff = new Date(Date.parse(now) - 2 * 3_600_000).toISOString();
122
- return db
123
- .prepare(
124
- `SELECT s.* FROM sessions s
125
- WHERE s.dream_state = 'pending'
126
- AND (? = 1 OR s.ended_at IS NOT NULL OR COALESCE(s.last_turn_at, s.started_at) < ?)
127
- AND EXISTS (SELECT 1 FROM user_turns t WHERE t.session_id = s.id)
128
- ORDER BY s.started_at LIMIT ?`,
129
- )
130
- .all(includeOpen ? 1 : 0, cutoff, limit);
293
+ function excerpt(text, query) {
294
+ const flat = text.replace(/\s+/g, ' ').trim();
295
+ if (flat.length <= EXCERPT_CHARS) return flat;
296
+ const lower = flat.toLowerCase();
297
+ const at = [...new Set(words(query))].map((w) => lower.indexOf(lower.includes(w) ? w : stem(w))).filter((i) => i >= 0);
298
+ const within = (from) => at.filter((i) => i >= from && i < from + EXCERPT_CHARS - EXCERPT_LEAD).length;
299
+ const first = at.reduce((best, i) => (best === null || within(i) > within(best) || (within(i) === within(best) && i < best) ? i : best), null) ?? 0;
300
+ const from = Math.max(0, Math.min(first - EXCERPT_LEAD, flat.length - EXCERPT_CHARS));
301
+ return `${from > 0 ? '…' : ''}${head(flat.slice(from), EXCERPT_CHARS).trim()}${from + EXCERPT_CHARS < flat.length ? '…' : ''}`;
131
302
  }
132
303
 
133
304
  /**
134
- * What the user actually typed, searched word for word: `match` is `ftsQuery`'s expression. A Chinese, Japanese or
135
- * Korean word sits inside a run the index keeps as one word, so it is looked for in the text itself: what holds the most
136
- * of those words, newest first, after what the index found.
305
+ * One half of the journal searched: the rows in scope that hold every word (`every`) or any of them, best first, and
306
+ * how many rows matched per project. The index answers for the words it keeps apart; a Chinese, Japanese or Korean
307
+ * word sits inside a run the index keeps as one word, so it is looked for in the text itself — a row found only that
308
+ * way ranks by the share of those words it holds, the best the index found counting as all of them.
137
309
  */
138
- export function searchTurns(db, match, limit = 8) {
139
- const found = db
140
- .prepare(
141
- `SELECT t.* FROM user_turns_fts JOIN user_turns t ON t.id = user_turns_fts.rowid
142
- WHERE user_turns_fts MATCH ? ORDER BY bm25(user_turns_fts), t.id DESC LIMIT ?`,
143
- )
144
- .all(match, limit);
145
- const terms = cjkTerms(match);
146
- if (terms.length === 0 || found.length >= limit) return found;
147
- const holds = terms.map(() => `(text LIKE ? ESCAPE '\\')`);
148
- const patterns = terms.map(containing);
149
- const seen = new Set(found.map((t) => t.id));
150
- const more = db
151
- .prepare(`SELECT * FROM user_turns WHERE ${holds.join(' OR ')} ORDER BY ${holds.join(' + ')} DESC, id DESC LIMIT ?`)
152
- .all(...patterns, ...patterns, limit + found.length);
153
- return [...found, ...more.filter((t) => !seen.has(t.id))].slice(0, limit);
310
+ function saidIn(db, kind, query, every, { project, everywhere, since, before, limit }) {
311
+ const { table, fts, mark } = JOURNAL[kind];
312
+ const cjk = cjkTerms(query);
313
+ const match = every ? ftsAllQuery(query) : ftsQuery(query);
314
+ const likes = cjk.map(() => `t.text LIKE ? ESCAPE '\\'`);
315
+ const patterns = cjk.map(containing);
316
+ const dated = `${since ? ' AND t.ts >= ?' : ''}${before ? ' AND t.ts < ?' : ''}`;
317
+ const dates = [since, before].filter(Boolean);
318
+ // What was said about no project is everyone's, as a global memory is; another project's is counted, never shown.
319
+ const mine = everywhere ? '' : project ? ' AND (t.project IS NULL OR t.project = ?)' : ' AND t.project IS NULL';
320
+ const mineParams = everywhere || !project ? [] : [project];
321
+
322
+ const inIndex = match ? `t.id IN (SELECT rowid FROM ${fts} WHERE ${fts} MATCH ?)` : null;
323
+ const holds = [inIndex, ...likes].filter(Boolean).join(every ? ' AND ' : ' OR ');
324
+ const counted = db.prepare(`SELECT t.project AS project, COUNT(*) AS n FROM ${table} t WHERE (${holds})${dated} GROUP BY t.project`).all(...(match ? [match] : []), ...patterns, ...dates);
325
+
326
+ const cols = 't.id, t.session_id, t.ts, t.text, t.project, t.full_len';
327
+ const indexed = match
328
+ ? db
329
+ .prepare(
330
+ `SELECT ${cols}, bm25(${fts}) AS rank FROM ${fts} JOIN ${table} t ON t.id = ${fts}.rowid
331
+ WHERE ${fts} MATCH ?${every ? likes.map((l) => ` AND ${l}`).join('') : ''}${dated}${mine} ORDER BY rank, t.ts DESC, t.id DESC LIMIT ?`,
332
+ )
333
+ .all(match, ...(every ? patterns : []), ...dates, ...mineParams, limit)
334
+ : [];
335
+ const byId = new Map(indexed.map((row) => [row.id, row]));
336
+ if (cjk.length > 0 && (!every || !match)) {
337
+ const best = indexed.reduce((min, row) => Math.min(min, row.rank), 0) || -1;
338
+ const inText = db
339
+ .prepare(`SELECT ${cols} FROM ${table} t WHERE (${likes.join(every ? ' AND ' : ' OR ')})${dated}${mine} ORDER BY ${likes.map((l) => `(${l})`).join(' + ')} DESC, t.ts DESC, t.id DESC LIMIT ?`)
340
+ .all(...patterns, ...dates, ...mineParams, ...patterns, limit);
341
+ for (const row of inText) {
342
+ const lower = row.text.toLowerCase();
343
+ const rank = (best * cjk.filter((t) => lower.includes(t)).length) / cjk.length;
344
+ const known = byId.get(row.id);
345
+ if (!known || rank < known.rank) byId.set(row.id, { ...row, rank });
346
+ }
347
+ }
348
+ const rows = [...byId.values()].map(({ text, ...row }) => ({ kind, ref: `${mark}${row.id}`, ...row, excerpt: excerpt(text, query), held: termsHeld(query, text) }));
349
+ return { rows, counted };
350
+ }
351
+
352
+ /**
353
+ * The conversations themselves, searched: what the user typed and what sumo answered, where the answer to "what did
354
+ * we say about …" lives when nobody made a memory of it. All the words first; only when no row in scope holds every
355
+ * one is it tried with any of them (`fellBack`); `terms: 'all' | 'any'` asks for one of the two alone.
356
+ *
357
+ * Scope is as memory has it: a row said about no project, and the rows of `project` (a slug); another project's rows
358
+ * are counted in `elsewhere` (`project:<slug>` → how many), never shown, unless `everywhere`. `since` and `before`
359
+ * (ISO times) hold it to when it was said. Each hit is `{ kind: 'turn' | 'reply', ref, id, session_id, ts, project,
360
+ * full_len, excerpt, held, rank }` — `ref` is the id as it is typed (`t481`, `r3`), `excerpt` the part around the
361
+ * words, `held` how many of the search's words the whole text holds. `more` counts what matched in scope past `limit`;
362
+ * `best` is the most `held` among the hits.
363
+ */
364
+ export function searchJournal(db, query, { terms, project = null, everywhere = false, since, before, limit = 8 } = {}) {
365
+ if (!ftsQuery(query)) throw new UsageError('nothing to search for');
366
+ const several = new Set(words(query)).size > 1;
367
+ let hits = [];
368
+ let counted = [];
369
+ let fellBack = false;
370
+ for (const every of terms === 'any' ? [false] : terms === 'all' || !several ? [true] : [true, false]) {
371
+ const halves = Object.keys(JOURNAL).map((kind) => saidIn(db, kind, query, every, { project, everywhere, since, before, limit }));
372
+ hits = halves.flatMap((h) => h.rows).sort((a, b) => a.rank - b.rank || (a.ts < b.ts ? 1 : a.ts > b.ts ? -1 : 0) || b.id - a.id);
373
+ counted = halves.flatMap((h) => h.counted);
374
+ fellBack = !every && terms !== 'any';
375
+ if (hits.length > 0) break;
376
+ }
377
+ const elsewhere = {};
378
+ let inScope = 0;
379
+ for (const { project: said, n } of counted) {
380
+ if (everywhere || said === null || said === project) inScope += n;
381
+ else elsewhere[`project:${said}`] = (elsewhere[`project:${said}`] ?? 0) + n;
382
+ }
383
+ const shown = hits.slice(0, limit);
384
+ return { hits: shown, elsewhere, more: inScope - shown.length, fellBack, best: shown.reduce((most, h) => Math.max(most, h.held), 0) };
385
+ }
386
+
387
+ /** How many turns and replies either side of one are shown with it. */
388
+ const AROUND = 2;
389
+
390
+ /**
391
+ * One turn or reply by the id a search printed (`t481`, `r3`), whole, with what was said around it in its
392
+ * conversation: `{ entry, around }`, both as `sessionJournal` gives rows, `around` in the order it was said and
393
+ * holding the entry itself. Null when `ref` is not such an id; a refusal when it names nothing.
394
+ */
395
+ export function journalEntry(db, ref) {
396
+ const typed = /^([tr])(\d+)$/.exec(String(ref ?? '').trim().toLowerCase());
397
+ if (!typed) return null;
398
+ const kind = typed[1] === 't' ? 'turn' : 'reply';
399
+ const id = Number(typed[2]);
400
+ const row = db.prepare(`SELECT session_id FROM ${JOURNAL[kind].table} WHERE id = ?`).get(id);
401
+ if (!row) throw new Refusal(`no ${kind} ${typed[0]} in the conversations — sumo search "<words>" prints the ids it finds`);
402
+ const said = sessionJournal(db, row.session_id).map((e) => ({ ...e, ref: `${JOURNAL[e.kind].mark}${e.id}` }));
403
+ const at = said.findIndex((e) => e.kind === kind && e.id === id);
404
+ return { entry: said[at], around: said.slice(Math.max(0, at - AROUND), at + AROUND + 1) };
154
405
  }
155
406
 
156
407
  /**
@@ -298,15 +549,15 @@ export function loadState(id) {
298
549
  function summaryOf(file) {
299
550
  try {
300
551
  const fd = openSync(file, 'r');
301
- let text;
302
552
  try {
303
553
  const buffer = Buffer.alloc(SUMMARY_BYTES);
304
- text = buffer.toString('utf8', 0, readSync(fd, buffer, 0, SUMMARY_BYTES, 0));
554
+ const text = buffer.toString('utf8', 0, readSync(fd, buffer, 0, SUMMARY_BYTES, 0));
555
+ const cut = text.indexOf(',"messages":');
556
+ // The read above was at a position, so the descriptor is still at the start: a head with no cut is read whole from it.
557
+ return JSON.parse(cut === -1 ? readFileSync(fd, 'utf8') : `${text.slice(0, cut)}}`);
305
558
  } finally {
306
559
  closeSync(fd);
307
560
  }
308
- const cut = text.indexOf(',"messages":');
309
- return JSON.parse(cut === -1 ? readFileSync(file, 'utf8') : `${text.slice(0, cut)}}`);
310
561
  } catch {
311
562
  return null;
312
563
  }