sumo-agents 0.6.1 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +6 -5
- package/CHANGELOG.md +72 -0
- package/README.md +47 -34
- package/guides/memory.md +21 -14
- package/package.json +2 -2
- package/prompts/dream.md +3 -16
- package/prompts/scribe.md +20 -15
- package/src/apply.mjs +364 -230
- package/src/card.mjs +16 -11
- package/src/catalog.mjs +3 -3
- package/src/chat.mjs +53 -15
- package/src/check.mjs +371 -0
- package/src/cli.mjs +230 -146
- package/src/db.mjs +147 -2
- package/src/dream.mjs +97 -189
- package/src/export.mjs +43 -18
- package/src/herdr.mjs +71 -36
- package/src/hooks.mjs +115 -22
- package/src/jobs.mjs +4 -2
- package/src/keeper.mjs +14 -51
- package/src/ledger.mjs +126 -0
- package/src/logs.mjs +10 -5
- package/src/loop.mjs +4 -4
- package/src/mcp.mjs +129 -51
- package/src/memeval.mjs +167 -0
- package/src/memory.mjs +261 -130
- package/src/page.mjs +3 -3
- package/src/paths.mjs +1 -0
- package/src/prime.mjs +82 -13
- package/src/rebuild.mjs +162 -0
- package/src/render.mjs +22 -0
- package/src/route.mjs +1 -1
- package/src/sandbox.mjs +5 -2
- package/src/scan.mjs +16 -4
- package/src/scribe.mjs +361 -113
- package/src/search.mjs +48 -0
- package/src/sessions.mjs +304 -53
- package/src/settings.mjs +868 -0
- package/src/setup.mjs +56 -64
- package/src/spool.mjs +308 -0
- package/src/text.mjs +118 -6
- package/src/tools.mjs +18 -7
- package/src/transcript.mjs +3 -47
- package/src/tty.mjs +48 -23
- package/src/ui.mjs +62 -25
- package/src/verify.mjs +23 -27
package/src/search.mjs
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import { LOOSE_BELOW, markHits, recordMiss, resolveProject, search } from './memory.mjs';
|
|
2
|
+
import { searchJournal } from './sessions.mjs';
|
|
3
|
+
import { words } from './text.mjs';
|
|
4
|
+
|
|
5
|
+
/** How many lines of the conversations follow memories that answered: the memories are the answer, these are where it was said. */
|
|
6
|
+
const SAID_AFTER_MEMORIES = 3;
|
|
7
|
+
const NONE = { results: [], hits: [], elsewhere: {}, more: 0, best: 0 };
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* The one search: memories first, then the conversations themselves — what the user typed and what sumo answered.
|
|
11
|
+
* Only extracted facts used to be searched, and a topic talked through at length and never made a memory of was a
|
|
12
|
+
* miss. All the words are tried first, across both; only when neither holds them all in scope is any word tried.
|
|
13
|
+
*
|
|
14
|
+
* When memories answer, a few lines of the conversations follow them; when none does, the conversations are the
|
|
15
|
+
* answer and the search is not a miss. A miss is kept when nothing is shown, or when what is shown is `loose`: the
|
|
16
|
+
* search had several words, fell back to any of them, and no line holds two. Memories shown count as used; rows of
|
|
17
|
+
* the conversations carry no such count.
|
|
18
|
+
*
|
|
19
|
+
* `turns` leaves memories out (and keeps no miss: nothing was asked of memory); `type` leaves the conversations out,
|
|
20
|
+
* a type being a kind of memory. `record: false` is a search nobody asked — a replay, a check: nothing shown counts
|
|
21
|
+
* as used and no miss is kept. The rest — `project`, `everywhere`, `all`, `since`, `before`, `limit` — is as
|
|
22
|
+
* `memory.search` has it, and holds for both. Returns `{ memories, conversations, more, moreSaid, elsewhere, loose }`:
|
|
23
|
+
* `more` and `moreSaid` count what matched in scope and was not shown, `elsewhere` what other projects hold, by scope.
|
|
24
|
+
*/
|
|
25
|
+
export function searchAll(db, query, opts = {}) {
|
|
26
|
+
const slug = opts.project ? resolveProject(db, opts.project) : null;
|
|
27
|
+
const now = opts.now ?? new Date().toISOString();
|
|
28
|
+
const limit = opts.limit ?? 8;
|
|
29
|
+
const { everywhere, since, before } = opts;
|
|
30
|
+
let memories = NONE;
|
|
31
|
+
let said = NONE;
|
|
32
|
+
let fellBack = false;
|
|
33
|
+
for (const terms of new Set(words(query)).size > 1 ? ['all', 'any'] : ['all']) {
|
|
34
|
+
memories = opts.turns ? NONE : search(db, query, { project: opts.project, type: opts.type, all: opts.all, everywhere, since, before, limit, now, terms, record: false });
|
|
35
|
+
said = opts.type ? NONE : searchJournal(db, query, { terms, project: slug, everywhere, since, before, limit: memories.results.length > 0 ? Math.min(limit, SAID_AFTER_MEMORIES) : limit });
|
|
36
|
+
fellBack = terms === 'any';
|
|
37
|
+
if (memories.results.length + said.hits.length > 0) break;
|
|
38
|
+
}
|
|
39
|
+
const shown = memories.results.length + said.hits.length;
|
|
40
|
+
const loose = fellBack && shown > 0 && Math.max(memories.best, said.best) < LOOSE_BELOW;
|
|
41
|
+
if (!opts.turns && opts.record !== false) {
|
|
42
|
+
markHits(db, memories.results.map((m) => m.id), now);
|
|
43
|
+
if (shown === 0 || loose) recordMiss(db, query, everywhere ? 'everywhere' : slug ? `project:${slug}` : 'global', now);
|
|
44
|
+
}
|
|
45
|
+
const elsewhere = { ...memories.elsewhere };
|
|
46
|
+
for (const [scope, n] of Object.entries(said.elsewhere)) elsewhere[scope] = (elsewhere[scope] ?? 0) + n;
|
|
47
|
+
return { memories: memories.results, conversations: said.hits, more: memories.more, moreSaid: said.more, elsewhere, loose };
|
|
48
|
+
}
|
package/src/sessions.mjs
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
1
2
|
import { closeSync, mkdirSync, openSync, readdirSync, readFileSync, readSync, renameSync, statSync, unlinkSync, writeFileSync } from 'node:fs';
|
|
2
3
|
import { join } from 'node:path';
|
|
3
4
|
import { ago } from './card.mjs';
|
|
@@ -5,10 +6,18 @@ import { heldBy } from './lock.mjs';
|
|
|
5
6
|
import { Refusal, UsageError } from './memory.mjs';
|
|
6
7
|
import { paths } from './paths.mjs';
|
|
7
8
|
import { redact } from './redact.mjs';
|
|
8
|
-
import { cjkTerms, clip, containing, head } from './text.mjs';
|
|
9
|
+
import { cjkTerms, clip, containing, ftsAllQuery, ftsQuery, head, stem, termsHeld, words } from './text.mjs';
|
|
9
10
|
import { readBackwards } from './transcript.mjs';
|
|
10
11
|
|
|
11
|
-
|
|
12
|
+
/**
|
|
13
|
+
* How much of one turn or reply the journal keeps: whole, in practice. The journal is what memory is rebuilt from and
|
|
14
|
+
* what search falls back on, and at 4,000 characters the pasted error or the long explanation a later question was
|
|
15
|
+
* about had been cut away. Past this it is a pasted file, and its head is kept.
|
|
16
|
+
*/
|
|
17
|
+
const JOURNAL_MAX_CHARS = 64_000;
|
|
18
|
+
const CUT = ' …[cut]';
|
|
19
|
+
/** A conversation nobody has spoken in for this long is over, as far as reading it for memory goes. */
|
|
20
|
+
const FINISHED_AFTER_MS = 30 * 60_000;
|
|
12
21
|
|
|
13
22
|
/**
|
|
14
23
|
* Where a session starts costing quality. The numbers are absolute tokens, not
|
|
@@ -33,25 +42,185 @@ export function ensureSession(db, { id, harness = 'claude', cwd = null, transcri
|
|
|
33
42
|
).run(id, harness, cwd, transcriptPath, now);
|
|
34
43
|
}
|
|
35
44
|
|
|
45
|
+
/** A slash command, a worker's brief and an empty line are not something the user said. A command's name may carry a dash or a plugin prefix (`/code-review`, `/codex:rescue`); a path (`/usr/bin`) still has a slash after its first word. */
|
|
46
|
+
const notSaid = (trimmed) => !trimmed || /^\/[\w:-]+(\s|$)/.test(trimmed) || trimmed.startsWith('JOB:');
|
|
47
|
+
|
|
36
48
|
/**
|
|
37
|
-
*
|
|
38
|
-
*
|
|
39
|
-
* worth keeping (a slash command, a worker's brief, an empty line).
|
|
49
|
+
* Words as the journal keeps them: secrets out first, then the cut — half a token left at the edge of a cut would
|
|
50
|
+
* still be a leak. `fullLen` is how long they were, secrets out, before any cut. Null for nothing at all.
|
|
40
51
|
*/
|
|
41
|
-
|
|
42
|
-
const trimmed =
|
|
43
|
-
|
|
44
|
-
if (!trimmed || /^\/[\w:-]+(\s|$)/.test(trimmed) || trimmed.startsWith('JOB:')) return null;
|
|
45
|
-
|
|
52
|
+
function journalText(raw) {
|
|
53
|
+
const trimmed = String(raw ?? '').trim();
|
|
54
|
+
if (!trimmed) return null;
|
|
46
55
|
const cleaned = redact(trimmed);
|
|
47
|
-
const
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
56
|
+
const cut = cleaned.text.length > JOURNAL_MAX_CHARS;
|
|
57
|
+
return { text: cut ? `${head(cleaned.text, JOURNAL_MAX_CHARS)}${CUT}` : cleaned.text, fullLen: cleaned.text.length, redacted: cleaned.count > 0 };
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* What the user typed, as the journal would keep it — `{ text, fullLen, redacted }` — or null when it is nothing
|
|
62
|
+
* the journal records (a slash command, a `JOB:` brief, an empty line). For whoever must keep a turn somewhere
|
|
63
|
+
* else first (the spool): the words leave here already redacted and cut.
|
|
64
|
+
*/
|
|
65
|
+
export const turnText = (raw) => (notSaid(String(raw ?? '').trim()) ? null : journalText(raw));
|
|
66
|
+
/** The same for the assistant's reply; null only when it said nothing. */
|
|
67
|
+
export const replyText = (raw) => journalText(raw);
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Words kept somewhere else first, on their way into the journal. They were redacted and cut when they were kept;
|
|
71
|
+
* they are redacted again here, since a file is not the hand that wrote it, and never cut a second time — a cut of a
|
|
72
|
+
* cut would be other words, and another digest.
|
|
73
|
+
*/
|
|
74
|
+
function fromKept(kept) {
|
|
75
|
+
if (typeof kept?.text !== 'string' || !kept.text.trim()) return null;
|
|
76
|
+
const cleaned = redact(kept.text.trim());
|
|
77
|
+
const text = cleaned.text.length > JOURNAL_MAX_CHARS + CUT.length ? `${head(cleaned.text, JOURNAL_MAX_CHARS)}${CUT}` : cleaned.text;
|
|
78
|
+
return { text, fullLen: Number.isInteger(kept.fullLen) && kept.fullLen >= 0 ? kept.fullLen : text.length, redacted: kept.redacted === true || cleaned.count > 0 };
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** One journal row's digest: its session, its time and its words as kept. The same event written twice is the same row. */
|
|
82
|
+
const shaOf = (sessionId, ts, text) => createHash('sha256').update(`${sessionId}\n${ts}\n${text}`).digest('hex');
|
|
83
|
+
|
|
84
|
+
/** The project a session was about at a moment: the one whose card had gone in most recently by then. */
|
|
85
|
+
function projectAsOf(db, sessionId, at) {
|
|
86
|
+
return db.prepare(`SELECT slug FROM session_injections WHERE session_id = ? AND slug NOT LIKE '%:%' AND ts <= ? ORDER BY ts DESC LIMIT 1`).get(sessionId, at)?.slug ?? null;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* Stores what the user typed, word for word and whole (the first 64,000 characters of a longer paste, the cut
|
|
91
|
+
* marked, `full_len` saying how long it was) — the ground truth every derived memory points back to. It is stamped
|
|
92
|
+
* with the project the session was about, and with a digest of its session, time and words: the same turn written
|
|
93
|
+
* again — an event replayed from the spool — inserts nothing and returns the row already there. Returns the turn id,
|
|
94
|
+
* or null when there was nothing worth keeping (a slash command, a worker's brief, an empty line).
|
|
95
|
+
*
|
|
96
|
+
* `kept` is for a replay: `{ text, fullLen, redacted }` as `turnText` made them when the event was set aside.
|
|
97
|
+
*/
|
|
98
|
+
export function recordTurn(db, { sessionId, text, now, kept = null }) {
|
|
99
|
+
const said = kept ? fromKept(kept) : turnText(text);
|
|
100
|
+
if (!said || notSaid(said.text)) return null;
|
|
101
|
+
const sha = shaOf(sessionId, now, said.text);
|
|
102
|
+
const { changes, lastInsertRowid } = db
|
|
103
|
+
.prepare('INSERT OR IGNORE INTO user_turns (session_id, ts, text, redacted, project, full_len, sha) VALUES (?, ?, ?, ?, ?, ?, ?)')
|
|
104
|
+
.run(sessionId, now, said.text, said.redacted ? 1 : 0, projectAsOf(db, sessionId, now), said.fullLen, sha);
|
|
105
|
+
if (Number(changes) === 0) return db.prepare('SELECT id FROM user_turns WHERE sha = ?').get(sha).id;
|
|
106
|
+
// A turn replayed late is older than what the session has heard since: the newest time stays.
|
|
107
|
+
db.prepare("UPDATE sessions SET last_turn_at = MAX(COALESCE(last_turn_at, ''), ?), dream_state = 'pending' WHERE id = ?").run(now, sessionId);
|
|
52
108
|
return Number(lastInsertRowid);
|
|
53
109
|
}
|
|
54
110
|
|
|
111
|
+
/**
|
|
112
|
+
* Stores the assistant's final reply of a turn in the journal, beside the user's turns: redacted like a turn, whole
|
|
113
|
+
* up to 64,000 characters, stamped with the session's project, and with the same kind of digest, so a replay inserts
|
|
114
|
+
* nothing twice. It is what search falls back on and what the memory pass reads for context — never itself a memory:
|
|
115
|
+
* only the user's words are. The session's row must exist (`ensureSession`). Returns the reply id, or null when
|
|
116
|
+
* nothing was said.
|
|
117
|
+
*/
|
|
118
|
+
export function recordReply(db, { sessionId, text, now, kept = null }) {
|
|
119
|
+
const said = kept ? fromKept(kept) : replyText(text);
|
|
120
|
+
if (!said) return null;
|
|
121
|
+
const sha = shaOf(sessionId, now, said.text);
|
|
122
|
+
const { changes, lastInsertRowid } = db
|
|
123
|
+
.prepare('INSERT OR IGNORE INTO replies (session_id, ts, text, project, full_len, sha) VALUES (?, ?, ?, ?, ?, ?)')
|
|
124
|
+
.run(sessionId, now, said.text, projectAsOf(db, sessionId, now), said.fullLen, sha);
|
|
125
|
+
return Number(changes) === 0 ? db.prepare('SELECT id FROM replies WHERE sha = ?').get(sha).id : Number(lastInsertRowid);
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* A session heard of late: an event that could not be written when it happened, replayed with the time it happened.
|
|
130
|
+
* Unlike `ensureSession` it never reopens a session that has ended since — the event is from before that — and an
|
|
131
|
+
* older start replaces a newer one.
|
|
132
|
+
*/
|
|
133
|
+
export function heardSession(db, { id, harness = 'claude', cwd = null, transcriptPath = null, now }) {
|
|
134
|
+
db.prepare(
|
|
135
|
+
`INSERT INTO sessions (id, harness, cwd, transcript_path, started_at) VALUES (?, ?, ?, ?, ?)
|
|
136
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
137
|
+
cwd = COALESCE(cwd, excluded.cwd),
|
|
138
|
+
transcript_path = COALESCE(transcript_path, excluded.transcript_path),
|
|
139
|
+
started_at = MIN(started_at, excluded.started_at)`,
|
|
140
|
+
).run(id, harness, cwd, transcriptPath, now);
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/** A goodbye replayed late: the session ended then, unless it has been heard from since or has ended already. */
|
|
144
|
+
export function endedThen(db, id, at) {
|
|
145
|
+
db.prepare('UPDATE sessions SET ended_at = ? WHERE id = ? AND ended_at IS NULL AND COALESCE(last_turn_at, started_at) <= ?').run(at, id, at);
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* One conversation as the journal holds it, in the order it was said: the user's turns and the assistant's final
|
|
150
|
+
* replies of a session, each `{ kind: 'turn' | 'reply', id, session_id, ts, text, project, full_len }`. `id` is the
|
|
151
|
+
* row's own in `user_turns` or `replies` (the two count separately). Ordered by time, not by id: a turn replayed
|
|
152
|
+
* from the spool has a late id and its own time; at the same instant the turn comes before the reply.
|
|
153
|
+
*/
|
|
154
|
+
export function sessionJournal(db, sessionId) {
|
|
155
|
+
return db
|
|
156
|
+
.prepare(
|
|
157
|
+
`SELECT 'turn' AS kind, id, session_id, ts, text, project, full_len FROM user_turns WHERE session_id = ?1
|
|
158
|
+
UNION ALL
|
|
159
|
+
SELECT 'reply' AS kind, id, session_id, ts, text, project, full_len FROM replies WHERE session_id = ?1
|
|
160
|
+
ORDER BY ts, kind DESC, id`,
|
|
161
|
+
)
|
|
162
|
+
.all(sessionId);
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
const COVERED = (versions) => `EXISTS (SELECT 1 FROM extractions x WHERE x.session_id = t.session_id AND x.status IN ('ok', 'quarantined')
|
|
166
|
+
${versions.length > 0 ? `AND x.extractor_version IN (${versions.map(() => '?').join(', ')}) ` : ''}AND t.id BETWEEN x.first_turn AND x.last_turn)`;
|
|
167
|
+
/** No version named is every version: a turn any extractor read counts as read. */
|
|
168
|
+
const listOf = (version) => (version === undefined || version === null ? [] : Array.isArray(version) ? version : [version]);
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* The finished conversations the memory pass has yet to read: sessions rows, oldest first, that are over — ended, or
|
|
172
|
+
* nobody has spoken or been answered in them for 30 minutes (a closed terminal never says goodbye) — and hold a turn
|
|
173
|
+
* no `ok` or `quarantined` extraction covers (its id within the row's `first_turn`…`last_turn`). A `failed` or
|
|
174
|
+
* `running` row covers nothing: the turn is still owed.
|
|
175
|
+
*
|
|
176
|
+
* With no `version` an extraction under any extractor covers, which is what the pass asks: a changed prompt, model
|
|
177
|
+
* or rule must not make every conversation owed again and have them all read anew by themselves — reading again under
|
|
178
|
+
* the current extractor is `sumo memory rebuild`, which shows its difference first. `version`, one or several,
|
|
179
|
+
* counts only those extractors' rows. `includeOpen` is for a pass the user asked for by hand, which waits for nothing.
|
|
180
|
+
*/
|
|
181
|
+
export function sessionsAwaitingExtraction(db, { version, now, limit = 5, includeOpen = false }) {
|
|
182
|
+
const versions = listOf(version);
|
|
183
|
+
const cutoff = new Date(Date.parse(now) - FINISHED_AFTER_MS).toISOString();
|
|
184
|
+
return db
|
|
185
|
+
.prepare(
|
|
186
|
+
`SELECT s.* FROM sessions s
|
|
187
|
+
WHERE (? = 1 OR s.ended_at IS NOT NULL
|
|
188
|
+
OR MAX(COALESCE(s.last_turn_at, s.started_at), COALESCE((SELECT MAX(r.ts) FROM replies r WHERE r.session_id = s.id), '')) < ?)
|
|
189
|
+
AND EXISTS (SELECT 1 FROM user_turns t WHERE t.session_id = s.id AND NOT ${COVERED(versions)})
|
|
190
|
+
ORDER BY s.started_at, s.id LIMIT ?`,
|
|
191
|
+
)
|
|
192
|
+
.all(includeOpen ? 1 : 0, cutoff, ...versions, limit);
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
/** The turns of one session that no `ok` or `quarantined` extraction covers — of any extractor, or of `version` (one, or several) when given — oldest first: what a pass still owes it. */
|
|
196
|
+
export function turnsAwaitingExtraction(db, sessionId, version) {
|
|
197
|
+
const versions = listOf(version);
|
|
198
|
+
return db.prepare(`SELECT t.* FROM user_turns t WHERE t.session_id = ? AND NOT ${COVERED(versions)} ORDER BY t.ts, t.id`).all(sessionId, ...versions);
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/**
|
|
202
|
+
* When the user was last heard: the time of the newest turn in the journal — or of a later prompt that left no turn
|
|
203
|
+
* (`noteHeard`) — or null when there is none. With a
|
|
204
|
+
* `harness` — 'claude' for what came in through `sumo hook`, 'sumo' for sumo's own chat — only that one's sessions:
|
|
205
|
+
* a harness whose hook lines are gone goes quiet while the other keeps talking.
|
|
206
|
+
*/
|
|
207
|
+
export function lastHeard(db, harness = null) {
|
|
208
|
+
const said = db.prepare('SELECT MAX(t.ts) AS at FROM user_turns t JOIN sessions s ON s.id = t.session_id WHERE (? IS NULL OR s.harness = ?)').get(harness, harness).at ?? null;
|
|
209
|
+
const stamped = db.prepare(`SELECT MAX(value) AS at FROM meta WHERE key LIKE ? ESCAPE '\\'`).get(`${HEARD_AT}${harness ?? '%'}`).at ?? null;
|
|
210
|
+
return said !== null && stamped !== null ? (said > stamped ? said : stamped) : (said ?? stamped);
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/** Where the time of a harness's last prompt that left no turn is kept: `heard_at.claude`, `heard_at.sumo`. */
|
|
214
|
+
const HEARD_AT = 'heard\\_at.';
|
|
215
|
+
|
|
216
|
+
/**
|
|
217
|
+
* A prompt that is nothing the journal keeps — a slash command, a brief — is still the harness being heard: its time
|
|
218
|
+
* is kept, so days of `/compact` and `/clear` are not read as hook lines gone missing.
|
|
219
|
+
*/
|
|
220
|
+
export function noteHeard(db, harness, now) {
|
|
221
|
+
db.prepare('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = MAX(value, excluded.value)').run(`heard_at.${harness}`, now);
|
|
222
|
+
}
|
|
223
|
+
|
|
55
224
|
/**
|
|
56
225
|
* A call that reached past the machine — a web search or fetch, an MCP server — written on its session, each tool once:
|
|
57
226
|
* what the conversation says after it may be what a page told it, so the memory pass holds what it would file from the
|
|
@@ -111,46 +280,128 @@ export function addCheckpoint(db, { sessionId = null, project, done, next = null
|
|
|
111
280
|
db.prepare('UPDATE projects SET last_touched_at = ? WHERE slug = ?').run(now, project);
|
|
112
281
|
}
|
|
113
282
|
|
|
283
|
+
/** The two halves of the journal, and the letter each one's ids are typed with: `t481` is a turn, `r3` a reply. */
|
|
284
|
+
const JOURNAL = { turn: { table: 'user_turns', fts: 'user_turns_fts', mark: 't' }, reply: { table: 'replies', fts: 'replies_fts', mark: 'r' } };
|
|
285
|
+
/** How much of a turn or a reply one line of a search shows: the part around the words asked for. */
|
|
286
|
+
const EXCERPT_CHARS = 200;
|
|
287
|
+
const EXCERPT_LEAD = 40;
|
|
288
|
+
|
|
114
289
|
/**
|
|
115
|
-
*
|
|
116
|
-
*
|
|
117
|
-
* or a crash never sends the goodbye. `includeOpen` is for a pass the user asked
|
|
118
|
-
* for by hand, which should not wait for anything.
|
|
290
|
+
* The stretch of a text that holds the most of the words searched for, on one line: a reply can be pages long, and
|
|
291
|
+
* its first words are seldom where the answer is. A word is looked for by its stem, so "deploys" shows for "deploy".
|
|
119
292
|
*/
|
|
120
|
-
|
|
121
|
-
const
|
|
122
|
-
return
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
)
|
|
130
|
-
.all(includeOpen ? 1 : 0, cutoff, limit);
|
|
293
|
+
function excerpt(text, query) {
|
|
294
|
+
const flat = text.replace(/\s+/g, ' ').trim();
|
|
295
|
+
if (flat.length <= EXCERPT_CHARS) return flat;
|
|
296
|
+
const lower = flat.toLowerCase();
|
|
297
|
+
const at = [...new Set(words(query))].map((w) => lower.indexOf(lower.includes(w) ? w : stem(w))).filter((i) => i >= 0);
|
|
298
|
+
const within = (from) => at.filter((i) => i >= from && i < from + EXCERPT_CHARS - EXCERPT_LEAD).length;
|
|
299
|
+
const first = at.reduce((best, i) => (best === null || within(i) > within(best) || (within(i) === within(best) && i < best) ? i : best), null) ?? 0;
|
|
300
|
+
const from = Math.max(0, Math.min(first - EXCERPT_LEAD, flat.length - EXCERPT_CHARS));
|
|
301
|
+
return `${from > 0 ? '…' : ''}${head(flat.slice(from), EXCERPT_CHARS).trim()}${from + EXCERPT_CHARS < flat.length ? '…' : ''}`;
|
|
131
302
|
}
|
|
132
303
|
|
|
133
304
|
/**
|
|
134
|
-
*
|
|
135
|
-
*
|
|
136
|
-
*
|
|
305
|
+
* One half of the journal searched: the rows in scope that hold every word (`every`) or any of them, best first, and
|
|
306
|
+
* how many rows matched per project. The index answers for the words it keeps apart; a Chinese, Japanese or Korean
|
|
307
|
+
* word sits inside a run the index keeps as one word, so it is looked for in the text itself — a row found only that
|
|
308
|
+
* way ranks by the share of those words it holds, the best the index found counting as all of them.
|
|
137
309
|
*/
|
|
138
|
-
|
|
139
|
-
const
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
const
|
|
146
|
-
|
|
147
|
-
const
|
|
148
|
-
const
|
|
149
|
-
|
|
150
|
-
const
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
310
|
+
function saidIn(db, kind, query, every, { project, everywhere, since, before, limit }) {
|
|
311
|
+
const { table, fts, mark } = JOURNAL[kind];
|
|
312
|
+
const cjk = cjkTerms(query);
|
|
313
|
+
const match = every ? ftsAllQuery(query) : ftsQuery(query);
|
|
314
|
+
const likes = cjk.map(() => `t.text LIKE ? ESCAPE '\\'`);
|
|
315
|
+
const patterns = cjk.map(containing);
|
|
316
|
+
const dated = `${since ? ' AND t.ts >= ?' : ''}${before ? ' AND t.ts < ?' : ''}`;
|
|
317
|
+
const dates = [since, before].filter(Boolean);
|
|
318
|
+
// What was said about no project is everyone's, as a global memory is; another project's is counted, never shown.
|
|
319
|
+
const mine = everywhere ? '' : project ? ' AND (t.project IS NULL OR t.project = ?)' : ' AND t.project IS NULL';
|
|
320
|
+
const mineParams = everywhere || !project ? [] : [project];
|
|
321
|
+
|
|
322
|
+
const inIndex = match ? `t.id IN (SELECT rowid FROM ${fts} WHERE ${fts} MATCH ?)` : null;
|
|
323
|
+
const holds = [inIndex, ...likes].filter(Boolean).join(every ? ' AND ' : ' OR ');
|
|
324
|
+
const counted = db.prepare(`SELECT t.project AS project, COUNT(*) AS n FROM ${table} t WHERE (${holds})${dated} GROUP BY t.project`).all(...(match ? [match] : []), ...patterns, ...dates);
|
|
325
|
+
|
|
326
|
+
const cols = 't.id, t.session_id, t.ts, t.text, t.project, t.full_len';
|
|
327
|
+
const indexed = match
|
|
328
|
+
? db
|
|
329
|
+
.prepare(
|
|
330
|
+
`SELECT ${cols}, bm25(${fts}) AS rank FROM ${fts} JOIN ${table} t ON t.id = ${fts}.rowid
|
|
331
|
+
WHERE ${fts} MATCH ?${every ? likes.map((l) => ` AND ${l}`).join('') : ''}${dated}${mine} ORDER BY rank, t.ts DESC, t.id DESC LIMIT ?`,
|
|
332
|
+
)
|
|
333
|
+
.all(match, ...(every ? patterns : []), ...dates, ...mineParams, limit)
|
|
334
|
+
: [];
|
|
335
|
+
const byId = new Map(indexed.map((row) => [row.id, row]));
|
|
336
|
+
if (cjk.length > 0 && (!every || !match)) {
|
|
337
|
+
const best = indexed.reduce((min, row) => Math.min(min, row.rank), 0) || -1;
|
|
338
|
+
const inText = db
|
|
339
|
+
.prepare(`SELECT ${cols} FROM ${table} t WHERE (${likes.join(every ? ' AND ' : ' OR ')})${dated}${mine} ORDER BY ${likes.map((l) => `(${l})`).join(' + ')} DESC, t.ts DESC, t.id DESC LIMIT ?`)
|
|
340
|
+
.all(...patterns, ...dates, ...mineParams, ...patterns, limit);
|
|
341
|
+
for (const row of inText) {
|
|
342
|
+
const lower = row.text.toLowerCase();
|
|
343
|
+
const rank = (best * cjk.filter((t) => lower.includes(t)).length) / cjk.length;
|
|
344
|
+
const known = byId.get(row.id);
|
|
345
|
+
if (!known || rank < known.rank) byId.set(row.id, { ...row, rank });
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
const rows = [...byId.values()].map(({ text, ...row }) => ({ kind, ref: `${mark}${row.id}`, ...row, excerpt: excerpt(text, query), held: termsHeld(query, text) }));
|
|
349
|
+
return { rows, counted };
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
/**
|
|
353
|
+
* The conversations themselves, searched: what the user typed and what sumo answered, where the answer to "what did
|
|
354
|
+
* we say about …" lives when nobody made a memory of it. All the words first; only when no row in scope holds every
|
|
355
|
+
* one is it tried with any of them (`fellBack`); `terms: 'all' | 'any'` asks for one of the two alone.
|
|
356
|
+
*
|
|
357
|
+
* Scope is as memory has it: a row said about no project, and the rows of `project` (a slug); another project's rows
|
|
358
|
+
* are counted in `elsewhere` (`project:<slug>` → how many), never shown, unless `everywhere`. `since` and `before`
|
|
359
|
+
* (ISO times) hold it to when it was said. Each hit is `{ kind: 'turn' | 'reply', ref, id, session_id, ts, project,
|
|
360
|
+
* full_len, excerpt, held, rank }` — `ref` is the id as it is typed (`t481`, `r3`), `excerpt` the part around the
|
|
361
|
+
* words, `held` how many of the search's words the whole text holds. `more` counts what matched in scope past `limit`;
|
|
362
|
+
* `best` is the most `held` among the hits.
|
|
363
|
+
*/
|
|
364
|
+
export function searchJournal(db, query, { terms, project = null, everywhere = false, since, before, limit = 8 } = {}) {
|
|
365
|
+
if (!ftsQuery(query)) throw new UsageError('nothing to search for');
|
|
366
|
+
const several = new Set(words(query)).size > 1;
|
|
367
|
+
let hits = [];
|
|
368
|
+
let counted = [];
|
|
369
|
+
let fellBack = false;
|
|
370
|
+
for (const every of terms === 'any' ? [false] : terms === 'all' || !several ? [true] : [true, false]) {
|
|
371
|
+
const halves = Object.keys(JOURNAL).map((kind) => saidIn(db, kind, query, every, { project, everywhere, since, before, limit }));
|
|
372
|
+
hits = halves.flatMap((h) => h.rows).sort((a, b) => a.rank - b.rank || (a.ts < b.ts ? 1 : a.ts > b.ts ? -1 : 0) || b.id - a.id);
|
|
373
|
+
counted = halves.flatMap((h) => h.counted);
|
|
374
|
+
fellBack = !every && terms !== 'any';
|
|
375
|
+
if (hits.length > 0) break;
|
|
376
|
+
}
|
|
377
|
+
const elsewhere = {};
|
|
378
|
+
let inScope = 0;
|
|
379
|
+
for (const { project: said, n } of counted) {
|
|
380
|
+
if (everywhere || said === null || said === project) inScope += n;
|
|
381
|
+
else elsewhere[`project:${said}`] = (elsewhere[`project:${said}`] ?? 0) + n;
|
|
382
|
+
}
|
|
383
|
+
const shown = hits.slice(0, limit);
|
|
384
|
+
return { hits: shown, elsewhere, more: inScope - shown.length, fellBack, best: shown.reduce((most, h) => Math.max(most, h.held), 0) };
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
/** How many turns and replies either side of one are shown with it. */
|
|
388
|
+
const AROUND = 2;
|
|
389
|
+
|
|
390
|
+
/**
|
|
391
|
+
* One turn or reply by the id a search printed (`t481`, `r3`), whole, with what was said around it in its
|
|
392
|
+
* conversation: `{ entry, around }`, both as `sessionJournal` gives rows, `around` in the order it was said and
|
|
393
|
+
* holding the entry itself. Null when `ref` is not such an id; a refusal when it names nothing.
|
|
394
|
+
*/
|
|
395
|
+
export function journalEntry(db, ref) {
|
|
396
|
+
const typed = /^([tr])(\d+)$/.exec(String(ref ?? '').trim().toLowerCase());
|
|
397
|
+
if (!typed) return null;
|
|
398
|
+
const kind = typed[1] === 't' ? 'turn' : 'reply';
|
|
399
|
+
const id = Number(typed[2]);
|
|
400
|
+
const row = db.prepare(`SELECT session_id FROM ${JOURNAL[kind].table} WHERE id = ?`).get(id);
|
|
401
|
+
if (!row) throw new Refusal(`no ${kind} ${typed[0]} in the conversations — sumo search "<words>" prints the ids it finds`);
|
|
402
|
+
const said = sessionJournal(db, row.session_id).map((e) => ({ ...e, ref: `${JOURNAL[e.kind].mark}${e.id}` }));
|
|
403
|
+
const at = said.findIndex((e) => e.kind === kind && e.id === id);
|
|
404
|
+
return { entry: said[at], around: said.slice(Math.max(0, at - AROUND), at + AROUND + 1) };
|
|
154
405
|
}
|
|
155
406
|
|
|
156
407
|
/**
|
|
@@ -298,15 +549,15 @@ export function loadState(id) {
|
|
|
298
549
|
function summaryOf(file) {
|
|
299
550
|
try {
|
|
300
551
|
const fd = openSync(file, 'r');
|
|
301
|
-
let text;
|
|
302
552
|
try {
|
|
303
553
|
const buffer = Buffer.alloc(SUMMARY_BYTES);
|
|
304
|
-
text = buffer.toString('utf8', 0, readSync(fd, buffer, 0, SUMMARY_BYTES, 0));
|
|
554
|
+
const text = buffer.toString('utf8', 0, readSync(fd, buffer, 0, SUMMARY_BYTES, 0));
|
|
555
|
+
const cut = text.indexOf(',"messages":');
|
|
556
|
+
// The read above was at a position, so the descriptor is still at the start: a head with no cut is read whole from it.
|
|
557
|
+
return JSON.parse(cut === -1 ? readFileSync(fd, 'utf8') : `${text.slice(0, cut)}}`);
|
|
305
558
|
} finally {
|
|
306
559
|
closeSync(fd);
|
|
307
560
|
}
|
|
308
|
-
const cut = text.indexOf(',"messages":');
|
|
309
|
-
return JSON.parse(cut === -1 ? readFileSync(file, 'utf8') : `${text.slice(0, cut)}}`);
|
|
310
561
|
} catch {
|
|
311
562
|
return null;
|
|
312
563
|
}
|