sumo-agents 0.4.0 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/AGENTS.md +8 -6
  2. package/CHANGELOG.md +187 -0
  3. package/README.md +76 -18
  4. package/bin/sumo.mjs +13 -2
  5. package/guides/feature.md +2 -1
  6. package/guides/memory.md +6 -4
  7. package/guides/projects.md +1 -1
  8. package/guides/review.md +1 -1
  9. package/package.json +5 -4
  10. package/prompts/dream.md +6 -0
  11. package/spinner.txt +25 -1
  12. package/src/apply.mjs +146 -18
  13. package/src/auth.mjs +32 -5
  14. package/src/card.mjs +39 -2
  15. package/src/catalog.mjs +6 -3
  16. package/src/chat.mjs +186 -75
  17. package/src/cli.mjs +359 -135
  18. package/src/cost.mjs +67 -8
  19. package/src/db.mjs +106 -3
  20. package/src/dream.mjs +445 -19
  21. package/src/editor.mjs +3 -2
  22. package/src/export.mjs +105 -7
  23. package/src/guard.mjs +57 -11
  24. package/src/guides.mjs +5 -4
  25. package/src/herdr.mjs +1 -1
  26. package/src/hooks.mjs +31 -18
  27. package/src/jobs.mjs +118 -39
  28. package/src/jobtools.mjs +6 -4
  29. package/src/keeper.mjs +77 -0
  30. package/src/local-server.mjs +9 -16
  31. package/src/lock.mjs +2 -1
  32. package/src/logs.mjs +165 -0
  33. package/src/loop.mjs +121 -44
  34. package/src/mcp.mjs +72 -16
  35. package/src/memory.mjs +142 -15
  36. package/src/model.mjs +41 -21
  37. package/src/paths.mjs +38 -0
  38. package/src/prime.mjs +19 -31
  39. package/src/projects.mjs +76 -12
  40. package/src/redact.mjs +10 -0
  41. package/src/render.mjs +2 -1
  42. package/src/route.mjs +16 -22
  43. package/src/sandbox.mjs +82 -3
  44. package/src/scan.mjs +35 -1
  45. package/src/scribe.mjs +10 -22
  46. package/src/sessions.mjs +46 -11
  47. package/src/setup.mjs +423 -78
  48. package/src/text.mjs +100 -1
  49. package/src/tools.mjs +258 -21
  50. package/src/tty.mjs +210 -44
  51. package/src/ui.mjs +816 -226
  52. package/src/verify.mjs +17 -1
package/src/dream.mjs CHANGED
@@ -1,7 +1,41 @@
1
- import { knownProjectsLine, askAndApply, runScribe, withLock, writerOff } from './scribe.mjs';
1
+ import { readFileSync } from 'node:fs';
2
+ import { join } from 'node:path';
3
+ import { keepNamed, namedPairs, nameForJudging, raiseNotice } from './apply.mjs';
4
+ import { getMeta, setMeta, tx } from './db.mjs';
5
+ import { storedKeys } from './memory.mjs';
6
+ import { callModel, logRun } from './model.mjs';
7
+ import { REPO_ROOT } from './paths.mjs';
8
+ import { redact } from './redact.mjs';
9
+ import { knownProjectsLine, askAndApply, relatedMemories, runScribe, withLock, writerOff } from './scribe.mjs';
2
10
  import { line } from './render.mjs';
3
11
  import { sessionsAwaitingDream } from './sessions.mjs';
4
- import { clip } from './text.mjs';
12
+ import { CONFIG_DEFAULTS } from './setup.mjs';
13
+ import { clip, stem, words } from './text.mjs';
14
+
15
+ /** Finished sessions waiting that make a pass worth running by itself. */
16
+ export const SESSIONS_TO_TIDY = 3;
17
+ /** How long memory goes between tidies — pairs judged, search terms filled in — when no conversation brings one sooner. */
18
+ const TIDY_EVERY_MS = 7 * 86_400_000;
19
+ /** The oldest memories a pass reads beside the related and the newest, moving on each pass so nothing old goes unread for long. */
20
+ const SWEEP_SLICE = 20;
21
+ /** Pairs put to the local model in one call, and calls in one pass: one when conversations bring it, a few when it is the weekly tidy. */
22
+ const PAIRS_PER_CALL = 10;
23
+ const JUDGE_CALLS = 1;
24
+ const TIDY_JUDGE_CALLS = 3;
25
+ /** Memories given search terms in one tidy: what the local model answers well in one call. */
26
+ const KEYS_PER_TIDY = 20;
27
+ const LABELS = ['same', 'replaces', 'conflict', 'unrelated'];
28
+ /** A topic as the scribe gives one: a lowercase word. */
29
+ const TOPIC = /^[a-z][a-z0-9-]{1,23}$/;
30
+ const SWEEP_CURSOR = 'dream.sweep_cursor';
31
+ const KEYS_CURSOR = 'dream.keys_cursor';
32
+ /** The last rule checked for being said already in other words: each is checked once. */
33
+ const REPEATS_CURSOR = 'dream.repeats_cursor';
34
+ /** Rules checked a call, and the most of the other rules in their scope shown beside them. */
35
+ const REPEATS_PER_CALL = 8;
36
+ const REPEAT_NEIGHBOURS = 40;
37
+ const LAST_TIDY = 'dream.last_tidy';
38
+ const NO_USAGE = { inputTokens: 0, outputTokens: 0, costUsd: 0, cacheReadTokens: null, cacheCreationTokens: null };
5
39
 
6
40
  const MAX_SESSIONS = 5;
7
41
  const TURN_MAX = 600;
@@ -37,12 +71,7 @@ function buildBundle(db, sessions) {
37
71
  const projects = db.prepare(`SELECT DISTINCT slug FROM session_injections WHERE session_id IN (${marks}) AND slug NOT LIKE '%:%'`).all(...ids).map((r) => r.slug);
38
72
 
39
73
  const scopes = ['global', ...projects.map((p) => `project:${p}`)];
40
- const memories = db
41
- .prepare(
42
- `SELECT * FROM memories WHERE state = 'active' AND scan_key IS NULL AND scope IN (${scopes.map(() => '?').join(', ')})
43
- ORDER BY id DESC LIMIT ?`,
44
- )
45
- .all(...scopes, MAX_MEMORIES);
74
+ const { memories, cursor } = shownMemories(db, scopes, turns.map((t) => t.text));
46
75
 
47
76
  const sections = [];
48
77
  for (const session of sessions) {
@@ -56,12 +85,393 @@ function buildBundle(db, sessions) {
56
85
  memories.length > 0 ? `What is already remembered:\n${memories.map(line).join('\n')}` : 'Nothing is remembered yet.',
57
86
  ...sections,
58
87
  ].join('\n');
59
- return { prompt, turns: new Map(turns.map((t) => [t.id, t])) };
88
+ // What it was shown is what it may name as saying the same thing.
89
+ return { prompt, turns: new Map(turns.map((t) => [t.id, t])), shown: new Set(memories.map((m) => m.id)), cursor };
90
+ }
91
+
92
+ /**
93
+ * What a pass is shown of memory, at most MAX_MEMORIES: the memories most related to what was said, a slice of the
94
+ * oldest that moves on with every pass — from where the last one stopped, round to the start again — and the newest
95
+ * for the rest. All of them when there are no more than that. Returns them by id, with where the slice stopped.
96
+ */
97
+ function shownMemories(db, scopes, texts) {
98
+ const pool = db.prepare(`SELECT * FROM memories WHERE state = 'active' AND scan_key IS NULL AND scope IN (${scopes.map(() => '?').join(', ')}) ORDER BY id`).all(...scopes);
99
+ if (pool.length <= MAX_MEMORIES) return { memories: pool, cursor: null };
100
+ const chosen = new Map(relatedMemories(db, texts, scopes).map((m) => [m.id, m]));
101
+ const from = Number(getMeta(db, SWEEP_CURSOR) ?? 0);
102
+ const slice = [...pool.filter((m) => m.id > from), ...pool.filter((m) => m.id <= from)].filter((m) => !chosen.has(m.id)).slice(0, SWEEP_SLICE);
103
+ for (const m of slice) chosen.set(m.id, m);
104
+ for (const m of [...pool].reverse()) {
105
+ if (chosen.size >= MAX_MEMORIES) break;
106
+ chosen.set(m.id, m);
107
+ }
108
+ return { memories: [...chosen.values()].sort((a, b) => a.id - b.id), cursor: slice.at(-1)?.id ?? null };
109
+ }
110
+
111
+ /** The memories the tidy steps read: active, the user's own rather than scanned from a repository, and not workflows. */
112
+ const tidyable = (db) => db.prepare(`SELECT * FROM memories WHERE state = 'active' AND scan_key IS NULL AND type != 'procedure' ORDER BY id`).all();
113
+
114
+ /** A stem that this many memories share says nothing about any two of them: it is left out of the index. */
115
+ const COMMON_STEM = 200;
116
+
117
+ /**
118
+ * Pairs of memories worth asking about, most alike first: in one scope, or one of them global; and sharing three
119
+ * stems, or two beside a topic or a search term — a topic alone is a shelf, not a likeness. Word overlap alone misses
120
+ * the same rule in other words and two values for one setting, so code only picks the pairs and the local model says
121
+ * what they are. A pair judged before, or already raised for the user, is never picked again. A memory is compared
122
+ * only with those it shares a stem with, through an index, and only the best `limit` are kept as they are found:
123
+ * thousands of memories cost a moment, not every pair of them.
124
+ */
125
+ export function candidatePairs(db, limit = PAIRS_PER_CALL) {
126
+ const rows = tidyable(db);
127
+ const judged = new Set(db.prepare('SELECT a, b FROM memory_pairs').all().map((p) => `${p.a}:${p.b}`));
128
+ for (const n of db.prepare('SELECT memory_a, memory_b FROM notices').all()) judged.add(`${Math.min(n.memory_a, n.memory_b)}:${Math.max(n.memory_a, n.memory_b)}`);
129
+ for (const p of namedPairs(db)) judged.add(`${p.a}:${p.b}`);
130
+ const stems = new Map(rows.map((m) => [m.id, new Set(words(m.body).map(stem))]));
131
+ const keys = new Map(rows.map((m) => [m.id, new Set(String(m.keys ?? '').split(',').map((k) => k.trim()).filter(Boolean))]));
132
+ const using = new Map();
133
+ for (const m of rows) for (const s of stems.get(m.id)) (using.get(s) ?? using.set(s, []).get(s)).push(m);
134
+ /** Negative when pair x goes before pair y: more alike first, then the newer. */
135
+ const before = (x, y) => y.rank - x.rank || y.b.id - x.b.id;
136
+ const best = [];
137
+ for (const a of rows) {
138
+ const shared = new Map();
139
+ for (const s of stems.get(a.id)) {
140
+ const others = using.get(s);
141
+ if (others.length > COMMON_STEM) continue;
142
+ for (const b of others) if (b.id > a.id) shared.set(b, (shared.get(b) ?? 0) + 1);
143
+ }
144
+ for (const [b, count] of shared) {
145
+ if (count < 2) continue;
146
+ if (a.scope !== b.scope && a.scope !== 'global' && b.scope !== 'global') continue;
147
+ if (judged.has(`${a.id}:${b.id}`)) continue;
148
+ const topic = Boolean(a.topic) && a.topic === b.topic;
149
+ const term = [...keys.get(a.id)].some((k) => keys.get(b.id).has(k));
150
+ if (count < 3 && !topic && !term) continue;
151
+ const pair = { a, b, rank: count * 2 + (term ? 2 : 0) + (topic ? 1 : 0) };
152
+ if (best.length >= limit && before(pair, best.at(-1)) >= 0) continue;
153
+ const at = best.findIndex((p) => before(pair, p) < 0);
154
+ best.splice(at === -1 ? best.length : at, 0, pair);
155
+ if (best.length > limit) best.pop();
156
+ }
157
+ }
158
+ return best;
159
+ }
160
+
161
+ const PAIRS_SCHEMA = {
162
+ type: 'object',
163
+ properties: {
164
+ pairs: {
165
+ type: 'array',
166
+ items: { type: 'object', properties: { pair: { type: 'integer' }, label: { type: 'string', enum: LABELS } }, required: ['pair', 'label'], additionalProperties: false },
167
+ },
168
+ },
169
+ required: ['pairs'],
170
+ additionalProperties: false,
171
+ };
172
+
173
+ const KEYS_SCHEMA = {
174
+ type: 'object',
175
+ properties: {
176
+ memories: {
177
+ type: 'array',
178
+ items: {
179
+ type: 'object',
180
+ properties: { id: { type: 'integer' }, topic: { type: 'string' }, keys: { type: 'array', items: { type: 'string' } } },
181
+ required: ['id', 'topic', 'keys'],
182
+ additionalProperties: false,
183
+ },
184
+ },
185
+ },
186
+ required: ['memories'],
187
+ additionalProperties: false,
188
+ };
189
+
190
+ /** Whether the tidy steps may run: they call the local model only, never one that is paid for. */
191
+ const tidiesLocally = (db) => (getMeta(db, 'config.memory.model') ?? CONFIG_DEFAULTS['memory.model']) === 'local';
192
+
193
+ /** One call of the tidy steps to the local model, with the consolidation pass's own instructions, and its ledger row. */
194
+ async function askLocal(db, { prompt, schema, kind, now }) {
195
+ const system = readFileSync(join(REPO_ROOT, 'prompts', 'dream.md'), 'utf8').trim();
196
+ const result = await callModel(db, { system, prompt, schema, model: 'local', kind });
197
+ return { result, log: (note) => logRun(db, { kind: 'dream', model: 'local', result, note, now }) };
60
198
  }
61
199
 
62
- export async function runDream(db, { now = new Date().toISOString(), force = false } = {}) {
63
- // Anything still unfiled belongs to the writer first; consolidation reads its results.
64
- await runScribe(db, { now });
200
+ const added = (x, y) => ({ ...x, inputTokens: x.inputTokens + y.inputTokens, outputTokens: x.outputTokens + y.outputTokens, costUsd: x.costUsd + y.costUsd });
201
+
202
+ const describe = (m) => `m${m.id} (${m.type}, ${m.scope}, ${String(m.created_at).slice(0, 10)}): ${clip(m.body, 300)}`;
203
+
204
+ /**
205
+ * The pairs a pass named that are still worth asking about, the first `limit` of them, as `{ a, b, named }`. One whose
206
+ * memory is gone, or no longer active, waits no longer.
207
+ */
208
+ function namedToJudge(db, limit = PAIRS_PER_CALL) {
209
+ const row = db.prepare(`SELECT * FROM memories WHERE id = ? AND state = 'active'`);
210
+ const named = namedPairs(db).map((p) => ({ a: row.get(p.a), b: row.get(p.b), named: p }));
211
+ const live = named.filter((p) => p.a && p.b);
212
+ if (live.length < named.length) keepNamed(db, live.map((p) => p.named));
213
+ return live.slice(0, limit);
214
+ }
215
+
216
+ /**
217
+ * Pairs judged, and what the judgement means raised for the user: the same thing said twice, one that may replace
218
+ * the other, or two that cannot both be true — the look-alike and contradiction notices the block already asks about.
219
+ * Nothing is superseded or forgotten here; the user settles it. Every pair answered is kept, so it is asked once. A
220
+ * model reading a list names pairs too — a rule said twice, a clash — and those are judged first, side by side like
221
+ * the rest: raised as the judgement says, the note it was named with kept when it is judged the way it was named.
222
+ * `fill` false judges only those.
223
+ */
224
+ async function judgePairs(db, { calls, now, fill = true }) {
225
+ const out = { applied: [], dropped: [], usage: NO_USAGE, calls: 0, error: null };
226
+ for (let call = 0; call < calls; call++) {
227
+ const named = namedToJudge(db);
228
+ const pairs = [...named, ...(fill && named.length < PAIRS_PER_CALL ? candidatePairs(db, PAIRS_PER_CALL - named.length) : [])];
229
+ if (pairs.length === 0) break;
230
+ const prompt = ['Pairs to judge — how memory A relates to memory B in each:', ...pairs.flatMap(({ a, b }, i) => [`[p${i + 1}]`, ` A: ${describe(a)}`, ` B: ${describe(b)}`])].join('\n');
231
+ const { result, log } = await askLocal(db, { prompt, schema: PAIRS_SCHEMA, kind: 'dream-pairs', now });
232
+ out.calls++;
233
+ out.usage = added(out.usage, result.usage);
234
+ // A model that could not answer leaves the tidy owed; an answer in the wrong shape is a bad answer, dropped.
235
+ if (!result.ok) {
236
+ out.error = result.error;
237
+ log(result.error);
238
+ break;
239
+ }
240
+ const labels = Array.isArray(result.data?.pairs) ? result.data.pairs : null;
241
+ if (!labels) {
242
+ out.dropped.push('pairs: the answer had no list of pairs');
243
+ log('the answer had no list of pairs');
244
+ break;
245
+ }
246
+ let judged = 0;
247
+ tx(db, () => {
248
+ for (const answer of labels) {
249
+ const pair = pairs[Number(answer?.pair) - 1];
250
+ if (!pair || !LABELS.includes(answer?.label)) {
251
+ out.dropped.push(`pair ${JSON.stringify(answer?.pair)}: not one of the pairs asked, or no label`);
252
+ continue;
253
+ }
254
+ const { a, b, named: asked } = pair;
255
+ if (db.prepare('SELECT 1 FROM memory_pairs WHERE a = ? AND b = ?').get(a.id, b.id)) continue;
256
+ db.prepare('INSERT INTO memory_pairs (a, b, label, judged_at) VALUES (?, ?, ?, ?)').run(a.id, b.id, answer.label, now);
257
+ judged++;
258
+ const [older, newer] = a.created_at <= b.created_at ? [a, b] : [b, a];
259
+ const raised =
260
+ answer.label === 'same' ? { kind: 'lookalike', note: 'they say the same thing — forget one, or supersede the older with the newer', said: 'say the same thing' }
261
+ : answer.label === 'replaces' ? { kind: 'lookalike', note: `m${newer.id} is newer and may replace m${older.id} — if it does, supersede m${older.id} with it; if both hold, leave them`, said: `may be replaced by m${newer.id}` }
262
+ : answer.label === 'conflict' ? { kind: 'contradiction', note: 'they cannot both be true — only the user knows which holds', said: 'cannot both be true' }
263
+ : null;
264
+ if (!raised) {
265
+ if (asked) out.dropped.push(`m${a.id} and m${b.id}, named as ${asked.label === 'same' ? 'saying the same thing' : 'a clash'}: judged on their own, they are unrelated`);
266
+ continue;
267
+ }
268
+ const kept = asked?.label === answer.label ? asked.note : null;
269
+ if (raiseNotice(db, { kind: raised.kind, a: a.id, b: b.id, note: kept ?? raised.note, now })) out.applied.push(`raised for the user: m${a.id} and m${b.id} ${raised.said}${kept ? ` — ${kept}` : ''}`);
270
+ }
271
+ // What was judged waits no longer; a pair the answer left out is asked again.
272
+ if (named.length > 0) keepNamed(db, namedPairs(db).filter((p) => !db.prepare('SELECT 1 FROM memory_pairs WHERE a = ? AND b = ?').get(p.a, p.b)));
273
+ });
274
+ log(`judged ${judged} of ${pairs.length} pairs`);
275
+ // Nothing judged: the same pairs would be asked again in the same words.
276
+ if (judged === 0) break;
277
+ }
278
+ return out;
279
+ }
280
+
281
+ const REPEATS_SCHEMA = {
282
+ type: 'object',
283
+ properties: {
284
+ repeats: {
285
+ type: 'array',
286
+ items: { type: 'object', properties: { rule: { type: 'integer' }, same_as: { type: 'integer' } }, required: ['rule', 'same_as'], additionalProperties: false },
287
+ },
288
+ },
289
+ required: ['repeats'],
290
+ additionalProperties: false,
291
+ };
292
+
293
+ /**
294
+ * The same rule said twice in other words — "I prefer short answers", then "keep your replies brief" — shares no word,
295
+ * so no pair index finds it. Each rule not yet checked is shown beside the other rules where it applies, and the local
296
+ * model names any it says again: one call for a batch, each rule checked once (mem0 checks a new fact against its
297
+ * neighbours the same way). What it names is judged on its own next, and raised for the user only as that judgement
298
+ * says; nothing is retired here.
299
+ */
300
+ async function checkRepeats(db, { now }) {
301
+ const out = { applied: [], dropped: [], usage: NO_USAGE, calls: 0, error: null };
302
+ const rules = tidyable(db).filter((m) => m.type === 'preference' || m.type === 'decision');
303
+ const from = Number(getMeta(db, REPEATS_CURSOR) ?? 0);
304
+ const batch = rules.filter((m) => m.id > from).slice(0, REPEATS_PER_CALL);
305
+ if (batch.length === 0) return out;
306
+ const reaches = new Set(batch.map((m) => m.scope));
307
+ const near = rules.filter((m) => m.scope === 'global' || reaches.has(m.scope) || reaches.has('global')).slice(-REPEAT_NEIGHBOURS);
308
+ const shown = new Map([...near, ...batch].map((m) => [m.id, m]));
309
+ if (shown.size < 2) {
310
+ setMeta(db, REPEATS_CURSOR, batch.at(-1).id);
311
+ return out;
312
+ }
313
+ const prompt = [
314
+ 'Rules said twice — for each rule checked, the rule in the list that says the same thing in other words, if one does:',
315
+ 'Rules checked:',
316
+ ...batch.map((m) => `[m${m.id}] ${clip(m.body, 300)}`),
317
+ 'All rules:',
318
+ ...[...shown.values()].sort((x, y) => x.id - y.id).map(describe),
319
+ ].join('\n');
320
+ const { result, log } = await askLocal(db, { prompt, schema: REPEATS_SCHEMA, kind: 'dream-repeats', now });
321
+ out.calls = 1;
322
+ out.usage = result.usage;
323
+ if (!result.ok) {
324
+ out.error = result.error;
325
+ log(result.error);
326
+ return out;
327
+ }
328
+ const answers = Array.isArray(result.data?.repeats) ? result.data.repeats : null;
329
+ // Checked, whatever came back: a bad answer is not asked again in the same words.
330
+ setMeta(db, REPEATS_CURSOR, batch.at(-1).id);
331
+ if (!answers) {
332
+ out.dropped.push('repeats: the answer had no list');
333
+ log('the answer had no list of repeats');
334
+ return out;
335
+ }
336
+ const checked = new Set(batch.map((m) => m.id));
337
+ tx(db, () => {
338
+ for (const answer of answers) {
339
+ const a = shown.get(Number(answer?.rule));
340
+ const b = shown.get(Number(answer?.same_as));
341
+ if (!a || !b || a.id === b.id || !checked.has(a.id)) {
342
+ out.dropped.push(`repeat m${JSON.stringify(answer?.rule)} of m${JSON.stringify(answer?.same_as)}: not a rule checked and another rule shown`);
343
+ continue;
344
+ }
345
+ if (a.scope !== b.scope && a.scope !== 'global' && b.scope !== 'global') {
346
+ out.dropped.push(`repeat m${a.id} of m${b.id}: they hold in different projects`);
347
+ continue;
348
+ }
349
+ // Named without a note of its own: the judgement says it.
350
+ nameForJudging(db, { label: 'same', a: a.id, b: b.id, note: null });
351
+ }
352
+ });
353
+ log(`checked ${batch.length} rules for repeats`);
354
+ return out;
355
+ }
356
+
357
+ /**
358
+ * Search terms and a topic for memories that have none, so a search in other words finds them: the next batch from
359
+ * where the last tidy stopped. Each term passes the same check as the scribe's (`storedKeys`); a bad one is dropped,
360
+ * never stored, and a topic is a lowercase word or nothing. What a memory already has is never replaced.
361
+ */
362
+ async function fillKeys(db, { now }) {
363
+ const out = { applied: [], dropped: [], usage: NO_USAGE, calls: 0, error: null };
364
+ const lacking = tidyable(db).filter((m) => m.keys === null || m.topic === null);
365
+ const from = Number(getMeta(db, KEYS_CURSOR) ?? 0);
366
+ const batch = [...lacking.filter((m) => m.id > from), ...lacking.filter((m) => m.id <= from)].slice(0, KEYS_PER_TIDY);
367
+ if (batch.length === 0) return out;
368
+ const prompt = ['Search terms — a topic and keys for each memory:', ...batch.map((m) => `[m${m.id}] ${clip(m.body, 300)}`)].join('\n');
369
+ const { result, log } = await askLocal(db, { prompt, schema: KEYS_SCHEMA, kind: 'dream-keys', now });
370
+ out.calls = 1;
371
+ out.usage = result.usage;
372
+ if (!result.ok) {
373
+ out.error = result.error;
374
+ log(result.error);
375
+ return out;
376
+ }
377
+ const answers = Array.isArray(result.data?.memories) ? result.data.memories : null;
378
+ if (!answers) {
379
+ out.dropped.push('search terms: the answer had no list of memories');
380
+ log('the answer had no list of memories');
381
+ setMeta(db, KEYS_CURSOR, batch.at(-1).id);
382
+ return out;
383
+ }
384
+ const asked = new Map(batch.map((m) => [m.id, m]));
385
+ const given = [];
386
+ tx(db, () => {
387
+ for (const answer of answers) {
388
+ const m = asked.get(Number(answer?.id));
389
+ if (!m) {
390
+ out.dropped.push(`search terms for m${JSON.stringify(answer?.id)}: not one of the memories asked`);
391
+ continue;
392
+ }
393
+ const topic = m.topic === null && typeof answer.topic === 'string' && TOPIC.test(answer.topic.trim()) && redact(answer.topic).count === 0 ? answer.topic.trim() : null;
394
+ const keys = m.keys === null ? storedKeys(answer.keys, m.body) : null;
395
+ if (!topic && !keys) continue;
396
+ db.prepare('UPDATE memories SET topic = COALESCE(topic, ?), keys = COALESCE(keys, ?) WHERE id = ?').run(topic, keys, m.id);
397
+ asked.delete(m.id);
398
+ given.push(`m${m.id}`);
399
+ }
400
+ });
401
+ setMeta(db, KEYS_CURSOR, batch.at(-1).id);
402
+ if (given.length > 0) out.applied.push(`search terms for ${given.length} memor${given.length === 1 ? 'y' : 'ies'}: ${given.join(', ')}`);
403
+ log(`search terms for ${given.length} of ${batch.length}`);
404
+ return out;
405
+ }
406
+
407
+ /**
408
+ * Whether the weekly tidy is owed: a week since the last one, with something to tidy. Never tidied, the week runs from
409
+ * the oldest memory — memory a day old has nothing to tidy yet.
410
+ */
411
+ function tidyOwed(db, now) {
412
+ const since = getMeta(db, LAST_TIDY) ?? db.prepare(`SELECT MIN(created_at) AS at FROM memories WHERE state = 'active' AND scan_key IS NULL`).get()?.at;
413
+ if (!since || Date.parse(now) - Date.parse(since) < TIDY_EVERY_MS) return false;
414
+ return Boolean(db.prepare(`SELECT 1 FROM memories WHERE state = 'active' AND scan_key IS NULL LIMIT 1`).get());
415
+ }
416
+
417
+ /**
418
+ * Whether a consolidation pass is worth running by itself now: three finished sessions are waiting to be read side by
419
+ * side, or a week has gone since memory was last tidied and there is memory to tidy. Pure: it reads, never runs.
420
+ */
421
+ export function tidyDue(db, now = new Date().toISOString()) {
422
+ return sessionsAwaitingDream(db, now, SESSIONS_TO_TIDY).length >= SESSIONS_TO_TIDY || tidyOwed(db, now);
423
+ }
424
+
425
+ /**
426
+ * The tidy steps of a pass: weekly, search terms filled in; rules checked for being said twice; and pairs judged — those
427
+ * the check named first — in one call, or a few in the weekly tidy.
428
+ */
429
+ async function tidyMemory(db, { weekly, now }) {
430
+ const steps = [];
431
+ if (weekly) steps.push(await fillKeys(db, { now }));
432
+ steps.push(await checkRepeats(db, { now }));
433
+ steps.push(await judgePairs(db, { calls: weekly ? TIDY_JUDGE_CALLS : JUDGE_CALLS, now }));
434
+ return {
435
+ applied: steps.flatMap((s) => s.applied),
436
+ dropped: steps.flatMap((s) => s.dropped),
437
+ usage: steps.reduce((u, s) => added(u, s.usage), NO_USAGE),
438
+ calls: steps.reduce((n, s) => n + s.calls, 0),
439
+ error: steps.find((s) => s.error)?.error ?? null,
440
+ };
441
+ }
442
+
443
+ /**
444
+ * The pairs the reading of the conversations named — a clash, one thing said twice — judged on their own before anything
445
+ * is put to the user. With no local model to judge them (the tidy steps never call a paid one), they are raised as named.
446
+ */
447
+ async function settleNamed(db, { now }) {
448
+ if (namedPairs(db).length === 0) return { applied: [], dropped: [], usage: NO_USAGE };
449
+ if (tidiesLocally(db)) {
450
+ const judged = await judgePairs(db, { calls: 1, now, fill: false });
451
+ // Not answered, they wait for the next tidy, which judges them first.
452
+ return judged.error ? { ...judged, dropped: [...judged.dropped, `the pairs named wait to be judged: ${judged.error}`] } : judged;
453
+ }
454
+ const applied = [];
455
+ tx(db, () => {
456
+ for (const { a, b, named } of namedToJudge(db, Infinity)) {
457
+ const [kind, said] = named.label === 'same' ? ['lookalike', 'say the same thing'] : ['contradiction', 'cannot both be true'];
458
+ const note = named.note ?? `they ${said}`;
459
+ if (raiseNotice(db, { kind, a: a.id, b: b.id, note, now })) applied.push(`raised for the user: m${a.id} and m${b.id} ${said} — ${note}`);
460
+ }
461
+ keepNamed(db, []);
462
+ });
463
+ return { applied, dropped: [], usage: NO_USAGE };
464
+ }
465
+
466
+ /**
467
+ * The consolidation pass, which is also memory's tidy: the conversations waiting are read side by side, and memory
468
+ * itself is gone over — pairs judged, and weekly, search terms filled in. `sweep` asks for the weekly part now; it is
469
+ * done anyway once a week has gone. With no conversation waiting and no tidy owed, there is nothing to do.
470
+ */
471
+ export async function runDream(db, { now = new Date().toISOString(), force = false, sweep = false, file = true } = {}) {
472
+ // Anything still unfiled belongs to the writer first; consolidation reads its results. The memory pass files first
473
+ // itself, so it can say what the filing did, and asks for none here.
474
+ if (file) await runScribe(db, { now });
65
475
 
66
476
  return withLock('dream', async () => {
67
477
  // A conversation the writer has not finished filing (it is busy, failing, or behind) waits for it.
@@ -73,14 +483,25 @@ export async function runDream(db, { now = new Date().toISOString(), force = fal
73
483
  ? sessionsAwaitingDream(db, now, MAX_SESSIONS, { includeOpen: true }).filter((s) => !unfiled.has(s.id) && !over.some((o) => o.id === s.id))
74
484
  : [];
75
485
  const sessions = [...over, ...open].slice(0, MAX_SESSIONS).sort((a, b) => a.started_at.localeCompare(b.started_at));
76
- if (sessions.length === 0) return { skipped: 'no finished sessions are waiting' };
486
+ const weekly = sweep || tidyOwed(db, now);
487
+ if (sessions.length === 0 && !weekly) return { skipped: 'no finished sessions are waiting' };
488
+
489
+ // Memory is tidied first, so what the conversations bring is read against it as it now stands.
490
+ const tidy = tidiesLocally(db) ? await tidyMemory(db, { weekly, now }) : { applied: [], dropped: [], usage: NO_USAGE, calls: 0, error: null };
491
+ // A tidy the local model could not answer is owed still; one that had nothing to ask is done.
492
+ if (weekly && !tidy.error) setMeta(db, LAST_TIDY, now);
493
+ if (sessions.length === 0) {
494
+ if (tidy.calls === 0) return { skipped: 'no finished sessions are waiting, and memory has nothing to tidy' };
495
+ if (tidy.error && tidy.applied.length === 0) return { ok: false, error: tidy.error, applied: [], dropped: tidy.dropped, usage: tidy.usage };
496
+ return { ok: true, error: null, applied: tidy.applied, dropped: tidy.dropped, usage: tidy.usage, sessions: 0 };
497
+ }
77
498
 
78
499
  const bundle = buildBundle(db, sessions);
79
500
  let lastTurn = 0;
80
501
  for (const id of bundle.turns.keys()) lastTurn = id;
81
502
  const outcome = await askAndApply(db, {
82
503
  kind: 'dream', promptFile: 'dream.md', bundle,
83
- ops: ['add', 'supersede', 'gotcha', 'checkpoint', 'contradiction', 'procedure'], sessionId: null, now,
504
+ ops: ['add', 'supersede', 'gotcha', 'checkpoint', 'contradiction', 'same', 'procedure'], sessionId: null, now,
84
505
  });
85
506
  if (outcome.ok && over.length > 0) {
86
507
  // A resumed session may have received a new turn while the model read the snapshot.
@@ -88,10 +509,15 @@ export async function runDream(db, { now = new Date().toISOString(), force = fal
88
509
  AND NOT EXISTS (SELECT 1 FROM user_turns t WHERE t.session_id = sessions.id AND t.id > ?)`)
89
510
  .run(...over.map((s) => s.id), lastTurn);
90
511
  }
91
- return { ...outcome, sessions: sessions.length };
512
+ // The slice of the oldest moves on only past a pass that read it.
513
+ if (outcome.ok && bundle.cursor !== null) setMeta(db, SWEEP_CURSOR, bundle.cursor);
514
+ const named = outcome.ok ? await settleNamed(db, { now }) : { applied: [], dropped: [], usage: NO_USAGE };
515
+ return {
516
+ ...outcome,
517
+ applied: [...tidy.applied, ...outcome.applied, ...named.applied],
518
+ dropped: [...tidy.dropped, ...outcome.dropped, ...named.dropped],
519
+ usage: added(added(tidy.usage, outcome.usage ?? NO_USAGE), named.usage),
520
+ sessions: sessions.length,
521
+ };
92
522
  });
93
523
  }
94
-
95
- export function dreamStatus(db, now = new Date().toISOString()) {
96
- return [`finished sessions waiting: ${sessionsAwaitingDream(db, now, 1000).length} (runs by itself at 3)`];
97
- }
package/src/editor.mjs CHANGED
@@ -81,10 +81,11 @@ export function matchFiles(list, query, max = 8) {
81
81
  /**
82
82
  * The menu for the text being written: the slash menu, or — given `files(query, from)`, which answers with the paths a
83
83
  * mention could mean — the files for the `@word` at the cursor; picking one leaves its path where the mention was. A
84
- * recalled entry has none, so the arrows keep walking history; nor has a text whose menu Esc shut (`shut`), until it changes.
84
+ * recalled entry, as it was sent, has none, so the arrows keep walking history — changed, it is being written again, and
85
+ * has; nor has a text whose menu Esc shut (`shut`), until it changes.
85
86
  */
86
87
  export function menuOf(state, commands, files = null) {
87
- if (state.at !== state.history.length || state.shut === state.text) return [];
88
+ if ((state.at !== state.history.length && state.text === state.history[state.at]) || state.shut === state.text) return [];
88
89
  const menu = menuFor(state.text, commands);
89
90
  const at = menu.length === 0 && files ? mentionAt(state) : null;
90
91
  if (!at) return menu;
package/src/export.mjs CHANGED
@@ -1,6 +1,7 @@
1
- import { chmodSync, existsSync, mkdirSync, readdirSync, rmSync } from 'node:fs';
2
- import { join } from 'node:path';
1
+ import { chmodSync, copyFileSync, existsSync, mkdirSync, readdirSync, renameSync, rmSync, statSync } from 'node:fs';
2
+ import { basename, dirname, join, resolve } from 'node:path';
3
3
  import { getMeta, setMeta } from './db.mjs';
4
+ import { UsageError } from './memory.mjs';
4
5
  import { paths } from './paths.mjs';
5
6
  import { DatabaseSync } from './sqlite.mjs';
6
7
 
@@ -89,20 +90,117 @@ export function backup(db, now = new Date()) {
89
90
  return { file, pruned: old.length };
90
91
  }
91
92
 
93
+ /**
94
+ * The copies a damaged database can be put back from, newest first: the weekly backups, and the copies made before a
95
+ * migration (`memory.db.pre-v<N>.bak`). An empty one — a copy a process died making — is no copy.
96
+ */
97
+ function backupsOnHand() {
98
+ const { backups, db } = paths();
99
+ const found = [];
100
+ const listed = (dir, keep) => {
101
+ try {
102
+ for (const name of readdirSync(dir)) if (keep(name)) found.push(join(dir, name));
103
+ } catch {
104
+ // Not there yet: nothing kept there.
105
+ }
106
+ };
107
+ listed(backups, (name) => /^memory-.*\.db$/.test(name));
108
+ listed(dirname(db), (name) => name.startsWith(`${basename(db)}.pre-v`) && name.endsWith('.bak'));
109
+ return found
110
+ .map((file) => ({ file, stat: statSync(file) }))
111
+ .filter(({ stat }) => stat.size > 0)
112
+ .map(({ file, stat }) => ({ file, at: stat.mtime }))
113
+ .sort((a, b) => b.at - a.at);
114
+ }
115
+
116
+ const minute = (date) => date.toISOString().slice(0, 16).replace('T', ' ');
117
+
118
+ /** What to say when the database cannot be opened: the newest copy, and the command that puts it back. */
119
+ export function restoreHint() {
120
+ const [newest] = backupsOnHand();
121
+ return newest ? `the newest backup is ${newest.file} (${minute(newest.at)}): sumo backup restore puts it back, keeping this one aside` : `no backup was found to put back (${paths().backups} is empty)`;
122
+ }
123
+
124
+ /** Whether an error is SQLite saying the file is not a database it can read. */
125
+ export const damaged = (cause) => /file is not a database|database disk image is malformed|SQLITE_(?:CORRUPT|NOTADB)/i.test(String(cause?.message ?? cause));
126
+
127
+ const memoriesIn = (n) => `${n} memor${n === 1 ? 'y' : 'ies'}`;
128
+
129
+ /**
130
+ * The copy a restore would put back — `file`, or the newest on hand — checked before anything moves: a copy SQLite cannot
131
+ * read changes nothing. `{ from, at, memories }`.
132
+ */
133
+ export function restoreCandidate(file = null) {
134
+ const chosen = file ? resolve(file) : backupsOnHand()[0]?.file;
135
+ if (!chosen) throw new UsageError(`no backup to put back: none in ${paths().backups}, and no copy made before a migration`);
136
+ if (!existsSync(chosen)) throw new UsageError(`${chosen} does not exist`);
137
+ try {
138
+ const copy = new DatabaseSync(chosen, { readOnly: true });
139
+ try {
140
+ const checked = Object.values(copy.prepare('PRAGMA quick_check').get())[0];
141
+ if (checked !== 'ok') throw new Error(checked);
142
+ return { from: chosen, at: statSync(chosen).mtime, memories: copy.prepare('SELECT COUNT(*) AS n FROM memories').get().n };
143
+ } finally {
144
+ copy.close();
145
+ }
146
+ } catch (cause) {
147
+ throw new UsageError(`${chosen} is not a usable database (${cause.message}) — nothing was changed`);
148
+ }
149
+ }
150
+
151
+ /** The database in place, when it opens and checks out: `{ memories }`. Null when it is missing or damaged — nothing to lose by replacing it. */
152
+ export function liveDatabase() {
153
+ const { db } = paths();
154
+ if (!existsSync(db)) return null;
155
+ try {
156
+ const live = new DatabaseSync(db, { readOnly: true });
157
+ try {
158
+ if (Object.values(live.prepare('PRAGMA quick_check').get())[0] !== 'ok') return null;
159
+ return { memories: live.prepare('SELECT COUNT(*) AS n FROM memories').get().n };
160
+ } finally {
161
+ live.close();
162
+ }
163
+ } catch {
164
+ return null;
165
+ }
166
+ }
167
+
168
+ /** What is asked before a copy goes over a database that is fine: both sides, counted, then a short question. */
169
+ export const restoreQuestion = (copy, live) =>
170
+ `memory.db is fine, with ${memoriesIn(live.memories)}. The copy is ${copy.from}, backed up ${minute(copy.at)}, with ${memoriesIn(copy.memories)}.\nPut the copy in its place, keeping the one there now aside? [y/N] `;
171
+
172
+ /**
173
+ * Puts a checked copy back. The database it replaces is kept aside with its -wal and -shm — as `memory.db.damaged-<time>`,
174
+ * or `memory.db.replaced-<time>` when it was fine (`live`): a restore never deletes. The copy is the user's only, at 0600.
175
+ * Returns the lines the user reads.
176
+ */
177
+ export function restoreLines(copy, live, now = new Date()) {
178
+ const { db } = paths();
179
+ const aside = `${db}.${live ? 'replaced' : 'damaged'}-${now.toISOString().replace(/[:.]/g, '-')}`;
180
+ for (const side of ['', '-wal', '-shm']) if (existsSync(`${db}${side}`)) renameSync(`${db}${side}`, `${aside}${side}`);
181
+ copyFileSync(copy.from, db);
182
+ chmodSync(db, 0o600);
183
+ const kept = !existsSync(aside) ? '' : live ? `; the database it replaced (${memoriesIn(live.memories)}) is kept as ${aside}` : `; the damaged database is kept as ${aside}`;
184
+ return [`restored ${copy.from} (backed up ${minute(copy.at)}) — ${memoriesIn(copy.memories)}${kept}`];
185
+ }
186
+
92
187
  /**
93
188
  * A purge reaches the backups too: "erased, not recoverable" must hold for every copy this machine keeps. Each backup
94
189
  * loses the memory and the sentence it came from, and is rewritten so the pages that held them are gone.
95
190
  * Returns how many were cleaned, and the ones that could not be — said, never hidden.
96
191
  */
97
192
  export function purgeFromBackups({ id, body, created_at: createdAt, source_turn: sourceTurn }) {
98
- const dir = paths().backups;
99
- const files = existsSync(dir) ? readdirSync(dir).filter((f) => /^memory-.*\.db$/.test(f)) : [];
193
+ const { backups, db: live } = paths();
194
+ const within = (dir, keep) => (existsSync(dir) ? readdirSync(dir).filter(keep).map((f) => join(dir, f)) : []);
195
+ // The weekly copies, and every copy set aside beside the database: before a migration, or by a restore.
196
+ const aside = (f) => f.startsWith(`${basename(live)}.`) && (/\.pre-v\d+\.bak$/.test(f) || /\.(?:damaged|replaced)-[^/]*$/.test(f)) && !/-(?:wal|shm)$/.test(f);
197
+ const files = [...within(backups, (f) => /^memory-.*\.db$/.test(f)), ...within(dirname(live), aside)];
100
198
  const failed = [];
101
199
  let cleaned = 0;
102
- for (const f of files) {
200
+ for (const file of files) {
103
201
  let db = null;
104
202
  try {
105
- db = new DatabaseSync(join(dir, f));
203
+ db = new DatabaseSync(file);
106
204
  db.exec('PRAGMA secure_delete = ON;');
107
205
  // The same memory, not just the same number: a backup from after a restore may hold another one under this id.
108
206
  const row = db.prepare('SELECT source_turn FROM memories WHERE id = ? AND body = ? AND created_at = ?').get(id, body, createdAt);
@@ -123,7 +221,7 @@ export function purgeFromBackups({ id, body, created_at: createdAt, source_turn:
123
221
  cleaned++;
124
222
  }
125
223
  } catch {
126
- failed.push(join(dir, f));
224
+ failed.push(file);
127
225
  } finally {
128
226
  db?.close();
129
227
  }