sumo-agents 0.5.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/memory.mjs CHANGED
@@ -1,16 +1,23 @@
1
1
  import { createHash } from 'node:crypto';
2
2
  import { getMeta, setMeta, tx } from './db.mjs';
3
3
  import { redact } from './redact.mjs';
4
- import { cjkTerms, clip, containing, ftsQuery, overlap, sharedStems, stem, words } from './text.mjs';
4
+ import { cjkTerms, clip, containing, ftsAllQuery, ftsQuery, overlap, sharedStems, stem, termsHeld, words } from './text.mjs';
5
5
  import { matchWithin } from './workflows.mjs';
6
6
 
7
7
  /** How much of a search that found nothing is kept as the record of it. */
8
8
  const MISS_KEPT = 200;
9
+ /** A search of several words that had to settle for any of them, and whose best result holds fewer of them than this, answered nothing. */
10
+ export const LOOSE_BELOW = 2;
9
11
 
10
12
  const TYPES = ['preference', 'fact', 'decision', 'gotcha', 'procedure'];
11
13
 
12
14
  const IMPORTANCE = { preference: 0.8, procedure: 0.8, decision: 0.7, gotcha: 0.6, fact: 0.5 };
13
15
  const PROVENANCE_WEIGHT = { stated: 1, observed: 0.9, scanned: 0.7, inferred: 0.5 };
16
+ /**
17
+ * Why a memory waits unconfirmed for the user: its conversation read the web or an MCP server ('outside', only the user
18
+ * confirms), or the model saved from its shell what the user never typed ('shell').
19
+ */
20
+ const HELD_FOR = ['outside', 'shell'];
14
21
  /** Days for a memory's rank to fall to 1/e. Absent means it does not age: a preference stays true until superseded. */
15
22
  const DECAY_DAYS = { fact: 365, gotcha: 365 };
16
23
  const PROJECT_MATCH_BOOST = 1.5;
@@ -130,16 +137,31 @@ export function storedKeys(raw, body) {
130
137
  return kept.length > 0 ? kept.join(', ') : null;
131
138
  }
132
139
 
140
+ /**
141
+ * One line of a memory's history (`memory_events`): what happened to it, when, by whom — `actor` is the writer of an
142
+ * add, 'user' or 'model-shell' where the caller says who asked, else 'sumo' — and in which pass (`runId`, an
143
+ * extractions row). Never the memory's words: the history outlives a purge of it.
144
+ */
145
+ function noteEvent(db, id, op, { actor = 'sumo', runId = null, reason = null, now }) {
146
+ db.prepare('INSERT INTO memory_events (ts, memory_id, op, actor, run_id, reason) VALUES (?, ?, ?, ?, ?, ?)').run(now, id, op, actor ?? 'sumo', runId ?? null, reason ?? null);
147
+ }
148
+
133
149
  /**
134
150
  * Stores one memory. Returns it with any active memories that look like the
135
151
  * same statement, so the caller can supersede instead of piling up duplicates.
152
+ * `derivedBy` is the extractor version that drew it from the journal (absent: primary, the user's to keep);
153
+ * `runId` is the extractions row of the pass that filed it.
136
154
  */
137
155
  export function add(db, input) {
138
156
  if (!TYPES.includes(input.type)) {
139
157
  throw new UsageError(`unknown type "${input.type}" — one of: ${TYPES.join(', ')}`);
140
158
  }
141
- const provenance = input.provenance ?? 'stated';
159
+ // Held for the user, and only for one of two reasons: nothing else is ever unconfirmed — a guess is not kept.
160
+ const heldFor = input.heldFor ?? null;
161
+ if (heldFor !== null && !HELD_FOR.includes(heldFor)) throw new UsageError(`unknown reason to hold a memory "${heldFor}" — one of: ${HELD_FOR.join(', ')}`);
162
+ const provenance = heldFor ? 'inferred' : (input.provenance ?? 'stated');
142
163
  if (!(provenance in PROVENANCE_WEIGHT)) throw new UsageError(`unknown provenance "${provenance}"`);
164
+ if (provenance === 'inferred' && !heldFor) throw new UsageError('a guess is not kept: a memory is held for the user only with why (heldFor)');
143
165
 
144
166
  const { text: body, redacted } = storedBody(input.type, input.body);
145
167
  if (!body) throw new UsageError('nothing to remember — the text is empty');
@@ -154,11 +176,13 @@ export function add(db, input) {
154
176
  const replaced = input.supersedes === undefined ? null : requireActive(db, input.supersedes, 'superseded');
155
177
  const similar = replaced ? [] : findSimilar(db, { type: input.type, title: input.title, body, scope });
156
178
 
179
+ const writer = input.writtenBy ?? 'chat-model';
180
+ const runId = input.runId ?? null;
157
181
  const { lastInsertRowid } = db
158
182
  .prepare(
159
183
  `INSERT INTO memories (type, scope, title, cue, gate, body, topic, keys, provenance, state, pinned, importance,
160
- scan_key, written_by, source_session, source_turn, source_quote, valid_from, created_at)
161
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
184
+ scan_key, written_by, source_session, source_turn, source_quote, valid_from, created_at, derived_by, held_for)
185
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
162
186
  )
163
187
  .run(
164
188
  input.type,
@@ -170,19 +194,23 @@ export function add(db, input) {
170
194
  input.topic?.trim().toLowerCase() || null,
171
195
  storedKeys(input.keys, body),
172
196
  provenance,
173
- provenance === 'inferred' ? 'unconfirmed' : 'active',
197
+ heldFor ? 'unconfirmed' : 'active',
174
198
  input.pin ? 1 : 0,
175
199
  IMPORTANCE[input.type],
176
200
  input.scanKey ?? null,
177
- input.writtenBy ?? 'chat-model',
201
+ writer,
178
202
  input.sourceSession ?? null,
179
203
  input.sourceTurn ?? null,
180
204
  input.sourceQuote ?? null,
181
205
  now,
182
206
  now,
207
+ // Null is what the user said to remember; a version is what an extractor drew from the journal, and may be rebuilt.
208
+ input.derivedBy ?? null,
209
+ heldFor,
183
210
  );
184
211
  const id = Number(lastInsertRowid);
185
- if (replaced) markSuperseded(db, replaced.id, id, now);
212
+ noteEvent(db, id, 'added', { actor: writer, runId, now });
213
+ if (replaced) markSuperseded(db, replaced.id, id, now, { actor: writer, runId });
186
214
 
187
215
  return { memory: get(db, id), similar, redacted };
188
216
  });
@@ -202,22 +230,23 @@ function noteRetired(db, id) {
202
230
  /** Whether a scanned fact with this text was let go by someone other than the scanner. */
203
231
  export const scanRetired = (db, scope, scanKey, body) => getMeta(db, retiredKey(scope, scanKey)) === digestOf(body);
204
232
 
205
- function markSuperseded(db, oldId, newId, now) {
233
+ function markSuperseded(db, oldId, newId, now, { actor, runId } = {}) {
206
234
  noteRetired(db, oldId);
207
235
  db.prepare(`UPDATE memories SET state = 'superseded', superseded_by = ?, invalid_at = ? WHERE id = ?`).run(
208
236
  newId,
209
237
  now,
210
238
  oldId,
211
239
  );
240
+ noteEvent(db, oldId, 'superseded', { actor, runId, reason: `by m${newId}`, now });
212
241
  }
213
242
 
214
- /** Links two memories that already exist: `oldId` stops being true because of `newId`. */
215
- export function supersede(db, oldId, newId, now = new Date().toISOString()) {
243
+ /** Links two memories that already exist: `oldId` stops being true because of `newId`. `actor` and `runId` say who asked, and in which pass. */
244
+ export function supersede(db, oldId, newId, now = new Date().toISOString(), { actor, runId } = {}) {
216
245
  if (oldId === newId) throw new UsageError('a memory cannot supersede itself');
217
246
  return tx(db, () => {
218
247
  requireActive(db, oldId, 'superseded');
219
248
  requireActive(db, newId, 'the replacement');
220
- markSuperseded(db, oldId, newId, now);
249
+ markSuperseded(db, oldId, newId, now, { actor, runId });
221
250
  return get(db, oldId);
222
251
  });
223
252
  }
@@ -265,6 +294,15 @@ export function markHits(db, ids, now) {
265
294
  db.prepare(`UPDATE memories SET hits = hits + 1, last_hit_at = ? WHERE id IN (${ids.map(() => '?').join(', ')})`).run(now, ...ids);
266
295
  }
267
296
 
297
+ /** The fields of a memory that are searched, in the order the index holds them. */
298
+ const SEARCHED = ['title', 'cue', 'body', 'topic', 'keys'];
299
+ /**
300
+ * How much a word found in each of them counts, in that order: a title and the search terms a writer added say what a
301
+ * memory is about; a word somewhere in a long body says less.
302
+ */
303
+ const RANKED = 'bm25(memories_fts, 3.0, 1.0, 1.0, 2.0, 2.0)';
304
+ const searchedText = (row) => SEARCHED.map((f) => row[f] ?? '').join(' ');
305
+
268
306
  /**
269
307
  * What the index found, with the memories holding a Chinese, Japanese or Korean word of the search: the index keeps a
270
308
  * run of those characters as one word, so a word inside it is looked for in the text itself, under the same `filters`.
@@ -273,15 +311,14 @@ export function markHits(db, ids, now) {
273
311
  */
274
312
  function withCjk(db, terms, found, filters, params) {
275
313
  if (terms.length === 0) return found;
276
- const fields = ['title', 'cue', 'body', 'topic', 'keys'];
277
- const holds = terms.flatMap(() => fields.map((f) => `m.${f} LIKE ? ESCAPE '\\'`)).join(' OR ');
314
+ const holds = terms.flatMap(() => SEARCHED.map((f) => `m.${f} LIKE ? ESCAPE '\\'`)).join(' OR ');
278
315
  const rows = db
279
316
  .prepare(`SELECT m.* FROM memories m WHERE (${holds})${filters.map((w) => ` AND ${w}`).join('')}`)
280
- .all(...terms.flatMap((t) => fields.map(() => containing(t))), ...params);
317
+ .all(...terms.flatMap((t) => SEARCHED.map(() => containing(t))), ...params);
281
318
  const best = found.length > 0 ? found.reduce((min, row) => Math.min(min, row.rank), 0) : -1;
282
319
  const byId = new Map(found.map((row) => [row.id, row]));
283
320
  for (const row of rows) {
284
- const text = fields.map((f) => row[f] ?? '').join(' ').toLowerCase();
321
+ const text = searchedText(row).toLowerCase();
285
322
  const rank = (best * terms.filter((t) => text.includes(t)).length) / terms.length;
286
323
  const known = byId.get(row.id);
287
324
  if (!known || rank < known.rank) byId.set(row.id, { ...row, rank });
@@ -289,18 +326,47 @@ function withCjk(db, terms, found, filters, params) {
289
326
  return [...byId.values()];
290
327
  }
291
328
 
329
+ /**
330
+ * The memories under `filters` that hold every word of the search (`every`), or any of them. Every word: the index
331
+ * answers for the words it keeps apart, and a Chinese, Japanese or Korean word must be in the text itself.
332
+ */
333
+ function matching(db, query, every, filters, params) {
334
+ const cjk = cjkTerms(query);
335
+ const also = filters.map((w) => ` AND ${w}`).join('');
336
+ const indexed = (match, extra = '', extraParams = []) =>
337
+ db
338
+ .prepare(`SELECT m.*, ${RANKED} AS rank FROM memories_fts JOIN memories m ON m.id = memories_fts.rowid WHERE memories_fts MATCH ?${extra}${also}`)
339
+ .all(match, ...extraParams, ...params);
340
+ if (!every) return withCjk(db, cjk, indexed(ftsQuery(query)), filters, params);
341
+ const holds = cjk.map(() => ` AND (${SEARCHED.map((f) => `m.${f} LIKE ? ESCAPE '\\'`).join(' OR ')})`).join('');
342
+ const patterns = cjk.flatMap((t) => SEARCHED.map(() => containing(t)));
343
+ const match = ftsAllQuery(query);
344
+ if (match) return indexed(match, holds, patterns);
345
+ // Only words the index cannot tell apart: the text itself answers, and none ranks above another.
346
+ return db.prepare(`SELECT m.*, -1.0 AS rank FROM memories m WHERE 1 = 1${holds}${also}`).all(...patterns, ...params);
347
+ }
348
+
349
+ /** Keeps the record of what a search could not answer: without secrets, and short — a pasted log is not a question worth keeping whole. */
350
+ export function recordMiss(db, query, scope, now) {
351
+ db.prepare('INSERT INTO search_misses (ts, query, scope) VALUES (?, ?, ?)').run(now, clip(redact(query).text, MISS_KEPT), scope);
352
+ }
353
+
292
354
  /**
293
355
  * Ranked search. Scope is `global` plus the named project; other projects stay
294
356
  * invisible unless `everywhere` is set — only their hit counts come back, so
295
357
  * the caller learns where to look without one project's memory leaking into
296
- * another's work. `record: false` is for a search nobody asked for: it counts
297
- * no hits and keeps no misses, and the caller decides what was used.
358
+ * another's work. `record: false` is for a search nobody asked for, or one whose
359
+ * caller decides what was shown: it counts no hits and keeps no misses.
298
360
  * `since` and `before` (ISO times) hold it to a time: since, what was said or
299
361
  * last changed — replaced, let go — at or after it; before, what was said before it.
362
+ *
363
+ * All the words first: only when no memory in scope holds every one of them is it tried with any of them
364
+ * (`fellBack`). `terms: 'all' | 'any'` asks for one of the two alone. `best` is the most of the search's words one of
365
+ * the results holds; a search of several words that fell back and whose best holds fewer than two is `loose`: something
366
+ * is shown, and it is still kept as a miss — one word in common is an accident, not an answer.
300
367
  */
301
368
  export function search(db, query, opts = {}) {
302
- const match = ftsQuery(query);
303
- if (!match) throw new UsageError('nothing to search for');
369
+ if (!ftsQuery(query)) throw new UsageError('nothing to search for');
304
370
  if (opts.type && !TYPES.includes(opts.type)) {
305
371
  // Said as the user knows them: a procedure is a workflow everywhere they look.
306
372
  throw new UsageError(`unknown type "${opts.type}" — one of: ${TYPES.map((t) => (t === 'procedure' ? 'workflow' : t)).join(', ')}`);
@@ -310,8 +376,8 @@ export function search(db, query, opts = {}) {
310
376
  const now = opts.now ?? new Date().toISOString();
311
377
  const limit = opts.limit ?? 8;
312
378
 
313
- const where = ['memories_fts MATCH ?'];
314
- const params = [match];
379
+ const where = [];
380
+ const params = [];
315
381
  if (!opts.all) where.push(`m.state = 'active'`);
316
382
  if (opts.type) {
317
383
  where.push('m.type = ?');
@@ -327,19 +393,20 @@ export function search(db, query, opts = {}) {
327
393
  params.push(opts.before);
328
394
  }
329
395
 
330
- const found = db
331
- .prepare(
332
- `SELECT m.*, bm25(memories_fts) AS rank FROM memories_fts JOIN memories m ON m.id = memories_fts.rowid
333
- WHERE ${where.join(' AND ')}`,
334
- )
335
- .all(...params);
336
- const rows = withCjk(db, cjkTerms(query), found, where.slice(1), params.slice(1));
337
-
338
396
  const inScope = (row) => opts.everywhere || row.scope === 'global' || row.scope === projectScope;
339
- // OR-ed words match on any one of them. Fine for words a caller picked; a caller searching with
397
+ // Any-word matches are on any one of them. Fine for words a caller picked; a caller searching with
340
398
  // words nobody picked — a sentence lifted from a reply — sets a floor on how many must be shared.
341
399
  const close = (row) => !opts.minShared || sharedStems(query, `${row.title ?? ''} ${row.cue ?? ''} ${row.body} ${row.keys ?? ''}`) >= opts.minShared;
342
- const candidates = rows.filter((row) => inScope(row) && close(row));
400
+ const several = new Set(words(query)).size > 1;
401
+ let rows = [];
402
+ let candidates = [];
403
+ let fellBack = false;
404
+ for (const every of opts.terms === 'any' ? [false] : opts.terms === 'all' || !several ? [true] : [true, false]) {
405
+ rows = matching(db, query, every, where, params);
406
+ candidates = rows.filter((row) => inScope(row) && close(row));
407
+ fellBack = !every && opts.terms !== 'any';
408
+ if (candidates.length > 0) break;
409
+ }
343
410
  // Not Math.min(...ranks): spread into arguments, a broad query's rows overflow the call stack.
344
411
  const bestRank = candidates.reduce((best, row) => Math.min(best, row.rank), Infinity);
345
412
  const results = candidates
@@ -351,22 +418,17 @@ export function search(db, query, opts = {}) {
351
418
  for (const row of rows) if (!inScope(row)) elsewhere[row.scope] = (elsewhere[row.scope] ?? 0) + 1;
352
419
  // Matches past the limit, counted, so a list cut short says so.
353
420
  const more = candidates.length - results.length;
354
-
355
- if (opts.record === false) return { results, elsewhere, more };
356
- if (results.length > 0) {
357
- markHits(db, results.map((r) => r.id), now);
358
- } else {
359
- // The record of what memory could not answer: the evidence for whether
360
- // keyword search is enough or embeddings are ever worth adding.
361
- // Kept like everything else here: without secrets, and short — a pasted log is not a question worth keeping whole.
362
- db.prepare('INSERT INTO search_misses (ts, query, scope) VALUES (?, ?, ?)').run(
363
- now,
364
- clip(redact(query).text, MISS_KEPT),
365
- opts.everywhere ? 'everywhere' : (projectScope ?? 'global'),
366
- );
367
- }
368
-
369
- return { results, elsewhere, more };
421
+ const best = results.reduce((most, row) => Math.max(most, termsHeld(query, searchedText(row))), 0);
422
+ const loose = fellBack && results.length > 0 && best < LOOSE_BELOW;
423
+ const found = { results, elsewhere, more, fellBack, best, loose };
424
+
425
+ if (opts.record === false) return found;
426
+ // Shown to someone who asked: used. What was matched and left out for scope or room was not.
427
+ markHits(db, results.map((r) => r.id), now);
428
+ // The record of what memory could not answer: the evidence for whether
429
+ // keyword search is enough or embeddings are ever worth adding.
430
+ if (results.length === 0 || loose) recordMiss(db, query, opts.everywhere ? 'everywhere' : (projectScope ?? 'global'), now);
431
+ return found;
370
432
  }
371
433
 
372
434
  /** Every memory connected to `id` by supersession, oldest first. */
@@ -397,46 +459,182 @@ function eraseFromIndexes(db) {
397
459
  db.exec(`
398
460
  INSERT INTO memories_fts(memories_fts, rank) VALUES ('secure-delete', 1);
399
461
  INSERT INTO user_turns_fts(user_turns_fts, rank) VALUES ('secure-delete', 1);
462
+ INSERT INTO replies_fts(replies_fts, rank) VALUES ('secure-delete', 1);
400
463
  `);
401
464
  }
402
465
 
403
- /** Stops a memory being true. With `purge` it is erased outright, for things that should never have been stored. */
404
- export function forget(db, id, { purge = false, now = new Date().toISOString() } = {}) {
466
+ const sameLetters = (text) => text.toLowerCase().replace(/\s+/g, ' ').trim();
467
+
468
+ /** A phrase shorter than both of these is too common to erase by: "we use bun" stands in conversations that are not about it. */
469
+ const PHRASE_MIN_WORDS = 5;
470
+ const PHRASE_MIN_CHARS = 25;
471
+
472
+ /**
473
+ * Whether a text says any of `phrases` whole, whatever the case or the line breaks — a phrase counting only when it
474
+ * is at least five words or twenty-five characters; never, when no phrase is that long.
475
+ */
476
+ export function saying(phrases) {
477
+ const needles = [...new Set(phrases.filter(Boolean).map(sameLetters))].filter((n) => n.length >= PHRASE_MIN_CHARS || n.split(' ').length >= PHRASE_MIN_WORDS);
478
+ return (text) => needles.length > 0 && needles.some((n) => sameLetters(String(text ?? '')).includes(n));
479
+ }
480
+
481
+ const hasTable = (db, name) => db.prepare(`SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?`).get(name) !== undefined;
482
+
483
+ /**
484
+ * What of the journal a purge of this memory erases: the turn it cites, and every turn and reply, in any
485
+ * conversation, that holds its whole body or the whole of the words it was drawn from — when that phrase is long
486
+ * enough to be this memory's and not three words any conversation may hold (`saying`). A purge once took a long
487
+ * plan in another conversation with a memory that said "we use bun". What only mentions part of it stays, and the
488
+ * purge says so. `{ source, turns, replies, sessions }`: `source` the cited turn's id when it is here; `turns` and
489
+ * `replies` as `{ id, ts, text }`, to be shown before they go.
490
+ */
491
+ export function purgeReach(db, row) {
492
+ const says = saying([row.source_quote, row.body]);
493
+ const source = row.source_turn === null || row.source_turn === undefined ? null : (db.prepare('SELECT id, session_id FROM user_turns WHERE id = ?').get(row.source_turn) ?? null);
494
+ const turns = [];
495
+ const replies = [];
496
+ const sessions = new Set([row.source_session ?? source?.session_id].filter(Boolean));
497
+ // One row at a time: a journal of pasted logs is not read into memory whole.
498
+ for (const t of db.prepare('SELECT id, session_id, ts, text FROM user_turns ORDER BY id').iterate()) {
499
+ if (t.id !== source?.id && !says(t.text)) continue;
500
+ turns.push({ id: t.id, ts: t.ts, text: t.text });
501
+ sessions.add(t.session_id);
502
+ }
503
+ // A copy from before replies were kept has none.
504
+ if (hasTable(db, 'replies')) {
505
+ for (const r of db.prepare('SELECT id, session_id, ts, text FROM replies ORDER BY id').iterate()) {
506
+ if (!says(r.text)) continue;
507
+ replies.push({ id: r.id, ts: r.ts, text: r.text });
508
+ sessions.add(r.session_id);
509
+ }
510
+ }
511
+ return { source: source?.id ?? null, turns, replies, sessions: [...sessions] };
512
+ }
513
+
514
+ /** Every string a value holds, at any depth. */
515
+ const stringsIn = (value) => (typeof value === 'string' ? [value] : value && typeof value === 'object' ? Object.values(value).flatMap(stringsIn) : []);
516
+
517
+ /** What the passes proposed (`extractions.ops_json`) cleared wherever `which(row)` says: they keep that they ran. Returns how many. */
518
+ function clearProposals(db, which) {
519
+ if (!hasTable(db, 'extractions')) return 0;
520
+ let cleared = 0;
521
+ for (const x of db.prepare('SELECT id, session_id, ops_json FROM extractions WHERE ops_json IS NOT NULL').all()) {
522
+ if (!which(x)) continue;
523
+ db.prepare('UPDATE extractions SET ops_json = NULL WHERE id = ?').run(x.id);
524
+ cleared++;
525
+ }
526
+ return cleared;
527
+ }
528
+
529
+ /**
530
+ * Erases what `purgeReach` finds, and what the passes over those conversations proposed: a pass keeps the model's
531
+ * answer, the quote among it — those of the conversations touched, of the passes that filed the memory (`runs`), and
532
+ * any other whose answer says it. The same rule for the database and for each backup. Returns the reach, with `ops`
533
+ * the answers cleared.
534
+ */
535
+ export function eraseSaid(db, row, { runs = [] } = {}) {
536
+ const reach = purgeReach(db, row);
537
+ for (const { id } of reach.turns) db.prepare('DELETE FROM user_turns WHERE id = ?').run(id);
538
+ for (const { id } of reach.replies) db.prepare('DELETE FROM replies WHERE id = ?').run(id);
539
+ const says = saying([row.source_quote, row.body]);
540
+ const proposed = (json) => {
541
+ try {
542
+ return stringsIn(JSON.parse(json));
543
+ } catch {
544
+ return [json];
545
+ }
546
+ };
547
+ const ops = clearProposals(db, (x) => reach.sessions.includes(x.session_id) || runs.includes(x.id) || proposed(x.ops_json).some(says));
548
+ return { ...reach, ops };
549
+ }
550
+
551
+ const JOURNAL_TABLE = { turn: 'user_turns', reply: 'replies' };
552
+
553
+ /**
554
+ * One turn or reply erased from a database — this one, or a backup: the row with that id and those very words (in a
555
+ * copy, the number alone may be another row), and what the passes over its conversation proposed. False when the
556
+ * row is not there.
557
+ */
558
+ export function eraseRow(db, { kind, id, text, session_id: session }) {
559
+ const table = JOURNAL_TABLE[kind];
560
+ if (!hasTable(db, table) || Number(db.prepare(`DELETE FROM ${table} WHERE id = ? AND text = ?`).run(id, text).changes) === 0) return false;
561
+ clearProposals(db, (x) => x.session_id === session);
562
+ return true;
563
+ }
564
+
565
+ /**
566
+ * `sumo forget --purge t12`: one turn or reply erased for good — what a memory's purge leaves when a conversation
567
+ * only mentions part of it. `entry` is the journal row (`journalEntry` in sessions.mjs). Returns the ids of the
568
+ * memories that cite the turn: they hold its words still, and are the user's to purge or keep.
569
+ */
570
+ export function purgeRow(db, entry) {
571
+ const cites = entry.kind === 'turn' ? db.prepare('SELECT id FROM memories WHERE source_turn = ? ORDER BY id').all(entry.id).map((m) => m.id) : [];
572
+ tx(db, () => {
573
+ eraseFromIndexes(db);
574
+ eraseRow(db, entry);
575
+ });
576
+ // As for a memory: the write-ahead log still holds the pages as they were.
577
+ db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
578
+ return cites;
579
+ }
580
+
581
+ /**
582
+ * Stops a memory being true. With `purge` it is erased outright, for things that should never have been stored: the
583
+ * memory, the sentence it came from, and every turn and reply of the journal that says it whole (`eraseSaid`;
584
+ * `erased_turns` and `erased_replies` are those rows). `actor`, `runId` and `reason` go into its history; `op`
585
+ * names a letting-go that is not the user's.
586
+ */
587
+ export function forget(db, id, { purge = false, now = new Date().toISOString(), actor, runId, reason, op = 'forgotten' } = {}) {
405
588
  const row = get(db, id);
406
589
  if (purge) {
590
+ let erased = null;
407
591
  tx(db, () => {
408
592
  eraseFromIndexes(db);
409
593
  // A scanned fact let go for good stays let go: only its digest is kept, never its words.
410
594
  noteRetired(db, id);
595
+ const runs = db.prepare('SELECT run_id FROM memory_events WHERE memory_id = ? AND run_id IS NOT NULL').all(id).map((e) => e.run_id);
411
596
  db.prepare('DELETE FROM memories WHERE id = ?').run(id);
412
- // Erasing the memory but keeping the sentence it came from would not be erasing it.
413
- if (row.source_turn !== null) db.prepare('DELETE FROM user_turns WHERE id = ?').run(row.source_turn);
597
+ // Erasing the memory but keeping the sentence it came from, or an answer that said it back, would not be erasing it.
598
+ erased = eraseSaid(db, row, { runs });
599
+ noteEvent(db, id, 'purged', { actor, runId, now });
414
600
  });
415
601
  // The write-ahead log still holds the pages as they were before the delete; copy it into the file and empty it.
416
602
  db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
417
- return { ...row, state: 'purged' };
603
+ return { ...row, state: 'purged', erased_source: erased.source, erased_turns: erased.turns, erased_replies: erased.replies };
418
604
  }
419
605
  if (row.state === 'invalid') return row;
420
- noteRetired(db, id);
421
- db.prepare(`UPDATE memories SET state = 'invalid', invalid_at = ? WHERE id = ?`).run(now, id);
422
- return get(db, id);
606
+ return tx(db, () => {
607
+ noteRetired(db, id);
608
+ db.prepare(`UPDATE memories SET state = 'invalid', invalid_at = ? WHERE id = ?`).run(now, id);
609
+ noteEvent(db, id, op, { actor, runId, reason, now });
610
+ return get(db, id);
611
+ });
423
612
  }
424
613
 
425
- /** The user said yes to a guess: from here on it carries the weight of something they stated. */
426
- export function confirm(db, id, { byUser = true } = {}) {
614
+ /**
615
+ * The user said yes to a memory held for them: from here on it carries the weight of something they stated, and it is
616
+ * theirs — primary (`derived_by` cleared), never rebuilt with what an extractor drew.
617
+ */
618
+ export function confirm(db, id, { byUser = true, now = new Date().toISOString() } = {}) {
427
619
  const row = get(db, id);
428
620
  if (row.state !== 'unconfirmed') throw new Refusal(`m${id} is ${row.state} — only an unconfirmed memory can be confirmed`);
429
621
  // Held because what it says came from a page or an MCP server the model read: the model may not vouch for it.
430
622
  if (row.held_for === 'outside' && !byUser) throw new Refusal(`m${id} came from a conversation that read the web or an MCP server: only the user confirms it — on the memory page (/memory), or with sumo confirm in their own terminal`);
431
- db.prepare(`UPDATE memories SET state = 'active', provenance = 'stated' WHERE id = ?`).run(id);
432
- return get(db, id);
623
+ return tx(db, () => {
624
+ db.prepare(`UPDATE memories SET state = 'active', provenance = 'stated', derived_by = NULL WHERE id = ?`).run(id);
625
+ noteEvent(db, id, 'confirmed', { actor: byUser ? 'user' : 'model-shell', now });
626
+ return get(db, id);
627
+ });
433
628
  }
434
629
 
435
- export function reject(db, id, now = new Date().toISOString()) {
630
+ export function reject(db, id, now = new Date().toISOString(), { actor } = {}) {
436
631
  const row = get(db, id);
437
632
  if (row.state !== 'unconfirmed') throw new Refusal(`m${id} is ${row.state} — only an unconfirmed memory can be rejected`);
438
- db.prepare(`UPDATE memories SET state = 'rejected', invalid_at = ? WHERE id = ?`).run(now, id);
439
- return get(db, id);
633
+ return tx(db, () => {
634
+ db.prepare(`UPDATE memories SET state = 'rejected', invalid_at = ? WHERE id = ?`).run(now, id);
635
+ noteEvent(db, id, 'rejected', { actor, now });
636
+ return get(db, id);
637
+ });
440
638
  }
441
639
 
442
640
  /** How long a gotcha an agent filed may go unshown before it is let go. */
@@ -454,74 +652,7 @@ export function expireUnusedGotchas(db, now = new Date().toISOString()) {
454
652
  .prepare(`SELECT id FROM memories WHERE type = 'gotcha' AND state = 'active' AND provenance = 'observed' AND pinned = 0 AND hits = 0 AND created_at < ?`)
455
653
  .all(before);
456
654
  tx(db, () => {
457
- for (const { id } of unused) forget(db, id, { now });
655
+ for (const { id } of unused) forget(db, id, { now, op: 'expired', reason: `never shown in ${GOTCHA_UNUSED_DAYS} days` });
458
656
  });
459
657
  return unused.length;
460
658
  }
461
-
462
- /** The writers whose memories start as a model's guess; one confirmed by the user becomes `stated` and keeps no quote. */
463
- const GUESSING_WRITERS = ['scribe', 'dream', 'worker'];
464
- /** How old a project's scan may be before memory health names it. */
465
- const SCAN_STALE_DAYS = 7;
466
-
467
- /**
468
- * How memory is doing, as plain values for whoever says it — doctor, the memory status:
469
- * - `lastTurnAt`: when the user was last heard (null: never);
470
- * - `lastPass`: the last memory pass, `{ at, ok, error }` (null: never ran);
471
- * - `waiting`, `oldestWaitingDays`: guesses waiting for the user, and the age of the oldest;
472
- * - `guesses`: of the guesses settled, how many the user confirmed and rejected, and the share confirmed;
473
- * - `scans`: active projects whose scan is more than a week old, oldest first, with its age in days.
474
- */
475
- export function memoryHealth(db, now = new Date().toISOString()) {
476
- const daysSince = (at) => Math.floor((Date.parse(now) - Date.parse(at)) / DAY_MS);
477
- const lastTurnAt = db.prepare('SELECT MAX(ts) AS at FROM user_turns').get().at ?? null;
478
- const pass = db.prepare(`SELECT ts, ok, note FROM model_runs WHERE kind IN ('scribe', 'dream') ORDER BY ts DESC, id DESC LIMIT 1`).get();
479
- const lastPass = pass ? { at: pass.ts, ok: pass.ok === 1, error: pass.ok === 1 ? null : pass.note } : null;
480
- const waitingRow = db.prepare(`SELECT COUNT(*) AS n, MIN(created_at) AS oldest FROM memories WHERE state = 'unconfirmed'`).get();
481
- const writers = GUESSING_WRITERS.map(() => '?').join(', ');
482
- const confirmed = db.prepare(`SELECT COUNT(*) AS n FROM memories WHERE provenance = 'stated' AND source_quote IS NULL AND written_by IN (${writers})`).get(...GUESSING_WRITERS).n;
483
- const rejected = db.prepare(`SELECT COUNT(*) AS n FROM memories WHERE state = 'rejected'`).get().n;
484
- const scans = db
485
- .prepare(`SELECT slug, scanned_at FROM projects WHERE status = 'active' AND scanned_at IS NOT NULL ORDER BY scanned_at`)
486
- .all()
487
- .map((p) => ({ slug: p.slug, days: daysSince(p.scanned_at) }))
488
- .filter((p) => p.days > SCAN_STALE_DAYS);
489
- return {
490
- lastTurnAt,
491
- lastPass,
492
- waiting: waitingRow.n,
493
- oldestWaitingDays: waitingRow.oldest ? daysSince(waitingRow.oldest) : null,
494
- guesses: { confirmed, rejected, rate: confirmed + rejected > 0 ? confirmed / (confirmed + rejected) : null },
495
- scans,
496
- };
497
- }
498
-
499
- /** How long ago, in the largest whole unit: `3d ago`, `5h ago`, `just now`. */
500
- function sinceSaid(at, now) {
501
- const hours = (Date.parse(now) - Date.parse(at)) / 3_600_000;
502
- if (hours >= 24) return `${Math.floor(hours / 24)}d ago`;
503
- return hours >= 1 ? `${Math.floor(hours)}h ago` : 'just now';
504
- }
505
- /** A scan this old is a warning, not a note. */
506
- const SCAN_WARN_DAYS = 30;
507
-
508
- /**
509
- * Memory health said in one line, for doctor and the memory status: `{ ok, label, fix }`. Not ok — a warning — when the
510
- * last pass failed or a project's scan is a month old; `fix` says what to do about each.
511
- */
512
- export function memoryHealthLine(health, now = new Date().toISOString()) {
513
- const parts = [
514
- health.lastTurnAt ? `last heard ${sinceSaid(health.lastTurnAt, now)}` : 'nothing heard yet',
515
- health.lastPass ? `last pass ${sinceSaid(health.lastPass.at, now)}${health.lastPass.ok ? '' : ', failed'}` : 'no memory pass yet',
516
- health.waiting > 0 ? `${health.waiting} guess${health.waiting === 1 ? '' : 'es'} waiting, the oldest ${health.oldestWaitingDays}d` : 'no guesses waiting',
517
- ];
518
- if (health.guesses.rate !== null) parts.push(`${Math.round(health.guesses.rate * 100)}% of guesses confirmed`);
519
- if (health.scans.length > 0) {
520
- const named = health.scans.slice(0, 3).map((p) => `${p.slug}, ${p.days}d`).join('; ');
521
- parts.push(`${health.scans.length} project scan${health.scans.length === 1 ? '' : 's'} over ${SCAN_STALE_DAYS} days old (${named}${health.scans.length > 3 ? '; …' : ''})`);
522
- }
523
- const failed = health.lastPass && !health.lastPass.ok;
524
- const old = health.scans.filter((p) => p.days > SCAN_WARN_DAYS);
525
- const fix = [failed ? `the last pass failed: ${health.lastPass.error}` : null, old.length > 0 ? `rescan: sumo project rescan ${old[0].slug}${old.length > 1 ? ` (and ${old.length - 1} more)` : ''}` : null].filter(Boolean).join('; ');
526
- return { ok: !failed && old.length === 0, label: `memory: ${parts.join(' · ')}`, fix };
527
- }
package/src/page.mjs CHANGED
@@ -14,9 +14,9 @@ const MAX_BODY = 64 * 1024;
14
14
 
15
15
  /** What the page may do. Erasing for good is not here: `sumo forget --purge` stays a command someone types. */
16
16
  const ACTIONS = {
17
- confirm: (db, id) => ({ memory: memory.confirm(db, id) }),
18
- reject: (db, id) => ({ memory: memory.reject(db, id) }),
19
- forget: (db, id) => ({ memory: memory.forget(db, id) }),
17
+ confirm: (db, id) => ({ memory: memory.confirm(db, id, { byUser: true }) }),
18
+ reject: (db, id) => ({ memory: memory.reject(db, id, undefined, { actor: 'user' }) }),
19
+ forget: (db, id) => ({ memory: memory.forget(db, id, { actor: 'user' }) }),
20
20
  edit: (db, id, { body }) => edit(db, id, body),
21
21
  };
22
22
 
package/src/paths.mjs CHANGED
@@ -39,6 +39,7 @@ export function paths() {
39
39
  jobs: join(home, 'jobs'),
40
40
  models: join(home, 'models'),
41
41
  scratch: join(home, 'scratch'),
42
+ spool: join(home, 'spool'),
42
43
  };
43
44
  }
44
45