sumo-agents 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +86 -0
- package/README.md +50 -37
- package/guides/memory.md +21 -14
- package/package.json +2 -2
- package/prompts/dream.md +3 -16
- package/prompts/scribe.md +20 -15
- package/src/apply.mjs +364 -230
- package/src/card.mjs +16 -11
- package/src/catalog.mjs +3 -3
- package/src/chat.mjs +44 -23
- package/src/check.mjs +371 -0
- package/src/cli.mjs +218 -144
- package/src/db.mjs +147 -2
- package/src/dream.mjs +97 -189
- package/src/export.mjs +43 -18
- package/src/herdr.mjs +1 -3
- package/src/hooks.mjs +115 -22
- package/src/jobs.mjs +4 -2
- package/src/keeper.mjs +14 -51
- package/src/ledger.mjs +126 -0
- package/src/logs.mjs +10 -5
- package/src/loop.mjs +4 -4
- package/src/mcp.mjs +129 -51
- package/src/memeval.mjs +167 -0
- package/src/memory.mjs +261 -130
- package/src/page.mjs +3 -3
- package/src/paths.mjs +1 -0
- package/src/prime.mjs +82 -13
- package/src/rebuild.mjs +162 -0
- package/src/render.mjs +22 -0
- package/src/route.mjs +1 -1
- package/src/sandbox.mjs +5 -2
- package/src/scan.mjs +16 -4
- package/src/scribe.mjs +361 -113
- package/src/search.mjs +48 -0
- package/src/sessions.mjs +304 -53
- package/src/settings.mjs +868 -0
- package/src/setup.mjs +57 -65
- package/src/spool.mjs +308 -0
- package/src/text.mjs +118 -6
- package/src/tools.mjs +18 -7
- package/src/transcript.mjs +3 -47
- package/src/tty.mjs +102 -63
- package/src/ui.mjs +313 -271
- package/src/verify.mjs +23 -27
package/src/memory.mjs
CHANGED
|
@@ -1,16 +1,23 @@
|
|
|
1
1
|
import { createHash } from 'node:crypto';
|
|
2
2
|
import { getMeta, setMeta, tx } from './db.mjs';
|
|
3
3
|
import { redact } from './redact.mjs';
|
|
4
|
-
import { cjkTerms, clip, containing, ftsQuery, overlap, sharedStems, stem, words } from './text.mjs';
|
|
4
|
+
import { cjkTerms, clip, containing, ftsAllQuery, ftsQuery, overlap, sharedStems, stem, termsHeld, words } from './text.mjs';
|
|
5
5
|
import { matchWithin } from './workflows.mjs';
|
|
6
6
|
|
|
7
7
|
/** How much of a search that found nothing is kept as the record of it. */
|
|
8
8
|
const MISS_KEPT = 200;
|
|
9
|
+
/** A search of several words that had to settle for any of them, and whose best result holds fewer of them than this, answered nothing. */
|
|
10
|
+
export const LOOSE_BELOW = 2;
|
|
9
11
|
|
|
10
12
|
const TYPES = ['preference', 'fact', 'decision', 'gotcha', 'procedure'];
|
|
11
13
|
|
|
12
14
|
const IMPORTANCE = { preference: 0.8, procedure: 0.8, decision: 0.7, gotcha: 0.6, fact: 0.5 };
|
|
13
15
|
const PROVENANCE_WEIGHT = { stated: 1, observed: 0.9, scanned: 0.7, inferred: 0.5 };
|
|
16
|
+
/**
|
|
17
|
+
* Why a memory waits unconfirmed for the user: its conversation read the web or an MCP server ('outside', only the user
|
|
18
|
+
* confirms), or the model saved from its shell what the user never typed ('shell').
|
|
19
|
+
*/
|
|
20
|
+
const HELD_FOR = ['outside', 'shell'];
|
|
14
21
|
/** Days for a memory's rank to fall to 1/e. Absent means it does not age: a preference stays true until superseded. */
|
|
15
22
|
const DECAY_DAYS = { fact: 365, gotcha: 365 };
|
|
16
23
|
const PROJECT_MATCH_BOOST = 1.5;
|
|
@@ -130,16 +137,31 @@ export function storedKeys(raw, body) {
|
|
|
130
137
|
return kept.length > 0 ? kept.join(', ') : null;
|
|
131
138
|
}
|
|
132
139
|
|
|
140
|
+
/**
|
|
141
|
+
* One line of a memory's history (`memory_events`): what happened to it, when, by whom — `actor` is the writer of an
|
|
142
|
+
* add, 'user' or 'model-shell' where the caller says who asked, else 'sumo' — and in which pass (`runId`, an
|
|
143
|
+
* extractions row). Never the memory's words: the history outlives a purge of it.
|
|
144
|
+
*/
|
|
145
|
+
function noteEvent(db, id, op, { actor = 'sumo', runId = null, reason = null, now }) {
|
|
146
|
+
db.prepare('INSERT INTO memory_events (ts, memory_id, op, actor, run_id, reason) VALUES (?, ?, ?, ?, ?, ?)').run(now, id, op, actor ?? 'sumo', runId ?? null, reason ?? null);
|
|
147
|
+
}
|
|
148
|
+
|
|
133
149
|
/**
|
|
134
150
|
* Stores one memory. Returns it with any active memories that look like the
|
|
135
151
|
* same statement, so the caller can supersede instead of piling up duplicates.
|
|
152
|
+
* `derivedBy` is the extractor version that drew it from the journal (absent: primary, the user's to keep);
|
|
153
|
+
* `runId` is the extractions row of the pass that filed it.
|
|
136
154
|
*/
|
|
137
155
|
export function add(db, input) {
|
|
138
156
|
if (!TYPES.includes(input.type)) {
|
|
139
157
|
throw new UsageError(`unknown type "${input.type}" — one of: ${TYPES.join(', ')}`);
|
|
140
158
|
}
|
|
141
|
-
|
|
159
|
+
// Held for the user, and only for one of two reasons: nothing else is ever unconfirmed — a guess is not kept.
|
|
160
|
+
const heldFor = input.heldFor ?? null;
|
|
161
|
+
if (heldFor !== null && !HELD_FOR.includes(heldFor)) throw new UsageError(`unknown reason to hold a memory "${heldFor}" — one of: ${HELD_FOR.join(', ')}`);
|
|
162
|
+
const provenance = heldFor ? 'inferred' : (input.provenance ?? 'stated');
|
|
142
163
|
if (!(provenance in PROVENANCE_WEIGHT)) throw new UsageError(`unknown provenance "${provenance}"`);
|
|
164
|
+
if (provenance === 'inferred' && !heldFor) throw new UsageError('a guess is not kept: a memory is held for the user only with why (heldFor)');
|
|
143
165
|
|
|
144
166
|
const { text: body, redacted } = storedBody(input.type, input.body);
|
|
145
167
|
if (!body) throw new UsageError('nothing to remember — the text is empty');
|
|
@@ -154,11 +176,13 @@ export function add(db, input) {
|
|
|
154
176
|
const replaced = input.supersedes === undefined ? null : requireActive(db, input.supersedes, 'superseded');
|
|
155
177
|
const similar = replaced ? [] : findSimilar(db, { type: input.type, title: input.title, body, scope });
|
|
156
178
|
|
|
179
|
+
const writer = input.writtenBy ?? 'chat-model';
|
|
180
|
+
const runId = input.runId ?? null;
|
|
157
181
|
const { lastInsertRowid } = db
|
|
158
182
|
.prepare(
|
|
159
183
|
`INSERT INTO memories (type, scope, title, cue, gate, body, topic, keys, provenance, state, pinned, importance,
|
|
160
|
-
scan_key, written_by, source_session, source_turn, source_quote, valid_from, created_at)
|
|
161
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
184
|
+
scan_key, written_by, source_session, source_turn, source_quote, valid_from, created_at, derived_by, held_for)
|
|
185
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
162
186
|
)
|
|
163
187
|
.run(
|
|
164
188
|
input.type,
|
|
@@ -170,19 +194,23 @@ export function add(db, input) {
|
|
|
170
194
|
input.topic?.trim().toLowerCase() || null,
|
|
171
195
|
storedKeys(input.keys, body),
|
|
172
196
|
provenance,
|
|
173
|
-
|
|
197
|
+
heldFor ? 'unconfirmed' : 'active',
|
|
174
198
|
input.pin ? 1 : 0,
|
|
175
199
|
IMPORTANCE[input.type],
|
|
176
200
|
input.scanKey ?? null,
|
|
177
|
-
|
|
201
|
+
writer,
|
|
178
202
|
input.sourceSession ?? null,
|
|
179
203
|
input.sourceTurn ?? null,
|
|
180
204
|
input.sourceQuote ?? null,
|
|
181
205
|
now,
|
|
182
206
|
now,
|
|
207
|
+
// Null is what the user said to remember; a version is what an extractor drew from the journal, and may be rebuilt.
|
|
208
|
+
input.derivedBy ?? null,
|
|
209
|
+
heldFor,
|
|
183
210
|
);
|
|
184
211
|
const id = Number(lastInsertRowid);
|
|
185
|
-
|
|
212
|
+
noteEvent(db, id, 'added', { actor: writer, runId, now });
|
|
213
|
+
if (replaced) markSuperseded(db, replaced.id, id, now, { actor: writer, runId });
|
|
186
214
|
|
|
187
215
|
return { memory: get(db, id), similar, redacted };
|
|
188
216
|
});
|
|
@@ -202,22 +230,23 @@ function noteRetired(db, id) {
|
|
|
202
230
|
/** Whether a scanned fact with this text was let go by someone other than the scanner. */
|
|
203
231
|
export const scanRetired = (db, scope, scanKey, body) => getMeta(db, retiredKey(scope, scanKey)) === digestOf(body);
|
|
204
232
|
|
|
205
|
-
function markSuperseded(db, oldId, newId, now) {
|
|
233
|
+
function markSuperseded(db, oldId, newId, now, { actor, runId } = {}) {
|
|
206
234
|
noteRetired(db, oldId);
|
|
207
235
|
db.prepare(`UPDATE memories SET state = 'superseded', superseded_by = ?, invalid_at = ? WHERE id = ?`).run(
|
|
208
236
|
newId,
|
|
209
237
|
now,
|
|
210
238
|
oldId,
|
|
211
239
|
);
|
|
240
|
+
noteEvent(db, oldId, 'superseded', { actor, runId, reason: `by m${newId}`, now });
|
|
212
241
|
}
|
|
213
242
|
|
|
214
|
-
/** Links two memories that already exist: `oldId` stops being true because of `newId`. */
|
|
215
|
-
export function supersede(db, oldId, newId, now = new Date().toISOString()) {
|
|
243
|
+
/** Links two memories that already exist: `oldId` stops being true because of `newId`. `actor` and `runId` say who asked, and in which pass. */
|
|
244
|
+
export function supersede(db, oldId, newId, now = new Date().toISOString(), { actor, runId } = {}) {
|
|
216
245
|
if (oldId === newId) throw new UsageError('a memory cannot supersede itself');
|
|
217
246
|
return tx(db, () => {
|
|
218
247
|
requireActive(db, oldId, 'superseded');
|
|
219
248
|
requireActive(db, newId, 'the replacement');
|
|
220
|
-
markSuperseded(db, oldId, newId, now);
|
|
249
|
+
markSuperseded(db, oldId, newId, now, { actor, runId });
|
|
221
250
|
return get(db, oldId);
|
|
222
251
|
});
|
|
223
252
|
}
|
|
@@ -265,6 +294,15 @@ export function markHits(db, ids, now) {
|
|
|
265
294
|
db.prepare(`UPDATE memories SET hits = hits + 1, last_hit_at = ? WHERE id IN (${ids.map(() => '?').join(', ')})`).run(now, ...ids);
|
|
266
295
|
}
|
|
267
296
|
|
|
297
|
+
/** The fields of a memory that are searched, in the order the index holds them. */
|
|
298
|
+
const SEARCHED = ['title', 'cue', 'body', 'topic', 'keys'];
|
|
299
|
+
/**
|
|
300
|
+
* How much a word found in each of them counts, in that order: a title and the search terms a writer added say what a
|
|
301
|
+
* memory is about; a word somewhere in a long body says less.
|
|
302
|
+
*/
|
|
303
|
+
const RANKED = 'bm25(memories_fts, 3.0, 1.0, 1.0, 2.0, 2.0)';
|
|
304
|
+
const searchedText = (row) => SEARCHED.map((f) => row[f] ?? '').join(' ');
|
|
305
|
+
|
|
268
306
|
/**
|
|
269
307
|
* What the index found, with the memories holding a Chinese, Japanese or Korean word of the search: the index keeps a
|
|
270
308
|
* run of those characters as one word, so a word inside it is looked for in the text itself, under the same `filters`.
|
|
@@ -273,15 +311,14 @@ export function markHits(db, ids, now) {
|
|
|
273
311
|
*/
|
|
274
312
|
function withCjk(db, terms, found, filters, params) {
|
|
275
313
|
if (terms.length === 0) return found;
|
|
276
|
-
const
|
|
277
|
-
const holds = terms.flatMap(() => fields.map((f) => `m.${f} LIKE ? ESCAPE '\\'`)).join(' OR ');
|
|
314
|
+
const holds = terms.flatMap(() => SEARCHED.map((f) => `m.${f} LIKE ? ESCAPE '\\'`)).join(' OR ');
|
|
278
315
|
const rows = db
|
|
279
316
|
.prepare(`SELECT m.* FROM memories m WHERE (${holds})${filters.map((w) => ` AND ${w}`).join('')}`)
|
|
280
|
-
.all(...terms.flatMap((t) =>
|
|
317
|
+
.all(...terms.flatMap((t) => SEARCHED.map(() => containing(t))), ...params);
|
|
281
318
|
const best = found.length > 0 ? found.reduce((min, row) => Math.min(min, row.rank), 0) : -1;
|
|
282
319
|
const byId = new Map(found.map((row) => [row.id, row]));
|
|
283
320
|
for (const row of rows) {
|
|
284
|
-
const text =
|
|
321
|
+
const text = searchedText(row).toLowerCase();
|
|
285
322
|
const rank = (best * terms.filter((t) => text.includes(t)).length) / terms.length;
|
|
286
323
|
const known = byId.get(row.id);
|
|
287
324
|
if (!known || rank < known.rank) byId.set(row.id, { ...row, rank });
|
|
@@ -289,18 +326,47 @@ function withCjk(db, terms, found, filters, params) {
|
|
|
289
326
|
return [...byId.values()];
|
|
290
327
|
}
|
|
291
328
|
|
|
329
|
+
/**
|
|
330
|
+
* The memories under `filters` that hold every word of the search (`every`), or any of them. Every word: the index
|
|
331
|
+
* answers for the words it keeps apart, and a Chinese, Japanese or Korean word must be in the text itself.
|
|
332
|
+
*/
|
|
333
|
+
function matching(db, query, every, filters, params) {
|
|
334
|
+
const cjk = cjkTerms(query);
|
|
335
|
+
const also = filters.map((w) => ` AND ${w}`).join('');
|
|
336
|
+
const indexed = (match, extra = '', extraParams = []) =>
|
|
337
|
+
db
|
|
338
|
+
.prepare(`SELECT m.*, ${RANKED} AS rank FROM memories_fts JOIN memories m ON m.id = memories_fts.rowid WHERE memories_fts MATCH ?${extra}${also}`)
|
|
339
|
+
.all(match, ...extraParams, ...params);
|
|
340
|
+
if (!every) return withCjk(db, cjk, indexed(ftsQuery(query)), filters, params);
|
|
341
|
+
const holds = cjk.map(() => ` AND (${SEARCHED.map((f) => `m.${f} LIKE ? ESCAPE '\\'`).join(' OR ')})`).join('');
|
|
342
|
+
const patterns = cjk.flatMap((t) => SEARCHED.map(() => containing(t)));
|
|
343
|
+
const match = ftsAllQuery(query);
|
|
344
|
+
if (match) return indexed(match, holds, patterns);
|
|
345
|
+
// Only words the index cannot tell apart: the text itself answers, and none ranks above another.
|
|
346
|
+
return db.prepare(`SELECT m.*, -1.0 AS rank FROM memories m WHERE 1 = 1${holds}${also}`).all(...patterns, ...params);
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
/** Keeps the record of what a search could not answer: without secrets, and short — a pasted log is not a question worth keeping whole. */
|
|
350
|
+
export function recordMiss(db, query, scope, now) {
|
|
351
|
+
db.prepare('INSERT INTO search_misses (ts, query, scope) VALUES (?, ?, ?)').run(now, clip(redact(query).text, MISS_KEPT), scope);
|
|
352
|
+
}
|
|
353
|
+
|
|
292
354
|
/**
|
|
293
355
|
* Ranked search. Scope is `global` plus the named project; other projects stay
|
|
294
356
|
* invisible unless `everywhere` is set — only their hit counts come back, so
|
|
295
357
|
* the caller learns where to look without one project's memory leaking into
|
|
296
|
-
* another's work. `record: false` is for a search nobody asked for
|
|
297
|
-
*
|
|
358
|
+
* another's work. `record: false` is for a search nobody asked for, or one whose
|
|
359
|
+
* caller decides what was shown: it counts no hits and keeps no misses.
|
|
298
360
|
* `since` and `before` (ISO times) hold it to a time: since, what was said or
|
|
299
361
|
* last changed — replaced, let go — at or after it; before, what was said before it.
|
|
362
|
+
*
|
|
363
|
+
* All the words first: only when no memory in scope holds every one of them is it tried with any of them
|
|
364
|
+
* (`fellBack`). `terms: 'all' | 'any'` asks for one of the two alone. `best` is the most of the search's words one of
|
|
365
|
+
* the results holds; a search of several words that fell back and whose best holds fewer than two is `loose`: something
|
|
366
|
+
* is shown, and it is still kept as a miss — one word in common is an accident, not an answer.
|
|
300
367
|
*/
|
|
301
368
|
export function search(db, query, opts = {}) {
|
|
302
|
-
|
|
303
|
-
if (!match) throw new UsageError('nothing to search for');
|
|
369
|
+
if (!ftsQuery(query)) throw new UsageError('nothing to search for');
|
|
304
370
|
if (opts.type && !TYPES.includes(opts.type)) {
|
|
305
371
|
// Said as the user knows them: a procedure is a workflow everywhere they look.
|
|
306
372
|
throw new UsageError(`unknown type "${opts.type}" — one of: ${TYPES.map((t) => (t === 'procedure' ? 'workflow' : t)).join(', ')}`);
|
|
@@ -310,8 +376,8 @@ export function search(db, query, opts = {}) {
|
|
|
310
376
|
const now = opts.now ?? new Date().toISOString();
|
|
311
377
|
const limit = opts.limit ?? 8;
|
|
312
378
|
|
|
313
|
-
const where = [
|
|
314
|
-
const params = [
|
|
379
|
+
const where = [];
|
|
380
|
+
const params = [];
|
|
315
381
|
if (!opts.all) where.push(`m.state = 'active'`);
|
|
316
382
|
if (opts.type) {
|
|
317
383
|
where.push('m.type = ?');
|
|
@@ -327,19 +393,20 @@ export function search(db, query, opts = {}) {
|
|
|
327
393
|
params.push(opts.before);
|
|
328
394
|
}
|
|
329
395
|
|
|
330
|
-
const found = db
|
|
331
|
-
.prepare(
|
|
332
|
-
`SELECT m.*, bm25(memories_fts) AS rank FROM memories_fts JOIN memories m ON m.id = memories_fts.rowid
|
|
333
|
-
WHERE ${where.join(' AND ')}`,
|
|
334
|
-
)
|
|
335
|
-
.all(...params);
|
|
336
|
-
const rows = withCjk(db, cjkTerms(query), found, where.slice(1), params.slice(1));
|
|
337
|
-
|
|
338
396
|
const inScope = (row) => opts.everywhere || row.scope === 'global' || row.scope === projectScope;
|
|
339
|
-
//
|
|
397
|
+
// Any-word matches are on any one of them. Fine for words a caller picked; a caller searching with
|
|
340
398
|
// words nobody picked — a sentence lifted from a reply — sets a floor on how many must be shared.
|
|
341
399
|
const close = (row) => !opts.minShared || sharedStems(query, `${row.title ?? ''} ${row.cue ?? ''} ${row.body} ${row.keys ?? ''}`) >= opts.minShared;
|
|
342
|
-
const
|
|
400
|
+
const several = new Set(words(query)).size > 1;
|
|
401
|
+
let rows = [];
|
|
402
|
+
let candidates = [];
|
|
403
|
+
let fellBack = false;
|
|
404
|
+
for (const every of opts.terms === 'any' ? [false] : opts.terms === 'all' || !several ? [true] : [true, false]) {
|
|
405
|
+
rows = matching(db, query, every, where, params);
|
|
406
|
+
candidates = rows.filter((row) => inScope(row) && close(row));
|
|
407
|
+
fellBack = !every && opts.terms !== 'any';
|
|
408
|
+
if (candidates.length > 0) break;
|
|
409
|
+
}
|
|
343
410
|
// Not Math.min(...ranks): spread into arguments, a broad query's rows overflow the call stack.
|
|
344
411
|
const bestRank = candidates.reduce((best, row) => Math.min(best, row.rank), Infinity);
|
|
345
412
|
const results = candidates
|
|
@@ -351,22 +418,17 @@ export function search(db, query, opts = {}) {
|
|
|
351
418
|
for (const row of rows) if (!inScope(row)) elsewhere[row.scope] = (elsewhere[row.scope] ?? 0) + 1;
|
|
352
419
|
// Matches past the limit, counted, so a list cut short says so.
|
|
353
420
|
const more = candidates.length - results.length;
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
opts.everywhere ? 'everywhere' : (projectScope ?? 'global'),
|
|
366
|
-
);
|
|
367
|
-
}
|
|
368
|
-
|
|
369
|
-
return { results, elsewhere, more };
|
|
421
|
+
const best = results.reduce((most, row) => Math.max(most, termsHeld(query, searchedText(row))), 0);
|
|
422
|
+
const loose = fellBack && results.length > 0 && best < LOOSE_BELOW;
|
|
423
|
+
const found = { results, elsewhere, more, fellBack, best, loose };
|
|
424
|
+
|
|
425
|
+
if (opts.record === false) return found;
|
|
426
|
+
// Shown to someone who asked: used. What was matched and left out for scope or room was not.
|
|
427
|
+
markHits(db, results.map((r) => r.id), now);
|
|
428
|
+
// The record of what memory could not answer: the evidence for whether
|
|
429
|
+
// keyword search is enough or embeddings are ever worth adding.
|
|
430
|
+
if (results.length === 0 || loose) recordMiss(db, query, opts.everywhere ? 'everywhere' : (projectScope ?? 'global'), now);
|
|
431
|
+
return found;
|
|
370
432
|
}
|
|
371
433
|
|
|
372
434
|
/** Every memory connected to `id` by supersession, oldest first. */
|
|
@@ -397,46 +459,182 @@ function eraseFromIndexes(db) {
|
|
|
397
459
|
db.exec(`
|
|
398
460
|
INSERT INTO memories_fts(memories_fts, rank) VALUES ('secure-delete', 1);
|
|
399
461
|
INSERT INTO user_turns_fts(user_turns_fts, rank) VALUES ('secure-delete', 1);
|
|
462
|
+
INSERT INTO replies_fts(replies_fts, rank) VALUES ('secure-delete', 1);
|
|
400
463
|
`);
|
|
401
464
|
}
|
|
402
465
|
|
|
403
|
-
|
|
404
|
-
|
|
466
|
+
const sameLetters = (text) => text.toLowerCase().replace(/\s+/g, ' ').trim();
|
|
467
|
+
|
|
468
|
+
/** A phrase shorter than both of these is too common to erase by: "we use bun" stands in conversations that are not about it. */
|
|
469
|
+
const PHRASE_MIN_WORDS = 5;
|
|
470
|
+
const PHRASE_MIN_CHARS = 25;
|
|
471
|
+
|
|
472
|
+
/**
|
|
473
|
+
* Whether a text says any of `phrases` whole, whatever the case or the line breaks — a phrase counting only when it
|
|
474
|
+
* is at least five words or twenty-five characters; never, when no phrase is that long.
|
|
475
|
+
*/
|
|
476
|
+
export function saying(phrases) {
|
|
477
|
+
const needles = [...new Set(phrases.filter(Boolean).map(sameLetters))].filter((n) => n.length >= PHRASE_MIN_CHARS || n.split(' ').length >= PHRASE_MIN_WORDS);
|
|
478
|
+
return (text) => needles.length > 0 && needles.some((n) => sameLetters(String(text ?? '')).includes(n));
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
const hasTable = (db, name) => db.prepare(`SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?`).get(name) !== undefined;
|
|
482
|
+
|
|
483
|
+
/**
|
|
484
|
+
* What of the journal a purge of this memory erases: the turn it cites, and every turn and reply, in any
|
|
485
|
+
* conversation, that holds its whole body or the whole of the words it was drawn from — when that phrase is long
|
|
486
|
+
* enough to be this memory's and not three words any conversation may hold (`saying`). A purge once took a long
|
|
487
|
+
* plan in another conversation with a memory that said "we use bun". What only mentions part of it stays, and the
|
|
488
|
+
* purge says so. `{ source, turns, replies, sessions }`: `source` the cited turn's id when it is here; `turns` and
|
|
489
|
+
* `replies` as `{ id, ts, text }`, to be shown before they go.
|
|
490
|
+
*/
|
|
491
|
+
export function purgeReach(db, row) {
|
|
492
|
+
const says = saying([row.source_quote, row.body]);
|
|
493
|
+
const source = row.source_turn === null || row.source_turn === undefined ? null : (db.prepare('SELECT id, session_id FROM user_turns WHERE id = ?').get(row.source_turn) ?? null);
|
|
494
|
+
const turns = [];
|
|
495
|
+
const replies = [];
|
|
496
|
+
const sessions = new Set([row.source_session ?? source?.session_id].filter(Boolean));
|
|
497
|
+
// One row at a time: a journal of pasted logs is not read into memory whole.
|
|
498
|
+
for (const t of db.prepare('SELECT id, session_id, ts, text FROM user_turns ORDER BY id').iterate()) {
|
|
499
|
+
if (t.id !== source?.id && !says(t.text)) continue;
|
|
500
|
+
turns.push({ id: t.id, ts: t.ts, text: t.text });
|
|
501
|
+
sessions.add(t.session_id);
|
|
502
|
+
}
|
|
503
|
+
// A copy from before replies were kept has none.
|
|
504
|
+
if (hasTable(db, 'replies')) {
|
|
505
|
+
for (const r of db.prepare('SELECT id, session_id, ts, text FROM replies ORDER BY id').iterate()) {
|
|
506
|
+
if (!says(r.text)) continue;
|
|
507
|
+
replies.push({ id: r.id, ts: r.ts, text: r.text });
|
|
508
|
+
sessions.add(r.session_id);
|
|
509
|
+
}
|
|
510
|
+
}
|
|
511
|
+
return { source: source?.id ?? null, turns, replies, sessions: [...sessions] };
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
/** Every string a value holds, at any depth. */
|
|
515
|
+
const stringsIn = (value) => (typeof value === 'string' ? [value] : value && typeof value === 'object' ? Object.values(value).flatMap(stringsIn) : []);
|
|
516
|
+
|
|
517
|
+
/** What the passes proposed (`extractions.ops_json`) cleared wherever `which(row)` says: they keep that they ran. Returns how many. */
|
|
518
|
+
function clearProposals(db, which) {
|
|
519
|
+
if (!hasTable(db, 'extractions')) return 0;
|
|
520
|
+
let cleared = 0;
|
|
521
|
+
for (const x of db.prepare('SELECT id, session_id, ops_json FROM extractions WHERE ops_json IS NOT NULL').all()) {
|
|
522
|
+
if (!which(x)) continue;
|
|
523
|
+
db.prepare('UPDATE extractions SET ops_json = NULL WHERE id = ?').run(x.id);
|
|
524
|
+
cleared++;
|
|
525
|
+
}
|
|
526
|
+
return cleared;
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
/**
|
|
530
|
+
* Erases what `purgeReach` finds, and what the passes over those conversations proposed: a pass keeps the model's
|
|
531
|
+
* answer, the quote among it — those of the conversations touched, of the passes that filed the memory (`runs`), and
|
|
532
|
+
* any other whose answer says it. The same rule for the database and for each backup. Returns the reach, with `ops`
|
|
533
|
+
* the answers cleared.
|
|
534
|
+
*/
|
|
535
|
+
export function eraseSaid(db, row, { runs = [] } = {}) {
|
|
536
|
+
const reach = purgeReach(db, row);
|
|
537
|
+
for (const { id } of reach.turns) db.prepare('DELETE FROM user_turns WHERE id = ?').run(id);
|
|
538
|
+
for (const { id } of reach.replies) db.prepare('DELETE FROM replies WHERE id = ?').run(id);
|
|
539
|
+
const says = saying([row.source_quote, row.body]);
|
|
540
|
+
const proposed = (json) => {
|
|
541
|
+
try {
|
|
542
|
+
return stringsIn(JSON.parse(json));
|
|
543
|
+
} catch {
|
|
544
|
+
return [json];
|
|
545
|
+
}
|
|
546
|
+
};
|
|
547
|
+
const ops = clearProposals(db, (x) => reach.sessions.includes(x.session_id) || runs.includes(x.id) || proposed(x.ops_json).some(says));
|
|
548
|
+
return { ...reach, ops };
|
|
549
|
+
}
|
|
550
|
+
|
|
551
|
+
const JOURNAL_TABLE = { turn: 'user_turns', reply: 'replies' };
|
|
552
|
+
|
|
553
|
+
/**
|
|
554
|
+
* One turn or reply erased from a database — this one, or a backup: the row with that id and those very words (in a
|
|
555
|
+
* copy, the number alone may be another row), and what the passes over its conversation proposed. False when the
|
|
556
|
+
* row is not there.
|
|
557
|
+
*/
|
|
558
|
+
export function eraseRow(db, { kind, id, text, session_id: session }) {
|
|
559
|
+
const table = JOURNAL_TABLE[kind];
|
|
560
|
+
if (!hasTable(db, table) || Number(db.prepare(`DELETE FROM ${table} WHERE id = ? AND text = ?`).run(id, text).changes) === 0) return false;
|
|
561
|
+
clearProposals(db, (x) => x.session_id === session);
|
|
562
|
+
return true;
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
/**
|
|
566
|
+
* `sumo forget --purge t12`: one turn or reply erased for good — what a memory's purge leaves when a conversation
|
|
567
|
+
* only mentions part of it. `entry` is the journal row (`journalEntry` in sessions.mjs). Returns the ids of the
|
|
568
|
+
* memories that cite the turn: they hold its words still, and are the user's to purge or keep.
|
|
569
|
+
*/
|
|
570
|
+
export function purgeRow(db, entry) {
|
|
571
|
+
const cites = entry.kind === 'turn' ? db.prepare('SELECT id FROM memories WHERE source_turn = ? ORDER BY id').all(entry.id).map((m) => m.id) : [];
|
|
572
|
+
tx(db, () => {
|
|
573
|
+
eraseFromIndexes(db);
|
|
574
|
+
eraseRow(db, entry);
|
|
575
|
+
});
|
|
576
|
+
// As for a memory: the write-ahead log still holds the pages as they were.
|
|
577
|
+
db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
|
|
578
|
+
return cites;
|
|
579
|
+
}
|
|
580
|
+
|
|
581
|
+
/**
|
|
582
|
+
* Stops a memory being true. With `purge` it is erased outright, for things that should never have been stored: the
|
|
583
|
+
* memory, the sentence it came from, and every turn and reply of the journal that says it whole (`eraseSaid`;
|
|
584
|
+
* `erased_turns` and `erased_replies` are those rows). `actor`, `runId` and `reason` go into its history; `op`
|
|
585
|
+
* names a letting-go that is not the user's.
|
|
586
|
+
*/
|
|
587
|
+
export function forget(db, id, { purge = false, now = new Date().toISOString(), actor, runId, reason, op = 'forgotten' } = {}) {
|
|
405
588
|
const row = get(db, id);
|
|
406
589
|
if (purge) {
|
|
590
|
+
let erased = null;
|
|
407
591
|
tx(db, () => {
|
|
408
592
|
eraseFromIndexes(db);
|
|
409
593
|
// A scanned fact let go for good stays let go: only its digest is kept, never its words.
|
|
410
594
|
noteRetired(db, id);
|
|
595
|
+
const runs = db.prepare('SELECT run_id FROM memory_events WHERE memory_id = ? AND run_id IS NOT NULL').all(id).map((e) => e.run_id);
|
|
411
596
|
db.prepare('DELETE FROM memories WHERE id = ?').run(id);
|
|
412
|
-
// Erasing the memory but keeping the sentence it came from would not be erasing it.
|
|
413
|
-
|
|
597
|
+
// Erasing the memory but keeping the sentence it came from, or an answer that said it back, would not be erasing it.
|
|
598
|
+
erased = eraseSaid(db, row, { runs });
|
|
599
|
+
noteEvent(db, id, 'purged', { actor, runId, now });
|
|
414
600
|
});
|
|
415
601
|
// The write-ahead log still holds the pages as they were before the delete; copy it into the file and empty it.
|
|
416
602
|
db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
|
|
417
|
-
return { ...row, state: 'purged' };
|
|
603
|
+
return { ...row, state: 'purged', erased_source: erased.source, erased_turns: erased.turns, erased_replies: erased.replies };
|
|
418
604
|
}
|
|
419
605
|
if (row.state === 'invalid') return row;
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
606
|
+
return tx(db, () => {
|
|
607
|
+
noteRetired(db, id);
|
|
608
|
+
db.prepare(`UPDATE memories SET state = 'invalid', invalid_at = ? WHERE id = ?`).run(now, id);
|
|
609
|
+
noteEvent(db, id, op, { actor, runId, reason, now });
|
|
610
|
+
return get(db, id);
|
|
611
|
+
});
|
|
423
612
|
}
|
|
424
613
|
|
|
425
|
-
/**
|
|
426
|
-
|
|
614
|
+
/**
|
|
615
|
+
* The user said yes to a memory held for them: from here on it carries the weight of something they stated, and it is
|
|
616
|
+
* theirs — primary (`derived_by` cleared), never rebuilt with what an extractor drew.
|
|
617
|
+
*/
|
|
618
|
+
export function confirm(db, id, { byUser = true, now = new Date().toISOString() } = {}) {
|
|
427
619
|
const row = get(db, id);
|
|
428
620
|
if (row.state !== 'unconfirmed') throw new Refusal(`m${id} is ${row.state} — only an unconfirmed memory can be confirmed`);
|
|
429
621
|
// Held because what it says came from a page or an MCP server the model read: the model may not vouch for it.
|
|
430
622
|
if (row.held_for === 'outside' && !byUser) throw new Refusal(`m${id} came from a conversation that read the web or an MCP server: only the user confirms it — on the memory page (/memory), or with sumo confirm in their own terminal`);
|
|
431
|
-
db
|
|
432
|
-
|
|
623
|
+
return tx(db, () => {
|
|
624
|
+
db.prepare(`UPDATE memories SET state = 'active', provenance = 'stated', derived_by = NULL WHERE id = ?`).run(id);
|
|
625
|
+
noteEvent(db, id, 'confirmed', { actor: byUser ? 'user' : 'model-shell', now });
|
|
626
|
+
return get(db, id);
|
|
627
|
+
});
|
|
433
628
|
}
|
|
434
629
|
|
|
435
|
-
export function reject(db, id, now = new Date().toISOString()) {
|
|
630
|
+
export function reject(db, id, now = new Date().toISOString(), { actor } = {}) {
|
|
436
631
|
const row = get(db, id);
|
|
437
632
|
if (row.state !== 'unconfirmed') throw new Refusal(`m${id} is ${row.state} — only an unconfirmed memory can be rejected`);
|
|
438
|
-
db
|
|
439
|
-
|
|
633
|
+
return tx(db, () => {
|
|
634
|
+
db.prepare(`UPDATE memories SET state = 'rejected', invalid_at = ? WHERE id = ?`).run(now, id);
|
|
635
|
+
noteEvent(db, id, 'rejected', { actor, now });
|
|
636
|
+
return get(db, id);
|
|
637
|
+
});
|
|
440
638
|
}
|
|
441
639
|
|
|
442
640
|
/** How long a gotcha an agent filed may go unshown before it is let go. */
|
|
@@ -454,74 +652,7 @@ export function expireUnusedGotchas(db, now = new Date().toISOString()) {
|
|
|
454
652
|
.prepare(`SELECT id FROM memories WHERE type = 'gotcha' AND state = 'active' AND provenance = 'observed' AND pinned = 0 AND hits = 0 AND created_at < ?`)
|
|
455
653
|
.all(before);
|
|
456
654
|
tx(db, () => {
|
|
457
|
-
for (const { id } of unused) forget(db, id, { now });
|
|
655
|
+
for (const { id } of unused) forget(db, id, { now, op: 'expired', reason: `never shown in ${GOTCHA_UNUSED_DAYS} days` });
|
|
458
656
|
});
|
|
459
657
|
return unused.length;
|
|
460
658
|
}
|
|
461
|
-
|
|
462
|
-
/** The writers whose memories start as a model's guess; one confirmed by the user becomes `stated` and keeps no quote. */
|
|
463
|
-
const GUESSING_WRITERS = ['scribe', 'dream', 'worker'];
|
|
464
|
-
/** How old a project's scan may be before memory health names it. */
|
|
465
|
-
const SCAN_STALE_DAYS = 7;
|
|
466
|
-
|
|
467
|
-
/**
|
|
468
|
-
* How memory is doing, as plain values for whoever says it — doctor, the memory status:
|
|
469
|
-
* - `lastTurnAt`: when the user was last heard (null: never);
|
|
470
|
-
* - `lastPass`: the last memory pass, `{ at, ok, error }` (null: never ran);
|
|
471
|
-
* - `waiting`, `oldestWaitingDays`: guesses waiting for the user, and the age of the oldest;
|
|
472
|
-
* - `guesses`: of the guesses settled, how many the user confirmed and rejected, and the share confirmed;
|
|
473
|
-
* - `scans`: active projects whose scan is more than a week old, oldest first, with its age in days.
|
|
474
|
-
*/
|
|
475
|
-
export function memoryHealth(db, now = new Date().toISOString()) {
|
|
476
|
-
const daysSince = (at) => Math.floor((Date.parse(now) - Date.parse(at)) / DAY_MS);
|
|
477
|
-
const lastTurnAt = db.prepare('SELECT MAX(ts) AS at FROM user_turns').get().at ?? null;
|
|
478
|
-
const pass = db.prepare(`SELECT ts, ok, note FROM model_runs WHERE kind IN ('scribe', 'dream') ORDER BY ts DESC, id DESC LIMIT 1`).get();
|
|
479
|
-
const lastPass = pass ? { at: pass.ts, ok: pass.ok === 1, error: pass.ok === 1 ? null : pass.note } : null;
|
|
480
|
-
const waitingRow = db.prepare(`SELECT COUNT(*) AS n, MIN(created_at) AS oldest FROM memories WHERE state = 'unconfirmed'`).get();
|
|
481
|
-
const writers = GUESSING_WRITERS.map(() => '?').join(', ');
|
|
482
|
-
const confirmed = db.prepare(`SELECT COUNT(*) AS n FROM memories WHERE provenance = 'stated' AND source_quote IS NULL AND written_by IN (${writers})`).get(...GUESSING_WRITERS).n;
|
|
483
|
-
const rejected = db.prepare(`SELECT COUNT(*) AS n FROM memories WHERE state = 'rejected'`).get().n;
|
|
484
|
-
const scans = db
|
|
485
|
-
.prepare(`SELECT slug, scanned_at FROM projects WHERE status = 'active' AND scanned_at IS NOT NULL ORDER BY scanned_at`)
|
|
486
|
-
.all()
|
|
487
|
-
.map((p) => ({ slug: p.slug, days: daysSince(p.scanned_at) }))
|
|
488
|
-
.filter((p) => p.days > SCAN_STALE_DAYS);
|
|
489
|
-
return {
|
|
490
|
-
lastTurnAt,
|
|
491
|
-
lastPass,
|
|
492
|
-
waiting: waitingRow.n,
|
|
493
|
-
oldestWaitingDays: waitingRow.oldest ? daysSince(waitingRow.oldest) : null,
|
|
494
|
-
guesses: { confirmed, rejected, rate: confirmed + rejected > 0 ? confirmed / (confirmed + rejected) : null },
|
|
495
|
-
scans,
|
|
496
|
-
};
|
|
497
|
-
}
|
|
498
|
-
|
|
499
|
-
/** How long ago, in the largest whole unit: `3d ago`, `5h ago`, `just now`. */
|
|
500
|
-
function sinceSaid(at, now) {
|
|
501
|
-
const hours = (Date.parse(now) - Date.parse(at)) / 3_600_000;
|
|
502
|
-
if (hours >= 24) return `${Math.floor(hours / 24)}d ago`;
|
|
503
|
-
return hours >= 1 ? `${Math.floor(hours)}h ago` : 'just now';
|
|
504
|
-
}
|
|
505
|
-
/** A scan this old is a warning, not a note. */
|
|
506
|
-
const SCAN_WARN_DAYS = 30;
|
|
507
|
-
|
|
508
|
-
/**
|
|
509
|
-
* Memory health said in one line, for doctor and the memory status: `{ ok, label, fix }`. Not ok — a warning — when the
|
|
510
|
-
* last pass failed or a project's scan is a month old; `fix` says what to do about each.
|
|
511
|
-
*/
|
|
512
|
-
export function memoryHealthLine(health, now = new Date().toISOString()) {
|
|
513
|
-
const parts = [
|
|
514
|
-
health.lastTurnAt ? `last heard ${sinceSaid(health.lastTurnAt, now)}` : 'nothing heard yet',
|
|
515
|
-
health.lastPass ? `last pass ${sinceSaid(health.lastPass.at, now)}${health.lastPass.ok ? '' : ', failed'}` : 'no memory pass yet',
|
|
516
|
-
health.waiting > 0 ? `${health.waiting} guess${health.waiting === 1 ? '' : 'es'} waiting, the oldest ${health.oldestWaitingDays}d` : 'no guesses waiting',
|
|
517
|
-
];
|
|
518
|
-
if (health.guesses.rate !== null) parts.push(`${Math.round(health.guesses.rate * 100)}% of guesses confirmed`);
|
|
519
|
-
if (health.scans.length > 0) {
|
|
520
|
-
const named = health.scans.slice(0, 3).map((p) => `${p.slug}, ${p.days}d`).join('; ');
|
|
521
|
-
parts.push(`${health.scans.length} project scan${health.scans.length === 1 ? '' : 's'} over ${SCAN_STALE_DAYS} days old (${named}${health.scans.length > 3 ? '; …' : ''})`);
|
|
522
|
-
}
|
|
523
|
-
const failed = health.lastPass && !health.lastPass.ok;
|
|
524
|
-
const old = health.scans.filter((p) => p.days > SCAN_WARN_DAYS);
|
|
525
|
-
const fix = [failed ? `the last pass failed: ${health.lastPass.error}` : null, old.length > 0 ? `rescan: sumo project rescan ${old[0].slug}${old.length > 1 ? ` (and ${old.length - 1} more)` : ''}` : null].filter(Boolean).join('; ');
|
|
526
|
-
return { ok: !failed && old.length === 0, label: `memory: ${parts.join(' · ')}`, fix };
|
|
527
|
-
}
|
package/src/page.mjs
CHANGED
|
@@ -14,9 +14,9 @@ const MAX_BODY = 64 * 1024;
|
|
|
14
14
|
|
|
15
15
|
/** What the page may do. Erasing for good is not here: `sumo forget --purge` stays a command someone types. */
|
|
16
16
|
const ACTIONS = {
|
|
17
|
-
confirm: (db, id) => ({ memory: memory.confirm(db, id) }),
|
|
18
|
-
reject: (db, id) => ({ memory: memory.reject(db, id) }),
|
|
19
|
-
forget: (db, id) => ({ memory: memory.forget(db, id) }),
|
|
17
|
+
confirm: (db, id) => ({ memory: memory.confirm(db, id, { byUser: true }) }),
|
|
18
|
+
reject: (db, id) => ({ memory: memory.reject(db, id, undefined, { actor: 'user' }) }),
|
|
19
|
+
forget: (db, id) => ({ memory: memory.forget(db, id, { actor: 'user' }) }),
|
|
20
20
|
edit: (db, id, { body }) => edit(db, id, body),
|
|
21
21
|
};
|
|
22
22
|
|