sumo-agents 0.6.1 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/db.mjs CHANGED
@@ -1,4 +1,4 @@
1
- import { chmodSync, closeSync, existsSync, mkdirSync, openSync, readdirSync, rmSync, statSync } from 'node:fs';
1
+ import { chmodSync, closeSync, mkdirSync, openSync, readdirSync, rmSync, statSync } from 'node:fs';
2
2
  import { basename, dirname, join } from 'node:path';
3
3
  import { homeProblem, paths } from './paths.mjs';
4
4
  import { DatabaseSync } from './sqlite.mjs';
@@ -314,6 +314,146 @@ const MIGRATIONS = [
314
314
  DELETE FROM meta WHERE key IN ('config.chat.plain', 'config.chat.motion', 'config.model.source', 'config.model.repo', 'config.model.file',
315
315
  'config.prime.budget', 'config.scratch.days', 'config.jobs.escalate', 'config.guides.auto');
316
316
  `,
317
+ // 16 — memory in layers. The journal is the source of truth: a turn is kept whole (full_len is how long it was before
318
+ // any cut), stamped with the project it was about, and carries a digest (sha) of its session, time and words, so an
319
+ // event kept aside while the database could not be written is replayed without ever landing twice; the assistant's
320
+ // final replies are kept beside the turns (replies), searchable like them. What a model drew from the journal is
321
+ // marked (memories.derived_by, the extractor's version; NULL is what the user said to remember) and every pass over
322
+ // a conversation is written down before it runs (extractions), so one that died or failed is seen. memory_events is
323
+ // each change of a memory's state from here on, started from what the state columns already say. Guesses are no
324
+ // longer kept: the ones still waiting are rejected here, once — their rows and history stay — except what is held
325
+ // because the conversation read the web or an MCP server (held_for), which only the user settles.
326
+ (db) => {
327
+ addColumnsIfMissing(db, 'user_turns', ['project TEXT', 'full_len INTEGER', 'sha TEXT']);
328
+ addColumnsIfMissing(db, 'memories', ['derived_by TEXT']);
329
+ db.exec(`
330
+ -- ADD COLUMN takes no UNIQUE: the index says it. Rows from before carry no digest, and any number of those may.
331
+ CREATE UNIQUE INDEX IF NOT EXISTS user_turns_sha ON user_turns(sha) WHERE sha IS NOT NULL;
332
+ CREATE INDEX IF NOT EXISTS user_turns_session ON user_turns(session_id, id);
333
+
334
+ CREATE TABLE IF NOT EXISTS replies (
335
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
336
+ session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE,
337
+ ts TEXT NOT NULL,
338
+ text TEXT NOT NULL,
339
+ project TEXT,
340
+ full_len INTEGER,
341
+ sha TEXT UNIQUE
342
+ ) STRICT;
343
+ CREATE INDEX IF NOT EXISTS replies_session ON replies(session_id, id);
344
+
345
+ CREATE VIRTUAL TABLE IF NOT EXISTS replies_fts USING fts5(
346
+ text, content='replies', content_rowid='id', tokenize='porter unicode61'
347
+ );
348
+ CREATE TRIGGER IF NOT EXISTS replies_fts_insert AFTER INSERT ON replies BEGIN
349
+ INSERT INTO replies_fts(rowid, text) VALUES (new.id, new.text);
350
+ END;
351
+ CREATE TRIGGER IF NOT EXISTS replies_fts_delete AFTER DELETE ON replies BEGIN
352
+ INSERT INTO replies_fts(replies_fts, rowid, text) VALUES ('delete', old.id, old.text);
353
+ END;
354
+
355
+ -- Written 'running' before the model is called: one still 'running' after its process died is a failure.
356
+ -- first_turn and last_turn are user_turns ids, both ends included.
357
+ CREATE TABLE IF NOT EXISTS extractions (
358
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
359
+ session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE,
360
+ extractor_version TEXT NOT NULL,
361
+ status TEXT NOT NULL CHECK (status IN ('running', 'ok', 'failed', 'quarantined')),
362
+ attempts INTEGER NOT NULL DEFAULT 0,
363
+ first_turn INTEGER,
364
+ last_turn INTEGER,
365
+ ops_json TEXT,
366
+ error TEXT,
367
+ started_at TEXT NOT NULL,
368
+ ended_at TEXT
369
+ ) STRICT;
370
+ CREATE INDEX IF NOT EXISTS extractions_session ON extractions(session_id, extractor_version, status);
371
+
372
+ -- memory_id has no REFERENCES on purpose: the history of a memory outlives a purge of it (ids are never handed
373
+ -- out twice), so nothing here may hold a memory's words. run_id is the extractions row a pass made the change in.
374
+ CREATE TABLE IF NOT EXISTS memory_events (
375
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
376
+ ts TEXT NOT NULL,
377
+ memory_id INTEGER NOT NULL,
378
+ op TEXT NOT NULL,
379
+ actor TEXT NOT NULL,
380
+ run_id INTEGER,
381
+ reason TEXT
382
+ ) STRICT;
383
+ CREATE INDEX IF NOT EXISTS memory_events_memory ON memory_events(memory_id, id);
384
+ `);
385
+ // What follows reads the rows as they are and is done once: a history already begun is not begun again, should
386
+ // this step ever be run against a database that has had it.
387
+ if (db.prepare('SELECT 1 FROM memory_events LIMIT 1').get()) return;
388
+ db.exec(`
389
+ -- What a model filed and the user never made their own is derived, by the extractor from before versions were
390
+ -- kept. One the user confirmed is stated and keeps no quote (memory.mjs): that one is theirs now.
391
+ UPDATE memories SET derived_by = 'v0'
392
+ WHERE derived_by IS NULL AND written_by IN ('scribe', 'dream', 'worker') AND NOT (provenance = 'stated' AND source_quote IS NULL);
393
+
394
+ -- History so far, as the state columns tell it: when each memory came, and how it ended if it has.
395
+ INSERT INTO memory_events (ts, memory_id, op, actor, reason)
396
+ SELECT created_at, id, 'added', written_by, 'backfilled' FROM memories ORDER BY id;
397
+ INSERT INTO memory_events (ts, memory_id, op, actor, reason)
398
+ SELECT valid_from, id, 'confirmed', 'user', 'backfilled' FROM memories
399
+ WHERE written_by IN ('scribe', 'dream', 'worker') AND provenance = 'stated' AND source_quote IS NULL AND state != 'unconfirmed' ORDER BY id;
400
+ INSERT INTO memory_events (ts, memory_id, op, actor, reason)
401
+ SELECT COALESCE(invalid_at, created_at), id,
402
+ CASE state WHEN 'superseded' THEN 'superseded' WHEN 'invalid' THEN 'forgotten' ELSE 'rejected' END, 'unknown', 'backfilled'
403
+ FROM memories WHERE state IN ('superseded', 'invalid', 'rejected') ORDER BY id;
404
+
405
+ -- The guesses still waiting, rejected once. The event first: it names the rows while they are still waiting.
406
+ INSERT INTO memory_events (ts, memory_id, op, actor, reason)
407
+ SELECT strftime('%Y-%m-%dT%H:%M:%fZ', 'now'), id, 'rejected', 'migration', 'a guess still waiting when guesses stopped being kept'
408
+ FROM memories WHERE state = 'unconfirmed' AND held_for IS NULL ORDER BY id;
409
+ UPDATE memories SET state = 'rejected', invalid_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now')
410
+ WHERE state = 'unconfirmed' AND held_for IS NULL;
411
+
412
+ -- A conversation the old writer had read to its end is not read again as if new.
413
+ INSERT INTO extractions (session_id, extractor_version, status, attempts, first_turn, last_turn, started_at, ended_at)
414
+ SELECT session_id, 'v0', 'ok', 1, MIN(id), MAX(id), MIN(ts), MAX(ts) FROM user_turns GROUP BY session_id HAVING MIN(scribed) = 1;
415
+ `);
416
+ },
417
+ // 17: two things migration 16 got wrong, mended for a database that has had it — 16 itself stays as it ran.
418
+ (db) => {
419
+ const now = new Date().toISOString();
420
+ // 16 rejected everything still waiting with no hold named. A pass's guess was meant; a memory or a taught workflow
421
+ // the model's shell had saved, which the user had not yet answered, was not. Those it rejected — its own line of
422
+ // the history says which — wait for the user again, under the name that hold has now. One the user has touched
423
+ // since is no longer `rejected` and is left alone.
424
+ const held = db
425
+ .prepare(
426
+ `SELECT m.id FROM memories m WHERE m.state = 'rejected' AND m.written_by = 'chat-model'
427
+ AND EXISTS (SELECT 1 FROM memory_events e WHERE e.memory_id = m.id AND e.op = 'rejected' AND e.actor = 'migration') ORDER BY m.id`,
428
+ )
429
+ .all();
430
+ for (const { id } of held) {
431
+ db.prepare(`UPDATE memories SET state = 'unconfirmed', held_for = 'shell', invalid_at = NULL WHERE id = ?`).run(id);
432
+ db.prepare(`INSERT INTO memory_events (ts, memory_id, op, actor, reason) VALUES (?, ?, 'held', 'migration', 'saved by the model''s shell and never answered: it waits for the user again')`).run(now, id);
433
+ }
434
+ // 16 counted a conversation as read only when the old writer had read all of it. Of one half read, the turns it
435
+ // had got to (`scribed`) and no pass has covered since are covered now, each run of them, so only the rest is owed.
436
+ db.exec(`
437
+ INSERT INTO extractions (session_id, extractor_version, status, attempts, first_turn, last_turn, started_at, ended_at)
438
+ SELECT session_id, 'v0', 'ok', 1, MIN(id), MAX(id), MIN(ts), MAX(ts)
439
+ FROM (SELECT id, session_id, ts, was_read,
440
+ ROW_NUMBER() OVER (PARTITION BY session_id ORDER BY id) - ROW_NUMBER() OVER (PARTITION BY session_id, was_read ORDER BY id) AS run
441
+ FROM (SELECT t.id, t.session_id, t.ts,
442
+ (t.scribed = 1 AND NOT EXISTS (SELECT 1 FROM extractions x WHERE x.session_id = t.session_id AND x.status IN ('ok', 'quarantined')
443
+ AND t.id BETWEEN x.first_turn AND x.last_turn)) AS was_read
444
+ FROM user_turns t))
445
+ WHERE was_read = 1 GROUP BY session_id, run ORDER BY MIN(id);
446
+ `);
447
+ },
448
+ // 18 — MCP servers, a row each: the spec as mcp.json held it (a command with its args and env, or a url with its headers
449
+ // or token, and the allow list), as JSON. The file is brought in once, the first time the servers are read (mcp.mjs), and
450
+ // never read again: the model's shell could make a file of that name, never a row here.
451
+ `
452
+ CREATE TABLE IF NOT EXISTS mcp_servers (
453
+ name TEXT PRIMARY KEY,
454
+ spec TEXT NOT NULL
455
+ ) STRICT;
456
+ `,
317
457
  ];
318
458
 
319
459
  /**
@@ -354,7 +494,12 @@ function opened(file) {
354
494
  mkdirSync(dirname(file), { recursive: true, mode: 0o700 });
355
495
  // SQLite gives the -wal and -shm files the database's own mode: the file is made private before SQLite first opens
356
496
  // it, or they are born readable by everyone (umask 022) and keep the latest writes there.
357
- if (!existsSync(file)) closeSync(openSync(file, 'a', 0o600));
497
+ try {
498
+ closeSync(openSync(file, 'wx', 0o600));
499
+ } catch (cause) {
500
+ // There already: as it is, whatever it is — SQLite says below when it is not a database it can open.
501
+ if (cause.code !== 'EEXIST') throw cause;
502
+ }
358
503
  for (const side of [file, `${file}-wal`, `${file}-shm`]) {
359
504
  try {
360
505
  if (statSync(side).mode & 0o077) chmodSync(side, 0o600);
package/src/dream.mjs CHANGED
@@ -1,33 +1,30 @@
1
1
  import { readFileSync } from 'node:fs';
2
2
  import { join } from 'node:path';
3
- import { keepNamed, namedPairs, nameForJudging, raiseNotice } from './apply.mjs';
3
+ import { raiseNotice } from './apply.mjs';
4
4
  import { getMeta, setMeta, tx } from './db.mjs';
5
5
  import { storedKeys } from './memory.mjs';
6
6
  import { callModel, logRun } from './model.mjs';
7
7
  import { REPO_ROOT } from './paths.mjs';
8
8
  import { redact } from './redact.mjs';
9
- import { knownProjectsLine, askAndApply, relatedMemories, runScribe, withLock, writerOff } from './scribe.mjs';
10
- import { line } from './render.mjs';
11
- import { sessionsAwaitingDream } from './sessions.mjs';
12
9
  import { CONFIG_DEFAULTS } from './setup.mjs';
13
10
  import { clip, stem, words } from './text.mjs';
14
11
 
15
- /** Finished sessions waiting that make a pass worth running by itself. */
16
- export const SESSIONS_TO_TIDY = 3;
17
- /** How long memory goes between tidies — pairs judged, search terms filled in — when no conversation brings one sooner. */
12
+ /**
13
+ * Memory's tidy, the second half of the one memory pass: weekly, memory itself is gone over — search terms filled in,
14
+ * rules checked for being said twice, pairs judged. It reads memory only; conversations are read once, by the filing
15
+ * half (scribe.mjs), which also asks the pair judge here about a new statement that looks like an old one.
16
+ */
17
+
18
+ /** How long memory goes between tidies. */
18
19
  const TIDY_EVERY_MS = 7 * 86_400_000;
19
- /** The oldest memories a pass reads beside the related and the newest, moving on each pass so nothing old goes unread for long. */
20
- const SWEEP_SLICE = 20;
21
- /** Pairs put to the local model in one call, and calls in one pass: one when conversations bring it, a few when it is the weekly tidy. */
20
+ /** Pairs put to the local model in one call, and calls in one tidy. */
22
21
  const PAIRS_PER_CALL = 10;
23
- const JUDGE_CALLS = 1;
24
22
  const TIDY_JUDGE_CALLS = 3;
25
23
  /** Memories given search terms in one tidy: what the local model answers well in one call. */
26
24
  const KEYS_PER_TIDY = 20;
27
25
  const LABELS = ['same', 'replaces', 'conflict', 'unrelated'];
28
26
  /** A topic as the scribe gives one: a lowercase word. */
29
27
  const TOPIC = /^[a-z][a-z0-9-]{1,23}$/;
30
- const SWEEP_CURSOR = 'dream.sweep_cursor';
31
28
  const KEYS_CURSOR = 'dream.keys_cursor';
32
29
  /** The last rule checked for being said already in other words: each is checked once. */
33
30
  const REPEATS_CURSOR = 'dream.repeats_cursor';
@@ -37,75 +34,34 @@ const REPEAT_NEIGHBOURS = 40;
37
34
  const LAST_TIDY = 'dream.last_tidy';
38
35
  const NO_USAGE = { inputTokens: 0, outputTokens: 0, costUsd: 0, cacheReadTokens: null, cacheCreationTokens: null };
39
36
 
40
- const MAX_SESSIONS = 5;
41
- const TURN_MAX = 600;
42
- /** Each session's share of what fits the local model's context: its latest turns, as many as fit. The writer has filed every one already. */
43
- const SESSION_CHARS = 8_000;
37
+ /** Where the pairs the repeat check named wait to be judged on their own. */
38
+ const NAMED_PAIRS = 'dream.named_pairs';
44
39
 
45
- /** The last of a session's turns that fit its share, each clipped; the rest are counted, not shown. */
46
- function latestThatFit(turns) {
47
- const shown = [];
48
- let used = 0;
49
- for (const t of [...turns].reverse()) {
50
- const line = `[t${t.id}] user: ${clip(t.text, TURN_MAX)}`;
51
- if (shown.length > 0 && used + line.length > SESSION_CHARS) break;
52
- shown.unshift(line);
53
- used += line.length;
40
+ /** The pairs named that wait to be judged on their own, in the order named: `{ label, a, b, note }`, a < b. */
41
+ function namedPairs(db) {
42
+ try {
43
+ const named = JSON.parse(getMeta(db, NAMED_PAIRS) ?? '[]');
44
+ return Array.isArray(named) ? named.filter((p) => Number.isInteger(p?.a) && Number.isInteger(p?.b)) : [];
45
+ } catch {
46
+ return [];
54
47
  }
55
- const left = turns.length - shown.length;
56
- return left > 0 ? [`[${left} earlier turn${left === 1 ? '' : 's'} not shown]`, ...shown] : shown;
57
48
  }
58
- const MAX_MEMORIES = 60;
59
49
 
60
- /**
61
- * The slow pass. The writer files each conversation as it happens; this one
62
- * reads several finished conversations side by side, which is the only way to
63
- * see what no single one shows: a correction the user keeps repeating, two
64
- * memories that cannot both be true, a routine worth naming.
65
- */
66
- function buildBundle(db, sessions) {
67
- const ids = sessions.map((s) => s.id);
68
- const marks = ids.map(() => '?').join(', ');
69
- const turns = db.prepare(`SELECT * FROM user_turns WHERE session_id IN (${marks}) ORDER BY id`).all(...ids);
70
- const checkpoints = db.prepare(`SELECT * FROM checkpoints WHERE session_id IN (${marks}) ORDER BY id`).all(...ids);
71
- const projects = db.prepare(`SELECT DISTINCT slug FROM session_injections WHERE session_id IN (${marks}) AND slug NOT LIKE '%:%'`).all(...ids).map((r) => r.slug);
72
-
73
- const scopes = ['global', ...projects.map((p) => `project:${p}`)];
74
- const { memories, cursor } = shownMemories(db, scopes, turns.map((t) => t.text));
75
-
76
- const sections = [];
77
- for (const session of sessions) {
78
- sections.push(`--- session started ${session.started_at.slice(0, 16)}`);
79
- sections.push(...latestThatFit(turns.filter((x) => x.session_id === session.id)));
80
- for (const c of checkpoints.filter((x) => x.session_id === session.id)) sections.push(`[done in ${c.project}]: ${c.done}${c.next_step ? ` → next: ${c.next_step}` : ''}`);
81
- }
82
-
83
- const prompt = [
84
- knownProjectsLine(db),
85
- memories.length > 0 ? `What is already remembered:\n${memories.map(line).join('\n')}` : 'Nothing is remembered yet.',
86
- ...sections,
87
- ].join('\n');
88
- // What it was shown is what it may name as saying the same thing.
89
- return { prompt, turns: new Map(turns.map((t) => [t.id, t])), shown: new Set(memories.map((m) => m.id)), cursor };
90
- }
50
+ /** What waits to be judged from here on: those named, less any judged now. */
51
+ const keepNamed = (db, named) => setMeta(db, NAMED_PAIRS, JSON.stringify(named));
91
52
 
92
53
  /**
93
- * What a pass is shown of memory, at most MAX_MEMORIES: the memories most related to what was said, a slice of the
94
- * oldest that moves on with every pass — from where the last one stopped, round to the start again — and the newest
95
- * for the rest. All of them when there are no more than that. Returns them by id, with where the slice stopped.
54
+ * What a model reading a list of rules by number says of two of them waits until the pair is judged on its own: it
55
+ * writes one number for another — a neighbour's for the one it meant — and a pair it never looked at side by side
56
+ * would be a question for the user. Asked once: a pair raised, judged, or waiting already is not named again.
96
57
  */
97
- function shownMemories(db, scopes, texts) {
98
- const pool = db.prepare(`SELECT * FROM memories WHERE state = 'active' AND scan_key IS NULL AND scope IN (${scopes.map(() => '?').join(', ')}) ORDER BY id`).all(...scopes);
99
- if (pool.length <= MAX_MEMORIES) return { memories: pool, cursor: null };
100
- const chosen = new Map(relatedMemories(db, texts, scopes).map((m) => [m.id, m]));
101
- const from = Number(getMeta(db, SWEEP_CURSOR) ?? 0);
102
- const slice = [...pool.filter((m) => m.id > from), ...pool.filter((m) => m.id <= from)].filter((m) => !chosen.has(m.id)).slice(0, SWEEP_SLICE);
103
- for (const m of slice) chosen.set(m.id, m);
104
- for (const m of [...pool].reverse()) {
105
- if (chosen.size >= MAX_MEMORIES) break;
106
- chosen.set(m.id, m);
107
- }
108
- return { memories: [...chosen.values()].sort((a, b) => a.id - b.id), cursor: slice.at(-1)?.id ?? null };
58
+ function nameForJudging(db, { label, a, b, note }) {
59
+ const [lo, hi] = a < b ? [a, b] : [b, a];
60
+ if (db.prepare('SELECT 1 FROM notices WHERE memory_a = ? AND memory_b = ?').get(lo, hi)) return;
61
+ if (db.prepare('SELECT 1 FROM memory_pairs WHERE a = ? AND b = ?').get(lo, hi)) return;
62
+ const waiting = namedPairs(db);
63
+ if (waiting.some((p) => p.a === lo && p.b === hi)) return;
64
+ keepNamed(db, [...waiting, { label, a: lo, b: hi, note }]);
109
65
  }
110
66
 
111
67
  /** The memories the tidy steps read: active, the user's own rather than scanned from a repository, and not workflows. */
@@ -190,7 +146,7 @@ const KEYS_SCHEMA = {
190
146
  /** Whether the tidy steps may run: they call the local model only, never one that is paid for. */
191
147
  const tidiesLocally = (db) => (getMeta(db, 'config.memory.model') ?? CONFIG_DEFAULTS['memory.model']) === 'local';
192
148
 
193
- /** One call of the tidy steps to the local model, with the consolidation pass's own instructions, and its ledger row. */
149
+ /** One call of the tidy steps to the local model, with the tidy's own instructions, and its ledger row. */
194
150
  async function askLocal(db, { prompt, schema, kind, now }) {
195
151
  const system = readFileSync(join(REPO_ROOT, 'prompts', 'dream.md'), 'utf8').trim();
196
152
  const result = await callModel(db, { system, prompt, schema, model: 'local', kind });
@@ -216,16 +172,15 @@ function namedToJudge(db, limit = PAIRS_PER_CALL) {
216
172
  /**
217
173
  * Pairs judged, and what the judgement means raised for the user: the same thing said twice, one that may replace
218
174
  * the other, or two that cannot both be true — the look-alike and contradiction notices the block already asks about.
219
- * Nothing is superseded or forgotten here; the user settles it. Every pair answered is kept, so it is asked once. A
220
- * model reading a list names pairs too — a rule said twice, a clash — and those are judged first, side by side like
221
- * the rest: raised as the judgement says, the note it was named with kept when it is judged the way it was named.
222
- * `fill` false judges only those.
175
+ * Nothing is superseded or forgotten here; the user settles it. Every pair answered is kept, so it is asked once. The
176
+ * repeat check names pairs too — a rule said twice — and those are judged first, side by side like the rest: raised
177
+ * as the judgement says, the note it was named with kept when it is judged the way it was named.
223
178
  */
224
- async function judgePairs(db, { calls, now, fill = true }) {
179
+ async function judgePairs(db, { calls, now }) {
225
180
  const out = { applied: [], dropped: [], usage: NO_USAGE, calls: 0, error: null };
226
181
  for (let call = 0; call < calls; call++) {
227
182
  const named = namedToJudge(db);
228
- const pairs = [...named, ...(fill && named.length < PAIRS_PER_CALL ? candidatePairs(db, PAIRS_PER_CALL - named.length) : [])];
183
+ const pairs = [...named, ...(named.length < PAIRS_PER_CALL ? candidatePairs(db, PAIRS_PER_CALL - named.length) : [])];
229
184
  if (pairs.length === 0) break;
230
185
  const prompt = ['Pairs to judge — how memory A relates to memory B in each:', ...pairs.flatMap(({ a, b }, i) => [`[p${i + 1}]`, ` A: ${describe(a)}`, ` B: ${describe(b)}`])].join('\n');
231
186
  const { result, log } = await askLocal(db, { prompt, schema: PAIRS_SCHEMA, kind: 'dream-pairs', now });
@@ -278,6 +233,39 @@ async function judgePairs(db, { calls, now, fill = true }) {
278
233
  return out;
279
234
  }
280
235
 
236
+ /**
237
+ * A statement about to be filed that looks like a memory already there — "slate uses pnpm", then "slate uses bun":
238
+ * the fact may have changed. Code picked each pair (`pairs`, as `applyOps` gives them); the local model says what it
239
+ * is, with the same four labels the tidy uses. Returns `labels`, a Map from each statement's place in the answer it
240
+ * came in to `{ old, label }` — only for pairs asked and labels of the four — or undefined when the model could not
241
+ * be asked or did not answer in shape: the pairs then go to the user as notices, as they always did.
242
+ */
243
+ export async function judgeNewPairs(db, pairs, now) {
244
+ const today = now.slice(0, 10);
245
+ const prompt = [
246
+ // Each with the day it was said: a rebuild reads old turns, and "just said" made an old statement the newer one.
247
+ 'Pairs to judge — how memory A relates to memory B in each (B is a statement of the user\'s, not yet filed; each has the day it was said):',
248
+ ...pairs.flatMap(({ old, type, scope, body, said }, i) => [`[p${i + 1}]`, ` A: ${describe(old)}`, ` B: new (${type}, ${scope}, ${(said ?? today).slice(0, 10)}): ${clip(body, 300)}`]),
249
+ ].join('\n');
250
+ const { result, log } = await askLocal(db, { prompt, schema: PAIRS_SCHEMA, kind: 'dream-pairs', now });
251
+ if (!result.ok) {
252
+ log(result.error);
253
+ return { labels: undefined, usage: result.usage };
254
+ }
255
+ const answers = Array.isArray(result.data?.pairs) ? result.data.pairs : null;
256
+ if (!answers) {
257
+ log('the answer had no list of pairs');
258
+ return { labels: undefined, usage: result.usage };
259
+ }
260
+ const labels = new Map();
261
+ for (const answer of answers) {
262
+ const pair = pairs[Number(answer?.pair) - 1];
263
+ if (pair && LABELS.includes(answer?.label)) labels.set(pair.index, { old: pair.old.id, label: answer.label });
264
+ }
265
+ log(`judged ${labels.size} of ${pairs.length} new statements against what they look like`);
266
+ return { labels, usage: result.usage };
267
+ }
268
+
281
269
  const REPEATS_SCHEMA = {
282
270
  type: 'object',
283
271
  properties: {
@@ -311,11 +299,13 @@ async function checkRepeats(db, { now }) {
311
299
  return out;
312
300
  }
313
301
  const prompt = [
314
- 'Rules said twice — for each rule checked, the rule in the list that says the same thing in other words, if one does:',
315
- 'Rules checked:',
316
- ...batch.map((m) => `[m${m.id}] ${clip(m.body, 300)}`),
317
- 'All rules:',
318
- ...[...shown.values()].sort((x, y) => x.id - y.id).map(describe),
302
+ // One list, said plainly to be the user's words as typed. Shown the rules twice — those checked, then all — the
303
+ // local model named "I prefer short answers, everywhere" and "keep your replies brief, always, in every project"
304
+ // or missed them according to what kind a third rule was; asked this way it named them every time, and two other
305
+ // pairs, and nothing among rules that differ (tried on the local model, Nov 18 2026). Code keeps only an answer
306
+ // about a rule being checked.
307
+ "Rules said twice — these are the user's own words, as they typed them. Name every two that ask for the same thing in other words (rule: the later one, same_as: the earlier):",
308
+ ...[...shown.values()].sort((x, y) => x.id - y.id).map((m) => `m${m.id} (${m.type}, ${m.scope}): ${clip(m.body, 300)}`),
319
309
  ].join('\n');
320
310
  const { result, log } = await askLocal(db, { prompt, schema: REPEATS_SCHEMA, kind: 'dream-repeats', now });
321
311
  out.calls = 1;
@@ -336,8 +326,9 @@ async function checkRepeats(db, { now }) {
336
326
  const checked = new Set(batch.map((m) => m.id));
337
327
  tx(db, () => {
338
328
  for (const answer of answers) {
339
- const a = shown.get(Number(answer?.rule));
340
- const b = shown.get(Number(answer?.same_as));
329
+ // Either way round: the pair is the answer, and one of the two must be a rule this call checks.
330
+ const named = [shown.get(Number(answer?.rule)), shown.get(Number(answer?.same_as))];
331
+ const [a, b] = named[0] && !checked.has(named[0].id) ? [named[1], named[0]] : named;
341
332
  if (!a || !b || a.id === b.id || !checked.has(a.id)) {
342
333
  out.dropped.push(`repeat m${JSON.stringify(answer?.rule)} of m${JSON.stringify(answer?.same_as)}: not a rule checked and another rule shown`);
343
334
  continue;
@@ -405,119 +396,36 @@ async function fillKeys(db, { now }) {
405
396
  }
406
397
 
407
398
  /**
408
- * Whether the weekly tidy is owed: a week since the last one, with something to tidy. Never tidied, the week runs from
409
- * the oldest memory — memory a day old has nothing to tidy yet.
399
+ * Whether the weekly tidy is due: a week since the last one, with something to tidy. Never tidied, the week runs from
400
+ * the oldest memory — memory a day old has nothing to tidy yet. Pure: it reads, never runs.
410
401
  */
411
- function tidyOwed(db, now) {
402
+ export function tidyDue(db, now = new Date().toISOString()) {
412
403
  const since = getMeta(db, LAST_TIDY) ?? db.prepare(`SELECT MIN(created_at) AS at FROM memories WHERE state = 'active' AND scan_key IS NULL`).get()?.at;
413
404
  if (!since || Date.parse(now) - Date.parse(since) < TIDY_EVERY_MS) return false;
414
405
  return Boolean(db.prepare(`SELECT 1 FROM memories WHERE state = 'active' AND scan_key IS NULL LIMIT 1`).get());
415
406
  }
416
407
 
417
408
  /**
418
- * Whether a consolidation pass is worth running by itself now: three finished sessions are waiting to be read side by
419
- * side, or a week has gone since memory was last tidied and there is memory to tidy. Pure: it reads, never runs.
409
+ * The tidy: search terms filled in; rules checked for being said twice; and pairs judged — those the check named
410
+ * first. Run when the week is up, or now when asked (`sumo memory run --tidy`); the caller decides which, and holds
411
+ * the lock. It calls the local model only, never one that is paid for: set to another model, there is nothing it does.
412
+ * A tidy the local model could not answer stays due.
420
413
  */
421
- export function tidyDue(db, now = new Date().toISOString()) {
422
- return sessionsAwaitingDream(db, now, SESSIONS_TO_TIDY).length >= SESSIONS_TO_TIDY || tidyOwed(db, now);
423
- }
424
-
425
- /**
426
- * The tidy steps of a pass: weekly, search terms filled in; rules checked for being said twice; and pairs judged — those
427
- * the check named first — in one call, or a few in the weekly tidy.
428
- */
429
- async function tidyMemory(db, { weekly, now }) {
430
- const steps = [];
431
- if (weekly) steps.push(await fillKeys(db, { now }));
432
- steps.push(await checkRepeats(db, { now }));
433
- steps.push(await judgePairs(db, { calls: weekly ? TIDY_JUDGE_CALLS : JUDGE_CALLS, now }));
434
- return {
414
+ export async function runDream(db, { now = new Date().toISOString() } = {}) {
415
+ if (!tidiesLocally(db)) {
416
+ setMeta(db, LAST_TIDY, now);
417
+ return { skipped: 'memory is tidied by the local model only, and memory.model is not local' };
418
+ }
419
+ const steps = [await fillKeys(db, { now }), await checkRepeats(db, { now }), await judgePairs(db, { calls: TIDY_JUDGE_CALLS, now })];
420
+ const tidy = {
435
421
  applied: steps.flatMap((s) => s.applied),
436
422
  dropped: steps.flatMap((s) => s.dropped),
437
423
  usage: steps.reduce((u, s) => added(u, s.usage), NO_USAGE),
438
424
  calls: steps.reduce((n, s) => n + s.calls, 0),
439
425
  error: steps.find((s) => s.error)?.error ?? null,
440
426
  };
441
- }
442
-
443
- /**
444
- * The pairs the reading of the conversations named — a clash, one thing said twice — judged on their own before anything
445
- * is put to the user. With no local model to judge them (the tidy steps never call a paid one), they are raised as named.
446
- */
447
- async function settleNamed(db, { now }) {
448
- if (namedPairs(db).length === 0) return { applied: [], dropped: [], usage: NO_USAGE };
449
- if (tidiesLocally(db)) {
450
- const judged = await judgePairs(db, { calls: 1, now, fill: false });
451
- // Not answered, they wait for the next tidy, which judges them first.
452
- return judged.error ? { ...judged, dropped: [...judged.dropped, `the pairs named wait to be judged: ${judged.error}`] } : judged;
453
- }
454
- const applied = [];
455
- tx(db, () => {
456
- for (const { a, b, named } of namedToJudge(db, Infinity)) {
457
- const [kind, said] = named.label === 'same' ? ['lookalike', 'say the same thing'] : ['contradiction', 'cannot both be true'];
458
- const note = named.note ?? `they ${said}`;
459
- if (raiseNotice(db, { kind, a: a.id, b: b.id, note, now })) applied.push(`raised for the user: m${a.id} and m${b.id} ${said} — ${note}`);
460
- }
461
- keepNamed(db, []);
462
- });
463
- return { applied, dropped: [], usage: NO_USAGE };
464
- }
465
-
466
- /**
467
- * The consolidation pass, which is also memory's tidy: the conversations waiting are read side by side, and memory
468
- * itself is gone over — pairs judged, and weekly, search terms filled in. `sweep` asks for the weekly part now; it is
469
- * done anyway once a week has gone. With no conversation waiting and no tidy owed, there is nothing to do.
470
- */
471
- export async function runDream(db, { now = new Date().toISOString(), force = false, sweep = false, file = true } = {}) {
472
- // Anything still unfiled belongs to the writer first; consolidation reads its results. The memory pass files first
473
- // itself, so it can say what the filing did, and asks for none here.
474
- if (file) await runScribe(db, { now });
475
-
476
- return withLock('dream', async () => {
477
- // A conversation the writer has not finished filing (it is busy, failing, or behind) waits for it.
478
- const unfiled = new Set(writerOff(db) ? [] : db.prepare('SELECT DISTINCT session_id FROM user_turns WHERE scribed = 0').all().map((r) => r.session_id));
479
- const over = sessionsAwaitingDream(db, now, MAX_SESSIONS).filter((s) => !unfiled.has(s.id));
480
- // Forced, a pass also reads conversations still going on, after the finished ones. It never closes them:
481
- // what is said in the rest of one is read once it is over.
482
- const open = force
483
- ? sessionsAwaitingDream(db, now, MAX_SESSIONS, { includeOpen: true }).filter((s) => !unfiled.has(s.id) && !over.some((o) => o.id === s.id))
484
- : [];
485
- const sessions = [...over, ...open].slice(0, MAX_SESSIONS).sort((a, b) => a.started_at.localeCompare(b.started_at));
486
- const weekly = sweep || tidyOwed(db, now);
487
- if (sessions.length === 0 && !weekly) return { skipped: 'no finished sessions are waiting' };
488
-
489
- // Memory is tidied first, so what the conversations bring is read against it as it now stands.
490
- const tidy = tidiesLocally(db) ? await tidyMemory(db, { weekly, now }) : { applied: [], dropped: [], usage: NO_USAGE, calls: 0, error: null };
491
- // A tidy the local model could not answer is owed still; one that had nothing to ask is done.
492
- if (weekly && !tidy.error) setMeta(db, LAST_TIDY, now);
493
- if (sessions.length === 0) {
494
- if (tidy.calls === 0) return { skipped: 'no finished sessions are waiting, and memory has nothing to tidy' };
495
- if (tidy.error && tidy.applied.length === 0) return { ok: false, error: tidy.error, applied: [], dropped: tidy.dropped, usage: tidy.usage };
496
- return { ok: true, error: null, applied: tidy.applied, dropped: tidy.dropped, usage: tidy.usage, sessions: 0 };
497
- }
498
-
499
- const bundle = buildBundle(db, sessions);
500
- let lastTurn = 0;
501
- for (const id of bundle.turns.keys()) lastTurn = id;
502
- const outcome = await askAndApply(db, {
503
- kind: 'dream', promptFile: 'dream.md', bundle,
504
- ops: ['add', 'supersede', 'gotcha', 'checkpoint', 'contradiction', 'same', 'procedure'], sessionId: null, now,
505
- });
506
- if (outcome.ok && over.length > 0) {
507
- // A resumed session may have received a new turn while the model read the snapshot.
508
- db.prepare(`UPDATE sessions SET dream_state = 'done' WHERE id IN (${over.map(() => '?').join(', ')})
509
- AND NOT EXISTS (SELECT 1 FROM user_turns t WHERE t.session_id = sessions.id AND t.id > ?)`)
510
- .run(...over.map((s) => s.id), lastTurn);
511
- }
512
- // The slice of the oldest moves on only past a pass that read it.
513
- if (outcome.ok && bundle.cursor !== null) setMeta(db, SWEEP_CURSOR, bundle.cursor);
514
- const named = outcome.ok ? await settleNamed(db, { now }) : { applied: [], dropped: [], usage: NO_USAGE };
515
- return {
516
- ...outcome,
517
- applied: [...tidy.applied, ...outcome.applied, ...named.applied],
518
- dropped: [...tidy.dropped, ...outcome.dropped, ...named.dropped],
519
- usage: added(added(tidy.usage, outcome.usage ?? NO_USAGE), named.usage),
520
- sessions: sessions.length,
521
- };
522
- });
427
+ if (!tidy.error) setMeta(db, LAST_TIDY, now);
428
+ if (tidy.calls === 0) return { skipped: 'memory has nothing to tidy' };
429
+ if (tidy.error && tidy.applied.length === 0) return { ok: false, error: tidy.error, applied: [], dropped: tidy.dropped, usage: tidy.usage };
430
+ return { ok: true, error: null, applied: tidy.applied, dropped: tidy.dropped, usage: tidy.usage };
523
431
  }