sumo-agents 0.5.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/export.mjs CHANGED
@@ -1,7 +1,7 @@
1
1
  import { chmodSync, copyFileSync, existsSync, mkdirSync, readdirSync, renameSync, rmSync, statSync } from 'node:fs';
2
2
  import { basename, dirname, join, resolve } from 'node:path';
3
3
  import { getMeta, setMeta } from './db.mjs';
4
- import { UsageError } from './memory.mjs';
4
+ import { eraseRow, eraseSaid, UsageError } from './memory.mjs';
5
5
  import { paths } from './paths.mjs';
6
6
  import { DatabaseSync } from './sqlite.mjs';
7
7
 
@@ -185,14 +185,14 @@ export function restoreLines(copy, live, now = new Date()) {
185
185
  }
186
186
 
187
187
  /**
188
- * A purge reaches the backups too: "erased, not recoverable" must hold for every copy this machine keeps. Each backup
189
- * loses the memory and the sentence it came from, and is rewritten so the pages that held them are gone.
190
- * Returns how many were cleaned, and the ones that could not be — said, never hidden.
188
+ * Every copy this machine keeps of the database — the weekly backups, and what was set aside beside it before a
189
+ * migration or by a restore — put through `erase(db, has)`, which says how much it took out: a copy it changed is
190
+ * rewritten whole, indexes and pages, so nothing of what went is left in it. All of it or none: a copy that cannot be
191
+ * cleaned whole is left as it was, and named. `{ cleaned, failed }`.
191
192
  */
192
- export function purgeFromBackups({ id, body, created_at: createdAt, source_turn: sourceTurn }) {
193
+ function cleanBackups(erase) {
193
194
  const { backups, db: live } = paths();
194
195
  const within = (dir, keep) => (existsSync(dir) ? readdirSync(dir).filter(keep).map((f) => join(dir, f)) : []);
195
- // The weekly copies, and every copy set aside beside the database: before a migration, or by a restore.
196
196
  const aside = (f) => f.startsWith(`${basename(live)}.`) && (/\.pre-v\d+\.bak$/.test(f) || /\.(?:damaged|replaced)-[^/]*$/.test(f)) && !/-(?:wal|shm)$/.test(f);
197
197
  const files = [...within(backups, (f) => /^memory-.*\.db$/.test(f)), ...within(dirname(live), aside)];
198
198
  const failed = [];
@@ -202,21 +202,23 @@ export function purgeFromBackups({ id, body, created_at: createdAt, source_turn:
202
202
  try {
203
203
  db = new DatabaseSync(file);
204
204
  db.exec('PRAGMA secure_delete = ON;');
205
- // The same memory, not just the same number: a backup from after a restore may hold another one under this id.
206
- const row = db.prepare('SELECT source_turn FROM memories WHERE id = ? AND body = ? AND created_at = ?').get(id, body, createdAt);
207
- if (row) {
208
- // All of it or none: a backup this cannot clean whole is left as it was, and named.
209
- db.exec('BEGIN');
210
- try {
211
- db.prepare('DELETE FROM memories WHERE id = ?').run(id);
212
- if (sourceTurn !== null) db.prepare('DELETE FROM user_turns WHERE id = ?').run(sourceTurn);
205
+ const has = (name) => db.prepare('SELECT 1 FROM sqlite_master WHERE name = ?').get(name) !== undefined;
206
+ let changed = 0;
207
+ db.exec('BEGIN');
208
+ try {
209
+ changed = erase(db, has);
210
+ if (changed > 0) {
213
211
  // A deleted row's words stay in the full-text index's pages until it is rebuilt; VACUUM keeps those pages.
214
212
  db.exec(`INSERT INTO memories_fts(memories_fts) VALUES ('rebuild'); INSERT INTO user_turns_fts(user_turns_fts) VALUES ('rebuild');`);
215
- db.exec('COMMIT');
216
- } catch (cause) {
217
- db.exec('ROLLBACK');
218
- throw cause;
213
+ // A copy from before replies were kept has none to clean.
214
+ if (has('replies_fts')) db.exec(`INSERT INTO replies_fts(replies_fts) VALUES ('rebuild');`);
219
215
  }
216
+ db.exec('COMMIT');
217
+ } catch (cause) {
218
+ db.exec('ROLLBACK');
219
+ throw cause;
220
+ }
221
+ if (changed > 0) {
220
222
  db.exec('VACUUM');
221
223
  cleaned++;
222
224
  }
@@ -228,3 +230,26 @@ export function purgeFromBackups({ id, body, created_at: createdAt, source_turn:
228
230
  }
229
231
  return { cleaned, failed };
230
232
  }
233
+
234
+ /**
235
+ * A purge reaches the backups too: "erased, not recoverable" must hold for every copy this machine keeps. Each backup
236
+ * loses the memory, every turn and reply that says it whole and what its passes proposed — by the one rule the
237
+ * database itself is held to (`eraseSaid` in memory.mjs). A copy from before the memory was filed may hold the
238
+ * sentence without the memory: it is cleaned all the same. Returns how many were cleaned, and the ones that could
239
+ * not be — said, never hidden.
240
+ */
241
+ export function purgeFromBackups({ id, body, created_at: createdAt, source_turn: sourceTurn, source_session: sourceSession = null, source_quote: sourceQuote = null }) {
242
+ return cleanBackups((db, has) => {
243
+ // The same memory, not just the same number: a backup from after a restore may hold another one under this id.
244
+ const row = db.prepare('SELECT source_turn FROM memories WHERE id = ? AND body = ? AND created_at = ?').get(id, body, createdAt);
245
+ // The passes that filed it, while its history still names them; a copy from before histories were kept has none.
246
+ const runs = row && has('memory_events') ? db.prepare('SELECT run_id FROM memory_events WHERE memory_id = ? AND run_id IS NOT NULL').all(id).map((e) => e.run_id) : [];
247
+ if (row) db.prepare('DELETE FROM memories WHERE id = ?').run(id);
248
+ // The turn it cites goes by its number only where the memory is this one: in another copy that number may be another turn.
249
+ const erased = eraseSaid(db, { body, source_quote: sourceQuote, source_turn: row ? sourceTurn : null, source_session: sourceSession }, { runs });
250
+ return (row ? 1 : 0) + erased.turns.length + erased.replies.length + erased.ops;
251
+ });
252
+ }
253
+
254
+ /** One turn or reply purged (`sumo forget --purge t12`) goes from every copy that holds it, the same way. */
255
+ export const purgeRowFromBackups = (entry) => cleanBackups((db) => (eraseRow(db, entry) ? 1 : 0));
package/src/herdr.mjs CHANGED
@@ -78,9 +78,7 @@ const screen = (herdr, pane) => {
78
78
  */
79
79
  export async function startWatcher(herdr, { pane, job, command, sleep = pause, tries = TRIES }) {
80
80
  const watching = new RegExp(`job watch ${job}(?!\\d)`);
81
- // The record's first line: the job named in the ledger's gutter, then who works on it.
82
- const shown = new RegExp(`(?:^|\\n) *j${job} {2,}\\S`);
83
- const started = (info) => info.foreground_processes?.some((p) => watching.test(String(p.cmdline))) || shown.test(screen(herdr, pane) ?? '');
81
+ const started = (info) => info.foreground_processes?.some((p) => watching.test(String(p.cmdline))) || screen(herdr, pane)?.includes(`⏺ j${job} `);
84
82
  for (let attempt = 0; attempt < tries; attempt++) {
85
83
  let last = null;
86
84
  for (let waited = 0; waited < READY_MS; waited += POLL_MS) {
package/src/hooks.mjs CHANGED
@@ -1,5 +1,5 @@
1
1
  import { card } from './card.mjs';
2
- import { getMeta, openDb, setMeta } from './db.mjs';
2
+ import { getMeta, openDb, setMeta, tx } from './db.mjs';
3
3
  import { backup } from './export.mjs';
4
4
  import { guardCommand, guardPath } from './guard.mjs';
5
5
  import { autoGuides, firingGuides, matchGuide, renderGuide } from './guides.mjs';
@@ -11,7 +11,8 @@ import { closingQuestions, recallBeforeAsking } from './recall.mjs';
11
11
  import { line } from './render.mjs';
12
12
  import { shouldRun, spawnDetached } from './scribe.mjs';
13
13
  import { tidyDue } from './dream.mjs';
14
- import { clearInjections, contextBand, contextNudge, contextUse, currentProject, endSession, ensureSession, injectedSlugs, markInjected, pruneStates, recordTurn } from './sessions.mjs';
14
+ import { clearInjections, contextBand, contextNudge, contextUse, currentProject, endedThen, endSession, ensureSession, heardSession, injectedSlugs, markInjected, noteHeard, pruneStates, recordReply, recordTurn, replyText, turnText } from './sessions.mjs';
15
+ import { drainSpool, retrySetAside, spoolEvent } from './spool.mjs';
15
16
  import { claim, renderWorkflow, withoutSumoCalls } from './workflows.mjs';
16
17
 
17
18
  /**
@@ -118,14 +119,21 @@ function unchecked(db, e) {
118
119
  return `<check>\nNot done yet: ${shown} ${were} edited and ${next}\n</check>`;
119
120
  }
120
121
 
122
+ /** The memory pass as a session starts it by itself: finished conversations only, and nothing filed while filing is paused. */
123
+ const PASS = ['memory', 'run', '--auto'];
124
+
125
+ /** The ways a session starts with its context wiped: Claude Code's compact and clear, the chat's /new. */
126
+ const CARDS_GONE = new Set(['compact', 'clear', 'new']);
127
+
121
128
  const HANDLERS = {
122
129
  'session-start'(db, e, now) {
123
- ensureSession(db, { id: e.sessionId, cwd: e.cwd, transcriptPath: e.transcriptPath, now });
124
- // A fresh start wipes the cards from context; forgetting they were shown lets them come back.
125
- if (e.source === 'compact' || e.source === 'new') clearInjections(db, e.sessionId);
130
+ ensureSession(db, { id: e.sessionId, harness: e.harness, cwd: e.cwd, transcriptPath: e.transcriptPath, now });
131
+ // A fresh start wipes the cards from context; forgetting they were shown lets them come back. Claude Code names its
132
+ // start startup, resume, clear or compact: a resume still holds the cards, a clear or a compaction does not.
133
+ if (CARDS_GONE.has(e.source)) clearInjections(db, e.sessionId);
126
134
 
127
- // One memory pass: it files what waits, and tidies too when conversations have finished or a week has gone by.
128
- if (shouldRun(db, { event: 'session-start', now }) || tidyDue(db, now)) spawnDetached(['memory', 'run']);
135
+ // One memory pass: it reads the conversations that have finished since the last one, and tidies when a week has gone by.
136
+ if (shouldRun(db, { now }) || tidyDue(db, now)) spawnDetached(PASS);
129
137
 
130
138
  const last = getMeta(db, 'last_backup');
131
139
  if (!last || Date.parse(now) - Date.parse(last) > BACKUP_EVERY_MS) {
@@ -161,8 +169,12 @@ const HANDLERS = {
161
169
  },
162
170
 
163
171
  prompt(db, e, now) {
164
- ensureSession(db, { id: e.sessionId, cwd: e.cwd, transcriptPath: e.transcriptPath, now });
165
- if (recordTurn(db, { sessionId: e.sessionId, text: e.prompt, now }) === null) return '';
172
+ ensureSession(db, { id: e.sessionId, harness: e.harness, cwd: e.cwd, transcriptPath: e.transcriptPath, now });
173
+ const turn = recordTurn(db, { sessionId: e.sessionId, text: e.prompt, now });
174
+ if (turn === null) {
175
+ noteHeard(db, e.harness, now);
176
+ return '';
177
+ }
166
178
 
167
179
  const shown = new Set(injectedSlugs(db, e.sessionId));
168
180
  const cards = [];
@@ -183,6 +195,8 @@ const HANDLERS = {
183
195
  // Its questions come with it, unless the block this session started with asked them: a session started inside the project.
184
196
  cards.push(card(db, slug, { now, asks: slug !== (e.cwd ? projectAt(db, e.cwd) : null) }));
185
197
  }
198
+ // The turn was stamped before its own words were read: one that names a project is about that project.
199
+ if (here || named.length > 0) db.prepare('UPDATE user_turns SET project = ? WHERE id = ?').run(currentProject(db, e.sessionId), turn);
186
200
 
187
201
  const asked = claim(db, { sessionId: e.sessionId, text: e.prompt, scopes: scopesOf(db, e.sessionId), now });
188
202
  if (asked.length > 0) {
@@ -204,7 +218,7 @@ const HANDLERS = {
204
218
  'pre-tool'(db, e, now) {
205
219
  // A question on its way to the user: what memory holds close to it is read first, and holds it back once.
206
220
  if (e.toolName === 'ask') {
207
- ensureSession(db, { id: e.sessionId, cwd: e.cwd, transcriptPath: e.transcriptPath, now });
221
+ ensureSession(db, { id: e.sessionId, harness: e.harness, cwd: e.cwd, transcriptPath: e.transcriptPath, now });
208
222
  const known = recallBeforeAsking(db, { sessionId: e.sessionId, agentId: e.agentId, asked: e.question, now });
209
223
  return known.length > 0 ? deny(`Not asked yet: ${alreadyKnown(known)} If none does, ask again.`) : '';
210
224
  }
@@ -212,7 +226,7 @@ const HANDLERS = {
212
226
  // Teaching or reading a workflow names its own cue; `sumo` must never be gated by what it manages — and nothing else rides through on it.
213
227
  const judged = withoutSumoCalls(e.command);
214
228
  if (!judged) return '';
215
- ensureSession(db, { id: e.sessionId, cwd: e.cwd, transcriptPath: e.transcriptPath, now });
229
+ ensureSession(db, { id: e.sessionId, harness: e.harness, cwd: e.cwd, transcriptPath: e.transcriptPath, now });
216
230
 
217
231
  const due = claim(db, { sessionId: e.sessionId, agentId: e.agentId, text: judged, scopes: [...scopesOf(db, e.sessionId), ...(e.project ? [`project:${e.project}`] : [])], forCommand: true, now });
218
232
  if (due.length === 0) return '';
@@ -223,8 +237,9 @@ const HANDLERS = {
223
237
  },
224
238
 
225
239
  stop(db, e, now) {
226
- ensureSession(db, { id: e.sessionId, cwd: e.cwd, transcriptPath: e.transcriptPath, now });
227
- if (shouldRun(db, { event: 'stop', now })) spawnDetached(['memory', 'run']);
240
+ ensureSession(db, { id: e.sessionId, harness: e.harness, cwd: e.cwd, transcriptPath: e.transcriptPath, now });
241
+ // The journal first: what the assistant ended the turn on, whether or not a gate then sends the turn on.
242
+ if (e.reply) recordReply(db, { sessionId: e.sessionId, text: e.reply, now });
228
243
 
229
244
  // The other gates: a turn that ends on an edit nothing checked, or by asking the user something memory may
230
245
  // already hold. Already continuing because of one → whatever the turn ends on now goes to the user.
@@ -235,24 +250,89 @@ const HANDLERS = {
235
250
  },
236
251
 
237
252
  'session-end'(db, e, now) {
238
- ensureSession(db, { id: e.sessionId, cwd: e.cwd, transcriptPath: e.transcriptPath, now });
253
+ ensureSession(db, { id: e.sessionId, harness: e.harness, cwd: e.cwd, transcriptPath: e.transcriptPath, now });
239
254
  endSession(db, e.sessionId, now);
240
- if (shouldRun(db, { event: 'session-end', now })) spawnDetached(['memory', 'run']);
255
+ // The conversation is over: it is read for memory now, once.
256
+ if (shouldRun(db, { now })) spawnDetached(PASS);
241
257
  return '';
242
258
  },
243
259
  };
244
260
 
245
- /** One event, in process: the guard first, then the handler. Throws only for an unknown event or a payload with no session. */
246
- export function handleEvent(db, event, payload, now = new Date().toISOString()) {
261
+ /**
262
+ * One event, in process: the guard first, then the handler. Throws only for an unknown event or a payload with no
263
+ * session. `harness` is who drives the session — sumo's own chat and jobs unless said; `runHook` says 'claude' — and
264
+ * is written on a session's row when it is first heard of. Events an earlier hook could not write are replayed first.
265
+ */
266
+ export function handleEvent(db, event, payload, now = new Date().toISOString(), { harness = 'sumo' } = {}) {
247
267
  const handler = HANDLERS[event];
248
268
  if (!handler) throw new Error(`no handler for ${event}`);
249
- const e = fieldsOf(payload);
269
+ const e = { ...fieldsOf(payload), harness };
250
270
  if (!e.sessionId) throw new Error('the event had no session id');
251
271
  const refused = event === 'pre-tool' ? earlyRefusal(e) : '';
252
272
  if (refused) return deny(refused);
273
+ drain(db);
253
274
  return handler(db, e, now);
254
275
  }
255
276
 
277
+ /** The events with something for the journal: a session's row, a turn, a reply, a goodbye. A tool about to run has nothing. */
278
+ const JOURNALED = new Set(['session-start', 'prompt', 'stop', 'session-end']);
279
+ /** The harness `sumo hook` is wired into. */
280
+ const HOOKED = 'claude';
281
+
282
+ /**
283
+ * An event that could not be written, kept in the spool for the next one that gets through: only what the journal
284
+ * needs — who, when, where, and the words, redacted and cut as the journal would keep them. Never the tool's input, the
285
+ * gates or the cards: those were for the moment that has passed. A slash command or a turn that ended on nothing has
286
+ * nothing to keep.
287
+ */
288
+ function keepForLater(event, payload, at) {
289
+ const e = fieldsOf(payload);
290
+ if (!JOURNALED.has(event) || !e.sessionId) return;
291
+ const said = event === 'prompt' ? turnText(e.prompt) : event === 'stop' ? replyText(e.reply) : null;
292
+ if ((event === 'prompt' || event === 'stop') && !said) return;
293
+ const text = (value) => (typeof value === 'string' ? value : null);
294
+ spoolEvent({ v: 1, at, event, harness: HOOKED, session_id: String(e.sessionId), cwd: text(e.cwd), transcript_path: text(e.transcriptPath), ...(said ? { text: said.text, full_len: said.fullLen, redacted: said.redacted } : {}) });
295
+ }
296
+
297
+ /** A line of the spool that is not an event: no later try would make it one, so it is set aside at once. */
298
+ function notAnEvent(why) {
299
+ return Object.assign(new Error(why), { bad: true });
300
+ }
301
+
302
+ /**
303
+ * One spooled event written into the journal with the time it happened: the session's row, and the turn, the reply or
304
+ * the goodbye. Nothing else an event does is done again — no card, no gate, no memory pass woken, no backup.
305
+ */
306
+ function replay(db, line) {
307
+ if (line.v !== 1 || !JOURNALED.has(line.event)) throw notAnEvent('not an event this version replays');
308
+ if (typeof line.session_id !== 'string' || !line.session_id) throw notAnEvent('no session id');
309
+ const happened = typeof line.at === 'string' ? Date.parse(line.at) : NaN;
310
+ if (Number.isNaN(happened)) throw notAnEvent('no time it happened');
311
+ const at = new Date(happened).toISOString();
312
+ const text = (value) => (typeof value === 'string' ? value : null);
313
+ const kept = { text: line.text, fullLen: line.full_len, redacted: line.redacted };
314
+ if ((line.event === 'prompt' || line.event === 'stop') && typeof line.text !== 'string') throw notAnEvent('no words');
315
+ heardSession(db, { id: line.session_id, harness: text(line.harness) ?? HOOKED, cwd: text(line.cwd), transcriptPath: text(line.transcript_path), now: at });
316
+ if (line.event === 'prompt') recordTurn(db, { sessionId: line.session_id, now: at, kept });
317
+ if (line.event === 'stop') recordReply(db, { sessionId: line.session_id, now: at, kept });
318
+ if (line.event === 'session-end') endedThen(db, line.session_id, at);
319
+ }
320
+
321
+ /** What waits in the spool, into the journal, before the event in hand. It never stops that event: what went wrong is logged, and the lines wait on. */
322
+ function drain(db) {
323
+ try {
324
+ const done = drainSpool((line) => replay(db, line), { within: (fn) => tx(db, fn) });
325
+ if (!done) return;
326
+ const parts = [`${done.replayed} replayed`, done.retried > 0 ? `${done.retried} to try again` : '', done.bad > 0 ? `${done.bad} set aside in spool/events.bad.jsonl (sumo memory check counts them)` : ''].filter(Boolean);
327
+ appendLog('hook.log', `${new Date().toISOString()} spool: ${parts.join(', ')}\n`);
328
+ } catch (cause) {
329
+ logFailure('spool', cause);
330
+ }
331
+ }
332
+
333
+ /** The lines set aside in the spool tried again, for `sumo memory check --fix`: `{ replayed, left }`. */
334
+ export const replaySetAside = (db) => retrySetAside((line) => replay(db, line), { within: (fn) => tx(db, fn) });
335
+
256
336
  /** What went wrong, in logs/hook.log. If even the log cannot be written there is nothing left to do but stay out of the way. */
257
337
  function logFailure(what, cause) {
258
338
  try {
@@ -266,21 +346,34 @@ function logFailure(what, cause) {
266
346
  * One event from outside the process, JSON on stdin. It never throws and never
267
347
  * exits non-zero: memory is a convenience, and a convenience that can break the
268
348
  * session it serves is worse than none. Whatever goes wrong is written to
269
- * logs/hook.log and the session simply carries on without this contribution.
349
+ * logs/hook.log and the session simply carries on without this contribution —
350
+ * but what was said is not lost with it: an event the database could not take,
351
+ * or a handler broke on, waits in the spool for the next one that gets through.
270
352
  */
271
353
  export function runHook(event, stdin) {
272
354
  let db = null;
355
+ let payload = null;
356
+ const now = new Date().toISOString();
273
357
  try {
274
- const payload = JSON.parse(stdin || '{}');
358
+ payload = JSON.parse(stdin || '{}');
275
359
  // The guard needs no database, and a database that cannot be opened must not stop it.
276
360
  const refused = event === 'pre-tool' ? earlyRefusal(fieldsOf(payload)) : '';
277
361
  if (refused) return deny(refused);
278
362
  db = openDb();
279
- return handleEvent(db, event, payload);
363
+ return handleEvent(db, event, payload, now, { harness: HOOKED });
280
364
  } catch (cause) {
281
365
  logFailure(event, cause);
366
+ try {
367
+ if (payload && typeof payload === 'object') keepForLater(event, payload, now);
368
+ } catch (unkept) {
369
+ logFailure(`${event} spool`, unkept);
370
+ }
282
371
  return '';
283
372
  } finally {
284
- db?.close();
373
+ try {
374
+ db?.close();
375
+ } catch (cause) {
376
+ logFailure(`${event} close`, cause);
377
+ }
285
378
  }
286
379
  }
package/src/jobs.mjs CHANGED
@@ -15,6 +15,7 @@ import { preferences } from './prime.mjs';
15
15
  import { getProject, isScratch } from './projects.mjs';
16
16
  import { chooseRoute, RETRY_IMPORTANT_THRESHOLD, statsLines } from './route.mjs';
17
17
  import { verifyCommands } from './scan.mjs';
18
+ import { filingState } from './ledger.mjs';
18
19
  import { runScribe } from './scribe.mjs';
19
20
  import { addCheckpoint, pendingTurns } from './sessions.mjs';
20
21
  import { CONFIG_DEFAULTS } from './setup.mjs';
@@ -377,8 +378,9 @@ export async function newJob(db, { project: nameOrAlias, title, agent = 'worker'
377
378
  const guide = chosen ? guideFor(chosen) : null;
378
379
  const judged = agent === 'reviewer' ? changeToJudge(db, { project, reviews }) : null;
379
380
 
380
- // A rule the user stated a minute ago may not be filed yet, and the brief is built from memory.
381
- if (pendingTurns(db, 1).length > 0) await runScribe(db, { now });
381
+ // A rule the user stated a minute ago may not be filed yet, and the brief is built from memory: the conversation
382
+ // still going on is read now. Not while filing is paused — a job starting is not the user asking for it by hand.
383
+ if (pendingTurns(db, 1).length > 0 && !filingState(db, now).paused) await runScribe(db, { now });
382
384
 
383
385
  const route = routed ?? (await chooseRoute(db, { agent, project, task, title: title.trim(), retryOf, reviews }));
384
386
 
package/src/keeper.mjs CHANGED
@@ -1,10 +1,6 @@
1
1
  import { getMeta, setMeta } from './db.mjs';
2
- import { runDream, SESSIONS_TO_TIDY, tidyDue } from './dream.mjs';
3
- import { localServerStatus } from './local-server.mjs';
4
- import { memoryHealth, memoryHealthLine } from './memory.mjs';
5
- import { modelStats, outcomeLines, runScribe } from './scribe.mjs';
6
- import { pendingTurns, sessionsAwaitingDream } from './sessions.mjs';
7
- import { CONFIG_DEFAULTS, modelFileOf } from './setup.mjs';
2
+ import { runDream, tidyDue } from './dream.mjs';
3
+ import { outcomeLines, runScribe, withLock } from './scribe.mjs';
8
4
 
9
5
  /** When the last tidy failed; until TIDY_RETRY_MS has gone by, a pass that finds a tidy due files only. */
10
6
  const TIDY_FAILED = 'memory.tidy_failed_at';
@@ -12,19 +8,20 @@ const TIDY_RETRY_MS = 6 * 3_600_000;
12
8
 
13
9
  /**
14
10
  * The memory pass: the one thing that looks after memory, so the user has one name, one status and one setting
15
- * (`memory.model`) instead of a writer and a tidier to keep track of. Each pass files what was said since the last one;
16
- * when tidying is due — conversations have finished, or a week has gone by — the same pass then tidies: reads them
17
- * side by side, raises pairs that say one thing or disagree, fills in search terms. It runs by itself after turns and
18
- * at session start and end; `sumo memory run` runs it now, `--tidy` tidies now whatever is due. A tidy that failed is
19
- * not tried again by itself for six hours: a local model that is down must not turn every turn's filing into a failed
20
- * tidy. Returns what each half did: `{ filed, tidied }`, `tidied` null when it did not tidy.
11
+ * (`memory.model`) instead of a writer and a tidier to keep track of. Each pass files what the user stated in the
12
+ * conversations that have finished since the last one; when the week is up the same pass then tidies memory itself:
13
+ * raises pairs that say one thing or disagree, fills in search terms. A session starting or ending runs it by itself
14
+ * (`auto`): finished conversations only, and no filing while filing is paused. `sumo memory run` runs it now — also
15
+ * over conversations still going on, and through a pause — and `--tidy` tidies now whatever is due. A tidy that failed
16
+ * is not tried again by itself for six hours: a local model that is down must not turn every pass into a failed tidy.
17
+ * Returns what each half did: `{ filed, tidied }`, `tidied` null when it did not tidy.
21
18
  */
22
- export async function runMemoryPass(db, { now = new Date().toISOString(), tidy = false } = {}) {
23
- const filed = await runScribe(db, { now });
19
+ export async function runMemoryPass(db, { now = new Date().toISOString(), tidy = false, auto = false } = {}) {
20
+ const filed = await runScribe(db, { now, auto });
24
21
  const failedAt = getMeta(db, TIDY_FAILED);
25
22
  const resting = failedAt && Date.parse(now) - Date.parse(failedAt) < TIDY_RETRY_MS;
26
23
  if (!tidy && (resting || !tidyDue(db, now))) return { filed, tidied: null };
27
- const tidied = await runDream(db, { now, force: tidy, file: false });
24
+ const tidied = await withLock('dream', () => runDream(db, { now }));
28
25
  if (tidied.ok === false && !tidied.skipped) setMeta(db, TIDY_FAILED, now);
29
26
  else if (failedAt) setMeta(db, TIDY_FAILED, '');
30
27
  return { filed, tidied };
@@ -36,42 +33,8 @@ export async function runMemoryPass(db, { now = new Date().toISOString(), tidy =
36
33
  */
37
34
  export function passLines({ filed, tidied }, { asked = false } = {}) {
38
35
  const tidying = tidied ? outcomeLines(tidied).map((l, i) => (i === 0 ? `tidy: ${l}` : l)) : [];
39
- const lines = [...(tidied && filed.skipped ? [] : outcomeLines(filed)), ...tidying];
36
+ // A filing that is paused is said even beside a tidy: it is not "nothing to do".
37
+ const lines = [...(tidied && filed.skipped && !filed.paused ? [] : outcomeLines(filed)), ...tidying];
40
38
  const failed = (filed.ok === false && !filed.skipped) || (asked && tidied?.ok === false && !tidied.skipped);
41
39
  return failed ? Object.assign(lines, { exitCode: 1 }) : lines;
42
40
  }
43
-
44
- /** `sumo memory status`: the pass as a whole — the model it runs on, what waits, how it last went — then the numbers. */
45
- /** The ledger's names for the pass's two halves, as the status says them. */
46
- const HALVES = { scribe: 'filing', dream: 'tidying' };
47
- /** A status line: its label in a column of its own, so the values stand in one. */
48
- const row = (label, value) => `${label.padEnd(LABEL_WIDTH)}${value}`;
49
- const LABEL_WIDTH = 11;
50
-
51
- export async function memoryStatus(db, now = new Date().toISOString()) {
52
- const health = memoryHealthLine(memoryHealth(db, now), now);
53
- const model = getMeta(db, 'config.memory.model') ?? CONFIG_DEFAULTS['memory.model'];
54
- const server = model === 'local' ? await localServerStatus(db, { file: modelFileOf(db) }) : null;
55
- const turns = pendingTurns(db, 1000).length;
56
- const finished = sessionsAwaitingDream(db, now, 1000).length;
57
- const tidied = db.prepare(`SELECT MAX(ts) AS at FROM model_runs WHERE kind = 'dream' AND ok = 1`).get()?.at ?? null;
58
- const sweep = getMeta(db, 'dream.last_tidy');
59
- // Due with fewer than three conversations waiting: the weekly sweep is what is owed.
60
- const owed = finished < SESSIONS_TO_TIDY && tidyDue(db, now);
61
- const failures = Number(getMeta(db, 'scribe.failures') ?? 0);
62
- const failedAt = getMeta(db, TIDY_FAILED);
63
- const day = (at) => (at ? at.slice(0, 10) : 'never');
64
- // The ledger holds every request; the pass's own are its filing and its tidying, named so.
65
- const spent = modelStats(db)
66
- .filter((l) => /^(scribe|dream): /.test(l))
67
- .map((l) => l.replace(/^(scribe|dream)/, (kind) => HALVES[kind]));
68
- return [
69
- `${health.ok ? 'ok ' : 'warn'} ${health.label}${health.fix ? ` — ${health.fix}` : ''}`,
70
- row('model', `${model}${server ? ` · ${server}` : ''}`),
71
- row('waiting', `${turns} turn${turns === 1 ? '' : 's'} to file · ${finished} finished conversation${finished === 1 ? '' : 's'} to read (tidied by itself at ${SESSIONS_TO_TIDY}, and weekly)`),
72
- row('last tidy', `${day(tidied)} · weekly sweep: ${day(sweep)}${owed ? ' — owed: the next pass sweeps' : ''}`),
73
- ...(failures > 0 ? [row('failing', `filing failed ${failures} time${failures === 1 ? '' : 's'} in a row — ${getMeta(db, 'scribe.last_error')}`)] : []),
74
- ...(failedAt ? [row('failing', `the last tidy, at ${failedAt.slice(0, 16)}; tried again by itself ${Date.parse(now) - Date.parse(failedAt) < TIDY_RETRY_MS ? 'six hours on' : 'with the next pass'}, or now: sumo memory run --tidy`)] : []),
75
- ...(spent.length > 0 ? [row('spent', spent[0]), ...spent.slice(1).map((l) => row('', l))] : []),
76
- ];
77
- }
package/src/ledger.mjs ADDED
@@ -0,0 +1,126 @@
1
+ import { createHash } from 'node:crypto';
2
+ import { readFileSync } from 'node:fs';
3
+ import { join } from 'node:path';
4
+ import { APPLY_RULES } from './apply.mjs';
5
+ import { getMeta, setMeta } from './db.mjs';
6
+ import { REPO_ROOT } from './paths.mjs';
7
+ import { CONFIG_DEFAULTS } from './setup.mjs';
8
+
9
+ /**
10
+ * The extraction ledger, read without a model: which extractor is filing, and how its filing is going. Every model
11
+ * call of the memory pass is a row of `extractions`, written `running` before the call and closed `ok`, `failed` or
12
+ * `quarantined`; what the pass derived carries the extractor's version, so a changed prompt, model or rule is a new
13
+ * extractor whose record starts clean.
14
+ */
15
+
16
+ /** Failed passes in a row before filing stops by itself: a model that is down must not be asked on every session. */
17
+ const FAILURES_TO_PAUSE = 3;
18
+ /** The share of what an extractor filed that the user then threw out, past which it stops filing by itself. */
19
+ const JUNK_TO_PAUSE = 0.5;
20
+ /** Fewer derived memories than this say nothing about a rate. */
21
+ const JUNK_MIN_DERIVED = 10;
22
+ /** A memory thrown out this long after it was filed was wanted for a while: it went stale, it was not junk. */
23
+ const JUNK_WITHIN_MS = 30 * 86_400_000;
24
+
25
+ const FAILURES = 'scribe.failures';
26
+ const FAILURES_VERSION = 'scribe.failures_version';
27
+ const LAST_ERROR = 'scribe.last_error';
28
+ const FALLBACK_USED = 'scribe.fallback_used';
29
+
30
+ /** The model the memory pass is set to: `memory.model`, the one setting. */
31
+ const memoryModel = (db) => getMeta(db, 'config.memory.model') ?? CONFIG_DEFAULTS['memory.model'];
32
+
33
+ /**
34
+ * The extractor's version: a short hash of what decides what gets filed — the prompt the model reads, the model
35
+ * `memory.model` names, and the rules its answer is held to (`APPLY_RULES`). Each derived memory and each row of the
36
+ * ledger carries it.
37
+ */
38
+ export function extractorVersion(db) {
39
+ const prompt = readFileSync(join(REPO_ROOT, 'prompts', 'scribe.md'), 'utf8');
40
+ return createHash('sha256').update(`${prompt}\n${memoryModel(db)}\n${APPLY_RULES}`).digest('hex').slice(0, 8);
41
+ }
42
+
43
+ /**
44
+ * Of the memories this extractor derived, the share the user then threw out within 30 days — forgot, rejected, or
45
+ * replaced by hand — read from `memory_events`; null under ten derived. A memory a rebuild retired is not the user's
46
+ * verdict, and one the pass itself replaced with a newer statement was right when it was filed.
47
+ */
48
+ function junkRate(db, version) {
49
+ const derived = db
50
+ .prepare(
51
+ `SELECT e.memory_id AS id, MIN(e.ts) AS at FROM memory_events e JOIN extractions x ON x.id = e.run_id
52
+ WHERE e.op = 'added' AND x.extractor_version = ? GROUP BY e.memory_id`,
53
+ )
54
+ .all(version);
55
+ if (derived.length < JUNK_MIN_DERIVED) return null;
56
+ const thrownOut = db.prepare(
57
+ `SELECT MIN(ts) AS at FROM memory_events
58
+ WHERE memory_id = ? AND COALESCE(reason, '') != 'rebuilt'
59
+ AND (op IN ('forgotten', 'rejected', 'purged') OR (op = 'superseded' AND actor IN ('user', 'model-shell')))`,
60
+ );
61
+ const junk = derived.filter((d) => {
62
+ const at = thrownOut.get(d.id)?.at;
63
+ return at && Date.parse(at) - Date.parse(d.at) <= JUNK_WITHIN_MS;
64
+ });
65
+ return junk.length / derived.length;
66
+ }
67
+
68
+ /** How many of the conversations last read are looked at for one that filed something. */
69
+ const EMPTY_LOOKS = 100;
70
+
71
+ /**
72
+ * For whoever checks memory's health: how many conversations in a row, counting back from the one read last, had
73
+ * nothing filed from them — no memory added by any pass that read them. Nothing is the normal answer for one conversation; a model that answers nothing to
74
+ * everything looks exactly like it. What the writer from before versions read (`v0`) kept no such record and is not counted.
75
+ */
76
+ export function emptyRun(db) {
77
+ const read = db
78
+ .prepare(
79
+ `SELECT x.session_id, MAX(x.id) AS last,
80
+ EXISTS (SELECT 1 FROM memory_events e JOIN extractions y ON y.id = e.run_id WHERE y.session_id = x.session_id AND e.op = 'added') AS filed
81
+ FROM extractions x WHERE x.status = 'ok' AND x.extractor_version != 'v0' GROUP BY x.session_id ORDER BY last DESC LIMIT ?`,
82
+ )
83
+ .all(EMPTY_LOOKS);
84
+ const first = read.findIndex((r) => r.filed);
85
+ return first === -1 ? read.length : first;
86
+ }
87
+
88
+ /**
89
+ * How filing stands, for the pass and for whoever reports on it:
90
+ * - `failures`: passes of this extractor that failed in a row;
91
+ * - `quarantined`: turns set aside after failing three times alone;
92
+ * - `fallbackUsed`: a pass ran on the hosted fallback because the local model could not answer, and none has been read on the local model since;
93
+ * - `junkRate`: see `junkRate`;
94
+ * - `paused` with its `reason`: three failures in a row, or more than half of what it filed thrown out. A paused pass
95
+ * files nothing by itself until `sumo memory run` is asked by hand or the extractor changes;
96
+ * - `lastOkAt`: when a conversation was last read through.
97
+ */
98
+ export function filingState(db) {
99
+ const version = extractorVersion(db);
100
+ // A count kept under another extractor is that one's: a changed prompt or model starts from none.
101
+ const failures = getMeta(db, FAILURES_VERSION) === version ? Number(getMeta(db, FAILURES) ?? 0) : 0;
102
+ const quarantined = db.prepare(`SELECT COUNT(*) AS n FROM extractions WHERE status = 'quarantined' AND extractor_version = ?`).get(version).n;
103
+ const junk = junkRate(db, version);
104
+ const lastOkAt = db.prepare(`SELECT MAX(ended_at) AS at FROM extractions WHERE status = 'ok'`).get().at ?? null;
105
+ const by = 'sumo memory run reads again by hand';
106
+ const reason =
107
+ failures >= FAILURES_TO_PAUSE ? `the last ${failures} passes failed (${getMeta(db, LAST_ERROR) ?? 'no reason kept'}) — ${by}`
108
+ : junk !== null && junk > JUNK_TO_PAUSE ? `${Math.round(junk * 100)}% of what this extractor filed was forgotten or rejected within 30 days — ${by}, and a changed prompt or model starts afresh`
109
+ : null;
110
+ return { paused: reason !== null, reason, failures, quarantined, junkRate: junk, fallbackUsed: getMeta(db, FALLBACK_USED) === '1', lastOkAt };
111
+ }
112
+
113
+ /**
114
+ * How a pass went, written where `filingState` reads it: a pass that read something clears the count, one that only
115
+ * failed adds to it. That the hosted fallback was used stays said until a pass reads a conversation on the model as
116
+ * set (`direct`): a pass that had nothing to read, or that failed, has not shown the local model is back.
117
+ */
118
+ export function notePass(db, { ok, error = null, fallbackUsed = false, direct = false }) {
119
+ const version = extractorVersion(db);
120
+ const before = getMeta(db, FAILURES_VERSION) === version ? Number(getMeta(db, FAILURES) ?? 0) : 0;
121
+ setMeta(db, FAILURES_VERSION, version);
122
+ setMeta(db, FAILURES, ok ? 0 : before + 1);
123
+ if (!ok) setMeta(db, LAST_ERROR, error ?? 'no reason given');
124
+ if (fallbackUsed) setMeta(db, FALLBACK_USED, '1');
125
+ else if (direct) setMeta(db, FALLBACK_USED, '');
126
+ }
package/src/logs.mjs CHANGED
@@ -1,4 +1,4 @@
1
- import { appendFileSync, chmodSync, copyFileSync, existsSync, mkdirSync, openSync, readdirSync, readFileSync, renameSync, rmSync, statSync, truncateSync, writeFileSync } from 'node:fs';
1
+ import { appendFileSync, chmodSync, closeSync, copyFileSync, existsSync, fstatSync, mkdirSync, openSync, readdirSync, readFileSync, renameSync, rmSync, statSync, truncateSync, writeFileSync } from 'node:fs';
2
2
  import { join } from 'node:path';
3
3
  import { paths } from './paths.mjs';
4
4
  import { redact } from './redact.mjs';
@@ -84,10 +84,15 @@ export function crashFiles() {
84
84
  export function latestCrash(since) {
85
85
  const [file] = crashFiles();
86
86
  if (!file) return null;
87
- const at = statSync(file).mtime;
88
- if (at < since) return null;
89
- const said = readFileSync(file, 'utf8').split('\n').find((line) => line.startsWith('error: '))?.slice('error: '.length) ?? 'see the file';
90
- return { file, at, said };
87
+ const fd = openSync(file, 'r');
88
+ try {
89
+ const at = fstatSync(fd).mtime;
90
+ if (at < since) return null;
91
+ const said = readFileSync(fd, 'utf8').split('\n').find((line) => line.startsWith('error: '))?.slice('error: '.length) ?? 'see the file';
92
+ return { file, at, said };
93
+ } finally {
94
+ closeSync(fd);
95
+ }
91
96
  }
92
97
 
93
98
  /**
package/src/loop.mjs CHANGED
@@ -145,7 +145,7 @@ function clearedBy(response) {
145
145
  return ` cleared ${uses} tool use${uses === 1 ? '' : 's'} (${Math.round(tokens / 1000)}k)`;
146
146
  }
147
147
 
148
- export const promptFile = (name) => readFileSync(join(REPO_ROOT, 'prompts', name), 'utf8').trim();
148
+ const promptFile = (name) => readFileSync(join(REPO_ROOT, 'prompts', name), 'utf8').trim();
149
149
  /**
150
150
  * A sub-agent works by the user's coding rules as the chat does: the Coding section of AGENTS.md, read from its one
151
151
  * home, without the lines that point at guides — a job is handed its guide in the brief.
@@ -741,7 +741,7 @@ export async function runJob(db, id, { send = sendToApi, now, signal = null, onS
741
741
  if (job.status !== 'running' && job.status !== 'needs_input') throw new UsageError(`j${id} is ${job.status} — only a running job can be run`);
742
742
  if (!job.model) throw new UsageError(`j${id} has no route — it was created before routing existed; create it again`);
743
743
  if (MODELS.includes(job.model) && !usableModels(db).includes(job.model)) {
744
- throw new UsageError(`j${id} is routed to ${job.model}, which is off — sumo models enable ${job.model}, or abandon it and create it again`);
744
+ throw new UsageError(`j${id} is routed to ${job.model}, which is off — turn it on under API models, on the settings page (sumo config), or abandon it and create it again`);
745
745
  }
746
746
  const project = getProject(db, job.project);
747
747
  const tree = treeOf(db, id);
@@ -890,8 +890,8 @@ export function stopReason(stop, detail = null) {
890
890
  if (stop === 'model_context_window_exceeded') return 'stopped: the conversation no longer fits in the model\'s context';
891
891
  if (stop === 'max_turns') return `stopped: ${MAX_TURNS} turns without an end`;
892
892
  if (stop === 'loop') return `stopped: the same call, or the same round of calls, was made ${REPEAT_STOP_AT} times in a row`;
893
- if (stop === 'budget') return 'stopped: the job reached its budget — sumo config jobs.budget sets it, 0 for none';
894
- if (stop === 'time') return 'stopped: the job reached its time limit — sumo config jobs.minutes sets it, 0 for none';
893
+ if (stop === 'budget') return 'stopped: the job reached its budget — Jobs › Budget, on the settings page (sumo config), sets it; 0 for none';
894
+ if (stop === 'time') return 'stopped: the job reached its time limit — Jobs › Time limit, on the settings page (sumo config), sets it; 0 for none';
895
895
  if (stop === 'advisor') return `stopped: the advisor was asked ${ADVISOR_STOP_AT} times`;
896
896
  if (stop === 'declined') return `stopped: the user declined ${DECLINE_STOP_AT} calls`;
897
897
  return null;