sumo-agents 0.6.1 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +6 -5
- package/CHANGELOG.md +72 -0
- package/README.md +47 -34
- package/guides/memory.md +21 -14
- package/package.json +2 -2
- package/prompts/dream.md +3 -16
- package/prompts/scribe.md +20 -15
- package/src/apply.mjs +364 -230
- package/src/card.mjs +16 -11
- package/src/catalog.mjs +3 -3
- package/src/chat.mjs +53 -15
- package/src/check.mjs +371 -0
- package/src/cli.mjs +230 -146
- package/src/db.mjs +147 -2
- package/src/dream.mjs +97 -189
- package/src/export.mjs +43 -18
- package/src/herdr.mjs +71 -36
- package/src/hooks.mjs +115 -22
- package/src/jobs.mjs +4 -2
- package/src/keeper.mjs +14 -51
- package/src/ledger.mjs +126 -0
- package/src/logs.mjs +10 -5
- package/src/loop.mjs +4 -4
- package/src/mcp.mjs +129 -51
- package/src/memeval.mjs +167 -0
- package/src/memory.mjs +261 -130
- package/src/page.mjs +3 -3
- package/src/paths.mjs +1 -0
- package/src/prime.mjs +82 -13
- package/src/rebuild.mjs +162 -0
- package/src/render.mjs +22 -0
- package/src/route.mjs +1 -1
- package/src/sandbox.mjs +5 -2
- package/src/scan.mjs +16 -4
- package/src/scribe.mjs +361 -113
- package/src/search.mjs +48 -0
- package/src/sessions.mjs +304 -53
- package/src/settings.mjs +868 -0
- package/src/setup.mjs +56 -64
- package/src/spool.mjs +308 -0
- package/src/text.mjs +118 -6
- package/src/tools.mjs +18 -7
- package/src/transcript.mjs +3 -47
- package/src/tty.mjs +48 -23
- package/src/ui.mjs +62 -25
- package/src/verify.mjs +23 -27
package/src/export.mjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { chmodSync, copyFileSync, existsSync, mkdirSync, readdirSync, renameSync, rmSync, statSync } from 'node:fs';
|
|
2
2
|
import { basename, dirname, join, resolve } from 'node:path';
|
|
3
3
|
import { getMeta, setMeta } from './db.mjs';
|
|
4
|
-
import { UsageError } from './memory.mjs';
|
|
4
|
+
import { eraseRow, eraseSaid, UsageError } from './memory.mjs';
|
|
5
5
|
import { paths } from './paths.mjs';
|
|
6
6
|
import { DatabaseSync } from './sqlite.mjs';
|
|
7
7
|
|
|
@@ -185,14 +185,14 @@ export function restoreLines(copy, live, now = new Date()) {
|
|
|
185
185
|
}
|
|
186
186
|
|
|
187
187
|
/**
|
|
188
|
-
*
|
|
189
|
-
*
|
|
190
|
-
*
|
|
188
|
+
* Every copy this machine keeps of the database — the weekly backups, and what was set aside beside it before a
|
|
189
|
+
* migration or by a restore — put through `erase(db, has)`, which says how much it took out: a copy it changed is
|
|
190
|
+
* rewritten whole, indexes and pages, so nothing of what went is left in it. All of it or none: a copy that cannot be
|
|
191
|
+
* cleaned whole is left as it was, and named. `{ cleaned, failed }`.
|
|
191
192
|
*/
|
|
192
|
-
|
|
193
|
+
function cleanBackups(erase) {
|
|
193
194
|
const { backups, db: live } = paths();
|
|
194
195
|
const within = (dir, keep) => (existsSync(dir) ? readdirSync(dir).filter(keep).map((f) => join(dir, f)) : []);
|
|
195
|
-
// The weekly copies, and every copy set aside beside the database: before a migration, or by a restore.
|
|
196
196
|
const aside = (f) => f.startsWith(`${basename(live)}.`) && (/\.pre-v\d+\.bak$/.test(f) || /\.(?:damaged|replaced)-[^/]*$/.test(f)) && !/-(?:wal|shm)$/.test(f);
|
|
197
197
|
const files = [...within(backups, (f) => /^memory-.*\.db$/.test(f)), ...within(dirname(live), aside)];
|
|
198
198
|
const failed = [];
|
|
@@ -202,21 +202,23 @@ export function purgeFromBackups({ id, body, created_at: createdAt, source_turn:
|
|
|
202
202
|
try {
|
|
203
203
|
db = new DatabaseSync(file);
|
|
204
204
|
db.exec('PRAGMA secure_delete = ON;');
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
db
|
|
210
|
-
|
|
211
|
-
db.prepare('DELETE FROM memories WHERE id = ?').run(id);
|
|
212
|
-
if (sourceTurn !== null) db.prepare('DELETE FROM user_turns WHERE id = ?').run(sourceTurn);
|
|
205
|
+
const has = (name) => db.prepare('SELECT 1 FROM sqlite_master WHERE name = ?').get(name) !== undefined;
|
|
206
|
+
let changed = 0;
|
|
207
|
+
db.exec('BEGIN');
|
|
208
|
+
try {
|
|
209
|
+
changed = erase(db, has);
|
|
210
|
+
if (changed > 0) {
|
|
213
211
|
// A deleted row's words stay in the full-text index's pages until it is rebuilt; VACUUM keeps those pages.
|
|
214
212
|
db.exec(`INSERT INTO memories_fts(memories_fts) VALUES ('rebuild'); INSERT INTO user_turns_fts(user_turns_fts) VALUES ('rebuild');`);
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
db.exec('ROLLBACK');
|
|
218
|
-
throw cause;
|
|
213
|
+
// A copy from before replies were kept has none to clean.
|
|
214
|
+
if (has('replies_fts')) db.exec(`INSERT INTO replies_fts(replies_fts) VALUES ('rebuild');`);
|
|
219
215
|
}
|
|
216
|
+
db.exec('COMMIT');
|
|
217
|
+
} catch (cause) {
|
|
218
|
+
db.exec('ROLLBACK');
|
|
219
|
+
throw cause;
|
|
220
|
+
}
|
|
221
|
+
if (changed > 0) {
|
|
220
222
|
db.exec('VACUUM');
|
|
221
223
|
cleaned++;
|
|
222
224
|
}
|
|
@@ -228,3 +230,26 @@ export function purgeFromBackups({ id, body, created_at: createdAt, source_turn:
|
|
|
228
230
|
}
|
|
229
231
|
return { cleaned, failed };
|
|
230
232
|
}
|
|
233
|
+
|
|
234
|
+
/**
|
|
235
|
+
* A purge reaches the backups too: "erased, not recoverable" must hold for every copy this machine keeps. Each backup
|
|
236
|
+
* loses the memory, every turn and reply that says it whole and what its passes proposed — by the one rule the
|
|
237
|
+
* database itself is held to (`eraseSaid` in memory.mjs). A copy from before the memory was filed may hold the
|
|
238
|
+
* sentence without the memory: it is cleaned all the same. Returns how many were cleaned, and the ones that could
|
|
239
|
+
* not be — said, never hidden.
|
|
240
|
+
*/
|
|
241
|
+
export function purgeFromBackups({ id, body, created_at: createdAt, source_turn: sourceTurn, source_session: sourceSession = null, source_quote: sourceQuote = null }) {
|
|
242
|
+
return cleanBackups((db, has) => {
|
|
243
|
+
// The same memory, not just the same number: a backup from after a restore may hold another one under this id.
|
|
244
|
+
const row = db.prepare('SELECT source_turn FROM memories WHERE id = ? AND body = ? AND created_at = ?').get(id, body, createdAt);
|
|
245
|
+
// The passes that filed it, while its history still names them; a copy from before histories were kept has none.
|
|
246
|
+
const runs = row && has('memory_events') ? db.prepare('SELECT run_id FROM memory_events WHERE memory_id = ? AND run_id IS NOT NULL').all(id).map((e) => e.run_id) : [];
|
|
247
|
+
if (row) db.prepare('DELETE FROM memories WHERE id = ?').run(id);
|
|
248
|
+
// The turn it cites goes by its number only where the memory is this one: in another copy that number may be another turn.
|
|
249
|
+
const erased = eraseSaid(db, { body, source_quote: sourceQuote, source_turn: row ? sourceTurn : null, source_session: sourceSession }, { runs });
|
|
250
|
+
return (row ? 1 : 0) + erased.turns.length + erased.replies.length + erased.ops;
|
|
251
|
+
});
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
/** One turn or reply purged (`sumo forget --purge t12`) goes from every copy that holds it, the same way. */
|
|
255
|
+
export const purgeRowFromBackups = (entry) => cleanBackups((db) => (eraseRow(db, entry) ? 1 : 0));
|
package/src/herdr.mjs
CHANGED
|
@@ -5,8 +5,9 @@ import { paths, SELF } from './paths.mjs';
|
|
|
5
5
|
/**
|
|
6
6
|
* Herdr is where a delegated job can be watched. The job itself runs in the chat's process; inside Herdr it also
|
|
7
7
|
* gets a tab of its own, listed on the left and named after it, where `sumo job watch` shows its work as it
|
|
8
|
-
* happens and reports it to Herdr as an agent with its state — so the chat keeps one line per job.
|
|
9
|
-
*
|
|
8
|
+
* happens and reports it to Herdr as an agent with its state — so the chat keeps one line per job. The chat opened
|
|
9
|
+
* the tab, so the chat closes it once the job's run is over, however it ended. Everything goes through the `herdr`
|
|
10
|
+
* CLI, which answers in JSON — an error on stderr.
|
|
10
11
|
*/
|
|
11
12
|
|
|
12
13
|
export const runHerdr = (args) => spawnSync('herdr', args, { encoding: 'utf8', timeout: 10_000 });
|
|
@@ -28,93 +29,127 @@ const shellWord = (text) => (/^[\w/.:=@%+-]+$/.test(text) ? text : `'${text.repl
|
|
|
28
29
|
|
|
29
30
|
/**
|
|
30
31
|
* A tab in the chat's workspace that shows a job's work: `sumo job watch`, not the job — that stays in the chat.
|
|
31
|
-
*
|
|
32
|
-
*
|
|
32
|
+
* Its pane and its tab, which the chat closes once the run is over. Throws with Herdr's reason when the tab cannot be
|
|
33
|
+
* opened; the job runs the same either way. The watcher is typed into the tab's shell after this returns, once the
|
|
34
|
+
* shell is ready for it, and `signal` calls that off when the run ends first; `onFail` hears why it never started.
|
|
33
35
|
*/
|
|
34
|
-
export function openWatchTab(herdr, { job, project, env = process.env, onFail = () => {}, sleep }) {
|
|
36
|
+
export function openWatchTab(herdr, { job, project, env = process.env, onFail = () => {}, sleep, signal = null }) {
|
|
35
37
|
const workspace = env.HERDR_WORKSPACE_ID ? ['--workspace', env.HERDR_WORKSPACE_ID] : [];
|
|
36
38
|
const made = herdr(['tab', 'create', ...workspace, '--cwd', project.path, '--label', `${agentLabel(job)} · ${job.title}`, '--no-focus']);
|
|
37
|
-
const tab = parse(made
|
|
39
|
+
const tab = parse(made);
|
|
38
40
|
if (tab.error) throw new Error(tab.error);
|
|
39
|
-
// Marked as the tab's own, so the watcher may close it behind
|
|
41
|
+
// Marked as the tab's own, so the watcher may close it behind the run; the home goes with the command,
|
|
40
42
|
// or a chat on another home would be watched from the default one.
|
|
41
43
|
// The launcher setup made, where there is one; else this very sumo, node and entry by their full paths — a clone not set up yet still has a watcher.
|
|
42
44
|
const sumo = existsSync(paths().launcher) ? shellWord(paths().launcher) : SELF.map(shellWord).join(' ');
|
|
43
45
|
const command = `env SUMO_JOB_TAB=1 SUMO_AGENTS_HOME=${shellWord(paths().home)} ${sumo} job watch ${job.id}`;
|
|
44
|
-
startWatcher(herdr, { pane: tab.
|
|
45
|
-
.then((started) => started
|
|
46
|
+
startWatcher(herdr, { pane: tab.pane, job, command, sleep, signal })
|
|
47
|
+
.then((started) => started === false && onFail('its Herdr tab never started the watcher'))
|
|
46
48
|
.catch((cause) => onFail(cause.message));
|
|
47
|
-
return tab
|
|
49
|
+
return tab;
|
|
48
50
|
}
|
|
49
51
|
|
|
52
|
+
/** Closes a tab sumo opened; one already gone is no matter. */
|
|
53
|
+
export const closeTab = (herdr, tab) => herdr(['tab', 'close', tab]);
|
|
54
|
+
|
|
50
55
|
const POLL_MS = 250;
|
|
51
|
-
/**
|
|
56
|
+
/**
|
|
57
|
+
* How long a new shell is given to reach its prompt; how long a typed watcher is given to start before a shell back
|
|
58
|
+
* at its prompt has it typed again; and how long anything else may hold the tab's foreground before it is given up.
|
|
59
|
+
*/
|
|
52
60
|
const READY_MS = 15_000;
|
|
53
61
|
const STARTED_MS = 3_000;
|
|
62
|
+
const GIVE_UP_MS = 30_000;
|
|
54
63
|
const TRIES = 3;
|
|
55
64
|
const pause = (ms) => new Promise((resolve) => setTimeout(resolve, ms).unref());
|
|
56
65
|
|
|
57
|
-
/**
|
|
58
|
-
function
|
|
59
|
-
const run = herdr(
|
|
60
|
-
if (run.error
|
|
66
|
+
/** Herdr's answer: `{ result }`, `{ gone: true }` for a pane that is closed, or null when Herdr cannot say — an older Herdr, or one not answering. */
|
|
67
|
+
function ask(herdr, args) {
|
|
68
|
+
const run = herdr(args);
|
|
69
|
+
if (run.error) return null;
|
|
61
70
|
try {
|
|
62
|
-
|
|
71
|
+
const answer = JSON.parse(run.status === 0 ? run.stdout : run.stderr);
|
|
72
|
+
if (answer.error) return answer.error.code === 'pane_not_found' ? { gone: true } : null;
|
|
73
|
+
return { result: answer.result ?? {} };
|
|
63
74
|
} catch {
|
|
64
75
|
return null;
|
|
65
76
|
}
|
|
66
77
|
}
|
|
67
78
|
|
|
79
|
+
/** What the pane runs in the foreground; `{ gone: true }` or null as `ask` says. */
|
|
80
|
+
function foreground(herdr, pane) {
|
|
81
|
+
const answer = ask(herdr, ['pane', 'process-info', '--pane', pane]);
|
|
82
|
+
return answer?.gone ? answer : (answer?.result.process_info ?? null);
|
|
83
|
+
}
|
|
84
|
+
|
|
68
85
|
const screen = (herdr, pane) => {
|
|
69
86
|
const run = herdr(['pane', 'read', pane, '--source', 'visible']);
|
|
70
87
|
return run.error || run.status !== 0 ? null : String(run.stdout);
|
|
71
88
|
};
|
|
72
89
|
|
|
73
90
|
/**
|
|
74
|
-
* Types the watcher into a new tab's shell and makes sure it runs.
|
|
75
|
-
*
|
|
76
|
-
*
|
|
77
|
-
*
|
|
91
|
+
* Types the watcher into a new tab's shell and makes sure it runs. The watcher is typed only once the shell stands
|
|
92
|
+
* alone in the foreground with its screen settled — what Herdr itself asks of a shell before it starts an agent there.
|
|
93
|
+
* It has started once Herdr shows its report on the pane (`reportAgent`), or once it runs in the foreground. It is
|
|
94
|
+
* typed again, a few times at most, only when the shell stands alone at its prompt again — the line was lost, or the
|
|
95
|
+
* watcher quit — never while anything runs there: a line typed into a watcher reaches the job as a message.
|
|
96
|
+
* True once it runs; false when it never did; null when it no longer matters — the run is over (`signal`), the pane
|
|
97
|
+
* is gone, or Herdr cannot say (one too old to ask).
|
|
78
98
|
*/
|
|
79
|
-
export async function startWatcher(herdr, { pane, job, command, sleep = pause, tries = TRIES }) {
|
|
80
|
-
const watching = new RegExp(`job watch ${job}(?!\\d)`);
|
|
81
|
-
const
|
|
99
|
+
export async function startWatcher(herdr, { pane, job, command, sleep = pause, tries = TRIES, signal = null }) {
|
|
100
|
+
const watching = new RegExp(`job watch ${job.id}(?!\\d)`);
|
|
101
|
+
const reported = () => ask(herdr, ['pane', 'get', pane])?.result?.pane?.agent === agentLabel(job);
|
|
102
|
+
const started = (info) => info.foreground_processes?.some((p) => watching.test(String(p.cmdline))) || reported();
|
|
103
|
+
const alone = (info) => info.foreground_processes?.every((p) => p.pid === info.shell_pid);
|
|
82
104
|
for (let attempt = 0; attempt < tries; attempt++) {
|
|
83
105
|
let last = null;
|
|
84
106
|
for (let waited = 0; waited < READY_MS; waited += POLL_MS) {
|
|
107
|
+
if (signal?.aborted) return null;
|
|
85
108
|
const info = foreground(herdr, pane);
|
|
109
|
+
if (info?.gone) return null;
|
|
86
110
|
if (info === null) break;
|
|
87
111
|
const now = screen(herdr, pane);
|
|
88
|
-
|
|
89
|
-
if (alone && now?.trim() && now === last) break;
|
|
112
|
+
if (alone(info) && now?.trim() && now === last) break;
|
|
90
113
|
last = now;
|
|
91
114
|
await sleep(POLL_MS);
|
|
92
115
|
}
|
|
116
|
+
if (signal?.aborted) return null;
|
|
93
117
|
const run = herdr(['pane', 'run', pane, command]);
|
|
94
|
-
if (run.error || run.status !== 0)
|
|
95
|
-
|
|
118
|
+
if (run.error || run.status !== 0) {
|
|
119
|
+
if (foreground(herdr, pane)?.gone) return null;
|
|
120
|
+
throw new Error(`Herdr opened the tab but would not run the watcher in it: ${String(run.stderr || run.stdout || run.error?.message).trim().slice(0, 200)}`);
|
|
121
|
+
}
|
|
122
|
+
for (let waited = POLL_MS; ; waited += POLL_MS) {
|
|
96
123
|
await sleep(POLL_MS);
|
|
124
|
+
if (signal?.aborted) return null;
|
|
97
125
|
const info = foreground(herdr, pane);
|
|
98
|
-
if (info === null ||
|
|
126
|
+
if (info === null || info.gone) return null;
|
|
127
|
+
if (started(info)) return true;
|
|
128
|
+
if (waited >= STARTED_MS && alone(info)) break;
|
|
129
|
+
if (waited >= GIVE_UP_MS) return false;
|
|
99
130
|
}
|
|
100
131
|
}
|
|
101
132
|
return false;
|
|
102
133
|
}
|
|
103
134
|
|
|
104
|
-
|
|
135
|
+
/** The tab `tab create` made: its root pane, and the tab to close once the run is over. */
|
|
136
|
+
function parse(run) {
|
|
105
137
|
if (run.error) return { error: run.error.code === 'ENOENT' ? '`herdr` is not on PATH' : run.error.message };
|
|
106
138
|
try {
|
|
107
|
-
const answer = JSON.parse(run.stdout);
|
|
139
|
+
const answer = JSON.parse(run.status === 0 ? run.stdout : run.stderr || run.stdout);
|
|
108
140
|
if (answer.error) return { error: answer.error.message ?? answer.error.code ?? 'Herdr refused' };
|
|
109
|
-
const
|
|
110
|
-
|
|
141
|
+
const pane = answer.result?.root_pane?.pane_id;
|
|
142
|
+
const tab = answer.result?.tab?.tab_id ?? answer.result?.root_pane?.tab_id;
|
|
143
|
+
return pane && tab ? { pane, tab } : { error: 'Herdr answered without a pane and tab id' };
|
|
111
144
|
} catch {
|
|
112
145
|
return { error: `Herdr did not answer in JSON: ${String(run.stderr || run.stdout).trim().slice(0, 200)}` };
|
|
113
146
|
}
|
|
114
147
|
}
|
|
115
148
|
|
|
116
|
-
/**
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
149
|
+
/**
|
|
150
|
+
* The watcher's side: the tab opened to watch a job closes once the run is over, however it ended. The chat closes
|
|
151
|
+
* it too; this is for a chat that could not. Only a tab sumo opened is closed — a tab the user opened is theirs.
|
|
152
|
+
*/
|
|
153
|
+
export function closeJobTab(herdr, env) {
|
|
154
|
+
if (env.SUMO_JOB_TAB && env.HERDR_TAB_ID) closeTab(herdr, env.HERDR_TAB_ID);
|
|
120
155
|
}
|
package/src/hooks.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { card } from './card.mjs';
|
|
2
|
-
import { getMeta, openDb, setMeta } from './db.mjs';
|
|
2
|
+
import { getMeta, openDb, setMeta, tx } from './db.mjs';
|
|
3
3
|
import { backup } from './export.mjs';
|
|
4
4
|
import { guardCommand, guardPath } from './guard.mjs';
|
|
5
5
|
import { autoGuides, firingGuides, matchGuide, renderGuide } from './guides.mjs';
|
|
@@ -11,7 +11,8 @@ import { closingQuestions, recallBeforeAsking } from './recall.mjs';
|
|
|
11
11
|
import { line } from './render.mjs';
|
|
12
12
|
import { shouldRun, spawnDetached } from './scribe.mjs';
|
|
13
13
|
import { tidyDue } from './dream.mjs';
|
|
14
|
-
import { clearInjections, contextBand, contextNudge, contextUse, currentProject, endSession, ensureSession, injectedSlugs, markInjected, pruneStates, recordTurn } from './sessions.mjs';
|
|
14
|
+
import { clearInjections, contextBand, contextNudge, contextUse, currentProject, endedThen, endSession, ensureSession, heardSession, injectedSlugs, markInjected, noteHeard, pruneStates, recordReply, recordTurn, replyText, turnText } from './sessions.mjs';
|
|
15
|
+
import { drainSpool, retrySetAside, spoolEvent } from './spool.mjs';
|
|
15
16
|
import { claim, renderWorkflow, withoutSumoCalls } from './workflows.mjs';
|
|
16
17
|
|
|
17
18
|
/**
|
|
@@ -118,14 +119,21 @@ function unchecked(db, e) {
|
|
|
118
119
|
return `<check>\nNot done yet: ${shown} ${were} edited and ${next}\n</check>`;
|
|
119
120
|
}
|
|
120
121
|
|
|
122
|
+
/** The memory pass as a session starts it by itself: finished conversations only, and nothing filed while filing is paused. */
|
|
123
|
+
const PASS = ['memory', 'run', '--auto'];
|
|
124
|
+
|
|
125
|
+
/** The ways a session starts with its context wiped: Claude Code's compact and clear, the chat's /new. */
|
|
126
|
+
const CARDS_GONE = new Set(['compact', 'clear', 'new']);
|
|
127
|
+
|
|
121
128
|
const HANDLERS = {
|
|
122
129
|
'session-start'(db, e, now) {
|
|
123
|
-
ensureSession(db, { id: e.sessionId, cwd: e.cwd, transcriptPath: e.transcriptPath, now });
|
|
124
|
-
// A fresh start wipes the cards from context; forgetting they were shown lets them come back.
|
|
125
|
-
|
|
130
|
+
ensureSession(db, { id: e.sessionId, harness: e.harness, cwd: e.cwd, transcriptPath: e.transcriptPath, now });
|
|
131
|
+
// A fresh start wipes the cards from context; forgetting they were shown lets them come back. Claude Code names its
|
|
132
|
+
// start startup, resume, clear or compact: a resume still holds the cards, a clear or a compaction does not.
|
|
133
|
+
if (CARDS_GONE.has(e.source)) clearInjections(db, e.sessionId);
|
|
126
134
|
|
|
127
|
-
// One memory pass: it
|
|
128
|
-
if (shouldRun(db, {
|
|
135
|
+
// One memory pass: it reads the conversations that have finished since the last one, and tidies when a week has gone by.
|
|
136
|
+
if (shouldRun(db, { now }) || tidyDue(db, now)) spawnDetached(PASS);
|
|
129
137
|
|
|
130
138
|
const last = getMeta(db, 'last_backup');
|
|
131
139
|
if (!last || Date.parse(now) - Date.parse(last) > BACKUP_EVERY_MS) {
|
|
@@ -161,8 +169,12 @@ const HANDLERS = {
|
|
|
161
169
|
},
|
|
162
170
|
|
|
163
171
|
prompt(db, e, now) {
|
|
164
|
-
ensureSession(db, { id: e.sessionId, cwd: e.cwd, transcriptPath: e.transcriptPath, now });
|
|
165
|
-
|
|
172
|
+
ensureSession(db, { id: e.sessionId, harness: e.harness, cwd: e.cwd, transcriptPath: e.transcriptPath, now });
|
|
173
|
+
const turn = recordTurn(db, { sessionId: e.sessionId, text: e.prompt, now });
|
|
174
|
+
if (turn === null) {
|
|
175
|
+
noteHeard(db, e.harness, now);
|
|
176
|
+
return '';
|
|
177
|
+
}
|
|
166
178
|
|
|
167
179
|
const shown = new Set(injectedSlugs(db, e.sessionId));
|
|
168
180
|
const cards = [];
|
|
@@ -183,6 +195,8 @@ const HANDLERS = {
|
|
|
183
195
|
// Its questions come with it, unless the block this session started with asked them: a session started inside the project.
|
|
184
196
|
cards.push(card(db, slug, { now, asks: slug !== (e.cwd ? projectAt(db, e.cwd) : null) }));
|
|
185
197
|
}
|
|
198
|
+
// The turn was stamped before its own words were read: one that names a project is about that project.
|
|
199
|
+
if (here || named.length > 0) db.prepare('UPDATE user_turns SET project = ? WHERE id = ?').run(currentProject(db, e.sessionId), turn);
|
|
186
200
|
|
|
187
201
|
const asked = claim(db, { sessionId: e.sessionId, text: e.prompt, scopes: scopesOf(db, e.sessionId), now });
|
|
188
202
|
if (asked.length > 0) {
|
|
@@ -204,7 +218,7 @@ const HANDLERS = {
|
|
|
204
218
|
'pre-tool'(db, e, now) {
|
|
205
219
|
// A question on its way to the user: what memory holds close to it is read first, and holds it back once.
|
|
206
220
|
if (e.toolName === 'ask') {
|
|
207
|
-
ensureSession(db, { id: e.sessionId, cwd: e.cwd, transcriptPath: e.transcriptPath, now });
|
|
221
|
+
ensureSession(db, { id: e.sessionId, harness: e.harness, cwd: e.cwd, transcriptPath: e.transcriptPath, now });
|
|
208
222
|
const known = recallBeforeAsking(db, { sessionId: e.sessionId, agentId: e.agentId, asked: e.question, now });
|
|
209
223
|
return known.length > 0 ? deny(`Not asked yet: ${alreadyKnown(known)} If none does, ask again.`) : '';
|
|
210
224
|
}
|
|
@@ -212,7 +226,7 @@ const HANDLERS = {
|
|
|
212
226
|
// Teaching or reading a workflow names its own cue; `sumo` must never be gated by what it manages — and nothing else rides through on it.
|
|
213
227
|
const judged = withoutSumoCalls(e.command);
|
|
214
228
|
if (!judged) return '';
|
|
215
|
-
ensureSession(db, { id: e.sessionId, cwd: e.cwd, transcriptPath: e.transcriptPath, now });
|
|
229
|
+
ensureSession(db, { id: e.sessionId, harness: e.harness, cwd: e.cwd, transcriptPath: e.transcriptPath, now });
|
|
216
230
|
|
|
217
231
|
const due = claim(db, { sessionId: e.sessionId, agentId: e.agentId, text: judged, scopes: [...scopesOf(db, e.sessionId), ...(e.project ? [`project:${e.project}`] : [])], forCommand: true, now });
|
|
218
232
|
if (due.length === 0) return '';
|
|
@@ -223,8 +237,9 @@ const HANDLERS = {
|
|
|
223
237
|
},
|
|
224
238
|
|
|
225
239
|
stop(db, e, now) {
|
|
226
|
-
ensureSession(db, { id: e.sessionId, cwd: e.cwd, transcriptPath: e.transcriptPath, now });
|
|
227
|
-
|
|
240
|
+
ensureSession(db, { id: e.sessionId, harness: e.harness, cwd: e.cwd, transcriptPath: e.transcriptPath, now });
|
|
241
|
+
// The journal first: what the assistant ended the turn on, whether or not a gate then sends the turn on.
|
|
242
|
+
if (e.reply) recordReply(db, { sessionId: e.sessionId, text: e.reply, now });
|
|
228
243
|
|
|
229
244
|
// The other gates: a turn that ends on an edit nothing checked, or by asking the user something memory may
|
|
230
245
|
// already hold. Already continuing because of one → whatever the turn ends on now goes to the user.
|
|
@@ -235,24 +250,89 @@ const HANDLERS = {
|
|
|
235
250
|
},
|
|
236
251
|
|
|
237
252
|
'session-end'(db, e, now) {
|
|
238
|
-
ensureSession(db, { id: e.sessionId, cwd: e.cwd, transcriptPath: e.transcriptPath, now });
|
|
253
|
+
ensureSession(db, { id: e.sessionId, harness: e.harness, cwd: e.cwd, transcriptPath: e.transcriptPath, now });
|
|
239
254
|
endSession(db, e.sessionId, now);
|
|
240
|
-
|
|
255
|
+
// The conversation is over: it is read for memory now, once.
|
|
256
|
+
if (shouldRun(db, { now })) spawnDetached(PASS);
|
|
241
257
|
return '';
|
|
242
258
|
},
|
|
243
259
|
};
|
|
244
260
|
|
|
245
|
-
/**
|
|
246
|
-
|
|
261
|
+
/**
|
|
262
|
+
* One event, in process: the guard first, then the handler. Throws only for an unknown event or a payload with no
|
|
263
|
+
* session. `harness` is who drives the session — sumo's own chat and jobs unless said; `runHook` says 'claude' — and
|
|
264
|
+
* is written on a session's row when it is first heard of. Events an earlier hook could not write are replayed first.
|
|
265
|
+
*/
|
|
266
|
+
export function handleEvent(db, event, payload, now = new Date().toISOString(), { harness = 'sumo' } = {}) {
|
|
247
267
|
const handler = HANDLERS[event];
|
|
248
268
|
if (!handler) throw new Error(`no handler for ${event}`);
|
|
249
|
-
const e = fieldsOf(payload);
|
|
269
|
+
const e = { ...fieldsOf(payload), harness };
|
|
250
270
|
if (!e.sessionId) throw new Error('the event had no session id');
|
|
251
271
|
const refused = event === 'pre-tool' ? earlyRefusal(e) : '';
|
|
252
272
|
if (refused) return deny(refused);
|
|
273
|
+
drain(db);
|
|
253
274
|
return handler(db, e, now);
|
|
254
275
|
}
|
|
255
276
|
|
|
277
|
+
/** The events with something for the journal: a session's row, a turn, a reply, a goodbye. A tool about to run has nothing. */
|
|
278
|
+
const JOURNALED = new Set(['session-start', 'prompt', 'stop', 'session-end']);
|
|
279
|
+
/** The harness `sumo hook` is wired into. */
|
|
280
|
+
const HOOKED = 'claude';
|
|
281
|
+
|
|
282
|
+
/**
|
|
283
|
+
* An event that could not be written, kept in the spool for the next one that gets through: only what the journal
|
|
284
|
+
* needs — who, when, where, and the words, redacted and cut as the journal would keep them. Never the tool's input, the
|
|
285
|
+
* gates or the cards: those were for the moment that has passed. A slash command or a turn that ended on nothing has
|
|
286
|
+
* nothing to keep.
|
|
287
|
+
*/
|
|
288
|
+
function keepForLater(event, payload, at) {
|
|
289
|
+
const e = fieldsOf(payload);
|
|
290
|
+
if (!JOURNALED.has(event) || !e.sessionId) return;
|
|
291
|
+
const said = event === 'prompt' ? turnText(e.prompt) : event === 'stop' ? replyText(e.reply) : null;
|
|
292
|
+
if ((event === 'prompt' || event === 'stop') && !said) return;
|
|
293
|
+
const text = (value) => (typeof value === 'string' ? value : null);
|
|
294
|
+
spoolEvent({ v: 1, at, event, harness: HOOKED, session_id: String(e.sessionId), cwd: text(e.cwd), transcript_path: text(e.transcriptPath), ...(said ? { text: said.text, full_len: said.fullLen, redacted: said.redacted } : {}) });
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
/** A line of the spool that is not an event: no later try would make it one, so it is set aside at once. */
|
|
298
|
+
function notAnEvent(why) {
|
|
299
|
+
return Object.assign(new Error(why), { bad: true });
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
/**
|
|
303
|
+
* One spooled event written into the journal with the time it happened: the session's row, and the turn, the reply or
|
|
304
|
+
* the goodbye. Nothing else an event does is done again — no card, no gate, no memory pass woken, no backup.
|
|
305
|
+
*/
|
|
306
|
+
function replay(db, line) {
|
|
307
|
+
if (line.v !== 1 || !JOURNALED.has(line.event)) throw notAnEvent('not an event this version replays');
|
|
308
|
+
if (typeof line.session_id !== 'string' || !line.session_id) throw notAnEvent('no session id');
|
|
309
|
+
const happened = typeof line.at === 'string' ? Date.parse(line.at) : NaN;
|
|
310
|
+
if (Number.isNaN(happened)) throw notAnEvent('no time it happened');
|
|
311
|
+
const at = new Date(happened).toISOString();
|
|
312
|
+
const text = (value) => (typeof value === 'string' ? value : null);
|
|
313
|
+
const kept = { text: line.text, fullLen: line.full_len, redacted: line.redacted };
|
|
314
|
+
if ((line.event === 'prompt' || line.event === 'stop') && typeof line.text !== 'string') throw notAnEvent('no words');
|
|
315
|
+
heardSession(db, { id: line.session_id, harness: text(line.harness) ?? HOOKED, cwd: text(line.cwd), transcriptPath: text(line.transcript_path), now: at });
|
|
316
|
+
if (line.event === 'prompt') recordTurn(db, { sessionId: line.session_id, now: at, kept });
|
|
317
|
+
if (line.event === 'stop') recordReply(db, { sessionId: line.session_id, now: at, kept });
|
|
318
|
+
if (line.event === 'session-end') endedThen(db, line.session_id, at);
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
/** What waits in the spool, into the journal, before the event in hand. It never stops that event: what went wrong is logged, and the lines wait on. */
|
|
322
|
+
function drain(db) {
|
|
323
|
+
try {
|
|
324
|
+
const done = drainSpool((line) => replay(db, line), { within: (fn) => tx(db, fn) });
|
|
325
|
+
if (!done) return;
|
|
326
|
+
const parts = [`${done.replayed} replayed`, done.retried > 0 ? `${done.retried} to try again` : '', done.bad > 0 ? `${done.bad} set aside in spool/events.bad.jsonl (sumo memory check counts them)` : ''].filter(Boolean);
|
|
327
|
+
appendLog('hook.log', `${new Date().toISOString()} spool: ${parts.join(', ')}\n`);
|
|
328
|
+
} catch (cause) {
|
|
329
|
+
logFailure('spool', cause);
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
/** The lines set aside in the spool tried again, for `sumo memory check --fix`: `{ replayed, left }`. */
|
|
334
|
+
export const replaySetAside = (db) => retrySetAside((line) => replay(db, line), { within: (fn) => tx(db, fn) });
|
|
335
|
+
|
|
256
336
|
/** What went wrong, in logs/hook.log. If even the log cannot be written there is nothing left to do but stay out of the way. */
|
|
257
337
|
function logFailure(what, cause) {
|
|
258
338
|
try {
|
|
@@ -266,21 +346,34 @@ function logFailure(what, cause) {
|
|
|
266
346
|
* One event from outside the process, JSON on stdin. It never throws and never
|
|
267
347
|
* exits non-zero: memory is a convenience, and a convenience that can break the
|
|
268
348
|
* session it serves is worse than none. Whatever goes wrong is written to
|
|
269
|
-
* logs/hook.log and the session simply carries on without this contribution
|
|
349
|
+
* logs/hook.log and the session simply carries on without this contribution —
|
|
350
|
+
* but what was said is not lost with it: an event the database could not take,
|
|
351
|
+
* or a handler broke on, waits in the spool for the next one that gets through.
|
|
270
352
|
*/
|
|
271
353
|
export function runHook(event, stdin) {
|
|
272
354
|
let db = null;
|
|
355
|
+
let payload = null;
|
|
356
|
+
const now = new Date().toISOString();
|
|
273
357
|
try {
|
|
274
|
-
|
|
358
|
+
payload = JSON.parse(stdin || '{}');
|
|
275
359
|
// The guard needs no database, and a database that cannot be opened must not stop it.
|
|
276
360
|
const refused = event === 'pre-tool' ? earlyRefusal(fieldsOf(payload)) : '';
|
|
277
361
|
if (refused) return deny(refused);
|
|
278
362
|
db = openDb();
|
|
279
|
-
return handleEvent(db, event, payload);
|
|
363
|
+
return handleEvent(db, event, payload, now, { harness: HOOKED });
|
|
280
364
|
} catch (cause) {
|
|
281
365
|
logFailure(event, cause);
|
|
366
|
+
try {
|
|
367
|
+
if (payload && typeof payload === 'object') keepForLater(event, payload, now);
|
|
368
|
+
} catch (unkept) {
|
|
369
|
+
logFailure(`${event} spool`, unkept);
|
|
370
|
+
}
|
|
282
371
|
return '';
|
|
283
372
|
} finally {
|
|
284
|
-
|
|
373
|
+
try {
|
|
374
|
+
db?.close();
|
|
375
|
+
} catch (cause) {
|
|
376
|
+
logFailure(`${event} close`, cause);
|
|
377
|
+
}
|
|
285
378
|
}
|
|
286
379
|
}
|
package/src/jobs.mjs
CHANGED
|
@@ -15,6 +15,7 @@ import { preferences } from './prime.mjs';
|
|
|
15
15
|
import { getProject, isScratch } from './projects.mjs';
|
|
16
16
|
import { chooseRoute, RETRY_IMPORTANT_THRESHOLD, statsLines } from './route.mjs';
|
|
17
17
|
import { verifyCommands } from './scan.mjs';
|
|
18
|
+
import { filingState } from './ledger.mjs';
|
|
18
19
|
import { runScribe } from './scribe.mjs';
|
|
19
20
|
import { addCheckpoint, pendingTurns } from './sessions.mjs';
|
|
20
21
|
import { CONFIG_DEFAULTS } from './setup.mjs';
|
|
@@ -377,8 +378,9 @@ export async function newJob(db, { project: nameOrAlias, title, agent = 'worker'
|
|
|
377
378
|
const guide = chosen ? guideFor(chosen) : null;
|
|
378
379
|
const judged = agent === 'reviewer' ? changeToJudge(db, { project, reviews }) : null;
|
|
379
380
|
|
|
380
|
-
// A rule the user stated a minute ago may not be filed yet, and the brief is built from memory
|
|
381
|
-
|
|
381
|
+
// A rule the user stated a minute ago may not be filed yet, and the brief is built from memory: the conversation
|
|
382
|
+
// still going on is read now. Not while filing is paused — a job starting is not the user asking for it by hand.
|
|
383
|
+
if (pendingTurns(db, 1).length > 0 && !filingState(db, now).paused) await runScribe(db, { now });
|
|
382
384
|
|
|
383
385
|
const route = routed ?? (await chooseRoute(db, { agent, project, task, title: title.trim(), retryOf, reviews }));
|
|
384
386
|
|
package/src/keeper.mjs
CHANGED
|
@@ -1,10 +1,6 @@
|
|
|
1
1
|
import { getMeta, setMeta } from './db.mjs';
|
|
2
|
-
import { runDream,
|
|
3
|
-
import {
|
|
4
|
-
import { memoryHealth, memoryHealthLine } from './memory.mjs';
|
|
5
|
-
import { modelStats, outcomeLines, runScribe } from './scribe.mjs';
|
|
6
|
-
import { pendingTurns, sessionsAwaitingDream } from './sessions.mjs';
|
|
7
|
-
import { CONFIG_DEFAULTS, modelFileOf } from './setup.mjs';
|
|
2
|
+
import { runDream, tidyDue } from './dream.mjs';
|
|
3
|
+
import { outcomeLines, runScribe, withLock } from './scribe.mjs';
|
|
8
4
|
|
|
9
5
|
/** When the last tidy failed; until TIDY_RETRY_MS has gone by, a pass that finds a tidy due files only. */
|
|
10
6
|
const TIDY_FAILED = 'memory.tidy_failed_at';
|
|
@@ -12,19 +8,20 @@ const TIDY_RETRY_MS = 6 * 3_600_000;
|
|
|
12
8
|
|
|
13
9
|
/**
|
|
14
10
|
* The memory pass: the one thing that looks after memory, so the user has one name, one status and one setting
|
|
15
|
-
* (`memory.model`) instead of a writer and a tidier to keep track of. Each pass files what
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
11
|
+
* (`memory.model`) instead of a writer and a tidier to keep track of. Each pass files what the user stated in the
|
|
12
|
+
* conversations that have finished since the last one; when the week is up the same pass then tidies memory itself:
|
|
13
|
+
* raises pairs that say one thing or disagree, fills in search terms. A session starting or ending runs it by itself
|
|
14
|
+
* (`auto`): finished conversations only, and no filing while filing is paused. `sumo memory run` runs it now — also
|
|
15
|
+
* over conversations still going on, and through a pause — and `--tidy` tidies now whatever is due. A tidy that failed
|
|
16
|
+
* is not tried again by itself for six hours: a local model that is down must not turn every pass into a failed tidy.
|
|
17
|
+
* Returns what each half did: `{ filed, tidied }`, `tidied` null when it did not tidy.
|
|
21
18
|
*/
|
|
22
|
-
export async function runMemoryPass(db, { now = new Date().toISOString(), tidy = false } = {}) {
|
|
23
|
-
const filed = await runScribe(db, { now });
|
|
19
|
+
export async function runMemoryPass(db, { now = new Date().toISOString(), tidy = false, auto = false } = {}) {
|
|
20
|
+
const filed = await runScribe(db, { now, auto });
|
|
24
21
|
const failedAt = getMeta(db, TIDY_FAILED);
|
|
25
22
|
const resting = failedAt && Date.parse(now) - Date.parse(failedAt) < TIDY_RETRY_MS;
|
|
26
23
|
if (!tidy && (resting || !tidyDue(db, now))) return { filed, tidied: null };
|
|
27
|
-
const tidied = await runDream(db, { now
|
|
24
|
+
const tidied = await withLock('dream', () => runDream(db, { now }));
|
|
28
25
|
if (tidied.ok === false && !tidied.skipped) setMeta(db, TIDY_FAILED, now);
|
|
29
26
|
else if (failedAt) setMeta(db, TIDY_FAILED, '');
|
|
30
27
|
return { filed, tidied };
|
|
@@ -36,42 +33,8 @@ export async function runMemoryPass(db, { now = new Date().toISOString(), tidy =
|
|
|
36
33
|
*/
|
|
37
34
|
export function passLines({ filed, tidied }, { asked = false } = {}) {
|
|
38
35
|
const tidying = tidied ? outcomeLines(tidied).map((l, i) => (i === 0 ? `tidy: ${l}` : l)) : [];
|
|
39
|
-
|
|
36
|
+
// A filing that is paused is said even beside a tidy: it is not "nothing to do".
|
|
37
|
+
const lines = [...(tidied && filed.skipped && !filed.paused ? [] : outcomeLines(filed)), ...tidying];
|
|
40
38
|
const failed = (filed.ok === false && !filed.skipped) || (asked && tidied?.ok === false && !tidied.skipped);
|
|
41
39
|
return failed ? Object.assign(lines, { exitCode: 1 }) : lines;
|
|
42
40
|
}
|
|
43
|
-
|
|
44
|
-
/** `sumo memory status`: the pass as a whole — the model it runs on, what waits, how it last went — then the numbers. */
|
|
45
|
-
/** The ledger's names for the pass's two halves, as the status says them. */
|
|
46
|
-
const HALVES = { scribe: 'filing', dream: 'tidying' };
|
|
47
|
-
/** A status line: its label in a column of its own, so the values stand in one. */
|
|
48
|
-
const row = (label, value) => `${label.padEnd(LABEL_WIDTH)}${value}`;
|
|
49
|
-
const LABEL_WIDTH = 11;
|
|
50
|
-
|
|
51
|
-
export async function memoryStatus(db, now = new Date().toISOString()) {
|
|
52
|
-
const health = memoryHealthLine(memoryHealth(db, now), now);
|
|
53
|
-
const model = getMeta(db, 'config.memory.model') ?? CONFIG_DEFAULTS['memory.model'];
|
|
54
|
-
const server = model === 'local' ? await localServerStatus(db, { file: modelFileOf(db) }) : null;
|
|
55
|
-
const turns = pendingTurns(db, 1000).length;
|
|
56
|
-
const finished = sessionsAwaitingDream(db, now, 1000).length;
|
|
57
|
-
const tidied = db.prepare(`SELECT MAX(ts) AS at FROM model_runs WHERE kind = 'dream' AND ok = 1`).get()?.at ?? null;
|
|
58
|
-
const sweep = getMeta(db, 'dream.last_tidy');
|
|
59
|
-
// Due with fewer than three conversations waiting: the weekly sweep is what is owed.
|
|
60
|
-
const owed = finished < SESSIONS_TO_TIDY && tidyDue(db, now);
|
|
61
|
-
const failures = Number(getMeta(db, 'scribe.failures') ?? 0);
|
|
62
|
-
const failedAt = getMeta(db, TIDY_FAILED);
|
|
63
|
-
const day = (at) => (at ? at.slice(0, 10) : 'never');
|
|
64
|
-
// The ledger holds every request; the pass's own are its filing and its tidying, named so.
|
|
65
|
-
const spent = modelStats(db)
|
|
66
|
-
.filter((l) => /^(scribe|dream): /.test(l))
|
|
67
|
-
.map((l) => l.replace(/^(scribe|dream)/, (kind) => HALVES[kind]));
|
|
68
|
-
return [
|
|
69
|
-
`${health.ok ? 'ok ' : 'warn'} ${health.label}${health.fix ? ` — ${health.fix}` : ''}`,
|
|
70
|
-
row('model', `${model}${server ? ` · ${server}` : ''}`),
|
|
71
|
-
row('waiting', `${turns} turn${turns === 1 ? '' : 's'} to file · ${finished} finished conversation${finished === 1 ? '' : 's'} to read (tidied by itself at ${SESSIONS_TO_TIDY}, and weekly)`),
|
|
72
|
-
row('last tidy', `${day(tidied)} · weekly sweep: ${day(sweep)}${owed ? ' — owed: the next pass sweeps' : ''}`),
|
|
73
|
-
...(failures > 0 ? [row('failing', `filing failed ${failures} time${failures === 1 ? '' : 's'} in a row — ${getMeta(db, 'scribe.last_error')}`)] : []),
|
|
74
|
-
...(failedAt ? [row('failing', `the last tidy, at ${failedAt.slice(0, 16)}; tried again by itself ${Date.parse(now) - Date.parse(failedAt) < TIDY_RETRY_MS ? 'six hours on' : 'with the next pass'}, or now: sumo memory run --tidy`)] : []),
|
|
75
|
-
...(spent.length > 0 ? [row('spent', spent[0]), ...spent.slice(1).map((l) => row('', l))] : []),
|
|
76
|
-
];
|
|
77
|
-
}
|