sumo-agents 0.6.1 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +6 -5
- package/CHANGELOG.md +72 -0
- package/README.md +47 -34
- package/guides/memory.md +21 -14
- package/package.json +2 -2
- package/prompts/dream.md +3 -16
- package/prompts/scribe.md +20 -15
- package/src/apply.mjs +364 -230
- package/src/card.mjs +16 -11
- package/src/catalog.mjs +3 -3
- package/src/chat.mjs +53 -15
- package/src/check.mjs +371 -0
- package/src/cli.mjs +230 -146
- package/src/db.mjs +147 -2
- package/src/dream.mjs +97 -189
- package/src/export.mjs +43 -18
- package/src/herdr.mjs +71 -36
- package/src/hooks.mjs +115 -22
- package/src/jobs.mjs +4 -2
- package/src/keeper.mjs +14 -51
- package/src/ledger.mjs +126 -0
- package/src/logs.mjs +10 -5
- package/src/loop.mjs +4 -4
- package/src/mcp.mjs +129 -51
- package/src/memeval.mjs +167 -0
- package/src/memory.mjs +261 -130
- package/src/page.mjs +3 -3
- package/src/paths.mjs +1 -0
- package/src/prime.mjs +82 -13
- package/src/rebuild.mjs +162 -0
- package/src/render.mjs +22 -0
- package/src/route.mjs +1 -1
- package/src/sandbox.mjs +5 -2
- package/src/scan.mjs +16 -4
- package/src/scribe.mjs +361 -113
- package/src/search.mjs +48 -0
- package/src/sessions.mjs +304 -53
- package/src/settings.mjs +868 -0
- package/src/setup.mjs +56 -64
- package/src/spool.mjs +308 -0
- package/src/text.mjs +118 -6
- package/src/tools.mjs +18 -7
- package/src/transcript.mjs +3 -47
- package/src/tty.mjs +48 -23
- package/src/ui.mjs +62 -25
- package/src/verify.mjs +23 -27
package/src/setup.mjs
CHANGED
|
@@ -5,7 +5,6 @@ import {
|
|
|
5
5
|
} from 'node:fs';
|
|
6
6
|
import { homedir, hostname } from 'node:os';
|
|
7
7
|
import { basename, delimiter, dirname, isAbsolute, join } from 'node:path';
|
|
8
|
-
import { createInterface } from 'node:readline/promises';
|
|
9
8
|
import { pipeline } from 'node:stream/promises';
|
|
10
9
|
import { Readable, Transform } from 'node:stream';
|
|
11
10
|
import { anthropicClient, authenticatedRequest, FALLBACK_BETA, resolveAnthropicCredential } from './auth.mjs';
|
|
@@ -14,7 +13,7 @@ import { getMeta, openDb, SCHEMA_VERSION, schemaVersion, setMeta } from './db.mj
|
|
|
14
13
|
import { damaged, restoreHint } from './export.mjs';
|
|
15
14
|
import { stopLocalServer } from './local-server.mjs';
|
|
16
15
|
import { latestCrash, crashFiles, logFiles, tailOf } from './logs.mjs';
|
|
17
|
-
import {
|
|
16
|
+
import { Refusal, UsageError } from './memory.mjs';
|
|
18
17
|
import { COMPILED, ENTRY, homeProblem, paths, REPO_ROOT, VERSION } from './paths.mjs';
|
|
19
18
|
import { redact } from './redact.mjs';
|
|
20
19
|
import { unsandboxable } from './sandbox.mjs';
|
|
@@ -22,11 +21,11 @@ import { DatabaseSync } from './sqlite.mjs';
|
|
|
22
21
|
import { closest } from './text.mjs';
|
|
23
22
|
import { tilde } from './tty.mjs';
|
|
24
23
|
|
|
25
|
-
const NODE_REQUIREMENT = '
|
|
24
|
+
const NODE_REQUIREMENT = '24.6+';
|
|
26
25
|
// route.mjs's EFFORTS, spelled out: importing it would close a cycle (route → model → setup). A test keeps them equal.
|
|
27
26
|
export const CHAT_EFFORTS = ['low', 'medium', 'high', 'xhigh', 'max'];
|
|
28
27
|
|
|
29
|
-
/** The settings `sumo config`
|
|
28
|
+
/** The settings the settings page shows and `sumo config` prints, with the value used when none is stored. */
|
|
30
29
|
export const CONFIG_DEFAULTS = {
|
|
31
30
|
'chat.budget': '5',
|
|
32
31
|
'chat.effort': 'medium',
|
|
@@ -47,11 +46,11 @@ export const CONFIG_DEFAULTS = {
|
|
|
47
46
|
|
|
48
47
|
/** What a setting that is gone became, said when its old name is used, instead of "unknown setting". */
|
|
49
48
|
const RETIRED = {
|
|
50
|
-
'chat.motion': 'chat.motion is now chat.screen
|
|
51
|
-
'chat.plain': 'chat.plain is now chat.screen
|
|
52
|
-
'model.source': 'model.source is now part of model.url, the model file\'s whole address
|
|
53
|
-
'model.repo': 'model.repo is now part of model.url, the model file\'s whole address
|
|
54
|
-
'model.file': 'model.file is now part of model.url, the model file\'s whole address
|
|
49
|
+
'chat.motion': 'chat.motion is now chat.screen — Chat › Screen, still, on the settings page (sumo config)',
|
|
50
|
+
'chat.plain': 'chat.plain is now chat.screen — Chat › Screen, plain, on the settings page (sumo config)',
|
|
51
|
+
'model.source': 'model.source is now part of model.url, the model file\'s whole address — Memory › Local model, on the settings page (sumo config)',
|
|
52
|
+
'model.repo': 'model.repo is now part of model.url, the model file\'s whole address — Memory › Local model, on the settings page (sumo config)',
|
|
53
|
+
'model.file': 'model.file is now part of model.url, the model file\'s whole address — Memory › Local model, on the settings page (sumo config)',
|
|
55
54
|
'prime.budget': 'prime.budget is no longer a setting: the memory block every session starts with holds 800 tokens; the rest is found by search',
|
|
56
55
|
'scratch.days': 'scratch.days is no longer a setting: a scratch folder nothing is written in for 14 days is deleted',
|
|
57
56
|
'jobs.escalate': 'jobs.escalate is no longer a setting: a worker\'s first attempt that fails is always retried once, one step up',
|
|
@@ -134,7 +133,7 @@ export const modelFileOfUrl = (url = CONFIG_DEFAULTS['model.url']) => new URL(ur
|
|
|
134
133
|
export const modelFileOf = (db) => modelFileOfUrl(modelUrlOf(db));
|
|
135
134
|
|
|
136
135
|
/**
|
|
137
|
-
* The address with another source in front:
|
|
136
|
+
* The address with another source in front: `setup({ modelSource })` names a host, or a JFrog
|
|
138
137
|
* Artifactory Hugging Face remote's huggingfaceml URL, and the file keeps its path there (`<org>/<repo>/resolve/main/<file>`
|
|
139
138
|
* in Hugging Face's layout, else the file alone). A source that is itself a .gguf address is the whole answer.
|
|
140
139
|
*/
|
|
@@ -145,12 +144,6 @@ function withSource(url, source) {
|
|
|
145
144
|
return `${source.replace(/\/+$/, '')}/${tail}`;
|
|
146
145
|
}
|
|
147
146
|
|
|
148
|
-
/** The source part of an address, what setup's question offers to keep. */
|
|
149
|
-
function sourceOf(url) {
|
|
150
|
-
const parts = url.split('/');
|
|
151
|
-
return parts.slice(0, parts.at(-3) === 'resolve' ? -5 : -1).join('/');
|
|
152
|
-
}
|
|
153
|
-
|
|
154
147
|
/**
|
|
155
148
|
* Downloads the router model unless it is already there at the size the server reports (HEAD first).
|
|
156
149
|
* Streams to `<file>.part` then renames, so a run that dies never leaves a file that looks finished.
|
|
@@ -186,7 +179,7 @@ async function ensureModel(db) {
|
|
|
186
179
|
const why = redactUrl(cause.message);
|
|
187
180
|
// Offline, with the model already here: that is a model that could not be checked, not one that is missing.
|
|
188
181
|
if (existsSync(dest)) return `model ${dest} (${formatBytes(statSync(dest).size)}, present; not checked against ${shown}: ${why})`;
|
|
189
|
-
return `could not download the router model from ${shown}: ${why}`;
|
|
182
|
+
return `could not download the router model from ${shown}: ${why} — another address: Memory › Local model, on the settings page (sumo config)`;
|
|
190
183
|
}
|
|
191
184
|
}
|
|
192
185
|
|
|
@@ -243,28 +236,18 @@ function modelUrlOrRefuse(value) {
|
|
|
243
236
|
return value;
|
|
244
237
|
}
|
|
245
238
|
|
|
246
|
-
/**
|
|
247
|
-
|
|
248
|
-
|
|
239
|
+
/**
|
|
240
|
+
* The local model fetched again for the address the settings page now holds, and the server that ran the old one
|
|
241
|
+
* stopped, so the next local call starts on the new file. Never throws: what happened is the line returned.
|
|
242
|
+
*/
|
|
243
|
+
export async function fetchModel() {
|
|
244
|
+
const db = openDb();
|
|
249
245
|
try {
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
const answer = (
|
|
254
|
-
await rl.question(`Download the router model from [${defaultSource}]: `).catch((cause) => {
|
|
255
|
-
if (cause?.name === 'AbortError') throw new UsageError('setup stopped — nothing was downloaded');
|
|
256
|
-
return '';
|
|
257
|
-
})
|
|
258
|
-
).trim();
|
|
259
|
-
if (!answer) return defaultSource;
|
|
260
|
-
try {
|
|
261
|
-
return sourceOrRefuse(answer);
|
|
262
|
-
} catch (cause) {
|
|
263
|
-
process.stdout.write(`${cause.message}\n`);
|
|
264
|
-
}
|
|
265
|
-
}
|
|
246
|
+
const line = await ensureModel(db);
|
|
247
|
+
stopLocalServer(db);
|
|
248
|
+
return line;
|
|
266
249
|
} finally {
|
|
267
|
-
|
|
250
|
+
db.close();
|
|
268
251
|
}
|
|
269
252
|
}
|
|
270
253
|
|
|
@@ -284,19 +267,17 @@ export async function setup({ binDir, link = true, modelSource, noModel = false
|
|
|
284
267
|
const llama = commandOnPath('llama-server');
|
|
285
268
|
if (llama) setMeta(db, 'llama.path', llama);
|
|
286
269
|
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
source = await askModelSource(sourceOf(CONFIG_DEFAULTS['model.url']));
|
|
290
|
-
}
|
|
270
|
+
// Setup asks nothing: the default address is fetched, and another is set on the settings page, which fetches it then.
|
|
271
|
+
const source = modelSource === undefined ? undefined : sourceOrRefuse(modelSource);
|
|
291
272
|
if (source !== undefined) setMeta(db, 'config.model.url', modelUrlOrRefuse(withSource(modelUrlOf(db), source)));
|
|
292
273
|
|
|
293
274
|
// SUMO_AGENTS_MODEL_CMD marks a hermetic test/dev harness, the same way it tells callModel and
|
|
294
|
-
// callLocalModel never to run a real model — an explicit
|
|
275
|
+
// callLocalModel never to run a real model — an explicit modelSource still means "go fetch it".
|
|
295
276
|
const harnessSkip = Boolean(process.env.SUMO_AGENTS_MODEL_CMD) && modelSource === undefined;
|
|
296
277
|
let modelLine = null;
|
|
297
278
|
if (!noModel) {
|
|
298
279
|
modelLine = harnessSkip
|
|
299
|
-
? 'model not downloaded — SUMO_AGENTS_MODEL_CMD is set (unset it
|
|
280
|
+
? 'model not downloaded — SUMO_AGENTS_MODEL_CMD is set (unset it to fetch it)'
|
|
300
281
|
: await ensureModel(db);
|
|
301
282
|
}
|
|
302
283
|
const modelsLine = await checkModels(db);
|
|
@@ -330,12 +311,12 @@ export async function setup({ binDir, link = true, modelSource, noModel = false
|
|
|
330
311
|
|
|
331
312
|
/**
|
|
332
313
|
* Which of Sumo's models this credential can use, asked of the API the first time only: a later setup
|
|
333
|
-
* leaves the switches as they are, since the user may have set some by hand —
|
|
314
|
+
* leaves the switches as they are, since the user may have set some by hand — API models › Discover, on the settings page, asks again.
|
|
334
315
|
* Never throws: a check that could not be made is not a reason for `sumo setup` to fail.
|
|
335
316
|
*/
|
|
336
317
|
async function checkModels(db) {
|
|
337
318
|
const checked = modelsChecked(db);
|
|
338
|
-
if (checked) return `models checked ${checked.slice(0, 10)} —
|
|
319
|
+
if (checked) return `models checked ${checked.slice(0, 10)} — API models › Discover, on the settings page (sumo config), checks again`;
|
|
339
320
|
const result = await discoverModels(db);
|
|
340
321
|
return result.ok ? `models ${discoverySummary(result)}` : `models not checked — ${result.error}`;
|
|
341
322
|
}
|
|
@@ -366,7 +347,7 @@ export async function doctor() {
|
|
|
366
347
|
check(code !== null, `code ${code ?? 'unknown'}`, 'neither a git checkout nor an installed package — reinstall: npm i -g sumo-agents', true);
|
|
367
348
|
|
|
368
349
|
const [major, minor] = process.versions.node.split('.').map(Number);
|
|
369
|
-
const nodeOk = (major ===
|
|
350
|
+
const nodeOk = (major === 24 && minor >= 6) || major > 24;
|
|
370
351
|
// The binary Bun builds carries its own runtime, whatever Node version that says it is.
|
|
371
352
|
if (process.versions.bun) check(true, `bun ${process.versions.bun} (a built binary: no Node needed)`, '');
|
|
372
353
|
else check(nodeOk, `node ${process.versions.node}`, `needs Node ${NODE_REQUIREMENT}`);
|
|
@@ -394,8 +375,11 @@ export async function doctor() {
|
|
|
394
375
|
checked: modelsChecked(db),
|
|
395
376
|
configured: ['chat.model', 'memory.model'].map((key) => [key, setting(key)]),
|
|
396
377
|
sandbox: setting('shell.sandbox'),
|
|
397
|
-
memory: memoryHealth(db),
|
|
398
378
|
};
|
|
379
|
+
// Loaded here: the hooks reach this module for its settings, and only doctor needs the checks.
|
|
380
|
+
const { memoryCheck, memoryHealth, memoryHealthLine } = await import('./check.mjs');
|
|
381
|
+
stored.memory = memoryHealthLine(memoryHealth(db));
|
|
382
|
+
stored.checks = memoryCheck(db);
|
|
399
383
|
} catch (cause) {
|
|
400
384
|
check(false, 'database opens', damaged(cause) ? `${cause.message} — ${restoreHint()}` : cause.message);
|
|
401
385
|
} finally {
|
|
@@ -406,10 +390,10 @@ export async function doctor() {
|
|
|
406
390
|
if (stored.models) {
|
|
407
391
|
const on = MODELS.filter((name) => stored.models[name].on);
|
|
408
392
|
const off = MODELS.filter((name) => !stored.models[name].on);
|
|
409
|
-
check(on.length > 0, `models on: ${on.join(', ') || 'none'}${off.length > 0 ? `; off: ${off.join(', ')}` : ''}`, '
|
|
410
|
-
check(stored.checked !== null, `models checked against the API${stored.checked ? ` (${stored.checked.slice(0, 10)})` : ''}`, '
|
|
393
|
+
check(on.length > 0, `models on: ${on.join(', ') || 'none'}${off.length > 0 ? `; off: ${off.join(', ')}` : ''}`, 'turn one on under API models, on the settings page: sumo config');
|
|
394
|
+
check(stored.checked !== null, `models checked against the API${stored.checked ? ` (${stored.checked.slice(0, 10)})` : ''}`, 'API models › Discover, on the settings page: sumo config', true);
|
|
411
395
|
for (const [key, value] of stored.configured) {
|
|
412
|
-
if (MODELS.includes(value)) check(stored.models[value].on, `${key} is on (${value})`, `
|
|
396
|
+
if (MODELS.includes(value)) check(stored.models[value].on, `${key} is on (${value})`, `turn ${value} on under API models, or pick another model, on the settings page: sumo config`);
|
|
413
397
|
}
|
|
414
398
|
}
|
|
415
399
|
|
|
@@ -433,28 +417,36 @@ export async function doctor() {
|
|
|
433
417
|
const crash = latestCrash(new Date(Date.now() - CRASH_SHOWN_MS));
|
|
434
418
|
if (crash) check(false, `crash on ${crash.at.toISOString().slice(0, 19).replace('T', ' ')}: ${crash.said}`, `${crash.file}; sumo doctor --report bundles it for a bug report`, true);
|
|
435
419
|
|
|
436
|
-
// Memory, in one line: whether the user is being heard and filed,
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
}
|
|
420
|
+
// Memory, in one line: whether the user is being heard and filed, and scans gone stale. Claude Code in use and not
|
|
421
|
+
// heard is a failure — capture once stopped for 35 days behind a line marked ok; silence alone is not. Then each
|
|
422
|
+
// check of the layer itself.
|
|
423
|
+
if (stored.memory) check(stored.memory.ok, stored.memory.label, stored.memory.fix, !stored.memory.fail);
|
|
424
|
+
for (const c of stored.checks ?? []) check(c.ok, `memory: ${c.label}`, c.fix);
|
|
441
425
|
|
|
442
426
|
// The shell's sandbox is on by default: where it cannot be, every command writes wherever the user can, and that is said.
|
|
443
427
|
if (stored.sandbox !== 'off') {
|
|
444
428
|
const why = unsandboxable();
|
|
445
|
-
check(why === null, 'shell sandbox (/usr/bin/sandbox-exec)', `${why} — the model's shell can write wherever you can; sumo config
|
|
429
|
+
check(why === null, 'shell sandbox (/usr/bin/sandbox-exec)', `${why} — the model's shell can write wherever you can; turning Sandbox off on the settings page (sumo config) says you know`, true);
|
|
446
430
|
}
|
|
447
431
|
check(commandOnPath('git') !== null, 'git on PATH', "install git — a job's worktree, its changes and the checks that judge it need it", true);
|
|
448
|
-
//
|
|
449
|
-
|
|
432
|
+
// The MCP servers moved from mcp.json into the database the first time anything read them — asked for here, so that
|
|
433
|
+
// has happened — and a file that would not read was left where it was: said, since its servers are not in use.
|
|
434
|
+
if (stored.models) {
|
|
450
435
|
const { readConfig } = await import('./mcp.mjs');
|
|
451
436
|
let unread = null;
|
|
452
437
|
try {
|
|
453
438
|
readConfig();
|
|
439
|
+
const db = openDb();
|
|
440
|
+
try {
|
|
441
|
+
unread = getMeta(db, 'mcp.import_failed');
|
|
442
|
+
} finally {
|
|
443
|
+
db.close();
|
|
444
|
+
}
|
|
454
445
|
} catch (cause) {
|
|
455
446
|
unread = cause.message;
|
|
456
447
|
}
|
|
457
|
-
|
|
448
|
+
if (unread) check(false, 'MCP servers moved from mcp.json', `${unread} — add them on the settings page (sumo config), then delete the file`);
|
|
449
|
+
else if (readdirSync(p.home).some((name) => name.startsWith('mcp.json.imported'))) check(true, 'MCP servers moved from mcp.json — mcp.json.imported is a copy nothing reads; delete it when you like');
|
|
458
450
|
}
|
|
459
451
|
|
|
460
452
|
const launcherOk = existsSync(p.launcher) && readFileSync(p.launcher, 'utf8') === launcherScript();
|
|
@@ -555,11 +547,11 @@ const LOOSE_SHOWN = 5;
|
|
|
555
547
|
/** How far back doctor looks for a crash to say. */
|
|
556
548
|
const CRASH_SHOWN_MS = 7 * 24 * 60 * 60 * 1000;
|
|
557
549
|
/** The folders of the home whose every file and folder holds the user's words or settings. */
|
|
558
|
-
const PRIVATE_TREES = ['logs', 'backups', 'scribe'];
|
|
550
|
+
const PRIVATE_TREES = ['logs', 'backups', 'scribe', 'spool'];
|
|
559
551
|
|
|
560
552
|
/**
|
|
561
|
-
* What in the home others can read, by full path: the home and its own files; everything in the logs, the backups
|
|
562
|
-
* the scribe's folder; the jobs folder, each job's folder and the files directly in it — never deeper, where a job's
|
|
553
|
+
* What in the home others can read, by full path: the home and its own files; everything in the logs, the backups,
|
|
554
|
+
* the scribe's folder and the spool; the jobs folder, each job's folder and the files directly in it — never deeper, where a job's
|
|
563
555
|
* worktree holds a copy of a project whose files keep their own modes. The models, the launcher's folder and scratch (a
|
|
564
556
|
* project) are not the user's words. Only group and other bits count: what the owner may do is the owner's.
|
|
565
557
|
*/
|
|
@@ -783,8 +775,8 @@ function launcherRuns() {
|
|
|
783
775
|
}
|
|
784
776
|
|
|
785
777
|
/** What sumo makes in its home, by name: an uninstall removes these and leaves anything else where it is. */
|
|
786
|
-
const MADE_HERE = new Set(['memory.db', 'memory.db-wal', 'memory.db-shm', 'backups', 'bin', 'logs', 'jobs', 'models', 'scratch', 'history', 'mcp.json', 'scribe', 'sumo.db']);
|
|
787
|
-
const madeHere = (name) => MADE_HERE.has(name) || /^memory\.db\.(?:pre-v\d+\.bak|damaged-|replaced-)/.test(name);
|
|
778
|
+
const MADE_HERE = new Set(['memory.db', 'memory.db-wal', 'memory.db-shm', 'backups', 'bin', 'logs', 'jobs', 'models', 'scratch', 'history', 'mcp.json', 'scribe', 'spool', 'sumo.db']);
|
|
779
|
+
const madeHere = (name) => MADE_HERE.has(name) || /^memory\.db\.(?:pre-v\d+\.bak|damaged-|replaced-)/.test(name) || /^mcp\.json\.imported(?:-\d+)?$/.test(name);
|
|
788
780
|
|
|
789
781
|
/** The local model's servers for this home: every llama-server whose command line names its models folder. */
|
|
790
782
|
function serversFor(home) {
|
package/src/spool.mjs
ADDED
|
@@ -0,0 +1,308 @@
|
|
|
1
|
+
import { appendFileSync, chmodSync, closeSync, fchmodSync, fstatSync, mkdirSync, openSync, readdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { alive } from './lock.mjs';
|
|
4
|
+
import { paths } from './paths.mjs';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Where an event waits when the journal could not take it: a hook that found the database busy, or broke part-way,
|
|
8
|
+
* appends the event here as one line of JSON, and the next event that gets through replays it. Without this a failed
|
|
9
|
+
* hook lost the turn for good and nobody was told.
|
|
10
|
+
*
|
|
11
|
+
* `events.jsonl` is what waits. A drain takes it by renaming it to a name of its own (`events.<pid>.<n>.draining`):
|
|
12
|
+
* one process gets it, the other finds nothing there, and an event arriving meanwhile starts a new file. A line that
|
|
13
|
+
* is not an event, or that failed to replay three times, goes to `events.bad.jsonl` — kept and counted, never tried
|
|
14
|
+
* for ever and never dropped. A database that is only busy is no failure, and a failure counts once a minute at most:
|
|
15
|
+
* every hook drains, a tool about to run included, and a write lock held for thirteen seconds once set a turn aside
|
|
16
|
+
* for good. `sumo memory check --fix` tries what was set aside again (`retrySetAside`). Everything here is private (the folder 0700, each file 0600) and holds only words that
|
|
17
|
+
* were redacted before they were written.
|
|
18
|
+
*/
|
|
19
|
+
const WAITING = 'events.jsonl';
|
|
20
|
+
const BAD = 'events.bad.jsonl';
|
|
21
|
+
const DRAINING = /^events\.(\d+)\.\d+\.draining$/;
|
|
22
|
+
/** How many times a line may fail to replay before it is set aside. */
|
|
23
|
+
const TRIES = 3;
|
|
24
|
+
/** Failures of one line closer together than this are one failure: the hooks of one busy minute are not three tries. */
|
|
25
|
+
const COUNTED_EVERY_MS = 60_000;
|
|
26
|
+
/** A drain's file this old was left by a process that died, whatever now runs under its number. */
|
|
27
|
+
const LEFT_AFTER_MS = 10 * 60_000;
|
|
28
|
+
/** Past this the spool takes no more: a database that never opens again must not fill the disk one prompt at a time. */
|
|
29
|
+
const SPOOL_MAX_BYTES = 64 * 1024 * 1024;
|
|
30
|
+
|
|
31
|
+
let claims = 0;
|
|
32
|
+
|
|
33
|
+
/** A line added to a file of the spool, the folder and the file made private — also when they were there before, and were not. */
|
|
34
|
+
function addLine(name, line) {
|
|
35
|
+
const dir = paths().spool;
|
|
36
|
+
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
37
|
+
if (statSync(dir).mode & 0o077) chmodSync(dir, 0o700);
|
|
38
|
+
const file = join(dir, name);
|
|
39
|
+
// A mode is given only to a file being made: one already there and readable by others is closed before more is added,
|
|
40
|
+
// through the descriptor the line goes in by, so the file made private is the file written.
|
|
41
|
+
const fd = openSync(file, 'a', 0o600);
|
|
42
|
+
try {
|
|
43
|
+
if (fstatSync(fd).mode & 0o077) fchmodSync(fd, 0o600);
|
|
44
|
+
appendFileSync(fd, `${line}\n`);
|
|
45
|
+
} finally {
|
|
46
|
+
closeSync(fd);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Keeps one event for later, as one line. The caller hands over only what the journal needs, its words already
|
|
52
|
+
* redacted (`turnText`, `replyText` in sessions.mjs): nothing is redacted here, and nothing else belongs in a spool.
|
|
53
|
+
* Throws when the spool is full or cannot be written: whoever keeps the event says so.
|
|
54
|
+
*/
|
|
55
|
+
export function spoolEvent(entry) {
|
|
56
|
+
let size = 0;
|
|
57
|
+
try {
|
|
58
|
+
size = statSync(join(paths().spool, WAITING)).size;
|
|
59
|
+
} catch {
|
|
60
|
+
// Nothing waiting yet.
|
|
61
|
+
}
|
|
62
|
+
if (size > SPOOL_MAX_BYTES) throw new Error(`the spool is full (${paths().spool}/${WAITING} is past ${SPOOL_MAX_BYTES / (1024 * 1024)} MB) — the event was not kept; sumo doctor says why the database cannot be written`);
|
|
63
|
+
addLine(WAITING, JSON.stringify(entry));
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
const names = () => {
|
|
67
|
+
try {
|
|
68
|
+
return readdirSync(paths().spool);
|
|
69
|
+
} catch (cause) {
|
|
70
|
+
if (cause.code === 'ENOENT') return [];
|
|
71
|
+
throw cause;
|
|
72
|
+
}
|
|
73
|
+
};
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* The files this process may drain now, each taken by a rename only one process can win: what is waiting, and what a
|
|
77
|
+
* drain that died left in its hand (its process is gone) — replaying a line twice inserts nothing twice.
|
|
78
|
+
*/
|
|
79
|
+
function claim() {
|
|
80
|
+
const dir = paths().spool;
|
|
81
|
+
const mine = [];
|
|
82
|
+
for (const name of names()) {
|
|
83
|
+
const left = DRAINING.exec(name);
|
|
84
|
+
if (name !== WAITING && !(left && (Number(left[1]) === process.pid || !alive(Number(left[1])) || olderThan(join(dir, name), LEFT_AFTER_MS)))) continue;
|
|
85
|
+
const taken = join(dir, `events.${process.pid}.${Date.now()}${claims++}.draining`);
|
|
86
|
+
try {
|
|
87
|
+
renameSync(join(dir, name), taken);
|
|
88
|
+
mine.push(taken);
|
|
89
|
+
} catch (cause) {
|
|
90
|
+
// Another process took it first.
|
|
91
|
+
if (cause.code !== 'ENOENT') throw cause;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
return mine;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function olderThan(file, ms) {
|
|
98
|
+
try {
|
|
99
|
+
return Date.now() - statSync(file).mtimeMs > ms;
|
|
100
|
+
} catch {
|
|
101
|
+
return false;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** SQLite saying the database is busy or locked: another process is writing, and the line has not been tried at all. */
|
|
106
|
+
const locked = (cause) => cause?.errcode === 5 || cause?.errcode === 6 || /database (?:table )?is locked|SQLITE_(?:BUSY|LOCKED)/i.test(String(cause?.message ?? cause));
|
|
107
|
+
|
|
108
|
+
const setAside = (raw, why) => addLine(BAD, JSON.stringify({ at: new Date().toISOString(), why: String(why).slice(0, 300), line: raw }));
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* Replays what waits, oldest first: `replay(entry)` writes one event into the journal and throws when it cannot — with
|
|
112
|
+
* `bad` set on the error when the line is not an event at all, which no later try would mend. `within(fn)` runs a
|
|
113
|
+
* file's replays as one piece of work (a transaction); when it throws, every line of that piece is counted as failed.
|
|
114
|
+
* A failed line goes back to wait, its tries counted — once a minute at most (`tried_at`) — and is set aside at the
|
|
115
|
+
* third; a line that found the database busy goes back uncounted, and nothing more is tried in this drain. Returns
|
|
116
|
+
* `{ replayed, retried, bad }`, or null when nothing was waiting — the usual case, which costs one look at a folder.
|
|
117
|
+
* `now` is the time of this drain, in milliseconds.
|
|
118
|
+
*/
|
|
119
|
+
export function drainSpool(replay, { within = (fn) => fn(), now = Date.now() } = {}) {
|
|
120
|
+
const files = claim();
|
|
121
|
+
if (files.length === 0) return null;
|
|
122
|
+
const done = { replayed: 0, retried: 0, bad: 0 };
|
|
123
|
+
// Once the database is found busy the rest waits untried: each further try would wait out the lock again.
|
|
124
|
+
let busy = false;
|
|
125
|
+
const wait = (entry) => {
|
|
126
|
+
addLine(WAITING, JSON.stringify(entry));
|
|
127
|
+
done.retried++;
|
|
128
|
+
};
|
|
129
|
+
const again = (entry, cause) => {
|
|
130
|
+
if (!cause?.bad && locked(cause)) {
|
|
131
|
+
busy = true;
|
|
132
|
+
return wait(entry);
|
|
133
|
+
}
|
|
134
|
+
const last = Date.parse(entry.tried_at ?? '');
|
|
135
|
+
const counts = Number.isNaN(last) || now - last >= COUNTED_EVERY_MS;
|
|
136
|
+
const tries = (Number.isInteger(entry.tries) ? entry.tries : 0) + (counts ? 1 : 0);
|
|
137
|
+
if (cause?.bad || tries >= TRIES) {
|
|
138
|
+
setAside(JSON.stringify(entry), cause?.message ?? cause);
|
|
139
|
+
done.bad++;
|
|
140
|
+
} else wait(counts ? { ...entry, tries, tried_at: new Date(now).toISOString() } : entry);
|
|
141
|
+
};
|
|
142
|
+
for (const file of files) {
|
|
143
|
+
// Read again after replaying until nothing new is there: a hook that had the file open when it was taken may
|
|
144
|
+
// have added its line after the first read. The replay takes far longer than that write.
|
|
145
|
+
for (let read = 0; ; ) {
|
|
146
|
+
const bytes = readFileSync(file);
|
|
147
|
+
const text = bytes.toString('utf8');
|
|
148
|
+
// `read` is past the last line end once a torn piece has been taken.
|
|
149
|
+
const end = Math.max(text.lastIndexOf('\n') + 1, read);
|
|
150
|
+
// The piece after the last line end is a line still being written — or, when nothing more comes, a torn one.
|
|
151
|
+
const torn = end === read && text.length > end ? text.slice(end) : '';
|
|
152
|
+
if (end === read && !torn) {
|
|
153
|
+
// Let go only if it is no longer than what this look read: a line added since that look is read, not removed with the file.
|
|
154
|
+
if (statSync(file).size > bytes.length) continue;
|
|
155
|
+
rmSync(file, { force: true });
|
|
156
|
+
break;
|
|
157
|
+
}
|
|
158
|
+
const entries = [];
|
|
159
|
+
for (const raw of torn ? [torn] : text.slice(read, end).split('\n').filter((l) => l.trim())) {
|
|
160
|
+
let entry = null;
|
|
161
|
+
try {
|
|
162
|
+
entry = JSON.parse(raw);
|
|
163
|
+
} catch {
|
|
164
|
+
// Said below.
|
|
165
|
+
}
|
|
166
|
+
if (entry && typeof entry === 'object' && !Array.isArray(entry)) entries.push(entry);
|
|
167
|
+
else {
|
|
168
|
+
setAside(raw, 'not a line of JSON');
|
|
169
|
+
done.bad++;
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
const failed = [];
|
|
173
|
+
let replayed = 0;
|
|
174
|
+
if (busy) for (const entry of entries) wait(entry);
|
|
175
|
+
else try {
|
|
176
|
+
within(() => {
|
|
177
|
+
for (const entry of entries) {
|
|
178
|
+
try {
|
|
179
|
+
replay(entry);
|
|
180
|
+
replayed++;
|
|
181
|
+
} catch (cause) {
|
|
182
|
+
failed.push([entry, cause]);
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
});
|
|
186
|
+
done.replayed += replayed;
|
|
187
|
+
for (const [entry, cause] of failed) again(entry, cause);
|
|
188
|
+
} catch (cause) {
|
|
189
|
+
// The piece of work itself failed — the database went busy — so nothing in it is known to have landed.
|
|
190
|
+
for (const entry of entries) again(entry, cause);
|
|
191
|
+
}
|
|
192
|
+
read = torn ? text.length : end;
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
return done;
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
/**
|
|
199
|
+
* `sumo memory check --fix`: every line set aside tried once more, each as its own piece of work — a turn set aside
|
|
200
|
+
* while the database was busy goes in now. What replays leaves `events.bad.jsonl`; what still cannot — a line that
|
|
201
|
+
* was never an event — stays as it was. Returns `{ replayed, left }`.
|
|
202
|
+
*/
|
|
203
|
+
export function retrySetAside(replay, { within = (fn) => fn() } = {}) {
|
|
204
|
+
const file = join(paths().spool, BAD);
|
|
205
|
+
let lines;
|
|
206
|
+
try {
|
|
207
|
+
lines = readFileSync(file, 'utf8').split('\n').filter((l) => l.trim());
|
|
208
|
+
} catch (cause) {
|
|
209
|
+
if (cause.code === 'ENOENT') return { replayed: 0, left: 0 };
|
|
210
|
+
throw cause;
|
|
211
|
+
}
|
|
212
|
+
const left = lines.filter((raw) => {
|
|
213
|
+
try {
|
|
214
|
+
const entry = JSON.parse(JSON.parse(raw).line);
|
|
215
|
+
if (!entry || typeof entry !== 'object' || Array.isArray(entry)) return true;
|
|
216
|
+
within(() => replay(entry));
|
|
217
|
+
return false;
|
|
218
|
+
} catch {
|
|
219
|
+
return true;
|
|
220
|
+
}
|
|
221
|
+
});
|
|
222
|
+
if (left.length === 0) rmSync(file, { force: true });
|
|
223
|
+
else if (left.length < lines.length) writeFileSync(file, `${left.join('\n')}\n`, { mode: 0o600 });
|
|
224
|
+
return { replayed: lines.length - left.length, left: left.length };
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
/** Every string a line of the spool holds: its own, and those of a line kept inside it (`events.bad.jsonl` keeps the line it set aside). */
|
|
228
|
+
function saidIn(raw) {
|
|
229
|
+
let value;
|
|
230
|
+
try {
|
|
231
|
+
value = JSON.parse(raw);
|
|
232
|
+
} catch {
|
|
233
|
+
return [raw];
|
|
234
|
+
}
|
|
235
|
+
const strings = (v) => (typeof v === 'string' ? [v, ...(v.trimStart().startsWith('{') ? saidIn(v).filter((s) => s !== v) : [])] : v && typeof v === 'object' ? Object.values(v).flatMap(strings) : []);
|
|
236
|
+
return strings(value);
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
/**
|
|
240
|
+
* A purge reaches the spool too: every line — waiting, in a drain's hand, or set aside — whose words `says` holds is
|
|
241
|
+
* taken out, the rest left as they were. Returns how many lines went.
|
|
242
|
+
*/
|
|
243
|
+
export function purgeSpool(says) {
|
|
244
|
+
let gone = 0;
|
|
245
|
+
for (const name of names()) {
|
|
246
|
+
if (name !== WAITING && name !== BAD && !DRAINING.test(name)) continue;
|
|
247
|
+
const file = join(paths().spool, name);
|
|
248
|
+
const lines = readFileSync(file, 'utf8').split('\n');
|
|
249
|
+
const kept = lines.filter((l) => !l.trim() || !saidIn(l).some(says));
|
|
250
|
+
if (kept.length === lines.length) continue;
|
|
251
|
+
gone += lines.length - kept.length;
|
|
252
|
+
if (kept.some((l) => l.trim())) writeFileSync(file, kept.join('\n'), { mode: 0o600 });
|
|
253
|
+
else rmSync(file, { force: true });
|
|
254
|
+
}
|
|
255
|
+
return gone;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/**
|
|
259
|
+
* When the oldest event still waiting happened — its own `at`, or the time of the file it waits in for a line that
|
|
260
|
+
* names none — in milliseconds, with how many wait: `{ waiting, since }`, `since` null when nothing does. For the
|
|
261
|
+
* block a session starts with: what has waited an hour was not replayed by the next thing said.
|
|
262
|
+
*/
|
|
263
|
+
export function spoolWaitingSince() {
|
|
264
|
+
let waiting = 0;
|
|
265
|
+
let since = null;
|
|
266
|
+
for (const name of names()) {
|
|
267
|
+
if (name !== WAITING && !DRAINING.test(name)) continue;
|
|
268
|
+
const file = join(paths().spool, name);
|
|
269
|
+
let text;
|
|
270
|
+
try {
|
|
271
|
+
text = readFileSync(file, 'utf8');
|
|
272
|
+
} catch {
|
|
273
|
+
// Taken by a drain since the folder was listed: no longer waiting here.
|
|
274
|
+
continue;
|
|
275
|
+
}
|
|
276
|
+
for (const raw of text.split('\n').filter((l) => l.trim())) {
|
|
277
|
+
waiting++;
|
|
278
|
+
let at = NaN;
|
|
279
|
+
try {
|
|
280
|
+
at = Date.parse(JSON.parse(raw).at);
|
|
281
|
+
} catch {
|
|
282
|
+
// Not a line of JSON: as old as its file.
|
|
283
|
+
}
|
|
284
|
+
if (Number.isNaN(at)) at = statSync(file).mtimeMs;
|
|
285
|
+
since = since === null ? at : Math.min(since, at);
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
return { waiting, since };
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
const linesIn = (name) => {
|
|
292
|
+
try {
|
|
293
|
+
return readFileSync(join(paths().spool, name), 'utf8').split('\n').filter((l) => l.trim()).length;
|
|
294
|
+
} catch (cause) {
|
|
295
|
+
if (cause.code === 'ENOENT') return 0;
|
|
296
|
+
throw cause;
|
|
297
|
+
}
|
|
298
|
+
};
|
|
299
|
+
|
|
300
|
+
/**
|
|
301
|
+
* How the spool stands, for whoever checks memory's health: `waiting`, the events not yet in the journal (what waits,
|
|
302
|
+
* and what a drain has in its hand or died holding), and `bad`, the lines set aside in `events.bad.jsonl` because
|
|
303
|
+
* they were not events or failed to replay three times, a minute or more apart. Both are 0 when there has never been a spool.
|
|
304
|
+
*/
|
|
305
|
+
export function spoolStatus() {
|
|
306
|
+
const all = names();
|
|
307
|
+
return { waiting: all.filter((n) => n === WAITING || DRAINING.test(n)).reduce((n, name) => n + linesIn(name), 0), bad: all.includes(BAD) ? linesIn(BAD) : 0 };
|
|
308
|
+
}
|