sumo-agents 0.0.0-stage → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/AGENTS.md +38 -0
  2. package/LICENSE +21 -0
  3. package/README.md +111 -2
  4. package/bin/sumo.mjs +33 -0
  5. package/guides/delegation.md +33 -0
  6. package/guides/feature.md +14 -0
  7. package/guides/fix.md +14 -0
  8. package/guides/memory.md +23 -0
  9. package/guides/projects.md +27 -0
  10. package/guides/review.md +16 -0
  11. package/guides/workflows.md +26 -0
  12. package/logo.txt +19 -0
  13. package/package.json +42 -4
  14. package/prompts/agent.md +7 -0
  15. package/prompts/dream.md +13 -0
  16. package/prompts/scribe.md +21 -0
  17. package/spinner.txt +90 -0
  18. package/src/advisor.mjs +59 -0
  19. package/src/apply.mjs +368 -0
  20. package/src/auth.mjs +80 -0
  21. package/src/card.mjs +72 -0
  22. package/src/catalog.mjs +167 -0
  23. package/src/chat.mjs +1162 -0
  24. package/src/cli.mjs +815 -0
  25. package/src/cost.mjs +128 -0
  26. package/src/db.mjs +384 -0
  27. package/src/dream.mjs +97 -0
  28. package/src/editor.mjs +198 -0
  29. package/src/export.mjs +132 -0
  30. package/src/guard.mjs +121 -0
  31. package/src/guides.mjs +86 -0
  32. package/src/herdr.mjs +117 -0
  33. package/src/hooks.mjs +270 -0
  34. package/src/images.mjs +89 -0
  35. package/src/jobs.mjs +1184 -0
  36. package/src/jobtools.mjs +82 -0
  37. package/src/local-server.mjs +169 -0
  38. package/src/lock.mjs +45 -0
  39. package/src/loop.mjs +838 -0
  40. package/src/mcp.mjs +604 -0
  41. package/src/memory.mjs +400 -0
  42. package/src/model.mjs +270 -0
  43. package/src/page.html +304 -0
  44. package/src/page.mjs +135 -0
  45. package/src/paths.mjs +33 -0
  46. package/src/prime.mjs +177 -0
  47. package/src/projects.mjs +232 -0
  48. package/src/recall.mjs +45 -0
  49. package/src/redact.mjs +173 -0
  50. package/src/render.mjs +45 -0
  51. package/src/route.mjs +287 -0
  52. package/src/sandbox.mjs +173 -0
  53. package/src/scan.mjs +204 -0
  54. package/src/scribe.mjs +349 -0
  55. package/src/sessions.mjs +371 -0
  56. package/src/setup.mjs +528 -0
  57. package/src/sqlite.mjs +66 -0
  58. package/src/text.mjs +97 -0
  59. package/src/tools.mjs +987 -0
  60. package/src/transcript.mjs +106 -0
  61. package/src/tty.mjs +529 -0
  62. package/src/ui.mjs +1363 -0
  63. package/src/verify.mjs +369 -0
  64. package/src/web.mjs +49 -0
  65. package/src/workflows.mjs +270 -0
  66. package/src/worktree.mjs +128 -0
package/src/cost.mjs ADDED
@@ -0,0 +1,128 @@
1
+ import { MODEL_IDS, MODELS, modelId } from './catalog.mjs';
2
+ import { CACHE_WRITE, PRICES } from './model.mjs';
3
+ import { getProject } from './projects.mjs';
4
+
5
+ /**
6
+ * Where the money goes, read from the ledger: every request `model_runs` holds, by the kind of conversation that made it
7
+ * and the model that answered, in dollars and in the four ways a request is paid for — input read fresh, input read from
8
+ * the cache, input written to it, and output. Then the cache rewrites: requests that paid to write their context again
9
+ * while the cache should still have held it. A conversation here is a job, or outside one a chat session, on one model:
10
+ * the advisor answers in the same session on a model of its own, and a cache is never shared between models.
11
+ */
12
+
13
+ /** A request that read less than this from the cache found next to nothing of its context there. */
14
+ const REWRITE_READ_UNDER = 10_000;
15
+ /** …and wrote more than this share of its input to the cache again. */
16
+ const REWRITE_SHARE = 0.5;
17
+ /** How long what is written to the cache lives unread: a request this soon after the one before should have found it. */
18
+ const CACHE_LIVES_MS = 5 * 60_000;
19
+
20
+ const PARTS = ['uncached', 'read', 'write', 'output'];
21
+
22
+ /** What one request paid for each part, from its tokens and the price list; a model the list does not know splits into nothing. */
23
+ function partsOf(row) {
24
+ const price = PRICES[modelId(row.model)];
25
+ if (!price) return { uncached: 0, read: 0, write: 0, output: 0 };
26
+ // The ledger keeps the whole input; what was read fresh is what the cache neither gave nor took.
27
+ const read = row.cache_read_tokens ?? 0;
28
+ const written = row.cache_creation_tokens ?? 0;
29
+ const plain = Math.max(0, (row.input_tokens ?? 0) - read - written);
30
+ return {
31
+ uncached: (plain * price.input) / 1_000_000,
32
+ read: (read * price.input * price.cacheRead) / 1_000_000,
33
+ write: (written * price.input * CACHE_WRITE) / 1_000_000,
34
+ output: ((row.output_tokens ?? 0) * price.output) / 1_000_000,
35
+ };
36
+ }
37
+
38
+ const empty = () => ({ requests: 0, usd: 0, uncached: 0, read: 0, write: 0, output: 0, search: 0 });
39
+
40
+ function add(into, row, parts) {
41
+ into.requests++;
42
+ into.usd += row.cost_usd ?? 0;
43
+ for (const part of PARTS) into[part] += parts[part];
44
+ // What the tokens do not explain is what a search cost on top of them.
45
+ into.search += Math.max(0, (row.cost_usd ?? 0) - PARTS.reduce((sum, part) => sum + parts[part], 0));
46
+ }
47
+
48
+ /**
49
+ * The ledger's rows, summed: the whole, each kind and model (dearest first), and the cache rewrites — requests that read
50
+ * under 10k from the cache and wrote over half their input to it, under five minutes after the request before in the same
51
+ * conversation. Only a request that answered counts as the one before: a failed one warmed nothing. Pure, so it is tested
52
+ * on rows alone.
53
+ */
54
+ export function breakdown(rows) {
55
+ const total = empty();
56
+ const groups = new Map();
57
+ const rewrites = { requests: 0, usd: 0 };
58
+ const last = new Map();
59
+ const ordered = [...rows].sort((a, b) => (a.ts < b.ts ? -1 : a.ts > b.ts ? 1 : 0));
60
+ for (const row of ordered) {
61
+ const parts = partsOf(row);
62
+ add(total, row, parts);
63
+ const key = `${row.kind}\n${row.model}`;
64
+ if (!groups.has(key)) groups.set(key, { kind: row.kind, model: row.model, ...empty() });
65
+ add(groups.get(key), row, parts);
66
+
67
+ const conversation = row.job_id !== null && row.job_id !== undefined ? `j${row.job_id}` : row.session_id ? `s${row.session_id}` : null;
68
+ if (conversation === null || !(row.input_tokens > 0)) continue;
69
+ const thread = `${conversation}\n${row.model}`;
70
+ const at = Date.parse(row.ts);
71
+ const before = last.get(thread);
72
+ last.set(thread, at);
73
+ const read = row.cache_read_tokens ?? 0;
74
+ const written = row.cache_creation_tokens ?? 0;
75
+ if (before !== undefined && at - before < CACHE_LIVES_MS && read < REWRITE_READ_UNDER && written > row.input_tokens * REWRITE_SHARE) {
76
+ rewrites.requests++;
77
+ rewrites.usd += parts.write;
78
+ }
79
+ }
80
+ return { total, groups: [...groups.values()].sort((a, b) => b.usd - a.usd), rewrites };
81
+ }
82
+
83
+ const dollars = (usd) => `$${usd.toFixed(2)}`;
84
+ const count = (n, word) => `${n.toLocaleString('en-US')} ${word}${n === 1 ? '' : 's'}`;
85
+ const percent = (part, whole) => (whole > 0 ? ` (${Math.round((part / whole) * 100)}%)` : '');
86
+ /** A model by the name Sumo calls it; a local model or one the list does not know, as the ledger has it. */
87
+ const nameOf = (model) => MODELS.find((name) => MODEL_IDS[name] === model) ?? model;
88
+
89
+ /** The four parts of a sum, and a search when one was paid for; with `of`, each part's share of it. */
90
+ function split(sum, of = null) {
91
+ const share = (part) => (of === null ? '' : percent(part, of));
92
+ const parts = [`uncached ${dollars(sum.uncached)}${share(sum.uncached)}`, `cache read ${dollars(sum.read)}${share(sum.read)}`, `cache write ${dollars(sum.write)}${share(sum.write)}`, `output ${dollars(sum.output)}${share(sum.output)}`];
93
+ if (sum.search >= 0.005) parts.push(`search ${dollars(sum.search)}`);
94
+ return parts.join(' · ');
95
+ }
96
+
97
+ /**
98
+ * `sumo cost [--days N] [--project S]`: what the ledger says was spent, the whole first, then each kind and model, then
99
+ * the cache rewrites. A project's spend is its jobs and the chat sessions its card came up in — a session that worked
100
+ * on two projects is counted under each.
101
+ */
102
+ export function costLines(db, { days, project } = {}) {
103
+ const slug = project === undefined ? null : getProject(db, project).slug;
104
+ const since = days === undefined ? null : new Date(Date.now() - days * 86_400_000).toISOString();
105
+ const rows = db
106
+ .prepare(
107
+ `SELECT ts, kind, model, input_tokens, output_tokens, cost_usd, cache_read_tokens, cache_creation_tokens, job_id, session_id
108
+ FROM model_runs
109
+ WHERE (? IS NULL OR ts >= ?)
110
+ AND (? IS NULL
111
+ OR job_id IN (SELECT id FROM jobs WHERE project = ?)
112
+ OR (job_id IS NULL AND session_id IN (SELECT session_id FROM session_injections WHERE slug = ?)))
113
+ ORDER BY ts, id`,
114
+ )
115
+ .all(since, since, slug, slug, slug);
116
+ const window = days === undefined ? null : `the last ${count(days, 'day')}`;
117
+ if (rows.length === 0) return [`nothing was spent ${window ? `in ${window}` : 'yet'}${slug ? ` on ${slug}` : ''}`];
118
+
119
+ const { total, groups, rewrites } = breakdown(rows);
120
+ const share = total.usd > 0 ? `, ${Math.round((rewrites.usd / total.usd) * 100)}% of the total` : '';
121
+ const scope = `${window ?? `since ${rows[0].ts.slice(0, 10)}`}${slug ? ` · ${slug}` : ''}`;
122
+ return [
123
+ `${scope}: ${count(total.requests, 'request')}, ${dollars(total.usd)} — ${split(total, total.usd)}`,
124
+ // A group that cost nothing — the local model's — has nothing to split.
125
+ ...groups.map((g) => ` ${g.kind} · ${nameOf(g.model)}: ${count(g.requests, 'request')}, ${dollars(g.usd)}${g.usd >= 0.005 ? ` — ${split(g)}` : ''}`),
126
+ `cache rewrites: ${count(rewrites.requests, 'request')} read under ${REWRITE_READ_UNDER / 1000}k from the cache and wrote most of their context again, under ${CACHE_LIVES_MS / 60_000} minutes after the request before — ${dollars(rewrites.usd)} of cache writes${share}`,
127
+ ];
128
+ }
package/src/db.mjs ADDED
@@ -0,0 +1,384 @@
1
+ import { chmodSync, closeSync, existsSync, mkdirSync, openSync, statSync } from 'node:fs';
2
+ import { dirname } from 'node:path';
3
+ import { paths } from './paths.mjs';
4
+ import { DatabaseSync } from './sqlite.mjs';
5
+
6
+ /**
7
+ * One entry per schema version, applied in order and never edited once shipped.
8
+ * Each phase of the plan adds its own tables here rather than the first one
9
+ * guessing at all of them. An entry is SQL, or a function of the database for a
10
+ * step plain SQL cannot make re-runnable (see addColumnsIfMissing).
11
+ */
12
+ const MIGRATIONS = [
13
+ `
14
+ CREATE TABLE meta (
15
+ key TEXT PRIMARY KEY,
16
+ value TEXT NOT NULL
17
+ ) STRICT;
18
+
19
+ CREATE TABLE projects (
20
+ slug TEXT PRIMARY KEY,
21
+ name TEXT NOT NULL,
22
+ path TEXT NOT NULL UNIQUE,
23
+ remote TEXT,
24
+ status TEXT NOT NULL DEFAULT 'active' CHECK (status IN ('active', 'archived')),
25
+ created_at TEXT NOT NULL,
26
+ last_touched_at TEXT,
27
+ scanned_at TEXT
28
+ ) STRICT;
29
+
30
+ CREATE TABLE project_aliases (
31
+ alias TEXT PRIMARY KEY,
32
+ slug TEXT NOT NULL REFERENCES projects(slug) ON DELETE CASCADE
33
+ ) STRICT;
34
+
35
+ -- AUTOINCREMENT so a purged id is never handed out again: "m12" must not
36
+ -- come to mean a different memory than the one someone remembers.
37
+ CREATE TABLE memories (
38
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
39
+ type TEXT NOT NULL CHECK (type IN ('preference', 'fact', 'decision', 'gotcha', 'procedure')),
40
+ scope TEXT NOT NULL,
41
+ title TEXT,
42
+ cue TEXT,
43
+ body TEXT NOT NULL,
44
+ topic TEXT,
45
+ provenance TEXT NOT NULL CHECK (provenance IN ('stated', 'observed', 'scanned', 'inferred')),
46
+ state TEXT NOT NULL CHECK (state IN ('active', 'unconfirmed', 'superseded', 'invalid', 'rejected')),
47
+ pinned INTEGER NOT NULL DEFAULT 0,
48
+ importance REAL NOT NULL,
49
+ scan_key TEXT,
50
+ written_by TEXT NOT NULL,
51
+ source_session TEXT,
52
+ source_turn INTEGER,
53
+ source_quote TEXT,
54
+ valid_from TEXT NOT NULL,
55
+ invalid_at TEXT,
56
+ superseded_by INTEGER REFERENCES memories(id) ON DELETE SET NULL,
57
+ created_at TEXT NOT NULL,
58
+ hits INTEGER NOT NULL DEFAULT 0,
59
+ last_hit_at TEXT
60
+ ) STRICT;
61
+
62
+ CREATE INDEX memories_scope_state ON memories(scope, state);
63
+ CREATE UNIQUE INDEX memories_active_scan_key ON memories(scope, scan_key)
64
+ WHERE scan_key IS NOT NULL AND state = 'active';
65
+
66
+ CREATE VIRTUAL TABLE memories_fts USING fts5(
67
+ title, cue, body, topic,
68
+ content='memories', content_rowid='id', tokenize='porter unicode61'
69
+ );
70
+
71
+ CREATE TRIGGER memories_fts_insert AFTER INSERT ON memories BEGIN
72
+ INSERT INTO memories_fts(rowid, title, cue, body, topic)
73
+ VALUES (new.id, new.title, new.cue, new.body, new.topic);
74
+ END;
75
+
76
+ CREATE TRIGGER memories_fts_delete AFTER DELETE ON memories BEGIN
77
+ INSERT INTO memories_fts(memories_fts, rowid, title, cue, body, topic)
78
+ VALUES ('delete', old.id, old.title, old.cue, old.body, old.topic);
79
+ END;
80
+
81
+ -- Only the indexed columns: a search bumps hits on every row it returns, and
82
+ -- that must not rewrite the index.
83
+ CREATE TRIGGER memories_fts_update AFTER UPDATE OF title, cue, body, topic ON memories BEGIN
84
+ INSERT INTO memories_fts(memories_fts, rowid, title, cue, body, topic)
85
+ VALUES ('delete', old.id, old.title, old.cue, old.body, old.topic);
86
+ INSERT INTO memories_fts(rowid, title, cue, body, topic)
87
+ VALUES (new.id, new.title, new.cue, new.body, new.topic);
88
+ END;
89
+
90
+ CREATE TABLE search_misses (
91
+ ts TEXT NOT NULL,
92
+ query TEXT NOT NULL,
93
+ scope TEXT NOT NULL
94
+ ) STRICT;
95
+ `,
96
+
97
+ // 2 — sessions: what was said, what was shown, what got done, what the cheap model cost.
98
+ `
99
+ CREATE TABLE sessions (
100
+ id TEXT PRIMARY KEY,
101
+ harness TEXT NOT NULL,
102
+ cwd TEXT,
103
+ transcript_path TEXT,
104
+ started_at TEXT NOT NULL,
105
+ last_turn_at TEXT,
106
+ ended_at TEXT,
107
+ dream_state TEXT NOT NULL DEFAULT 'pending' CHECK (dream_state IN ('pending', 'done'))
108
+ ) STRICT;
109
+
110
+ CREATE TABLE user_turns (
111
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
112
+ session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE,
113
+ ts TEXT NOT NULL,
114
+ text TEXT NOT NULL,
115
+ redacted INTEGER NOT NULL DEFAULT 0,
116
+ scribed INTEGER NOT NULL DEFAULT 0
117
+ ) STRICT;
118
+
119
+ CREATE INDEX user_turns_pending ON user_turns(scribed, session_id);
120
+
121
+ CREATE VIRTUAL TABLE user_turns_fts USING fts5(
122
+ text, content='user_turns', content_rowid='id', tokenize='porter unicode61'
123
+ );
124
+ CREATE TRIGGER user_turns_fts_insert AFTER INSERT ON user_turns BEGIN
125
+ INSERT INTO user_turns_fts(rowid, text) VALUES (new.id, new.text);
126
+ END;
127
+ CREATE TRIGGER user_turns_fts_delete AFTER DELETE ON user_turns BEGIN
128
+ INSERT INTO user_turns_fts(user_turns_fts, rowid, text) VALUES ('delete', old.id, old.text);
129
+ END;
130
+
131
+ CREATE TABLE session_injections (
132
+ session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE,
133
+ slug TEXT NOT NULL,
134
+ ts TEXT NOT NULL,
135
+ PRIMARY KEY (session_id, slug)
136
+ ) STRICT;
137
+
138
+ CREATE TABLE checkpoints (
139
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
140
+ session_id TEXT,
141
+ project TEXT NOT NULL REFERENCES projects(slug) ON DELETE CASCADE,
142
+ done TEXT NOT NULL,
143
+ next_step TEXT,
144
+ ts TEXT NOT NULL
145
+ ) STRICT;
146
+
147
+ CREATE INDEX checkpoints_project_ts ON checkpoints(project, ts);
148
+
149
+ CREATE TABLE model_runs (
150
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
151
+ ts TEXT NOT NULL,
152
+ kind TEXT NOT NULL,
153
+ model TEXT NOT NULL,
154
+ input_tokens INTEGER,
155
+ output_tokens INTEGER,
156
+ cost_usd REAL,
157
+ ok INTEGER NOT NULL,
158
+ note TEXT
159
+ ) STRICT;
160
+
161
+ -- Things only the user can settle, surfaced at session start until they stop being true.
162
+ CREATE TABLE notices (
163
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
164
+ kind TEXT NOT NULL,
165
+ memory_a INTEGER REFERENCES memories(id) ON DELETE CASCADE,
166
+ memory_b INTEGER REFERENCES memories(id) ON DELETE CASCADE,
167
+ note TEXT NOT NULL,
168
+ created_at TEXT NOT NULL
169
+ ) STRICT;
170
+ `,
171
+
172
+ // 3 — delegated jobs. The files live in ~/.sumo-agents/jobs/<id>/; this is the index.
173
+ `
174
+ CREATE TABLE jobs (
175
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
176
+ project TEXT NOT NULL REFERENCES projects(slug) ON DELETE CASCADE,
177
+ title TEXT NOT NULL,
178
+ agent TEXT NOT NULL CHECK (agent IN ('scout', 'worker')),
179
+ status TEXT NOT NULL CHECK (status IN ('running', 'needs_input', 'done', 'failed', 'abandoned')),
180
+ session_id TEXT,
181
+ created_at TEXT NOT NULL,
182
+ updated_at TEXT NOT NULL
183
+ ) STRICT;
184
+ `,
185
+
186
+ // 4 — a workflow may name, as a regular expression, the shell commands it must come before.
187
+ `
188
+ ALTER TABLE memories ADD COLUMN gate TEXT;
189
+ `,
190
+
191
+ // 5 — a third kind of sub-agent: the reviewer. SQLite cannot alter a CHECK, so the table is rebuilt; ids are kept.
192
+ `
193
+ CREATE TABLE jobs_next (
194
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
195
+ project TEXT NOT NULL REFERENCES projects(slug) ON DELETE CASCADE,
196
+ title TEXT NOT NULL,
197
+ agent TEXT NOT NULL CHECK (agent IN ('scout', 'worker', 'reviewer')),
198
+ status TEXT NOT NULL CHECK (status IN ('running', 'needs_input', 'done', 'failed', 'abandoned')),
199
+ session_id TEXT,
200
+ created_at TEXT NOT NULL,
201
+ updated_at TEXT NOT NULL
202
+ ) STRICT;
203
+ INSERT INTO jobs_next (id, project, title, agent, status, session_id, created_at, updated_at)
204
+ SELECT id, project, title, agent, status, session_id, created_at, updated_at FROM jobs;
205
+ -- A job's files live in a directory named by its id, so an id must never be handed out twice:
206
+ -- carry the counter over, not just the rows that happen to be left.
207
+ DELETE FROM sqlite_sequence WHERE name = 'jobs_next';
208
+ INSERT INTO sqlite_sequence (name, seq) SELECT 'jobs_next', seq FROM sqlite_sequence WHERE name = 'jobs';
209
+ DROP TABLE jobs;
210
+ ALTER TABLE jobs_next RENAME TO jobs;
211
+ `,
212
+
213
+ // 6 — a job's chosen model and effort, why, whether it is a retry, and — once reviewed — how many
214
+ // Important findings it drew. The agent CHECK is unchanged; these are all nullable, so a job
215
+ // created before this migration keeps printing nothing for its route.
216
+ `
217
+ ALTER TABLE jobs ADD COLUMN model TEXT;
218
+ ALTER TABLE jobs ADD COLUMN effort TEXT;
219
+ ALTER TABLE jobs ADD COLUMN route_reason TEXT;
220
+ ALTER TABLE jobs ADD COLUMN important INTEGER;
221
+ ALTER TABLE jobs ADD COLUMN retry_of INTEGER REFERENCES jobs(id) ON DELETE SET NULL;
222
+ `,
223
+
224
+ // 7 — the ledger splits cached input from fresh and names the job and session a call belonged to.
225
+ // All nullable: a row from before this migration, or a backend that reports no cache, reads as NULL.
226
+ // job_id has no REFERENCES on purpose: the ledger outlives the jobs it counts.
227
+ (db) => addColumnsIfMissing(db, 'model_runs', [
228
+ 'cache_read_tokens INTEGER',
229
+ 'cache_creation_tokens INTEGER',
230
+ 'job_id INTEGER',
231
+ 'session_id TEXT',
232
+ ]),
233
+
234
+ // 8 — a memory may carry search terms beside the user's words: what it is about, in words it does not use itself.
235
+ // They are searched, never shown as the user's words. A full-text table takes no new column, so the index is made
236
+ // again with them, and filled from every memory already stored.
237
+ (db) => {
238
+ addColumnsIfMissing(db, 'memories', ['keys TEXT']);
239
+ db.exec(`
240
+ DROP TRIGGER IF EXISTS memories_fts_insert;
241
+ DROP TRIGGER IF EXISTS memories_fts_delete;
242
+ DROP TRIGGER IF EXISTS memories_fts_update;
243
+ DROP TABLE IF EXISTS memories_fts;
244
+
245
+ CREATE VIRTUAL TABLE memories_fts USING fts5(
246
+ title, cue, body, topic, keys,
247
+ content='memories', content_rowid='id', tokenize='porter unicode61'
248
+ );
249
+ INSERT INTO memories_fts(memories_fts) VALUES ('rebuild');
250
+
251
+ CREATE TRIGGER memories_fts_insert AFTER INSERT ON memories BEGIN
252
+ INSERT INTO memories_fts(rowid, title, cue, body, topic, keys)
253
+ VALUES (new.id, new.title, new.cue, new.body, new.topic, new.keys);
254
+ END;
255
+
256
+ CREATE TRIGGER memories_fts_delete AFTER DELETE ON memories BEGIN
257
+ INSERT INTO memories_fts(memories_fts, rowid, title, cue, body, topic, keys)
258
+ VALUES ('delete', old.id, old.title, old.cue, old.body, old.topic, old.keys);
259
+ END;
260
+
261
+ CREATE TRIGGER memories_fts_update AFTER UPDATE OF title, cue, body, topic, keys ON memories BEGIN
262
+ INSERT INTO memories_fts(memories_fts, rowid, title, cue, body, topic, keys)
263
+ VALUES ('delete', old.id, old.title, old.cue, old.body, old.topic, old.keys);
264
+ INSERT INTO memories_fts(rowid, title, cue, body, topic, keys)
265
+ VALUES (new.id, new.title, new.cue, new.body, new.topic, new.keys);
266
+ END;
267
+ `);
268
+ },
269
+ // 9 — whether the router called a job's work risky (security, credentials, money, migrations, concurrency): 1 or 0, so such
270
+ // work is reviewed once it is done. Nullable: a job routed before this migration was never asked.
271
+ (db) => addColumnsIfMissing(db, 'jobs', ['risky INTEGER']),
272
+ ];
273
+
274
+ /**
275
+ * ALTER TABLE … ADD COLUMN for each definition whose column the table does not have yet, so a
276
+ * migration that only adds columns can be re-run against any shape of the table — a half-applied
277
+ * upgrade, or a database whose user_version was lowered without undoing what came after it.
278
+ * Every ADD COLUMN migration goes through here.
279
+ */
280
+ function addColumnsIfMissing(db, table, defs) {
281
+ const have = new Set(db.prepare(`PRAGMA table_info(${table})`).all().map((c) => c.name));
282
+ for (const def of defs) {
283
+ if (!have.has(def.split(/\s+/)[0])) db.exec(`ALTER TABLE ${table} ADD COLUMN ${def}`);
284
+ }
285
+ }
286
+
287
+ export const SCHEMA_VERSION = MIGRATIONS.length;
288
+
289
+ /** How long one process waits for another to let go of the file before it gives up. */
290
+ const BUSY_MS = 3000;
291
+
292
+ /**
293
+ * Every `sumo` command is a process of its own on this file, and several run at once: the chat, the scribe it
294
+ * wakes, a job in its tab. The wait is set before anything else, so the first statement queues like the rest.
295
+ * secure_delete zeroes what is deleted, so an erased memory does not linger in the file's free space.
296
+ */
297
+ export function openDb(file = paths().db) {
298
+ mkdirSync(dirname(file), { recursive: true, mode: 0o700 });
299
+ // SQLite gives the -wal and -shm files the database's own mode: the file is made private before SQLite first opens
300
+ // it, or they are born readable by everyone (umask 022) and keep the latest writes there.
301
+ if (!existsSync(file)) closeSync(openSync(file, 'a', 0o600));
302
+ for (const side of [file, `${file}-wal`, `${file}-shm`]) {
303
+ try {
304
+ if (statSync(side).mode & 0o077) chmodSync(side, 0o600);
305
+ } catch {
306
+ // Not there yet: made with the database's mode when it is.
307
+ }
308
+ }
309
+ const db = new DatabaseSync(file);
310
+ db.exec(`PRAGMA busy_timeout = ${BUSY_MS}; PRAGMA foreign_keys = ON; PRAGMA secure_delete = ON;`);
311
+ useWal(db);
312
+ migrate(db);
313
+ return db;
314
+ }
315
+
316
+ /**
317
+ * The switch to WAL needs the file to itself, and SQLite does not wait for that the way it waits for a lock:
318
+ * of two processes opening a new database at the same moment, one is told "locked" at once. It asks again.
319
+ */
320
+ function useWal(db) {
321
+ const giveUp = Date.now() + BUSY_MS;
322
+ for (;;) {
323
+ try {
324
+ return db.exec('PRAGMA journal_mode = WAL');
325
+ } catch (cause) {
326
+ if (!/database is locked/.test(cause.message) || Date.now() > giveUp) throw cause;
327
+ Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 25);
328
+ }
329
+ }
330
+ }
331
+
332
+ export function schemaVersion(db) {
333
+ return db.prepare('PRAGMA user_version').get().user_version;
334
+ }
335
+
336
+ /** The version is read again inside each step: a process that waited for another to migrate finds the step already taken. */
337
+ function migrate(db) {
338
+ while (schemaVersion(db) < MIGRATIONS.length) {
339
+ tx(db, () => {
340
+ const v = schemaVersion(db);
341
+ if (v >= MIGRATIONS.length) return;
342
+ if (typeof MIGRATIONS[v] === 'function') MIGRATIONS[v](db);
343
+ else db.exec(MIGRATIONS[v]);
344
+ db.exec(`PRAGMA user_version = ${v + 1}`);
345
+ });
346
+ }
347
+ }
348
+
349
+ const inTransaction = new WeakSet();
350
+
351
+ /**
352
+ * IMMEDIATE so two sessions writing at once queue on busy_timeout instead of
353
+ * failing mid-way. Re-entrant: a call made inside a transaction joins it, and
354
+ * the outermost call alone commits or rolls back.
355
+ */
356
+ export function tx(db, fn) {
357
+ if (inTransaction.has(db)) return fn();
358
+ db.exec('BEGIN IMMEDIATE');
359
+ inTransaction.add(db);
360
+ try {
361
+ const result = fn();
362
+ db.exec('COMMIT');
363
+ return result;
364
+ } catch (cause) {
365
+ try {
366
+ db.exec('ROLLBACK');
367
+ } catch {
368
+ // SQLite had already rolled back by itself (a full disk, an I/O error): the reason worth reporting is the first one.
369
+ }
370
+ throw cause;
371
+ } finally {
372
+ inTransaction.delete(db);
373
+ }
374
+ }
375
+
376
+ export function getMeta(db, key) {
377
+ return db.prepare('SELECT value FROM meta WHERE key = ?').get(key)?.value ?? null;
378
+ }
379
+
380
+ export function setMeta(db, key, value) {
381
+ db.prepare(
382
+ 'INSERT INTO meta(key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value',
383
+ ).run(key, String(value));
384
+ }
package/src/dream.mjs ADDED
@@ -0,0 +1,97 @@
1
+ import { knownProjectsLine, askAndApply, runScribe, withLock, writerOff } from './scribe.mjs';
2
+ import { line } from './render.mjs';
3
+ import { sessionsAwaitingDream } from './sessions.mjs';
4
+ import { clip } from './text.mjs';
5
+
6
+ const MAX_SESSIONS = 5;
7
+ const TURN_MAX = 600;
8
+ /** Each session's share of what fits the local model's context: its latest turns, as many as fit. The writer has filed every one already. */
9
+ const SESSION_CHARS = 8_000;
10
+
11
+ /** The last of a session's turns that fit its share, each clipped; the rest are counted, not shown. */
12
+ function latestThatFit(turns) {
13
+ const shown = [];
14
+ let used = 0;
15
+ for (const t of [...turns].reverse()) {
16
+ const line = `[t${t.id}] user: ${clip(t.text, TURN_MAX)}`;
17
+ if (shown.length > 0 && used + line.length > SESSION_CHARS) break;
18
+ shown.unshift(line);
19
+ used += line.length;
20
+ }
21
+ const left = turns.length - shown.length;
22
+ return left > 0 ? [`[${left} earlier turn${left === 1 ? '' : 's'} not shown]`, ...shown] : shown;
23
+ }
24
+ const MAX_MEMORIES = 60;
25
+
26
+ /**
27
+ * The slow pass. The writer files each conversation as it happens; this one
28
+ * reads several finished conversations side by side, which is the only way to
29
+ * see what no single one shows: a correction the user keeps repeating, two
30
+ * memories that cannot both be true, a routine worth naming.
31
+ */
32
+ function buildBundle(db, sessions) {
33
+ const ids = sessions.map((s) => s.id);
34
+ const marks = ids.map(() => '?').join(', ');
35
+ const turns = db.prepare(`SELECT * FROM user_turns WHERE session_id IN (${marks}) ORDER BY id`).all(...ids);
36
+ const checkpoints = db.prepare(`SELECT * FROM checkpoints WHERE session_id IN (${marks}) ORDER BY id`).all(...ids);
37
+ const projects = db.prepare(`SELECT DISTINCT slug FROM session_injections WHERE session_id IN (${marks}) AND slug NOT LIKE '%:%'`).all(...ids).map((r) => r.slug);
38
+
39
+ const scopes = ['global', ...projects.map((p) => `project:${p}`)];
40
+ const memories = db
41
+ .prepare(
42
+ `SELECT * FROM memories WHERE state = 'active' AND scan_key IS NULL AND scope IN (${scopes.map(() => '?').join(', ')})
43
+ ORDER BY id DESC LIMIT ?`,
44
+ )
45
+ .all(...scopes, MAX_MEMORIES);
46
+
47
+ const sections = [];
48
+ for (const session of sessions) {
49
+ sections.push(`--- session started ${session.started_at.slice(0, 16)}`);
50
+ sections.push(...latestThatFit(turns.filter((x) => x.session_id === session.id)));
51
+ for (const c of checkpoints.filter((x) => x.session_id === session.id)) sections.push(`[done in ${c.project}]: ${c.done}${c.next_step ? ` → next: ${c.next_step}` : ''}`);
52
+ }
53
+
54
+ const prompt = [
55
+ knownProjectsLine(db),
56
+ memories.length > 0 ? `What is already remembered:\n${memories.map(line).join('\n')}` : 'Nothing is remembered yet.',
57
+ ...sections,
58
+ ].join('\n');
59
+ return { prompt, turns: new Map(turns.map((t) => [t.id, t])) };
60
+ }
61
+
62
+ export async function runDream(db, { now = new Date().toISOString(), force = false } = {}) {
63
+ // Anything still unfiled belongs to the writer first; consolidation reads its results.
64
+ await runScribe(db, { now });
65
+
66
+ return withLock('dream', async () => {
67
+ // A conversation the writer has not finished filing (it is busy, failing, or behind) waits for it.
68
+ const unfiled = new Set(writerOff(db) ? [] : db.prepare('SELECT DISTINCT session_id FROM user_turns WHERE scribed = 0').all().map((r) => r.session_id));
69
+ const over = sessionsAwaitingDream(db, now, MAX_SESSIONS).filter((s) => !unfiled.has(s.id));
70
+ // Forced, a pass also reads conversations still going on, after the finished ones. It never closes them:
71
+ // what is said in the rest of one is read once it is over.
72
+ const open = force
73
+ ? sessionsAwaitingDream(db, now, MAX_SESSIONS, { includeOpen: true }).filter((s) => !unfiled.has(s.id) && !over.some((o) => o.id === s.id))
74
+ : [];
75
+ const sessions = [...over, ...open].slice(0, MAX_SESSIONS).sort((a, b) => a.started_at.localeCompare(b.started_at));
76
+ if (sessions.length === 0) return { skipped: 'no finished sessions are waiting' };
77
+
78
+ const bundle = buildBundle(db, sessions);
79
+ let lastTurn = 0;
80
+ for (const id of bundle.turns.keys()) lastTurn = id;
81
+ const outcome = await askAndApply(db, {
82
+ kind: 'dream', promptFile: 'dream.md', bundle,
83
+ ops: ['add', 'supersede', 'gotcha', 'checkpoint', 'contradiction', 'procedure'], sessionId: null, now,
84
+ });
85
+ if (outcome.ok && over.length > 0) {
86
+ // A resumed session may have received a new turn while the model read the snapshot.
87
+ db.prepare(`UPDATE sessions SET dream_state = 'done' WHERE id IN (${over.map(() => '?').join(', ')})
88
+ AND NOT EXISTS (SELECT 1 FROM user_turns t WHERE t.session_id = sessions.id AND t.id > ?)`)
89
+ .run(...over.map((s) => s.id), lastTurn);
90
+ }
91
+ return { ...outcome, sessions: sessions.length };
92
+ });
93
+ }
94
+
95
+ export function dreamStatus(db, now = new Date().toISOString()) {
96
+ return [`finished sessions waiting: ${sessionsAwaitingDream(db, now, 1000).length} (runs by itself at 3)`];
97
+ }