sumo-agents 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +86 -0
- package/README.md +50 -37
- package/guides/memory.md +21 -14
- package/package.json +2 -2
- package/prompts/dream.md +3 -16
- package/prompts/scribe.md +20 -15
- package/src/apply.mjs +364 -230
- package/src/card.mjs +16 -11
- package/src/catalog.mjs +3 -3
- package/src/chat.mjs +44 -23
- package/src/check.mjs +371 -0
- package/src/cli.mjs +218 -144
- package/src/db.mjs +147 -2
- package/src/dream.mjs +97 -189
- package/src/export.mjs +43 -18
- package/src/herdr.mjs +1 -3
- package/src/hooks.mjs +115 -22
- package/src/jobs.mjs +4 -2
- package/src/keeper.mjs +14 -51
- package/src/ledger.mjs +126 -0
- package/src/logs.mjs +10 -5
- package/src/loop.mjs +4 -4
- package/src/mcp.mjs +129 -51
- package/src/memeval.mjs +167 -0
- package/src/memory.mjs +261 -130
- package/src/page.mjs +3 -3
- package/src/paths.mjs +1 -0
- package/src/prime.mjs +82 -13
- package/src/rebuild.mjs +162 -0
- package/src/render.mjs +22 -0
- package/src/route.mjs +1 -1
- package/src/sandbox.mjs +5 -2
- package/src/scan.mjs +16 -4
- package/src/scribe.mjs +361 -113
- package/src/search.mjs +48 -0
- package/src/sessions.mjs +304 -53
- package/src/settings.mjs +868 -0
- package/src/setup.mjs +57 -65
- package/src/spool.mjs +308 -0
- package/src/text.mjs +118 -6
- package/src/tools.mjs +18 -7
- package/src/transcript.mjs +3 -47
- package/src/tty.mjs +102 -63
- package/src/ui.mjs +313 -271
- package/src/verify.mjs +23 -27
package/src/db.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { chmodSync, closeSync,
|
|
1
|
+
import { chmodSync, closeSync, mkdirSync, openSync, readdirSync, rmSync, statSync } from 'node:fs';
|
|
2
2
|
import { basename, dirname, join } from 'node:path';
|
|
3
3
|
import { homeProblem, paths } from './paths.mjs';
|
|
4
4
|
import { DatabaseSync } from './sqlite.mjs';
|
|
@@ -314,6 +314,146 @@ const MIGRATIONS = [
|
|
|
314
314
|
DELETE FROM meta WHERE key IN ('config.chat.plain', 'config.chat.motion', 'config.model.source', 'config.model.repo', 'config.model.file',
|
|
315
315
|
'config.prime.budget', 'config.scratch.days', 'config.jobs.escalate', 'config.guides.auto');
|
|
316
316
|
`,
|
|
317
|
+
// 16 — memory in layers. The journal is the source of truth: a turn is kept whole (full_len is how long it was before
|
|
318
|
+
// any cut), stamped with the project it was about, and carries a digest (sha) of its session, time and words, so an
|
|
319
|
+
// event kept aside while the database could not be written is replayed without ever landing twice; the assistant's
|
|
320
|
+
// final replies are kept beside the turns (replies), searchable like them. What a model drew from the journal is
|
|
321
|
+
// marked (memories.derived_by, the extractor's version; NULL is what the user said to remember) and every pass over
|
|
322
|
+
// a conversation is written down before it runs (extractions), so one that died or failed is seen. memory_events is
|
|
323
|
+
// each change of a memory's state from here on, started from what the state columns already say. Guesses are no
|
|
324
|
+
// longer kept: the ones still waiting are rejected here, once — their rows and history stay — except what is held
|
|
325
|
+
// because the conversation read the web or an MCP server (held_for), which only the user settles.
|
|
326
|
+
(db) => {
|
|
327
|
+
addColumnsIfMissing(db, 'user_turns', ['project TEXT', 'full_len INTEGER', 'sha TEXT']);
|
|
328
|
+
addColumnsIfMissing(db, 'memories', ['derived_by TEXT']);
|
|
329
|
+
db.exec(`
|
|
330
|
+
-- ADD COLUMN takes no UNIQUE: the index says it. Rows from before carry no digest, and any number of those may.
|
|
331
|
+
CREATE UNIQUE INDEX IF NOT EXISTS user_turns_sha ON user_turns(sha) WHERE sha IS NOT NULL;
|
|
332
|
+
CREATE INDEX IF NOT EXISTS user_turns_session ON user_turns(session_id, id);
|
|
333
|
+
|
|
334
|
+
CREATE TABLE IF NOT EXISTS replies (
|
|
335
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
336
|
+
session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE,
|
|
337
|
+
ts TEXT NOT NULL,
|
|
338
|
+
text TEXT NOT NULL,
|
|
339
|
+
project TEXT,
|
|
340
|
+
full_len INTEGER,
|
|
341
|
+
sha TEXT UNIQUE
|
|
342
|
+
) STRICT;
|
|
343
|
+
CREATE INDEX IF NOT EXISTS replies_session ON replies(session_id, id);
|
|
344
|
+
|
|
345
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS replies_fts USING fts5(
|
|
346
|
+
text, content='replies', content_rowid='id', tokenize='porter unicode61'
|
|
347
|
+
);
|
|
348
|
+
CREATE TRIGGER IF NOT EXISTS replies_fts_insert AFTER INSERT ON replies BEGIN
|
|
349
|
+
INSERT INTO replies_fts(rowid, text) VALUES (new.id, new.text);
|
|
350
|
+
END;
|
|
351
|
+
CREATE TRIGGER IF NOT EXISTS replies_fts_delete AFTER DELETE ON replies BEGIN
|
|
352
|
+
INSERT INTO replies_fts(replies_fts, rowid, text) VALUES ('delete', old.id, old.text);
|
|
353
|
+
END;
|
|
354
|
+
|
|
355
|
+
-- Written 'running' before the model is called: one still 'running' after its process died is a failure.
|
|
356
|
+
-- first_turn and last_turn are user_turns ids, both ends included.
|
|
357
|
+
CREATE TABLE IF NOT EXISTS extractions (
|
|
358
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
359
|
+
session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE,
|
|
360
|
+
extractor_version TEXT NOT NULL,
|
|
361
|
+
status TEXT NOT NULL CHECK (status IN ('running', 'ok', 'failed', 'quarantined')),
|
|
362
|
+
attempts INTEGER NOT NULL DEFAULT 0,
|
|
363
|
+
first_turn INTEGER,
|
|
364
|
+
last_turn INTEGER,
|
|
365
|
+
ops_json TEXT,
|
|
366
|
+
error TEXT,
|
|
367
|
+
started_at TEXT NOT NULL,
|
|
368
|
+
ended_at TEXT
|
|
369
|
+
) STRICT;
|
|
370
|
+
CREATE INDEX IF NOT EXISTS extractions_session ON extractions(session_id, extractor_version, status);
|
|
371
|
+
|
|
372
|
+
-- memory_id has no REFERENCES on purpose: the history of a memory outlives a purge of it (ids are never handed
|
|
373
|
+
-- out twice), so nothing here may hold a memory's words. run_id is the extractions row a pass made the change in.
|
|
374
|
+
CREATE TABLE IF NOT EXISTS memory_events (
|
|
375
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
376
|
+
ts TEXT NOT NULL,
|
|
377
|
+
memory_id INTEGER NOT NULL,
|
|
378
|
+
op TEXT NOT NULL,
|
|
379
|
+
actor TEXT NOT NULL,
|
|
380
|
+
run_id INTEGER,
|
|
381
|
+
reason TEXT
|
|
382
|
+
) STRICT;
|
|
383
|
+
CREATE INDEX IF NOT EXISTS memory_events_memory ON memory_events(memory_id, id);
|
|
384
|
+
`);
|
|
385
|
+
// What follows reads the rows as they are and is done once: a history already begun is not begun again, should
|
|
386
|
+
// this step ever be run against a database that has had it.
|
|
387
|
+
if (db.prepare('SELECT 1 FROM memory_events LIMIT 1').get()) return;
|
|
388
|
+
db.exec(`
|
|
389
|
+
-- What a model filed and the user never made their own is derived, by the extractor from before versions were
|
|
390
|
+
-- kept. One the user confirmed is stated and keeps no quote (memory.mjs): that one is theirs now.
|
|
391
|
+
UPDATE memories SET derived_by = 'v0'
|
|
392
|
+
WHERE derived_by IS NULL AND written_by IN ('scribe', 'dream', 'worker') AND NOT (provenance = 'stated' AND source_quote IS NULL);
|
|
393
|
+
|
|
394
|
+
-- History so far, as the state columns tell it: when each memory came, and how it ended if it has.
|
|
395
|
+
INSERT INTO memory_events (ts, memory_id, op, actor, reason)
|
|
396
|
+
SELECT created_at, id, 'added', written_by, 'backfilled' FROM memories ORDER BY id;
|
|
397
|
+
INSERT INTO memory_events (ts, memory_id, op, actor, reason)
|
|
398
|
+
SELECT valid_from, id, 'confirmed', 'user', 'backfilled' FROM memories
|
|
399
|
+
WHERE written_by IN ('scribe', 'dream', 'worker') AND provenance = 'stated' AND source_quote IS NULL AND state != 'unconfirmed' ORDER BY id;
|
|
400
|
+
INSERT INTO memory_events (ts, memory_id, op, actor, reason)
|
|
401
|
+
SELECT COALESCE(invalid_at, created_at), id,
|
|
402
|
+
CASE state WHEN 'superseded' THEN 'superseded' WHEN 'invalid' THEN 'forgotten' ELSE 'rejected' END, 'unknown', 'backfilled'
|
|
403
|
+
FROM memories WHERE state IN ('superseded', 'invalid', 'rejected') ORDER BY id;
|
|
404
|
+
|
|
405
|
+
-- The guesses still waiting, rejected once. The event first: it names the rows while they are still waiting.
|
|
406
|
+
INSERT INTO memory_events (ts, memory_id, op, actor, reason)
|
|
407
|
+
SELECT strftime('%Y-%m-%dT%H:%M:%fZ', 'now'), id, 'rejected', 'migration', 'a guess still waiting when guesses stopped being kept'
|
|
408
|
+
FROM memories WHERE state = 'unconfirmed' AND held_for IS NULL ORDER BY id;
|
|
409
|
+
UPDATE memories SET state = 'rejected', invalid_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now')
|
|
410
|
+
WHERE state = 'unconfirmed' AND held_for IS NULL;
|
|
411
|
+
|
|
412
|
+
-- A conversation the old writer had read to its end is not read again as if new.
|
|
413
|
+
INSERT INTO extractions (session_id, extractor_version, status, attempts, first_turn, last_turn, started_at, ended_at)
|
|
414
|
+
SELECT session_id, 'v0', 'ok', 1, MIN(id), MAX(id), MIN(ts), MAX(ts) FROM user_turns GROUP BY session_id HAVING MIN(scribed) = 1;
|
|
415
|
+
`);
|
|
416
|
+
},
|
|
417
|
+
// 17: two things migration 16 got wrong, mended for a database that has had it — 16 itself stays as it ran.
|
|
418
|
+
(db) => {
|
|
419
|
+
const now = new Date().toISOString();
|
|
420
|
+
// 16 rejected everything still waiting with no hold named. A pass's guess was meant; a memory or a taught workflow
|
|
421
|
+
// the model's shell had saved, which the user had not yet answered, was not. Those it rejected — its own line of
|
|
422
|
+
// the history says which — wait for the user again, under the name that hold has now. One the user has touched
|
|
423
|
+
// since is no longer `rejected` and is left alone.
|
|
424
|
+
const held = db
|
|
425
|
+
.prepare(
|
|
426
|
+
`SELECT m.id FROM memories m WHERE m.state = 'rejected' AND m.written_by = 'chat-model'
|
|
427
|
+
AND EXISTS (SELECT 1 FROM memory_events e WHERE e.memory_id = m.id AND e.op = 'rejected' AND e.actor = 'migration') ORDER BY m.id`,
|
|
428
|
+
)
|
|
429
|
+
.all();
|
|
430
|
+
for (const { id } of held) {
|
|
431
|
+
db.prepare(`UPDATE memories SET state = 'unconfirmed', held_for = 'shell', invalid_at = NULL WHERE id = ?`).run(id);
|
|
432
|
+
db.prepare(`INSERT INTO memory_events (ts, memory_id, op, actor, reason) VALUES (?, ?, 'held', 'migration', 'saved by the model''s shell and never answered: it waits for the user again')`).run(now, id);
|
|
433
|
+
}
|
|
434
|
+
// 16 counted a conversation as read only when the old writer had read all of it. Of one half read, the turns it
|
|
435
|
+
// had got to (`scribed`) and no pass has covered since are covered now, each run of them, so only the rest is owed.
|
|
436
|
+
db.exec(`
|
|
437
|
+
INSERT INTO extractions (session_id, extractor_version, status, attempts, first_turn, last_turn, started_at, ended_at)
|
|
438
|
+
SELECT session_id, 'v0', 'ok', 1, MIN(id), MAX(id), MIN(ts), MAX(ts)
|
|
439
|
+
FROM (SELECT id, session_id, ts, was_read,
|
|
440
|
+
ROW_NUMBER() OVER (PARTITION BY session_id ORDER BY id) - ROW_NUMBER() OVER (PARTITION BY session_id, was_read ORDER BY id) AS run
|
|
441
|
+
FROM (SELECT t.id, t.session_id, t.ts,
|
|
442
|
+
(t.scribed = 1 AND NOT EXISTS (SELECT 1 FROM extractions x WHERE x.session_id = t.session_id AND x.status IN ('ok', 'quarantined')
|
|
443
|
+
AND t.id BETWEEN x.first_turn AND x.last_turn)) AS was_read
|
|
444
|
+
FROM user_turns t))
|
|
445
|
+
WHERE was_read = 1 GROUP BY session_id, run ORDER BY MIN(id);
|
|
446
|
+
`);
|
|
447
|
+
},
|
|
448
|
+
// 18 — MCP servers, a row each: the spec as mcp.json held it (a command with its args and env, or a url with its headers
|
|
449
|
+
// or token, and the allow list), as JSON. The file is brought in once, the first time the servers are read (mcp.mjs), and
|
|
450
|
+
// never read again: the model's shell could make a file of that name, never a row here.
|
|
451
|
+
`
|
|
452
|
+
CREATE TABLE IF NOT EXISTS mcp_servers (
|
|
453
|
+
name TEXT PRIMARY KEY,
|
|
454
|
+
spec TEXT NOT NULL
|
|
455
|
+
) STRICT;
|
|
456
|
+
`,
|
|
317
457
|
];
|
|
318
458
|
|
|
319
459
|
/**
|
|
@@ -354,7 +494,12 @@ function opened(file) {
|
|
|
354
494
|
mkdirSync(dirname(file), { recursive: true, mode: 0o700 });
|
|
355
495
|
// SQLite gives the -wal and -shm files the database's own mode: the file is made private before SQLite first opens
|
|
356
496
|
// it, or they are born readable by everyone (umask 022) and keep the latest writes there.
|
|
357
|
-
|
|
497
|
+
try {
|
|
498
|
+
closeSync(openSync(file, 'wx', 0o600));
|
|
499
|
+
} catch (cause) {
|
|
500
|
+
// There already: as it is, whatever it is — SQLite says below when it is not a database it can open.
|
|
501
|
+
if (cause.code !== 'EEXIST') throw cause;
|
|
502
|
+
}
|
|
358
503
|
for (const side of [file, `${file}-wal`, `${file}-shm`]) {
|
|
359
504
|
try {
|
|
360
505
|
if (statSync(side).mode & 0o077) chmodSync(side, 0o600);
|
package/src/dream.mjs
CHANGED
|
@@ -1,33 +1,30 @@
|
|
|
1
1
|
import { readFileSync } from 'node:fs';
|
|
2
2
|
import { join } from 'node:path';
|
|
3
|
-
import {
|
|
3
|
+
import { raiseNotice } from './apply.mjs';
|
|
4
4
|
import { getMeta, setMeta, tx } from './db.mjs';
|
|
5
5
|
import { storedKeys } from './memory.mjs';
|
|
6
6
|
import { callModel, logRun } from './model.mjs';
|
|
7
7
|
import { REPO_ROOT } from './paths.mjs';
|
|
8
8
|
import { redact } from './redact.mjs';
|
|
9
|
-
import { knownProjectsLine, askAndApply, relatedMemories, runScribe, withLock, writerOff } from './scribe.mjs';
|
|
10
|
-
import { line } from './render.mjs';
|
|
11
|
-
import { sessionsAwaitingDream } from './sessions.mjs';
|
|
12
9
|
import { CONFIG_DEFAULTS } from './setup.mjs';
|
|
13
10
|
import { clip, stem, words } from './text.mjs';
|
|
14
11
|
|
|
15
|
-
/**
|
|
16
|
-
|
|
17
|
-
|
|
12
|
+
/**
|
|
13
|
+
* Memory's tidy, the second half of the one memory pass: weekly, memory itself is gone over — search terms filled in,
|
|
14
|
+
* rules checked for being said twice, pairs judged. It reads memory only; conversations are read once, by the filing
|
|
15
|
+
* half (scribe.mjs), which also asks the pair judge here about a new statement that looks like an old one.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
/** How long memory goes between tidies. */
|
|
18
19
|
const TIDY_EVERY_MS = 7 * 86_400_000;
|
|
19
|
-
/**
|
|
20
|
-
const SWEEP_SLICE = 20;
|
|
21
|
-
/** Pairs put to the local model in one call, and calls in one pass: one when conversations bring it, a few when it is the weekly tidy. */
|
|
20
|
+
/** Pairs put to the local model in one call, and calls in one tidy. */
|
|
22
21
|
const PAIRS_PER_CALL = 10;
|
|
23
|
-
const JUDGE_CALLS = 1;
|
|
24
22
|
const TIDY_JUDGE_CALLS = 3;
|
|
25
23
|
/** Memories given search terms in one tidy: what the local model answers well in one call. */
|
|
26
24
|
const KEYS_PER_TIDY = 20;
|
|
27
25
|
const LABELS = ['same', 'replaces', 'conflict', 'unrelated'];
|
|
28
26
|
/** A topic as the scribe gives one: a lowercase word. */
|
|
29
27
|
const TOPIC = /^[a-z][a-z0-9-]{1,23}$/;
|
|
30
|
-
const SWEEP_CURSOR = 'dream.sweep_cursor';
|
|
31
28
|
const KEYS_CURSOR = 'dream.keys_cursor';
|
|
32
29
|
/** The last rule checked for being said already in other words: each is checked once. */
|
|
33
30
|
const REPEATS_CURSOR = 'dream.repeats_cursor';
|
|
@@ -37,75 +34,34 @@ const REPEAT_NEIGHBOURS = 40;
|
|
|
37
34
|
const LAST_TIDY = 'dream.last_tidy';
|
|
38
35
|
const NO_USAGE = { inputTokens: 0, outputTokens: 0, costUsd: 0, cacheReadTokens: null, cacheCreationTokens: null };
|
|
39
36
|
|
|
40
|
-
|
|
41
|
-
const
|
|
42
|
-
/** Each session's share of what fits the local model's context: its latest turns, as many as fit. The writer has filed every one already. */
|
|
43
|
-
const SESSION_CHARS = 8_000;
|
|
37
|
+
/** Where the pairs the repeat check named wait to be judged on their own. */
|
|
38
|
+
const NAMED_PAIRS = 'dream.named_pairs';
|
|
44
39
|
|
|
45
|
-
/** The
|
|
46
|
-
function
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
shown.unshift(line);
|
|
53
|
-
used += line.length;
|
|
40
|
+
/** The pairs named that wait to be judged on their own, in the order named: `{ label, a, b, note }`, a < b. */
|
|
41
|
+
function namedPairs(db) {
|
|
42
|
+
try {
|
|
43
|
+
const named = JSON.parse(getMeta(db, NAMED_PAIRS) ?? '[]');
|
|
44
|
+
return Array.isArray(named) ? named.filter((p) => Number.isInteger(p?.a) && Number.isInteger(p?.b)) : [];
|
|
45
|
+
} catch {
|
|
46
|
+
return [];
|
|
54
47
|
}
|
|
55
|
-
const left = turns.length - shown.length;
|
|
56
|
-
return left > 0 ? [`[${left} earlier turn${left === 1 ? '' : 's'} not shown]`, ...shown] : shown;
|
|
57
48
|
}
|
|
58
|
-
const MAX_MEMORIES = 60;
|
|
59
49
|
|
|
60
|
-
/**
|
|
61
|
-
|
|
62
|
-
* reads several finished conversations side by side, which is the only way to
|
|
63
|
-
* see what no single one shows: a correction the user keeps repeating, two
|
|
64
|
-
* memories that cannot both be true, a routine worth naming.
|
|
65
|
-
*/
|
|
66
|
-
function buildBundle(db, sessions) {
|
|
67
|
-
const ids = sessions.map((s) => s.id);
|
|
68
|
-
const marks = ids.map(() => '?').join(', ');
|
|
69
|
-
const turns = db.prepare(`SELECT * FROM user_turns WHERE session_id IN (${marks}) ORDER BY id`).all(...ids);
|
|
70
|
-
const checkpoints = db.prepare(`SELECT * FROM checkpoints WHERE session_id IN (${marks}) ORDER BY id`).all(...ids);
|
|
71
|
-
const projects = db.prepare(`SELECT DISTINCT slug FROM session_injections WHERE session_id IN (${marks}) AND slug NOT LIKE '%:%'`).all(...ids).map((r) => r.slug);
|
|
72
|
-
|
|
73
|
-
const scopes = ['global', ...projects.map((p) => `project:${p}`)];
|
|
74
|
-
const { memories, cursor } = shownMemories(db, scopes, turns.map((t) => t.text));
|
|
75
|
-
|
|
76
|
-
const sections = [];
|
|
77
|
-
for (const session of sessions) {
|
|
78
|
-
sections.push(`--- session started ${session.started_at.slice(0, 16)}`);
|
|
79
|
-
sections.push(...latestThatFit(turns.filter((x) => x.session_id === session.id)));
|
|
80
|
-
for (const c of checkpoints.filter((x) => x.session_id === session.id)) sections.push(`[done in ${c.project}]: ${c.done}${c.next_step ? ` → next: ${c.next_step}` : ''}`);
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
const prompt = [
|
|
84
|
-
knownProjectsLine(db),
|
|
85
|
-
memories.length > 0 ? `What is already remembered:\n${memories.map(line).join('\n')}` : 'Nothing is remembered yet.',
|
|
86
|
-
...sections,
|
|
87
|
-
].join('\n');
|
|
88
|
-
// What it was shown is what it may name as saying the same thing.
|
|
89
|
-
return { prompt, turns: new Map(turns.map((t) => [t.id, t])), shown: new Set(memories.map((m) => m.id)), cursor };
|
|
90
|
-
}
|
|
50
|
+
/** What waits to be judged from here on: those named, less any judged now. */
|
|
51
|
+
const keepNamed = (db, named) => setMeta(db, NAMED_PAIRS, JSON.stringify(named));
|
|
91
52
|
|
|
92
53
|
/**
|
|
93
|
-
* What a
|
|
94
|
-
*
|
|
95
|
-
*
|
|
54
|
+
* What a model reading a list of rules by number says of two of them waits until the pair is judged on its own: it
|
|
55
|
+
* writes one number for another — a neighbour's for the one it meant — and a pair it never looked at side by side
|
|
56
|
+
* would be a question for the user. Asked once: a pair raised, judged, or waiting already is not named again.
|
|
96
57
|
*/
|
|
97
|
-
function
|
|
98
|
-
const
|
|
99
|
-
if (
|
|
100
|
-
|
|
101
|
-
const
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
for (const m of [...pool].reverse()) {
|
|
105
|
-
if (chosen.size >= MAX_MEMORIES) break;
|
|
106
|
-
chosen.set(m.id, m);
|
|
107
|
-
}
|
|
108
|
-
return { memories: [...chosen.values()].sort((a, b) => a.id - b.id), cursor: slice.at(-1)?.id ?? null };
|
|
58
|
+
function nameForJudging(db, { label, a, b, note }) {
|
|
59
|
+
const [lo, hi] = a < b ? [a, b] : [b, a];
|
|
60
|
+
if (db.prepare('SELECT 1 FROM notices WHERE memory_a = ? AND memory_b = ?').get(lo, hi)) return;
|
|
61
|
+
if (db.prepare('SELECT 1 FROM memory_pairs WHERE a = ? AND b = ?').get(lo, hi)) return;
|
|
62
|
+
const waiting = namedPairs(db);
|
|
63
|
+
if (waiting.some((p) => p.a === lo && p.b === hi)) return;
|
|
64
|
+
keepNamed(db, [...waiting, { label, a: lo, b: hi, note }]);
|
|
109
65
|
}
|
|
110
66
|
|
|
111
67
|
/** The memories the tidy steps read: active, the user's own rather than scanned from a repository, and not workflows. */
|
|
@@ -190,7 +146,7 @@ const KEYS_SCHEMA = {
|
|
|
190
146
|
/** Whether the tidy steps may run: they call the local model only, never one that is paid for. */
|
|
191
147
|
const tidiesLocally = (db) => (getMeta(db, 'config.memory.model') ?? CONFIG_DEFAULTS['memory.model']) === 'local';
|
|
192
148
|
|
|
193
|
-
/** One call of the tidy steps to the local model, with the
|
|
149
|
+
/** One call of the tidy steps to the local model, with the tidy's own instructions, and its ledger row. */
|
|
194
150
|
async function askLocal(db, { prompt, schema, kind, now }) {
|
|
195
151
|
const system = readFileSync(join(REPO_ROOT, 'prompts', 'dream.md'), 'utf8').trim();
|
|
196
152
|
const result = await callModel(db, { system, prompt, schema, model: 'local', kind });
|
|
@@ -216,16 +172,15 @@ function namedToJudge(db, limit = PAIRS_PER_CALL) {
|
|
|
216
172
|
/**
|
|
217
173
|
* Pairs judged, and what the judgement means raised for the user: the same thing said twice, one that may replace
|
|
218
174
|
* the other, or two that cannot both be true — the look-alike and contradiction notices the block already asks about.
|
|
219
|
-
* Nothing is superseded or forgotten here; the user settles it. Every pair answered is kept, so it is asked once.
|
|
220
|
-
*
|
|
221
|
-
*
|
|
222
|
-
* `fill` false judges only those.
|
|
175
|
+
* Nothing is superseded or forgotten here; the user settles it. Every pair answered is kept, so it is asked once. The
|
|
176
|
+
* repeat check names pairs too — a rule said twice — and those are judged first, side by side like the rest: raised
|
|
177
|
+
* as the judgement says, the note it was named with kept when it is judged the way it was named.
|
|
223
178
|
*/
|
|
224
|
-
async function judgePairs(db, { calls, now
|
|
179
|
+
async function judgePairs(db, { calls, now }) {
|
|
225
180
|
const out = { applied: [], dropped: [], usage: NO_USAGE, calls: 0, error: null };
|
|
226
181
|
for (let call = 0; call < calls; call++) {
|
|
227
182
|
const named = namedToJudge(db);
|
|
228
|
-
const pairs = [...named, ...(
|
|
183
|
+
const pairs = [...named, ...(named.length < PAIRS_PER_CALL ? candidatePairs(db, PAIRS_PER_CALL - named.length) : [])];
|
|
229
184
|
if (pairs.length === 0) break;
|
|
230
185
|
const prompt = ['Pairs to judge — how memory A relates to memory B in each:', ...pairs.flatMap(({ a, b }, i) => [`[p${i + 1}]`, ` A: ${describe(a)}`, ` B: ${describe(b)}`])].join('\n');
|
|
231
186
|
const { result, log } = await askLocal(db, { prompt, schema: PAIRS_SCHEMA, kind: 'dream-pairs', now });
|
|
@@ -278,6 +233,39 @@ async function judgePairs(db, { calls, now, fill = true }) {
|
|
|
278
233
|
return out;
|
|
279
234
|
}
|
|
280
235
|
|
|
236
|
+
/**
|
|
237
|
+
* A statement about to be filed that looks like a memory already there — "slate uses pnpm", then "slate uses bun":
|
|
238
|
+
* the fact may have changed. Code picked each pair (`pairs`, as `applyOps` gives them); the local model says what it
|
|
239
|
+
* is, with the same four labels the tidy uses. Returns `labels`, a Map from each statement's place in the answer it
|
|
240
|
+
* came in to `{ old, label }` — only for pairs asked and labels of the four — or undefined when the model could not
|
|
241
|
+
* be asked or did not answer in shape: the pairs then go to the user as notices, as they always did.
|
|
242
|
+
*/
|
|
243
|
+
export async function judgeNewPairs(db, pairs, now) {
|
|
244
|
+
const today = now.slice(0, 10);
|
|
245
|
+
const prompt = [
|
|
246
|
+
// Each with the day it was said: a rebuild reads old turns, and "just said" made an old statement the newer one.
|
|
247
|
+
'Pairs to judge — how memory A relates to memory B in each (B is a statement of the user\'s, not yet filed; each has the day it was said):',
|
|
248
|
+
...pairs.flatMap(({ old, type, scope, body, said }, i) => [`[p${i + 1}]`, ` A: ${describe(old)}`, ` B: new (${type}, ${scope}, ${(said ?? today).slice(0, 10)}): ${clip(body, 300)}`]),
|
|
249
|
+
].join('\n');
|
|
250
|
+
const { result, log } = await askLocal(db, { prompt, schema: PAIRS_SCHEMA, kind: 'dream-pairs', now });
|
|
251
|
+
if (!result.ok) {
|
|
252
|
+
log(result.error);
|
|
253
|
+
return { labels: undefined, usage: result.usage };
|
|
254
|
+
}
|
|
255
|
+
const answers = Array.isArray(result.data?.pairs) ? result.data.pairs : null;
|
|
256
|
+
if (!answers) {
|
|
257
|
+
log('the answer had no list of pairs');
|
|
258
|
+
return { labels: undefined, usage: result.usage };
|
|
259
|
+
}
|
|
260
|
+
const labels = new Map();
|
|
261
|
+
for (const answer of answers) {
|
|
262
|
+
const pair = pairs[Number(answer?.pair) - 1];
|
|
263
|
+
if (pair && LABELS.includes(answer?.label)) labels.set(pair.index, { old: pair.old.id, label: answer.label });
|
|
264
|
+
}
|
|
265
|
+
log(`judged ${labels.size} of ${pairs.length} new statements against what they look like`);
|
|
266
|
+
return { labels, usage: result.usage };
|
|
267
|
+
}
|
|
268
|
+
|
|
281
269
|
const REPEATS_SCHEMA = {
|
|
282
270
|
type: 'object',
|
|
283
271
|
properties: {
|
|
@@ -311,11 +299,13 @@ async function checkRepeats(db, { now }) {
|
|
|
311
299
|
return out;
|
|
312
300
|
}
|
|
313
301
|
const prompt = [
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
302
|
+
// One list, said plainly to be the user's words as typed. Shown the rules twice — those checked, then all — the
|
|
303
|
+
// local model named "I prefer short answers, everywhere" and "keep your replies brief, always, in every project"
|
|
304
|
+
// or missed them according to what kind a third rule was; asked this way it named them every time, and two other
|
|
305
|
+
// pairs, and nothing among rules that differ (tried on the local model, Nov 18 2026). Code keeps only an answer
|
|
306
|
+
// about a rule being checked.
|
|
307
|
+
"Rules said twice — these are the user's own words, as they typed them. Name every two that ask for the same thing in other words (rule: the later one, same_as: the earlier):",
|
|
308
|
+
...[...shown.values()].sort((x, y) => x.id - y.id).map((m) => `m${m.id} (${m.type}, ${m.scope}): ${clip(m.body, 300)}`),
|
|
319
309
|
].join('\n');
|
|
320
310
|
const { result, log } = await askLocal(db, { prompt, schema: REPEATS_SCHEMA, kind: 'dream-repeats', now });
|
|
321
311
|
out.calls = 1;
|
|
@@ -336,8 +326,9 @@ async function checkRepeats(db, { now }) {
|
|
|
336
326
|
const checked = new Set(batch.map((m) => m.id));
|
|
337
327
|
tx(db, () => {
|
|
338
328
|
for (const answer of answers) {
|
|
339
|
-
|
|
340
|
-
const
|
|
329
|
+
// Either way round: the pair is the answer, and one of the two must be a rule this call checks.
|
|
330
|
+
const named = [shown.get(Number(answer?.rule)), shown.get(Number(answer?.same_as))];
|
|
331
|
+
const [a, b] = named[0] && !checked.has(named[0].id) ? [named[1], named[0]] : named;
|
|
341
332
|
if (!a || !b || a.id === b.id || !checked.has(a.id)) {
|
|
342
333
|
out.dropped.push(`repeat m${JSON.stringify(answer?.rule)} of m${JSON.stringify(answer?.same_as)}: not a rule checked and another rule shown`);
|
|
343
334
|
continue;
|
|
@@ -405,119 +396,36 @@ async function fillKeys(db, { now }) {
|
|
|
405
396
|
}
|
|
406
397
|
|
|
407
398
|
/**
|
|
408
|
-
* Whether the weekly tidy is
|
|
409
|
-
* the oldest memory — memory a day old has nothing to tidy yet.
|
|
399
|
+
* Whether the weekly tidy is due: a week since the last one, with something to tidy. Never tidied, the week runs from
|
|
400
|
+
* the oldest memory — memory a day old has nothing to tidy yet. Pure: it reads, never runs.
|
|
410
401
|
*/
|
|
411
|
-
function
|
|
402
|
+
export function tidyDue(db, now = new Date().toISOString()) {
|
|
412
403
|
const since = getMeta(db, LAST_TIDY) ?? db.prepare(`SELECT MIN(created_at) AS at FROM memories WHERE state = 'active' AND scan_key IS NULL`).get()?.at;
|
|
413
404
|
if (!since || Date.parse(now) - Date.parse(since) < TIDY_EVERY_MS) return false;
|
|
414
405
|
return Boolean(db.prepare(`SELECT 1 FROM memories WHERE state = 'active' AND scan_key IS NULL LIMIT 1`).get());
|
|
415
406
|
}
|
|
416
407
|
|
|
417
408
|
/**
|
|
418
|
-
*
|
|
419
|
-
*
|
|
409
|
+
* The tidy: search terms filled in; rules checked for being said twice; and pairs judged — those the check named
|
|
410
|
+
* first. Run when the week is up, or now when asked (`sumo memory run --tidy`); the caller decides which, and holds
|
|
411
|
+
* the lock. It calls the local model only, never one that is paid for: set to another model, there is nothing it does.
|
|
412
|
+
* A tidy the local model could not answer stays due.
|
|
420
413
|
*/
|
|
421
|
-
export function
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
*/
|
|
429
|
-
async function tidyMemory(db, { weekly, now }) {
|
|
430
|
-
const steps = [];
|
|
431
|
-
if (weekly) steps.push(await fillKeys(db, { now }));
|
|
432
|
-
steps.push(await checkRepeats(db, { now }));
|
|
433
|
-
steps.push(await judgePairs(db, { calls: weekly ? TIDY_JUDGE_CALLS : JUDGE_CALLS, now }));
|
|
434
|
-
return {
|
|
414
|
+
export async function runDream(db, { now = new Date().toISOString() } = {}) {
|
|
415
|
+
if (!tidiesLocally(db)) {
|
|
416
|
+
setMeta(db, LAST_TIDY, now);
|
|
417
|
+
return { skipped: 'memory is tidied by the local model only, and memory.model is not local' };
|
|
418
|
+
}
|
|
419
|
+
const steps = [await fillKeys(db, { now }), await checkRepeats(db, { now }), await judgePairs(db, { calls: TIDY_JUDGE_CALLS, now })];
|
|
420
|
+
const tidy = {
|
|
435
421
|
applied: steps.flatMap((s) => s.applied),
|
|
436
422
|
dropped: steps.flatMap((s) => s.dropped),
|
|
437
423
|
usage: steps.reduce((u, s) => added(u, s.usage), NO_USAGE),
|
|
438
424
|
calls: steps.reduce((n, s) => n + s.calls, 0),
|
|
439
425
|
error: steps.find((s) => s.error)?.error ?? null,
|
|
440
426
|
};
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
* is put to the user. With no local model to judge them (the tidy steps never call a paid one), they are raised as named.
|
|
446
|
-
*/
|
|
447
|
-
async function settleNamed(db, { now }) {
|
|
448
|
-
if (namedPairs(db).length === 0) return { applied: [], dropped: [], usage: NO_USAGE };
|
|
449
|
-
if (tidiesLocally(db)) {
|
|
450
|
-
const judged = await judgePairs(db, { calls: 1, now, fill: false });
|
|
451
|
-
// Not answered, they wait for the next tidy, which judges them first.
|
|
452
|
-
return judged.error ? { ...judged, dropped: [...judged.dropped, `the pairs named wait to be judged: ${judged.error}`] } : judged;
|
|
453
|
-
}
|
|
454
|
-
const applied = [];
|
|
455
|
-
tx(db, () => {
|
|
456
|
-
for (const { a, b, named } of namedToJudge(db, Infinity)) {
|
|
457
|
-
const [kind, said] = named.label === 'same' ? ['lookalike', 'say the same thing'] : ['contradiction', 'cannot both be true'];
|
|
458
|
-
const note = named.note ?? `they ${said}`;
|
|
459
|
-
if (raiseNotice(db, { kind, a: a.id, b: b.id, note, now })) applied.push(`raised for the user: m${a.id} and m${b.id} ${said} — ${note}`);
|
|
460
|
-
}
|
|
461
|
-
keepNamed(db, []);
|
|
462
|
-
});
|
|
463
|
-
return { applied, dropped: [], usage: NO_USAGE };
|
|
464
|
-
}
|
|
465
|
-
|
|
466
|
-
/**
|
|
467
|
-
* The consolidation pass, which is also memory's tidy: the conversations waiting are read side by side, and memory
|
|
468
|
-
* itself is gone over — pairs judged, and weekly, search terms filled in. `sweep` asks for the weekly part now; it is
|
|
469
|
-
* done anyway once a week has gone. With no conversation waiting and no tidy owed, there is nothing to do.
|
|
470
|
-
*/
|
|
471
|
-
export async function runDream(db, { now = new Date().toISOString(), force = false, sweep = false, file = true } = {}) {
|
|
472
|
-
// Anything still unfiled belongs to the writer first; consolidation reads its results. The memory pass files first
|
|
473
|
-
// itself, so it can say what the filing did, and asks for none here.
|
|
474
|
-
if (file) await runScribe(db, { now });
|
|
475
|
-
|
|
476
|
-
return withLock('dream', async () => {
|
|
477
|
-
// A conversation the writer has not finished filing (it is busy, failing, or behind) waits for it.
|
|
478
|
-
const unfiled = new Set(writerOff(db) ? [] : db.prepare('SELECT DISTINCT session_id FROM user_turns WHERE scribed = 0').all().map((r) => r.session_id));
|
|
479
|
-
const over = sessionsAwaitingDream(db, now, MAX_SESSIONS).filter((s) => !unfiled.has(s.id));
|
|
480
|
-
// Forced, a pass also reads conversations still going on, after the finished ones. It never closes them:
|
|
481
|
-
// what is said in the rest of one is read once it is over.
|
|
482
|
-
const open = force
|
|
483
|
-
? sessionsAwaitingDream(db, now, MAX_SESSIONS, { includeOpen: true }).filter((s) => !unfiled.has(s.id) && !over.some((o) => o.id === s.id))
|
|
484
|
-
: [];
|
|
485
|
-
const sessions = [...over, ...open].slice(0, MAX_SESSIONS).sort((a, b) => a.started_at.localeCompare(b.started_at));
|
|
486
|
-
const weekly = sweep || tidyOwed(db, now);
|
|
487
|
-
if (sessions.length === 0 && !weekly) return { skipped: 'no finished sessions are waiting' };
|
|
488
|
-
|
|
489
|
-
// Memory is tidied first, so what the conversations bring is read against it as it now stands.
|
|
490
|
-
const tidy = tidiesLocally(db) ? await tidyMemory(db, { weekly, now }) : { applied: [], dropped: [], usage: NO_USAGE, calls: 0, error: null };
|
|
491
|
-
// A tidy the local model could not answer is owed still; one that had nothing to ask is done.
|
|
492
|
-
if (weekly && !tidy.error) setMeta(db, LAST_TIDY, now);
|
|
493
|
-
if (sessions.length === 0) {
|
|
494
|
-
if (tidy.calls === 0) return { skipped: 'no finished sessions are waiting, and memory has nothing to tidy' };
|
|
495
|
-
if (tidy.error && tidy.applied.length === 0) return { ok: false, error: tidy.error, applied: [], dropped: tidy.dropped, usage: tidy.usage };
|
|
496
|
-
return { ok: true, error: null, applied: tidy.applied, dropped: tidy.dropped, usage: tidy.usage, sessions: 0 };
|
|
497
|
-
}
|
|
498
|
-
|
|
499
|
-
const bundle = buildBundle(db, sessions);
|
|
500
|
-
let lastTurn = 0;
|
|
501
|
-
for (const id of bundle.turns.keys()) lastTurn = id;
|
|
502
|
-
const outcome = await askAndApply(db, {
|
|
503
|
-
kind: 'dream', promptFile: 'dream.md', bundle,
|
|
504
|
-
ops: ['add', 'supersede', 'gotcha', 'checkpoint', 'contradiction', 'same', 'procedure'], sessionId: null, now,
|
|
505
|
-
});
|
|
506
|
-
if (outcome.ok && over.length > 0) {
|
|
507
|
-
// A resumed session may have received a new turn while the model read the snapshot.
|
|
508
|
-
db.prepare(`UPDATE sessions SET dream_state = 'done' WHERE id IN (${over.map(() => '?').join(', ')})
|
|
509
|
-
AND NOT EXISTS (SELECT 1 FROM user_turns t WHERE t.session_id = sessions.id AND t.id > ?)`)
|
|
510
|
-
.run(...over.map((s) => s.id), lastTurn);
|
|
511
|
-
}
|
|
512
|
-
// The slice of the oldest moves on only past a pass that read it.
|
|
513
|
-
if (outcome.ok && bundle.cursor !== null) setMeta(db, SWEEP_CURSOR, bundle.cursor);
|
|
514
|
-
const named = outcome.ok ? await settleNamed(db, { now }) : { applied: [], dropped: [], usage: NO_USAGE };
|
|
515
|
-
return {
|
|
516
|
-
...outcome,
|
|
517
|
-
applied: [...tidy.applied, ...outcome.applied, ...named.applied],
|
|
518
|
-
dropped: [...tidy.dropped, ...outcome.dropped, ...named.dropped],
|
|
519
|
-
usage: added(added(tidy.usage, outcome.usage ?? NO_USAGE), named.usage),
|
|
520
|
-
sessions: sessions.length,
|
|
521
|
-
};
|
|
522
|
-
});
|
|
427
|
+
if (!tidy.error) setMeta(db, LAST_TIDY, now);
|
|
428
|
+
if (tidy.calls === 0) return { skipped: 'memory has nothing to tidy' };
|
|
429
|
+
if (tidy.error && tidy.applied.length === 0) return { ok: false, error: tidy.error, applied: [], dropped: tidy.dropped, usage: tidy.usage };
|
|
430
|
+
return { ok: true, error: null, applied: tidy.applied, dropped: tidy.dropped, usage: tidy.usage };
|
|
523
431
|
}
|