sumo-agents 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +8 -6
- package/CHANGELOG.md +155 -0
- package/README.md +82 -19
- package/bin/sumo.mjs +13 -2
- package/guides/feature.md +2 -1
- package/guides/memory.md +6 -4
- package/guides/projects.md +1 -1
- package/guides/review.md +1 -1
- package/package.json +5 -4
- package/prompts/dream.md +6 -0
- package/spinner.txt +25 -1
- package/src/apply.mjs +146 -18
- package/src/auth.mjs +32 -5
- package/src/card.mjs +39 -2
- package/src/catalog.mjs +6 -3
- package/src/chat.mjs +289 -89
- package/src/cli.mjs +359 -135
- package/src/cost.mjs +67 -8
- package/src/db.mjs +106 -3
- package/src/dream.mjs +445 -19
- package/src/editor.mjs +113 -11
- package/src/export.mjs +105 -7
- package/src/guard.mjs +57 -11
- package/src/guides.mjs +5 -4
- package/src/herdr.mjs +9 -4
- package/src/hooks.mjs +35 -19
- package/src/jobs.mjs +118 -39
- package/src/jobtools.mjs +6 -4
- package/src/keeper.mjs +77 -0
- package/src/local-server.mjs +9 -16
- package/src/lock.mjs +2 -1
- package/src/logs.mjs +165 -0
- package/src/loop.mjs +121 -44
- package/src/mcp.mjs +72 -16
- package/src/memory.mjs +142 -15
- package/src/model.mjs +41 -21
- package/src/paths.mjs +38 -0
- package/src/prime.mjs +19 -31
- package/src/projects.mjs +94 -12
- package/src/redact.mjs +10 -0
- package/src/render.mjs +2 -1
- package/src/route.mjs +39 -25
- package/src/sandbox.mjs +82 -3
- package/src/scan.mjs +35 -1
- package/src/scribe.mjs +10 -22
- package/src/sessions.mjs +49 -13
- package/src/setup.mjs +425 -76
- package/src/text.mjs +100 -1
- package/src/tools.mjs +258 -21
- package/src/tty.mjs +481 -105
- package/src/ui.mjs +1205 -276
- package/src/verify.mjs +17 -1
package/src/dream.mjs
CHANGED
|
@@ -1,7 +1,41 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { readFileSync } from 'node:fs';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { keepNamed, namedPairs, nameForJudging, raiseNotice } from './apply.mjs';
|
|
4
|
+
import { getMeta, setMeta, tx } from './db.mjs';
|
|
5
|
+
import { storedKeys } from './memory.mjs';
|
|
6
|
+
import { callModel, logRun } from './model.mjs';
|
|
7
|
+
import { REPO_ROOT } from './paths.mjs';
|
|
8
|
+
import { redact } from './redact.mjs';
|
|
9
|
+
import { knownProjectsLine, askAndApply, relatedMemories, runScribe, withLock, writerOff } from './scribe.mjs';
|
|
2
10
|
import { line } from './render.mjs';
|
|
3
11
|
import { sessionsAwaitingDream } from './sessions.mjs';
|
|
4
|
-
import {
|
|
12
|
+
import { CONFIG_DEFAULTS } from './setup.mjs';
|
|
13
|
+
import { clip, stem, words } from './text.mjs';
|
|
14
|
+
|
|
15
|
+
/** Finished sessions waiting that make a pass worth running by itself. */
|
|
16
|
+
export const SESSIONS_TO_TIDY = 3;
|
|
17
|
+
/** How long memory goes between tidies — pairs judged, search terms filled in — when no conversation brings one sooner. */
|
|
18
|
+
const TIDY_EVERY_MS = 7 * 86_400_000;
|
|
19
|
+
/** The oldest memories a pass reads beside the related and the newest, moving on each pass so nothing old goes unread for long. */
|
|
20
|
+
const SWEEP_SLICE = 20;
|
|
21
|
+
/** Pairs put to the local model in one call, and calls in one pass: one when conversations bring it, a few when it is the weekly tidy. */
|
|
22
|
+
const PAIRS_PER_CALL = 10;
|
|
23
|
+
const JUDGE_CALLS = 1;
|
|
24
|
+
const TIDY_JUDGE_CALLS = 3;
|
|
25
|
+
/** Memories given search terms in one tidy: what the local model answers well in one call. */
|
|
26
|
+
const KEYS_PER_TIDY = 20;
|
|
27
|
+
const LABELS = ['same', 'replaces', 'conflict', 'unrelated'];
|
|
28
|
+
/** A topic as the scribe gives one: a lowercase word. */
|
|
29
|
+
const TOPIC = /^[a-z][a-z0-9-]{1,23}$/;
|
|
30
|
+
const SWEEP_CURSOR = 'dream.sweep_cursor';
|
|
31
|
+
const KEYS_CURSOR = 'dream.keys_cursor';
|
|
32
|
+
/** The last rule checked for being said already in other words: each is checked once. */
|
|
33
|
+
const REPEATS_CURSOR = 'dream.repeats_cursor';
|
|
34
|
+
/** Rules checked a call, and the most of the other rules in their scope shown beside them. */
|
|
35
|
+
const REPEATS_PER_CALL = 8;
|
|
36
|
+
const REPEAT_NEIGHBOURS = 40;
|
|
37
|
+
const LAST_TIDY = 'dream.last_tidy';
|
|
38
|
+
const NO_USAGE = { inputTokens: 0, outputTokens: 0, costUsd: 0, cacheReadTokens: null, cacheCreationTokens: null };
|
|
5
39
|
|
|
6
40
|
const MAX_SESSIONS = 5;
|
|
7
41
|
const TURN_MAX = 600;
|
|
@@ -37,12 +71,7 @@ function buildBundle(db, sessions) {
|
|
|
37
71
|
const projects = db.prepare(`SELECT DISTINCT slug FROM session_injections WHERE session_id IN (${marks}) AND slug NOT LIKE '%:%'`).all(...ids).map((r) => r.slug);
|
|
38
72
|
|
|
39
73
|
const scopes = ['global', ...projects.map((p) => `project:${p}`)];
|
|
40
|
-
const memories = db
|
|
41
|
-
.prepare(
|
|
42
|
-
`SELECT * FROM memories WHERE state = 'active' AND scan_key IS NULL AND scope IN (${scopes.map(() => '?').join(', ')})
|
|
43
|
-
ORDER BY id DESC LIMIT ?`,
|
|
44
|
-
)
|
|
45
|
-
.all(...scopes, MAX_MEMORIES);
|
|
74
|
+
const { memories, cursor } = shownMemories(db, scopes, turns.map((t) => t.text));
|
|
46
75
|
|
|
47
76
|
const sections = [];
|
|
48
77
|
for (const session of sessions) {
|
|
@@ -56,12 +85,393 @@ function buildBundle(db, sessions) {
|
|
|
56
85
|
memories.length > 0 ? `What is already remembered:\n${memories.map(line).join('\n')}` : 'Nothing is remembered yet.',
|
|
57
86
|
...sections,
|
|
58
87
|
].join('\n');
|
|
59
|
-
|
|
88
|
+
// What it was shown is what it may name as saying the same thing.
|
|
89
|
+
return { prompt, turns: new Map(turns.map((t) => [t.id, t])), shown: new Set(memories.map((m) => m.id)), cursor };
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* What a pass is shown of memory, at most MAX_MEMORIES: the memories most related to what was said, a slice of the
|
|
94
|
+
* oldest that moves on with every pass — from where the last one stopped, round to the start again — and the newest
|
|
95
|
+
* for the rest. All of them when there are no more than that. Returns them by id, with where the slice stopped.
|
|
96
|
+
*/
|
|
97
|
+
function shownMemories(db, scopes, texts) {
|
|
98
|
+
const pool = db.prepare(`SELECT * FROM memories WHERE state = 'active' AND scan_key IS NULL AND scope IN (${scopes.map(() => '?').join(', ')}) ORDER BY id`).all(...scopes);
|
|
99
|
+
if (pool.length <= MAX_MEMORIES) return { memories: pool, cursor: null };
|
|
100
|
+
const chosen = new Map(relatedMemories(db, texts, scopes).map((m) => [m.id, m]));
|
|
101
|
+
const from = Number(getMeta(db, SWEEP_CURSOR) ?? 0);
|
|
102
|
+
const slice = [...pool.filter((m) => m.id > from), ...pool.filter((m) => m.id <= from)].filter((m) => !chosen.has(m.id)).slice(0, SWEEP_SLICE);
|
|
103
|
+
for (const m of slice) chosen.set(m.id, m);
|
|
104
|
+
for (const m of [...pool].reverse()) {
|
|
105
|
+
if (chosen.size >= MAX_MEMORIES) break;
|
|
106
|
+
chosen.set(m.id, m);
|
|
107
|
+
}
|
|
108
|
+
return { memories: [...chosen.values()].sort((a, b) => a.id - b.id), cursor: slice.at(-1)?.id ?? null };
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/** The memories the tidy steps read: active, the user's own rather than scanned from a repository, and not workflows. */
|
|
112
|
+
const tidyable = (db) => db.prepare(`SELECT * FROM memories WHERE state = 'active' AND scan_key IS NULL AND type != 'procedure' ORDER BY id`).all();
|
|
113
|
+
|
|
114
|
+
/** A stem that this many memories share says nothing about any two of them: it is left out of the index. */
|
|
115
|
+
const COMMON_STEM = 200;
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* Pairs of memories worth asking about, most alike first: in one scope, or one of them global; and sharing three
|
|
119
|
+
* stems, or two beside a topic or a search term — a topic alone is a shelf, not a likeness. Word overlap alone misses
|
|
120
|
+
* the same rule in other words and two values for one setting, so code only picks the pairs and the local model says
|
|
121
|
+
* what they are. A pair judged before, or already raised for the user, is never picked again. A memory is compared
|
|
122
|
+
* only with those it shares a stem with, through an index, and only the best `limit` are kept as they are found:
|
|
123
|
+
* thousands of memories cost a moment, not every pair of them.
|
|
124
|
+
*/
|
|
125
|
+
export function candidatePairs(db, limit = PAIRS_PER_CALL) {
|
|
126
|
+
const rows = tidyable(db);
|
|
127
|
+
const judged = new Set(db.prepare('SELECT a, b FROM memory_pairs').all().map((p) => `${p.a}:${p.b}`));
|
|
128
|
+
for (const n of db.prepare('SELECT memory_a, memory_b FROM notices').all()) judged.add(`${Math.min(n.memory_a, n.memory_b)}:${Math.max(n.memory_a, n.memory_b)}`);
|
|
129
|
+
for (const p of namedPairs(db)) judged.add(`${p.a}:${p.b}`);
|
|
130
|
+
const stems = new Map(rows.map((m) => [m.id, new Set(words(m.body).map(stem))]));
|
|
131
|
+
const keys = new Map(rows.map((m) => [m.id, new Set(String(m.keys ?? '').split(',').map((k) => k.trim()).filter(Boolean))]));
|
|
132
|
+
const using = new Map();
|
|
133
|
+
for (const m of rows) for (const s of stems.get(m.id)) (using.get(s) ?? using.set(s, []).get(s)).push(m);
|
|
134
|
+
/** Negative when pair x goes before pair y: more alike first, then the newer. */
|
|
135
|
+
const before = (x, y) => y.rank - x.rank || y.b.id - x.b.id;
|
|
136
|
+
const best = [];
|
|
137
|
+
for (const a of rows) {
|
|
138
|
+
const shared = new Map();
|
|
139
|
+
for (const s of stems.get(a.id)) {
|
|
140
|
+
const others = using.get(s);
|
|
141
|
+
if (others.length > COMMON_STEM) continue;
|
|
142
|
+
for (const b of others) if (b.id > a.id) shared.set(b, (shared.get(b) ?? 0) + 1);
|
|
143
|
+
}
|
|
144
|
+
for (const [b, count] of shared) {
|
|
145
|
+
if (count < 2) continue;
|
|
146
|
+
if (a.scope !== b.scope && a.scope !== 'global' && b.scope !== 'global') continue;
|
|
147
|
+
if (judged.has(`${a.id}:${b.id}`)) continue;
|
|
148
|
+
const topic = Boolean(a.topic) && a.topic === b.topic;
|
|
149
|
+
const term = [...keys.get(a.id)].some((k) => keys.get(b.id).has(k));
|
|
150
|
+
if (count < 3 && !topic && !term) continue;
|
|
151
|
+
const pair = { a, b, rank: count * 2 + (term ? 2 : 0) + (topic ? 1 : 0) };
|
|
152
|
+
if (best.length >= limit && before(pair, best.at(-1)) >= 0) continue;
|
|
153
|
+
const at = best.findIndex((p) => before(pair, p) < 0);
|
|
154
|
+
best.splice(at === -1 ? best.length : at, 0, pair);
|
|
155
|
+
if (best.length > limit) best.pop();
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
return best;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
const PAIRS_SCHEMA = {
|
|
162
|
+
type: 'object',
|
|
163
|
+
properties: {
|
|
164
|
+
pairs: {
|
|
165
|
+
type: 'array',
|
|
166
|
+
items: { type: 'object', properties: { pair: { type: 'integer' }, label: { type: 'string', enum: LABELS } }, required: ['pair', 'label'], additionalProperties: false },
|
|
167
|
+
},
|
|
168
|
+
},
|
|
169
|
+
required: ['pairs'],
|
|
170
|
+
additionalProperties: false,
|
|
171
|
+
};
|
|
172
|
+
|
|
173
|
+
const KEYS_SCHEMA = {
|
|
174
|
+
type: 'object',
|
|
175
|
+
properties: {
|
|
176
|
+
memories: {
|
|
177
|
+
type: 'array',
|
|
178
|
+
items: {
|
|
179
|
+
type: 'object',
|
|
180
|
+
properties: { id: { type: 'integer' }, topic: { type: 'string' }, keys: { type: 'array', items: { type: 'string' } } },
|
|
181
|
+
required: ['id', 'topic', 'keys'],
|
|
182
|
+
additionalProperties: false,
|
|
183
|
+
},
|
|
184
|
+
},
|
|
185
|
+
},
|
|
186
|
+
required: ['memories'],
|
|
187
|
+
additionalProperties: false,
|
|
188
|
+
};
|
|
189
|
+
|
|
190
|
+
/** Whether the tidy steps may run: they call the local model only, never one that is paid for. */
|
|
191
|
+
const tidiesLocally = (db) => (getMeta(db, 'config.memory.model') ?? CONFIG_DEFAULTS['memory.model']) === 'local';
|
|
192
|
+
|
|
193
|
+
/** One call of the tidy steps to the local model, with the consolidation pass's own instructions, and its ledger row. */
|
|
194
|
+
async function askLocal(db, { prompt, schema, kind, now }) {
|
|
195
|
+
const system = readFileSync(join(REPO_ROOT, 'prompts', 'dream.md'), 'utf8').trim();
|
|
196
|
+
const result = await callModel(db, { system, prompt, schema, model: 'local', kind });
|
|
197
|
+
return { result, log: (note) => logRun(db, { kind: 'dream', model: 'local', result, note, now }) };
|
|
60
198
|
}
|
|
61
199
|
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
200
|
+
const added = (x, y) => ({ ...x, inputTokens: x.inputTokens + y.inputTokens, outputTokens: x.outputTokens + y.outputTokens, costUsd: x.costUsd + y.costUsd });
|
|
201
|
+
|
|
202
|
+
const describe = (m) => `m${m.id} (${m.type}, ${m.scope}, ${String(m.created_at).slice(0, 10)}): ${clip(m.body, 300)}`;
|
|
203
|
+
|
|
204
|
+
/**
|
|
205
|
+
* The pairs a pass named that are still worth asking about, the first `limit` of them, as `{ a, b, named }`. One whose
|
|
206
|
+
* memory is gone, or no longer active, waits no longer.
|
|
207
|
+
*/
|
|
208
|
+
function namedToJudge(db, limit = PAIRS_PER_CALL) {
|
|
209
|
+
const row = db.prepare(`SELECT * FROM memories WHERE id = ? AND state = 'active'`);
|
|
210
|
+
const named = namedPairs(db).map((p) => ({ a: row.get(p.a), b: row.get(p.b), named: p }));
|
|
211
|
+
const live = named.filter((p) => p.a && p.b);
|
|
212
|
+
if (live.length < named.length) keepNamed(db, live.map((p) => p.named));
|
|
213
|
+
return live.slice(0, limit);
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/**
|
|
217
|
+
* Pairs judged, and what the judgement means raised for the user: the same thing said twice, one that may replace
|
|
218
|
+
* the other, or two that cannot both be true — the look-alike and contradiction notices the block already asks about.
|
|
219
|
+
* Nothing is superseded or forgotten here; the user settles it. Every pair answered is kept, so it is asked once. A
|
|
220
|
+
* model reading a list names pairs too — a rule said twice, a clash — and those are judged first, side by side like
|
|
221
|
+
* the rest: raised as the judgement says, the note it was named with kept when it is judged the way it was named.
|
|
222
|
+
* `fill` false judges only those.
|
|
223
|
+
*/
|
|
224
|
+
async function judgePairs(db, { calls, now, fill = true }) {
|
|
225
|
+
const out = { applied: [], dropped: [], usage: NO_USAGE, calls: 0, error: null };
|
|
226
|
+
for (let call = 0; call < calls; call++) {
|
|
227
|
+
const named = namedToJudge(db);
|
|
228
|
+
const pairs = [...named, ...(fill && named.length < PAIRS_PER_CALL ? candidatePairs(db, PAIRS_PER_CALL - named.length) : [])];
|
|
229
|
+
if (pairs.length === 0) break;
|
|
230
|
+
const prompt = ['Pairs to judge — how memory A relates to memory B in each:', ...pairs.flatMap(({ a, b }, i) => [`[p${i + 1}]`, ` A: ${describe(a)}`, ` B: ${describe(b)}`])].join('\n');
|
|
231
|
+
const { result, log } = await askLocal(db, { prompt, schema: PAIRS_SCHEMA, kind: 'dream-pairs', now });
|
|
232
|
+
out.calls++;
|
|
233
|
+
out.usage = added(out.usage, result.usage);
|
|
234
|
+
// A model that could not answer leaves the tidy owed; an answer in the wrong shape is a bad answer, dropped.
|
|
235
|
+
if (!result.ok) {
|
|
236
|
+
out.error = result.error;
|
|
237
|
+
log(result.error);
|
|
238
|
+
break;
|
|
239
|
+
}
|
|
240
|
+
const labels = Array.isArray(result.data?.pairs) ? result.data.pairs : null;
|
|
241
|
+
if (!labels) {
|
|
242
|
+
out.dropped.push('pairs: the answer had no list of pairs');
|
|
243
|
+
log('the answer had no list of pairs');
|
|
244
|
+
break;
|
|
245
|
+
}
|
|
246
|
+
let judged = 0;
|
|
247
|
+
tx(db, () => {
|
|
248
|
+
for (const answer of labels) {
|
|
249
|
+
const pair = pairs[Number(answer?.pair) - 1];
|
|
250
|
+
if (!pair || !LABELS.includes(answer?.label)) {
|
|
251
|
+
out.dropped.push(`pair ${JSON.stringify(answer?.pair)}: not one of the pairs asked, or no label`);
|
|
252
|
+
continue;
|
|
253
|
+
}
|
|
254
|
+
const { a, b, named: asked } = pair;
|
|
255
|
+
if (db.prepare('SELECT 1 FROM memory_pairs WHERE a = ? AND b = ?').get(a.id, b.id)) continue;
|
|
256
|
+
db.prepare('INSERT INTO memory_pairs (a, b, label, judged_at) VALUES (?, ?, ?, ?)').run(a.id, b.id, answer.label, now);
|
|
257
|
+
judged++;
|
|
258
|
+
const [older, newer] = a.created_at <= b.created_at ? [a, b] : [b, a];
|
|
259
|
+
const raised =
|
|
260
|
+
answer.label === 'same' ? { kind: 'lookalike', note: 'they say the same thing — forget one, or supersede the older with the newer', said: 'say the same thing' }
|
|
261
|
+
: answer.label === 'replaces' ? { kind: 'lookalike', note: `m${newer.id} is newer and may replace m${older.id} — if it does, supersede m${older.id} with it; if both hold, leave them`, said: `may be replaced by m${newer.id}` }
|
|
262
|
+
: answer.label === 'conflict' ? { kind: 'contradiction', note: 'they cannot both be true — only the user knows which holds', said: 'cannot both be true' }
|
|
263
|
+
: null;
|
|
264
|
+
if (!raised) {
|
|
265
|
+
if (asked) out.dropped.push(`m${a.id} and m${b.id}, named as ${asked.label === 'same' ? 'saying the same thing' : 'a clash'}: judged on their own, they are unrelated`);
|
|
266
|
+
continue;
|
|
267
|
+
}
|
|
268
|
+
const kept = asked?.label === answer.label ? asked.note : null;
|
|
269
|
+
if (raiseNotice(db, { kind: raised.kind, a: a.id, b: b.id, note: kept ?? raised.note, now })) out.applied.push(`raised for the user: m${a.id} and m${b.id} ${raised.said}${kept ? ` — ${kept}` : ''}`);
|
|
270
|
+
}
|
|
271
|
+
// What was judged waits no longer; a pair the answer left out is asked again.
|
|
272
|
+
if (named.length > 0) keepNamed(db, namedPairs(db).filter((p) => !db.prepare('SELECT 1 FROM memory_pairs WHERE a = ? AND b = ?').get(p.a, p.b)));
|
|
273
|
+
});
|
|
274
|
+
log(`judged ${judged} of ${pairs.length} pairs`);
|
|
275
|
+
// Nothing judged: the same pairs would be asked again in the same words.
|
|
276
|
+
if (judged === 0) break;
|
|
277
|
+
}
|
|
278
|
+
return out;
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
const REPEATS_SCHEMA = {
|
|
282
|
+
type: 'object',
|
|
283
|
+
properties: {
|
|
284
|
+
repeats: {
|
|
285
|
+
type: 'array',
|
|
286
|
+
items: { type: 'object', properties: { rule: { type: 'integer' }, same_as: { type: 'integer' } }, required: ['rule', 'same_as'], additionalProperties: false },
|
|
287
|
+
},
|
|
288
|
+
},
|
|
289
|
+
required: ['repeats'],
|
|
290
|
+
additionalProperties: false,
|
|
291
|
+
};
|
|
292
|
+
|
|
293
|
+
/**
|
|
294
|
+
* The same rule said twice in other words — "I prefer short answers", then "keep your replies brief" — shares no word,
|
|
295
|
+
* so no pair index finds it. Each rule not yet checked is shown beside the other rules where it applies, and the local
|
|
296
|
+
* model names any it says again: one call for a batch, each rule checked once (mem0 checks a new fact against its
|
|
297
|
+
* neighbours the same way). What it names is judged on its own next, and raised for the user only as that judgement
|
|
298
|
+
* says; nothing is retired here.
|
|
299
|
+
*/
|
|
300
|
+
async function checkRepeats(db, { now }) {
|
|
301
|
+
const out = { applied: [], dropped: [], usage: NO_USAGE, calls: 0, error: null };
|
|
302
|
+
const rules = tidyable(db).filter((m) => m.type === 'preference' || m.type === 'decision');
|
|
303
|
+
const from = Number(getMeta(db, REPEATS_CURSOR) ?? 0);
|
|
304
|
+
const batch = rules.filter((m) => m.id > from).slice(0, REPEATS_PER_CALL);
|
|
305
|
+
if (batch.length === 0) return out;
|
|
306
|
+
const reaches = new Set(batch.map((m) => m.scope));
|
|
307
|
+
const near = rules.filter((m) => m.scope === 'global' || reaches.has(m.scope) || reaches.has('global')).slice(-REPEAT_NEIGHBOURS);
|
|
308
|
+
const shown = new Map([...near, ...batch].map((m) => [m.id, m]));
|
|
309
|
+
if (shown.size < 2) {
|
|
310
|
+
setMeta(db, REPEATS_CURSOR, batch.at(-1).id);
|
|
311
|
+
return out;
|
|
312
|
+
}
|
|
313
|
+
const prompt = [
|
|
314
|
+
'Rules said twice — for each rule checked, the rule in the list that says the same thing in other words, if one does:',
|
|
315
|
+
'Rules checked:',
|
|
316
|
+
...batch.map((m) => `[m${m.id}] ${clip(m.body, 300)}`),
|
|
317
|
+
'All rules:',
|
|
318
|
+
...[...shown.values()].sort((x, y) => x.id - y.id).map(describe),
|
|
319
|
+
].join('\n');
|
|
320
|
+
const { result, log } = await askLocal(db, { prompt, schema: REPEATS_SCHEMA, kind: 'dream-repeats', now });
|
|
321
|
+
out.calls = 1;
|
|
322
|
+
out.usage = result.usage;
|
|
323
|
+
if (!result.ok) {
|
|
324
|
+
out.error = result.error;
|
|
325
|
+
log(result.error);
|
|
326
|
+
return out;
|
|
327
|
+
}
|
|
328
|
+
const answers = Array.isArray(result.data?.repeats) ? result.data.repeats : null;
|
|
329
|
+
// Checked, whatever came back: a bad answer is not asked again in the same words.
|
|
330
|
+
setMeta(db, REPEATS_CURSOR, batch.at(-1).id);
|
|
331
|
+
if (!answers) {
|
|
332
|
+
out.dropped.push('repeats: the answer had no list');
|
|
333
|
+
log('the answer had no list of repeats');
|
|
334
|
+
return out;
|
|
335
|
+
}
|
|
336
|
+
const checked = new Set(batch.map((m) => m.id));
|
|
337
|
+
tx(db, () => {
|
|
338
|
+
for (const answer of answers) {
|
|
339
|
+
const a = shown.get(Number(answer?.rule));
|
|
340
|
+
const b = shown.get(Number(answer?.same_as));
|
|
341
|
+
if (!a || !b || a.id === b.id || !checked.has(a.id)) {
|
|
342
|
+
out.dropped.push(`repeat m${JSON.stringify(answer?.rule)} of m${JSON.stringify(answer?.same_as)}: not a rule checked and another rule shown`);
|
|
343
|
+
continue;
|
|
344
|
+
}
|
|
345
|
+
if (a.scope !== b.scope && a.scope !== 'global' && b.scope !== 'global') {
|
|
346
|
+
out.dropped.push(`repeat m${a.id} of m${b.id}: they hold in different projects`);
|
|
347
|
+
continue;
|
|
348
|
+
}
|
|
349
|
+
// Named without a note of its own: the judgement says it.
|
|
350
|
+
nameForJudging(db, { label: 'same', a: a.id, b: b.id, note: null });
|
|
351
|
+
}
|
|
352
|
+
});
|
|
353
|
+
log(`checked ${batch.length} rules for repeats`);
|
|
354
|
+
return out;
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
/**
|
|
358
|
+
* Search terms and a topic for memories that have none, so a search in other words finds them: the next batch from
|
|
359
|
+
* where the last tidy stopped. Each term passes the same check as the scribe's (`storedKeys`); a bad one is dropped,
|
|
360
|
+
* never stored, and a topic is a lowercase word or nothing. What a memory already has is never replaced.
|
|
361
|
+
*/
|
|
362
|
+
async function fillKeys(db, { now }) {
|
|
363
|
+
const out = { applied: [], dropped: [], usage: NO_USAGE, calls: 0, error: null };
|
|
364
|
+
const lacking = tidyable(db).filter((m) => m.keys === null || m.topic === null);
|
|
365
|
+
const from = Number(getMeta(db, KEYS_CURSOR) ?? 0);
|
|
366
|
+
const batch = [...lacking.filter((m) => m.id > from), ...lacking.filter((m) => m.id <= from)].slice(0, KEYS_PER_TIDY);
|
|
367
|
+
if (batch.length === 0) return out;
|
|
368
|
+
const prompt = ['Search terms — a topic and keys for each memory:', ...batch.map((m) => `[m${m.id}] ${clip(m.body, 300)}`)].join('\n');
|
|
369
|
+
const { result, log } = await askLocal(db, { prompt, schema: KEYS_SCHEMA, kind: 'dream-keys', now });
|
|
370
|
+
out.calls = 1;
|
|
371
|
+
out.usage = result.usage;
|
|
372
|
+
if (!result.ok) {
|
|
373
|
+
out.error = result.error;
|
|
374
|
+
log(result.error);
|
|
375
|
+
return out;
|
|
376
|
+
}
|
|
377
|
+
const answers = Array.isArray(result.data?.memories) ? result.data.memories : null;
|
|
378
|
+
if (!answers) {
|
|
379
|
+
out.dropped.push('search terms: the answer had no list of memories');
|
|
380
|
+
log('the answer had no list of memories');
|
|
381
|
+
setMeta(db, KEYS_CURSOR, batch.at(-1).id);
|
|
382
|
+
return out;
|
|
383
|
+
}
|
|
384
|
+
const asked = new Map(batch.map((m) => [m.id, m]));
|
|
385
|
+
const given = [];
|
|
386
|
+
tx(db, () => {
|
|
387
|
+
for (const answer of answers) {
|
|
388
|
+
const m = asked.get(Number(answer?.id));
|
|
389
|
+
if (!m) {
|
|
390
|
+
out.dropped.push(`search terms for m${JSON.stringify(answer?.id)}: not one of the memories asked`);
|
|
391
|
+
continue;
|
|
392
|
+
}
|
|
393
|
+
const topic = m.topic === null && typeof answer.topic === 'string' && TOPIC.test(answer.topic.trim()) && redact(answer.topic).count === 0 ? answer.topic.trim() : null;
|
|
394
|
+
const keys = m.keys === null ? storedKeys(answer.keys, m.body) : null;
|
|
395
|
+
if (!topic && !keys) continue;
|
|
396
|
+
db.prepare('UPDATE memories SET topic = COALESCE(topic, ?), keys = COALESCE(keys, ?) WHERE id = ?').run(topic, keys, m.id);
|
|
397
|
+
asked.delete(m.id);
|
|
398
|
+
given.push(`m${m.id}`);
|
|
399
|
+
}
|
|
400
|
+
});
|
|
401
|
+
setMeta(db, KEYS_CURSOR, batch.at(-1).id);
|
|
402
|
+
if (given.length > 0) out.applied.push(`search terms for ${given.length} memor${given.length === 1 ? 'y' : 'ies'}: ${given.join(', ')}`);
|
|
403
|
+
log(`search terms for ${given.length} of ${batch.length}`);
|
|
404
|
+
return out;
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
/**
|
|
408
|
+
* Whether the weekly tidy is owed: a week since the last one, with something to tidy. Never tidied, the week runs from
|
|
409
|
+
* the oldest memory — memory a day old has nothing to tidy yet.
|
|
410
|
+
*/
|
|
411
|
+
function tidyOwed(db, now) {
|
|
412
|
+
const since = getMeta(db, LAST_TIDY) ?? db.prepare(`SELECT MIN(created_at) AS at FROM memories WHERE state = 'active' AND scan_key IS NULL`).get()?.at;
|
|
413
|
+
if (!since || Date.parse(now) - Date.parse(since) < TIDY_EVERY_MS) return false;
|
|
414
|
+
return Boolean(db.prepare(`SELECT 1 FROM memories WHERE state = 'active' AND scan_key IS NULL LIMIT 1`).get());
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
/**
|
|
418
|
+
* Whether a consolidation pass is worth running by itself now: three finished sessions are waiting to be read side by
|
|
419
|
+
* side, or a week has gone since memory was last tidied and there is memory to tidy. Pure: it reads, never runs.
|
|
420
|
+
*/
|
|
421
|
+
export function tidyDue(db, now = new Date().toISOString()) {
|
|
422
|
+
return sessionsAwaitingDream(db, now, SESSIONS_TO_TIDY).length >= SESSIONS_TO_TIDY || tidyOwed(db, now);
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
/**
|
|
426
|
+
* The tidy steps of a pass: weekly, search terms filled in; rules checked for being said twice; and pairs judged — those
|
|
427
|
+
* the check named first — in one call, or a few in the weekly tidy.
|
|
428
|
+
*/
|
|
429
|
+
async function tidyMemory(db, { weekly, now }) {
|
|
430
|
+
const steps = [];
|
|
431
|
+
if (weekly) steps.push(await fillKeys(db, { now }));
|
|
432
|
+
steps.push(await checkRepeats(db, { now }));
|
|
433
|
+
steps.push(await judgePairs(db, { calls: weekly ? TIDY_JUDGE_CALLS : JUDGE_CALLS, now }));
|
|
434
|
+
return {
|
|
435
|
+
applied: steps.flatMap((s) => s.applied),
|
|
436
|
+
dropped: steps.flatMap((s) => s.dropped),
|
|
437
|
+
usage: steps.reduce((u, s) => added(u, s.usage), NO_USAGE),
|
|
438
|
+
calls: steps.reduce((n, s) => n + s.calls, 0),
|
|
439
|
+
error: steps.find((s) => s.error)?.error ?? null,
|
|
440
|
+
};
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
/**
|
|
444
|
+
* The pairs the reading of the conversations named — a clash, one thing said twice — judged on their own before anything
|
|
445
|
+
* is put to the user. With no local model to judge them (the tidy steps never call a paid one), they are raised as named.
|
|
446
|
+
*/
|
|
447
|
+
async function settleNamed(db, { now }) {
|
|
448
|
+
if (namedPairs(db).length === 0) return { applied: [], dropped: [], usage: NO_USAGE };
|
|
449
|
+
if (tidiesLocally(db)) {
|
|
450
|
+
const judged = await judgePairs(db, { calls: 1, now, fill: false });
|
|
451
|
+
// Not answered, they wait for the next tidy, which judges them first.
|
|
452
|
+
return judged.error ? { ...judged, dropped: [...judged.dropped, `the pairs named wait to be judged: ${judged.error}`] } : judged;
|
|
453
|
+
}
|
|
454
|
+
const applied = [];
|
|
455
|
+
tx(db, () => {
|
|
456
|
+
for (const { a, b, named } of namedToJudge(db, Infinity)) {
|
|
457
|
+
const [kind, said] = named.label === 'same' ? ['lookalike', 'say the same thing'] : ['contradiction', 'cannot both be true'];
|
|
458
|
+
const note = named.note ?? `they ${said}`;
|
|
459
|
+
if (raiseNotice(db, { kind, a: a.id, b: b.id, note, now })) applied.push(`raised for the user: m${a.id} and m${b.id} ${said} — ${note}`);
|
|
460
|
+
}
|
|
461
|
+
keepNamed(db, []);
|
|
462
|
+
});
|
|
463
|
+
return { applied, dropped: [], usage: NO_USAGE };
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
/**
|
|
467
|
+
* The consolidation pass, which is also memory's tidy: the conversations waiting are read side by side, and memory
|
|
468
|
+
* itself is gone over — pairs judged, and weekly, search terms filled in. `sweep` asks for the weekly part now; it is
|
|
469
|
+
* done anyway once a week has gone. With no conversation waiting and no tidy owed, there is nothing to do.
|
|
470
|
+
*/
|
|
471
|
+
export async function runDream(db, { now = new Date().toISOString(), force = false, sweep = false, file = true } = {}) {
|
|
472
|
+
// Anything still unfiled belongs to the writer first; consolidation reads its results. The memory pass files first
|
|
473
|
+
// itself, so it can say what the filing did, and asks for none here.
|
|
474
|
+
if (file) await runScribe(db, { now });
|
|
65
475
|
|
|
66
476
|
return withLock('dream', async () => {
|
|
67
477
|
// A conversation the writer has not finished filing (it is busy, failing, or behind) waits for it.
|
|
@@ -73,14 +483,25 @@ export async function runDream(db, { now = new Date().toISOString(), force = fal
|
|
|
73
483
|
? sessionsAwaitingDream(db, now, MAX_SESSIONS, { includeOpen: true }).filter((s) => !unfiled.has(s.id) && !over.some((o) => o.id === s.id))
|
|
74
484
|
: [];
|
|
75
485
|
const sessions = [...over, ...open].slice(0, MAX_SESSIONS).sort((a, b) => a.started_at.localeCompare(b.started_at));
|
|
76
|
-
|
|
486
|
+
const weekly = sweep || tidyOwed(db, now);
|
|
487
|
+
if (sessions.length === 0 && !weekly) return { skipped: 'no finished sessions are waiting' };
|
|
488
|
+
|
|
489
|
+
// Memory is tidied first, so what the conversations bring is read against it as it now stands.
|
|
490
|
+
const tidy = tidiesLocally(db) ? await tidyMemory(db, { weekly, now }) : { applied: [], dropped: [], usage: NO_USAGE, calls: 0, error: null };
|
|
491
|
+
// A tidy the local model could not answer is owed still; one that had nothing to ask is done.
|
|
492
|
+
if (weekly && !tidy.error) setMeta(db, LAST_TIDY, now);
|
|
493
|
+
if (sessions.length === 0) {
|
|
494
|
+
if (tidy.calls === 0) return { skipped: 'no finished sessions are waiting, and memory has nothing to tidy' };
|
|
495
|
+
if (tidy.error && tidy.applied.length === 0) return { ok: false, error: tidy.error, applied: [], dropped: tidy.dropped, usage: tidy.usage };
|
|
496
|
+
return { ok: true, error: null, applied: tidy.applied, dropped: tidy.dropped, usage: tidy.usage, sessions: 0 };
|
|
497
|
+
}
|
|
77
498
|
|
|
78
499
|
const bundle = buildBundle(db, sessions);
|
|
79
500
|
let lastTurn = 0;
|
|
80
501
|
for (const id of bundle.turns.keys()) lastTurn = id;
|
|
81
502
|
const outcome = await askAndApply(db, {
|
|
82
503
|
kind: 'dream', promptFile: 'dream.md', bundle,
|
|
83
|
-
ops: ['add', 'supersede', 'gotcha', 'checkpoint', 'contradiction', 'procedure'], sessionId: null, now,
|
|
504
|
+
ops: ['add', 'supersede', 'gotcha', 'checkpoint', 'contradiction', 'same', 'procedure'], sessionId: null, now,
|
|
84
505
|
});
|
|
85
506
|
if (outcome.ok && over.length > 0) {
|
|
86
507
|
// A resumed session may have received a new turn while the model read the snapshot.
|
|
@@ -88,10 +509,15 @@ export async function runDream(db, { now = new Date().toISOString(), force = fal
|
|
|
88
509
|
AND NOT EXISTS (SELECT 1 FROM user_turns t WHERE t.session_id = sessions.id AND t.id > ?)`)
|
|
89
510
|
.run(...over.map((s) => s.id), lastTurn);
|
|
90
511
|
}
|
|
91
|
-
|
|
512
|
+
// The slice of the oldest moves on only past a pass that read it.
|
|
513
|
+
if (outcome.ok && bundle.cursor !== null) setMeta(db, SWEEP_CURSOR, bundle.cursor);
|
|
514
|
+
const named = outcome.ok ? await settleNamed(db, { now }) : { applied: [], dropped: [], usage: NO_USAGE };
|
|
515
|
+
return {
|
|
516
|
+
...outcome,
|
|
517
|
+
applied: [...tidy.applied, ...outcome.applied, ...named.applied],
|
|
518
|
+
dropped: [...tidy.dropped, ...outcome.dropped, ...named.dropped],
|
|
519
|
+
usage: added(added(tidy.usage, outcome.usage ?? NO_USAGE), named.usage),
|
|
520
|
+
sessions: sessions.length,
|
|
521
|
+
};
|
|
92
522
|
});
|
|
93
523
|
}
|
|
94
|
-
|
|
95
|
-
export function dreamStatus(db, now = new Date().toISOString()) {
|
|
96
|
-
return [`finished sessions waiting: ${sessionsAwaitingDream(db, now, 1000).length} (runs by itself at 3)`];
|
|
97
|
-
}
|