sumo-agents 0.6.1 → 0.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/scribe.mjs CHANGED
@@ -4,53 +4,50 @@ import { join } from 'node:path';
4
4
  import { applyOps } from './apply.mjs';
5
5
  import { resolveAnthropicCredential } from './auth.mjs';
6
6
  import { MODELS, offLine, usableModels } from './catalog.mjs';
7
- import { getMeta, openDb, setMeta, tx } from './db.mjs';
7
+ import { getMeta, openDb, tx } from './db.mjs';
8
+ import { judgeNewPairs } from './dream.mjs';
9
+ import { extractorVersion, filingState, notePass } from './ledger.mjs';
8
10
  import { callModel, failure, logRun } from './model.mjs';
9
11
  import { appendLog } from './logs.mjs';
10
12
  import { paths, REPO_ROOT, SELF } from './paths.mjs';
11
13
  import { aliasesOf, listProjects } from './projects.mjs';
12
14
  import { line } from './render.mjs';
13
- import { currentProject, markScribed, pendingTurns } from './sessions.mjs';
15
+ import { currentProject, markScribed, sessionJournal, sessionsAwaitingExtraction, turnsAwaitingExtraction } from './sessions.mjs';
14
16
  import { CONFIG_DEFAULTS } from './setup.mjs';
15
- import { ftsQuery, looksDurable } from './text.mjs';
16
- import { assistantReplies } from './transcript.mjs';
17
+ import { ftsQuery, head } from './text.mjs';
18
+ import { abridge } from './transcript.mjs';
17
19
 
18
20
  const LOCK_STALE_MS = 5 * 60_000;
19
- const QUIET_MS = 5 * 60_000;
20
- const TURNS_TO_RUN = 3;
21
- const MAX_REPLIES_PER_SESSION = 6;
22
21
  const MAX_RELATED = 10;
22
+ /** Conversations one pass reads, and model calls it makes: the rest wait for the next pass, which is a session away. */
23
+ const MAX_SESSIONS = 5;
24
+ const MAX_CALLS = 8;
25
+ /** Calls on the hosted fallback a pass started by itself may make: nobody is there to see what a longer run costs. */
26
+ const MAX_PAID_AUTO = 2;
23
27
  /**
24
- * How much of the user's words one run reads, so a batch always fits the local model's 32k context with room to think
25
- * and answer. Counted in characters, which a pasted log turns into more tokens than prose does — the room is left for
26
- * that. Thirty stored turns of 4,000 characters each would not fit; what does not fit waits for the next run.
28
+ * How much of a conversation one model call reads, so a batch always fits the local model's 32k context with room to
29
+ * think and answer. Counted in characters, which a pasted log turns into more tokens than prose does — the room is
30
+ * left for that. A long conversation is read in several calls.
27
31
  */
28
32
  const BATCH_CHARS = 40_000;
33
+ /** How often one batch is tried before it is split in half, and one turn alone before it is set aside. */
34
+ const ATTEMPTS = 3;
35
+ const NO_USAGE = { inputTokens: 0, outputTokens: 0, costUsd: 0, cacheReadTokens: null, cacheCreationTokens: null };
36
+ const added = (x, y) => ({ ...x, inputTokens: x.inputTokens + y.inputTokens, outputTokens: x.outputTokens + y.outputTokens, costUsd: x.costUsd + y.costUsd });
29
37
 
30
- /** The pending turns one run reads: the oldest first, as many as fit, and always at least one. */
31
- function batchOf(turns) {
32
- const batch = [];
33
- let used = 0;
34
- for (const t of turns) {
35
- if (batch.length > 0 && used + t.text.length > BATCH_CHARS) break;
36
- batch.push(t);
37
- used += t.text.length;
38
- }
39
- return batch;
40
- }
38
+ /** The model the pass is set to: `memory.model`, the one setting. */
39
+ const memoryModel = (db) => getMeta(db, 'config.memory.model') ?? CONFIG_DEFAULTS['memory.model'];
40
+ /** Whether the user turned the writer off: nothing gets filed, so nothing should wait for it to be. */
41
+ const writerOff = (db) => memoryModel(db) === 'off';
41
42
 
42
43
  /**
43
- * Whether the writer should run now. It is batched on purpose: each call
44
- * carries a few thousand tokens of fixed overhead, so one call per message
45
- * would cost many times more for the same memories.
44
+ * Whether a session starting or ending should start the memory pass: a finished conversation waits to be read, and
45
+ * filing is neither off nor paused. A conversation is read once, when it is over — a turn ending starts nothing.
46
46
  */
47
- export function shouldRun(db, { event, now }) {
48
- const pending = pendingTurns(db);
49
- if (pending.length === 0) return false;
50
- if (event !== 'stop') return true; // session start and end always sweep up what is left
51
- if (pending.some((t) => looksDurable(t.text)) || pending.length >= TURNS_TO_RUN) return true;
52
- const last = getMeta(db, 'scribe.last_run');
53
- return !last || Date.parse(now) - Date.parse(last) >= QUIET_MS;
47
+ export function shouldRun(db, { now }) {
48
+ if (writerOff(db)) return false;
49
+ if (sessionsAwaitingExtraction(db, { now, limit: 1 }).length === 0) return false;
50
+ return !filingState(db, now).paused;
54
51
  }
55
52
 
56
53
  /**
@@ -138,7 +135,7 @@ function projectEnum(db) {
138
135
  return listProjects(db, { includeArchived: true }).map((p) => p.slug);
139
136
  }
140
137
 
141
- export function knownProjectsLine(db) {
138
+ function knownProjectsLine(db) {
142
139
  const all = listProjects(db);
143
140
  if (all.length === 0) return 'Known projects: none';
144
141
  return `Known projects: ${all.map((p) => [p.slug, ...aliasesOf(db, p.slug)].join(' / ')).join(' · ')}`;
@@ -151,29 +148,28 @@ export function opsSchema(db, ops) {
151
148
  type: { type: 'string', enum: ['preference', 'fact', 'decision'] },
152
149
  scope: { type: 'string', enum: ['global', ...slugs.map((s) => `project:${s}`)] },
153
150
  topic: { type: 'string', description: 'one lowercase word: git, testing, style, writing, tooling, deploy…' },
151
+ // Only a trap has a sentence the model writes. What the user stated is stored in their words: no body is asked for.
154
152
  body: { type: 'string', description: 'one plain sentence, written as a standing fact or rule' },
155
- turn: { type: ['integer', 'null'], description: 'the number from the [tN] tag, or null when this is an inferred pattern' },
156
- quote: { type: ['string', 'null'], description: 'an exact run of words from that user turn, or null when this is an inferred pattern' },
153
+ // Never null: a statement with no turn and no words of the user's behind it is a guess, and guesses are not kept.
154
+ turn: { type: 'integer', description: 'the number from the [tN] tag of the user turn it was said in' },
155
+ quote: { type: 'string', description: "the user's exact words, copied from that turn letter for letter, on one line: they are stored as the memory, so do not reword, shorten or correct them" },
157
156
  // Required like the rest, and may be empty: an optional field is a combination more for the grammar to compile.
158
- keys: { type: 'array', items: { type: 'string' }, description: 'up to 6 short lowercase search terms it is about that are not words of body, or []' },
157
+ keys: { type: 'array', items: { type: 'string' }, description: 'up to 6 short lowercase search terms it is about that are not among its own words, or []' },
159
158
  };
160
159
  const branch = (op, properties) => {
161
160
  const all = { op: { type: 'string', enum: [op] }, ...properties };
162
161
  return { type: 'object', properties: all, required: Object.keys(all), additionalProperties: false };
163
162
  };
164
163
  const branches = {
165
- add: branch('add', { type: field.type, scope: field.scope, topic: field.topic, body: field.body, turn: field.turn, quote: field.quote, keys: field.keys }),
164
+ add: branch('add', { type: field.type, scope: field.scope, topic: field.topic, turn: field.turn, quote: field.quote, keys: field.keys }),
166
165
  supersede: branch('supersede', {
167
166
  old: { type: 'integer', description: 'the id of the existing memory this replaces' },
168
- type: field.type, scope: field.scope, topic: field.topic, body: field.body, turn: field.turn, quote: field.quote, keys: field.keys,
167
+ type: field.type, scope: field.scope, topic: field.topic, turn: field.turn, quote: field.quote, keys: field.keys,
169
168
  }),
170
169
  gotcha: branch('gotcha', { scope: field.scope, body: field.body, keys: field.keys }),
171
170
  checkpoint: slugs.length > 0
172
171
  ? branch('checkpoint', { project: { type: 'string', enum: slugs }, done: { type: 'string' }, next: { type: ['string', 'null'] } })
173
172
  : null,
174
- contradiction: branch('contradiction', { ids: { type: 'array', items: { type: 'integer' } }, note: { type: 'string' } }),
175
- same: branch('same', { ids: { type: 'array', items: { type: 'integer' } }, note: { type: 'string' } }),
176
- procedure: branch('procedure', { scope: field.scope, title: { type: 'string' }, cue: { type: 'string' }, body: field.body }),
177
173
  };
178
174
  return {
179
175
  type: 'object',
@@ -183,7 +179,7 @@ export function opsSchema(db, ops) {
183
179
  };
184
180
  }
185
181
 
186
- export function relatedMemories(db, texts, scopes) {
182
+ function relatedMemories(db, texts, scopes) {
187
183
  const seen = new Map();
188
184
  for (const text of texts) {
189
185
  const match = ftsQuery(text);
@@ -201,112 +197,363 @@ export function relatedMemories(db, texts, scopes) {
201
197
  return [...seen.values()].slice(0, MAX_RELATED);
202
198
  }
203
199
 
204
- /** What the writer is shown: the user's words, the assistant's replies around them, and nothing else. */
205
- export function buildBundle(db) {
206
- const turns = batchOf(pendingTurns(db));
207
- if (turns.length === 0) return null;
200
+ /** The most of one turn a model call is shown, and of the replies after it. */
201
+ const TURN_SHOWN = 30_000;
202
+ const REPLIES_PER_TURN = 2;
208
203
 
209
- const bySession = new Map();
210
- for (const t of turns) bySession.set(t.session_id, [...(bySession.get(t.session_id) ?? []), t]);
211
-
212
- const scopes = new Set(['global']);
213
- const sections = [];
214
- for (const [sessionId, sessionTurns] of bySession) {
215
- const project = currentProject(db, sessionId);
216
- if (project) scopes.add(`project:${project}`);
217
- const session = db.prepare('SELECT transcript_path FROM sessions WHERE id = ?').get(sessionId);
218
- const replies = session?.transcript_path ? assistantReplies(session.transcript_path, sessionTurns[0].ts, MAX_REPLIES_PER_SESSION).slice(-MAX_REPLIES_PER_SESSION) : [];
219
-
220
- const events = [
221
- ...sessionTurns.map((t) => ({ ts: t.ts, text: `[t${t.id}] user: ${t.text}` })),
222
- ...replies.map((r) => ({ ts: r.ts, text: `[assistant]: ${r.text}` })),
223
- ].sort((a, b) => a.ts.localeCompare(b.ts));
224
-
225
- sections.push(
226
- `--- session${project ? ` (the conversation is about the project "${project}": statements that are not clearly general belong to project:${project})` : ' (no project in focus: scope is global unless a project is named)'}`,
227
- ...events.map((e) => e.text),
228
- );
204
+ /**
205
+ * A turn as the model is shown it: whole when it fits, else its head and its tail — a pasted log says what it is at
206
+ * the top and how it ended at the bottom. The journal keeps the turn whole, and a quote is checked against that.
207
+ */
208
+ function shownTurn(text) {
209
+ if (text.length <= TURN_SHOWN) return text;
210
+ const half = Math.floor(TURN_SHOWN / 2);
211
+ const tail = text.slice(-half);
212
+ // Either cut can fall inside an emoji; a lone half of one makes the whole request one the API refuses.
213
+ return `${head(text, half)}\n[… ${text.length - 2 * half} characters of this turn not shown …]\n${/^[\uDC00-\uDFFF]/.test(tail) ? tail.slice(1) : tail}`;
214
+ }
215
+
216
+ /**
217
+ * A conversation's turns that wait to be read, each with the replies that answered it, in the order of their ids —
218
+ * the order the ledger's `first_turn`…`last_turn` ranges are in, which a turn replayed late does not share with time.
219
+ */
220
+ function unitsOf(journal, waiting) {
221
+ const units = [];
222
+ let current = null;
223
+ for (const entry of journal) {
224
+ if (entry.kind === 'turn') {
225
+ current = waiting.has(entry.id) ? { turn: entry, text: shownTurn(entry.text), replies: [] } : null;
226
+ if (current) units.push(current);
227
+ } else if (current) {
228
+ current.replies = [...current.replies, { ts: entry.ts, text: abridge(entry.text) }].slice(-REPLIES_PER_TURN);
229
+ }
229
230
  }
231
+ for (const unit of units) unit.size = unit.text.length + unit.replies.reduce((n, r) => n + r.text.length, 0);
232
+ return units.sort((a, b) => a.turn.id - b.turn.id);
233
+ }
230
234
 
231
- const related = relatedMemories(db, turns.map((t) => t.text), [...scopes]);
235
+ /** The turns one model call reads: the first of those waiting, as many as fit, and always at least one. */
236
+ function naturalBatch(units) {
237
+ const batch = [];
238
+ let used = 0;
239
+ for (const unit of units) {
240
+ if (batch.length > 0 && used + unit.size > BATCH_CHARS) break;
241
+ batch.push(unit);
242
+ used += unit.size;
243
+ }
244
+ return batch;
245
+ }
246
+
247
+ /**
248
+ * What the model is shown for one batch: today's date, the projects, memories that may be related, then the user's
249
+ * words with the assistant's replies to them, as they were said — and nothing else. `related` false leaves memory
250
+ * out: what an extractor would file from a conversation is then a matter of the conversation alone.
251
+ */
252
+ function bundleOf(db, sessionId, batch, { now, related = true }) {
253
+ const project = batch.map((u) => u.turn.project).filter(Boolean).at(-1) ?? currentProject(db, sessionId);
254
+ const scopes = ['global', ...(project ? [`project:${project}`] : [])];
255
+ const said = [...batch].sort((a, b) => a.turn.ts.localeCompare(b.turn.ts) || a.turn.id - b.turn.id);
256
+ const known = related ? relatedMemories(db, batch.map((u) => u.turn.text), scopes) : [];
232
257
  const prompt = [
258
+ `Today's date: ${now.slice(0, 10)}`,
233
259
  knownProjectsLine(db),
234
- related.length > 0 ? `Existing memories that may be related (use supersede, not add, when one of these is being replaced):\n${related.map(line).join('\n')}` : 'Existing related memories: none',
235
- ...sections,
260
+ ...(related ? [known.length > 0 ? `Existing memories that may be related (use supersede, not add, when one of these is being replaced):\n${known.map(line).join('\n')}` : 'Existing related memories: none'] : []),
261
+ `--- session${project ? ` (the conversation is about the project "${project}": statements that are not clearly general belong to project:${project})` : ' (no project in focus: scope is global unless a project is named)'}`,
262
+ ...said.flatMap((u) => [`[t${u.turn.id}] user: ${u.text}`, ...u.replies.map((r) => `[assistant]: ${r.text}`)]),
236
263
  ].join('\n');
264
+ return { prompt, turns: new Map(batch.map((u) => [u.turn.id, u.turn])) };
265
+ }
266
+
267
+ /**
268
+ * How often this exact batch has failed under this extractor, by its own fault: a row's `attempts` is the number of
269
+ * tries that count, its own included, and a try made while the model answered nothing at all does not (`runScribe`).
270
+ */
271
+ const failedTimes = (db, sessionId, version, batch) =>
272
+ db
273
+ .prepare(`SELECT COALESCE(MAX(attempts), 0) AS n FROM extractions WHERE session_id = ? AND extractor_version = ? AND first_turn = ? AND last_turn = ? AND status = 'failed'`)
274
+ .get(sessionId, version, batch[0].turn.id, batch.at(-1).turn.id).n;
275
+
276
+ /**
277
+ * The next batch of a conversation to read, or null when nothing of it waits: `{ batch, attempts }`. A batch that has
278
+ * failed three times is not asked again whole — its first half is, and so on down; a single turn that failed three
279
+ * times alone comes back with `quarantine`, to be set aside so the turns after it are read.
280
+ */
281
+ function nextBatch(db, sessionId, version) {
282
+ // Read under any extractor is read: only a rebuild reads a conversation again under this one.
283
+ const waiting = turnsAwaitingExtraction(db, sessionId);
284
+ if (waiting.length === 0) return null;
285
+ let batch = naturalBatch(unitsOf(sessionJournal(db, sessionId), new Set(waiting.map((t) => t.id))));
286
+ for (;;) {
287
+ const failed = failedTimes(db, sessionId, version, batch);
288
+ if (failed < ATTEMPTS) return { batch, attempts: failed + 1 };
289
+ if (batch.length === 1) return { batch, attempts: failed, quarantine: true };
290
+ batch = batch.slice(0, Math.ceil(batch.length / 2));
291
+ }
292
+ }
237
293
 
238
- return { prompt, turns: new Map(turns.map((t) => [t.id, t])) };
294
+ /** Exactly what the next pass asked for by hand would send first, or null when no conversation waits to be read. */
295
+ export function buildBundle(db, { now = new Date().toISOString() } = {}) {
296
+ const version = extractorVersion(db);
297
+ for (const session of sessionsAwaitingExtraction(db, { now, limit: MAX_SESSIONS, includeOpen: true })) {
298
+ const next = nextBatch(db, session.id, version);
299
+ if (next && !next.quarantine) return bundleOf(db, session.id, next.batch, { now });
300
+ }
301
+ return null;
239
302
  }
240
303
 
241
- /** The API model a pass is retried on when the local one could not answer: the cheapest one that is on, or none. */
304
+ /** The API model a call is retried on when the local one could not answer: the cheapest one that is on, or none. */
242
305
  const fallbackModel = (db) => usableModels(db)[0] ?? null;
306
+ const EXTRACTS = ['add', 'supersede', 'gotcha', 'checkpoint'];
243
307
 
244
- /** Shared by the writer and the consolidation pass: ask, validate, record what it cost. */
245
- export async function askAndApply(db, { kind, promptFile, bundle, ops, sessionId, now }) {
246
- // One setting for the whole memory pass, filing and tidying alike.
247
- let model = getMeta(db, 'config.memory.model') ?? CONFIG_DEFAULTS['memory.model'];
248
- const system = readFileSync(join(REPO_ROOT, 'prompts', promptFile), 'utf8').trim();
249
- const schema = opsSchema(db, ops);
308
+ /**
309
+ * One batch put to the model `memory.model` names: `{ ok, error, ops, usage, model, result, fallbackUsed }`. A local
310
+ * model that could not answer (server down, bundle past its context, answer cut off) is one failed row of the model
311
+ * ledger, then one more try on the hosted fallback — when a credential is there to pay for it and a model is on to
312
+ * take it — and `fallbackUsed` says so; `fallback: false` keeps the call on the model as set, for a pass that has
313
+ * paid for as much as it may. `quiet` keeps the model ledger out of it: a dry run writes nothing.
314
+ */
315
+ async function ask(db, { bundle, now, quiet = false, fallback: mayPay = true }) {
316
+ let model = memoryModel(db);
317
+ const log = (result, note) => {
318
+ if (!quiet) logRun(db, { kind: 'scribe', model, result, note, now });
319
+ };
320
+ const system = readFileSync(join(REPO_ROOT, 'prompts', 'scribe.md'), 'utf8').trim();
321
+ const schema = opsSchema(db, EXTRACTS);
250
322
  // A writer set to a model the user has since turned off is not called: the refusal is the pass's failure, in the ledger like any other.
251
- const off = MODELS.includes(model) ? offLine(db, model, 'sumo config memory.model local') : null;
252
- let result = off ? failure(off) : await callModel(db, { system, prompt: bundle.prompt, schema, model, kind });
323
+ const off = MODELS.includes(model) ? offLine(db, model, 'set Memory › Filed by to local there') : null;
324
+ let result = off ? failure(off) : await callModel(db, { system, prompt: bundle.prompt, schema, model, kind: 'scribe' });
253
325
  // A local answer that failed (server down, bundle past its context, answer cut off) is one failed row, then one more try —
254
326
  // when a credential is there to pay for it and a model is on to take it.
255
- const fallback = !result.ok && model === 'local' && resolveAnthropicCredential() ? fallbackModel(db) : null;
327
+ const fallback = mayPay && !result.ok && model === 'local' && resolveAnthropicCredential() ? fallbackModel(db) : null;
256
328
  if (fallback) {
257
- logRun(db, { kind, model, result, note: `${result.error} — retrying on ${fallback}`, now });
329
+ log(result, `${result.error} — retrying on ${fallback}`);
258
330
  model = fallback;
259
- result = await callModel(db, { system, prompt: bundle.prompt, schema, model, kind });
331
+ result = await callModel(db, { system, prompt: bundle.prompt, schema, model, kind: 'scribe' });
260
332
  }
261
-
333
+ const answered = { usage: result.usage, model, result, fallbackUsed: Boolean(fallback), log };
262
334
  if (!result.ok) {
263
- logRun(db, { kind, model, result, note: result.error, now });
264
- return { ok: false, error: result.error, applied: [], dropped: [], usage: result.usage };
335
+ log(result, result.error);
336
+ return { ...answered, ok: false, error: result.error, ops: null };
265
337
  }
266
338
  // An answer in the wrong shape is a failed call, not an empty one: what it was shown must be read again.
267
339
  if (!Array.isArray(result.data?.ops)) {
268
340
  const error = 'the answer had no list of operations';
269
- logRun(db, { kind, model, result: { ...result, ok: false }, note: error, now });
270
- return { ok: false, error, applied: [], dropped: [], usage: result.usage };
341
+ log({ ...result, ok: false }, error);
342
+ return { ...answered, ok: false, error, ops: null };
271
343
  }
272
- const outcome = applyOps(db, result.data.ops, { source: kind, turns: bundle.turns, shown: bundle.shown, sessionId, now });
273
- logRun(db, { kind, model, result, note: `applied=${outcome.applied.length} dropped=${outcome.dropped.length}`, now });
344
+ return { ...answered, ok: true, error: null, ops: result.data.ops };
345
+ }
346
+
347
+ /**
348
+ * What the pair judge says of each statement in `ops` that has a look-alike in memory — a fact that may have changed.
349
+ * Code picks the pairs (a dry run of the rules); the local model only labels them. `labels` is undefined when nobody
350
+ * could be asked: the rules then put each pair to the user as a notice. Only the local model judges, never a paid one.
351
+ */
352
+ export async function labelsFor(db, ops, ctx) {
353
+ if (memoryModel(db) !== 'local') return { labels: undefined, usage: NO_USAGE };
354
+ const { pairs } = applyOps(db, ops, { ...ctx, dry: true });
355
+ if (pairs.length === 0) return { labels: undefined, usage: NO_USAGE };
356
+ return judgeNewPairs(db, pairs, ctx.now);
357
+ }
358
+
359
+ /** A model's operations through the rules: look-alikes judged first, then everything applied in one transaction. */
360
+ async function fileOps(db, ops, ctx) {
361
+ const { labels, usage } = await labelsFor(db, ops, ctx);
362
+ return { ...applyOps(db, ops, { ...ctx, labels }), usage };
363
+ }
364
+
365
+ /**
366
+ * One batch read and filed, with its row of the extraction ledger: written `running` before the model is called, so a
367
+ * pass that dies leaves the attempt behind it; closed `ok` with the model's answer as it came, or `failed` with why.
368
+ */
369
+ async function readBatch(db, { sessionId, batch, attempts, version, now, fallback }) {
370
+ const bundle = bundleOf(db, sessionId, batch, { now });
371
+ const ids = batch.map((u) => u.turn.id);
372
+ const runId = Number(
373
+ db
374
+ .prepare(`INSERT INTO extractions (session_id, extractor_version, status, attempts, first_turn, last_turn, started_at) VALUES (?, ?, 'running', ?, ?, ?, ?)`)
375
+ .run(sessionId, version, attempts, ids[0], ids.at(-1), now).lastInsertRowid,
376
+ );
377
+ const close = (status, { ops = null, error = null } = {}) =>
378
+ db.prepare('UPDATE extractions SET status = ?, ops_json = ?, error = ?, ended_at = ? WHERE id = ?').run(status, ops === null ? null : JSON.stringify(ops), error, now, runId);
379
+
380
+ const asked = await ask(db, { bundle, now, fallback });
381
+ if (!asked.ok) {
382
+ close('failed', { error: asked.error });
383
+ return { ok: false, error: asked.error, usage: asked.usage, fallbackUsed: asked.fallbackUsed, attempts, runId };
384
+ }
385
+ let outcome;
386
+ try {
387
+ outcome = await fileOps(db, asked.ops, { source: 'scribe', turns: bundle.turns, sessionId, now, derivedBy: version, runId });
388
+ } catch (cause) {
389
+ // A bug in the rules is this attempt's failure too, and still surfaces.
390
+ close('failed', { ops: asked.ops, error: `could not be applied: ${cause.message}` });
391
+ throw cause;
392
+ }
393
+ // Closed and marked together: a turn is never both read and still owed.
394
+ tx(db, () => {
395
+ close('ok', { ops: asked.ops });
396
+ markScribed(db, ids);
397
+ });
398
+ asked.log(asked.result, `applied=${outcome.applied.length} dropped=${outcome.dropped.length}`);
274
399
  for (const reason of outcome.dropped) appendLog('scribe.log', `${now} dropped — ${reason}\n`);
275
- return { ok: true, error: null, ...outcome, usage: result.usage };
400
+ if (outcome.refused) appendLog('scribe.log', `${now} nothing retired — ${outcome.refused}\n`);
401
+ return { ok: true, error: null, ...outcome, usage: added(asked.usage, outcome.usage), fallbackUsed: asked.fallbackUsed, model: asked.model, turns: ids.length };
276
402
  }
277
403
 
278
- /** Whether the user turned the writer off: nothing gets filed, so nothing should wait for it to be. */
279
- export const writerOff = (db) => (getMeta(db, 'config.memory.model') ?? CONFIG_DEFAULTS['memory.model']) === 'off';
404
+ /** A turn that failed three times alone is set aside, in the ledger and counted, so the turns after it are read. */
405
+ function quarantine(db, { sessionId, turn, attempts, version, now }) {
406
+ const last = db
407
+ .prepare(`SELECT error FROM extractions WHERE session_id = ? AND extractor_version = ? AND first_turn = ? AND last_turn = ? AND status = 'failed' ORDER BY id DESC LIMIT 1`)
408
+ .get(sessionId, version, turn.id, turn.id);
409
+ tx(db, () => {
410
+ db.prepare(`INSERT INTO extractions (session_id, extractor_version, status, attempts, first_turn, last_turn, error, started_at, ended_at) VALUES (?, ?, 'quarantined', ?, ?, ?, ?, ?, ?)`)
411
+ .run(sessionId, version, attempts, turn.id, turn.id, last?.error ?? null, now, now);
412
+ markScribed(db, [turn.id]);
413
+ });
414
+ return `t${turn.id} was set aside: read alone it failed ${attempts} times — ${last?.error ?? 'no reason kept'}`;
415
+ }
280
416
 
281
417
  /**
282
- * Files what was said since the last run. Turns are marked done only after the
283
- * answer was applied, so a failed call costs nothing but a retry.
418
+ * Whether the model answers at all: one turn that says nothing, put to it as any batch is. A batch that fails while
419
+ * this is answered failed by its own fault; one that fails while this fails too says nothing about the batch.
420
+ * `{ ok, fallbackUsed }`.
284
421
  */
285
- export async function runScribe(db, { now = new Date().toISOString() } = {}) {
422
+ async function modelAnswers(db, now, fallback) {
423
+ const turn = { id: 0, session_id: '', ts: now, text: 'hello', project: null };
424
+ const bundle = bundleOf(db, '', [{ turn, text: turn.text, replies: [] }], { now, related: false });
425
+ const asked = await ask(db, { bundle, now, quiet: true, fallback });
426
+ // In the model ledger when it read or cost something; a call that reached no model is the failed row already there.
427
+ if (asked.usage.inputTokens + asked.usage.outputTokens > 0) {
428
+ logRun(db, { kind: 'scribe', model: asked.model, result: asked.ok ? asked.result : { ...asked.result, ok: false }, note: `asked whether the model answers at all: ${asked.ok ? 'it does' : asked.error}`, now });
429
+ }
430
+ return { ok: asked.ok, fallbackUsed: asked.fallbackUsed };
431
+ }
432
+
433
+ /**
434
+ * The filing half of the memory pass: each finished conversation read once, in batches that fit the local model, and
435
+ * what the user stated in it filed. `auto` is a pass a session started by itself: it reads only conversations that
436
+ * are over, and nothing while filing is paused (ledger.mjs). Asked for by hand it also reads conversations still
437
+ * going on, and is how a paused filing is tried again.
438
+ *
439
+ * One batch failing stops only its own conversation for this pass, and is tried again by the next: three failures
440
+ * and it is read in halves, down to the one turn that cannot be read, which is set aside. That ladder is for a batch
441
+ * the model cannot read; a model that answers nothing at all is another matter — the pass fails, the try is not held
442
+ * against the batch, and three such passes in a row pause filing.
443
+ */
444
+ export async function runScribe(db, { now = new Date().toISOString(), auto = false } = {}) {
286
445
  if (writerOff(db)) return { skipped: 'memory.model is off' };
287
446
 
288
447
  return withLock('scribe', async () => {
289
- const bundle = buildBundle(db);
290
- if (!bundle) return { skipped: 'nothing new was said' };
448
+ const version = extractorVersion(db);
449
+ const state = filingState(db, now);
450
+ if (auto && state.paused) return { skipped: `filing is paused: ${state.reason}`, paused: true };
451
+ // The lock is ours: a row still `running` belongs to a pass that died, and is one failed attempt.
452
+ db.prepare(`UPDATE extractions SET status = 'failed', error = 'the pass that started it died before it finished', ended_at = ? WHERE status = 'running'`).run(now);
291
453
 
292
- const sessions = new Set([...bundle.turns.values()].map((t) => t.session_id));
293
- const outcome = await askAndApply(db, {
294
- kind: 'scribe', promptFile: 'scribe.md', bundle, ops: ['add', 'supersede', 'gotcha', 'checkpoint'],
295
- sessionId: sessions.size === 1 ? [...sessions][0] : null, now,
296
- });
454
+ const sessions = sessionsAwaitingExtraction(db, { now, limit: MAX_SESSIONS, includeOpen: !auto });
455
+ if (sessions.length === 0) return { skipped: 'nothing new was said' };
297
456
 
298
- setMeta(db, 'scribe.last_run', now);
299
- if (outcome.ok) {
300
- markScribed(db, [...bundle.turns.keys()]);
301
- setMeta(db, 'scribe.failures', 0);
302
- } else {
303
- setMeta(db, 'scribe.failures', Number(getMeta(db, 'scribe.failures') ?? 0) + 1);
304
- setMeta(db, 'scribe.last_error', outcome.error);
457
+ const pass = { applied: [], dropped: [], notes: [], usage: NO_USAGE, turns: 0, stuck: 0, down: false, error: null, fallbackUsed: false, direct: false, written: [] };
458
+ let calls = 0;
459
+ let alive = false;
460
+ // What the hosted fallback was asked, answered or not: a pass nobody asked for pays for two such calls and no more.
461
+ let paid = 0;
462
+ const mayPay = () => !auto || paid < MAX_PAID_AUTO;
463
+ reading: for (const session of sessions) {
464
+ while (calls < MAX_CALLS) {
465
+ const next = nextBatch(db, session.id, version);
466
+ if (!next) break;
467
+ if (next.quarantine) {
468
+ pass.notes.push(quarantine(db, { sessionId: session.id, turn: next.batch[0].turn, attempts: next.attempts, version, now }));
469
+ continue;
470
+ }
471
+ if (!mayPay()) {
472
+ pass.notes.push(`the hosted fallback read ${paid} batches, as many as a pass started by itself may pay for: the other conversations wait for the local model, or for sumo memory run`);
473
+ break reading;
474
+ }
475
+ calls++;
476
+ const outcome = await readBatch(db, { sessionId: session.id, batch: next.batch, attempts: next.attempts, version, now, fallback: true });
477
+ if (outcome.fallbackUsed) paid++;
478
+ pass.usage = added(pass.usage, outcome.usage);
479
+ pass.fallbackUsed ||= outcome.fallbackUsed;
480
+ if (outcome.ok) {
481
+ alive = true;
482
+ pass.direct ||= !outcome.fallbackUsed;
483
+ pass.turns += outcome.turns;
484
+ pass.applied.push(...outcome.applied);
485
+ pass.dropped.push(...outcome.dropped);
486
+ pass.written.push(...outcome.written);
487
+ if (outcome.fallbackUsed) pass.notes.push(`the local model could not answer: read on ${outcome.model} instead`);
488
+ continue;
489
+ }
490
+ pass.error = outcome.error;
491
+ // Nothing has shown the model answers in this pass: is it this batch, or is the model not answering anything?
492
+ if (!alive) {
493
+ const answers = await modelAnswers(db, now, mayPay());
494
+ if (answers.fallbackUsed) paid++;
495
+ alive = answers.ok;
496
+ }
497
+ if (!alive) {
498
+ db.prepare('UPDATE extractions SET attempts = attempts - 1 WHERE id = ?').run(outcome.runId);
499
+ pass.down = true;
500
+ break reading;
501
+ }
502
+ // Its third failure: asked again at once in halves, or set aside if it is one turn — the next look says which.
503
+ if (outcome.attempts >= ATTEMPTS) continue;
504
+ // Otherwise the rest of this conversation waits behind it; the other conversations do not.
505
+ pass.stuck++;
506
+ break;
507
+ }
305
508
  }
306
- return { ...outcome, turns: bundle.turns.size };
509
+
510
+ notePass(db, { ok: !pass.down, error: pass.error, fallbackUsed: pass.fallbackUsed, direct: pass.direct });
511
+ if (pass.down) return { ok: false, error: pass.error, applied: [], dropped: [], notes: pass.notes, usage: pass.usage };
512
+ if (pass.stuck > 0) pass.notes.push(`${pass.stuck} conversation${pass.stuck === 1 ? '' : 's'} could not be read and wait${pass.stuck === 1 ? 's' : ''} for the next pass — ${pass.error}`);
513
+ return { ok: true, error: null, applied: pass.applied, dropped: pass.dropped, notes: [...new Set(pass.notes)], usage: pass.usage, turns: pass.turns, written: pass.written };
307
514
  });
308
515
  }
309
516
 
517
+ /**
518
+ * The current extractor over journal turns, changing nothing: what it would file from them, judged by the rules as
519
+ * memory stands. `{ ok, error, wouldFile, dropped, batches }` — `wouldFile` as `applyOps` gives `filed`, `dropped`
520
+ * the proposals the rules refused; `batches` what was read, as `{ sessionId, turnIds, ops }`, for whoever then applies it (rebuild.mjs).
521
+ * The model is shown the conversation and no memory, so the answer is the conversation's alone. Only what the user
522
+ * stated is asked of it here: a trap or a checkpoint drawn from a reply cites no turn to be rebuilt from.
523
+ */
524
+ export async function extract(db, { turnIds, now = new Date().toISOString() }) {
525
+ const out = { ok: true, error: null, wouldFile: [], dropped: [], batches: [], usage: NO_USAGE };
526
+ if (writerOff(db)) return { ...out, ok: false, error: 'memory.model is off' };
527
+ const wanted = [...new Set(turnIds)].filter(Number.isInteger);
528
+ const bySession = new Map();
529
+ for (const id of wanted) {
530
+ const row = db.prepare('SELECT id, session_id FROM user_turns WHERE id = ?').get(id);
531
+ if (row) bySession.set(row.session_id, [...(bySession.get(row.session_id) ?? []), row.id]);
532
+ }
533
+ for (const [sessionId, ids] of bySession) {
534
+ let rest = unitsOf(sessionJournal(db, sessionId), new Set(ids));
535
+ while (rest.length > 0) {
536
+ const batch = naturalBatch(rest);
537
+ rest = rest.slice(batch.length);
538
+ const bundle = bundleOf(db, sessionId, batch, { now, related: false });
539
+ const asked = await ask(db, { bundle, now, quiet: true });
540
+ out.usage = added(out.usage, asked.usage);
541
+ if (!asked.ok) {
542
+ out.ok = false;
543
+ out.error = asked.error;
544
+ continue;
545
+ }
546
+ const ops = asked.ops.filter((op) => op?.op === 'add' || op?.op === 'supersede');
547
+ const dry = applyOps(db, ops, { source: 'scribe', turns: bundle.turns, sessionId, now, dry: true });
548
+ out.wouldFile.push(...dry.filed);
549
+ // What memory holds already was not refused: it is in `wouldFile`, with the memory that holds it.
550
+ out.dropped.push(...dry.dropped.filter((reason) => !dry.repeats.includes(reason)));
551
+ out.batches.push({ sessionId, turnIds: batch.map((u) => u.turn.id), ops, turns: bundle.turns });
552
+ }
553
+ }
554
+ return out;
555
+ }
556
+
310
557
  export function modelStats(db) {
311
558
  const rows = db
312
559
  .prepare(
@@ -323,15 +570,16 @@ export function modelStats(db) {
323
570
  );
324
571
  }
325
572
 
326
- /** What a pass did, as a person reads it: how much it read and cost, and each thing it filed or dropped. */
573
+ /** What a pass did, as a person reads it: how much it read and cost, each thing it filed or dropped, and what it has to say of itself. */
327
574
  export function outcomeLines(outcome) {
328
575
  if (outcome.skipped) return [`nothing to do — ${outcome.skipped}`];
329
- if (!outcome.ok) return [`failed — ${outcome.error}`];
576
+ if (!outcome.ok) return [`failed — ${outcome.error}`, ...(outcome.notes ?? []).map((l) => ` ${l}`)];
330
577
  const cost = `${outcome.usage.inputTokens.toLocaleString('en-US')} tokens in, ${outcome.usage.outputTokens.toLocaleString('en-US')} out, $${outcome.usage.costUsd.toFixed(4)}`;
331
578
  return [
332
- `read ${outcome.turns ?? outcome.sessions} ${outcome.turns === undefined ? 'sessions' : 'turns'} (${cost})`,
579
+ outcome.turns === undefined ? `went over memory (${cost})` : `read ${outcome.turns} turns (${cost})`,
333
580
  ...outcome.applied.map((l) => ` ${l}`),
334
581
  ...outcome.dropped.map((l) => ` dropped — ${l}`),
335
- ...(outcome.applied.length + outcome.dropped.length === 0 ? [' nothing worth remembering'] : []),
582
+ ...(outcome.notes ?? []).map((l) => ` ${l}`),
583
+ ...(outcome.applied.length + outcome.dropped.length === 0 ? [outcome.turns === undefined ? ' nothing to raise or fill in' : ' nothing worth remembering'] : []),
336
584
  ];
337
585
  }