sumo-agents 0.6.1 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/apply.mjs CHANGED
@@ -1,34 +1,63 @@
1
- import { getMeta, setMeta, tx } from './db.mjs';
1
+ import { tx } from './db.mjs';
2
2
  import * as memory from './memory.mjs';
3
3
  import { line } from './render.mjs';
4
4
  import { addCheckpoint, currentProject, outsideReadsOf } from './sessions.mjs';
5
- import { asksForWork, onlyAlternative, overlap, sameWayRound, sharedStems } from './text.mjs';
5
+ import { asksForWork, coverage, overlap, saysAs, sharedStems, standsAlone, words } from './text.mjs';
6
6
 
7
7
  /**
8
8
  * The only door through which a model's suggestion becomes a memory.
9
9
  *
10
10
  * A model proposes; this code decides. Nothing here trusts the model's word:
11
11
  * a claim that the user said something is checked against what the user
12
- * actually typed, and anything that cannot be checked is held back as a guess
13
- * for the user to confirm. One bad operation is dropped and logged; it never
14
- * takes the good ones down with it.
12
+ * actually typed, in the turn the model names, and anything that cannot be
13
+ * checked is dropped with its reason — never kept as a guess for the user to
14
+ * settle. One bad operation is dropped and logged; it never takes the good
15
+ * ones down with it.
16
+ *
17
+ * What the user stated is stored in the user's own words: the model points at a turn and quotes it, and the memory is
18
+ * the paragraph of that turn the quote sits in, whole and as typed. The model writes no sentence that is kept, so a memory cannot
19
+ * say what the user did not type — no list of opposites or borrowed words has to catch a paraphrase that went wrong
20
+ * (two independent reviews, Nov 18 2026, got eleven of twelve such paraphrases past the lists).
15
21
  */
16
22
 
23
+ /**
24
+ * The rules below as a version: part of the extractor's version (ledger.mjs), so memories derived under other rules
25
+ * can be told apart and rebuilt. Bump it whenever what is accepted, dropped or replaced here changes.
26
+ */
27
+ export const APPLY_RULES = '2026-11-18.6';
28
+
17
29
  const STATED_TYPES = ['preference', 'fact', 'decision'];
18
30
  const BODY_MAX = 300;
19
31
  /** Enough to be unmistakable, short enough that a real "no emojis" still counts. */
20
32
  const QUOTE_MIN_WORDS = 2;
21
33
  const QUOTE_MIN_CHARS = 6;
22
- /** A paraphrase shares words with its source. Below this, the quote is real but the claim is about something else. */
23
- const BODY_MATCHES_TURN = 0.34;
24
- const SAME_STATEMENT = 0.8;
25
- /** What the slow pass proposes from a sentence already filed is that memory again when it shares this much with it. */
26
- const ALREADY_FILED = 0.5;
34
+ /**
35
+ * Two statements may be about one thing when they share this much of the shorter one's words, and two words at
36
+ * least. It is enough to ask about the pair, never to retire a memory: "never force-push on slate" and "always push
37
+ * slate tags after a release" pass it.
38
+ */
39
+ const BODY_MATCHES_TURN = 0.3;
40
+ /**
41
+ * A statement replaces a derived memory by itself only when each holds this share of the other's words — one word
42
+ * changed in a sentence of four. Anything looser is the pair judge's to call a replacement.
43
+ */
44
+ const SAME_MATTER = 0.75;
45
+ /** A quote longer than this is not a statement picked out: it is a paragraph. */
46
+ const QUOTE_MAX_WORDS = 40;
47
+ /** A memory the chat saved from a turn is in what is filed from that turn when this share of its words is. */
48
+ const SAID_HERE = 0.8;
49
+ /** A turn of this many paragraphs or more holds something pasted: only its first and last are taken as the user's own. */
50
+ const PASTED_PARAGRAPHS = 3;
51
+ /** What stands for a line break in a memory's one line: a paragraph of several lines is kept as it was typed, on one. */
52
+ const LINE_BREAK = ' / ';
53
+ /** A turn past this length is mostly something pasted: only what opens or closes it is taken as the user's own. */
54
+ const LONG_TURN_CHARS = 1500;
55
+ const LONG_TURN_OWN = 500;
27
56
  /** "use eslint for linting" / "use biome for linting": three words of four. Close enough to ask about, too different to call a repeat. */
28
57
  const LOOKS_ALIKE = 0.7;
29
58
 
30
59
  /**
31
- * How much of memory the slow pass may retire in one go. A playbook a model was asked to rewrite whole shrank to a
60
+ * How much of memory a pass may retire in one go. A playbook a model was asked to rewrite whole shrank to a
32
61
  * fraction of itself, and the agent did worse with it than with none. A few replacements a pass — a correction the user
33
62
  * kept repeating — are its job; past either bound it is rewriting, and none of its replacements is taken.
34
63
  */
@@ -39,52 +68,104 @@ const RETIRING = ['supersede'];
39
68
 
40
69
  const OPS_BY_SOURCE = {
41
70
  scribe: ['add', 'supersede', 'gotcha', 'checkpoint'],
42
- dream: ['add', 'supersede', 'gotcha', 'checkpoint', 'contradiction', 'same', 'procedure'],
71
+ dream: ['add', 'supersede', 'gotcha', 'checkpoint', 'contradiction', 'same'],
43
72
  // A delegated sub-agent's report: it may tell us about traps it hit, and nothing else.
44
73
  worker: ['gotcha'],
45
74
  };
46
75
 
47
76
  class Rejected extends Error {}
77
+ /** Dropped because memory holds it already, or held it until the user let it go: nothing wrong with the proposal. */
78
+ class Known extends Rejected {}
48
79
 
49
80
  const normalize = (text) => text.toLowerCase().replace(/[‘’]/g, "'").replace(/[“”]/g, '"').replace(/\s+/g, ' ').trim();
81
+ /** Runs of space made one, and nothing else: how the user's words are stored. */
82
+ const spaced = (text) => text.replace(/\s+/g, ' ').trim();
83
+ /** Case and the shape of quote marks aside, letter for letter: a place in the one is the same place in the other. */
84
+ const folded = (text) => {
85
+ const lower = text.toLowerCase().replace(/[‘’]/g, "'").replace(/[“”]/g, '"');
86
+ return lower.length === text.length ? lower : text;
87
+ };
50
88
 
51
89
  /**
52
- * The sentences a quote was taken from, whole: the shortest run of the turn's sentences that holds every quoted word.
53
- * The model picks where a quote starts and ends; what the user was doing in that sentence is not its to trim away.
90
+ * A turn as its paragraphs — the blocks between blank lines, the whole turn when it has none — each with its lines
91
+ * (their spaces made one, and where each starts in the turn), whether it holds something pasted (a fenced code
92
+ * block, a line quoted with `>`), and its `body`: the paragraph whole, as typed, its line breaks written " / ".
93
+ *
94
+ * sumo cuts nothing out of what the user typed: every cut that was tried — the sentence, the line, the words after
95
+ * "e.g." — at some point kept the half that said the opposite ("Things I never want you to do:" over a list; "Don't.
96
+ * Ever. Push straight to main."). A turn that fits in one memory is stored whole; of a longer one, a paragraph that
97
+ * stands by itself (`placeOf`).
54
98
  */
55
- function sentencesAround(text, quote) {
56
- const sentences = text.split(/(?<=[.!?])\s+|\n+/).map(normalize).filter(Boolean);
57
- // One pass: where each sentence starts in the turn as normalized, then the ones the quote's span touches.
58
- const starts = [];
99
+ function paragraphsOf(text) {
100
+ const paragraphs = [];
101
+ let current = null;
59
102
  let at = 0;
60
- for (const s of sentences) {
61
- starts.push(at);
62
- at += s.length + 1;
103
+ let fenced = false;
104
+ for (const raw of text.split('\n')) {
105
+ const start = at;
106
+ at += raw.length + 1;
107
+ const fence = /^\s*(?:```|~~~)/.test(raw);
108
+ // A blank line ends a paragraph — not inside a code block, which is one thing however it is spaced.
109
+ if (!raw.trim() && !fenced) {
110
+ current = null;
111
+ continue;
112
+ }
113
+ if (!current) paragraphs.push((current = { lines: [], pasted: false }));
114
+ if (fence || fenced || /^\s*>/.test(raw)) current.pasted = true;
115
+ if (fence) fenced = !fenced;
116
+ if (raw.trim()) current.lines.push({ start, text: spaced(raw) });
63
117
  }
64
- const from = sentences.join(' ').indexOf(normalize(quote));
65
- if (from === -1) return text;
66
- const to = from + normalize(quote).length;
67
- return sentences.filter((s, i) => starts[i] < to && starts[i] + s.length > from).join(' ');
118
+ for (const p of paragraphs) p.body = p.lines.map((l) => l.text).join(LINE_BREAK);
119
+ return paragraphs;
68
120
  }
69
121
 
70
- /** The clauses of the quote's sentence that the quote touches, whole — split at commas, semicolons, colons, dashes and "but". */
71
- function clausesAround(text, quote) {
72
- const sentence = sentencesAround(text, quote);
73
- const q = normalize(quote);
74
- const from = sentence.indexOf(q);
75
- if (from === -1) return sentence;
76
- const to = from + q.length;
77
- const parts = [];
78
- const cut = /[,;:]|\s[—–-]\s|\sbut\s/g;
79
- let start = 0;
80
- for (const m of sentence.matchAll(cut)) {
81
- parts.push([start, m.index]);
82
- start = m.index + m[0].length;
122
+ /** What opens a list item: a dash, a star, a bullet, a plus, a number. */
123
+ const LIST_ITEM = /^(?:[-*•+]|\d+[.)])\s/;
124
+
125
+ /**
126
+ * Whether a paragraph of a turn can be taken without the one before it: not when that one leads into it (it ends in
127
+ * a colon or a comma), not when it is a list or opens with a list item, a caret pointing back, or a small letter —
128
+ * a continuation. "Things I never want you to do:", a blank line, and then the list was stored as the list.
129
+ */
130
+ function standsApart(paragraphs, i) {
131
+ const lines = paragraphs[i].lines.map((l) => l.text);
132
+ if (i > 0 && /[:,]$/.test(paragraphs[i - 1].body)) return false;
133
+ return !lines.every((l) => LIST_ITEM.test(l)) && !LIST_ITEM.test(lines[0]) && !lines[0].startsWith('^') && !/^\p{Ll}/u.test(lines[0]);
134
+ }
135
+
136
+ /**
137
+ * What of a turn a quote stands for, as the user's own words: `{ paragraph }`, the body to store — or `{ why }` it
138
+ * cannot be taken. A quote sits on one line, in a paragraph that holds nothing pasted. A turn that fits in one
139
+ * memory, and holds nothing pasted, is taken whole, whichever paragraph the quote is in: a blank line between what
140
+ * negates and what is negated cuts nothing off. Of a longer turn the quote's paragraph is taken — the first or the
141
+ * last of three or more (a page pasted between what the user typed is a paragraph like any other), opening or closing
142
+ * a turn long enough to be mostly a paste, and only one that stands apart from the paragraph before it.
143
+ */
144
+ function placeOf(text, quote) {
145
+ const sought = folded(quote);
146
+ const paragraphs = paragraphsOf(text);
147
+ const whole = paragraphs.map((p) => p.body).join(LINE_BREAK);
148
+ const fits = whole.length <= BODY_MAX && !paragraphs.some((p) => p.pasted);
149
+ const why = { pasted: false, inner: false, middle: false, leaning: false };
150
+ for (const [i, p] of paragraphs.entries()) {
151
+ for (const line of p.lines) {
152
+ const from = folded(line.text).indexOf(sought);
153
+ if (from === -1) continue;
154
+ const [starts, ends] = [line.start + from, line.start + from + sought.length];
155
+ if (p.pasted) why.pasted = true;
156
+ else if (fits) return { paragraph: whole };
157
+ else if (paragraphs.length >= PASTED_PARAGRAPHS && i !== 0 && i !== paragraphs.length - 1) why.inner = true;
158
+ else if (text.length > LONG_TURN_CHARS && ends > LONG_TURN_OWN && starts < text.length - LONG_TURN_OWN) why.middle = true;
159
+ // One too long to keep is said to be that, below: the reason that needs no reading of its first letter.
160
+ else if (p.body.length <= BODY_MAX && !standsApart(paragraphs, i)) why.leaning = true;
161
+ else return { paragraph: p.body };
162
+ }
83
163
  }
84
- parts.push([start, sentence.length]);
85
- // The touched clauses are side by side: one slice of the sentence keeps the commas between them, which say what is set aside.
86
- const touched = parts.filter(([a, b]) => a < to && b > from);
87
- return sentence.slice(touched[0][0], touched.at(-1)[1]).trim();
164
+ if (why.middle) return { why: `the quote is from the middle of a long turn: past ${LONG_TURN_CHARS.toLocaleString('en-US')} characters only its first and last ${LONG_TURN_OWN} are taken as the user's own` };
165
+ if (why.inner) return { why: "the quote is from a paragraph in the middle of a turn of several: only its first and last paragraph are taken as the user's own" };
166
+ if (why.leaning) return { why: 'the paragraph does not stand by itself (it follows one that leads into it, is a list, or opens as a continuation), and the turn is too long to keep whole' };
167
+ if (why.pasted) return { why: 'the quote is from a pasted block — code, or a quoted line — or the paragraph it sits in holds one' };
168
+ return { why: 'the quote runs across a line break: a statement is on one line' };
88
169
  }
89
170
 
90
171
  function resolveScope(db, scope) {
@@ -106,32 +187,41 @@ function cleanBody(op) {
106
187
  return body;
107
188
  }
108
189
 
190
+ /** A turn's number as a model writes it: 12, or "t12" copied whole from the tag. */
191
+ const turnOf = (raw) => Number(/^t?(\d+)$/i.exec(String(raw ?? '').trim())?.[1] ?? NaN);
192
+
109
193
  /**
110
- * Did the user really say it? The quoted words must appear in something the
111
- * user typed, and the memory must be about that sentence. The turn the model
112
- * names is tried first, then every other turn it was shown: models mix up turn
113
- * numbers, and the words being the user's own is what matters, not the label.
194
+ * What the user stated, in their words: the turn a model names, its quote, and — as the memory's body — the whole
195
+ * turn when it fits in one memory, else the paragraph of it the quote sits in — exactly as typed, runs of space made
196
+ * one and line breaks written " / ". Nothing is cut out of it, and whatever the model wrote as a body is not looked at here. Only the named turn is looked in: a quote found somewhere else in what the model was
197
+ * shown is one turn's words under another's number, and which turn a statement came from is what a memory cites.
198
+ * Throws why it is not kept; nothing that cannot be shown to be the user's own words is kept in any state.
114
199
  */
115
- function verify(op, body, turns) {
116
- const named = turns.get(Number(op.turn)) ?? null;
117
- if (typeof op.quote !== 'string' || !op.quote.trim()) return { turn: named, verified: false, why: 'no quote given' };
200
+ function stated(op, turns) {
201
+ const not = (why) => new Rejected(`not the user's words, not kept — ${why}`);
202
+ const turn = turns.get(turnOf(op.turn)) ?? null;
203
+ if (!turn) throw not('it names no turn the model was shown');
204
+ if (typeof op.quote !== 'string' || !op.quote.trim()) throw not('no quote given');
118
205
  // The words, without the quote marks or the full stop a model may put round them.
119
206
  const said = op.quote.trim().replace(/^["'“‘]+|["'”’]+$/g, '').replace(/[.!?]+$/, '');
120
207
  const quote = normalize(said);
121
- if (quote.length < QUOTE_MIN_CHARS || quote.split(' ').length < QUOTE_MIN_WORDS) return { turn: named, verified: false, why: 'the quote is too short to prove anything' };
122
-
123
- const candidates = [...(named ? [named] : []), ...[...turns.values()].filter((t) => t !== named)];
124
- const source = candidates.find((t) => normalize(t.text).includes(quote));
125
- if (!source) return { turn: named, verified: false, why: 'the user never typed the quoted words' };
126
- if (overlap(body, source.text) < BODY_MATCHES_TURN) return { turn: source, verified: false, why: 'the quote is real but the memory is about something else' };
127
- // The user's words, turned round — "never" filed from an "always" — are not the user's word. Judged on the clauses the quote
128
- // is in, whole: "we use biome now, not eslint" negates the old tool, not the rule; a quote cut short of its "never" still has it.
129
- // An alternative set aside ("…, never npm") is looked for in the whole sentence: a quote that stops at the comma left it
130
- // there. A quote of nothing but the alternative is judged with the sentence it is set against.
131
- const sentence = sentencesAround(source.text, said);
132
- const clauses = clausesAround(source.text, said);
133
- if (!sameWayRound(body, onlyAlternative(clauses) ? sentence : clauses, sentence)) return { turn: source, verified: false, why: 'the quote is real but the memory says it the other way round' };
134
- return { turn: source, verified: true, why: null };
208
+ if (quote.length < QUOTE_MIN_CHARS || quote.split(' ').length < QUOTE_MIN_WORDS) throw not('the quote is too short to prove anything');
209
+ if (!normalize(turn.text).includes(quote)) throw not('the user never typed the quoted words in the turn it names');
210
+ if (quote.split(' ').length > QUOTE_MAX_WORDS) throw not(`the quote is over ${QUOTE_MAX_WORDS} words: it picks out a statement, it is not the statement`);
211
+ const place = placeOf(turn.text, spaced(said));
212
+ if (place.why) throw not(place.why);
213
+ const { paragraph } = place;
214
+ // Asked, not stated — also when it holds the words of a rule: "Do I want you to always force-push to main?".
215
+ if (/\?["'”’)\]]*$/.test(paragraph)) throw new Rejected('the quote is from a question; it states nothing');
216
+ // The words being the user's is not enough: asked to describe or fix something, the user stated nothing that lasts.
217
+ // Judged on the quote and on the whole paragraph it came from, so a quote cut short of its "delegate" proves no more than what it sits in.
218
+ if (asksForWork(said) || asksForWork(paragraph)) throw new Rejected('the quote is from a sentence that asks for work or an answer; it states nothing that lasts');
219
+ // Nor is agreeing: "ok that plan sounds good" is the user's, and the plan was the assistant's.
220
+ if (!standsAlone(paragraph)) throw new Rejected('the sentence agrees or points at something said elsewhere, and says nothing by itself');
221
+ // Whole, or not at all: a part of it can say the opposite of it ("it is not the case … that we always deploy on
222
+ // fridays without review"). What was said stays in the journal.
223
+ if (paragraph.length > BODY_MAX) throw new Rejected(`the paragraph is over ${BODY_MAX} characters: too long to keep whole, and a part of it is not what was said — it stays in the journal, where sumo search finds it`);
224
+ return { turn, body: paragraph, quote: op.quote.trim() };
135
225
  }
136
226
 
137
227
  /**
@@ -141,7 +231,8 @@ function verify(op, body, turns) {
141
231
  * already known would leave memory wrong for good — so look-alikes are never dropped, only flagged.
142
232
  */
143
233
  // Identifier case and punctuation carry meaning (Release/release, feature/a/feature-a).
144
- const statementOf = (text) => text.replace(/[‘’]/g, "'").replace(/[“”]/g, '"').replace(/\s+/g, ' ').trim();
234
+ // The full stop a sentence ends on does not: the same rule typed with and without one is one rule.
235
+ const statementOf = (text) => text.replace(/[‘’]/g, "'").replace(/[“”]/g, '"').replace(/\s+/g, ' ').trim().replace(/[.!?]+$/, '');
145
236
 
146
237
  function lookalike(db, scope, body, exclude) {
147
238
  const inScope = db
@@ -155,25 +246,47 @@ function lookalike(db, scope, body, exclude) {
155
246
  return alike ? { memory: alike, same: false } : null;
156
247
  }
157
248
 
158
- /** A guess already put to the user in other words — still waiting, or refused: asked once is enough. */
159
- function askedBefore(db, scope, body) {
160
- const asked = db.prepare(`SELECT * FROM memories WHERE scope = ? AND type != 'procedure' AND state IN ('unconfirmed', 'rejected')`).all(scope);
161
- return asked.find((m) => statementOf(m.body) !== statementOf(body) && overlap(body, m.body) >= LOOKS_ALIKE) ?? null;
162
- }
163
-
164
249
  /**
165
- * The same sentence is often proposed twice from one turn — once by a retry, once by the chat model saving it directly.
166
- * A repeat quotes the same words; two rules given in one sentence quote different ones, and are both kept.
250
+ * The same sentence is often proposed twice from one turn — once by a retry, once by the chat model saving it
251
+ * directly, once by an extractor from before memories were the user's own sentences. A repeat is the same statement,
252
+ * or one drawn from the same words: its quote holds this one's, or this one's holds its.
167
253
  */
168
254
  function savedFromTurn(db, turnId, scope, body, exclude, quote) {
169
255
  if (turnId === null) return null;
170
256
  const fromTurn = db.prepare(`SELECT * FROM memories WHERE source_turn = ? AND scope = ? AND state != 'unconfirmed' AND id IS NOT ?`).all(turnId, scope, exclude ?? null);
171
257
  const said = typeof quote === 'string' ? normalize(quote) : '';
172
- const quotesAgree = (m) => {
258
+ const sameWords = (m) => {
173
259
  const theirs = m.source_quote ? normalize(m.source_quote) : '';
174
- return !said || !theirs || said.includes(theirs) || theirs.includes(said);
260
+ return Boolean(said) && Boolean(theirs) && (said.includes(theirs) || theirs.includes(said));
175
261
  };
176
- return fromTurn.find((m) => statementOf(body) === statementOf(m.body) && quotesAgree(m)) ?? null;
262
+ return fromTurn.find((m) => statementOf(body) === statementOf(m.body) || sameWords(m)) ?? null;
263
+ }
264
+
265
+ /**
266
+ * A memory the user let go — forgot or rejected — after the turn in hand was said, that this sentence would bring
267
+ * back: one drawn from the same words of this very turn — its quote holds this one's, or this one's holds its, so
268
+ * the other sentences of the turn are not let go with it — or one that says what the sentence says: the same words,
269
+ * or nearly the same either way (`saysAs`). Sharing words is not saying it: "slate uses bun for installs now" is the correction of a
270
+ * forgotten "slate uses pnpm for installs", and was dropped as it. In every scope: let go under one, it does not come back under
271
+ * another. Reading a conversation again must not return what they threw out; said again later, it is theirs again.
272
+ * Only the user's letting go counts, never a rebuild's, a migration's or a trap's expiry.
273
+ */
274
+ function letGo(db, body, turn, quote) {
275
+ // A turn handed in without its time cannot be placed before or after anything.
276
+ if (typeof turn.ts !== 'string') return null;
277
+ const gone = db
278
+ .prepare(
279
+ `SELECT m.*, (SELECT MAX(e.ts) FROM memory_events e WHERE e.memory_id = m.id AND e.op IN ('forgotten', 'rejected')
280
+ AND e.actor IN ('user', 'model-shell', 'unknown') AND COALESCE(e.reason, '') != 'rebuilt') AS let_go_at
281
+ FROM memories m WHERE m.type != 'procedure' AND m.state IN ('rejected', 'invalid') ORDER BY m.id`,
282
+ )
283
+ .all()
284
+ .filter((m) => m.let_go_at !== null && m.let_go_at >= turn.ts);
285
+ const said = normalize(quote);
286
+ const sameWords = (m) => m.source_turn === turn.id && Boolean(m.source_quote) && (said.includes(normalize(m.source_quote)) || normalize(m.source_quote).includes(said));
287
+ // Saved from this turn by the chat, which keeps no quote: let go, it takes with it whatever of the turn holds its words.
288
+ const savedHere = (m) => m.source_turn === turn.id && !m.source_quote && coverage(m.body, body) >= SAID_HERE;
289
+ return gone.find((m) => sameWords(m) || savedHere(m) || statementOf(m.body) === statementOf(body) || saysAs(body, m.body) || saysAs(m.body, body)) ?? null;
177
290
  }
178
291
 
179
292
  /**
@@ -187,117 +300,127 @@ export function raiseNotice(db, { kind, a, b, note, now }) {
187
300
  return true;
188
301
  }
189
302
 
190
- /** Where the pairs a pass named wait to be judged on their own. */
191
- const NAMED_PAIRS = 'dream.named_pairs';
192
-
193
- /** The pairs a pass named that wait to be judged on their own, in the order named: `{ label, a, b, note }`, a < b. */
194
- export function namedPairs(db) {
195
- try {
196
- const named = JSON.parse(getMeta(db, NAMED_PAIRS) ?? '[]');
197
- return Array.isArray(named) ? named.filter((p) => Number.isInteger(p?.a) && Number.isInteger(p?.b)) : [];
198
- } catch {
199
- return [];
200
- }
201
- }
202
-
203
- /** What waits to be judged from here on: those named, less any judged now. */
204
- export const keepNamed = (db, named) => setMeta(db, NAMED_PAIRS, JSON.stringify(named));
205
-
206
- /**
207
- * What a pass that read memories by number says of two of them waits until the pair is judged on its own (dream.mjs):
208
- * reading sixty memories and a dozen turns, a small model writes one number for another — a turn's for a memory's, a
209
- * neighbour's for the one it meant — and a pair it never looked at side by side would be a question for the user. The
210
- * reason it is not named, or null: asked once — a pair raised, judged, or waiting already is not named again.
211
- */
212
- export function nameForJudging(db, { label, a, b, note }) {
213
- const [lo, hi] = a < b ? [a, b] : [b, a];
214
- if (db.prepare('SELECT 1 FROM notices WHERE memory_a = ? AND memory_b = ?').get(lo, hi)) return 'already raised';
215
- const judged = db.prepare('SELECT label FROM memory_pairs WHERE a = ? AND b = ?').get(lo, hi);
216
- if (judged) return `m${lo} and m${hi} were judged on their own already: ${judged.label}`;
217
- const waiting = namedPairs(db);
218
- if (waiting.some((p) => p.a === lo && p.b === hi)) return 'already waiting to be judged';
219
- keepNamed(db, [...waiting, { label, a: lo, b: hi, note }]);
220
- return null;
221
- }
222
-
223
303
  /** Two active memories that look alike but are not the same: only the user can say whether one replaces the other. */
224
304
  function flagLookalike(db, saved, other, now) {
225
305
  raiseNotice(db, { kind: 'lookalike', a: saved.id, b: other.id, note: 'they look alike — if the newer one replaces the older, supersede it; if both hold, leave them', now });
226
306
  }
227
307
 
228
- /** The user has now said, in so many words, what was only a guess: the guess becomes a stated memory. */
229
- function promote(db, guess, { turn, quote, supersedes, now }) {
230
- db.prepare(`UPDATE memories SET state = 'active', provenance = 'stated', source_session = ?, source_turn = ?, source_quote = ? WHERE id = ?`).run(
231
- turn.session_id, turn.id, quote, guess.id,
232
- );
233
- if (supersedes !== undefined) memory.supersede(db, supersedes, guess.id, now);
234
- return memory.get(db, guess.id);
308
+ /** A pair the pass had judged as it filed is never asked about again by the tidy. */
309
+ function noteJudged(db, a, b, label, now) {
310
+ const [lo, hi] = a < b ? [a, b] : [b, a];
311
+ db.prepare('INSERT OR IGNORE INTO memory_pairs (a, b, label, judged_at) VALUES (?, ?, ?, ?)').run(lo, hi, label, now);
235
312
  }
236
313
 
237
- function addStated(db, op, ctx, { supersedes } = {}) {
314
+ /**
315
+ * One statement of the user's through the rules. `supersedes` is the memory it replaces; `beside` one it was meant to
316
+ * replace and may not — `why` says so (too wide a rewrite, a memory the user wrote, a pair nobody called a
317
+ * replacement): the statement is kept, that memory left, and the user asked about the two.
318
+ */
319
+ function addStated(db, op, ctx, { supersedes, beside, why = ctx.refused, pending } = {}) {
238
320
  if (!STATED_TYPES.includes(op.type)) throw new Rejected(`type "${op.type}" cannot come from a model — one of: ${STATED_TYPES.join(', ')}`);
239
- const body = cleanBody(op);
240
- const { turn, verified, why } = verify(op, body, ctx.turns);
241
- // The words being the user's is not enough: asked to describe or fix something, the user stated nothing that lasts.
242
- // Judged on the quote and on the whole sentence it came from, so a quote cut short of its "delegate" proves no more than the sentence.
243
- if (verified && (asksForWork(op.quote) || asksForWork(sentencesAround(turn.text, op.quote)))) {
244
- throw new Rejected('the quote is from a sentence that asks for work or an answer; it states nothing that lasts');
245
- }
246
- // A guess costs the user a question; one drawn from a request — describe this, fix that — is the model answering it, not the user saying it.
247
- if (!verified && turn && asksForWork(turn.text)) throw new Rejected(`a guess from a request, not kept — ${why}`);
321
+ const { turn, body, quote } = stated(op, ctx.turns);
248
322
  // No scope given: what was said in the middle of work on a project belongs to that project.
249
- const focus = op.scope === undefined && turn ? currentProject(db, turn.session_id) : null;
323
+ const focus = op.scope === undefined ? currentProject(db, turn.session_id) : null;
250
324
  const { scope, project } = resolveScope(db, focus ? `project:${focus}` : op.scope);
251
- // Nor is a guess kept that shares nothing with the turn it names: that is the assistant's reply, or what is remembered already,
252
- // said in the user's name. The slow pass alone may guess past that — a pattern across conversations is what it reads for.
253
- const aboutTurn = ctx.source === 'dream' || (turn !== null && overlap(body, turn.text) >= BODY_MATCHES_TURN);
254
- if (!verified && !aboutTurn) throw new Rejected(`a guess about nothing the user typed, not kept — ${why}`);
255
- const asked = verified ? null : askedBefore(db, scope, body);
256
- if (asked) throw new Rejected(`a guess like m${asked.id}, which is ${asked.state} — not kept`);
257
-
258
- const again = savedFromTurn(db, turn?.id ?? null, scope, body, supersedes, op.quote);
259
- if (again) throw new Rejected(`already saved from this turn as m${again.id}`);
260
- // The slow pass reads conversations the writer has filed already: a sentence it filed is not filed again in other
261
- // words. Another rule from the same sentence, sharing little with what was filed, is one of its own.
262
- if (ctx.source === 'dream' && turn && supersedes === undefined) {
263
- const filed = db.prepare(`SELECT id, body FROM memories WHERE source_turn = ? AND state = 'active'`).all(turn.id).find((m) => overlap(body, m.body) >= ALREADY_FILED);
264
- if (filed) throw new Rejected(`already filed from that sentence: m${filed.id}`);
325
+ /** What this statement is, for whoever asks what the rules let through; `existing` when memory holds it already. */
326
+ const passed = (existing = null, more = {}) => ctx.filed.push({ turnId: turn.id, type: op.type, scope, body, quote, existing, replaces: null, mayReplace: null, later: null, ...more });
327
+ /** An older statement than the memory that covers its matter: not filed over what the user said after it. */
328
+ const saidBefore = (m) => {
329
+ passed(m.id, { later: m.id });
330
+ return new Known(`said before m${m.id}, which holds what the user said after it`);
331
+ };
332
+ const thrownOut = letGo(db, body, turn, quote);
333
+ if (thrownOut) {
334
+ passed(thrownOut.id);
335
+ throw new Known(`the user let this go (m${thrownOut.id})`);
336
+ }
337
+
338
+ const again = savedFromTurn(db, turn.id, scope, body, supersedes ?? beside, quote);
339
+ if (again) {
340
+ passed(again.id);
341
+ throw new Known(`already saved from this turn as m${again.id}`);
265
342
  }
266
343
 
267
344
  // What is being replaced naturally resembles its replacement, so it is left out of the comparison.
268
- const twin = lookalike(db, scope, body, supersedes);
269
- if (twin?.same) {
345
+ const twin = lookalike(db, scope, body, supersedes ?? beside);
346
+ // The same statement, active: known. One the user refused as a guess, or that is held for them, they have now said.
347
+ if (twin?.same && twin.memory.state === 'active') {
270
348
  const known = twin.memory;
271
- if (known.state === 'unconfirmed' && verified) {
272
- return `saved ${line(promote(db, known, { turn, quote: op.quote.trim(), supersedes, now: ctx.now }))}`;
273
- }
274
- if (known.state === 'active' && verified && supersedes !== undefined) {
275
- // The statement was filed earlier; only now is it clear what it replaces.
276
- memory.supersede(db, supersedes, known.id, ctx.now);
277
- return `m${known.id} now replaces m${supersedes}`;
278
- }
279
- if (known.state === 'rejected' && !verified) throw new Rejected(`the user already said no to this (m${known.id})`);
280
- if (known.state !== 'rejected') throw new Rejected(`already known as m${known.id}`);
349
+ passed(known.id);
350
+ if (supersedes === undefined) throw new Known(`already known as m${known.id}`);
351
+ // The statement was filed earlier; only now is it clear what it replaces.
352
+ memory.supersede(db, supersedes, known.id, ctx.now, { actor: ctx.source, runId: ctx.runId });
353
+ return `m${known.id} now replaces m${supersedes}`;
281
354
  }
282
355
 
356
+ // A look-alike is a fact that may have changed ("slate uses pnpm" / "slate uses bun"). Code picked the pair; what it
357
+ // is was asked of the model before this call (`ctx.labels`, by the operation's place in the answer). Not asked — no
358
+ // local model to ask — it is put to the user as a notice. Asked and not answered, both stand.
359
+ const alike = twin && !twin.same ? twin.memory : null;
360
+ // Read late — a rebuild, a pass that was paused — an old turn's statement meets the memory of a later one that looks
361
+ // like it. It never replaces that memory ("quartz uses yarn" once replaced a later "quartz uses deno"); whether it is
362
+ // the thing that memory corrected, and so not filed at all, is judged below — sharing words is not being it, and
363
+ // "slate tests run on node 22" is not what "slate builds run on node 22" corrected.
364
+ const older = Boolean(alike) && saidLater(db, alike, turn);
365
+ const judged = alike && ctx.labels ? ctx.labels.get(ctx.index) : null;
366
+ const label = alike && judged?.old === alike.id ? judged.label : null;
367
+ if (alike && !ctx.labels) ctx.pairs.push({ index: ctx.index, old: alike, type: op.type, scope, body, said: turn.ts });
368
+ if (label === 'same') {
369
+ passed(alike.id);
370
+ throw new Known(`already known as m${alike.id}, in other words`);
371
+ }
372
+ // A replacement a pass may not make (too wide a rewrite) leaves both standing, and the user is asked.
373
+ // Nearly the same words as what was said after it, and the judge says the two are one matter: the later one stands
374
+ // in its place. Anything less sure is filed beside it, and the user asked.
375
+ if (older && sameMatter(body, alike.body) && (label === 'replaces' || label === 'conflict')) throw saidBefore(alike);
376
+ // Nor one of a memory the user wrote themselves: that is theirs to replace. Nor of one said after this.
377
+ const replaces = label === 'replaces' && !ctx.refused && alike.derived_by !== null && !older ? alike.id : null;
378
+
283
379
  const { memory: saved } = memory.add(db, {
284
380
  type: op.type,
285
381
  body,
286
382
  project,
287
383
  topic: typeof op.topic === 'string' ? op.topic : undefined,
288
- // Search terms only, checked where they are stored: a bad one is dropped there, never the memory.
289
- keys: op.keys,
290
- provenance: verified ? 'stated' : 'inferred',
291
- // A guess must never be able to hide something true: only a verified statement supersedes.
292
- supersedes: verified ? supersedes : undefined,
384
+ // Search terms only, checked where they are stored: a bad one is dropped there, never the memory. A sentence the
385
+ // model wrote of its own is no more than that — its words that the user's sentence lacks help the memory be found.
386
+ keys: [...(Array.isArray(op.keys) ? op.keys : []), ...(typeof op.body === 'string' ? words(op.body) : [])],
387
+ provenance: 'stated',
388
+ supersedes: supersedes ?? replaces ?? undefined,
293
389
  writtenBy: ctx.source,
294
- sourceSession: turn?.session_id ?? null,
295
- sourceTurn: turn?.id ?? null,
296
- sourceQuote: verified ? op.quote.trim() : null,
390
+ derivedBy: ctx.derivedBy ?? ctx.source,
391
+ runId: ctx.runId,
392
+ sourceSession: turn.session_id,
393
+ sourceTurn: turn.id,
394
+ sourceQuote: quote,
297
395
  now: ctx.now,
298
396
  });
299
- if (verified && twin && !twin.same) flagLookalike(db, saved, twin.memory, ctx.now);
300
- return verified ? `saved ${line(saved)}` : `held for confirmation ${line(saved)} — ${why}`;
397
+ const unjudged = alike && !ctx.labels ? alike.id : null;
398
+ passed(null, { replaces: supersedes ?? replaces, mayReplace: pending ?? (unjudged !== null && !older && alike.derived_by !== null ? unjudged : null), mayBeOlder: older ? unjudged : null });
399
+ // Replacing what the model was asked to replace and what it was judged to replace: both are retired, history kept.
400
+ if (replaces !== null && supersedes !== undefined) memory.supersede(db, replaces, saved.id, ctx.now, { actor: ctx.source, runId: ctx.runId });
401
+ if (label) noteJudged(db, saved.id, alike.id, label, ctx.now);
402
+ if (alike && (!ctx.labels || (label === 'replaces' && replaces === null))) flagLookalike(db, saved, alike, ctx.now);
403
+ if (label === 'conflict') raiseNotice(db, { kind: 'contradiction', a: saved.id, b: alike.id, note: 'they cannot both be true — only the user knows which holds', now: ctx.now });
404
+ if (beside !== undefined) flagLookalike(db, saved, { id: beside }, ctx.now);
405
+ const said =
406
+ beside !== undefined ? ` — m${beside} is left as it is, and the user is asked: ${why}`
407
+ : replaces !== null ? ` — it replaces m${replaces}`
408
+ : label === 'conflict' ? ` — raised for the user: it cannot be true beside m${alike.id}`
409
+ : '';
410
+ return `saved ${line(saved)}${said}`;
411
+ }
412
+
413
+ const aboutOneThing = (a, b) => overlap(a, b) >= BODY_MATCHES_TURN && sharedStems(a, b) >= 2;
414
+ const sameMatter = (a, b) => coverage(a, b) >= SAME_MATTER && coverage(b, a) >= SAME_MATTER;
415
+
416
+ /**
417
+ * Whether a memory holds something said after a turn: the time of the turn it was drawn from, or of its filing when
418
+ * it cites none. A turn handed in without its time is not placed.
419
+ */
420
+ function saidLater(db, m, turn) {
421
+ if (typeof turn.ts !== 'string') return false;
422
+ const at = (m.source_turn === null || m.source_turn === undefined ? null : db.prepare('SELECT ts FROM user_turns WHERE id = ?').get(m.source_turn)?.ts) ?? m.created_at;
423
+ return typeof at === 'string' && at > turn.ts;
301
424
  }
302
425
 
303
426
  /** The scope a model named, as it is stored; null when it names nothing that exists. */
@@ -323,9 +446,29 @@ const HANDLERS = {
323
446
  // Nor does it replace a taught workflow — its steps and its gate are not a sentence to overwrite — or a memory of another
324
447
  // reach: a rule said for one project leaves the one that holds everywhere, and the one of another project, in force.
325
448
  const reach = op.scope ? reachOf(db, op.scope) : (old?.scope ?? null);
326
- const related = old?.state === 'active' && old.type !== 'procedure' && (reach === old.scope || reach === 'global') && overlap(cleanBody(op), old.body) >= BODY_MATCHES_TURN;
449
+ // Compared as the user's own words, the paragraph that would be stored — never as a model worded it.
450
+ const { body, turn } = stated(op, ctx.turns);
451
+ const related = old?.state === 'active' && old.type !== 'procedure' && (reach === old.scope || reach === 'global') && aboutOneThing(body, old.body);
327
452
  if (!related) return addStated(db, { ...op, type }, ctx);
328
- return addStated(db, { ...op, scope: op.scope ?? old.scope, type }, ctx, { supersedes: old.id });
453
+ const next = { ...op, scope: op.scope ?? old.scope, type };
454
+ const beside = (why, pending) => addStated(db, next, ctx, { beside: old.id, why, pending });
455
+ // What the memory holds was said after this: it is not replaced by it, whatever the model names as replacing what.
456
+ // The statement goes through as any other, and meets that memory there if the two look alike.
457
+ if (saidLater(db, old, turn)) return addStated(db, { ...op, type }, ctx);
458
+ // What the user wrote themselves is never a pass's to retire, whatever a model says replaces it: "never force-push
459
+ // on slate" was once superseded by "always push slate tags after a release". Both stand, and the user is asked.
460
+ if (old.derived_by === null) return beside(`the user wrote m${old.id} themselves, and a pass does not replace what they wrote`);
461
+ // Refused as too wide a rewrite, the user's words from the conversation just read are still theirs: kept beside the old.
462
+ if (ctx.refused) return beside(ctx.refused);
463
+ if (sameMatter(body, old.body)) return addStated(db, next, ctx, { supersedes: old.id });
464
+ // Sharing words is not being about one thing. The model that read the conversation named the memory; the pair
465
+ // judge, shown only the two, must agree — asked before this call, like any look-alike: that the new one replaces
466
+ // the old, or that the two cannot both be true, which of a memory and what the user has just said is the same
467
+ // verdict ("slate uses pnpm, never npm" / "in slate we moved from pnpm to bun last week" it calls a conflict).
468
+ if (!ctx.labels) ctx.pairs.push({ index: ctx.index, old, type, scope: reach ?? old.scope, body, said: turn.ts });
469
+ const judged = ctx.labels?.get(ctx.index);
470
+ if (judged?.old === old.id && (judged.label === 'replaces' || judged.label === 'conflict')) return addStated(db, next, ctx, { supersedes: old.id });
471
+ return beside('the two share words, and nothing says the new one replaces the old', ctx.labels ? undefined : old.id);
329
472
  },
330
473
 
331
474
  /** Something learned by doing the work, not something the user said — so no quote, and only this one kind. */
@@ -336,10 +479,13 @@ const HANDLERS = {
336
479
  const twin = lookalike(db, scope, body);
337
480
  if (twin) throw new Rejected(`already known as m${twin.memory.id}`);
338
481
  // A trap read in a conversation that searched or fetched the web, or called an MCP server, may be what a page told
339
- // it: held for the user, never filed as observed. A gotcha names no turn, so every conversation the pass read counts.
482
+ // it: held for the user, never filed as observed — the one thing still held, since guesses are not kept. A gotcha
483
+ // names no turn, so every conversation the pass read counts.
340
484
  const outside = outsideReadsOf(db, [ctx.sessionId, ...[...(ctx.turns?.values() ?? [])].map((t) => t.session_id)]);
341
- const { memory: saved } = memory.add(db, { type: 'gotcha', body, project, keys: op.keys, provenance: outside.length > 0 ? 'inferred' : 'observed', writtenBy: ctx.source, now: ctx.now });
342
- if (outside.length > 0) db.prepare(`UPDATE memories SET held_for = 'outside' WHERE id = ?`).run(saved.id);
485
+ const { memory: saved } = memory.add(db, {
486
+ type: 'gotcha', body, project, keys: op.keys, provenance: 'observed', heldFor: outside.length > 0 ? 'outside' : undefined,
487
+ writtenBy: ctx.source, derivedBy: ctx.derivedBy ?? ctx.source, runId: ctx.runId, now: ctx.now,
488
+ });
343
489
  if (outside.length > 0) return `held for confirmation ${line(saved)} — the conversation read ${outside.join(', ')}, and a trap it passes on may be what it was told`;
344
490
  return `saved ${line(saved)}`;
345
491
  },
@@ -359,16 +505,13 @@ const HANDLERS = {
359
505
  },
360
506
 
361
507
  /**
362
- * Two memories that cannot both be true. Never resolved here: only the user knows which one is. Only memories the pass
363
- * was shown, still active, can be named: one it never saw is a wrong number, whatever the note says.
508
+ * Two memories that cannot both be true. Never resolved here: only the user knows which one is. Only memories still
509
+ * active can be named.
364
510
  */
365
511
  contradiction(db, op, ctx) {
366
512
  const [a, b] = (Array.isArray(op.ids) ? op.ids : []).map(idOf);
367
513
  if (!a || !b || a === b) throw new Rejected('a contradiction names two different memories');
368
- for (const id of [a, b]) {
369
- if (ctx.shown && !ctx.shown.has(id)) throw new Rejected(`m${id} is not one of the memories this pass was shown`);
370
- if (memory.get(db, id).state !== 'active') throw new Rejected(`m${id} is not active`);
371
- }
514
+ for (const id of [a, b]) if (memory.get(db, id).state !== 'active') throw new Rejected(`m${id} is not active`);
372
515
  // Two things that clash are about something in common, or the note says what each is about: a memory sharing no word
373
516
  // with either is a wrong number, not a clash to put to the user.
374
517
  const said = typeof op.note === 'string' ? op.note : '';
@@ -377,83 +520,58 @@ const HANDLERS = {
377
520
  }
378
521
  const [lo, hi] = a < b ? [a, b] : [b, a];
379
522
  const note = typeof op.note === 'string' && op.note.trim() ? op.note.trim().slice(0, 200) : 'they cannot both be true';
380
- if (ctx.shown) return judgedFirst(db, { label: 'conflict', lo, hi, note, said: 'cannot both be true' });
381
523
  if (!raiseNotice(db, { kind: 'contradiction', a: lo, b: hi, note, now: ctx.now })) throw new Rejected('already raised');
382
524
  return `raised for the user: m${lo} vs m${hi} — ${note}`;
383
525
  },
384
526
 
385
527
  /**
386
528
  * Two memories that say one thing in other words — what no word overlap finds. Raised as a look-alike for the user to
387
- * settle; nothing is retired here. Only memories the pass was shown, still active, can be named.
529
+ * settle; nothing is retired here. Only memories still active can be named.
388
530
  */
389
531
  same(db, op, ctx) {
390
532
  const [a, b] = (Array.isArray(op.ids) ? op.ids : []).map(idOf);
391
533
  if (!a || !b || a === b) throw new Rejected('same names two different memories');
392
- for (const id of [a, b]) {
393
- if (ctx.shown && !ctx.shown.has(id)) throw new Rejected(`m${id} is not one of the memories this pass was shown`);
394
- if (memory.get(db, id).state !== 'active') throw new Rejected(`m${id} is not active`);
395
- }
534
+ for (const id of [a, b]) if (memory.get(db, id).state !== 'active') throw new Rejected(`m${id} is not active`);
396
535
  const [lo, hi] = a < b ? [a, b] : [b, a];
397
536
  const note = typeof op.note === 'string' && op.note.trim() ? op.note.replace(/\s+/g, ' ').trim().slice(0, 200) : 'they say one thing in other words';
398
- if (ctx.shown) return judgedFirst(db, { label: 'same', lo, hi, note, said: 'say the same thing' });
399
537
  if (!raiseNotice(db, { kind: 'lookalike', a: lo, b: hi, note, now: ctx.now })) throw new Rejected('already raised');
400
538
  return `raised for the user: m${lo} and m${hi} say the same thing — ${note}`;
401
539
  },
402
-
403
- /** A workflow the model thinks it saw the user repeat. Always a proposal. */
404
- procedure(db, op, ctx) {
405
- const title = typeof op.title === 'string' ? op.title.trim() : '';
406
- const body = typeof op.body === 'string' ? op.body.trim() : '';
407
- if (!title || !body) throw new Rejected('a procedure needs a title and steps');
408
- const { scope, project } = resolveScope(db, op.scope);
409
- // A routine is walked through more than once: what starts it, said in one turn only, was a one-off ask. Said nowhere, it is
410
- // the model's own words for a pattern, and the user is asked.
411
- const trigger = (typeof op.cue === 'string' && op.cue.trim() ? op.cue : title).replace(/\s+/g, ' ').trim();
412
- const turns = db.prepare('SELECT text FROM user_turns ORDER BY id DESC LIMIT 2000').all();
413
- if (turns.filter((t) => overlap(trigger, t.text) >= SAME_STATEMENT).length === 1) throw new Rejected(`a routine is something done more than once, and "${trigger.slice(0, 80)}" was said once`);
414
- const existing = db.prepare(`SELECT * FROM memories WHERE scope = ? AND type = 'procedure' AND state IN ('active', 'unconfirmed', 'rejected')`).all(scope);
415
- const twin = existing.find((m) => overlap(title, m.title) >= SAME_STATEMENT);
416
- if (twin) throw new Rejected(`a workflow like this exists already (m${twin.id}, ${twin.state})`);
417
- const { memory: saved } = memory.add(db, {
418
- type: 'procedure', title, cue: typeof op.cue === 'string' ? op.cue : undefined, body: body.slice(0, 2000), project,
419
- provenance: 'inferred', writtenBy: ctx.source, now: ctx.now,
420
- });
421
- return `held for confirmation ${line(saved)}`;
422
- },
423
540
  };
424
541
 
425
- /** A pair a pass that read memories by number named, put to the pair judge before the user (`nameForJudging`). */
426
- function judgedFirst(db, { label, lo, hi, note, said }) {
427
- const not = nameForJudging(db, { label, a: lo, b: hi, note });
428
- if (not) throw new Rejected(not);
429
- return `to be judged on their own: m${lo} and m${hi} ${said} — ${note}`;
430
- }
431
-
432
542
  /** Each operation in turn. With `refused` — why — every one that would retire a memory is dropped for that reason. */
433
543
  function applyEach(db, ops, ctx, refused = null) {
434
544
  const applied = [];
435
545
  const dropped = [];
546
+ const repeats = [];
547
+ const run = { ...ctx, refused, pairs: [], filed: [] };
436
548
  const allowed = OPS_BY_SOURCE[ctx.source] ?? [];
437
- for (const op of ops) {
549
+ ops.forEach((op, index) => {
438
550
  const name = op?.op;
439
551
  try {
440
552
  if (!allowed.includes(name)) throw new Rejected(`operation "${name}" is not allowed from ${ctx.source}`);
441
- if (refused && RETIRING.includes(name)) {
553
+ // A rewrite too wide to take: from a file of operations the replacement is dropped whole; the pass that read the
554
+ // conversation keeps what the user said and retires nothing (`supersede`).
555
+ if (refused && RETIRING.includes(name) && ctx.source !== 'scribe') {
442
556
  const old = idOf(op.old);
443
557
  throw new Rejected(`${Number.isNaN(old) ? 'what it replaces' : `m${old}`} is left as it is — ${refused}`);
444
558
  }
445
- applied.push(HANDLERS[name](db, op, ctx));
559
+ applied.push(HANDLERS[name](db, op, { ...run, index }));
446
560
  } catch (cause) {
447
561
  // Rejected and UsageError are verdicts on the model's suggestion; anything else is a bug and must surface.
448
562
  if (!(cause instanceof Rejected) && !(cause instanceof memory.UsageError)) throw cause;
449
563
  dropped.push(`${name ?? '?'}: ${cause.message}`);
564
+ if (cause instanceof Known) repeats.push(dropped.at(-1));
450
565
  }
451
- }
452
- return { applied, dropped };
566
+ });
567
+ return { applied, dropped, repeats, pairs: run.pairs, filed: run.filed, refused };
453
568
  }
454
569
 
455
- /** Why the memories retired since `before` (id → scope of every active memory then) are too many to take, or null. */
456
- function tooWide(db, before) {
570
+ /** Every active memory's scope by id: what `tooWide` counts a change against. */
571
+ export const activeScopes = (db) => new Map(db.prepare(`SELECT id, scope FROM memories WHERE state = 'active'`).all().map((m) => [m.id, m.scope]));
572
+
573
+ /** Why the memories retired since `before` (`activeScopes` then) are too many to take, or null. */
574
+ export function tooWide(db, before) {
457
575
  const still = new Set(db.prepare(`SELECT id FROM memories WHERE state = 'active'`).all().map((m) => m.id));
458
576
  const retired = [...before].filter(([id]) => !still.has(id)).map(([, scope]) => scope);
459
577
  const wide = ': a rewrite that wide is not taken from a model';
@@ -470,27 +588,43 @@ function tooWide(db, before) {
470
588
 
471
589
  /**
472
590
  * @param ops what the model returned
473
- * @param ctx.source 'scribe' | 'dream' — decides which operations are even allowed
474
- * @param ctx.turns Map of turn id → user_turns row the model was shown
591
+ * @param ctx.source 'scribe' | 'dream' | 'worker' — decides which operations are even allowed
592
+ * @param ctx.turns Map of turn id → user_turns row the model was shown
593
+ * @param ctx.derivedBy the extractor's version, written on what is filed; `ctx.runId` the ledger row of the call
594
+ * @param ctx.labels Map of an operation's place in `ops` → `{ old, label }`: what the pair judge said of the
595
+ * look-alike that operation's statement has. Absent, a look-alike becomes a notice.
596
+ * @param ctx.dry true to change nothing: the answer is what would have happened
597
+ *
598
+ * Returns `applied` and `dropped` (lines; `repeats` are those of `dropped` that only say memory holds it already),
599
+ * `written` (ids of the memories this call made), `pairs` (each statement
600
+ * that has a look-alike nobody judged: `{ index, old, type, scope, body, said }`, `said` when its turn was) and
601
+ * `filed` (each statement that passed the rules: `{ turnId, type, scope, body, quote, existing, replaces, mayReplace,
602
+ * later }` — `existing` the memory already holding it, or null; `replaces` the memory it retires; `mayReplace` one it
603
+ * retires if the pair judge, not yet asked, agrees; `mayBeOlder` a memory said after it, in whose place it is not
604
+ * filed if the judge calls the two one matter; `later` that memory, once the judge has),
605
+ * and `refused`: why nothing was retired, when the answer would have retired too much, else null.
475
606
  */
476
607
  export function applyOps(db, ops, ctx) {
477
- if (!Array.isArray(ops)) return { applied: [], dropped: ['the answer had no list of operations'], written: [] };
608
+ if (!Array.isArray(ops)) return { applied: [], dropped: ['the answer had no list of operations'], written: [], repeats: [], pairs: [], filed: [], refused: null };
478
609
 
479
610
  return tx(db, () => {
480
611
  // `written`: the memories this call made, by id. The transaction holds the write lock, so every id past the highest
481
612
  // at its start is this call's — never another process's that finished a moment before or after.
482
613
  const from = db.prepare('SELECT COALESCE(MAX(id), 0) AS id FROM memories').get().id;
483
- const made = (outcome) => ({ ...outcome, written: db.prepare('SELECT id FROM memories WHERE id > ? ORDER BY id').all(from).map((m) => m.id) });
484
- if (ctx.source !== 'dream') return made(applyEach(db, ops, ctx));
485
- // The slow pass's answer is tried first and counted, then kept or undone: what it would retire is known exactly
486
- // that way — a replacement that is only a guess, or aimed at the wrong memory, retires nothing. Too wide, it is
487
- // applied again without what retires anything, and what it adds is kept.
488
- const before = new Map(db.prepare(`SELECT id, scope FROM memories WHERE state = 'active'`).all().map((m) => [m.id, m.scope]));
489
- db.exec('SAVEPOINT dream_ops');
490
- const tried = applyEach(db, ops, ctx);
491
- const refused = tooWide(db, before);
492
- if (refused) db.exec('ROLLBACK TO dream_ops');
493
- db.exec('RELEASE dream_ops');
494
- return made(refused ? applyEach(db, ops, ctx, refused) : tried);
614
+ // The answer is tried first and counted, then kept or undone: what it would retire is known exactly that way — a
615
+ // replacement aimed at the wrong memory retires nothing. Too wide, it is applied again without what retires
616
+ // anything, and what it adds is kept.
617
+ const before = activeScopes(db);
618
+ const attempt = (refused, keep) => {
619
+ db.exec('SAVEPOINT apply_ops');
620
+ const outcome = applyEach(db, ops, ctx, refused);
621
+ const wide = refused ? null : tooWide(db, before);
622
+ if (wide || !keep) db.exec('ROLLBACK TO apply_ops');
623
+ db.exec('RELEASE apply_ops');
624
+ return { outcome, wide };
625
+ };
626
+ const tried = attempt(null, !ctx.dry);
627
+ const outcome = tried.wide ? attempt(tried.wide, !ctx.dry).outcome : tried.outcome;
628
+ return { ...outcome, written: ctx.dry ? [] : db.prepare('SELECT id FROM memories WHERE id > ? ORDER BY id').all(from).map((m) => m.id) };
495
629
  });
496
630
  }