wendkeep 0.57.2 → 0.58.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +35 -0
- package/README.en.md +42 -8
- package/README.md +42 -8
- package/bin/wendkeep.mjs +16 -4
- package/hooks/brain-inject.mjs +186 -19
- package/hooks/change-core.mjs +42 -5
- package/hooks/lessons-core.mjs +8 -2
- package/hooks/memory-handoff.mjs +199 -0
- package/hooks/memory-mode.mjs +39 -0
- package/hooks/memory-schema.mjs +310 -0
- package/hooks/memory-store.mjs +660 -0
- package/hooks/obsidian-common.mjs +241 -6
- package/hooks/session-ensure.mjs +20 -0
- package/hooks/session-start.mjs +13 -4
- package/hooks/session-stop.mjs +138 -16
- package/hooks/vault-health.mjs +161 -1
- package/package.json +2 -2
- package/src/init.mjs +2 -0
- package/src/memory.mjs +235 -0
- package/src/taxonomy.mjs +3 -0
- package/src/validate-memory.mjs +115 -0
|
@@ -0,0 +1,660 @@
|
|
|
1
|
+
// Durable, local-only store for shared-memory events.
|
|
2
|
+
//
|
|
3
|
+
// Producers only create immutable outbox files. The projector is the sole ledger writer:
|
|
4
|
+
// it takes MEMORY.lock, durably appends events, deterministically replays the complete
|
|
5
|
+
// ledger, then atomically publishes SHARED_MEMORY.md and MEMORY_CANDIDATES.jsonl.
|
|
6
|
+
import { createHash } from 'node:crypto';
|
|
7
|
+
import {
|
|
8
|
+
closeSync, copyFileSync, existsSync, fsyncSync, mkdirSync, openSync, readFileSync,
|
|
9
|
+
readdirSync, unlinkSync, writeFileSync,
|
|
10
|
+
} from 'node:fs';
|
|
11
|
+
import { join } from 'node:path';
|
|
12
|
+
import {
|
|
13
|
+
renderSharedMemory,
|
|
14
|
+
sanitizeMemoryText,
|
|
15
|
+
validateMemoryEvent,
|
|
16
|
+
} from './memory-schema.mjs';
|
|
17
|
+
import { LOCK_BUSY, withPathLock, writeFileAtomic } from './session-note-io.mjs';
|
|
18
|
+
|
|
19
|
+
const EVENT_ID = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
|
|
20
|
+
|
|
21
|
+
export class MemoryEventCollision extends Error {
|
|
22
|
+
constructor(eventId, message = `Memory event ID collision: ${eventId}`) {
|
|
23
|
+
super(message);
|
|
24
|
+
this.name = 'MemoryEventCollision';
|
|
25
|
+
this.code = 'MEMORY_EVENT_COLLISION';
|
|
26
|
+
this.eventId = eventId;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export class MemoryLedgerCorruption extends Error {
|
|
31
|
+
constructor(errors, message = 'Memory ledger is corrupt; run `wendkeep memory repair`.') {
|
|
32
|
+
super(message);
|
|
33
|
+
this.name = 'MemoryLedgerCorruption';
|
|
34
|
+
this.code = 'MEMORY_LEDGER_CORRUPT';
|
|
35
|
+
this.errors = Array.isArray(errors) ? errors : [];
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export class MemoryOutboxCorruption extends Error {
|
|
40
|
+
constructor(path, cause) {
|
|
41
|
+
super(`Memory outbox file is corrupt: ${path}`);
|
|
42
|
+
this.name = 'MemoryOutboxCorruption';
|
|
43
|
+
this.code = 'MEMORY_OUTBOX_CORRUPT';
|
|
44
|
+
this.path = path;
|
|
45
|
+
this.cause = cause;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function brainDir(vaultBase) {
|
|
50
|
+
return join(vaultBase, '.brain');
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function outboxDir(vaultBase) {
|
|
54
|
+
return join(brainDir(vaultBase), 'memory-outbox');
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function ledgerPath(vaultBase) {
|
|
58
|
+
return join(brainDir(vaultBase), 'MEMORY_EVENTS.jsonl');
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function sharedPath(vaultBase) {
|
|
62
|
+
return join(brainDir(vaultBase), 'SHARED_MEMORY.md');
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function corePath(vaultBase) {
|
|
66
|
+
return join(brainDir(vaultBase), 'CORE.md');
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function candidatesPath(vaultBase) {
|
|
70
|
+
return join(brainDir(vaultBase), 'MEMORY_CANDIDATES.jsonl');
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function lockTarget(vaultBase) {
|
|
74
|
+
return join(brainDir(vaultBase), 'MEMORY');
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function canonicalize(value) {
|
|
78
|
+
if (Array.isArray(value)) return value.map(canonicalize);
|
|
79
|
+
if (value && typeof value === 'object') {
|
|
80
|
+
const out = {};
|
|
81
|
+
for (const key of Object.keys(value).sort()) {
|
|
82
|
+
if (value[key] !== undefined) out[key] = canonicalize(value[key]);
|
|
83
|
+
}
|
|
84
|
+
return out;
|
|
85
|
+
}
|
|
86
|
+
return value;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
export function canonicalMemoryJson(value) {
|
|
90
|
+
return JSON.stringify(canonicalize(value));
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function sha256(value) {
|
|
94
|
+
return createHash('sha256').update(String(value)).digest('hex');
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
export function hashMemoryValue(value) {
|
|
98
|
+
return sha256(canonicalMemoryJson(value));
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
function sanitizeValue(value) {
|
|
102
|
+
if (typeof value === 'string') return sanitizeMemoryText(value);
|
|
103
|
+
if (Array.isArray(value)) return value.map(sanitizeValue);
|
|
104
|
+
if (value && typeof value === 'object') {
|
|
105
|
+
return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, sanitizeValue(item)]));
|
|
106
|
+
}
|
|
107
|
+
return value;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
function sanitizeEvent(event) {
|
|
111
|
+
return sanitizeValue(structuredClone(event));
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function projectIdForVault(vaultBase) {
|
|
115
|
+
try {
|
|
116
|
+
const project = JSON.parse(readFileSync(join(brainDir(vaultBase), 'PROJECT.json'), 'utf8'));
|
|
117
|
+
return typeof project.projectId === 'string' && project.projectId ? project.projectId : undefined;
|
|
118
|
+
} catch {
|
|
119
|
+
return undefined;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
function assertValidEvent(event, vaultBase) {
|
|
124
|
+
const sanitized = sanitizeEvent(event);
|
|
125
|
+
const projectId = vaultBase ? projectIdForVault(vaultBase) : undefined;
|
|
126
|
+
const result = validateMemoryEvent(sanitized, projectId ? { projectId } : {});
|
|
127
|
+
if (!result.ok) {
|
|
128
|
+
const error = new TypeError(`Invalid memory event: ${result.errors.join(' ')}`);
|
|
129
|
+
error.code = 'MEMORY_EVENT_INVALID';
|
|
130
|
+
error.errors = result.errors;
|
|
131
|
+
throw error;
|
|
132
|
+
}
|
|
133
|
+
if (!EVENT_ID.test(sanitized.event_id)) {
|
|
134
|
+
const error = new TypeError('Invalid memory event: event_id is not filename-safe.');
|
|
135
|
+
error.code = 'MEMORY_EVENT_INVALID';
|
|
136
|
+
throw error;
|
|
137
|
+
}
|
|
138
|
+
return sanitized;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
function eventHash(event) {
|
|
142
|
+
return sha256(canonicalMemoryJson(event));
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* Persist one immutable producer event using exclusive creation. A retry with the same
|
|
147
|
+
* canonical payload is a no-op; reusing the ID for different bytes is an observable error.
|
|
148
|
+
*/
|
|
149
|
+
export function enqueueMemoryEvent(vaultBase, event) {
|
|
150
|
+
const checked = assertValidEvent(event, vaultBase);
|
|
151
|
+
const dir = outboxDir(vaultBase);
|
|
152
|
+
mkdirSync(dir, { recursive: true });
|
|
153
|
+
const path = join(dir, `${checked.event_id}.json`);
|
|
154
|
+
const payload = `${canonicalMemoryJson(checked)}\n`;
|
|
155
|
+
const hash = eventHash(checked);
|
|
156
|
+
|
|
157
|
+
let fd;
|
|
158
|
+
try {
|
|
159
|
+
fd = openSync(path, 'wx');
|
|
160
|
+
writeFileSync(fd, payload, 'utf8');
|
|
161
|
+
fsyncSync(fd);
|
|
162
|
+
return { status: 'enqueued', path, eventId: checked.event_id, hash };
|
|
163
|
+
} catch (error) {
|
|
164
|
+
if (error?.code !== 'EEXIST') throw error;
|
|
165
|
+
let existing;
|
|
166
|
+
try {
|
|
167
|
+
existing = JSON.parse(readFileSync(path, 'utf8'));
|
|
168
|
+
} catch (cause) {
|
|
169
|
+
throw new MemoryEventCollision(checked.event_id, `Existing outbox event is unreadable: ${checked.event_id}`);
|
|
170
|
+
}
|
|
171
|
+
if (eventHash(existing) !== hash || canonicalMemoryJson(existing) !== canonicalMemoryJson(checked)) {
|
|
172
|
+
throw new MemoryEventCollision(checked.event_id);
|
|
173
|
+
}
|
|
174
|
+
return { status: 'duplicate', path, eventId: checked.event_id, hash };
|
|
175
|
+
} finally {
|
|
176
|
+
if (fd !== undefined) closeSync(fd);
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
function ledgerError(line, message, partial = false) {
|
|
181
|
+
return { line, message, partial };
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/** Read a ledger without hiding a valid prefix when its tail is corrupt or partial. */
|
|
185
|
+
export function readMemoryLedger(vaultBase) {
|
|
186
|
+
const path = ledgerPath(vaultBase);
|
|
187
|
+
if (!existsSync(path)) {
|
|
188
|
+
return { status: 'ok', path, raw: '', events: [], eventIds: new Set(), errors: [] };
|
|
189
|
+
}
|
|
190
|
+
const raw = readFileSync(path, 'utf8').replace(/\r\n/g, '\n');
|
|
191
|
+
const lines = raw.split('\n');
|
|
192
|
+
const hasPartialTail = raw.length > 0 && !raw.endsWith('\n');
|
|
193
|
+
if (!hasPartialTail) lines.pop();
|
|
194
|
+
const events = [];
|
|
195
|
+
const eventIds = new Set();
|
|
196
|
+
const eventPayloads = new Map();
|
|
197
|
+
const errors = [];
|
|
198
|
+
const projectId = projectIdForVault(vaultBase);
|
|
199
|
+
|
|
200
|
+
lines.forEach((line, index) => {
|
|
201
|
+
const lineNumber = index + 1;
|
|
202
|
+
const partial = hasPartialTail && index === lines.length - 1;
|
|
203
|
+
if (!line.trim()) {
|
|
204
|
+
errors.push(ledgerError(lineNumber, 'blank ledger line', partial));
|
|
205
|
+
return;
|
|
206
|
+
}
|
|
207
|
+
let parsed;
|
|
208
|
+
try {
|
|
209
|
+
parsed = JSON.parse(line);
|
|
210
|
+
} catch (error) {
|
|
211
|
+
errors.push(ledgerError(lineNumber, `invalid JSON: ${error.message}`, partial));
|
|
212
|
+
return;
|
|
213
|
+
}
|
|
214
|
+
const validation = validateMemoryEvent(parsed, projectId ? { projectId } : {});
|
|
215
|
+
if (!validation.ok) {
|
|
216
|
+
errors.push(ledgerError(lineNumber, validation.errors.join(' '), partial));
|
|
217
|
+
return;
|
|
218
|
+
}
|
|
219
|
+
const payload = canonicalMemoryJson(parsed);
|
|
220
|
+
if (eventIds.has(parsed.event_id)) {
|
|
221
|
+
if (eventPayloads.get(parsed.event_id) !== payload) {
|
|
222
|
+
errors.push(ledgerError(lineNumber, `event_id collision: ${parsed.event_id}`, partial));
|
|
223
|
+
}
|
|
224
|
+
return;
|
|
225
|
+
}
|
|
226
|
+
eventIds.add(parsed.event_id);
|
|
227
|
+
eventPayloads.set(parsed.event_id, payload);
|
|
228
|
+
events.push(parsed);
|
|
229
|
+
});
|
|
230
|
+
|
|
231
|
+
return {
|
|
232
|
+
status: errors.length ? 'corrupt' : 'ok',
|
|
233
|
+
path,
|
|
234
|
+
raw,
|
|
235
|
+
events,
|
|
236
|
+
eventIds,
|
|
237
|
+
errors,
|
|
238
|
+
};
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
function eventOrder(left, right) {
|
|
242
|
+
return (Number(left.base_revision ?? 0) - Number(right.base_revision ?? 0))
|
|
243
|
+
|| String(left.effective_at || left.observed_at).localeCompare(String(right.effective_at || right.observed_at))
|
|
244
|
+
|| Number(left.turn_sequence ?? 0) - Number(right.turn_sequence ?? 0)
|
|
245
|
+
|| String(left.event_id).localeCompare(String(right.event_id));
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
function comparable(left, right) {
|
|
249
|
+
if (left.activation_id === right.activation_id) return true;
|
|
250
|
+
const leftSupersedes = left.supersedes_event_id || left.supersedes;
|
|
251
|
+
const rightSupersedes = right.supersedes_event_id || right.supersedes;
|
|
252
|
+
return leftSupersedes === right.event_id || rightSupersedes === left.event_id
|
|
253
|
+
|| (Array.isArray(leftSupersedes) && leftSupersedes.includes(right.event_id))
|
|
254
|
+
|| (Array.isArray(rightSupersedes) && rightSupersedes.includes(left.event_id));
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
function conflictGroupKey(event) {
|
|
258
|
+
if (event.operation !== 'replace') return null;
|
|
259
|
+
if (!Number.isInteger(event.base_revision) || typeof event.base_value_hash !== 'string') return null;
|
|
260
|
+
return `${event.memory_key}\u0000${event.base_revision}\u0000${event.base_value_hash}`;
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
function candidateId(reason, memoryKey, eventIds) {
|
|
264
|
+
return `memcand-${sha256(`${reason}\u0000${memoryKey}\u0000${eventIds.join('\u0000')}`).slice(0, 16)}`;
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
// CORE is prose, not an operational data store. Only an explicit, single-line marker
|
|
268
|
+
// participates in precedence so the projector never guesses meaning from human text:
|
|
269
|
+
// <!-- wk-memory: release.push="manual-only" -->
|
|
270
|
+
function readCoreInvariants(vaultBase) {
|
|
271
|
+
let core;
|
|
272
|
+
try {
|
|
273
|
+
core = readFileSync(corePath(vaultBase), 'utf8').replace(/\r\n/g, '\n');
|
|
274
|
+
} catch {
|
|
275
|
+
return new Map();
|
|
276
|
+
}
|
|
277
|
+
const invariants = new Map();
|
|
278
|
+
const marker = /^<!--\s*wk-memory:\s*([A-Za-z0-9][A-Za-z0-9._-]*)=(.+)\s*-->$/;
|
|
279
|
+
for (const line of core.split('\n')) {
|
|
280
|
+
const match = line.trim().match(marker);
|
|
281
|
+
if (!match) continue;
|
|
282
|
+
try {
|
|
283
|
+
invariants.set(match[1], JSON.parse(match[2].trim()));
|
|
284
|
+
} catch { /* malformed prose markers do not become implicit authority */ }
|
|
285
|
+
}
|
|
286
|
+
return invariants;
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
function blockedByCoreCandidate(event, coreValue) {
|
|
290
|
+
return {
|
|
291
|
+
v: 1,
|
|
292
|
+
candidate_id: candidateId('blocked_by_core', event.memory_key, [event.event_id]),
|
|
293
|
+
reason: 'blocked_by_core',
|
|
294
|
+
status: 'blocked_by_core',
|
|
295
|
+
memory_key: event.memory_key,
|
|
296
|
+
event_ids: [event.event_id],
|
|
297
|
+
proposed_value: event.value,
|
|
298
|
+
core_value: coreValue,
|
|
299
|
+
provenance: {
|
|
300
|
+
authority: 'core',
|
|
301
|
+
source: '.brain/CORE.md',
|
|
302
|
+
core_value_hash: hashMemoryValue(coreValue),
|
|
303
|
+
},
|
|
304
|
+
events: [event],
|
|
305
|
+
};
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
function conflictCandidate(memoryKey, events, currentEvent = null) {
|
|
309
|
+
const ordered = [...events].sort(eventOrder);
|
|
310
|
+
const eventIds = ordered.map((item) => item.event_id).sort();
|
|
311
|
+
const byId = new Map(ordered.map((item) => [item.event_id, item]));
|
|
312
|
+
return {
|
|
313
|
+
v: 1,
|
|
314
|
+
candidate_id: candidateId('conflict', memoryKey, eventIds),
|
|
315
|
+
reason: 'conflict',
|
|
316
|
+
memory_key: memoryKey,
|
|
317
|
+
event_ids: eventIds,
|
|
318
|
+
values: eventIds.map((id) => byId.get(id).value),
|
|
319
|
+
base_revision: ordered[0]?.base_revision ?? currentEvent?.revision ?? 0,
|
|
320
|
+
base_value_hash: ordered[0]?.base_value_hash ?? (currentEvent ? hashMemoryValue(currentEvent.value) : null),
|
|
321
|
+
events: eventIds.map((id) => byId.get(id)),
|
|
322
|
+
};
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
function sortedObject(entries) {
|
|
326
|
+
return Object.fromEntries([...entries].sort(([left], [right]) => left.localeCompare(right)));
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
function currentEventFromRecord(record) {
|
|
330
|
+
if (!record) return null;
|
|
331
|
+
return { ...record.source, value: record.value, revision: record.revision };
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
function isCausallyOlder(event, current) {
|
|
335
|
+
if (!current) return false;
|
|
336
|
+
if (event.activation_id === current.activation_id) {
|
|
337
|
+
return Number(event.turn_sequence) < Number(current.turn_sequence);
|
|
338
|
+
}
|
|
339
|
+
if (Number.isInteger(event.activation_epoch) && Number.isInteger(current.activation_epoch)
|
|
340
|
+
&& event.activation_epoch !== current.activation_epoch) {
|
|
341
|
+
return event.activation_epoch < current.activation_epoch;
|
|
342
|
+
}
|
|
343
|
+
const eventEffective = Date.parse(event.effective_at || '');
|
|
344
|
+
const currentEffective = Date.parse(current.effective_at || '');
|
|
345
|
+
return Number.isFinite(eventEffective) && Number.isFinite(currentEffective)
|
|
346
|
+
&& eventEffective < currentEffective;
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
/**
|
|
350
|
+
* Pure deterministic reducer. It pre-detects incomparable scalar siblings so replay order
|
|
351
|
+
* never turns one concurrent writer into an accidental winner.
|
|
352
|
+
*/
|
|
353
|
+
export function reduceMemoryEvents(inputEvents = [], { coreInvariants = new Map() } = {}) {
|
|
354
|
+
const protectedValues = coreInvariants instanceof Map
|
|
355
|
+
? coreInvariants
|
|
356
|
+
: new Map(Object.entries(coreInvariants || {}));
|
|
357
|
+
const unique = new Map();
|
|
358
|
+
for (const raw of inputEvents) {
|
|
359
|
+
const event = assertValidEvent(raw);
|
|
360
|
+
const existing = unique.get(event.event_id);
|
|
361
|
+
if (existing && canonicalMemoryJson(existing) !== canonicalMemoryJson(event)) {
|
|
362
|
+
throw new MemoryEventCollision(event.event_id, `Ledger contains divergent payloads for ${event.event_id}`);
|
|
363
|
+
}
|
|
364
|
+
if (!existing) unique.set(event.event_id, event);
|
|
365
|
+
}
|
|
366
|
+
const events = [...unique.values()].sort(eventOrder);
|
|
367
|
+
|
|
368
|
+
const peerGroups = new Map();
|
|
369
|
+
for (const item of events) {
|
|
370
|
+
const key = conflictGroupKey(item);
|
|
371
|
+
if (!key) continue;
|
|
372
|
+
if (!peerGroups.has(key)) peerGroups.set(key, []);
|
|
373
|
+
peerGroups.get(key).push(item);
|
|
374
|
+
}
|
|
375
|
+
const conflictingIds = new Set();
|
|
376
|
+
const groupedCandidates = new Map();
|
|
377
|
+
for (const [key, group] of peerGroups) {
|
|
378
|
+
const incomparable = group.filter((item, index) => group.some((other, otherIndex) => index !== otherIndex && !comparable(item, other)));
|
|
379
|
+
if (incomparable.length < 2) continue;
|
|
380
|
+
const ordered = [...incomparable].sort(eventOrder);
|
|
381
|
+
ordered.forEach((item) => conflictingIds.add(item.event_id));
|
|
382
|
+
groupedCandidates.set(key, ordered);
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
const records = new Map();
|
|
386
|
+
const tombstones = new Map();
|
|
387
|
+
const candidates = [];
|
|
388
|
+
const emittedGroups = new Set();
|
|
389
|
+
const appliedEventIds = [];
|
|
390
|
+
const superseded = [];
|
|
391
|
+
let revision = 0;
|
|
392
|
+
|
|
393
|
+
for (const item of events) {
|
|
394
|
+
if (protectedValues.has(item.memory_key)) {
|
|
395
|
+
const coreValue = protectedValues.get(item.memory_key);
|
|
396
|
+
const agreesWithCore = item.operation === 'assert'
|
|
397
|
+
&& hashMemoryValue(item.value) === hashMemoryValue(coreValue);
|
|
398
|
+
if (!agreesWithCore) {
|
|
399
|
+
candidates.push(blockedByCoreCandidate(item, coreValue));
|
|
400
|
+
continue;
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
const groupKey = conflictGroupKey(item);
|
|
405
|
+
if (conflictingIds.has(item.event_id)) {
|
|
406
|
+
if (!emittedGroups.has(groupKey)) {
|
|
407
|
+
candidates.push(conflictCandidate(item.memory_key, groupedCandidates.get(groupKey)));
|
|
408
|
+
emittedGroups.add(groupKey);
|
|
409
|
+
}
|
|
410
|
+
continue;
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
const current = records.get(item.memory_key);
|
|
414
|
+
const currentSource = current?.source;
|
|
415
|
+
if (isCausallyOlder(item, currentSource)) {
|
|
416
|
+
superseded.push({ event_id: item.event_id, by_event_id: currentSource.event_id });
|
|
417
|
+
continue;
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
if (item.operation === 'assert') {
|
|
421
|
+
if (current && hashMemoryValue(current.value) !== hashMemoryValue(item.value)) {
|
|
422
|
+
candidates.push(conflictCandidate(item.memory_key, [currentEventFromRecord(current), item]));
|
|
423
|
+
continue;
|
|
424
|
+
}
|
|
425
|
+
if (!current) {
|
|
426
|
+
records.set(item.memory_key, { value: item.value, revision: 1, source: item });
|
|
427
|
+
tombstones.delete(item.memory_key);
|
|
428
|
+
revision += 1;
|
|
429
|
+
}
|
|
430
|
+
appliedEventIds.push(item.event_id);
|
|
431
|
+
continue;
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
if (item.operation === 'add') {
|
|
435
|
+
const oldValues = Array.isArray(current?.value) ? current.value : (current ? [current.value] : []);
|
|
436
|
+
const additions = Array.isArray(item.value) ? item.value : [item.value];
|
|
437
|
+
const byHash = new Map(oldValues.map((value) => [hashMemoryValue(value), value]));
|
|
438
|
+
additions.forEach((value) => byHash.set(hashMemoryValue(value), value));
|
|
439
|
+
const value = [...byHash.entries()].sort(([left], [right]) => left.localeCompare(right)).map(([, entry]) => entry);
|
|
440
|
+
records.set(item.memory_key, { value, revision: (current?.revision || 0) + 1, source: item });
|
|
441
|
+
tombstones.delete(item.memory_key);
|
|
442
|
+
revision += 1;
|
|
443
|
+
appliedEventIds.push(item.event_id);
|
|
444
|
+
continue;
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
if (item.operation === 'remove') {
|
|
448
|
+
if (!current) {
|
|
449
|
+
appliedEventIds.push(item.event_id);
|
|
450
|
+
continue;
|
|
451
|
+
}
|
|
452
|
+
const baseMatches = item.base_revision === undefined
|
|
453
|
+
|| (item.base_revision === current.revision
|
|
454
|
+
&& (!item.base_value_hash || item.base_value_hash === hashMemoryValue(current.value)));
|
|
455
|
+
if (!baseMatches) {
|
|
456
|
+
candidates.push(conflictCandidate(item.memory_key, [currentEventFromRecord(current), item]));
|
|
457
|
+
continue;
|
|
458
|
+
}
|
|
459
|
+
if (item.value !== null && item.value !== undefined && Array.isArray(current.value)) {
|
|
460
|
+
const removalHash = hashMemoryValue(item.value);
|
|
461
|
+
const value = current.value.filter((entry) => hashMemoryValue(entry) !== removalHash);
|
|
462
|
+
records.set(item.memory_key, { value, revision: current.revision + 1, source: item });
|
|
463
|
+
tombstones.set(`${item.memory_key}:${removalHash}`, {
|
|
464
|
+
event_id: item.event_id, removed_event_id: current.source.event_id, value_hash: removalHash,
|
|
465
|
+
});
|
|
466
|
+
} else {
|
|
467
|
+
records.delete(item.memory_key);
|
|
468
|
+
tombstones.set(item.memory_key, {
|
|
469
|
+
event_id: item.event_id,
|
|
470
|
+
removed_event_id: current.source.event_id,
|
|
471
|
+
value_hash: hashMemoryValue(current.value),
|
|
472
|
+
});
|
|
473
|
+
}
|
|
474
|
+
revision += 1;
|
|
475
|
+
appliedEventIds.push(item.event_id);
|
|
476
|
+
continue;
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
// replace
|
|
480
|
+
const explicitlySupersedes = item.supersedes_event_id === currentSource?.event_id
|
|
481
|
+
|| (Array.isArray(item.supersedes) && item.supersedes.includes(currentSource?.event_id));
|
|
482
|
+
const baseMatches = current
|
|
483
|
+
&& item.base_revision === current.revision
|
|
484
|
+
&& item.base_value_hash === hashMemoryValue(current.value);
|
|
485
|
+
if (!current || (!baseMatches && !explicitlySupersedes)) {
|
|
486
|
+
candidates.push(conflictCandidate(item.memory_key, current ? [currentEventFromRecord(current), item] : [item]));
|
|
487
|
+
continue;
|
|
488
|
+
}
|
|
489
|
+
records.set(item.memory_key, { value: item.value, revision: current.revision + 1, source: item });
|
|
490
|
+
tombstones.delete(item.memory_key);
|
|
491
|
+
revision += 1;
|
|
492
|
+
appliedEventIds.push(item.event_id);
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
const stateEntries = [...records].map(([key, record]) => [key, record.value]);
|
|
496
|
+
const recordEntries = [...records].map(([key, record]) => [key, record]);
|
|
497
|
+
const tombstoneEntries = [...tombstones];
|
|
498
|
+
const state = sortedObject(stateEntries);
|
|
499
|
+
const recordObject = sortedObject(recordEntries);
|
|
500
|
+
const tombstoneObject = sortedObject(tombstoneEntries);
|
|
501
|
+
candidates.sort((left, right) => left.candidate_id.localeCompare(right.candidate_id));
|
|
502
|
+
superseded.sort((left, right) => left.event_id.localeCompare(right.event_id));
|
|
503
|
+
const activeEvents = Object.entries(recordObject).map(([memoryKey, record]) => ({
|
|
504
|
+
...record.source,
|
|
505
|
+
memory_key: memoryKey,
|
|
506
|
+
operation: 'assert',
|
|
507
|
+
value: record.value,
|
|
508
|
+
}));
|
|
509
|
+
const eventCursor = events.at(-1)?.event_id || 'none';
|
|
510
|
+
const stateHash = hashMemoryValue({ state, tombstones: tombstoneObject });
|
|
511
|
+
|
|
512
|
+
return {
|
|
513
|
+
state,
|
|
514
|
+
records: recordObject,
|
|
515
|
+
candidates,
|
|
516
|
+
tombstones: tombstoneObject,
|
|
517
|
+
superseded,
|
|
518
|
+
appliedEventIds,
|
|
519
|
+
eventIds: events.map((item) => item.event_id),
|
|
520
|
+
activeEvents,
|
|
521
|
+
revision,
|
|
522
|
+
eventCursor,
|
|
523
|
+
stateHash,
|
|
524
|
+
};
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
function readOutbox(vaultBase) {
|
|
528
|
+
const dir = outboxDir(vaultBase);
|
|
529
|
+
if (!existsSync(dir)) return [];
|
|
530
|
+
return readdirSync(dir, { withFileTypes: true })
|
|
531
|
+
.filter((entry) => entry.isFile() && entry.name.endsWith('.json'))
|
|
532
|
+
.sort((left, right) => left.name.localeCompare(right.name))
|
|
533
|
+
.map((entry) => {
|
|
534
|
+
const path = join(dir, entry.name);
|
|
535
|
+
try {
|
|
536
|
+
const event = assertValidEvent(JSON.parse(readFileSync(path, 'utf8')), vaultBase);
|
|
537
|
+
if (`${event.event_id}.json` !== entry.name) throw new Error('filename does not match event_id');
|
|
538
|
+
return { event, path };
|
|
539
|
+
} catch (error) {
|
|
540
|
+
throw new MemoryOutboxCorruption(path, error);
|
|
541
|
+
}
|
|
542
|
+
});
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
function appendLedgerDurably(path, events) {
|
|
546
|
+
if (!events.length) return;
|
|
547
|
+
let fd;
|
|
548
|
+
try {
|
|
549
|
+
fd = openSync(path, 'a');
|
|
550
|
+
const payload = events.map((item) => canonicalMemoryJson(item)).join('\n') + '\n';
|
|
551
|
+
writeFileSync(fd, payload, 'utf8');
|
|
552
|
+
fsyncSync(fd);
|
|
553
|
+
} finally {
|
|
554
|
+
if (fd !== undefined) closeSync(fd);
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
function writeAtomicIfChanged(path, content) {
|
|
559
|
+
try {
|
|
560
|
+
if (readFileSync(path, 'utf8') === content) return false;
|
|
561
|
+
} catch { /* missing/unreadable projections are replaced atomically */ }
|
|
562
|
+
writeFileAtomic(path, content);
|
|
563
|
+
return true;
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
function injectFault(faultAt, boundary) {
|
|
567
|
+
if (faultAt === boundary) throw new Error(`Injected memory-store fault: ${boundary}`);
|
|
568
|
+
}
|
|
569
|
+
|
|
570
|
+
function projectLocked(vaultBase, { faultAt } = {}) {
|
|
571
|
+
const ledger = readMemoryLedger(vaultBase);
|
|
572
|
+
if (ledger.status !== 'ok') throw new MemoryLedgerCorruption(ledger.errors);
|
|
573
|
+
const outbox = readOutbox(vaultBase);
|
|
574
|
+
const ledgerById = new Map(ledger.events.map((item) => [item.event_id, item]));
|
|
575
|
+
const newEvents = [];
|
|
576
|
+
|
|
577
|
+
for (const { event: item } of outbox) {
|
|
578
|
+
const existing = ledgerById.get(item.event_id);
|
|
579
|
+
if (existing) {
|
|
580
|
+
if (canonicalMemoryJson(existing) !== canonicalMemoryJson(item)) throw new MemoryEventCollision(item.event_id);
|
|
581
|
+
continue;
|
|
582
|
+
}
|
|
583
|
+
ledgerById.set(item.event_id, item);
|
|
584
|
+
newEvents.push(item);
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
appendLedgerDurably(ledger.path, newEvents);
|
|
588
|
+
injectFault(faultAt, 'after-ledger');
|
|
589
|
+
|
|
590
|
+
const allEvents = [...ledger.events, ...newEvents];
|
|
591
|
+
const reduced = reduceMemoryEvents(allEvents, { coreInvariants: readCoreInvariants(vaultBase) });
|
|
592
|
+
const updatedAt = allEvents
|
|
593
|
+
.map((item) => item.observed_at)
|
|
594
|
+
.filter(Boolean)
|
|
595
|
+
.sort()
|
|
596
|
+
.at(-1);
|
|
597
|
+
const shared = renderSharedMemory({
|
|
598
|
+
revision: reduced.revision,
|
|
599
|
+
eventCursor: reduced.eventCursor,
|
|
600
|
+
events: reduced.activeEvents,
|
|
601
|
+
stateHash: reduced.stateHash,
|
|
602
|
+
updatedAt,
|
|
603
|
+
});
|
|
604
|
+
const candidates = reduced.candidates.map((item) => canonicalMemoryJson(item)).join('\n')
|
|
605
|
+
+ (reduced.candidates.length ? '\n' : '');
|
|
606
|
+
const sharedWritten = writeAtomicIfChanged(sharedPath(vaultBase), shared);
|
|
607
|
+
const candidatesWritten = writeAtomicIfChanged(candidatesPath(vaultBase), candidates);
|
|
608
|
+
injectFault(faultAt, 'after-projection');
|
|
609
|
+
|
|
610
|
+
for (const entry of outbox) unlinkSync(entry.path);
|
|
611
|
+
return {
|
|
612
|
+
status: 'projected',
|
|
613
|
+
appended: newEvents.length,
|
|
614
|
+
consumed: outbox.length,
|
|
615
|
+
pending: 0,
|
|
616
|
+
revision: reduced.revision,
|
|
617
|
+
eventCursor: reduced.eventCursor,
|
|
618
|
+
stateHash: reduced.stateHash,
|
|
619
|
+
candidates: reduced.candidates.length,
|
|
620
|
+
projectionsWritten: sharedWritten || candidatesWritten,
|
|
621
|
+
};
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
/** Serialize ledger append + full deterministic replay under .brain/MEMORY.lock. */
|
|
625
|
+
export function projectMemoryOutbox(vaultBase, options = {}) {
|
|
626
|
+
mkdirSync(brainDir(vaultBase), { recursive: true });
|
|
627
|
+
const pending = existsSync(outboxDir(vaultBase))
|
|
628
|
+
? readdirSync(outboxDir(vaultBase)).filter((name) => name.endsWith('.json')).length
|
|
629
|
+
: 0;
|
|
630
|
+
const result = withPathLock(
|
|
631
|
+
lockTarget(vaultBase),
|
|
632
|
+
() => projectLocked(vaultBase, options),
|
|
633
|
+
options.lock || {},
|
|
634
|
+
);
|
|
635
|
+
if (result === LOCK_BUSY) return { status: 'busy', pending };
|
|
636
|
+
return result;
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
/** Explicit repair: preserve exact corrupt bytes, then retain every independently valid line. */
|
|
640
|
+
export function repairMemoryLedger(vaultBase, options = {}) {
|
|
641
|
+
mkdirSync(brainDir(vaultBase), { recursive: true });
|
|
642
|
+
const result = withPathLock(lockTarget(vaultBase), () => {
|
|
643
|
+
const ledger = readMemoryLedger(vaultBase);
|
|
644
|
+
if (ledger.status === 'ok') return { status: 'unchanged', repairedLines: 0, backupPath: null };
|
|
645
|
+
const backupPath = `${ledger.path}.corrupt-${Date.now()}.bak`;
|
|
646
|
+
// copyFileSync preserves the exact evidence before any canonical rewrite.
|
|
647
|
+
copyFileSync(ledger.path, backupPath);
|
|
648
|
+
const repaired = ledger.events.map((item) => canonicalMemoryJson(item)).join('\n')
|
|
649
|
+
+ (ledger.events.length ? '\n' : '');
|
|
650
|
+
writeFileAtomic(ledger.path, repaired);
|
|
651
|
+
return {
|
|
652
|
+
status: 'repaired',
|
|
653
|
+
repairedLines: ledger.errors.length,
|
|
654
|
+
retainedEvents: ledger.events.length,
|
|
655
|
+
backupPath,
|
|
656
|
+
};
|
|
657
|
+
}, options.lock || {});
|
|
658
|
+
if (result === LOCK_BUSY) return { status: 'busy', repairedLines: 0, backupPath: null };
|
|
659
|
+
return result;
|
|
660
|
+
}
|