pi-hermes-memory 0.7.22 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -2
- package/package.json +3 -2
- package/src/config.ts +14 -1
- package/src/constants.ts +72 -0
- package/src/extension-root-migration.ts +429 -5
- package/src/handlers/auto-consolidate.ts +118 -8
- package/src/handlers/background-review.ts +164 -64
- package/src/handlers/child-process-watchdog.mjs +90 -0
- package/src/handlers/correction-detector.ts +52 -36
- package/src/handlers/pi-child-process.ts +270 -37
- package/src/handlers/review-memory-ops.ts +383 -0
- package/src/handlers/session-flush.ts +71 -4
- package/src/handlers/sync-markdown-memories.ts +143 -51
- package/src/index.ts +36 -17
- package/src/store/atomic-lock-coordinator.ts +252 -0
- package/src/store/canonical-storage-path.ts +54 -0
- package/src/store/db.ts +244 -9
- package/src/store/markdown-mutation-lock.ts +38 -0
- package/src/store/memory-store.ts +455 -57
- package/src/store/sqlite-memory-store.ts +121 -4
- package/src/tools/memory-tool.ts +48 -9
- package/src/tools/session-search-tool.ts +70 -12
- package/src/types.ts +6 -0
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
* - Content scanning before any write
|
|
12
12
|
*/
|
|
13
13
|
|
|
14
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
14
15
|
import * as fs from "node:fs/promises";
|
|
15
16
|
import * as path from "node:path";
|
|
16
17
|
import { scanContent } from "./content-scanner.js";
|
|
@@ -26,13 +27,29 @@ import {
|
|
|
26
27
|
} from "../constants.js";
|
|
27
28
|
import type { MemoryConfig, MemoryResult, MemorySnapshot, ConsolidationResult, MemoryCategory, MemoryOverflowStrategy } from "../types.js";
|
|
28
29
|
import { AGENT_ROOT } from "../paths.js";
|
|
30
|
+
import { canonicalMarkdownIdentity, withMarkdownMutationLock } from "./markdown-mutation-lock.js";
|
|
31
|
+
|
|
32
|
+
const MAX_EXTERNAL_WRITE_RETRIES = 2;
|
|
33
|
+
const RECOVERY_ACTIVE_GRACE_MS = 7 * 24 * 60 * 60 * 1000;
|
|
34
|
+
const RETIRED_RECOVERY_MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000;
|
|
35
|
+
const RETIRED_RECOVERY_MAX_COUNT = 32;
|
|
36
|
+
const RETIRED_RECOVERY_MAX_BYTES = 64 * 1024 * 1024;
|
|
37
|
+
const CONFLICT_ACTIVE_GRACE_MS = 7 * 24 * 60 * 60 * 1000;
|
|
38
|
+
const CONFLICT_MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000;
|
|
39
|
+
const CONFLICT_MAX_COUNT = 32;
|
|
40
|
+
const CONFLICT_MAX_BYTES = 64 * 1024 * 1024;
|
|
41
|
+
|
|
42
|
+
class ExternalMemoryWriteConflict extends Error {}
|
|
29
43
|
|
|
30
44
|
export class MemoryStore {
|
|
31
45
|
private memoryEntries: string[] = [];
|
|
32
46
|
private userEntries: string[] = [];
|
|
33
47
|
private failureEntries: string[] = [];
|
|
48
|
+
private fileFingerprints: Record<string, string> = {};
|
|
49
|
+
private storagePaths: Partial<Record<"memory" | "user" | "failure", string>> = {};
|
|
34
50
|
private snapshot: MemorySnapshot = { memory: "", user: "" };
|
|
35
51
|
private consolidator: ((target: "memory" | "user" | "failure", signal?: AbortSignal) => Promise<ConsolidationResult>) | null = null;
|
|
52
|
+
private mutationObserver: ((target: "memory" | "user" | "failure", entries: string[]) => Promise<string | null | undefined>) | null = null;
|
|
36
53
|
|
|
37
54
|
constructor(private config: MemoryConfig) {}
|
|
38
55
|
|
|
@@ -44,6 +61,12 @@ export class MemoryStore {
|
|
|
44
61
|
this.consolidator = fn;
|
|
45
62
|
}
|
|
46
63
|
|
|
64
|
+
setMutationObserver(
|
|
65
|
+
fn: (target: "memory" | "user" | "failure", entries: string[]) => Promise<string | null | undefined>,
|
|
66
|
+
): void {
|
|
67
|
+
this.mutationObserver = fn;
|
|
68
|
+
}
|
|
69
|
+
|
|
47
70
|
// ─── Path helpers ───
|
|
48
71
|
|
|
49
72
|
private get memoryDir(): string {
|
|
@@ -56,6 +79,18 @@ export class MemoryStore {
|
|
|
56
79
|
return path.join(this.memoryDir, MEMORY_FILE);
|
|
57
80
|
}
|
|
58
81
|
|
|
82
|
+
async getStorageIdentity(target: "memory" | "user" | "failure"): Promise<string> {
|
|
83
|
+
return this.resolveStoragePath(target);
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
private async resolveStoragePath(target: "memory" | "user" | "failure"): Promise<string> {
|
|
87
|
+
const cached = this.storagePaths[target];
|
|
88
|
+
if (cached) return cached;
|
|
89
|
+
const resolved = await canonicalMarkdownIdentity(this.pathFor(target));
|
|
90
|
+
this.storagePaths[target] = resolved;
|
|
91
|
+
return resolved;
|
|
92
|
+
}
|
|
93
|
+
|
|
59
94
|
private entriesFor(target: "memory" | "user" | "failure"): string[] {
|
|
60
95
|
if (target === "user") return this.userEntries;
|
|
61
96
|
if (target === "failure") return this.failureEntries;
|
|
@@ -86,15 +121,14 @@ export class MemoryStore {
|
|
|
86
121
|
|
|
87
122
|
async loadFromDisk(): Promise<void> {
|
|
88
123
|
await fs.mkdir(this.memoryDir, { recursive: true });
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
124
|
+
for (const target of ["memory", "user", "failure"] as const) {
|
|
125
|
+
const filePath = await this.resolveStoragePath(target);
|
|
126
|
+
const state = await this.readFileState(filePath);
|
|
127
|
+
this.setEntries(target, [...new Set(state.entries)]);
|
|
128
|
+
this.fileFingerprints[filePath] = state.fingerprint;
|
|
129
|
+
}
|
|
92
130
|
|
|
93
131
|
// Deduplicate preserving order
|
|
94
|
-
this.memoryEntries = [...new Set(this.memoryEntries)];
|
|
95
|
-
this.userEntries = [...new Set(this.userEntries)];
|
|
96
|
-
this.failureEntries = [...new Set(this.failureEntries)];
|
|
97
|
-
|
|
98
132
|
// Capture frozen snapshot for system prompt injection
|
|
99
133
|
// Strip metadata comments — the LLM doesn't need to see timestamps
|
|
100
134
|
const strippedMemory = this.memoryEntries.map((e) => this.stripMetadata(e));
|
|
@@ -108,7 +142,7 @@ export class MemoryStore {
|
|
|
108
142
|
// ─── CRUD ───
|
|
109
143
|
|
|
110
144
|
async add(target: "memory" | "user" | "failure", content: string, signal?: AbortSignal): Promise<MemoryResult> {
|
|
111
|
-
return this.
|
|
145
|
+
return this.addWithConsolidation(target, content, signal, 1, "Entry added.");
|
|
112
146
|
}
|
|
113
147
|
|
|
114
148
|
async addFailure(content: string, options: {
|
|
@@ -119,7 +153,9 @@ export class MemoryStore {
|
|
|
119
153
|
project?: string;
|
|
120
154
|
}): Promise<MemoryResult> {
|
|
121
155
|
const failureText = this.buildFailureMemoryText(content, options);
|
|
122
|
-
return this.
|
|
156
|
+
return this.addWithConsolidation(
|
|
157
|
+
"failure", failureText, undefined, 1, "Failure memory saved: " + options.category, options.project,
|
|
158
|
+
);
|
|
123
159
|
}
|
|
124
160
|
|
|
125
161
|
getFailureEntries(maxAgeDays = 7): string[] {
|
|
@@ -139,8 +175,8 @@ export class MemoryStore {
|
|
|
139
175
|
target: "memory" | "user" | "failure",
|
|
140
176
|
content: string,
|
|
141
177
|
signal?: AbortSignal,
|
|
142
|
-
_retriesLeft = 1,
|
|
143
178
|
addedMessage = "Entry added.",
|
|
179
|
+
project?: string,
|
|
144
180
|
): Promise<MemoryResult> {
|
|
145
181
|
content = content.trim();
|
|
146
182
|
if (!content) return { success: false, error: "Content cannot be empty." };
|
|
@@ -148,18 +184,24 @@ export class MemoryStore {
|
|
|
148
184
|
const scanError = scanContent(content);
|
|
149
185
|
if (scanError) return { success: false, error: scanError };
|
|
150
186
|
|
|
187
|
+
await this.syncTargetFromDiskIfChanged(target);
|
|
151
188
|
const entries = this.entriesFor(target);
|
|
152
189
|
const limit = this.charLimit(target);
|
|
153
190
|
|
|
154
191
|
// Check for duplicate — strip metadata from existing entries before comparing
|
|
155
|
-
const
|
|
156
|
-
|
|
192
|
+
const normalizedProject = project?.trim() || null;
|
|
193
|
+
const duplicate = entries.some((entry) => {
|
|
194
|
+
const decoded = this.decodeEntry(entry);
|
|
195
|
+
return decoded.text === content
|
|
196
|
+
&& (target !== "failure" || decoded.project === normalizedProject);
|
|
197
|
+
});
|
|
198
|
+
if (duplicate) {
|
|
157
199
|
return this.successResponse(target, "Entry already exists (no duplicate added).");
|
|
158
200
|
}
|
|
159
201
|
|
|
160
202
|
// Encode metadata: both dates = today
|
|
161
203
|
const today = new Date().toISOString().split("T")[0];
|
|
162
|
-
const encoded = this.encodeEntry(content, today, today);
|
|
204
|
+
const encoded = this.encodeEntry(content, today, today, project);
|
|
163
205
|
|
|
164
206
|
const newTotal = [...entries, encoded].join(ENTRY_DELIMITER).length;
|
|
165
207
|
if (newTotal > limit) {
|
|
@@ -169,20 +211,6 @@ export class MemoryStore {
|
|
|
169
211
|
return this.fifoEvictAndAdd(target, entries, encoded, content.length, limit);
|
|
170
212
|
}
|
|
171
213
|
|
|
172
|
-
// Auto-consolidate once if configured — limit retries to prevent infinite loops
|
|
173
|
-
if (strategy === "auto-consolidate" && this.consolidator && _retriesLeft > 0) {
|
|
174
|
-
try {
|
|
175
|
-
const result = await this.consolidator(target, signal);
|
|
176
|
-
if (result.consolidated) {
|
|
177
|
-
// CRITICAL: reload from disk — child process modified files, our arrays are stale
|
|
178
|
-
await this.loadFromDisk();
|
|
179
|
-
// Retry the add exactly once (retriesLeft = 0 means no more consolidation)
|
|
180
|
-
return this._add(target, content, signal, _retriesLeft - 1, addedMessage);
|
|
181
|
-
}
|
|
182
|
-
} catch {
|
|
183
|
-
// Consolidation failed — fall through to error
|
|
184
|
-
}
|
|
185
|
-
}
|
|
186
214
|
return this.memoryFullError(target, content.length);
|
|
187
215
|
}
|
|
188
216
|
|
|
@@ -193,6 +221,39 @@ export class MemoryStore {
|
|
|
193
221
|
return this.successResponse(target, addedMessage);
|
|
194
222
|
}
|
|
195
223
|
|
|
224
|
+
private async addWithConsolidation(
|
|
225
|
+
target: "memory" | "user" | "failure",
|
|
226
|
+
content: string,
|
|
227
|
+
signal: AbortSignal | undefined,
|
|
228
|
+
retriesLeft: number,
|
|
229
|
+
addedMessage: string,
|
|
230
|
+
project?: string,
|
|
231
|
+
): Promise<MemoryResult> {
|
|
232
|
+
const result = await this.runTargetMutation(
|
|
233
|
+
target,
|
|
234
|
+
() => this._add(target, content, signal, addedMessage, project),
|
|
235
|
+
);
|
|
236
|
+
if (
|
|
237
|
+
result.success
|
|
238
|
+
|| retriesLeft <= 0
|
|
239
|
+
|| this.memoryOverflowStrategy() !== "auto-consolidate"
|
|
240
|
+
|| !this.consolidator
|
|
241
|
+
|| !result.error?.startsWith("Memory at ")
|
|
242
|
+
) {
|
|
243
|
+
return result;
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
try {
|
|
247
|
+
const consolidation = await this.consolidator(target, signal);
|
|
248
|
+
if (consolidation.consolidated) {
|
|
249
|
+
await this.loadFromDisk();
|
|
250
|
+
return this.addWithConsolidation(target, content, signal, retriesLeft - 1, addedMessage, project);
|
|
251
|
+
}
|
|
252
|
+
} catch {
|
|
253
|
+
}
|
|
254
|
+
return result;
|
|
255
|
+
}
|
|
256
|
+
|
|
196
257
|
private async fifoEvictAndAdd(
|
|
197
258
|
target: "memory" | "user" | "failure",
|
|
198
259
|
entries: string[],
|
|
@@ -236,6 +297,10 @@ export class MemoryStore {
|
|
|
236
297
|
}
|
|
237
298
|
|
|
238
299
|
async replace(target: "memory" | "user" | "failure", oldText: string, newContent: string): Promise<MemoryResult> {
|
|
300
|
+
return this.runTargetMutation(target, () => this.replaceUnlocked(target, oldText, newContent));
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
private async replaceUnlocked(target: "memory" | "user" | "failure", oldText: string, newContent: string): Promise<MemoryResult> {
|
|
239
304
|
oldText = normalizeMemoryLookupText(oldText);
|
|
240
305
|
newContent = newContent.trim();
|
|
241
306
|
if (!oldText) return { success: false, error: "old_text cannot be empty." };
|
|
@@ -244,12 +309,13 @@ export class MemoryStore {
|
|
|
244
309
|
const scanError = scanContent(newContent);
|
|
245
310
|
if (scanError) return { success: false, error: scanError };
|
|
246
311
|
|
|
312
|
+
await this.syncTargetFromDiskIfChanged(target);
|
|
247
313
|
const entries = this.entriesFor(target);
|
|
248
314
|
// Match against stripped text (entries may have metadata comments)
|
|
249
315
|
const matches = entries.filter((e) => this.stripMetadata(e).includes(oldText));
|
|
250
316
|
|
|
251
317
|
if (matches.length === 0) return { success: false, error: `No entry matched '${oldText}'.` };
|
|
252
|
-
if (matches.length > 1 &&
|
|
318
|
+
if (matches.length > 1 && !this.areDistinctScopedFailureCopies(target, matches)) {
|
|
253
319
|
return {
|
|
254
320
|
success: false,
|
|
255
321
|
error: `Multiple entries matched '${oldText}'. Be more specific.`,
|
|
@@ -257,14 +323,12 @@ export class MemoryStore {
|
|
|
257
323
|
};
|
|
258
324
|
}
|
|
259
325
|
|
|
260
|
-
const idx = entries.indexOf(matches[0]);
|
|
261
|
-
// Preserve original created date, update last_referenced to today
|
|
262
|
-
const decoded = this.decodeEntry(matches[0]);
|
|
263
326
|
const today = new Date().toISOString().split("T")[0];
|
|
264
|
-
const
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
327
|
+
const replacements = new Map(matches.map((entry) => {
|
|
328
|
+
const decoded = this.decodeEntry(entry);
|
|
329
|
+
return [entry, this.encodeEntry(newContent, decoded.created, today, decoded.project ?? undefined)];
|
|
330
|
+
}));
|
|
331
|
+
const testEntries = entries.map((entry) => replacements.get(entry) ?? entry);
|
|
268
332
|
const newTotal = testEntries.join(ENTRY_DELIMITER).length;
|
|
269
333
|
|
|
270
334
|
if (newTotal > this.charLimit(target)) {
|
|
@@ -274,22 +338,26 @@ export class MemoryStore {
|
|
|
274
338
|
};
|
|
275
339
|
}
|
|
276
340
|
|
|
277
|
-
|
|
278
|
-
this.setEntries(target, entries);
|
|
341
|
+
this.setEntries(target, testEntries);
|
|
279
342
|
await this.saveToDisk(target);
|
|
280
343
|
|
|
281
344
|
return this.successResponse(target, "Entry replaced.");
|
|
282
345
|
}
|
|
283
346
|
|
|
284
347
|
async remove(target: "memory" | "user" | "failure", oldText: string): Promise<MemoryResult> {
|
|
348
|
+
return this.runTargetMutation(target, () => this.removeUnlocked(target, oldText));
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
private async removeUnlocked(target: "memory" | "user" | "failure", oldText: string): Promise<MemoryResult> {
|
|
285
352
|
oldText = normalizeMemoryLookupText(oldText);
|
|
286
353
|
if (!oldText) return { success: false, error: "old_text cannot be empty." };
|
|
287
354
|
|
|
355
|
+
await this.syncTargetFromDiskIfChanged(target);
|
|
288
356
|
const entries = this.entriesFor(target);
|
|
289
357
|
const matches = entries.filter((e) => this.stripMetadata(e).includes(oldText));
|
|
290
358
|
|
|
291
359
|
if (matches.length === 0) return { success: false, error: `No entry matched '${oldText}'.` };
|
|
292
|
-
if (matches.length > 1 &&
|
|
360
|
+
if (matches.length > 1 && !this.areDistinctScopedFailureCopies(target, matches)) {
|
|
293
361
|
return {
|
|
294
362
|
success: false,
|
|
295
363
|
error: `Multiple entries matched '${oldText}'. Be more specific.`,
|
|
@@ -297,9 +365,8 @@ export class MemoryStore {
|
|
|
297
365
|
};
|
|
298
366
|
}
|
|
299
367
|
|
|
300
|
-
const
|
|
301
|
-
|
|
302
|
-
this.setEntries(target, entries);
|
|
368
|
+
const matchedEntries = new Set(matches);
|
|
369
|
+
this.setEntries(target, entries.filter((entry) => !matchedEntries.has(entry)));
|
|
303
370
|
await this.saveToDisk(target);
|
|
304
371
|
|
|
305
372
|
return this.successResponse(target, "Entry removed.");
|
|
@@ -355,28 +422,40 @@ export class MemoryStore {
|
|
|
355
422
|
return this.userEntries.map((e) => this.stripMetadata(e));
|
|
356
423
|
}
|
|
357
424
|
|
|
425
|
+
/** Raw Markdown entries, including metadata, for exact SQLite reconciliation. */
|
|
426
|
+
getRawEntriesForSync(target: "memory" | "user" | "failure"): string[] {
|
|
427
|
+
return [...this.entriesFor(target)];
|
|
428
|
+
}
|
|
429
|
+
|
|
358
430
|
// ─── Internal helpers ───
|
|
359
431
|
|
|
360
432
|
/**
|
|
361
433
|
* Encode metadata (created, lastReferenced) as an HTML comment appended to entry text.
|
|
362
434
|
* The comment is invisible in markdown and transparent to the § delimiter.
|
|
363
435
|
*/
|
|
364
|
-
private encodeEntry(text: string, created: string, lastReferenced: string): string {
|
|
365
|
-
|
|
436
|
+
private encodeEntry(text: string, created: string, lastReferenced: string, project?: string): string {
|
|
437
|
+
const projectMetadata = project?.trim()
|
|
438
|
+
? `, project64=${Buffer.from(project.trim(), "utf-8").toString("base64url")}`
|
|
439
|
+
: "";
|
|
440
|
+
return `${text} <!-- created=${created}, last=${lastReferenced}${projectMetadata} -->`;
|
|
366
441
|
}
|
|
367
442
|
|
|
368
443
|
/**
|
|
369
444
|
* Decode entry text, extracting metadata if present.
|
|
370
445
|
* Falls back to today's date for legacy entries without metadata.
|
|
371
446
|
*/
|
|
372
|
-
private decodeEntry(raw: string): { text: string; created: string; lastReferenced: string } {
|
|
373
|
-
const match = raw.match(/^(.*?)\s*<!--\s*created=([^,]+),\s*last=([
|
|
447
|
+
private decodeEntry(raw: string): { text: string; created: string; lastReferenced: string; project: string | null } {
|
|
448
|
+
const match = raw.match(/^(.*?)\s*<!--\s*created=([^,]+),\s*last=([^,>]+)(?:,\s*project64=([A-Za-z0-9_-]+))?\s*-->\s*$/);
|
|
374
449
|
if (match) {
|
|
375
|
-
|
|
450
|
+
let project: string | null = null;
|
|
451
|
+
if (match[4]) {
|
|
452
|
+
try { project = Buffer.from(match[4], "base64url").toString("utf-8").trim() || null; } catch {}
|
|
453
|
+
}
|
|
454
|
+
return { text: match[1].trim(), created: match[2].trim(), lastReferenced: match[3].trim(), project };
|
|
376
455
|
}
|
|
377
456
|
// Legacy entry without metadata — use today as default
|
|
378
457
|
const today = new Date().toISOString().split("T")[0];
|
|
379
|
-
return { text: raw.trim(), created: today, lastReferenced: today };
|
|
458
|
+
return { text: raw.trim(), created: today, lastReferenced: today, project: null };
|
|
380
459
|
}
|
|
381
460
|
|
|
382
461
|
/** Strip metadata comment from entry text for display. */
|
|
@@ -384,6 +463,16 @@ export class MemoryStore {
|
|
|
384
463
|
return this.decodeEntry(text).text;
|
|
385
464
|
}
|
|
386
465
|
|
|
466
|
+
private areDistinctScopedFailureCopies(
|
|
467
|
+
target: "memory" | "user" | "failure",
|
|
468
|
+
entries: string[],
|
|
469
|
+
): boolean {
|
|
470
|
+
if (target !== "failure") return false;
|
|
471
|
+
const visibleTexts = new Set(entries.map((entry) => this.stripMetadata(entry)));
|
|
472
|
+
const scopes = new Set(entries.map((entry) => this.decodeEntry(entry).project));
|
|
473
|
+
return visibleTexts.size === 1 && scopes.size === entries.length;
|
|
474
|
+
}
|
|
475
|
+
|
|
387
476
|
private buildFailureMemoryText(content: string, options: {
|
|
388
477
|
category: MemoryCategory;
|
|
389
478
|
failureReason?: string;
|
|
@@ -397,7 +486,6 @@ export class MemoryStore {
|
|
|
397
486
|
if (options.failureReason) parts.push("Failed: " + options.failureReason);
|
|
398
487
|
if (options.toolState) parts.push("Tool state: " + options.toolState);
|
|
399
488
|
if (options.correctedTo) parts.push("Corrected to: " + options.correctedTo);
|
|
400
|
-
if (options.project) parts.push("Project: " + options.project);
|
|
401
489
|
return parts.join(" — ");
|
|
402
490
|
}
|
|
403
491
|
|
|
@@ -470,16 +558,79 @@ export class MemoryStore {
|
|
|
470
558
|
return `${header}\n${bulletList}`;
|
|
471
559
|
}
|
|
472
560
|
|
|
473
|
-
private
|
|
561
|
+
private fingerprint(content: Buffer | string): string {
|
|
562
|
+
return createHash("sha256").update(content).digest("hex");
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
private async readFileState(filePath: string): Promise<{ entries: string[]; fingerprint: string }> {
|
|
474
566
|
try {
|
|
475
|
-
const raw = await fs.readFile(filePath
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
567
|
+
const raw = await fs.readFile(filePath);
|
|
568
|
+
const content = raw.toString("utf-8");
|
|
569
|
+
const entries = content.trim()
|
|
570
|
+
? content.split(ENTRY_DELIMITER).map((entry) => entry.trim()).filter(Boolean)
|
|
571
|
+
: [];
|
|
572
|
+
return { entries, fingerprint: this.fingerprint(raw) };
|
|
573
|
+
} catch (error) {
|
|
574
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
|
|
575
|
+
return { entries: [], fingerprint: "missing" };
|
|
576
|
+
}
|
|
577
|
+
throw error;
|
|
480
578
|
}
|
|
481
579
|
}
|
|
482
580
|
|
|
581
|
+
private async syncTargetFromDiskIfChanged(target: "memory" | "user" | "failure"): Promise<void> {
|
|
582
|
+
const filePath = await this.resolveStoragePath(target);
|
|
583
|
+
const state = await this.readFileState(filePath);
|
|
584
|
+
if (this.fileFingerprints[filePath] === state.fingerprint) return;
|
|
585
|
+
|
|
586
|
+
this.setEntries(target, [...new Set(state.entries)]);
|
|
587
|
+
this.fileFingerprints[filePath] = state.fingerprint;
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
private async runTargetMutation(
|
|
591
|
+
target: "memory" | "user" | "failure",
|
|
592
|
+
mutation: () => Promise<MemoryResult>,
|
|
593
|
+
): Promise<MemoryResult> {
|
|
594
|
+
const storagePath = await this.resolveStoragePath(target);
|
|
595
|
+
return withMarkdownMutationLock(storagePath, async () => {
|
|
596
|
+
for (let attempt = 0; ; attempt++) {
|
|
597
|
+
try {
|
|
598
|
+
const result = await mutation();
|
|
599
|
+
if (result.success && this.mutationObserver) {
|
|
600
|
+
const filePath = storagePath;
|
|
601
|
+
const state = await this.readFileState(filePath);
|
|
602
|
+
this.setEntries(target, [...new Set(state.entries)]);
|
|
603
|
+
this.fileFingerprints[filePath] = state.fingerprint;
|
|
604
|
+
const warning = await this.mutationObserver(target, [...state.entries]);
|
|
605
|
+
if (warning) {
|
|
606
|
+
const warnings = [...(result.warnings ?? []), warning];
|
|
607
|
+
return {
|
|
608
|
+
...result,
|
|
609
|
+
message: result.message ? `${result.message} Warning: ${warning}` : warning,
|
|
610
|
+
warning,
|
|
611
|
+
warnings,
|
|
612
|
+
};
|
|
613
|
+
}
|
|
614
|
+
}
|
|
615
|
+
return result;
|
|
616
|
+
} catch (error) {
|
|
617
|
+
const filePath = storagePath;
|
|
618
|
+
delete this.fileFingerprints[filePath];
|
|
619
|
+
const state = await this.readFileState(filePath);
|
|
620
|
+
this.setEntries(target, [...new Set(state.entries)]);
|
|
621
|
+
this.fileFingerprints[filePath] = state.fingerprint;
|
|
622
|
+
if (!(error instanceof ExternalMemoryWriteConflict)) throw error;
|
|
623
|
+
if (attempt >= MAX_EXTERNAL_WRITE_RETRIES) {
|
|
624
|
+
return {
|
|
625
|
+
success: false,
|
|
626
|
+
error: "Memory file changed repeatedly during this update. No external changes were overwritten.",
|
|
627
|
+
};
|
|
628
|
+
}
|
|
629
|
+
}
|
|
630
|
+
}
|
|
631
|
+
});
|
|
632
|
+
}
|
|
633
|
+
|
|
483
634
|
/**
|
|
484
635
|
* Atomic write: temp file + fs.rename().
|
|
485
636
|
* Creates temp files in the same directory as the target to avoid
|
|
@@ -487,17 +638,87 @@ export class MemoryStore {
|
|
|
487
638
|
* drive than the memory directory (common on Windows).
|
|
488
639
|
*/
|
|
489
640
|
private async saveToDisk(target: "memory" | "user" | "failure"): Promise<void> {
|
|
490
|
-
const filePath = this.
|
|
641
|
+
const filePath = await this.resolveStoragePath(target);
|
|
491
642
|
const entries = this.entriesFor(target);
|
|
492
643
|
const content = entries.length ? entries.join(ENTRY_DELIMITER) : "";
|
|
644
|
+
const expectedFingerprint = this.fileFingerprints[filePath] ?? "missing";
|
|
493
645
|
|
|
494
646
|
// Use the memory directory for temp files so rename stays on the same device
|
|
495
|
-
const tmpDir = await fs.mkdtemp(path.join(
|
|
647
|
+
const tmpDir = await fs.mkdtemp(path.join(path.dirname(filePath), ".tmp-"));
|
|
496
648
|
const tmpPath = path.join(tmpDir, "write.tmp");
|
|
497
649
|
|
|
498
650
|
try {
|
|
499
651
|
await fs.writeFile(tmpPath, content, "utf-8");
|
|
500
|
-
await
|
|
652
|
+
await this.pruneRecoveryFiles(filePath);
|
|
653
|
+
const currentState = await this.readFileState(filePath);
|
|
654
|
+
if (currentState.fingerprint !== expectedFingerprint) {
|
|
655
|
+
throw new ExternalMemoryWriteConflict();
|
|
656
|
+
}
|
|
657
|
+
|
|
658
|
+
if (expectedFingerprint === "missing") {
|
|
659
|
+
try {
|
|
660
|
+
await fs.link(tmpPath, filePath);
|
|
661
|
+
} catch (error) {
|
|
662
|
+
if ((error as NodeJS.ErrnoException).code === "EEXIST") {
|
|
663
|
+
throw new ExternalMemoryWriteConflict();
|
|
664
|
+
}
|
|
665
|
+
throw error;
|
|
666
|
+
}
|
|
667
|
+
} else {
|
|
668
|
+
const recoveryPath = this.recoveryPathFor(filePath);
|
|
669
|
+
const publishedIdentity = await this.fileIdentity(tmpPath);
|
|
670
|
+
try {
|
|
671
|
+
await fs.rename(filePath, recoveryPath);
|
|
672
|
+
} catch (error) {
|
|
673
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
|
|
674
|
+
throw new ExternalMemoryWriteConflict();
|
|
675
|
+
}
|
|
676
|
+
throw error;
|
|
677
|
+
}
|
|
678
|
+
let published = false;
|
|
679
|
+
try {
|
|
680
|
+
const displacedState = await this.readFileState(recoveryPath);
|
|
681
|
+
if (displacedState.fingerprint !== expectedFingerprint) {
|
|
682
|
+
throw new ExternalMemoryWriteConflict();
|
|
683
|
+
}
|
|
684
|
+
|
|
685
|
+
await fs.link(tmpPath, filePath);
|
|
686
|
+
published = true;
|
|
687
|
+
|
|
688
|
+
const verifiedDisplacedState = await this.readFileState(recoveryPath);
|
|
689
|
+
if (verifiedDisplacedState.fingerprint !== expectedFingerprint) {
|
|
690
|
+
throw new ExternalMemoryWriteConflict();
|
|
691
|
+
}
|
|
692
|
+
} catch (error) {
|
|
693
|
+
let rollbackError: unknown;
|
|
694
|
+
if (published) {
|
|
695
|
+
try {
|
|
696
|
+
await this.preserveConflictFile(tmpPath, filePath, "local");
|
|
697
|
+
} catch {
|
|
698
|
+
}
|
|
699
|
+
try {
|
|
700
|
+
await this.rollbackPublishedFile(recoveryPath, filePath, publishedIdentity);
|
|
701
|
+
} catch (restorePublishedError) {
|
|
702
|
+
rollbackError = restorePublishedError;
|
|
703
|
+
}
|
|
704
|
+
} else {
|
|
705
|
+
try {
|
|
706
|
+
await this.restoreDisplacedFile(recoveryPath, filePath);
|
|
707
|
+
} catch (restoreError) {
|
|
708
|
+
rollbackError = restoreError;
|
|
709
|
+
}
|
|
710
|
+
}
|
|
711
|
+
if (rollbackError) throw rollbackError;
|
|
712
|
+
if ((error as NodeJS.ErrnoException).code === "EEXIST"
|
|
713
|
+
|| error instanceof ExternalMemoryWriteConflict) {
|
|
714
|
+
throw new ExternalMemoryWriteConflict();
|
|
715
|
+
}
|
|
716
|
+
throw error;
|
|
717
|
+
}
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
try { await this.unlinkPublishedTempLink(tmpPath); } catch { /* ignore */ }
|
|
721
|
+
this.fileFingerprints[filePath] = this.fingerprint(content);
|
|
501
722
|
} catch (err) {
|
|
502
723
|
try { await fs.unlink(tmpPath); } catch { /* ignore */ }
|
|
503
724
|
throw err;
|
|
@@ -505,4 +726,181 @@ export class MemoryStore {
|
|
|
505
726
|
try { await fs.rm(tmpDir, { recursive: true, force: true }); } catch { /* ignore */ }
|
|
506
727
|
}
|
|
507
728
|
}
|
|
729
|
+
|
|
730
|
+
private async restoreDisplacedFile(displacedPath: string, filePath: string): Promise<void> {
|
|
731
|
+
try {
|
|
732
|
+
await fs.link(displacedPath, filePath);
|
|
733
|
+
} catch (error) {
|
|
734
|
+
if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
|
|
735
|
+
}
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
private async fileIdentity(filePath: string): Promise<{ dev: number; ino: number }> {
|
|
739
|
+
const state = await fs.lstat(filePath);
|
|
740
|
+
return { dev: state.dev, ino: state.ino };
|
|
741
|
+
}
|
|
742
|
+
|
|
743
|
+
private sameFileIdentity(
|
|
744
|
+
left: { dev: number; ino: number },
|
|
745
|
+
right: { dev: number; ino: number },
|
|
746
|
+
): boolean {
|
|
747
|
+
return left.dev === right.dev && left.ino === right.ino;
|
|
748
|
+
}
|
|
749
|
+
|
|
750
|
+
private async rollbackPublishedFile(
|
|
751
|
+
displacedPath: string,
|
|
752
|
+
filePath: string,
|
|
753
|
+
publishedIdentity: { dev: number; ino: number },
|
|
754
|
+
): Promise<void> {
|
|
755
|
+
const conflictPath = path.join(
|
|
756
|
+
path.dirname(filePath),
|
|
757
|
+
`.${path.basename(filePath)}.conflict-local-${Date.now()}-${randomUUID()}`,
|
|
758
|
+
);
|
|
759
|
+
try {
|
|
760
|
+
await fs.rename(filePath, conflictPath);
|
|
761
|
+
} catch (error) {
|
|
762
|
+
if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
|
|
763
|
+
await this.restoreDisplacedFile(displacedPath, filePath);
|
|
764
|
+
return;
|
|
765
|
+
}
|
|
766
|
+
|
|
767
|
+
const movedIdentity = await this.fileIdentity(conflictPath);
|
|
768
|
+
if (this.sameFileIdentity(movedIdentity, publishedIdentity)) {
|
|
769
|
+
await this.restoreDisplacedFile(displacedPath, filePath);
|
|
770
|
+
return;
|
|
771
|
+
}
|
|
772
|
+
|
|
773
|
+
try {
|
|
774
|
+
await fs.link(conflictPath, filePath);
|
|
775
|
+
} catch (error) {
|
|
776
|
+
if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
|
|
777
|
+
}
|
|
778
|
+
}
|
|
779
|
+
|
|
780
|
+
private recoveryPathFor(filePath: string): string {
|
|
781
|
+
return path.join(
|
|
782
|
+
path.dirname(filePath),
|
|
783
|
+
`.${path.basename(filePath)}.recovery-${Date.now()}-${randomUUID()}`,
|
|
784
|
+
);
|
|
785
|
+
}
|
|
786
|
+
|
|
787
|
+
private retiredRecoveryPathFor(filePath: string): string {
|
|
788
|
+
return path.join(
|
|
789
|
+
path.dirname(filePath),
|
|
790
|
+
`.${path.basename(filePath)}.retired-${Date.now()}-${randomUUID()}`,
|
|
791
|
+
);
|
|
792
|
+
}
|
|
793
|
+
|
|
794
|
+
private async unlinkPublishedTempLink(tmpPath: string): Promise<void> {
|
|
795
|
+
await fs.unlink(tmpPath);
|
|
796
|
+
}
|
|
797
|
+
|
|
798
|
+
private async pruneRecoveryFiles(filePath: string): Promise<void> {
|
|
799
|
+
const directory = path.dirname(filePath);
|
|
800
|
+
const escapedName = path.basename(filePath).replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
801
|
+
const uuidPattern = "[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}";
|
|
802
|
+
const recoveryPattern = new RegExp(`^\\.${escapedName}\\.recovery-\\d+-${uuidPattern}$`, "i");
|
|
803
|
+
const retiredPattern = new RegExp(`^\\.${escapedName}\\.retired-\\d+-${uuidPattern}$`, "i");
|
|
804
|
+
const conflictPattern = new RegExp(
|
|
805
|
+
`^\\.${escapedName}\\.conflict-local-\\d+-[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`,
|
|
806
|
+
"i",
|
|
807
|
+
);
|
|
808
|
+
const activeCutoff = Date.now() - RECOVERY_ACTIVE_GRACE_MS;
|
|
809
|
+
try {
|
|
810
|
+
const names = await fs.readdir(directory);
|
|
811
|
+
await Promise.all(names.filter((name) => recoveryPattern.test(name)).map(async (name) => {
|
|
812
|
+
const recoveryPath = path.join(directory, name);
|
|
813
|
+
try {
|
|
814
|
+
const state = await fs.lstat(recoveryPath);
|
|
815
|
+
if (!state.isFile()) return;
|
|
816
|
+
if (state.mtimeMs >= activeCutoff) return;
|
|
817
|
+
await this.retireRecoveryFile(recoveryPath, filePath);
|
|
818
|
+
} catch {
|
|
819
|
+
}
|
|
820
|
+
}));
|
|
821
|
+
|
|
822
|
+
const retiredNames = (await fs.readdir(directory)).filter((name) => retiredPattern.test(name));
|
|
823
|
+
const retired = await Promise.all(retiredNames.map(async (name) => {
|
|
824
|
+
const retiredPath = path.join(directory, name);
|
|
825
|
+
try {
|
|
826
|
+
const state = await fs.lstat(retiredPath);
|
|
827
|
+
return state.isFile() ? { path: retiredPath, state } : null;
|
|
828
|
+
} catch {
|
|
829
|
+
return null;
|
|
830
|
+
}
|
|
831
|
+
}));
|
|
832
|
+
const maxAgeCutoff = Date.now() - RETIRED_RECOVERY_MAX_AGE_MS;
|
|
833
|
+
const candidates = retired
|
|
834
|
+
.filter((item): item is NonNullable<typeof item> => item !== null)
|
|
835
|
+
.sort((left, right) => right.state.mtimeMs - left.state.mtimeMs);
|
|
836
|
+
let retainedCount = 0;
|
|
837
|
+
let retainedBytes = 0;
|
|
838
|
+
for (const item of candidates) {
|
|
839
|
+
const withinAge = item.state.mtimeMs >= maxAgeCutoff;
|
|
840
|
+
const withinCount = retainedCount < RETIRED_RECOVERY_MAX_COUNT;
|
|
841
|
+
const withinBytes = retainedBytes + item.state.size <= RETIRED_RECOVERY_MAX_BYTES;
|
|
842
|
+
if (withinAge && withinCount && withinBytes) {
|
|
843
|
+
retainedCount++;
|
|
844
|
+
retainedBytes += item.state.size;
|
|
845
|
+
continue;
|
|
846
|
+
}
|
|
847
|
+
try { await fs.unlink(item.path); } catch {}
|
|
848
|
+
}
|
|
849
|
+
|
|
850
|
+
const conflictNames = (await fs.readdir(directory)).filter((name) => conflictPattern.test(name));
|
|
851
|
+
const conflicts = await Promise.all(conflictNames.map(async (name) => {
|
|
852
|
+
const conflictPath = path.join(directory, name);
|
|
853
|
+
try {
|
|
854
|
+
const state = await fs.lstat(conflictPath);
|
|
855
|
+
return state.isFile() ? { path: conflictPath, state } : null;
|
|
856
|
+
} catch {
|
|
857
|
+
return null;
|
|
858
|
+
}
|
|
859
|
+
}));
|
|
860
|
+
const graceCutoff = Date.now() - CONFLICT_ACTIVE_GRACE_MS;
|
|
861
|
+
const conflictMaxAgeCutoff = Date.now() - CONFLICT_MAX_AGE_MS;
|
|
862
|
+
const conflictCandidates = conflicts
|
|
863
|
+
.filter((item): item is NonNullable<typeof item> => item !== null)
|
|
864
|
+
.sort((left, right) => right.state.mtimeMs - left.state.mtimeMs);
|
|
865
|
+
let conflictCount = 0;
|
|
866
|
+
let conflictBytes = 0;
|
|
867
|
+
for (const item of conflictCandidates) {
|
|
868
|
+
const withinCount = conflictCount < CONFLICT_MAX_COUNT;
|
|
869
|
+
const withinBytes = conflictBytes + item.state.size <= CONFLICT_MAX_BYTES;
|
|
870
|
+
const withinGrace = item.state.mtimeMs >= graceCutoff;
|
|
871
|
+
const withinAge = item.state.mtimeMs >= conflictMaxAgeCutoff;
|
|
872
|
+
if ((withinGrace || withinAge) && withinCount && withinBytes) {
|
|
873
|
+
conflictCount++;
|
|
874
|
+
conflictBytes += item.state.size;
|
|
875
|
+
continue;
|
|
876
|
+
}
|
|
877
|
+
try { await fs.unlink(item.path); } catch {}
|
|
878
|
+
}
|
|
879
|
+
} catch {
|
|
880
|
+
}
|
|
881
|
+
}
|
|
882
|
+
|
|
883
|
+
private async retireRecoveryFile(recoveryPath: string, filePath: string): Promise<void> {
|
|
884
|
+
const retiredPath = this.retiredRecoveryPathFor(filePath);
|
|
885
|
+
const snapshotPath = `${retiredPath}.tmp`;
|
|
886
|
+
const snapshot = await fs.readFile(recoveryPath);
|
|
887
|
+
const handle = await fs.open(snapshotPath, "wx", 0o600);
|
|
888
|
+
try {
|
|
889
|
+
await handle.writeFile(snapshot);
|
|
890
|
+
await handle.sync();
|
|
891
|
+
} finally {
|
|
892
|
+
await handle.close();
|
|
893
|
+
}
|
|
894
|
+
await fs.rename(snapshotPath, retiredPath);
|
|
895
|
+
await fs.unlink(recoveryPath);
|
|
896
|
+
}
|
|
897
|
+
|
|
898
|
+
private async preserveConflictFile(sourcePath: string, filePath: string, kind: string): Promise<string> {
|
|
899
|
+
const conflictPath = path.join(
|
|
900
|
+
path.dirname(filePath),
|
|
901
|
+
`.${path.basename(filePath)}.conflict-${kind}-${Date.now()}-${randomUUID()}`,
|
|
902
|
+
);
|
|
903
|
+
await fs.copyFile(sourcePath, conflictPath);
|
|
904
|
+
return conflictPath;
|
|
905
|
+
}
|
|
508
906
|
}
|