pi-hermes-memory 0.7.23 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -5
- package/package.json +2 -2
- package/src/config.ts +6 -0
- package/src/constants.ts +53 -11
- package/src/extension-root-migration.ts +429 -5
- package/src/handlers/auto-consolidate.ts +118 -8
- package/src/handlers/background-review.ts +29 -23
- package/src/handlers/child-process-watchdog.mjs +90 -0
- package/src/handlers/correction-detector.ts +52 -36
- package/src/handlers/pi-child-process.ts +270 -37
- package/src/handlers/review-memory-ops.ts +18 -116
- package/src/handlers/session-flush.ts +71 -4
- package/src/handlers/sync-markdown-memories.ts +143 -51
- package/src/index.ts +32 -16
- package/src/store/atomic-lock-coordinator.ts +252 -0
- package/src/store/canonical-storage-path.ts +54 -0
- package/src/store/db.ts +244 -9
- package/src/store/markdown-mutation-lock.ts +38 -0
- package/src/store/memory-store.ts +455 -57
- package/src/store/sqlite-memory-store.ts +121 -4
- package/src/tools/memory-tool.ts +48 -9
- package/src/tools/session-search-tool.ts +70 -12
- package/src/types.ts +2 -0
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { randomUUID } from 'node:crypto';
|
|
4
|
+
import { createRequire } from 'node:module';
|
|
5
|
+
import { spawnSync } from 'node:child_process';
|
|
6
|
+
|
|
7
|
+
type StatementLike = {
|
|
8
|
+
run: (...args: unknown[]) => unknown;
|
|
9
|
+
get: (...args: unknown[]) => unknown;
|
|
10
|
+
all: (...args: unknown[]) => unknown[];
|
|
11
|
+
};
|
|
12
|
+
|
|
13
|
+
type DatabaseLike = {
|
|
14
|
+
prepare: (sql: string) => StatementLike;
|
|
15
|
+
exec: (sql: string) => void;
|
|
16
|
+
close: () => void;
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
type DatabaseCtor = new (dbPath: string) => DatabaseLike;
|
|
20
|
+
|
|
21
|
+
export interface AtomicLockOptions {
|
|
22
|
+
staleMs: number;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export interface AtomicLockLease {
|
|
26
|
+
token: string;
|
|
27
|
+
release: () => void;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export interface AtomicLockCoordinatorOptions {
|
|
31
|
+
pid?: number;
|
|
32
|
+
incarnation?: string;
|
|
33
|
+
probeIncarnation?: (pid: number) => string | null;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function loadDatabaseCtor(): DatabaseCtor {
|
|
37
|
+
const require = createRequire(import.meta.url);
|
|
38
|
+
if (typeof (globalThis as { Bun?: unknown }).Bun !== 'undefined') {
|
|
39
|
+
const bunSqlite = require('bun:sqlite') as { Database: DatabaseCtor };
|
|
40
|
+
return bunSqlite.Database;
|
|
41
|
+
}
|
|
42
|
+
const mod = require('better-sqlite3') as { default?: DatabaseCtor } | DatabaseCtor;
|
|
43
|
+
return (mod as { default?: DatabaseCtor }).default ?? (mod as DatabaseCtor);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
const Database = loadDatabaseCtor();
|
|
47
|
+
|
|
48
|
+
function processIsAlive(pid: number): boolean {
|
|
49
|
+
try {
|
|
50
|
+
process.kill(pid, 0);
|
|
51
|
+
return true;
|
|
52
|
+
} catch (error) {
|
|
53
|
+
return (error as NodeJS.ErrnoException).code !== 'ESRCH';
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function probeProcessIncarnation(pid: number): string | null {
|
|
58
|
+
if (!Number.isSafeInteger(pid) || pid <= 0) return null;
|
|
59
|
+
if (process.platform === 'linux') {
|
|
60
|
+
try {
|
|
61
|
+
const stat = fs.readFileSync(`/proc/${pid}/stat`, 'utf-8');
|
|
62
|
+
const end = stat.lastIndexOf(')');
|
|
63
|
+
const fields = stat.slice(end + 2).split(' ');
|
|
64
|
+
return fields[19] || null;
|
|
65
|
+
} catch {
|
|
66
|
+
return null;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
if (process.platform !== 'win32') {
|
|
71
|
+
const result = spawnSync('ps', ['-o', 'lstart=', '-p', String(pid)], {
|
|
72
|
+
encoding: 'utf-8',
|
|
73
|
+
timeout: 250,
|
|
74
|
+
});
|
|
75
|
+
return result.status === 0 ? result.stdout.trim() || null : null;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
const result = spawnSync(
|
|
79
|
+
'powershell.exe',
|
|
80
|
+
['-NoProfile', '-NonInteractive', '-Command', `(Get-Process -Id ${pid} -ErrorAction SilentlyContinue).StartTime.ToUniversalTime().Ticks`],
|
|
81
|
+
{ encoding: 'utf-8', timeout: 500 },
|
|
82
|
+
);
|
|
83
|
+
return result.status === 0 ? result.stdout.trim() || null : null;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
const currentProcessIncarnation = probeProcessIncarnation(process.pid);
|
|
87
|
+
const RELEASE_ATTEMPTS = 3;
|
|
88
|
+
const pendingReleases = new Map<string, () => void>();
|
|
89
|
+
|
|
90
|
+
export class AtomicLockCoordinator {
|
|
91
|
+
private readonly pid: number;
|
|
92
|
+
private readonly incarnation: string | null;
|
|
93
|
+
private readonly probeIncarnation: (pid: number) => string | null;
|
|
94
|
+
|
|
95
|
+
constructor(private readonly dbPath: string, options: AtomicLockCoordinatorOptions = {}) {
|
|
96
|
+
this.pid = options.pid ?? process.pid;
|
|
97
|
+
this.probeIncarnation = options.probeIncarnation
|
|
98
|
+
?? ((pid) => pid === process.pid ? currentProcessIncarnation : probeProcessIncarnation(pid));
|
|
99
|
+
this.incarnation = options.incarnation
|
|
100
|
+
?? this.probeIncarnation(this.pid)
|
|
101
|
+
?? null;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
tryAcquire(key: string, options: AtomicLockOptions): AtomicLockLease | null {
|
|
105
|
+
this.retryPendingReleases(key);
|
|
106
|
+
const token = randomUUID();
|
|
107
|
+
const now = Date.now();
|
|
108
|
+
const db = this.open();
|
|
109
|
+
let acquired = false;
|
|
110
|
+
|
|
111
|
+
try {
|
|
112
|
+
db.exec('BEGIN IMMEDIATE');
|
|
113
|
+
try {
|
|
114
|
+
const owner = db.prepare(`
|
|
115
|
+
SELECT token, pid, incarnation, acquired_at
|
|
116
|
+
FROM locks
|
|
117
|
+
WHERE lock_key = ?
|
|
118
|
+
`).get(key) as { token: string; pid: number; incarnation: string | null; acquired_at: number } | undefined;
|
|
119
|
+
|
|
120
|
+
if (!owner) {
|
|
121
|
+
db.prepare(`
|
|
122
|
+
INSERT INTO locks (lock_key, token, pid, incarnation, acquired_at)
|
|
123
|
+
VALUES (?, ?, ?, ?, ?)
|
|
124
|
+
`).run(key, token, this.pid, this.incarnation, now);
|
|
125
|
+
acquired = true;
|
|
126
|
+
} else {
|
|
127
|
+
const observedIncarnation = this.probeIncarnation(owner.pid);
|
|
128
|
+
const alive = observedIncarnation !== null || processIsAlive(owner.pid);
|
|
129
|
+
const sameIncarnation = alive
|
|
130
|
+
&& owner.incarnation !== null
|
|
131
|
+
&& observedIncarnation !== null
|
|
132
|
+
&& owner.incarnation === observedIncarnation;
|
|
133
|
+
const unknownIncarnation = alive && (owner.incarnation === null || observedIncarnation === null);
|
|
134
|
+
// A lease is reclaimable once it has been held for longer than staleMs,
|
|
135
|
+
// regardless of whether the owning process is still alive — it may be
|
|
136
|
+
// making no progress (blocked I/O, wedged, suspended) rather than dead.
|
|
137
|
+
// This is the sole backstop for that case: liveness/incarnation checks
|
|
138
|
+
// alone cannot distinguish "alive and working" from "alive and stuck".
|
|
139
|
+
// staleMs <= 0 disables time-based takeover (liveness checks only).
|
|
140
|
+
const stale = options.staleMs > 0 && now - owner.acquired_at >= options.staleMs;
|
|
141
|
+
if (stale || (!sameIncarnation && !unknownIncarnation)) {
|
|
142
|
+
db.prepare(`
|
|
143
|
+
UPDATE locks
|
|
144
|
+
SET token = ?, pid = ?, incarnation = ?, acquired_at = ?
|
|
145
|
+
WHERE lock_key = ? AND token = ?
|
|
146
|
+
`).run(token, this.pid, this.incarnation, now, key, owner.token);
|
|
147
|
+
acquired = true;
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
db.exec('COMMIT');
|
|
152
|
+
} catch (error) {
|
|
153
|
+
try { db.exec('ROLLBACK'); } catch {}
|
|
154
|
+
throw error;
|
|
155
|
+
}
|
|
156
|
+
} finally {
|
|
157
|
+
db.close();
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
if (!acquired) return null;
|
|
161
|
+
return {
|
|
162
|
+
token,
|
|
163
|
+
release: () => this.release(key, token),
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* Fencing check for destructive operations that lack their own independent
|
|
169
|
+
* compare-and-swap (e.g. a plain fs.renameSync with no content/inode
|
|
170
|
+
* verification). A lease can be legitimately stolen from a stale-but-alive
|
|
171
|
+
* holder (see tryAcquire); a holder resuming after being stuck must verify
|
|
172
|
+
* it is still the current owner immediately before publishing, or abort.
|
|
173
|
+
* This narrows — it cannot fully close — the check-then-act race, since
|
|
174
|
+
* synchronous work between this call and the actual write is not atomic
|
|
175
|
+
* with it.
|
|
176
|
+
*/
|
|
177
|
+
isCurrentOwner(key: string, token: string): boolean {
|
|
178
|
+
const db = this.open();
|
|
179
|
+
try {
|
|
180
|
+
const row = db.prepare('SELECT token FROM locks WHERE lock_key = ?').get(key) as { token: string } | undefined;
|
|
181
|
+
return row?.token === token;
|
|
182
|
+
} finally {
|
|
183
|
+
db.close();
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
release(key: string, token: string): void {
|
|
188
|
+
const pendingKey = this.pendingReleaseKey(key, token);
|
|
189
|
+
for (let attempt = 0; attempt < RELEASE_ATTEMPTS; attempt++) {
|
|
190
|
+
try {
|
|
191
|
+
this.deleteOwnedLock(key, token);
|
|
192
|
+
pendingReleases.delete(pendingKey);
|
|
193
|
+
return;
|
|
194
|
+
} catch {
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
pendingReleases.set(pendingKey, () => this.release(key, token));
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
private deleteOwnedLock(key: string, token: string): void {
|
|
201
|
+
const db = this.open();
|
|
202
|
+
try {
|
|
203
|
+
db.prepare('DELETE FROM locks WHERE lock_key = ? AND token = ?').run(key, token);
|
|
204
|
+
} finally {
|
|
205
|
+
db.close();
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
private retryPendingReleases(key: string): void {
|
|
210
|
+
const prefix = `${path.resolve(this.dbPath)}\0${key}\0`;
|
|
211
|
+
for (const [pendingKey, release] of [...pendingReleases.entries()]) {
|
|
212
|
+
if (pendingKey.startsWith(prefix)) release();
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
private pendingReleaseKey(key: string, token: string): string {
|
|
217
|
+
return `${path.resolve(this.dbPath)}\0${key}\0${token}`;
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
private open(): DatabaseLike {
|
|
221
|
+
fs.mkdirSync(path.dirname(this.dbPath), { recursive: true });
|
|
222
|
+
const existed = fs.existsSync(this.dbPath);
|
|
223
|
+
const db = new Database(this.dbPath);
|
|
224
|
+
try {
|
|
225
|
+
db.exec(`
|
|
226
|
+
PRAGMA busy_timeout = 5000;
|
|
227
|
+
PRAGMA journal_mode = WAL;
|
|
228
|
+
CREATE TABLE IF NOT EXISTS locks (
|
|
229
|
+
lock_key TEXT PRIMARY KEY,
|
|
230
|
+
token TEXT NOT NULL,
|
|
231
|
+
pid INTEGER NOT NULL,
|
|
232
|
+
incarnation TEXT,
|
|
233
|
+
acquired_at INTEGER NOT NULL
|
|
234
|
+
);
|
|
235
|
+
`);
|
|
236
|
+
const columns = db.prepare('PRAGMA table_info(locks)').all() as Array<{ name: string }>;
|
|
237
|
+
if (!columns.some(({ name }) => name === 'incarnation')) {
|
|
238
|
+
try {
|
|
239
|
+
db.exec('ALTER TABLE locks ADD COLUMN incarnation TEXT');
|
|
240
|
+
} catch (error) {
|
|
241
|
+
const refreshed = db.prepare('PRAGMA table_info(locks)').all() as Array<{ name: string }>;
|
|
242
|
+
if (!refreshed.some(({ name }) => name === 'incarnation')) throw error;
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
if (!existed) fs.chmodSync(this.dbPath, 0o600);
|
|
246
|
+
return db;
|
|
247
|
+
} catch (error) {
|
|
248
|
+
db.close();
|
|
249
|
+
throw error;
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
|
|
4
|
+
const MAX_SYMLINK_DEPTH = 40;
|
|
5
|
+
|
|
6
|
+
function pathParts(absolutePath: string): { root: string; parts: string[] } {
|
|
7
|
+
const root = path.parse(absolutePath).root;
|
|
8
|
+
return {
|
|
9
|
+
root,
|
|
10
|
+
parts: absolutePath.slice(root.length).split(path.sep).filter(Boolean),
|
|
11
|
+
};
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export function canonicalStoragePathSync(filePath: string): string {
|
|
15
|
+
const input = path.resolve(filePath);
|
|
16
|
+
let { root, parts } = pathParts(input);
|
|
17
|
+
let current = root;
|
|
18
|
+
let depth = 0;
|
|
19
|
+
|
|
20
|
+
while (parts.length > 0) {
|
|
21
|
+
const part = parts.shift()!;
|
|
22
|
+
const candidate = path.join(current, part);
|
|
23
|
+
let state: fs.Stats;
|
|
24
|
+
try {
|
|
25
|
+
state = fs.lstatSync(candidate);
|
|
26
|
+
} catch (error) {
|
|
27
|
+
if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
|
|
28
|
+
return path.join(fs.realpathSync.native(current), part, ...parts);
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
if (!state.isSymbolicLink()) {
|
|
32
|
+
current = candidate;
|
|
33
|
+
continue;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
if (depth++ >= MAX_SYMLINK_DEPTH) {
|
|
37
|
+
const error = new Error(`Symbolic link loop detected while resolving ${input}`) as NodeJS.ErrnoException;
|
|
38
|
+
error.code = "ELOOP";
|
|
39
|
+
throw error;
|
|
40
|
+
}
|
|
41
|
+
const target = fs.readlinkSync(candidate);
|
|
42
|
+
const targetPath = path.resolve(path.dirname(candidate), target);
|
|
43
|
+
const targetParts = pathParts(targetPath);
|
|
44
|
+
root = targetParts.root;
|
|
45
|
+
current = root;
|
|
46
|
+
parts = [...targetParts.parts, ...parts];
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
return fs.realpathSync.native(current);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export async function canonicalStoragePath(filePath: string): Promise<string> {
|
|
53
|
+
return canonicalStoragePathSync(filePath);
|
|
54
|
+
}
|
package/src/store/db.ts
CHANGED
|
@@ -2,6 +2,8 @@ import path from 'node:path';
|
|
|
2
2
|
import fs from 'node:fs';
|
|
3
3
|
import { createRequire } from 'node:module';
|
|
4
4
|
import { SCHEMA_SQL } from './schema.js';
|
|
5
|
+
import { AtomicLockCoordinator } from './atomic-lock-coordinator.js';
|
|
6
|
+
import { canonicalStoragePathSync } from './canonical-storage-path.js';
|
|
5
7
|
|
|
6
8
|
type StatementLike = {
|
|
7
9
|
run: (...args: any[]) => any;
|
|
@@ -34,12 +36,30 @@ type MovedDatabaseFile = {
|
|
|
34
36
|
};
|
|
35
37
|
|
|
36
38
|
export interface DatabaseRecoveryResult {
|
|
37
|
-
strategy: 'rebuilt' | 'recreated-empty';
|
|
39
|
+
strategy: 'rebuilt' | 'recreated-empty' | 'reused';
|
|
38
40
|
backupPaths: string[];
|
|
39
41
|
recoveredRows?: Record<string, number>;
|
|
40
42
|
error?: string;
|
|
41
43
|
}
|
|
42
44
|
|
|
45
|
+
export interface DatabaseRecoveryOptions {
|
|
46
|
+
recoveryLockWaitMs?: number;
|
|
47
|
+
recoveryLockPollMs?: number;
|
|
48
|
+
recoveryLockStaleMs?: number;
|
|
49
|
+
recoveryCircuitLimit?: number;
|
|
50
|
+
recoveryCircuitWindowMs?: number;
|
|
51
|
+
recoveryBackupRetention?: number;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
interface ResolvedDatabaseRecoveryOptions {
|
|
55
|
+
recoveryLockWaitMs: number;
|
|
56
|
+
recoveryLockPollMs: number;
|
|
57
|
+
recoveryLockStaleMs: number;
|
|
58
|
+
recoveryCircuitLimit: number;
|
|
59
|
+
recoveryCircuitWindowMs: number;
|
|
60
|
+
recoveryBackupRetention: number;
|
|
61
|
+
}
|
|
62
|
+
|
|
43
63
|
class DatabaseCorruptionError extends Error {
|
|
44
64
|
code = 'SQLITE_CORRUPT';
|
|
45
65
|
|
|
@@ -54,6 +74,14 @@ export const SQLITE_WAL_AUTOCHECKPOINT_PAGES = 1000;
|
|
|
54
74
|
const DATABASE_FILE_SUFFIXES: readonly DatabaseFileSuffix[] = ['', '-wal', '-shm'];
|
|
55
75
|
const MEMORY_TARGETS = new Set(['memory', 'user', 'failure']);
|
|
56
76
|
const MEMORY_CATEGORIES = new Set(['failure', 'correction', 'insight', 'preference', 'convention', 'tool-quirk']);
|
|
77
|
+
const DEFAULT_RECOVERY_OPTIONS: ResolvedDatabaseRecoveryOptions = {
|
|
78
|
+
recoveryLockWaitMs: 5000,
|
|
79
|
+
recoveryLockPollMs: 50,
|
|
80
|
+
recoveryLockStaleMs: 300000,
|
|
81
|
+
recoveryCircuitLimit: 3,
|
|
82
|
+
recoveryCircuitWindowMs: 300000,
|
|
83
|
+
recoveryBackupRetention: 3,
|
|
84
|
+
};
|
|
57
85
|
|
|
58
86
|
function quoteIdentifier(identifier: string): string {
|
|
59
87
|
return `"${identifier.replace(/"/g, '""')}"`;
|
|
@@ -110,11 +138,27 @@ const Database = loadDatabaseCtor();
|
|
|
110
138
|
|
|
111
139
|
export class DatabaseManager {
|
|
112
140
|
private db: DatabaseLike | null = null;
|
|
113
|
-
private readonly
|
|
141
|
+
private readonly displayDbPath: string;
|
|
142
|
+
private canonicalDbPath: string | null = null;
|
|
143
|
+
private readonly recoveryOptions: ResolvedDatabaseRecoveryOptions;
|
|
114
144
|
private lastRecovery: DatabaseRecoveryResult | null = null;
|
|
145
|
+
private openGuard: (() => void) | null = null;
|
|
146
|
+
private activeRecoveryLease: { coordinator: AtomicLockCoordinator; key: string; token: string } | null = null;
|
|
147
|
+
|
|
148
|
+
constructor(memoryDir: string, recoveryOptions: DatabaseRecoveryOptions = {}) {
|
|
149
|
+
this.displayDbPath = path.join(memoryDir, 'sessions.db');
|
|
150
|
+
this.recoveryOptions = { ...DEFAULT_RECOVERY_OPTIONS, ...recoveryOptions };
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
private get dbPath(): string {
|
|
154
|
+
if (!this.canonicalDbPath) {
|
|
155
|
+
this.canonicalDbPath = canonicalStoragePathSync(this.displayDbPath);
|
|
156
|
+
}
|
|
157
|
+
return this.canonicalDbPath;
|
|
158
|
+
}
|
|
115
159
|
|
|
116
|
-
|
|
117
|
-
this.
|
|
160
|
+
setOpenGuard(guard: (() => void) | null): void {
|
|
161
|
+
this.openGuard = guard;
|
|
118
162
|
}
|
|
119
163
|
|
|
120
164
|
/**
|
|
@@ -147,6 +191,7 @@ export class DatabaseManager {
|
|
|
147
191
|
*/
|
|
148
192
|
getDb(): DatabaseLike {
|
|
149
193
|
if (!this.db) {
|
|
194
|
+
this.openGuard?.();
|
|
150
195
|
this.db = this.open();
|
|
151
196
|
}
|
|
152
197
|
return this.db;
|
|
@@ -181,7 +226,15 @@ export class DatabaseManager {
|
|
|
181
226
|
*/
|
|
182
227
|
recoverFromCorruption(cause?: unknown): DatabaseRecoveryResult {
|
|
183
228
|
this.close();
|
|
184
|
-
|
|
229
|
+
let verifiedDb: DatabaseLike | null = null;
|
|
230
|
+
let recovery: DatabaseRecoveryResult;
|
|
231
|
+
try {
|
|
232
|
+
recovery = this.recoverDatabaseFile(cause, () => {
|
|
233
|
+
verifiedDb = this.openUnchecked();
|
|
234
|
+
});
|
|
235
|
+
} finally {
|
|
236
|
+
if (verifiedDb) this.safeClose(verifiedDb);
|
|
237
|
+
}
|
|
185
238
|
this.lastRecovery = recovery;
|
|
186
239
|
return recovery;
|
|
187
240
|
}
|
|
@@ -202,9 +255,19 @@ export class DatabaseManager {
|
|
|
202
255
|
throw err;
|
|
203
256
|
}
|
|
204
257
|
|
|
205
|
-
|
|
258
|
+
let recoveredDb: DatabaseLike | null = null;
|
|
259
|
+
let recovery: DatabaseRecoveryResult;
|
|
260
|
+
try {
|
|
261
|
+
recovery = this.recoverDatabaseFile(err, () => {
|
|
262
|
+
recoveredDb = this.openUnchecked();
|
|
263
|
+
});
|
|
264
|
+
} catch (error) {
|
|
265
|
+
if (recoveredDb) this.safeClose(recoveredDb);
|
|
266
|
+
throw error;
|
|
267
|
+
}
|
|
206
268
|
this.lastRecovery = recovery;
|
|
207
|
-
|
|
269
|
+
if (!recoveredDb) throw new Error(`SQLite recovery verification did not open ${this.displayDbPath}`);
|
|
270
|
+
return recoveredDb;
|
|
208
271
|
}
|
|
209
272
|
}
|
|
210
273
|
|
|
@@ -297,7 +360,57 @@ export class DatabaseManager {
|
|
|
297
360
|
}
|
|
298
361
|
}
|
|
299
362
|
|
|
300
|
-
private recoverDatabaseFile(cause
|
|
363
|
+
private recoverDatabaseFile(cause: unknown, verify: () => void): DatabaseRecoveryResult {
|
|
364
|
+
const coordinator = new AtomicLockCoordinator(path.join(path.dirname(this.dbPath), '.pi-hermes-locks.sqlite'));
|
|
365
|
+
const lockKey = `recovery:${this.dbPath}`;
|
|
366
|
+
const deadline = Date.now() + Math.max(0, this.recoveryOptions.recoveryLockWaitMs);
|
|
367
|
+
|
|
368
|
+
while (true) {
|
|
369
|
+
const lease = coordinator.tryAcquire(lockKey, { staleMs: this.recoveryOptions.recoveryLockStaleMs });
|
|
370
|
+
if (!lease) {
|
|
371
|
+
if (Date.now() >= deadline) {
|
|
372
|
+
throw new Error(`SQLite recovery already in progress for ${this.displayDbPath}; timed out after ${this.recoveryOptions.recoveryLockWaitMs}ms`);
|
|
373
|
+
}
|
|
374
|
+
DatabaseManager.sleepSync(Math.min(
|
|
375
|
+
this.recoveryOptions.recoveryLockPollMs,
|
|
376
|
+
Math.max(1, deadline - Date.now()),
|
|
377
|
+
));
|
|
378
|
+
continue;
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
this.activeRecoveryLease = { coordinator, key: lockKey, token: lease.token };
|
|
382
|
+
try {
|
|
383
|
+
if (this.currentDatabaseIsHealthy()) {
|
|
384
|
+
try {
|
|
385
|
+
verify();
|
|
386
|
+
this.clearRecoveryFailuresBestEffort();
|
|
387
|
+
return { strategy: 'reused', backupPaths: [] };
|
|
388
|
+
} catch (error) {
|
|
389
|
+
this.recordRecoveryFailure();
|
|
390
|
+
throw error;
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
this.assertRecoveryCircuitClosed();
|
|
395
|
+
try {
|
|
396
|
+
this.cleanupRecoveryArtifactsBestEffort();
|
|
397
|
+
const result = this.recoverDatabaseFileUnlocked(cause);
|
|
398
|
+
verify();
|
|
399
|
+
this.cleanupRecoveryArtifactsBestEffort();
|
|
400
|
+
this.clearRecoveryFailuresBestEffort();
|
|
401
|
+
return result;
|
|
402
|
+
} catch (error) {
|
|
403
|
+
this.recordRecoveryFailure();
|
|
404
|
+
throw error;
|
|
405
|
+
}
|
|
406
|
+
} finally {
|
|
407
|
+
this.activeRecoveryLease = null;
|
|
408
|
+
lease.release();
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
private recoverDatabaseFileUnlocked(cause?: unknown): DatabaseRecoveryResult {
|
|
301
414
|
const backupBase = this.corruptBackupBase();
|
|
302
415
|
let rebuildError: unknown;
|
|
303
416
|
|
|
@@ -317,6 +430,112 @@ export class DatabaseManager {
|
|
|
317
430
|
};
|
|
318
431
|
}
|
|
319
432
|
|
|
433
|
+
private currentDatabaseIsHealthy(): boolean {
|
|
434
|
+
if (!this.hasExistingMainDatabaseFile()) return false;
|
|
435
|
+
let db: DatabaseLike | null = null;
|
|
436
|
+
try {
|
|
437
|
+
db = new Database(this.dbPath);
|
|
438
|
+
this.assertIntegrityOk(db, 'quick_check', 'while joining corruption recovery');
|
|
439
|
+
return true;
|
|
440
|
+
} catch {
|
|
441
|
+
return false;
|
|
442
|
+
} finally {
|
|
443
|
+
if (db) this.safeClose(db);
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
private recoveryCircuitPath(): string {
|
|
448
|
+
return `${this.dbPath}.recovery-state.json`;
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
private recentRecoveryFailures(): number[] {
|
|
452
|
+
try {
|
|
453
|
+
const parsed = JSON.parse(fs.readFileSync(this.recoveryCircuitPath(), 'utf-8')) as { failures?: unknown };
|
|
454
|
+
if (!Array.isArray(parsed.failures)) return [];
|
|
455
|
+
const cutoff = Date.now() - Math.max(0, this.recoveryOptions.recoveryCircuitWindowMs);
|
|
456
|
+
return parsed.failures.filter((value): value is number => typeof value === 'number' && value >= cutoff);
|
|
457
|
+
} catch {
|
|
458
|
+
return [];
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
private assertRecoveryCircuitClosed(): void {
|
|
463
|
+
if (this.recentRecoveryFailures().length >= Math.max(1, this.recoveryOptions.recoveryCircuitLimit)) {
|
|
464
|
+
throw new Error(
|
|
465
|
+
`SQLite recovery circuit is open for ${this.displayDbPath}: too many failed recovery attempts within ${this.recoveryOptions.recoveryCircuitWindowMs}ms`,
|
|
466
|
+
);
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
private recordRecoveryFailure(): void {
|
|
471
|
+
const statePath = this.recoveryCircuitPath();
|
|
472
|
+
const tempPath = `${statePath}.tmp-${process.pid}-${Math.random().toString(16).slice(2, 8)}`;
|
|
473
|
+
const failures = [...this.recentRecoveryFailures(), Date.now()];
|
|
474
|
+
try {
|
|
475
|
+
fs.writeFileSync(tempPath, JSON.stringify({ failures }), { encoding: 'utf-8', mode: 0o600 });
|
|
476
|
+
fs.renameSync(tempPath, statePath);
|
|
477
|
+
} finally {
|
|
478
|
+
fs.rmSync(tempPath, { force: true });
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
private clearRecoveryFailures(): void {
|
|
483
|
+
fs.rmSync(this.recoveryCircuitPath(), { force: true });
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
private clearRecoveryFailuresBestEffort(): void {
|
|
487
|
+
try { this.clearRecoveryFailures(); } catch {}
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
private cleanupRecoveryArtifactsBestEffort(): void {
|
|
491
|
+
try { this.cleanupRecoveryArtifacts(); } catch {}
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
private cleanupRecoveryArtifacts(): void {
|
|
495
|
+
const dir = path.dirname(this.dbPath);
|
|
496
|
+
const databaseName = path.basename(this.dbPath);
|
|
497
|
+
let names: string[];
|
|
498
|
+
try {
|
|
499
|
+
names = fs.readdirSync(dir);
|
|
500
|
+
} catch {
|
|
501
|
+
return;
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
for (const name of names) {
|
|
505
|
+
if (name.startsWith(`${databaseName}.rebuild-`)) {
|
|
506
|
+
fs.rmSync(path.join(dir, name), { recursive: true, force: true });
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
const backupGroups = new Map<string, number>();
|
|
511
|
+
for (const name of names) {
|
|
512
|
+
if (!name.startsWith(`${databaseName}.corrupt-`)) continue;
|
|
513
|
+
const group = name.replace(/-(?:wal|shm)$/, '');
|
|
514
|
+
try {
|
|
515
|
+
const mtimeMs = fs.statSync(path.join(dir, name)).mtimeMs;
|
|
516
|
+
backupGroups.set(group, Math.max(backupGroups.get(group) ?? 0, mtimeMs));
|
|
517
|
+
} catch {
|
|
518
|
+
// Artifact disappeared while scanning.
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
const retained = Math.max(0, this.recoveryOptions.recoveryBackupRetention);
|
|
523
|
+
const expired = [...backupGroups.entries()]
|
|
524
|
+
.sort((left, right) => right[1] - left[1])
|
|
525
|
+
.slice(retained);
|
|
526
|
+
for (const [group] of expired) {
|
|
527
|
+
for (const suffix of DATABASE_FILE_SUFFIXES) {
|
|
528
|
+
fs.rmSync(path.join(dir, `${group}${suffix}`), { force: true });
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
private static sleepSync(milliseconds: number): void {
|
|
534
|
+
if (milliseconds <= 0) return;
|
|
535
|
+
const signal = new Int32Array(new SharedArrayBuffer(4));
|
|
536
|
+
Atomics.wait(signal, 0, 0, milliseconds);
|
|
537
|
+
}
|
|
538
|
+
|
|
320
539
|
private rebuildDatabaseFromReadableRows(backupBase: string): DatabaseRecoveryResult {
|
|
321
540
|
const tempPath = this.rebuildTempPath();
|
|
322
541
|
this.removeDatabaseFileSet(tempPath);
|
|
@@ -555,6 +774,7 @@ export class DatabaseManager {
|
|
|
555
774
|
}
|
|
556
775
|
|
|
557
776
|
private moveDatabaseFilesToBackup(backupBase: string): MovedDatabaseFile[] {
|
|
777
|
+
this.assertStillRecoveryOwner();
|
|
558
778
|
const moved: MovedDatabaseFile[] = [];
|
|
559
779
|
for (const suffix of DATABASE_FILE_SUFFIXES) {
|
|
560
780
|
const original = `${this.dbPath}${suffix}`;
|
|
@@ -568,6 +788,21 @@ export class DatabaseManager {
|
|
|
568
788
|
return moved;
|
|
569
789
|
}
|
|
570
790
|
|
|
791
|
+
/**
|
|
792
|
+
* Verifies this instance still holds the recovery lease immediately before
|
|
793
|
+
* a destructive rename that has no independent compare-and-swap of its
|
|
794
|
+
* own (unlike the Markdown mutation path, which re-checks a content
|
|
795
|
+
* fingerprint at publish time). If the lease was reclaimed as stale while
|
|
796
|
+
* this call was in flight, abort rather than race the new owner.
|
|
797
|
+
*/
|
|
798
|
+
private assertStillRecoveryOwner(): void {
|
|
799
|
+
const active = this.activeRecoveryLease;
|
|
800
|
+
if (!active) return;
|
|
801
|
+
if (!active.coordinator.isCurrentOwner(active.key, active.token)) {
|
|
802
|
+
throw new Error(`SQLite recovery lease lost for ${this.displayDbPath}; another process took over`);
|
|
803
|
+
}
|
|
804
|
+
}
|
|
805
|
+
|
|
571
806
|
private restoreMovedDatabaseFiles(moved: MovedDatabaseFile[]): void {
|
|
572
807
|
for (const file of [...moved].reverse()) {
|
|
573
808
|
try {
|
|
@@ -765,7 +1000,7 @@ export class DatabaseManager {
|
|
|
765
1000
|
* Get the database file path.
|
|
766
1001
|
*/
|
|
767
1002
|
getPath(): string {
|
|
768
|
-
return this.
|
|
1003
|
+
return this.displayDbPath;
|
|
769
1004
|
}
|
|
770
1005
|
|
|
771
1006
|
/**
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import * as path from "node:path";
|
|
2
|
+
import { AtomicLockCoordinator, type AtomicLockLease } from "./atomic-lock-coordinator.js";
|
|
3
|
+
import { canonicalStoragePath } from "./canonical-storage-path.js";
|
|
4
|
+
|
|
5
|
+
const MUTATION_WAIT_MS = 5_000;
|
|
6
|
+
const MUTATION_STALE_MS = 300_000;
|
|
7
|
+
|
|
8
|
+
export async function canonicalMarkdownIdentity(filePath: string): Promise<string> {
|
|
9
|
+
return canonicalStoragePath(filePath);
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export async function acquireMarkdownMutationLock(filePath: string): Promise<AtomicLockLease> {
|
|
13
|
+
const identity = await canonicalMarkdownIdentity(filePath);
|
|
14
|
+
const coordinatorDir = path.dirname(path.dirname(identity));
|
|
15
|
+
const coordinator = new AtomicLockCoordinator(path.join(coordinatorDir, ".pi-hermes-locks.sqlite"));
|
|
16
|
+
const lockKey = `mutation:${identity}`;
|
|
17
|
+
const deadline = Date.now() + MUTATION_WAIT_MS;
|
|
18
|
+
let lease = coordinator.tryAcquire(lockKey, { staleMs: MUTATION_STALE_MS });
|
|
19
|
+
|
|
20
|
+
while (!lease) {
|
|
21
|
+
if (Date.now() >= deadline) {
|
|
22
|
+
throw new Error(`Memory mutation already in progress for ${identity}`);
|
|
23
|
+
}
|
|
24
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
25
|
+
lease = coordinator.tryAcquire(lockKey, { staleMs: MUTATION_STALE_MS });
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
return lease;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export async function withMarkdownMutationLock<T>(filePath: string, operation: () => Promise<T> | T): Promise<T> {
|
|
32
|
+
const lease = await acquireMarkdownMutationLock(filePath);
|
|
33
|
+
try {
|
|
34
|
+
return await operation();
|
|
35
|
+
} finally {
|
|
36
|
+
lease.release();
|
|
37
|
+
}
|
|
38
|
+
}
|