rag-memory-epf-mcp 3.5.2 → 3.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -0
- package/dist/index.d.ts +50 -6
- package/dist/index.js +782 -188
- package/dist/src/backfillCoordinator.d.ts +59 -0
- package/dist/src/backfillCoordinator.js +552 -0
- package/dist/src/embeddingGate.d.ts +68 -0
- package/dist/src/embeddingGate.js +227 -0
- package/dist/src/migrations/migrations.js +71 -0
- package/dist/src/modelCache.d.ts +33 -0
- package/dist/src/modelCache.js +235 -0
- package/docs/UPDATING.md +121 -0
- package/package.json +8 -4
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
// v3.6 lite install (spec 2026-07-18 v5 §3): EmbeddingGate owns the embedding
|
|
2
|
+
// model lifecycle — nothing else. State machine + single-flight load/retry +
|
|
3
|
+
// prioritized serial inference queue. It does NOT touch the DB, backfill
|
|
4
|
+
// policy, FTS SQL, or tool response shapes (A′ boundary).
|
|
5
|
+
//
|
|
6
|
+
// Consumers never read state and then call separately (TOCTOU): embed() decides
|
|
7
|
+
// and executes inside one boundary, rejecting with typed errors when the model
|
|
8
|
+
// is unavailable so each tool can honor its own not-ready contract.
|
|
9
|
+
export class GateDisabledError extends Error {
|
|
10
|
+
code = 'EMBEDDINGS_DISABLED';
|
|
11
|
+
state = 'disabled';
|
|
12
|
+
constructor() {
|
|
13
|
+
super('Embeddings are disabled (RAG_MEMORY_EMBEDDINGS=off)');
|
|
14
|
+
this.name = 'GateDisabledError';
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
export class GateNotReadyError extends Error {
|
|
18
|
+
state;
|
|
19
|
+
retryAfterMs;
|
|
20
|
+
code = 'MODEL_NOT_READY';
|
|
21
|
+
constructor(state, retryAfterMs) {
|
|
22
|
+
super(`Embedding model not ready (state=${state})`);
|
|
23
|
+
this.state = state;
|
|
24
|
+
this.retryAfterMs = retryAfterMs;
|
|
25
|
+
this.name = 'GateNotReadyError';
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
// Configuration incompatibility (e.g. a non-1024-dim custom model): retrying or
|
|
29
|
+
// quarantining the cache cannot fix it — the gate parks in terminal 'failed'
|
|
30
|
+
// with no reload schedule (beta 2R B3).
|
|
31
|
+
export class TerminalConfigError extends Error {
|
|
32
|
+
terminal = true;
|
|
33
|
+
constructor(message) {
|
|
34
|
+
super(message);
|
|
35
|
+
this.name = 'TerminalConfigError';
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
const PRIO = { interactive: 0, bulk: 1, backfill: 2 };
|
|
39
|
+
const DEFAULT_BACKOFF_MS = [30_000, 120_000, 600_000, 3_600_000];
|
|
40
|
+
export class EmbeddingGate {
|
|
41
|
+
opts;
|
|
42
|
+
state;
|
|
43
|
+
embedFn = null;
|
|
44
|
+
startPromise = null;
|
|
45
|
+
queue = [];
|
|
46
|
+
running = false;
|
|
47
|
+
seq = 0;
|
|
48
|
+
generation = 0;
|
|
49
|
+
shuttingDown = false;
|
|
50
|
+
retryTimer = null;
|
|
51
|
+
attempt = 0;
|
|
52
|
+
// Systemic-failure detector (beta 2R B4): DISTINCT failed inputs within the
|
|
53
|
+
// current load epoch (cleared on successful load AND on any inference
|
|
54
|
+
// success). Same-input retries are data-specific (poison row) and never
|
|
55
|
+
// demote. `demotions` escalates the reload backoff across repeated
|
|
56
|
+
// demote-reload cycles and only resets on a real inference success.
|
|
57
|
+
failedInputs = new Set();
|
|
58
|
+
demotions = 0;
|
|
59
|
+
terminalFailure = false;
|
|
60
|
+
abort = new AbortController(); // loader passes this to lock waits / fetches
|
|
61
|
+
readySince;
|
|
62
|
+
lastError;
|
|
63
|
+
retryAt;
|
|
64
|
+
backoff;
|
|
65
|
+
constructor(opts) {
|
|
66
|
+
this.opts = opts;
|
|
67
|
+
this.backoff = opts.backoffMs ?? DEFAULT_BACKOFF_MS;
|
|
68
|
+
this.state = opts.mode === 'off' ? 'disabled' : 'idle';
|
|
69
|
+
}
|
|
70
|
+
get status() {
|
|
71
|
+
return { state: this.state, readySince: this.readySince, lastError: this.lastError, retryAt: this.retryAt };
|
|
72
|
+
}
|
|
73
|
+
get isReady() { return this.state === 'ready'; }
|
|
74
|
+
get isDisabled() { return this.state === 'disabled'; }
|
|
75
|
+
setState(s) {
|
|
76
|
+
if (this.state !== s) {
|
|
77
|
+
this.state = s;
|
|
78
|
+
this.opts.onStateChange?.(s);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
// Single-flight: concurrent calls share one load; after a failure the next
|
|
82
|
+
// scheduled retry (or explicit call) re-enters loadOnce.
|
|
83
|
+
start() {
|
|
84
|
+
if (this.state === 'disabled' || this.shuttingDown)
|
|
85
|
+
return Promise.resolve();
|
|
86
|
+
if (!this.startPromise)
|
|
87
|
+
this.startPromise = this.loadOnce();
|
|
88
|
+
return this.startPromise;
|
|
89
|
+
}
|
|
90
|
+
// loading -> downloading is reported by the loader via markDownloading().
|
|
91
|
+
markDownloading() {
|
|
92
|
+
if (this.state === 'loading')
|
|
93
|
+
this.setState('downloading');
|
|
94
|
+
}
|
|
95
|
+
async loadOnce() {
|
|
96
|
+
this.setState('loading');
|
|
97
|
+
try {
|
|
98
|
+
this.embedFn = await this.opts.loadModel();
|
|
99
|
+
this.attempt = 0;
|
|
100
|
+
this.retryAt = undefined;
|
|
101
|
+
this.lastError = undefined;
|
|
102
|
+
this.failedInputs.clear(); // fresh load epoch (beta 2R B4)
|
|
103
|
+
this.readySince = new Date().toISOString();
|
|
104
|
+
this.setState('ready');
|
|
105
|
+
this.opts.onReady?.();
|
|
106
|
+
this.pump();
|
|
107
|
+
}
|
|
108
|
+
catch (e) {
|
|
109
|
+
this.lastError = e instanceof Error ? e.message : String(e);
|
|
110
|
+
this.setState('failed');
|
|
111
|
+
this.startPromise = null;
|
|
112
|
+
if (e?.terminal === true) {
|
|
113
|
+
// Config incompatibility (beta 2R B3): retrying or quarantining cannot
|
|
114
|
+
// fix it — no reload schedule; a restart with a fixed config is the
|
|
115
|
+
// only recovery.
|
|
116
|
+
this.terminalFailure = true;
|
|
117
|
+
this.retryAt = undefined;
|
|
118
|
+
console.error(`❌ embeddings disabled for this run (config): ${this.lastError}`);
|
|
119
|
+
throw e;
|
|
120
|
+
}
|
|
121
|
+
this.scheduleRetry(this.attempt++);
|
|
122
|
+
throw e;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
// Shared retry scheduler with jitter; escalation index picks the backoff slot.
|
|
126
|
+
scheduleRetry(escalation) {
|
|
127
|
+
if (this.shuttingDown || this.terminalFailure)
|
|
128
|
+
return;
|
|
129
|
+
const base = this.backoff[Math.min(escalation, this.backoff.length - 1)];
|
|
130
|
+
const delay = Math.round(base * (0.8 + Math.random() * 0.4)); // jitter ±20%
|
|
131
|
+
this.retryAt = new Date(Date.now() + delay).toISOString();
|
|
132
|
+
this.retryTimer = setTimeout(() => {
|
|
133
|
+
this.retryTimer = null;
|
|
134
|
+
void this.start().catch(() => { });
|
|
135
|
+
}, delay);
|
|
136
|
+
this.retryTimer.unref?.();
|
|
137
|
+
}
|
|
138
|
+
embed(text, o) {
|
|
139
|
+
if (this.state === 'disabled')
|
|
140
|
+
return Promise.reject(new GateDisabledError());
|
|
141
|
+
if (this.shuttingDown)
|
|
142
|
+
return Promise.reject(new GateNotReadyError(this.state));
|
|
143
|
+
if (this.state !== 'ready') {
|
|
144
|
+
const retryAfterMs = this.retryAt ? Math.max(0, Date.parse(this.retryAt) - Date.now()) : undefined;
|
|
145
|
+
return Promise.reject(new GateNotReadyError(this.state, retryAfterMs));
|
|
146
|
+
}
|
|
147
|
+
return new Promise((resolve, reject) => {
|
|
148
|
+
this.queue.push({
|
|
149
|
+
text, dims: o.dims ?? 1024, isQuery: o.isQuery ?? false,
|
|
150
|
+
prio: PRIO[o.priority], seq: this.seq++, gen: this.generation, resolve, reject,
|
|
151
|
+
});
|
|
152
|
+
this.queue.sort((a, b) => a.prio - b.prio || a.seq - b.seq);
|
|
153
|
+
this.pump();
|
|
154
|
+
});
|
|
155
|
+
}
|
|
156
|
+
pump() {
|
|
157
|
+
if (this.running || !this.embedFn)
|
|
158
|
+
return;
|
|
159
|
+
const job = this.queue.shift();
|
|
160
|
+
if (!job)
|
|
161
|
+
return;
|
|
162
|
+
this.running = true;
|
|
163
|
+
void this.embedFn(job.text, job.dims, job.isQuery)
|
|
164
|
+
.then(v => {
|
|
165
|
+
this.failedInputs.clear();
|
|
166
|
+
this.demotions = 0; // a real success resets escalation
|
|
167
|
+
if (job.gen === this.generation)
|
|
168
|
+
job.resolve(v);
|
|
169
|
+
else
|
|
170
|
+
job.reject(new Error('embedding discarded: gate shut down'));
|
|
171
|
+
})
|
|
172
|
+
.catch(e => {
|
|
173
|
+
// Systemic-failure detection (beta 2R B4): 3 DISTINCT failed inputs in
|
|
174
|
+
// this load epoch -> demote to failed + reload with ESCALATING backoff
|
|
175
|
+
// (demotions counter survives reloads; only an inference success
|
|
176
|
+
// resets it — so a persistently broken runtime backs off instead of
|
|
177
|
+
// hot-looping). Same-input repeats (A→A) and A→B→A count 2 distinct.
|
|
178
|
+
this.failedInputs.add(`${job.text}|${job.dims}|${job.isQuery}`);
|
|
179
|
+
if (this.failedInputs.size >= 3 && this.state === 'ready' && !this.shuttingDown) {
|
|
180
|
+
this.lastError = e instanceof Error ? e.message : String(e);
|
|
181
|
+
this.embedFn = null;
|
|
182
|
+
this.startPromise = null;
|
|
183
|
+
this.failedInputs.clear();
|
|
184
|
+
this.setState('failed');
|
|
185
|
+
this.scheduleRetry(this.demotions++);
|
|
186
|
+
for (const j of this.queue.splice(0))
|
|
187
|
+
j.reject(new GateNotReadyError('failed'));
|
|
188
|
+
}
|
|
189
|
+
job.reject(e);
|
|
190
|
+
})
|
|
191
|
+
.finally(() => {
|
|
192
|
+
this.running = false;
|
|
193
|
+
this.pump();
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
// Was the load still in flight when shutdown settled? (index.ts uses this to
|
|
197
|
+
// decide whether a bounded exit is needed — see shutdownAll.)
|
|
198
|
+
get loadInFlight() {
|
|
199
|
+
return this.startPromise !== null && this.state !== 'ready' && this.state !== 'failed' && this.state !== 'disabled';
|
|
200
|
+
}
|
|
201
|
+
// Graceful shutdown (spec §3, beta B1): block new work, abort what we can
|
|
202
|
+
// (lock waits via this.abort), discard in-flight inference results via the
|
|
203
|
+
// generation token, clear retry timers, then wait (bounded) for BOTH the
|
|
204
|
+
// current inference and an in-flight model load to settle.
|
|
205
|
+
async shutdown(deadlineMs = 5000) {
|
|
206
|
+
this.shuttingDown = true;
|
|
207
|
+
this.generation++;
|
|
208
|
+
this.abort.abort();
|
|
209
|
+
if (this.retryTimer) {
|
|
210
|
+
clearTimeout(this.retryTimer);
|
|
211
|
+
this.retryTimer = null;
|
|
212
|
+
}
|
|
213
|
+
for (const j of this.queue.splice(0))
|
|
214
|
+
j.reject(new Error('gate shutdown'));
|
|
215
|
+
const deadline = Date.now() + deadlineMs;
|
|
216
|
+
if (this.startPromise) {
|
|
217
|
+
const remaining = () => Math.max(0, deadline - Date.now());
|
|
218
|
+
await Promise.race([
|
|
219
|
+
this.startPromise.catch(() => { }),
|
|
220
|
+
new Promise(r => setTimeout(r, remaining())),
|
|
221
|
+
]);
|
|
222
|
+
}
|
|
223
|
+
while (this.running && Date.now() < deadline) {
|
|
224
|
+
await new Promise(r => setTimeout(r, 20));
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
}
|
|
@@ -570,5 +570,76 @@ export const migrations = [
|
|
|
570
570
|
// No clean reversal: would need the original document content to recompute
|
|
571
571
|
// UTF-16 indices, and we never recorded which chunks were touched. No-op.
|
|
572
572
|
}
|
|
573
|
+
},
|
|
574
|
+
// v3.6 (spec 2026-07-18 lite-install v5 §6c): embedding provenance + narrowed FTS trigger.
|
|
575
|
+
// ADD COLUMN is guarded by PRAGMA table_info so re-runs are idempotent.
|
|
576
|
+
{
|
|
577
|
+
version: 12,
|
|
578
|
+
description: 'Embedding provenance (profiles, input_hash, provenance_state), backfill failures, narrowed chunks_fts_update trigger',
|
|
579
|
+
up: (db) => {
|
|
580
|
+
const hasCol = (table, col) => db.prepare(`PRAGMA table_info(${table})`).all().some(c => c.name === col);
|
|
581
|
+
for (const table of ['chunk_metadata', 'entity_embedding_metadata']) {
|
|
582
|
+
if (!hasCol(table, 'input_hash'))
|
|
583
|
+
db.exec(`ALTER TABLE ${table} ADD COLUMN input_hash TEXT`);
|
|
584
|
+
if (!hasCol(table, 'profile_id'))
|
|
585
|
+
db.exec(`ALTER TABLE ${table} ADD COLUMN profile_id INTEGER`);
|
|
586
|
+
if (!hasCol(table, 'provenance_state'))
|
|
587
|
+
db.exec(`ALTER TABLE ${table} ADD COLUMN provenance_state TEXT`);
|
|
588
|
+
}
|
|
589
|
+
db.exec(`
|
|
590
|
+
CREATE TABLE IF NOT EXISTS embedding_profiles (
|
|
591
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
592
|
+
model_id TEXT NOT NULL,
|
|
593
|
+
revision TEXT NOT NULL,
|
|
594
|
+
dtype TEXT NOT NULL,
|
|
595
|
+
dims INTEGER NOT NULL,
|
|
596
|
+
pooling TEXT NOT NULL,
|
|
597
|
+
normalize INTEGER NOT NULL,
|
|
598
|
+
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
599
|
+
UNIQUE(model_id, revision, dtype, dims, pooling, normalize)
|
|
600
|
+
)
|
|
601
|
+
`);
|
|
602
|
+
db.exec(`
|
|
603
|
+
CREATE TABLE IF NOT EXISTS embedding_backfill_failures (
|
|
604
|
+
kind TEXT NOT NULL,
|
|
605
|
+
target_id TEXT NOT NULL,
|
|
606
|
+
input_hash TEXT,
|
|
607
|
+
profile_id INTEGER,
|
|
608
|
+
attempts INTEGER NOT NULL DEFAULT 0,
|
|
609
|
+
last_error TEXT,
|
|
610
|
+
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
611
|
+
UNIQUE(kind, target_id)
|
|
612
|
+
)
|
|
613
|
+
`);
|
|
614
|
+
db.exec(`CREATE TABLE IF NOT EXISTS server_meta (key TEXT PRIMARY KEY, value TEXT)`);
|
|
615
|
+
// Narrow the broad AFTER UPDATE trigger (installed by v8) so provenance-only
|
|
616
|
+
// updates do not rewrite the FTS index (advisor 4R must-fix 1).
|
|
617
|
+
db.exec(`DROP TRIGGER IF EXISTS chunks_fts_update`);
|
|
618
|
+
db.exec(`
|
|
619
|
+
CREATE TRIGGER chunks_fts_update AFTER UPDATE OF text, chunk_id ON chunk_metadata BEGIN
|
|
620
|
+
INSERT INTO chunks_fts(chunks_fts, rowid, text, chunk_id)
|
|
621
|
+
VALUES ('delete', old.rowid, old.text, old.chunk_id);
|
|
622
|
+
INSERT INTO chunks_fts(rowid, text, chunk_id)
|
|
623
|
+
VALUES (new.rowid, new.text, new.chunk_id);
|
|
624
|
+
END
|
|
625
|
+
`);
|
|
626
|
+
},
|
|
627
|
+
down: (db) => {
|
|
628
|
+
// Restore the broad v8 trigger; drop v12 tables. Added columns are left in
|
|
629
|
+
// place (SQLite DROP COLUMN restrictions with dependent objects) — pre-v12
|
|
630
|
+
// code ignores unknown columns, so rollback stays safe (spec §8-4).
|
|
631
|
+
db.exec(`DROP TRIGGER IF EXISTS chunks_fts_update`);
|
|
632
|
+
db.exec(`
|
|
633
|
+
CREATE TRIGGER chunks_fts_update AFTER UPDATE ON chunk_metadata BEGIN
|
|
634
|
+
INSERT INTO chunks_fts(chunks_fts, rowid, text, chunk_id)
|
|
635
|
+
VALUES ('delete', old.rowid, old.text, old.chunk_id);
|
|
636
|
+
INSERT INTO chunks_fts(rowid, text, chunk_id)
|
|
637
|
+
VALUES (new.rowid, new.text, new.chunk_id);
|
|
638
|
+
END
|
|
639
|
+
`);
|
|
640
|
+
db.exec(`DROP TABLE IF EXISTS embedding_backfill_failures`);
|
|
641
|
+
db.exec(`DROP TABLE IF EXISTS embedding_profiles`);
|
|
642
|
+
db.exec(`DROP TABLE IF EXISTS server_meta`);
|
|
643
|
+
}
|
|
573
644
|
}
|
|
574
645
|
];
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
export declare function resolveModelCacheDir(env: Record<string, string | undefined>, platform: string, homedir: string): string;
|
|
2
|
+
export declare function preflightCacheDir(dir: string): {
|
|
3
|
+
ok: boolean;
|
|
4
|
+
error?: string;
|
|
5
|
+
};
|
|
6
|
+
export declare function artifactKey(modelId: string, revision: string, dtype: string): string;
|
|
7
|
+
export declare class ModelDownloadLock {
|
|
8
|
+
readonly lockPath: string;
|
|
9
|
+
readonly markerPath: string;
|
|
10
|
+
private held;
|
|
11
|
+
constructor(cacheDir: string, key: string);
|
|
12
|
+
private tryAcquire;
|
|
13
|
+
private isStale;
|
|
14
|
+
private holderPid;
|
|
15
|
+
invalidateMarker(): void;
|
|
16
|
+
acquireOrWait(opts: {
|
|
17
|
+
pollMs?: number;
|
|
18
|
+
timeoutMs: number;
|
|
19
|
+
signal?: AbortSignal;
|
|
20
|
+
}): Promise<'owner' | 'ready'>;
|
|
21
|
+
markComplete(): void;
|
|
22
|
+
release(): void;
|
|
23
|
+
}
|
|
24
|
+
export declare function isCacheIntegrityError(e: unknown, cacheDir?: string): boolean;
|
|
25
|
+
export declare function handleLoaderFailure(opts: {
|
|
26
|
+
role: 'owner' | 'ready';
|
|
27
|
+
error: unknown;
|
|
28
|
+
lock: ModelDownloadLock;
|
|
29
|
+
cacheDir: string;
|
|
30
|
+
modelId: string;
|
|
31
|
+
terminal: boolean;
|
|
32
|
+
}): 'none' | 'marker-invalidated' | 'quarantined';
|
|
33
|
+
export declare function quarantinePartialCache(cacheDir: string, modelId: string): void;
|
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
// v3.6 lite install (spec 2026-07-18 v5 §7): version-independent model cache.
|
|
2
|
+
// The transformers.js default cache lives INSIDE the package directory, which is
|
|
3
|
+
// npx-slot scoped — every engine version bump re-downloads ~1.2GB. This module
|
|
4
|
+
// resolves a user-level cache dir and provides a cross-process download lock so
|
|
5
|
+
// concurrent MCP servers on one machine never write the same model concurrently
|
|
6
|
+
// (transformers' FileCache.put is not atomic).
|
|
7
|
+
//
|
|
8
|
+
// Lock/marker keys use ONLY cache-artifact fields (model id, revision, dtype) —
|
|
9
|
+
// never retrieval config (spec §6c layer 1).
|
|
10
|
+
import { mkdirSync, writeFileSync, readFileSync, unlinkSync, existsSync, renameSync, rmSync, readdirSync } from 'node:fs';
|
|
11
|
+
import { join, sep } from 'node:path';
|
|
12
|
+
import { createHash } from 'node:crypto';
|
|
13
|
+
export function resolveModelCacheDir(env, platform, homedir) {
|
|
14
|
+
if (env.RAG_MEMORY_MODEL_CACHE_DIR)
|
|
15
|
+
return env.RAG_MEMORY_MODEL_CACHE_DIR;
|
|
16
|
+
if (env.XDG_CACHE_HOME)
|
|
17
|
+
return join(env.XDG_CACHE_HOME, 'rag-memory-epf-mcp');
|
|
18
|
+
if (platform === 'darwin')
|
|
19
|
+
return join(homedir, 'Library', 'Caches', 'rag-memory-epf-mcp');
|
|
20
|
+
if (platform === 'win32' && env.LOCALAPPDATA)
|
|
21
|
+
return join(env.LOCALAPPDATA, 'rag-memory-epf-mcp');
|
|
22
|
+
return join(homedir, '.cache', 'rag-memory-epf-mcp');
|
|
23
|
+
}
|
|
24
|
+
// mkdir -p + write probe. On failure the caller must transition the gate to
|
|
25
|
+
// `failed` — silently falling back to the package-internal cache is forbidden.
|
|
26
|
+
export function preflightCacheDir(dir) {
|
|
27
|
+
try {
|
|
28
|
+
mkdirSync(dir, { recursive: true });
|
|
29
|
+
const probe = join(dir, `.write-probe-${process.pid}`);
|
|
30
|
+
writeFileSync(probe, 'ok');
|
|
31
|
+
unlinkSync(probe);
|
|
32
|
+
return { ok: true };
|
|
33
|
+
}
|
|
34
|
+
catch (e) {
|
|
35
|
+
return { ok: false, error: e instanceof Error ? e.message : String(e) };
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
export function artifactKey(modelId, revision, dtype) {
|
|
39
|
+
return createHash('sha1').update(`${modelId}@${revision}#${dtype}`).digest('hex').slice(0, 12);
|
|
40
|
+
}
|
|
41
|
+
const STALE_LOCK_MS = 30 * 60_000;
|
|
42
|
+
export class ModelDownloadLock {
|
|
43
|
+
lockPath;
|
|
44
|
+
markerPath;
|
|
45
|
+
held = false;
|
|
46
|
+
constructor(cacheDir, key) {
|
|
47
|
+
this.lockPath = join(cacheDir, `.download-${key}.lock`);
|
|
48
|
+
this.markerPath = join(cacheDir, `.complete-${key}`);
|
|
49
|
+
}
|
|
50
|
+
tryAcquire() {
|
|
51
|
+
try {
|
|
52
|
+
writeFileSync(this.lockPath, JSON.stringify({ pid: process.pid, startedAt: Date.now() }), { flag: 'wx' });
|
|
53
|
+
this.held = true;
|
|
54
|
+
return true;
|
|
55
|
+
}
|
|
56
|
+
catch {
|
|
57
|
+
return false;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
// Stale = holder pid is dead or the lock is unreadable. A LIVE pid is never
|
|
61
|
+
// reclaimed by age alone (beta B5): a slow 1.2GB download has no heartbeat,
|
|
62
|
+
// so an mtime rule would mint a second concurrent owner — the exact failure
|
|
63
|
+
// this lock exists to prevent. A hung-but-alive holder instead surfaces as a
|
|
64
|
+
// waiter timeout -> gate 'failed' + backoff, with the lock path and holder
|
|
65
|
+
// pid in the error so an operator can verify and remove it manually (the
|
|
66
|
+
// recovery procedure lives in docs/UPDATING.md). No pid-reuse heuristic:
|
|
67
|
+
// there is no portable process-start identity to compare against (beta 2R).
|
|
68
|
+
isStale() {
|
|
69
|
+
try {
|
|
70
|
+
const info = JSON.parse(readFileSync(this.lockPath, 'utf-8'));
|
|
71
|
+
try {
|
|
72
|
+
process.kill(info.pid, 0);
|
|
73
|
+
}
|
|
74
|
+
catch {
|
|
75
|
+
return true;
|
|
76
|
+
} // pid dead
|
|
77
|
+
return false;
|
|
78
|
+
}
|
|
79
|
+
catch {
|
|
80
|
+
return true; // unreadable lock = stale
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
holderPid() {
|
|
84
|
+
try {
|
|
85
|
+
return JSON.parse(readFileSync(this.lockPath, 'utf-8')).pid;
|
|
86
|
+
}
|
|
87
|
+
catch {
|
|
88
|
+
return null;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
// Corrupted-cache recovery (beta B5): a marker only proves a PAST verified
|
|
92
|
+
// load. When a marker-holder ('ready' role) later fails to load, the caller
|
|
93
|
+
// must invalidate the marker so the next attempt becomes a locked owner.
|
|
94
|
+
invalidateMarker() {
|
|
95
|
+
try {
|
|
96
|
+
unlinkSync(this.markerPath);
|
|
97
|
+
}
|
|
98
|
+
catch { /* already gone */ }
|
|
99
|
+
}
|
|
100
|
+
// (beta 4R M1) Quarantine decisions are made by ERROR CLASS, not by failure
|
|
101
|
+
// count — see isCacheIntegrityError below.
|
|
102
|
+
// Returns 'owner' (caller must download, then markComplete + release) or
|
|
103
|
+
// 'ready' (a completed, verified cache already exists). Async polling only —
|
|
104
|
+
// never blocks the event loop (spec §7 / 3R M14).
|
|
105
|
+
async acquireOrWait(opts) {
|
|
106
|
+
const poll = opts.pollMs ?? 500;
|
|
107
|
+
const deadline = Date.now() + opts.timeoutMs;
|
|
108
|
+
for (;;) {
|
|
109
|
+
// Abort FIRST (beta 2R B5): a shutdown-aborted waiter must never become
|
|
110
|
+
// an owner or report 'ready' and start a pipeline load.
|
|
111
|
+
if (opts.signal?.aborted)
|
|
112
|
+
throw new Error('model download lock wait aborted');
|
|
113
|
+
if (existsSync(this.markerPath))
|
|
114
|
+
return 'ready';
|
|
115
|
+
if (this.tryAcquire())
|
|
116
|
+
return 'owner';
|
|
117
|
+
if (this.isStale()) {
|
|
118
|
+
try {
|
|
119
|
+
unlinkSync(this.lockPath);
|
|
120
|
+
}
|
|
121
|
+
catch { /* raced with another reclaimer */ }
|
|
122
|
+
continue;
|
|
123
|
+
}
|
|
124
|
+
if (Date.now() > deadline) {
|
|
125
|
+
throw new Error(`model download lock wait timed out — holder pid=${this.holderPid() ?? 'unknown'}, lock=${this.lockPath}. If that process is hung, verify and remove the lock file manually (see docs/UPDATING.md).`);
|
|
126
|
+
}
|
|
127
|
+
// Abortable sleep with symmetric listener cleanup (beta 3R M4): a
|
|
128
|
+
// 10-minute wait at 500ms polls must not accumulate ~1200 abort
|
|
129
|
+
// listeners on the shared shutdown signal.
|
|
130
|
+
await new Promise(resolve => {
|
|
131
|
+
const onAbort = () => { clearTimeout(t); resolve(); };
|
|
132
|
+
const t = setTimeout(() => {
|
|
133
|
+
opts.signal?.removeEventListener('abort', onAbort);
|
|
134
|
+
resolve();
|
|
135
|
+
}, poll);
|
|
136
|
+
opts.signal?.addEventListener('abort', onAbort, { once: true });
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
// Only call after a verified cache load — a memory-only load success without
|
|
141
|
+
// durable cache files must NOT produce a marker (3R M14).
|
|
142
|
+
markComplete() {
|
|
143
|
+
const tmp = `${this.markerPath}.tmp.${process.pid}`;
|
|
144
|
+
writeFileSync(tmp, new Date().toISOString());
|
|
145
|
+
renameSync(tmp, this.markerPath);
|
|
146
|
+
}
|
|
147
|
+
release() {
|
|
148
|
+
if (this.held) {
|
|
149
|
+
try {
|
|
150
|
+
unlinkSync(this.lockPath);
|
|
151
|
+
}
|
|
152
|
+
catch { /* already gone */ }
|
|
153
|
+
this.held = false;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
// (beta 4R M1 -> 5R M2) Error classification for cache handling: ONLY strong
|
|
158
|
+
// serialization/truncation/corruption signatures justify quarantining a 1.2GB
|
|
159
|
+
// cache. Generic words (parse, invalid model, byte length) and bare ENOENT
|
|
160
|
+
// match far too much outside the cache — ENOENT counts only when the message
|
|
161
|
+
// points INSIDE the model cache directory. OOM / allocation / network errors
|
|
162
|
+
// preserve the cache regardless of repetition. Residual risk (documented): a
|
|
163
|
+
// corruption with no matching signature stays put; docs/UPDATING.md tells the
|
|
164
|
+
// operator to delete the model cache directory manually.
|
|
165
|
+
const CACHE_INTEGRITY_SIGNATURES = [
|
|
166
|
+
/protobuf/i, /corrupt/i, /unexpected end/i, /deseriali[sz]e/i,
|
|
167
|
+
/magic number/i, /truncated/i, /checksum/i,
|
|
168
|
+
];
|
|
169
|
+
const CACHE_PRESERVE_SIGNATURES = [
|
|
170
|
+
/out of memory/i, /bad_alloc/i, /allocation/i, /OOM/i, /ETIMEDOUT/, /ECONNRESET/, /fetch/i, /network/i,
|
|
171
|
+
];
|
|
172
|
+
export function isCacheIntegrityError(e, cacheDir) {
|
|
173
|
+
const msg = e instanceof Error ? `${e.name}: ${e.message}` : String(e);
|
|
174
|
+
if (CACHE_PRESERVE_SIGNATURES.some(re => re.test(msg)))
|
|
175
|
+
return false;
|
|
176
|
+
if (CACHE_INTEGRITY_SIGNATURES.some(re => re.test(msg)))
|
|
177
|
+
return true;
|
|
178
|
+
// Missing-file errors are integrity ONLY when they point INSIDE the cache
|
|
179
|
+
// dir (path-boundary-safe: '/cache' must not match '/cache-old' — 6R note).
|
|
180
|
+
if (cacheDir && (/ENOENT/.test(msg) || /no such file/i.test(msg))
|
|
181
|
+
&& (msg.includes(cacheDir + sep) || msg.includes(cacheDir + '/')))
|
|
182
|
+
return true;
|
|
183
|
+
return false;
|
|
184
|
+
}
|
|
185
|
+
// (beta 5R M1) Loader-failure cache policy, extracted for unit testing.
|
|
186
|
+
// Quarantine requires EXCLUSIVITY: only the lock-holding OWNER may rename or
|
|
187
|
+
// delete shared cache files — a ready-role process (no lock) racing other
|
|
188
|
+
// readers must never touch the directory; it only drops the marker so the
|
|
189
|
+
// next retry becomes a locked owner and re-proves the same integrity error
|
|
190
|
+
// before any destructive action.
|
|
191
|
+
export function handleLoaderFailure(opts) {
|
|
192
|
+
if (opts.terminal)
|
|
193
|
+
return 'none'; // config error: cache is fine
|
|
194
|
+
if (!isCacheIntegrityError(opts.error, opts.cacheDir)) {
|
|
195
|
+
// OOM / network / unknown: preserve everything (marker included — a
|
|
196
|
+
// transient error does not disprove a verified cache).
|
|
197
|
+
return 'none';
|
|
198
|
+
}
|
|
199
|
+
if (opts.role === 'ready') {
|
|
200
|
+
opts.lock.invalidateMarker();
|
|
201
|
+
return 'marker-invalidated';
|
|
202
|
+
}
|
|
203
|
+
opts.lock.invalidateMarker();
|
|
204
|
+
quarantinePartialCache(opts.cacheDir, opts.modelId);
|
|
205
|
+
return 'quarantined';
|
|
206
|
+
}
|
|
207
|
+
// Owner-side failure handling: partially downloaded model dirs are quarantined
|
|
208
|
+
// (renamed aside) rather than left in place, so the next attempt starts clean.
|
|
209
|
+
// transformers.js FileCache keys are `<org>/<model>/...` — the on-disk layout
|
|
210
|
+
// nests the model id's path segments under cacheDir (beta B5: a flattened
|
|
211
|
+
// `org_model` path would miss the real files entirely).
|
|
212
|
+
export function quarantinePartialCache(cacheDir, modelId) {
|
|
213
|
+
const dir = join(cacheDir, ...modelId.split('/'));
|
|
214
|
+
if (existsSync(dir)) {
|
|
215
|
+
try {
|
|
216
|
+
renameSync(dir, `${dir}.quarantine.${Date.now()}`);
|
|
217
|
+
}
|
|
218
|
+
catch {
|
|
219
|
+
rmSync(dir, { recursive: true, force: true });
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
// Retention (beta 2R B3): keep only the newest quarantine — repeated failures
|
|
223
|
+
// must not accumulate 1.2GB directories.
|
|
224
|
+
try {
|
|
225
|
+
const parent = join(cacheDir, ...modelId.split('/').slice(0, -1));
|
|
226
|
+
const leaf = modelId.split('/').pop();
|
|
227
|
+
const quarantines = readdirSync(parent)
|
|
228
|
+
.filter(f => f.startsWith(`${leaf}.quarantine.`))
|
|
229
|
+
.sort();
|
|
230
|
+
for (const old of quarantines.slice(0, -1)) {
|
|
231
|
+
rmSync(join(parent, old), { recursive: true, force: true });
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
catch { /* parent may not exist */ }
|
|
235
|
+
}
|