pinokiod 8.0.39 → 8.0.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/kernel/api/hf/index.js +2 -2
- package/kernel/api/index.js +12 -3
- package/kernel/connect/index.js +2 -2
- package/kernel/connect/providers/huggingface/index.js +14 -5
- package/kernel/index.js +14 -2
- package/kernel/shell.js +3 -2
- package/kernel/shells.js +6 -0
- package/kernel/vault/constants.js +13 -0
- package/kernel/vault/hash_worker.js +9 -2
- package/kernel/vault/index.js +1856 -316
- package/kernel/vault/registry.js +526 -52
- package/kernel/vault/snapshot.js +29 -0
- package/kernel/vault/sweeper.js +480 -210
- package/kernel/vault/walker.js +142 -0
- package/package.json +1 -1
- package/server/index.js +79 -90
- package/server/lib/privacy_filter_cache.js +4 -1
- package/server/public/storage-size.js +12 -0
- package/server/public/style.css +13 -0
- package/server/public/tab-link-popover.js +3 -0
- package/server/public/vault-nav.js +96 -0
- package/server/public/vault.css +1079 -0
- package/server/public/vault.js +1835 -0
- package/server/views/app.ejs +93 -16
- package/server/views/connect/huggingface.ejs +7 -9
- package/server/views/partials/main_sidebar.ejs +6 -1
- package/server/views/partials/vault_workspace.ejs +55 -0
- package/server/views/vault.ejs +5 -1532
- package/server/views/vault_app.ejs +22 -0
- package/test/hf-api.test.js +4 -2
- package/test/huggingface-connect.test.js +7 -11
- package/test/huggingface-token-validity.test.js +135 -0
- package/test/plugin-action-functions.test.js +19 -0
- package/test/privacy-filter-cache.test.js +1 -0
- package/test/vault-engine.test.js +1276 -26
- package/test/vault-sweep.test.js +1043 -35
- package/test/vault-ui.test.js +2184 -65
|
@@ -6,6 +6,7 @@ const path = require('path')
|
|
|
6
6
|
const crypto = require('crypto')
|
|
7
7
|
const { execSync } = require('child_process')
|
|
8
8
|
const Vault = require('../kernel/vault')
|
|
9
|
+
const Registry = require('../kernel/vault/registry')
|
|
9
10
|
|
|
10
11
|
const sha256 = (buf) => crypto.createHash('sha256').update(buf).digest('hex')
|
|
11
12
|
|
|
@@ -70,6 +71,95 @@ describe('vault engine (phase 1)', () => {
|
|
|
70
71
|
}
|
|
71
72
|
})
|
|
72
73
|
|
|
74
|
+
test('kill switch blocks dashboard actions without creating mounts', async () => {
|
|
75
|
+
const h = await home()
|
|
76
|
+
const external = await home()
|
|
77
|
+
await writeFile(path.resolve(h, 'ENVIRONMENT'), 'PINOKIO_VAULT=false\n')
|
|
78
|
+
const vault = new Vault(fakeKernel(h))
|
|
79
|
+
await vault.init()
|
|
80
|
+
|
|
81
|
+
const result = await vault.perform('add_source', { path: external })
|
|
82
|
+
|
|
83
|
+
assert.match(result.error, /disabled/i)
|
|
84
|
+
assert.deepStrictEqual(await fs.promises.readdir(path.resolve(h, 'api')), [])
|
|
85
|
+
assert.strictEqual(fs.existsSync(path.resolve(h, 'vault')), false)
|
|
86
|
+
})
|
|
87
|
+
|
|
88
|
+
test('existing-only startup leaves a fresh install untouched until Vault is opened', async () => {
|
|
89
|
+
const h = await home()
|
|
90
|
+
const vault = new Vault(fakeKernel(h))
|
|
91
|
+
|
|
92
|
+
const startup = await vault.init({ existingOnly: true })
|
|
93
|
+
|
|
94
|
+
assert.deepStrictEqual(startup, { enabled: true, fresh: true })
|
|
95
|
+
assert.strictEqual(fs.existsSync(path.resolve(h, 'vault')), false)
|
|
96
|
+
assert.deepStrictEqual(vault.navigationStatus(), {
|
|
97
|
+
any_scan: false,
|
|
98
|
+
scanned: false,
|
|
99
|
+
scanning: false,
|
|
100
|
+
pending_bytes: 0
|
|
101
|
+
})
|
|
102
|
+
assert.strictEqual(fs.existsSync(path.resolve(h, 'vault')), false, 'navigation status is read-only')
|
|
103
|
+
await vault.ensureInitialized()
|
|
104
|
+
assert.strictEqual(fs.existsSync(path.resolve(h, 'vault', 'registry.json')), true)
|
|
105
|
+
})
|
|
106
|
+
|
|
107
|
+
test('navigation status distinguishes global and app scans without filesystem work', async () => {
|
|
108
|
+
const h = await home()
|
|
109
|
+
const pending = await writeFile(path.resolve(h, 'api', 'appA', 'pending.bin'), 'pending')
|
|
110
|
+
const vault = await makeVault(h)
|
|
111
|
+
const hash = sha256(Buffer.from('pending'))
|
|
112
|
+
const sourceId = 'app:appA'
|
|
113
|
+
vault.registry.addBlob(hash, { size: 7000 })
|
|
114
|
+
vault.registry.setDuplicate(pending, {
|
|
115
|
+
hash,
|
|
116
|
+
size: 7000,
|
|
117
|
+
app: 'appA',
|
|
118
|
+
source_id: sourceId
|
|
119
|
+
})
|
|
120
|
+
vault.registry.setLastScan({ ts: 1, files: 1 }, sourceId)
|
|
121
|
+
|
|
122
|
+
assert.deepStrictEqual(vault.navigationStatus(), {
|
|
123
|
+
any_scan: true,
|
|
124
|
+
scanned: false,
|
|
125
|
+
scanning: false,
|
|
126
|
+
pending_bytes: 7000,
|
|
127
|
+
incomplete: false
|
|
128
|
+
})
|
|
129
|
+
assert.deepStrictEqual(vault.navigationStatusForApp('appA'), {
|
|
130
|
+
any_scan: true,
|
|
131
|
+
scanned: true,
|
|
132
|
+
scanning: false,
|
|
133
|
+
pending_bytes: 7000,
|
|
134
|
+
incomplete: false
|
|
135
|
+
})
|
|
136
|
+
assert.deepStrictEqual(vault.navigationStatusForApp('appB'), {
|
|
137
|
+
any_scan: true,
|
|
138
|
+
scanned: false,
|
|
139
|
+
scanning: false,
|
|
140
|
+
pending_bytes: 0,
|
|
141
|
+
incomplete: false
|
|
142
|
+
})
|
|
143
|
+
|
|
144
|
+
vault.registry.exclude(pending, {
|
|
145
|
+
ts: Date.now(),
|
|
146
|
+
size: 7000,
|
|
147
|
+
source_id: sourceId,
|
|
148
|
+
unverified: true
|
|
149
|
+
})
|
|
150
|
+
assert.strictEqual(vault.navigationStatus().incomplete, true)
|
|
151
|
+
assert.strictEqual(vault.navigationStatusForApp('appA').incomplete, true)
|
|
152
|
+
|
|
153
|
+
vault.sweeper.state = Object.assign(vault.sweeper.idleState(), {
|
|
154
|
+
active: true,
|
|
155
|
+
phase: 'discovering',
|
|
156
|
+
scope_id: sourceId
|
|
157
|
+
})
|
|
158
|
+
assert.strictEqual(vault.navigationStatus().scanning, true)
|
|
159
|
+
assert.strictEqual(vault.navigationStatusForApp('appA').scanning, true)
|
|
160
|
+
assert.strictEqual(vault.navigationStatusForApp('appB').scanning, false)
|
|
161
|
+
})
|
|
162
|
+
|
|
73
163
|
test('adopt: metadata-only, store name shares the inode', async () => {
|
|
74
164
|
const h = await home()
|
|
75
165
|
const vault = await makeVault(h)
|
|
@@ -86,6 +176,37 @@ describe('vault engine (phase 1)', () => {
|
|
|
86
176
|
assert.strictEqual(vault.registry.links.get(file).hash, hash)
|
|
87
177
|
})
|
|
88
178
|
|
|
179
|
+
test('adopt never assigns a stale hash when the source is replaced before linking', async () => {
|
|
180
|
+
const h = await home()
|
|
181
|
+
const vault = await makeVault(h)
|
|
182
|
+
const original = Buffer.alloc(4096, 1)
|
|
183
|
+
const replacement = Buffer.alloc(4096, 2)
|
|
184
|
+
const hash = sha256(original)
|
|
185
|
+
const file = await writeFile(path.resolve(h, 'api', 'appA', 'm.bin'), original)
|
|
186
|
+
const storePath = vault.storePathFor(hash)
|
|
187
|
+
const realLink = fs.promises.link
|
|
188
|
+
fs.promises.link = async (source, target) => {
|
|
189
|
+
if (source === file && target === storePath) {
|
|
190
|
+
const writerPath = file + '.writer-replacement'
|
|
191
|
+
await fs.promises.writeFile(writerPath, replacement)
|
|
192
|
+
await fs.promises.rename(writerPath, file)
|
|
193
|
+
}
|
|
194
|
+
return realLink(source, target)
|
|
195
|
+
}
|
|
196
|
+
let result
|
|
197
|
+
try {
|
|
198
|
+
result = await vault.adopt(file, hash, { app: 'appA' })
|
|
199
|
+
} finally {
|
|
200
|
+
fs.promises.link = realLink
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
assert.strictEqual(result.status, 'stale')
|
|
204
|
+
assert.deepStrictEqual(await fs.promises.readFile(file), replacement)
|
|
205
|
+
assert.strictEqual(fs.existsSync(storePath), false)
|
|
206
|
+
assert.strictEqual(vault.registry.blobs.has(hash), false)
|
|
207
|
+
assert.strictEqual(vault.registry.links.has(file), false)
|
|
208
|
+
})
|
|
209
|
+
|
|
89
210
|
test('item 5: convert links duplicate to blob, content intact, other copies untouched', async () => {
|
|
90
211
|
const h = await home()
|
|
91
212
|
const vault = await makeVault(h)
|
|
@@ -101,24 +222,195 @@ describe('vault engine (phase 1)', () => {
|
|
|
101
222
|
assert.strictEqual(stA.ino, stB.ino)
|
|
102
223
|
assert.strictEqual(stB.nlink, 3)
|
|
103
224
|
assert.deepStrictEqual(await fs.promises.readFile(b), content)
|
|
104
|
-
assert.strictEqual(vault.registry.totals.lifetime_bytes_saved, content.length)
|
|
105
225
|
const events = await vault.registry.readEvents()
|
|
106
226
|
assert.ok(events.some((e) => e.kind === 'convert' && e.batch_id === 'batch1'))
|
|
107
227
|
})
|
|
108
228
|
|
|
229
|
+
test('convert stays registered when metadata fails after the atomic replacement', async () => {
|
|
230
|
+
const h = await home()
|
|
231
|
+
const vault = await makeVault(h)
|
|
232
|
+
const content = crypto.randomBytes(4096)
|
|
233
|
+
const hash = sha256(content)
|
|
234
|
+
const original = await writeFile(path.resolve(h, 'api', 'appA', 'm.bin'), content)
|
|
235
|
+
const duplicate = await writeFile(path.resolve(h, 'api', 'appB', 'm.bin'), content)
|
|
236
|
+
await vault.adopt(original, hash, { app: 'appA' })
|
|
237
|
+
const realRename = fs.promises.rename
|
|
238
|
+
const realLstat = fs.promises.lstat
|
|
239
|
+
let committed = false
|
|
240
|
+
let failedRead = false
|
|
241
|
+
fs.promises.rename = async (source, target) => {
|
|
242
|
+
const result = await realRename(source, target)
|
|
243
|
+
if (source === duplicate + Vault.TMP_SUFFIX && target === duplicate) committed = true
|
|
244
|
+
return result
|
|
245
|
+
}
|
|
246
|
+
fs.promises.lstat = async (target, options) => {
|
|
247
|
+
if (committed && !failedRead && path.resolve(String(target)) === duplicate) {
|
|
248
|
+
failedRead = true
|
|
249
|
+
const error = new Error('temporary post-rename metadata failure')
|
|
250
|
+
error.code = 'EIO'
|
|
251
|
+
throw error
|
|
252
|
+
}
|
|
253
|
+
return realLstat(target, options)
|
|
254
|
+
}
|
|
255
|
+
let result
|
|
256
|
+
try {
|
|
257
|
+
result = await vault.convert(duplicate, hash, { app: 'appB', batch_id: 'committed' })
|
|
258
|
+
} finally {
|
|
259
|
+
fs.promises.rename = realRename
|
|
260
|
+
fs.promises.lstat = realLstat
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
assert.strictEqual(failedRead, true)
|
|
264
|
+
assert.strictEqual(result.status, 'converted')
|
|
265
|
+
const storeStat = await fs.promises.stat(vault.storePathFor(hash))
|
|
266
|
+
const duplicateStat = await fs.promises.stat(duplicate)
|
|
267
|
+
assert.strictEqual(duplicateStat.ino, storeStat.ino)
|
|
268
|
+
assert.strictEqual(vault.registry.links.get(duplicate).batch_id, 'committed')
|
|
269
|
+
})
|
|
270
|
+
|
|
271
|
+
test('a failed activity append does not misreport a completed conversion', async () => {
|
|
272
|
+
const h = await home()
|
|
273
|
+
const vault = await makeVault(h)
|
|
274
|
+
const content = crypto.randomBytes(4096)
|
|
275
|
+
const hash = sha256(content)
|
|
276
|
+
const original = await writeFile(path.resolve(h, 'api', 'appA', 'm.bin'), content)
|
|
277
|
+
const duplicate = await writeFile(path.resolve(h, 'api', 'appB', 'm.bin'), content)
|
|
278
|
+
await vault.adopt(original, hash, { app: 'appA' })
|
|
279
|
+
const realAppendFile = vault.registry.appendFile
|
|
280
|
+
vault.registry.appendFile = async () => {
|
|
281
|
+
const error = new Error('temporary activity write failure')
|
|
282
|
+
error.code = 'EIO'
|
|
283
|
+
throw error
|
|
284
|
+
}
|
|
285
|
+
let result
|
|
286
|
+
try {
|
|
287
|
+
result = await vault.convert(duplicate, hash, { app: 'appB', batch_id: 'activity-failed' })
|
|
288
|
+
} finally {
|
|
289
|
+
vault.registry.appendFile = realAppendFile
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
assert.strictEqual(result.status, 'converted')
|
|
293
|
+
assert.ok(vault.registry.links.has(duplicate))
|
|
294
|
+
assert.match(vault.registry.eventError, /temporary activity write failure/)
|
|
295
|
+
assert.match((await vault.status()).activity_error, /temporary activity write failure/)
|
|
296
|
+
await vault.registry.appendEvent({ kind: 'scan' })
|
|
297
|
+
assert.strictEqual(vault.registry.eventError, null)
|
|
298
|
+
})
|
|
299
|
+
|
|
300
|
+
test('convert does not register a writer replacement that wins after the atomic rename', async () => {
|
|
301
|
+
const h = await home()
|
|
302
|
+
const vault = await makeVault(h)
|
|
303
|
+
const content = crypto.randomBytes(4096)
|
|
304
|
+
const replacement = Buffer.from('newer-writer-content')
|
|
305
|
+
const hash = sha256(content)
|
|
306
|
+
const original = await writeFile(path.resolve(h, 'api', 'appA', 'm.bin'), content)
|
|
307
|
+
const duplicate = await writeFile(path.resolve(h, 'api', 'appB', 'm.bin'), content)
|
|
308
|
+
await vault.adopt(original, hash, { app: 'appA' })
|
|
309
|
+
const realRename = fs.promises.rename
|
|
310
|
+
fs.promises.rename = async (source, target) => {
|
|
311
|
+
const result = await realRename(source, target)
|
|
312
|
+
if (source === duplicate + Vault.TMP_SUFFIX && target === duplicate) {
|
|
313
|
+
const writerPath = duplicate + '.writer-replacement'
|
|
314
|
+
await fs.promises.writeFile(writerPath, replacement)
|
|
315
|
+
await realRename(writerPath, duplicate)
|
|
316
|
+
}
|
|
317
|
+
return result
|
|
318
|
+
}
|
|
319
|
+
let result
|
|
320
|
+
try {
|
|
321
|
+
result = await vault.convert(duplicate, hash, { app: 'appB', batch_id: 'raced' })
|
|
322
|
+
} finally {
|
|
323
|
+
fs.promises.rename = realRename
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
assert.strictEqual(result.status, 'stale')
|
|
327
|
+
assert.deepStrictEqual(await fs.promises.readFile(duplicate), replacement)
|
|
328
|
+
assert.strictEqual(vault.registry.links.has(duplicate), false)
|
|
329
|
+
})
|
|
330
|
+
|
|
331
|
+
test('convert refuses a final store metadata change', {
|
|
332
|
+
skip: process.platform === 'win32'
|
|
333
|
+
}, async () => {
|
|
334
|
+
const h = await home()
|
|
335
|
+
const vault = await makeVault(h)
|
|
336
|
+
const content = crypto.randomBytes(4096)
|
|
337
|
+
const hash = sha256(content)
|
|
338
|
+
const original = await writeFile(path.resolve(h, 'api', 'appA', 'm.bin'), content)
|
|
339
|
+
const duplicate = await writeFile(path.resolve(h, 'api', 'appB', 'm.bin'), content)
|
|
340
|
+
await vault.adopt(original, hash, { app: 'appA' })
|
|
341
|
+
const storePath = vault.storePathFor(hash)
|
|
342
|
+
const originalMode = (await fs.promises.stat(storePath)).mode & 0o7777
|
|
343
|
+
const changedMode = originalMode === 0o600 ? 0o640 : 0o600
|
|
344
|
+
const realLstat = fs.promises.lstat
|
|
345
|
+
let targetReads = 0
|
|
346
|
+
fs.promises.lstat = async (target, options) => {
|
|
347
|
+
if (path.resolve(String(target)) === duplicate) {
|
|
348
|
+
targetReads += 1
|
|
349
|
+
if (targetReads === 2) await fs.promises.chmod(storePath, changedMode)
|
|
350
|
+
}
|
|
351
|
+
return realLstat(target, options)
|
|
352
|
+
}
|
|
353
|
+
let result
|
|
354
|
+
try {
|
|
355
|
+
result = await vault.convert(duplicate, hash, { app: 'appB' })
|
|
356
|
+
} finally {
|
|
357
|
+
fs.promises.lstat = realLstat
|
|
358
|
+
}
|
|
359
|
+
assert.strictEqual(result.status, 'stale-blob')
|
|
360
|
+
assert.strictEqual((await fs.promises.stat(duplicate)).nlink, 1)
|
|
361
|
+
assert.deepStrictEqual(await fs.promises.readFile(duplicate), content)
|
|
362
|
+
assert.strictEqual(fs.existsSync(duplicate + Vault.TMP_SUFFIX), false)
|
|
363
|
+
})
|
|
364
|
+
|
|
109
365
|
test('item 5: simulated crash between link and rename leaves target intact; verify cleans stray tmp', async () => {
|
|
110
366
|
const h = await home()
|
|
111
367
|
const vault = await makeVault(h)
|
|
112
368
|
const content = crypto.randomBytes(4096)
|
|
113
369
|
const hash = sha256(content)
|
|
114
370
|
const a = await writeFile(path.resolve(h, 'api', 'appA', 'm.bin'), content)
|
|
371
|
+
const duplicate = await writeFile(path.resolve(h, 'api', 'appB', 'm.bin'), content)
|
|
372
|
+
await vault.refreshSources()
|
|
115
373
|
await vault.adopt(a, hash, { app: 'appA' })
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
374
|
+
const duplicateStat = await fs.promises.stat(duplicate)
|
|
375
|
+
const source = vault.sources().find((item) => item.kind === 'app' && item.app === 'appB')
|
|
376
|
+
vault.registry.setDuplicate(duplicate, {
|
|
377
|
+
hash, size: content.length, app: 'appB', source_id: source.id,
|
|
378
|
+
dev: duplicateStat.dev, ino: duplicateStat.ino,
|
|
379
|
+
mtime: duplicateStat.mtimeMs, ctime: duplicateStat.ctimeMs
|
|
380
|
+
})
|
|
381
|
+
vault.registry.setScanEntry(duplicate, {
|
|
382
|
+
hash, size: content.length, source_id: source.id,
|
|
383
|
+
dev: duplicateStat.dev, ino: duplicateStat.ino,
|
|
384
|
+
mtime: duplicateStat.mtimeMs, ctime: duplicateStat.ctimeMs
|
|
385
|
+
})
|
|
386
|
+
// This is the actual crash state: the target is still pending and the
|
|
387
|
+
// temporary name already shares the stored file's identity.
|
|
388
|
+
await fs.promises.link(vault.storePathFor(hash), duplicate + Vault.TMP_SUFFIX)
|
|
389
|
+
assert.deepStrictEqual(await fs.promises.readFile(duplicate), content)
|
|
390
|
+
await vault.verify()
|
|
391
|
+
assert.strictEqual(fs.existsSync(duplicate + Vault.TMP_SUFFIX), false)
|
|
392
|
+
assert.deepStrictEqual(await fs.promises.readFile(duplicate), content)
|
|
393
|
+
assert.ok(vault.registry.duplicates.has(duplicate))
|
|
394
|
+
})
|
|
395
|
+
|
|
396
|
+
test('conversion and verification never delete an unrelated reserved-suffix file', async () => {
|
|
397
|
+
const h = await home()
|
|
398
|
+
const vault = await makeVault(h)
|
|
399
|
+
const content = crypto.randomBytes(4096)
|
|
400
|
+
const hash = sha256(content)
|
|
401
|
+
const original = await writeFile(path.resolve(h, 'api', 'appA', 'm.bin'), content)
|
|
402
|
+
const duplicate = await writeFile(path.resolve(h, 'api', 'appB', 'm.bin'), content)
|
|
403
|
+
await vault.adopt(original, hash, { app: 'appA' })
|
|
404
|
+
const tmp = duplicate + Vault.TMP_SUFFIX
|
|
405
|
+
await writeFile(tmp, 'user data')
|
|
406
|
+
|
|
407
|
+
assert.strictEqual((await vault.convert(duplicate, hash)).status, 'conflict')
|
|
408
|
+
assert.strictEqual(await fs.promises.readFile(tmp, 'utf8'), 'user data')
|
|
409
|
+
|
|
410
|
+
const registeredTmp = original + Vault.TMP_SUFFIX
|
|
411
|
+
await writeFile(registeredTmp, 'other user data')
|
|
119
412
|
await vault.verify()
|
|
120
|
-
assert.strictEqual(fs.
|
|
121
|
-
assert.deepStrictEqual(await fs.promises.readFile(a), content)
|
|
413
|
+
assert.strictEqual(await fs.promises.readFile(registeredTmp, 'utf8'), 'other user data')
|
|
122
414
|
})
|
|
123
415
|
|
|
124
416
|
test('verify: orphan detection via nlink, dead link pruning, store re-adoption', async () => {
|
|
@@ -147,6 +439,143 @@ describe('vault engine (phase 1)', () => {
|
|
|
147
439
|
assert.strictEqual(st.nlink, 2)
|
|
148
440
|
})
|
|
149
441
|
|
|
442
|
+
test('verification preserves registry intent after a transient canonical-path failure', async () => {
|
|
443
|
+
const h = await home()
|
|
444
|
+
const vault = await makeVault(h)
|
|
445
|
+
const content = crypto.randomBytes(4096)
|
|
446
|
+
const hash = sha256(content)
|
|
447
|
+
const file = await writeFile(path.resolve(h, 'api', 'appA', 'm.bin'), content)
|
|
448
|
+
await vault.adopt(file, hash, { app: 'appA' })
|
|
449
|
+
vault.registry.links.get(file).batch_id = 'keep-batch'
|
|
450
|
+
const realRealpath = fs.promises.realpath
|
|
451
|
+
fs.promises.realpath = async (target, options) => {
|
|
452
|
+
if (path.resolve(String(target)) === file) {
|
|
453
|
+
const error = new Error('temporary canonical-path failure')
|
|
454
|
+
error.code = 'EIO'
|
|
455
|
+
throw error
|
|
456
|
+
}
|
|
457
|
+
return realRealpath(target, options)
|
|
458
|
+
}
|
|
459
|
+
try {
|
|
460
|
+
await vault.verify()
|
|
461
|
+
} finally {
|
|
462
|
+
fs.promises.realpath = realRealpath
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
assert.strictEqual(vault.registry.links.get(file).batch_id, 'keep-batch')
|
|
466
|
+
assert.strictEqual(vault.registry.links.get(file).unverified, true)
|
|
467
|
+
assert.ok(vault.registry.blobs.has(hash))
|
|
468
|
+
assert.deepStrictEqual(await fs.promises.readFile(file), content)
|
|
469
|
+
})
|
|
470
|
+
|
|
471
|
+
test('verify re-adopts a trusted linked name without claiming an independent copy', async () => {
|
|
472
|
+
const h = await home()
|
|
473
|
+
const vault = await makeVault(h)
|
|
474
|
+
const content = crypto.randomBytes(4096)
|
|
475
|
+
const hash = sha256(content)
|
|
476
|
+
const linked = await writeFile(path.resolve(h, 'api', 'appA', 'm.bin'), content)
|
|
477
|
+
await writeFile(path.resolve(h, 'api', 'appB', 'm.bin'), content)
|
|
478
|
+
await vault.adopt(linked, hash, { app: 'appA' })
|
|
479
|
+
await fs.promises.unlink(vault.storePathFor(hash))
|
|
480
|
+
|
|
481
|
+
await vault.verify()
|
|
482
|
+
|
|
483
|
+
const storeStat = await fs.promises.stat(vault.storePathFor(hash))
|
|
484
|
+
const linkedStat = await fs.promises.stat(linked)
|
|
485
|
+
assert.strictEqual(storeStat.ino, linkedStat.ino)
|
|
486
|
+
assert.strictEqual(storeStat.dev, linkedStat.dev)
|
|
487
|
+
assert.strictEqual(vault.registry.blobs.get(hash).orphan, false)
|
|
488
|
+
})
|
|
489
|
+
|
|
490
|
+
test('verify never re-adopts a path outside the configured scan sources', async (t) => {
|
|
491
|
+
const h = await home()
|
|
492
|
+
const external = await home()
|
|
493
|
+
const vault = await makeVault(h)
|
|
494
|
+
const content = crypto.randomBytes(4096)
|
|
495
|
+
const hash = sha256(content)
|
|
496
|
+
const managed = await writeFile(path.resolve(h, 'api', 'appA', 'm.bin'), content)
|
|
497
|
+
const outside = path.resolve(external, 'm.bin')
|
|
498
|
+
await vault.adopt(managed, hash, { app: 'appA' })
|
|
499
|
+
try {
|
|
500
|
+
await fs.promises.link(vault.storePathFor(hash), outside)
|
|
501
|
+
} catch (error) {
|
|
502
|
+
if (error.code === 'EXDEV' || error.code === 'ENOTSUP') {
|
|
503
|
+
t.skip(`hardlinks unavailable across test paths: ${error.message}`)
|
|
504
|
+
return
|
|
505
|
+
}
|
|
506
|
+
throw error
|
|
507
|
+
}
|
|
508
|
+
const outsideStat = await fs.promises.stat(outside)
|
|
509
|
+
vault.registry.addLink(outside, {
|
|
510
|
+
hash, source_id: 'external:removed', dev: outsideStat.dev, ino: outsideStat.ino, mode: 'link'
|
|
511
|
+
})
|
|
512
|
+
vault.registry.scanIndex.set(outside, {
|
|
513
|
+
hash, dev: outsideStat.dev, ino: outsideStat.ino, size: outsideStat.size,
|
|
514
|
+
mtime: outsideStat.mtimeMs, ctime: outsideStat.ctimeMs
|
|
515
|
+
})
|
|
516
|
+
await fs.promises.unlink(managed)
|
|
517
|
+
await fs.promises.unlink(vault.storePathFor(hash))
|
|
518
|
+
|
|
519
|
+
await vault.verify()
|
|
520
|
+
|
|
521
|
+
assert.deepStrictEqual(await fs.promises.readFile(outside), content)
|
|
522
|
+
assert.strictEqual((await fs.promises.stat(outside)).nlink, 1)
|
|
523
|
+
assert.strictEqual(fs.existsSync(vault.storePathFor(hash)), false)
|
|
524
|
+
assert.strictEqual(vault.registry.links.has(outside), false)
|
|
525
|
+
assert.strictEqual(vault.registry.blobs.has(hash), false)
|
|
526
|
+
})
|
|
527
|
+
|
|
528
|
+
test('verify never re-adopts changed bytes under a stale hash filename', async () => {
|
|
529
|
+
const h = await home()
|
|
530
|
+
const vault = await makeVault(h)
|
|
531
|
+
const original = Buffer.from('original-content')
|
|
532
|
+
const changed = Buffer.from('modified-content')
|
|
533
|
+
const hash = sha256(original)
|
|
534
|
+
const file = await writeFile(path.resolve(h, 'api', 'appA', 'm.bin'), original)
|
|
535
|
+
await vault.adopt(file, hash, { app: 'appA' })
|
|
536
|
+
await fs.promises.unlink(vault.storePathFor(hash))
|
|
537
|
+
await fs.promises.writeFile(file, changed)
|
|
538
|
+
const future = new Date(Date.now() + 2000)
|
|
539
|
+
await fs.promises.utimes(file, future, future)
|
|
540
|
+
|
|
541
|
+
await vault.verify()
|
|
542
|
+
|
|
543
|
+
assert.strictEqual(fs.existsSync(vault.storePathFor(hash)), false)
|
|
544
|
+
assert.strictEqual(vault.registry.blobs.has(hash), false)
|
|
545
|
+
assert.deepStrictEqual(await fs.promises.readFile(file), changed)
|
|
546
|
+
})
|
|
547
|
+
|
|
548
|
+
test('verify rejects a source replaced during store re-adoption', async () => {
|
|
549
|
+
const h = await home()
|
|
550
|
+
const vault = await makeVault(h)
|
|
551
|
+
const original = Buffer.alloc(4096, 1)
|
|
552
|
+
const replacement = Buffer.alloc(4096, 2)
|
|
553
|
+
const hash = sha256(original)
|
|
554
|
+
const file = await writeFile(path.resolve(h, 'api', 'appA', 'm.bin'), original)
|
|
555
|
+
const storePath = vault.storePathFor(hash)
|
|
556
|
+
await vault.adopt(file, hash, { app: 'appA' })
|
|
557
|
+
await fs.promises.unlink(storePath)
|
|
558
|
+
const realLink = fs.promises.link
|
|
559
|
+
fs.promises.link = async (source, target) => {
|
|
560
|
+
if (source === file && target === storePath) {
|
|
561
|
+
const writerPath = file + '.writer-replacement'
|
|
562
|
+
await fs.promises.writeFile(writerPath, replacement)
|
|
563
|
+
await fs.promises.rename(writerPath, file)
|
|
564
|
+
}
|
|
565
|
+
return realLink(source, target)
|
|
566
|
+
}
|
|
567
|
+
try {
|
|
568
|
+
await vault.verify()
|
|
569
|
+
} finally {
|
|
570
|
+
fs.promises.link = realLink
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
assert.deepStrictEqual(await fs.promises.readFile(file), replacement)
|
|
574
|
+
assert.strictEqual(fs.existsSync(storePath), false)
|
|
575
|
+
assert.strictEqual(vault.registry.blobs.has(hash), false)
|
|
576
|
+
assert.strictEqual(vault.registry.links.has(file), false)
|
|
577
|
+
})
|
|
578
|
+
|
|
150
579
|
test('reclaim: refuses while in use, frees orphans', async () => {
|
|
151
580
|
const h = await home()
|
|
152
581
|
const vault = await makeVault(h)
|
|
@@ -155,19 +584,170 @@ describe('vault engine (phase 1)', () => {
|
|
|
155
584
|
const a = await writeFile(path.resolve(h, 'api', 'appA', 'm.bin'), content)
|
|
156
585
|
await vault.adopt(a, hash)
|
|
157
586
|
assert.strictEqual((await vault.reclaim(hash)).status, 'in-use')
|
|
587
|
+
const pending = await writeFile(path.resolve(h, 'api', 'appB', 'm.bin'), content)
|
|
588
|
+
const pendingStat = await fs.promises.stat(pending)
|
|
589
|
+
vault.registry.setDuplicate(pending, {
|
|
590
|
+
hash, size: pendingStat.size, dev: pendingStat.dev, ino: pendingStat.ino,
|
|
591
|
+
mtime: pendingStat.mtimeMs, ctime: pendingStat.ctimeMs
|
|
592
|
+
})
|
|
593
|
+
vault.registry.scanIndex.set(pending, {
|
|
594
|
+
hash, size: pendingStat.size, dev: pendingStat.dev, ino: pendingStat.ino,
|
|
595
|
+
mtime: pendingStat.mtimeMs, ctime: pendingStat.ctimeMs
|
|
596
|
+
})
|
|
158
597
|
await fs.promises.unlink(a)
|
|
159
598
|
const result = await vault.reclaim(hash)
|
|
160
599
|
assert.strictEqual(result.status, 'reclaimed')
|
|
161
600
|
assert.strictEqual(result.bytes_freed, content.length)
|
|
162
601
|
assert.strictEqual(fs.existsSync(vault.storePathFor(hash)), false)
|
|
163
602
|
assert.strictEqual(vault.registry.blobs.has(hash), false)
|
|
603
|
+
assert.strictEqual(vault.registry.duplicates.has(pending), false,
|
|
604
|
+
'pending rows cannot reference a Vault copy that was reclaimed')
|
|
605
|
+
assert.strictEqual(vault.registry.scanIndex.has(pending), false)
|
|
606
|
+
})
|
|
607
|
+
|
|
608
|
+
test('vault content identifiers cannot escape the managed file tree', async () => {
|
|
609
|
+
const h = await home()
|
|
610
|
+
const vault = await makeVault(h)
|
|
611
|
+
const sentinel = await writeFile(path.resolve(h, 'do-not-remove.txt'), 'safe')
|
|
612
|
+
|
|
613
|
+
await assert.rejects(vault.reclaim('../../do-not-remove.txt'), /Invalid vault content identifier/)
|
|
614
|
+
assert.strictEqual(await fs.promises.readFile(sentinel, 'utf8'), 'safe')
|
|
164
615
|
})
|
|
165
616
|
|
|
166
|
-
test('
|
|
617
|
+
test('vault storage symlinks never redirect initialization or reclaim', {
|
|
618
|
+
skip: process.platform === 'win32'
|
|
619
|
+
}, async () => {
|
|
620
|
+
const h = await home()
|
|
621
|
+
const outside = await home()
|
|
622
|
+
const redirectedRoot = path.resolve(outside, 'redirected-vault')
|
|
623
|
+
await fs.promises.mkdir(redirectedRoot)
|
|
624
|
+
await fs.promises.symlink(redirectedRoot, path.resolve(h, 'vault'))
|
|
625
|
+
|
|
626
|
+
const redirected = new Vault(fakeKernel(h))
|
|
627
|
+
await assert.rejects(redirected.init(), (error) => error && error.code === 'EVAULTPATH')
|
|
628
|
+
assert.deepStrictEqual(await fs.promises.readdir(redirectedRoot), [])
|
|
629
|
+
|
|
630
|
+
await fs.promises.unlink(path.resolve(h, 'vault'))
|
|
631
|
+
const vault = await makeVault(h)
|
|
632
|
+
const content = Buffer.from('preserve external bytes')
|
|
633
|
+
const hash = sha256(content)
|
|
634
|
+
const outsideShard = path.resolve(outside, 'outside-shard')
|
|
635
|
+
const outsideBlob = await writeFile(path.resolve(outsideShard, hash), content)
|
|
636
|
+
await fs.promises.symlink(outsideShard, path.resolve(vault.blobRoot, hash.slice(0, 2)))
|
|
637
|
+
vault.registry.addBlob(hash, { size: content.length })
|
|
638
|
+
|
|
639
|
+
await assert.rejects(vault.reclaim(hash), (error) => error && error.code === 'EVAULTPATH')
|
|
640
|
+
assert.deepStrictEqual(await fs.promises.readFile(outsideBlob), content)
|
|
641
|
+
})
|
|
642
|
+
|
|
643
|
+
test('deduplication refuses a file changed after discovery, even at the same size', async () => {
|
|
644
|
+
const h = await home()
|
|
645
|
+
const vault = await makeVault(h)
|
|
646
|
+
vault.sizeThreshold = 1
|
|
647
|
+
const originalContent = Buffer.from('first version')
|
|
648
|
+
const changedContent = Buffer.from('other version')
|
|
649
|
+
assert.strictEqual(originalContent.length, changedContent.length)
|
|
650
|
+
await writeFile(path.resolve(h, 'api', 'appA', 'model.bin'), originalContent)
|
|
651
|
+
await writeFile(path.resolve(h, 'api', 'appB', 'model.bin'), originalContent)
|
|
652
|
+
await vault.sweeper.scan()
|
|
653
|
+
|
|
654
|
+
const [pendingPath] = vault.registry.duplicates.keys()
|
|
655
|
+
assert.ok(pendingPath, 'scan found a pending duplicate')
|
|
656
|
+
await fs.promises.writeFile(pendingPath, changedContent)
|
|
657
|
+
const future = new Date(Date.now() + 2000)
|
|
658
|
+
await fs.promises.utimes(pendingPath, future, future)
|
|
659
|
+
|
|
660
|
+
const result = await vault.perform('deduplicate', {
|
|
661
|
+
scope_id: vault.registry.duplicates.get(pendingPath).source_id,
|
|
662
|
+
paths: [pendingPath]
|
|
663
|
+
})
|
|
664
|
+
assert.strictEqual(result.stale, 1)
|
|
665
|
+
assert.strictEqual(result.converted, 0)
|
|
666
|
+
assert.ok(vault.registry.duplicates.has(pendingPath), 'changed file remains pending for a fresh scan')
|
|
667
|
+
assert.deepStrictEqual(await fs.promises.readFile(pendingPath), changedContent)
|
|
668
|
+
})
|
|
669
|
+
|
|
670
|
+
test('deduplication revalidates canonical content before replacing a pending file', async () => {
|
|
671
|
+
const h = await home()
|
|
672
|
+
const vault = await makeVault(h)
|
|
673
|
+
vault.sizeThreshold = 1
|
|
674
|
+
const originalContent = Buffer.from('AAAA')
|
|
675
|
+
const changedContent = Buffer.from('BBBB')
|
|
676
|
+
const original = await writeFile(path.resolve(h, 'api', 'appA', 'model.bin'), originalContent)
|
|
677
|
+
const pending = await writeFile(path.resolve(h, 'api', 'appB', 'model.bin'), originalContent)
|
|
678
|
+
await vault.sweeper.scan()
|
|
679
|
+
await fs.promises.writeFile(original, changedContent)
|
|
680
|
+
|
|
681
|
+
const pendingEntry = vault.registry.duplicates.get(pending)
|
|
682
|
+
const result = await vault.perform('deduplicate', {
|
|
683
|
+
scope_id: pendingEntry.source_id,
|
|
684
|
+
paths: [pending]
|
|
685
|
+
})
|
|
686
|
+
|
|
687
|
+
assert.strictEqual(result.converted, 0)
|
|
688
|
+
assert.strictEqual(result.stale, 1)
|
|
689
|
+
assert.deepStrictEqual(await fs.promises.readFile(pending), originalContent)
|
|
690
|
+
assert.ok(vault.registry.duplicates.has(pending))
|
|
691
|
+
})
|
|
692
|
+
|
|
693
|
+
test('deduplication preserves executable and ownership metadata semantics', async () => {
|
|
694
|
+
const h = await home()
|
|
695
|
+
const vault = await makeVault(h)
|
|
696
|
+
vault.sizeThreshold = 1
|
|
697
|
+
const content = crypto.randomBytes(4096)
|
|
698
|
+
const original = await writeFile(path.resolve(h, 'api', 'appA', 'model.bin'), content)
|
|
699
|
+
const pending = await writeFile(path.resolve(h, 'api', 'appB', 'model.bin'), content)
|
|
700
|
+
if (process.platform !== 'win32') {
|
|
701
|
+
await fs.promises.chmod(original, 0o755)
|
|
702
|
+
await fs.promises.chmod(pending, 0o600)
|
|
703
|
+
}
|
|
704
|
+
await vault.sweeper.scan()
|
|
705
|
+
const before = await fs.promises.stat(pending)
|
|
706
|
+
const pendingEntry = vault.registry.duplicates.get(pending)
|
|
707
|
+
const result = await vault.perform('deduplicate', {
|
|
708
|
+
scope_id: pendingEntry.source_id,
|
|
709
|
+
paths: [pending]
|
|
710
|
+
})
|
|
711
|
+
const after = await fs.promises.stat(pending)
|
|
712
|
+
|
|
713
|
+
if (process.platform === 'win32') assert.ok(result.converted >= 0)
|
|
714
|
+
else {
|
|
715
|
+
assert.strictEqual(result.converted, 0)
|
|
716
|
+
assert.strictEqual(result.incompatible, 1)
|
|
717
|
+
assert.strictEqual(after.mode & 0o7777, before.mode & 0o7777)
|
|
718
|
+
assert.strictEqual(after.ino, before.ino)
|
|
719
|
+
assert.ok(vault.registry.duplicates.has(pending))
|
|
720
|
+
}
|
|
721
|
+
})
|
|
722
|
+
|
|
723
|
+
test('deduplication requires a trustworthy scan snapshot', async () => {
|
|
724
|
+
const h = await home()
|
|
725
|
+
const vault = await makeVault(h)
|
|
726
|
+
const content = crypto.randomBytes(4096)
|
|
727
|
+
const hash = sha256(content)
|
|
728
|
+
const original = await writeFile(path.resolve(h, 'api', 'appA', 'model.bin'), content)
|
|
729
|
+
const pending = await writeFile(path.resolve(h, 'api', 'appB', 'model.bin'), content)
|
|
730
|
+
await vault.refreshSources()
|
|
731
|
+
const appA = vault.sources().find((source) => source.kind === 'app' && source.app === 'appA')
|
|
732
|
+
const appB = vault.sources().find((source) => source.kind === 'app' && source.app === 'appB')
|
|
733
|
+
await vault.adopt(original, hash, { app: 'appA', source_id: appA.id })
|
|
734
|
+
const pendingStat = await fs.promises.stat(pending)
|
|
735
|
+
vault.registry.duplicates.set(pending, {
|
|
736
|
+
hash, size: content.length, app: 'appB', source_id: appB.id,
|
|
737
|
+
dev: pendingStat.dev, ino: pendingStat.ino, mtime: pendingStat.mtimeMs
|
|
738
|
+
})
|
|
739
|
+
|
|
740
|
+
const result = await vault.perform('deduplicate', { scope_id: appB.id, paths: [pending] })
|
|
741
|
+
|
|
742
|
+
assert.strictEqual(result.stale, 1)
|
|
743
|
+
assert.strictEqual(result.converted, 0)
|
|
744
|
+
assert.strictEqual((await fs.promises.stat(pending)).nlink, 1)
|
|
745
|
+
assert.ok(vault.registry.duplicates.has(pending))
|
|
746
|
+
})
|
|
747
|
+
|
|
748
|
+
test('item 9: registry deleted => startup recovers only managed blobs', async () => {
|
|
167
749
|
const h = await home()
|
|
168
750
|
const vault = await makeVault(h)
|
|
169
|
-
// Rebuild's ino-match walk only considers files >= SIZE_THRESHOLD, so
|
|
170
|
-
// lower the threshold surrogate by using large-enough sparse-ish files.
|
|
171
751
|
const big = Buffer.alloc(Vault.SIZE_THRESHOLD, 7)
|
|
172
752
|
const hash = sha256(big)
|
|
173
753
|
const a = await writeFile(path.resolve(h, 'api', 'appA', 'big.bin'), big)
|
|
@@ -180,19 +760,17 @@ describe('vault engine (phase 1)', () => {
|
|
|
180
760
|
|
|
181
761
|
await fs.promises.unlink(path.resolve(vault.root, 'registry.json')).catch(() => {})
|
|
182
762
|
await fs.promises.unlink(path.resolve(vault.root, 'events.ndjson')).catch(() => {})
|
|
183
|
-
const fresh =
|
|
184
|
-
await fresh.
|
|
763
|
+
const fresh = new Vault(fakeKernel(h))
|
|
764
|
+
const initResult = await fresh.init()
|
|
765
|
+
assert.strictEqual(initResult.missing_recovered, true)
|
|
185
766
|
assert.ok(fresh.registry.blobs.has(hash))
|
|
186
767
|
assert.ok(fresh.registry.blobs.has(orphanHash))
|
|
187
768
|
assert.strictEqual(fresh.registry.blobs.get(orphanHash).orphan, true)
|
|
188
|
-
assert.strictEqual(fresh.registry.blobs.get(hash).orphan, false
|
|
189
|
-
|
|
190
|
-
assert.strictEqual(fresh.registry.links.
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
false,
|
|
194
|
-
'the internal vault name is never exposed as a tracked location'
|
|
195
|
-
)
|
|
769
|
+
assert.strictEqual(fresh.registry.blobs.get(hash).orphan, false,
|
|
770
|
+
'the visible hardlink still keeps the managed inode live')
|
|
771
|
+
assert.strictEqual(fresh.registry.links.size, 0,
|
|
772
|
+
'startup recovery never walks visible source trees')
|
|
773
|
+
assert.strictEqual(fresh.registry.links.has(a), false)
|
|
196
774
|
})
|
|
197
775
|
|
|
198
776
|
test('item 9: torn events line tolerated; corrupt registry.json triggers rebuild, not failure', async () => {
|
|
@@ -216,6 +794,76 @@ describe('vault engine (phase 1)', () => {
|
|
|
216
794
|
assert.ok(fresh.registry.blobs.has(hash))
|
|
217
795
|
})
|
|
218
796
|
|
|
797
|
+
test('parseable non-object registry and event records are treated as corrupt cache data', async () => {
|
|
798
|
+
const h = await home()
|
|
799
|
+
const vault = await makeVault(h)
|
|
800
|
+
const content = crypto.randomBytes(4096)
|
|
801
|
+
const hash = sha256(content)
|
|
802
|
+
const file = await writeFile(path.resolve(h, 'api', 'appA', 'm.bin'), content)
|
|
803
|
+
await vault.adopt(file, hash)
|
|
804
|
+
await vault.registry.flush()
|
|
805
|
+
await fs.promises.writeFile(vault.registry.snapshotPath, 'null')
|
|
806
|
+
await fs.promises.writeFile(vault.registry.eventsPath, 'null\n{"kind":"found"}\n')
|
|
807
|
+
|
|
808
|
+
const fresh = new Vault(fakeKernel(h))
|
|
809
|
+
const result = await fresh.init()
|
|
810
|
+
const events = await fresh.registry.readEvents()
|
|
811
|
+
|
|
812
|
+
assert.strictEqual(result.corrupt_recovered, true)
|
|
813
|
+
assert.strictEqual(fresh.registry.blobs.has(hash), true)
|
|
814
|
+
assert.deepStrictEqual(events.map((event) => event.kind), ['found'])
|
|
815
|
+
})
|
|
816
|
+
|
|
817
|
+
test('transient registry read errors never become an empty replacement registry', async () => {
|
|
818
|
+
const h = await home()
|
|
819
|
+
const vault = await makeVault(h)
|
|
820
|
+
const file = await writeFile(path.resolve(h, 'api', 'appA', 'm.bin'), crypto.randomBytes(4096))
|
|
821
|
+
await vault.adopt(file, sha256(await fs.promises.readFile(file)), { app: 'appA' })
|
|
822
|
+
vault.registry.excluded.set(path.resolve(h, 'keep-independent.bin'), { ts: Date.now(), size: 123 })
|
|
823
|
+
await vault.registry.flush()
|
|
824
|
+
const registryPath = path.resolve(vault.root, 'registry.json')
|
|
825
|
+
const before = await fs.promises.readFile(registryPath, 'utf8')
|
|
826
|
+
const realReadFile = Registry.prototype.readFile
|
|
827
|
+
let fresh
|
|
828
|
+
Registry.prototype.readFile = async function (target, ...args) {
|
|
829
|
+
if (path.resolve(String(target)) === registryPath) {
|
|
830
|
+
const error = new Error('temporary read failure')
|
|
831
|
+
error.code = 'EIO'
|
|
832
|
+
throw error
|
|
833
|
+
}
|
|
834
|
+
return realReadFile.call(this, target, ...args)
|
|
835
|
+
}
|
|
836
|
+
try {
|
|
837
|
+
fresh = new Vault(fakeKernel(h))
|
|
838
|
+
await assert.rejects(fresh.init(), (error) => error && error.code === 'EIO')
|
|
839
|
+
} finally {
|
|
840
|
+
Registry.prototype.readFile = realReadFile
|
|
841
|
+
}
|
|
842
|
+
assert.strictEqual(await fs.promises.readFile(registryPath, 'utf8'), before)
|
|
843
|
+
await fresh.ensureInitialized()
|
|
844
|
+
assert.strictEqual(fresh.registry.excluded.has(path.resolve(h, 'keep-independent.bin')), true)
|
|
845
|
+
})
|
|
846
|
+
|
|
847
|
+
test('transient event-log read errors do not cache or compact an empty history', async () => {
|
|
848
|
+
const h = await home()
|
|
849
|
+
const vault = await makeVault(h)
|
|
850
|
+
await vault.registry.appendEvent({ kind: 'found', marker: 'preserve-me' })
|
|
851
|
+
vault.registry.eventsCache = null
|
|
852
|
+
const realReadEventTail = vault.registry.readEventTail
|
|
853
|
+
vault.registry.readEventTail = async () => {
|
|
854
|
+
const error = new Error('temporary read failure')
|
|
855
|
+
error.code = 'EIO'
|
|
856
|
+
throw error
|
|
857
|
+
}
|
|
858
|
+
try {
|
|
859
|
+
await assert.rejects(vault.registry.readEvents(), (error) => error && error.code === 'EIO')
|
|
860
|
+
} finally {
|
|
861
|
+
vault.registry.readEventTail = realReadEventTail
|
|
862
|
+
}
|
|
863
|
+
const events = await vault.registry.readEvents()
|
|
864
|
+
assert.strictEqual(events.some((event) => event.marker === 'preserve-me'), true)
|
|
865
|
+
})
|
|
866
|
+
|
|
219
867
|
test('registry snapshot flushes serialize without racing the shared temp file', async () => {
|
|
220
868
|
const h = await home()
|
|
221
869
|
const vault = await makeVault(h)
|
|
@@ -229,6 +877,408 @@ describe('vault engine (phase 1)', () => {
|
|
|
229
877
|
assert.strictEqual(fs.existsSync(path.resolve(vault.root, 'registry.json.tmp')), false)
|
|
230
878
|
})
|
|
231
879
|
|
|
880
|
+
test('a failed registry flush stays dirty and retries the in-memory state', async () => {
|
|
881
|
+
const h = await home()
|
|
882
|
+
const vault = await makeVault(h)
|
|
883
|
+
const hash = 'd'.repeat(64)
|
|
884
|
+
const newerHash = 'e'.repeat(64)
|
|
885
|
+
const realAtomicWrite = vault.registry.atomicWrite.bind(vault.registry)
|
|
886
|
+
let writes = 0
|
|
887
|
+
vault.registry.persistDelay = 5
|
|
888
|
+
vault.registry.atomicWrite = async (...args) => {
|
|
889
|
+
writes += 1
|
|
890
|
+
if (writes === 1) {
|
|
891
|
+
const error = new Error('temporary write failure')
|
|
892
|
+
error.code = 'EIO'
|
|
893
|
+
throw error
|
|
894
|
+
}
|
|
895
|
+
return realAtomicWrite(...args)
|
|
896
|
+
}
|
|
897
|
+
|
|
898
|
+
try {
|
|
899
|
+
vault.registry.addBlob(hash, { size: 123 })
|
|
900
|
+
await assert.rejects(vault.registry.flush(), (error) => error && error.code === 'EIO')
|
|
901
|
+
assert.strictEqual(vault.registry.persistDirty, true)
|
|
902
|
+
assert.ok(vault.registry.persistTimer, 'failed write scheduled a retry')
|
|
903
|
+
vault.registry.addBlob(newerHash, { size: 456 })
|
|
904
|
+
|
|
905
|
+
const deadline = Date.now() + 1000
|
|
906
|
+
while ((writes < 2 || !fs.existsSync(vault.registry.snapshotPath)) && Date.now() < deadline) {
|
|
907
|
+
await new Promise((resolve) => setTimeout(resolve, 10))
|
|
908
|
+
}
|
|
909
|
+
assert.ok(writes >= 2, 'registry write was retried')
|
|
910
|
+
assert.strictEqual(vault.registry.persistDirty, false)
|
|
911
|
+
const snapshot = JSON.parse(await fs.promises.readFile(vault.registry.snapshotPath, 'utf8'))
|
|
912
|
+
assert.strictEqual(snapshot.blobs[hash].size, 123)
|
|
913
|
+
assert.strictEqual(snapshot.blobs[newerHash].size, 456)
|
|
914
|
+
} finally {
|
|
915
|
+
vault.registry.atomicWrite = realAtomicWrite
|
|
916
|
+
}
|
|
917
|
+
})
|
|
918
|
+
|
|
919
|
+
test('registry writes preserve a temporary path replaced before rename', async () => {
|
|
920
|
+
const h = await home()
|
|
921
|
+
const vault = await makeVault(h)
|
|
922
|
+
vault.registry.addBlob('b'.repeat(64), { size: 1 })
|
|
923
|
+
await vault.registry.flush()
|
|
924
|
+
const before = await fs.promises.readFile(vault.registry.snapshotPath, 'utf8')
|
|
925
|
+
const realLstat = fs.promises.lstat
|
|
926
|
+
let tempPath = null
|
|
927
|
+
let tempReads = 0
|
|
928
|
+
fs.promises.lstat = async (target, options) => {
|
|
929
|
+
const resolved = path.resolve(String(target))
|
|
930
|
+
if (path.dirname(resolved) === vault.root && path.basename(resolved).startsWith('.registry.json.')) {
|
|
931
|
+
tempPath = resolved
|
|
932
|
+
tempReads += 1
|
|
933
|
+
if (tempReads === 2) {
|
|
934
|
+
await fs.promises.rename(resolved, `${resolved}.saved`)
|
|
935
|
+
await fs.promises.writeFile(resolved, 'unrelated')
|
|
936
|
+
}
|
|
937
|
+
}
|
|
938
|
+
return realLstat(target, options)
|
|
939
|
+
}
|
|
940
|
+
try {
|
|
941
|
+
await assert.rejects(
|
|
942
|
+
vault.registry.atomicWrite(vault.registry.snapshotPath, '{"replacement":true}\n'),
|
|
943
|
+
(error) => error && error.code === 'EVAULTPATH'
|
|
944
|
+
)
|
|
945
|
+
} finally {
|
|
946
|
+
fs.promises.lstat = realLstat
|
|
947
|
+
}
|
|
948
|
+
|
|
949
|
+
assert.strictEqual(await fs.promises.readFile(vault.registry.snapshotPath, 'utf8'), before)
|
|
950
|
+
assert.strictEqual(await fs.promises.readFile(tempPath, 'utf8'), 'unrelated')
|
|
951
|
+
})
|
|
952
|
+
|
|
953
|
+
test('registry files cannot redirect writes through symlinks', {
|
|
954
|
+
skip: process.platform === 'win32'
|
|
955
|
+
}, async () => {
|
|
956
|
+
const h = await home()
|
|
957
|
+
const vault = await makeVault(h)
|
|
958
|
+
const externalEvent = await writeFile(path.resolve(h, 'external-event.txt'), 'event sentinel')
|
|
959
|
+
const externalSnapshot = await writeFile(path.resolve(h, 'external-snapshot.txt'), 'snapshot sentinel')
|
|
960
|
+
await fs.promises.symlink(externalEvent, vault.registry.eventsPath)
|
|
961
|
+
await fs.promises.symlink(externalSnapshot, vault.registry.snapshotPath)
|
|
962
|
+
|
|
963
|
+
await assert.rejects(
|
|
964
|
+
vault.registry.appendEvent({ kind: 'found' }),
|
|
965
|
+
(error) => error && error.code === 'EVAULTPATH'
|
|
966
|
+
)
|
|
967
|
+
assert.match(vault.registry.eventError, /not safe/i)
|
|
968
|
+
vault.registry.addBlob('a'.repeat(64), { size: 1 })
|
|
969
|
+
await vault.registry.flush()
|
|
970
|
+
|
|
971
|
+
assert.strictEqual(await fs.promises.readFile(externalEvent, 'utf8'), 'event sentinel')
|
|
972
|
+
assert.strictEqual(await fs.promises.readFile(externalSnapshot, 'utf8'), 'snapshot sentinel')
|
|
973
|
+
assert.strictEqual((await fs.promises.lstat(vault.registry.snapshotPath)).isFile(), true)
|
|
974
|
+
})
|
|
975
|
+
|
|
976
|
+
test('loading an exclusion removes its stale derived scan classification', async () => {
|
|
977
|
+
const h = await home()
|
|
978
|
+
const vault = await makeVault(h)
|
|
979
|
+
const file = await writeFile(path.resolve(h, 'api', 'appA', 'model.bin'), crypto.randomBytes(4096))
|
|
980
|
+
const st = await fs.promises.stat(file)
|
|
981
|
+
vault.registry.scanIndex.set(file, {
|
|
982
|
+
hash: sha256(await fs.promises.readFile(file)), size: st.size,
|
|
983
|
+
dev: st.dev, ino: st.ino, mtime: st.mtimeMs, ctime: st.ctimeMs
|
|
984
|
+
})
|
|
985
|
+
vault.registry.excluded.set(file, { ts: Date.now(), size: st.size })
|
|
986
|
+
await vault.registry.flush()
|
|
987
|
+
|
|
988
|
+
const reloaded = await makeVault(h)
|
|
989
|
+
assert.ok(reloaded.registry.excluded.has(file))
|
|
990
|
+
assert.strictEqual(reloaded.registry.scanIndex.has(file), false)
|
|
991
|
+
})
|
|
992
|
+
|
|
993
|
+
test('event appends use one ordered writer', async () => {
|
|
994
|
+
const h = await home()
|
|
995
|
+
const vault = await makeVault(h)
|
|
996
|
+
const realAppendFile = vault.registry.appendFile
|
|
997
|
+
let active = 0
|
|
998
|
+
let maxActive = 0
|
|
999
|
+
vault.registry.appendFile = async (...args) => {
|
|
1000
|
+
active += 1
|
|
1001
|
+
maxActive = Math.max(maxActive, active)
|
|
1002
|
+
await new Promise((resolve) => setTimeout(resolve, 5))
|
|
1003
|
+
try {
|
|
1004
|
+
return await realAppendFile.call(vault.registry, ...args)
|
|
1005
|
+
} finally {
|
|
1006
|
+
active -= 1
|
|
1007
|
+
}
|
|
1008
|
+
}
|
|
1009
|
+
try {
|
|
1010
|
+
await Promise.all(Array.from({ length: 12 }, (_, index) => vault.registry.appendEvent({ kind: 'found', index })))
|
|
1011
|
+
} finally {
|
|
1012
|
+
vault.registry.appendFile = realAppendFile
|
|
1013
|
+
}
|
|
1014
|
+
assert.strictEqual(maxActive, 1)
|
|
1015
|
+
assert.deepStrictEqual((await vault.registry.readEvents()).map((event) => event.index), [...Array(12).keys()])
|
|
1016
|
+
})
|
|
1017
|
+
|
|
1018
|
+
test('activity history compacts to a bounded recent window', async () => {
|
|
1019
|
+
const h = await home()
|
|
1020
|
+
const vault = await makeVault(h)
|
|
1021
|
+
vault.registry.maxEvents = 5
|
|
1022
|
+
vault.registry.compactEvery = 2
|
|
1023
|
+
|
|
1024
|
+
for (let index = 0; index < 9; index++) {
|
|
1025
|
+
await vault.registry.appendEvent({ kind: 'found', index })
|
|
1026
|
+
}
|
|
1027
|
+
|
|
1028
|
+
let readWrites = 0
|
|
1029
|
+
const realAtomicWrite = vault.registry.atomicWrite
|
|
1030
|
+
vault.registry.atomicWrite = async (...args) => {
|
|
1031
|
+
readWrites += 1
|
|
1032
|
+
return realAtomicWrite.call(vault.registry, ...args)
|
|
1033
|
+
}
|
|
1034
|
+
const events = await vault.registry.readEvents()
|
|
1035
|
+
vault.registry.atomicWrite = realAtomicWrite
|
|
1036
|
+
assert.deepStrictEqual(events.map((event) => event.index), [4, 5, 6, 7, 8])
|
|
1037
|
+
assert.strictEqual(readWrites, 0)
|
|
1038
|
+
const persisted = (await fs.promises.readFile(vault.registry.eventsPath, 'utf8')).trim().split('\n')
|
|
1039
|
+
assert.ok(persisted.length >= vault.registry.maxEvents)
|
|
1040
|
+
assert.ok(persisted.length <= vault.registry.maxEvents + vault.registry.compactEvery)
|
|
1041
|
+
})
|
|
1042
|
+
|
|
1043
|
+
test('activity reads only a bounded tail of an oversized history', async () => {
|
|
1044
|
+
const h = await home()
|
|
1045
|
+
const vault = await makeVault(h)
|
|
1046
|
+
vault.registry.maxEvents = 2
|
|
1047
|
+
vault.registry.maxEventBytesPerEntry = 32
|
|
1048
|
+
const oversized = JSON.stringify({ kind: 'old', payload: 'x'.repeat(80 * 1024) })
|
|
1049
|
+
await fs.promises.writeFile(vault.registry.eventsPath,
|
|
1050
|
+
`${oversized}\n${JSON.stringify({ kind: 'found', index: 1 })}\n${JSON.stringify({ kind: 'found', index: 2 })}\n`)
|
|
1051
|
+
vault.registry.eventsCache = null
|
|
1052
|
+
|
|
1053
|
+
const events = await vault.registry.readEvents()
|
|
1054
|
+
|
|
1055
|
+
assert.deepStrictEqual(events.map((event) => event.index), [1, 2])
|
|
1056
|
+
})
|
|
1057
|
+
|
|
1058
|
+
test('capability probing preserves a colliding path it did not create', async () => {
|
|
1059
|
+
const h = await home()
|
|
1060
|
+
const vault = await makeVault(h)
|
|
1061
|
+
vault.volumeModes.clear()
|
|
1062
|
+
const realLink = fs.promises.link
|
|
1063
|
+
let collision = null
|
|
1064
|
+
fs.promises.link = async (source, target) => {
|
|
1065
|
+
collision = target
|
|
1066
|
+
await fs.promises.writeFile(target, 'unrelated', { flag: 'wx' })
|
|
1067
|
+
const error = new Error('occupied')
|
|
1068
|
+
error.code = 'EEXIST'
|
|
1069
|
+
throw error
|
|
1070
|
+
}
|
|
1071
|
+
try {
|
|
1072
|
+
await assert.rejects(vault.probe(h), (error) => error && error.code === 'EEXIST')
|
|
1073
|
+
} finally {
|
|
1074
|
+
fs.promises.link = realLink
|
|
1075
|
+
}
|
|
1076
|
+
|
|
1077
|
+
assert.strictEqual(await fs.promises.readFile(collision, 'utf8'), 'unrelated')
|
|
1078
|
+
assert.strictEqual(vault.volumeModes.size, 0)
|
|
1079
|
+
assert.strictEqual(await vault.probe(h), 'link')
|
|
1080
|
+
})
|
|
1081
|
+
|
|
1082
|
+
test('a transient capability probe failure is retryable and never cached as copy mode', async () => {
|
|
1083
|
+
const h = await home()
|
|
1084
|
+
const vault = await makeVault(h)
|
|
1085
|
+
vault.volumeModes.clear()
|
|
1086
|
+
const realLink = fs.promises.link
|
|
1087
|
+
fs.promises.link = async () => {
|
|
1088
|
+
const error = new Error('temporary I/O failure')
|
|
1089
|
+
error.code = 'EIO'
|
|
1090
|
+
throw error
|
|
1091
|
+
}
|
|
1092
|
+
try {
|
|
1093
|
+
await assert.rejects(vault.probe(h), (error) => error && error.code === 'EIO')
|
|
1094
|
+
} finally {
|
|
1095
|
+
fs.promises.link = realLink
|
|
1096
|
+
}
|
|
1097
|
+
|
|
1098
|
+
assert.strictEqual(vault.volumeModes.size, 0)
|
|
1099
|
+
assert.strictEqual(await vault.probe(h), 'link')
|
|
1100
|
+
})
|
|
1101
|
+
|
|
1102
|
+
test('dashboard mutations are serialized in request order', async () => {
|
|
1103
|
+
const h = await home()
|
|
1104
|
+
const vault = await makeVault(h)
|
|
1105
|
+
const order = []
|
|
1106
|
+
const first = vault.runExclusive(async () => {
|
|
1107
|
+
order.push('first:start')
|
|
1108
|
+
await new Promise((resolve) => setTimeout(resolve, 15))
|
|
1109
|
+
order.push('first:end')
|
|
1110
|
+
})
|
|
1111
|
+
const second = vault.runExclusive(async () => {
|
|
1112
|
+
order.push('second:start')
|
|
1113
|
+
order.push('second:end')
|
|
1114
|
+
})
|
|
1115
|
+
await Promise.all([first, second])
|
|
1116
|
+
assert.deepStrictEqual(order, ['first:start', 'first:end', 'second:start', 'second:end'])
|
|
1117
|
+
})
|
|
1118
|
+
|
|
1119
|
+
test('each deduplicate request receives a collision-proof undo batch id', async () => {
|
|
1120
|
+
const h = await home()
|
|
1121
|
+
const vault = await makeVault(h)
|
|
1122
|
+
const batches = []
|
|
1123
|
+
vault.deduplicateScope = async (scopeId, options) => {
|
|
1124
|
+
batches.push(options.batch_id)
|
|
1125
|
+
return { converted: 0 }
|
|
1126
|
+
}
|
|
1127
|
+
|
|
1128
|
+
await Promise.all([
|
|
1129
|
+
vault.perform('deduplicate', { scope_id: 'app:a' }),
|
|
1130
|
+
vault.perform('deduplicate', { scope_id: 'app:a' })
|
|
1131
|
+
])
|
|
1132
|
+
|
|
1133
|
+
assert.strictEqual(new Set(batches).size, 2)
|
|
1134
|
+
assert.ok(batches.every((batch) => /^batch-[0-9a-f-]{36}$/.test(batch)))
|
|
1135
|
+
})
|
|
1136
|
+
|
|
1137
|
+
test('deduplicate batches expose queued and exact completed file progress', async () => {
|
|
1138
|
+
const h = await home()
|
|
1139
|
+
const vault = await makeVault(h)
|
|
1140
|
+
vault.sizeThreshold = 1
|
|
1141
|
+
const files = []
|
|
1142
|
+
for (let index = 0; index < 3; index += 1) {
|
|
1143
|
+
const content = crypto.randomBytes(4096)
|
|
1144
|
+
files.push({
|
|
1145
|
+
content,
|
|
1146
|
+
hash: sha256(content),
|
|
1147
|
+
original: await writeFile(path.resolve(h, 'api', 'appA', `${index}.bin`), content),
|
|
1148
|
+
pending: await writeFile(path.resolve(h, 'api', 'appB', `${index}.bin`), content)
|
|
1149
|
+
})
|
|
1150
|
+
}
|
|
1151
|
+
await vault.refreshSources()
|
|
1152
|
+
const appA = vault.sources().find((source) => source.kind === 'app' && source.app === 'appA')
|
|
1153
|
+
const appB = vault.sources().find((source) => source.kind === 'app' && source.app === 'appB')
|
|
1154
|
+
for (const file of files) {
|
|
1155
|
+
await vault.adopt(file.original, file.hash, { app: 'appA', source_id: appA.id })
|
|
1156
|
+
const st = await fs.promises.stat(file.pending)
|
|
1157
|
+
const entry = {
|
|
1158
|
+
hash: file.hash,
|
|
1159
|
+
size: st.size,
|
|
1160
|
+
app: 'appB',
|
|
1161
|
+
source_id: appB.id,
|
|
1162
|
+
dev: st.dev,
|
|
1163
|
+
ino: st.ino,
|
|
1164
|
+
mtime: st.mtimeMs,
|
|
1165
|
+
ctime: st.ctimeMs
|
|
1166
|
+
}
|
|
1167
|
+
vault.registry.setDuplicate(file.pending, entry, entry)
|
|
1168
|
+
}
|
|
1169
|
+
|
|
1170
|
+
let releaseQueue
|
|
1171
|
+
const queueGate = new Promise((resolve) => { releaseQueue = resolve })
|
|
1172
|
+
const blocker = vault.runExclusive(() => queueGate)
|
|
1173
|
+
const samples = []
|
|
1174
|
+
const realConvert = vault.convert.bind(vault)
|
|
1175
|
+
vault.convert = async (...args) => {
|
|
1176
|
+
const action = vault.progressStatus(appB.id).file_actions
|
|
1177
|
+
.find((item) => item.kind === 'deduplicate')
|
|
1178
|
+
samples.push({
|
|
1179
|
+
phase: action.phase,
|
|
1180
|
+
completed: action.files_completed,
|
|
1181
|
+
total: action.files_total
|
|
1182
|
+
})
|
|
1183
|
+
return realConvert(...args)
|
|
1184
|
+
}
|
|
1185
|
+
|
|
1186
|
+
const pending = vault.perform('deduplicate', {
|
|
1187
|
+
scope_id: appB.id,
|
|
1188
|
+
paths: files.map((file) => file.pending)
|
|
1189
|
+
})
|
|
1190
|
+
await new Promise((resolve) => setImmediate(resolve))
|
|
1191
|
+
const queued = vault.progressStatus(appB.id).file_actions
|
|
1192
|
+
.find((item) => item.kind === 'deduplicate')
|
|
1193
|
+
assert.strictEqual(queued.phase, 'queued')
|
|
1194
|
+
assert.strictEqual(queued.files_completed, 0)
|
|
1195
|
+
assert.strictEqual(queued.files_total, 3)
|
|
1196
|
+
|
|
1197
|
+
releaseQueue()
|
|
1198
|
+
await blocker
|
|
1199
|
+
const result = await pending
|
|
1200
|
+
|
|
1201
|
+
assert.strictEqual(result.converted, 3)
|
|
1202
|
+
assert.deepStrictEqual(samples, [
|
|
1203
|
+
{ phase: 'deduplicating', completed: 0, total: 3 },
|
|
1204
|
+
{ phase: 'deduplicating', completed: 1, total: 3 },
|
|
1205
|
+
{ phase: 'deduplicating', completed: 2, total: 3 }
|
|
1206
|
+
])
|
|
1207
|
+
assert.deepStrictEqual(vault.progressStatus(appB.id).file_actions, [])
|
|
1208
|
+
})
|
|
1209
|
+
|
|
1210
|
+
test('Deduplicate all reports aggregate progress and keeps one batch per location', async () => {
|
|
1211
|
+
const h = await home()
|
|
1212
|
+
const vault = await makeVault(h)
|
|
1213
|
+
const batches = []
|
|
1214
|
+
const samples = []
|
|
1215
|
+
vault.deduplicateScope = async (scopeId, options) => {
|
|
1216
|
+
batches.push({ scopeId, batchId: options.batch_id, paths: options.paths })
|
|
1217
|
+
for (const filePath of options.paths) {
|
|
1218
|
+
options.onFileComplete()
|
|
1219
|
+
const action = vault.progressStatus().file_actions
|
|
1220
|
+
.find((item) => item.kind === 'deduplicate_all')
|
|
1221
|
+
samples.push({
|
|
1222
|
+
path: filePath,
|
|
1223
|
+
completed: action.files_completed,
|
|
1224
|
+
total: action.files_total,
|
|
1225
|
+
sourceId: action.source_id
|
|
1226
|
+
})
|
|
1227
|
+
}
|
|
1228
|
+
return { converted: options.paths.length, bytes_saved: options.paths.length * 100 }
|
|
1229
|
+
}
|
|
1230
|
+
|
|
1231
|
+
const result = await vault.perform('deduplicate_all', {
|
|
1232
|
+
groups: [
|
|
1233
|
+
{ scope_id: 'app:a', paths: ['/a/one', '/a/two'] },
|
|
1234
|
+
{ scope_id: 'app:b', paths: ['/b/one'] }
|
|
1235
|
+
]
|
|
1236
|
+
})
|
|
1237
|
+
|
|
1238
|
+
assert.strictEqual(result.converted, 3)
|
|
1239
|
+
assert.strictEqual(result.bytes_saved, 300)
|
|
1240
|
+
assert.deepStrictEqual(samples, [
|
|
1241
|
+
{ path: '/a/one', completed: 1, total: 3, sourceId: null },
|
|
1242
|
+
{ path: '/a/two', completed: 2, total: 3, sourceId: null },
|
|
1243
|
+
{ path: '/b/one', completed: 3, total: 3, sourceId: null }
|
|
1244
|
+
])
|
|
1245
|
+
assert.deepStrictEqual(batches.map((batch) => batch.scopeId), ['app:a', 'app:b'])
|
|
1246
|
+
assert.strictEqual(new Set(batches.map((batch) => batch.batchId)).size, 2)
|
|
1247
|
+
assert.ok(batches.every((batch) => /^batch-[0-9a-f-]{36}$/.test(batch.batchId)))
|
|
1248
|
+
assert.deepStrictEqual(vault.progressStatus().file_actions, [])
|
|
1249
|
+
})
|
|
1250
|
+
|
|
1251
|
+
test('legacy copy entries migrate to unavailable pending suggestions', async () => {
|
|
1252
|
+
const h = await home()
|
|
1253
|
+
const root = path.resolve(h, 'vault')
|
|
1254
|
+
await fs.promises.mkdir(root, { recursive: true })
|
|
1255
|
+
const content = crypto.randomBytes(4096)
|
|
1256
|
+
const hash = sha256(content)
|
|
1257
|
+
const file = await writeFile(path.resolve(h, 'api', 'appA', 'model.bin'), content)
|
|
1258
|
+
const st = await fs.promises.stat(file)
|
|
1259
|
+
const registry = new Registry(root)
|
|
1260
|
+
await fs.promises.writeFile(registry.snapshotPath, JSON.stringify({
|
|
1261
|
+
blobs: { [hash]: { size: content.length } },
|
|
1262
|
+
links: {
|
|
1263
|
+
[file]: {
|
|
1264
|
+
hash, app: 'appA', source_id: 'app:appA',
|
|
1265
|
+
dev: st.dev, ino: st.ino, mode: 'copy'
|
|
1266
|
+
}
|
|
1267
|
+
},
|
|
1268
|
+
scan_index: {
|
|
1269
|
+
[file]: {
|
|
1270
|
+
hash, size: st.size, source_id: 'app:appA',
|
|
1271
|
+
dev: st.dev, ino: st.ino, mtime: st.mtimeMs, ctime: st.ctimeMs
|
|
1272
|
+
}
|
|
1273
|
+
}
|
|
1274
|
+
}))
|
|
1275
|
+
|
|
1276
|
+
await registry.load()
|
|
1277
|
+
|
|
1278
|
+
assert.strictEqual(registry.links.has(file), false)
|
|
1279
|
+
assert.strictEqual(registry.duplicates.get(file).unavailable_reason, 'unsupported_disk')
|
|
1280
|
+
})
|
|
1281
|
+
|
|
232
1282
|
test('manual index repair preserves user decisions, history, and pending review state', async () => {
|
|
233
1283
|
const h = await home()
|
|
234
1284
|
const vault = await makeVault(h)
|
|
@@ -237,22 +1287,194 @@ describe('vault engine (phase 1)', () => {
|
|
|
237
1287
|
const original = await writeFile(path.resolve(h, 'api', 'appA', 'm.bin'), content)
|
|
238
1288
|
const pending = await writeFile(path.resolve(h, 'api', 'appB', 'm.bin'), content)
|
|
239
1289
|
const independent = await writeFile(path.resolve(h, 'cache', 'keep.bin'), crypto.randomBytes(2048))
|
|
1290
|
+
vault.sizeThreshold = 1
|
|
240
1291
|
await vault.adopt(original, hash, { app: 'appA', source_urls: ['https://example.test/model'] })
|
|
241
|
-
|
|
1292
|
+
const pendingStat = await fs.promises.stat(pending)
|
|
1293
|
+
const pendingEntry = {
|
|
1294
|
+
hash, size: content.length, app: 'appB', discovered: Date.now(),
|
|
1295
|
+
dev: pendingStat.dev, ino: pendingStat.ino,
|
|
1296
|
+
mtime: pendingStat.mtimeMs, ctime: pendingStat.ctimeMs
|
|
1297
|
+
}
|
|
1298
|
+
vault.registry.duplicates.set(pending, pendingEntry)
|
|
1299
|
+
vault.registry.scanIndex.set(pending, pendingEntry)
|
|
242
1300
|
vault.registry.excluded.set(independent, { ts: Date.now(), size: 2048 })
|
|
243
|
-
vault.registry.totals.lifetime_bytes_saved = 12345
|
|
244
1301
|
vault.registry.lastScan = { ts: 111, dirs: 2, files: 3, bytes_total: 999 }
|
|
245
1302
|
|
|
246
|
-
await vault.rebuild(
|
|
1303
|
+
await vault.rebuild()
|
|
247
1304
|
|
|
248
1305
|
assert.ok(vault.registry.excluded.has(independent), 'keep-independent decision survives')
|
|
249
1306
|
assert.strictEqual(vault.registry.excluded.get(independent).size, 2048)
|
|
250
|
-
assert.strictEqual(vault.registry.totals.lifetime_bytes_saved, 12345)
|
|
251
1307
|
assert.deepStrictEqual(vault.registry.lastScan, { ts: 111, dirs: 2, files: 3, bytes_total: 999 })
|
|
252
1308
|
assert.ok(vault.registry.duplicates.has(pending), 'pending review item survives when still valid')
|
|
253
1309
|
assert.deepStrictEqual(vault.registry.blobs.get(hash).source_urls, ['https://example.test/model'])
|
|
254
1310
|
})
|
|
255
1311
|
|
|
1312
|
+
test('manual repair never transfers an undo batch to a different tracked inode', async () => {
|
|
1313
|
+
const h = await home()
|
|
1314
|
+
const vault = await makeVault(h)
|
|
1315
|
+
vault.sizeThreshold = 1
|
|
1316
|
+
const firstContent = crypto.randomBytes(4096)
|
|
1317
|
+
const secondContent = crypto.randomBytes(4096)
|
|
1318
|
+
const firstHash = sha256(firstContent)
|
|
1319
|
+
const secondHash = sha256(secondContent)
|
|
1320
|
+
const first = await writeFile(path.resolve(h, 'api', 'appA', 'first.bin'), firstContent)
|
|
1321
|
+
const target = await writeFile(path.resolve(h, 'api', 'appB', 'target.bin'), firstContent)
|
|
1322
|
+
const second = await writeFile(path.resolve(h, 'api', 'appC', 'second.bin'), secondContent)
|
|
1323
|
+
await vault.adopt(first, firstHash, { app: 'appA' })
|
|
1324
|
+
assert.strictEqual((await vault.convert(target, firstHash, {
|
|
1325
|
+
app: 'appB', batch_id: 'batch-old'
|
|
1326
|
+
})).status, 'converted')
|
|
1327
|
+
await vault.adopt(second, secondHash, { app: 'appC' })
|
|
1328
|
+
|
|
1329
|
+
const replacementPath = `${target}.replacement`
|
|
1330
|
+
await fs.promises.link(vault.storePathFor(secondHash), replacementPath)
|
|
1331
|
+
await fs.promises.rename(replacementPath, target)
|
|
1332
|
+
const replacementStat = await fs.promises.stat(target)
|
|
1333
|
+
vault.registry.maxEvents = 2
|
|
1334
|
+
vault.registry.compactEvery = 1
|
|
1335
|
+
for (let index = 0; index < 3; index += 1) {
|
|
1336
|
+
await vault.registry.appendEvent({ kind: 'scan', marker: index })
|
|
1337
|
+
}
|
|
1338
|
+
|
|
1339
|
+
await vault.rebuild()
|
|
1340
|
+
|
|
1341
|
+
const repaired = vault.registry.links.get(target)
|
|
1342
|
+
assert.strictEqual(repaired.hash, secondHash)
|
|
1343
|
+
assert.strictEqual(repaired.batch_id, null)
|
|
1344
|
+
assert.deepStrictEqual(await vault.undoBatch('batch-old'), {
|
|
1345
|
+
undone: 0, bytes: 0, failed: 0
|
|
1346
|
+
})
|
|
1347
|
+
assert.strictEqual((await fs.promises.stat(target)).ino, replacementStat.ino)
|
|
1348
|
+
assert.deepStrictEqual(await fs.promises.readFile(target), secondContent)
|
|
1349
|
+
})
|
|
1350
|
+
|
|
1351
|
+
test('manual repair never publishes a partial index after a transient filesystem error', async () => {
|
|
1352
|
+
const h = await home()
|
|
1353
|
+
const vault = await makeVault(h)
|
|
1354
|
+
vault.sizeThreshold = 1
|
|
1355
|
+
const content = crypto.randomBytes(4096)
|
|
1356
|
+
const hash = sha256(content)
|
|
1357
|
+
const file = await writeFile(path.resolve(h, 'api', 'appA', 'model.bin'), content)
|
|
1358
|
+
await vault.adopt(file, hash, { app: 'appA' })
|
|
1359
|
+
await vault.registry.flush()
|
|
1360
|
+
const before = await fs.promises.readFile(vault.registry.snapshotPath, 'utf8')
|
|
1361
|
+
const realStat = fs.promises.lstat
|
|
1362
|
+
fs.promises.lstat = async (target, ...args) => {
|
|
1363
|
+
if (path.resolve(String(target)) === file) {
|
|
1364
|
+
const error = new Error('temporary metadata failure')
|
|
1365
|
+
error.code = 'EIO'
|
|
1366
|
+
throw error
|
|
1367
|
+
}
|
|
1368
|
+
return realStat(target, ...args)
|
|
1369
|
+
}
|
|
1370
|
+
try {
|
|
1371
|
+
await assert.rejects(vault.rebuild(), (error) => error && error.code === 'EIO')
|
|
1372
|
+
} finally {
|
|
1373
|
+
fs.promises.lstat = realStat
|
|
1374
|
+
}
|
|
1375
|
+
|
|
1376
|
+
assert.ok(vault.registry.blobs.has(hash))
|
|
1377
|
+
assert.ok(vault.registry.links.has(file))
|
|
1378
|
+
assert.strictEqual(await fs.promises.readFile(vault.registry.snapshotPath, 'utf8'), before)
|
|
1379
|
+
})
|
|
1380
|
+
|
|
1381
|
+
test('manual repair preserves pending review after a transient canonical-path error', async () => {
|
|
1382
|
+
const h = await home()
|
|
1383
|
+
const vault = await makeVault(h)
|
|
1384
|
+
vault.sizeThreshold = 1
|
|
1385
|
+
const content = crypto.randomBytes(4096)
|
|
1386
|
+
const hash = sha256(content)
|
|
1387
|
+
await writeFile(path.resolve(h, 'api', 'appA', 'model.bin'), content)
|
|
1388
|
+
await writeFile(path.resolve(h, 'api', 'appB', 'model.bin'), content)
|
|
1389
|
+
await vault.sweeper.scan()
|
|
1390
|
+
const pending = [...vault.registry.duplicates.keys()][0]
|
|
1391
|
+
const original = [...vault.registry.links.keys()].find((filePath) => filePath !== vault.storePathFor(hash))
|
|
1392
|
+
assert.ok(vault.registry.blobs.has(hash))
|
|
1393
|
+
assert.ok(pending && vault.registry.duplicates.has(pending))
|
|
1394
|
+
assert.ok(original)
|
|
1395
|
+
const before = await fs.promises.readFile(vault.registry.snapshotPath, 'utf8')
|
|
1396
|
+
const realRealpath = fs.promises.realpath
|
|
1397
|
+
fs.promises.realpath = async (target, options) => {
|
|
1398
|
+
if (path.resolve(String(target)) === pending) {
|
|
1399
|
+
const error = new Error('temporary canonical-path failure')
|
|
1400
|
+
error.code = 'EIO'
|
|
1401
|
+
throw error
|
|
1402
|
+
}
|
|
1403
|
+
return realRealpath(target, options)
|
|
1404
|
+
}
|
|
1405
|
+
try {
|
|
1406
|
+
await assert.rejects(vault.rebuild(), (error) => error && error.code === 'EIO')
|
|
1407
|
+
} finally {
|
|
1408
|
+
fs.promises.realpath = realRealpath
|
|
1409
|
+
}
|
|
1410
|
+
|
|
1411
|
+
assert.ok(vault.registry.links.has(original))
|
|
1412
|
+
assert.ok(vault.registry.duplicates.has(pending))
|
|
1413
|
+
assert.strictEqual(await fs.promises.readFile(vault.registry.snapshotPath, 'utf8'), before)
|
|
1414
|
+
})
|
|
1415
|
+
|
|
1416
|
+
test('manual repair never reclassifies an excluded path as shared', async () => {
|
|
1417
|
+
const h = await home()
|
|
1418
|
+
const vault = await makeVault(h)
|
|
1419
|
+
const content = crypto.randomBytes(4096)
|
|
1420
|
+
const hash = sha256(content)
|
|
1421
|
+
const file = await writeFile(path.resolve(h, 'api', 'appA', 'model.bin'), content)
|
|
1422
|
+
await vault.adopt(file, hash, { app: 'appA' })
|
|
1423
|
+
vault.registry.exclude(file, { ts: Date.now(), size: content.length })
|
|
1424
|
+
|
|
1425
|
+
await vault.rebuild()
|
|
1426
|
+
|
|
1427
|
+
assert.ok(vault.registry.excluded.has(file))
|
|
1428
|
+
assert.strictEqual(vault.registry.links.has(file), false)
|
|
1429
|
+
assert.strictEqual(vault.registry.scanIndex.has(file), false)
|
|
1430
|
+
})
|
|
1431
|
+
|
|
1432
|
+
test('manual repair drops a pending file that changed without changing size', async () => {
|
|
1433
|
+
const h = await home()
|
|
1434
|
+
const vault = await makeVault(h)
|
|
1435
|
+
const originalContent = Buffer.from('first version')
|
|
1436
|
+
const changedContent = Buffer.from('other version')
|
|
1437
|
+
const hash = sha256(originalContent)
|
|
1438
|
+
const original = await writeFile(path.resolve(h, 'api', 'appA', 'model.bin'), originalContent)
|
|
1439
|
+
const pending = await writeFile(path.resolve(h, 'api', 'appB', 'model.bin'), originalContent)
|
|
1440
|
+
await vault.adopt(original, hash, { app: 'appA' })
|
|
1441
|
+
const before = await fs.promises.stat(pending)
|
|
1442
|
+
const snapshot = {
|
|
1443
|
+
hash, size: before.size, dev: before.dev, ino: before.ino,
|
|
1444
|
+
mtime: before.mtimeMs, ctime: before.ctimeMs, source_id: null
|
|
1445
|
+
}
|
|
1446
|
+
vault.registry.duplicates.set(pending, snapshot)
|
|
1447
|
+
vault.registry.scanIndex.set(pending, snapshot)
|
|
1448
|
+
await fs.promises.writeFile(pending, changedContent)
|
|
1449
|
+
const future = new Date(Date.now() + 2000)
|
|
1450
|
+
await fs.promises.utimes(pending, future, future)
|
|
1451
|
+
|
|
1452
|
+
await vault.rebuild()
|
|
1453
|
+
|
|
1454
|
+
assert.strictEqual(vault.registry.duplicates.has(pending), false)
|
|
1455
|
+
assert.deepStrictEqual(await fs.promises.readFile(pending), changedContent)
|
|
1456
|
+
})
|
|
1457
|
+
|
|
1458
|
+
test('manual repair drops pending conversions when the stored content is no longer verified', async () => {
|
|
1459
|
+
const h = await home()
|
|
1460
|
+
const vault = await makeVault(h)
|
|
1461
|
+
vault.sizeThreshold = 1
|
|
1462
|
+
const original = Buffer.alloc(4096, 21)
|
|
1463
|
+
const changed = Buffer.alloc(4096, 22)
|
|
1464
|
+
await writeFile(path.resolve(h, 'api', 'appA', 'm.bin'), original)
|
|
1465
|
+
await writeFile(path.resolve(h, 'api', 'appB', 'm.bin'), original)
|
|
1466
|
+
await vault.sweeper.scan()
|
|
1467
|
+
const pendingPath = [...vault.registry.duplicates.keys()][0]
|
|
1468
|
+
const linkedPath = [...vault.registry.links.keys()].find((filePath) => filePath !== pendingPath)
|
|
1469
|
+
|
|
1470
|
+
await fs.promises.writeFile(linkedPath, changed)
|
|
1471
|
+
await vault.rebuild()
|
|
1472
|
+
|
|
1473
|
+
assert.strictEqual(vault.registry.duplicates.has(pendingPath), false)
|
|
1474
|
+
assert.strictEqual(vault.registry.scanIndex.has(linkedPath), false)
|
|
1475
|
+
assert.deepStrictEqual(await fs.promises.readFile(pendingPath), original)
|
|
1476
|
+
})
|
|
1477
|
+
|
|
256
1478
|
test('rebuild walks every directory without name heuristics and still skips symlinks', async () => {
|
|
257
1479
|
const h = await home()
|
|
258
1480
|
const vault = await makeVault(h)
|
|
@@ -275,11 +1497,12 @@ describe('vault engine (phase 1)', () => {
|
|
|
275
1497
|
const firstFile = await writeFile(path.resolve(h, 'api', 'appA', 'x.bin'), first)
|
|
276
1498
|
const secondFile = await writeFile(path.resolve(h, 'api', 'appA', 'y.bin'), second)
|
|
277
1499
|
const firstResult = await vault.hashFile(firstFile)
|
|
1500
|
+
const worker = vault.worker
|
|
278
1501
|
const secondResult = await vault.hashFile(secondFile)
|
|
279
1502
|
assert.strictEqual(firstResult.hash, sha256(first))
|
|
280
1503
|
assert.strictEqual(firstResult.size, first.length)
|
|
281
1504
|
assert.strictEqual(secondResult.hash, sha256(second))
|
|
282
|
-
assert.strictEqual(vault.
|
|
1505
|
+
assert.strictEqual(vault.worker, worker, 'worker startup is amortized across files')
|
|
283
1506
|
})
|
|
284
1507
|
|
|
285
1508
|
test('unsupported conversion leaves the independent copy pending and unregistered', async () => {
|
|
@@ -309,6 +1532,33 @@ describe('vault engine (phase 1)', () => {
|
|
|
309
1532
|
assert.deepStrictEqual(await fs.promises.readFile(pending), content)
|
|
310
1533
|
})
|
|
311
1534
|
|
|
1535
|
+
test('a transient adoption permission error is retried instead of becoming copy mode', async () => {
|
|
1536
|
+
const h = await home()
|
|
1537
|
+
const vault = await makeVault(h)
|
|
1538
|
+
const content = crypto.randomBytes(4096)
|
|
1539
|
+
const hash = sha256(content)
|
|
1540
|
+
const file = await writeFile(path.resolve(h, 'api', 'appA', 'm.bin'), content)
|
|
1541
|
+
const storePath = vault.storePathFor(hash)
|
|
1542
|
+
const realLink = fs.promises.link
|
|
1543
|
+
fs.promises.link = async (source, target) => {
|
|
1544
|
+
if (target === storePath) {
|
|
1545
|
+
const error = new Error('permission temporarily unavailable')
|
|
1546
|
+
error.code = 'EPERM'
|
|
1547
|
+
throw error
|
|
1548
|
+
}
|
|
1549
|
+
return realLink(source, target)
|
|
1550
|
+
}
|
|
1551
|
+
try {
|
|
1552
|
+
await assert.rejects(vault.adopt(file, hash, { app: 'appA' }), (error) => error.code === 'EPERM')
|
|
1553
|
+
} finally {
|
|
1554
|
+
fs.promises.link = realLink
|
|
1555
|
+
}
|
|
1556
|
+
assert.strictEqual(vault.registry.links.has(file), false)
|
|
1557
|
+
assert.strictEqual(vault.registry.blobs.has(hash), false)
|
|
1558
|
+
|
|
1559
|
+
assert.strictEqual((await vault.adopt(file, hash, { app: 'appA' })).status, 'adopted')
|
|
1560
|
+
})
|
|
1561
|
+
|
|
312
1562
|
test('permission errors during conversion are retryable locks, not unsupported disks', async () => {
|
|
313
1563
|
const h = await home()
|
|
314
1564
|
const vault = await makeVault(h)
|
|
@@ -336,7 +1586,7 @@ describe('vault engine (phase 1)', () => {
|
|
|
336
1586
|
assert.deepStrictEqual(await fs.promises.readFile(pending), content)
|
|
337
1587
|
})
|
|
338
1588
|
|
|
339
|
-
test('item 11: exFAT volume
|
|
1589
|
+
test('item 11: exFAT volume is unsupported (macOS loopback; skipped elsewhere)', { skip: process.platform !== 'darwin' }, async (t) => {
|
|
340
1590
|
const h = await home()
|
|
341
1591
|
const vault = await makeVault(h)
|
|
342
1592
|
const dmg = path.resolve(h, 'exfat-test.dmg')
|
|
@@ -352,7 +1602,7 @@ describe('vault engine (phase 1)', () => {
|
|
|
352
1602
|
}
|
|
353
1603
|
try {
|
|
354
1604
|
const mode = await vault.probe(mountPoint)
|
|
355
|
-
assert.strictEqual(mode, '
|
|
1605
|
+
assert.strictEqual(mode, 'unsupported')
|
|
356
1606
|
} finally {
|
|
357
1607
|
if (mountPoint) execSync(`hdiutil detach "${mountPoint}" -force`, { stdio: 'pipe' })
|
|
358
1608
|
}
|