pinokiod 8.0.74 → 8.0.76
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/kernel/vault/automatic_scans.js +168 -58
- package/kernel/vault/registry_core.js +65 -23
- package/package.json +1 -1
- package/server/public/vault.css +34 -0
- package/server/public/vault.js +8 -4
- package/test/vault-automatic-scans.test.js +1325 -139
- package/test/vault-engine.test.js +7 -0
- package/test/vault-ui.test.js +62 -0
|
@@ -226,6 +226,117 @@ describe("automatic app checks", () => {
|
|
|
226
226
|
await close(vault)
|
|
227
227
|
})
|
|
228
228
|
|
|
229
|
+
test("uncompleted watcher paths are not persisted across restart", async () => {
|
|
230
|
+
const home = await makeHome()
|
|
231
|
+
const appRoot = path.join(home, "api", "transient-path-app")
|
|
232
|
+
const scriptPath = path.join(appRoot, "start.js")
|
|
233
|
+
const changedPath = path.join(appRoot, "model.bin")
|
|
234
|
+
await fs.promises.mkdir(appRoot)
|
|
235
|
+
await fs.promises.writeFile(changedPath, "model")
|
|
236
|
+
const first = await makeVault(home, { deferStorage: true })
|
|
237
|
+
first.automaticScans.handleStarted(scriptPath)
|
|
238
|
+
first.automaticScans.recordChangedPath(changedPath)
|
|
239
|
+
assert.deepEqual(first.automaticScans.changedPaths.get(
|
|
240
|
+
"transient-path-app"), new Set([changedPath]))
|
|
241
|
+
assert.equal(fs.existsSync(path.join(home, "vault")), false)
|
|
242
|
+
await close(first)
|
|
243
|
+
|
|
244
|
+
const restored = await makeVault(home, { deferStorage: true })
|
|
245
|
+
assert.equal(restored.automaticScans.changedPaths.size, 0)
|
|
246
|
+
assert.equal(restored.automaticScans.entries.size, 0)
|
|
247
|
+
assert.equal(fs.existsSync(path.join(home, "vault")), false)
|
|
248
|
+
await close(restored)
|
|
249
|
+
})
|
|
250
|
+
|
|
251
|
+
test("watcher startup performs no filesystem discovery or queued work", async (t) => {
|
|
252
|
+
const home = await makeHome()
|
|
253
|
+
let subscribedRoot = null
|
|
254
|
+
const vault = {
|
|
255
|
+
enabled: true,
|
|
256
|
+
kernel: {
|
|
257
|
+
homedir: home,
|
|
258
|
+
platform: "darwin",
|
|
259
|
+
automaticWatcher: {
|
|
260
|
+
subscribe: async (root) => {
|
|
261
|
+
subscribedRoot = root
|
|
262
|
+
return { unsubscribe: async () => {} }
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
},
|
|
266
|
+
hashFile: async () => {
|
|
267
|
+
throw new Error("Watcher startup must not hash content.")
|
|
268
|
+
},
|
|
269
|
+
scanner: {
|
|
270
|
+
walk: async () => {
|
|
271
|
+
throw new Error("Watcher startup must not walk the API tree.")
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
const automatic = makeAutomatic(vault)
|
|
276
|
+
automatic.queueApp = () => {
|
|
277
|
+
throw new Error("Watcher startup must not queue work.")
|
|
278
|
+
}
|
|
279
|
+
const blocked = ["lstat", "stat", "readdir", "opendir", "readFile"]
|
|
280
|
+
const originals = new Map(blocked.map((name) =>
|
|
281
|
+
[name, fs.promises[name]]))
|
|
282
|
+
t.after(() => {
|
|
283
|
+
for (const [name, method] of originals) fs.promises[name] = method
|
|
284
|
+
})
|
|
285
|
+
for (const name of blocked) {
|
|
286
|
+
fs.promises[name] = async () => {
|
|
287
|
+
throw new Error(`Watcher startup must not call fs.promises.${name}.`)
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
assert.equal(await automatic.startWatcher(), true)
|
|
292
|
+
assert.equal(subscribedRoot, path.join(home, "api"))
|
|
293
|
+
assert.equal(automatic.changedPaths.size, 0)
|
|
294
|
+
assert.equal(automatic.entries.size, 0)
|
|
295
|
+
await automatic.stopWatcher()
|
|
296
|
+
})
|
|
297
|
+
|
|
298
|
+
test("watcher failures log without queuing recovery scans", async () => {
|
|
299
|
+
const home = await makeHome()
|
|
300
|
+
const app = "watcher-failure-app"
|
|
301
|
+
const appRoot = path.join(home, "api", app)
|
|
302
|
+
const changedPath = path.join(appRoot, "model.bin")
|
|
303
|
+
await fs.promises.mkdir(appRoot)
|
|
304
|
+
const vault = await makeVault(home, {
|
|
305
|
+
deferStorage: true,
|
|
306
|
+
automaticWatcher: {
|
|
307
|
+
subscribe: async () => {
|
|
308
|
+
throw new Error("subscription unavailable")
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
})
|
|
312
|
+
const logs = []
|
|
313
|
+
vault.automaticScans.log = (event, details) => {
|
|
314
|
+
logs.push({ event, details })
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
assert.equal(await vault.automaticScans.startWatcher(), false)
|
|
318
|
+
vault.automaticScans.observedApps.add(app)
|
|
319
|
+
vault.automaticScans.handleWatcherEvents(
|
|
320
|
+
new Error("event stream overflow"),
|
|
321
|
+
[{ type: "update", path: changedPath }]
|
|
322
|
+
)
|
|
323
|
+
|
|
324
|
+
assert.equal(logs.filter((record) =>
|
|
325
|
+
record.event === "watcher-error").length, 2)
|
|
326
|
+
assert.equal(vault.automaticScans.changedPaths.size, 0)
|
|
327
|
+
assert.equal(vault.automaticScans.entries.size, 0)
|
|
328
|
+
assert.equal(vault.registry, null)
|
|
329
|
+
assert.equal(fs.existsSync(path.join(home, "vault")), false)
|
|
330
|
+
const manual = await vault.perform("scan", {
|
|
331
|
+
scope_id: `app:${app}`,
|
|
332
|
+
candidate_size: 0
|
|
333
|
+
})
|
|
334
|
+
assert.equal(manual.started, true)
|
|
335
|
+
await waitFor(() => !vault.scanPromise && !vault.scanCompletionPromise,
|
|
336
|
+
"manual scan after watcher failure")
|
|
337
|
+
await close(vault)
|
|
338
|
+
})
|
|
339
|
+
|
|
229
340
|
test("Windows watcher deletes remove retained paths and descendants", async () => {
|
|
230
341
|
const home = await makeHome()
|
|
231
342
|
const appRoot = path.join(home, "api", "windows-app")
|
|
@@ -849,6 +960,13 @@ describe("automatic app checks", () => {
|
|
|
849
960
|
INSERT INTO automatic_precheck_files(app, path, size, dev, ino)
|
|
850
961
|
VALUES (?, ?, 100, 1, ?)
|
|
851
962
|
`)
|
|
963
|
+
insert.run("canonical-path", "C:\\Äpp\\MODEL.BIN", 1)
|
|
964
|
+
insert.run("canonical-path", "c:\\äpp\\model.bin", 2)
|
|
965
|
+
assert.deepEqual(
|
|
966
|
+
registry.automaticPrecheckEntries("canonical-path").entries,
|
|
967
|
+
[]
|
|
968
|
+
)
|
|
969
|
+
registry.abortAutomaticPrecheck("canonical-path")
|
|
852
970
|
insert.run("demo", "C:\\App\\Z.bin", 1)
|
|
853
971
|
insert.run("demo", "C:\\App\\a.bin", 2)
|
|
854
972
|
const hash = "a".repeat(64)
|
|
@@ -869,6 +987,66 @@ describe("automatic app checks", () => {
|
|
|
869
987
|
}
|
|
870
988
|
})
|
|
871
989
|
|
|
990
|
+
test("automatic result signatures cover the proof and complete changed batch", async (t) => {
|
|
991
|
+
const home = await makeHome()
|
|
992
|
+
const registry = new RegistryCore(path.join(home, "vault"))
|
|
993
|
+
await registry.load()
|
|
994
|
+
t.after(() => registry.close())
|
|
995
|
+
const app = "signature-batch-app"
|
|
996
|
+
const proofHash = "a".repeat(64)
|
|
997
|
+
const replacementHash = "b".repeat(64)
|
|
998
|
+
const proof = {
|
|
999
|
+
path: path.join(home, "api", app, "proof.bin"),
|
|
1000
|
+
dev: 1,
|
|
1001
|
+
ino: 11,
|
|
1002
|
+
size: 100,
|
|
1003
|
+
mtime: 101,
|
|
1004
|
+
ctime: 102,
|
|
1005
|
+
nlink: 1,
|
|
1006
|
+
mode: 0,
|
|
1007
|
+
uid: 0,
|
|
1008
|
+
gid: 0
|
|
1009
|
+
}
|
|
1010
|
+
const unmatched = {
|
|
1011
|
+
path: path.join(home, "api", app, "unmatched.bin"),
|
|
1012
|
+
dev: 2,
|
|
1013
|
+
ino: 21,
|
|
1014
|
+
size: 200,
|
|
1015
|
+
mtime: 201,
|
|
1016
|
+
ctime: 202,
|
|
1017
|
+
nlink: 1,
|
|
1018
|
+
mode: 0,
|
|
1019
|
+
uid: 0,
|
|
1020
|
+
gid: 0
|
|
1021
|
+
}
|
|
1022
|
+
const signature = (entries, hash = proofHash) => {
|
|
1023
|
+
registry.beginAutomaticPrecheck(app)
|
|
1024
|
+
registry.stageAutomaticPrecheckFiles(app, entries)
|
|
1025
|
+
return registry.automaticPrecheckResult(app, [{
|
|
1026
|
+
path: proof.path,
|
|
1027
|
+
hash
|
|
1028
|
+
}]).signature
|
|
1029
|
+
}
|
|
1030
|
+
|
|
1031
|
+
const baseline = signature([proof, unmatched])
|
|
1032
|
+
assert.equal(signature([unmatched, proof]), baseline)
|
|
1033
|
+
assert.notEqual(signature([proof, unmatched], replacementHash), baseline)
|
|
1034
|
+
for (const [field, value] of [
|
|
1035
|
+
["path", path.join(home, "api", app, "renamed.bin")],
|
|
1036
|
+
["dev", 3],
|
|
1037
|
+
["ino", 22],
|
|
1038
|
+
["size", 201],
|
|
1039
|
+
["mtime", 203],
|
|
1040
|
+
["ctime", 204]
|
|
1041
|
+
]) {
|
|
1042
|
+
assert.notEqual(
|
|
1043
|
+
signature([proof, Object.assign({}, unmatched, { [field]: value })]),
|
|
1044
|
+
baseline,
|
|
1045
|
+
`signature must include changed-path ${field}`
|
|
1046
|
+
)
|
|
1047
|
+
}
|
|
1048
|
+
})
|
|
1049
|
+
|
|
872
1050
|
test("the final app-process stop verifies only changed candidates", async () => {
|
|
873
1051
|
const home = await makeHome()
|
|
874
1052
|
const appRoot = path.join(home, "api", "demo")
|
|
@@ -971,13 +1149,15 @@ describe("automatic app checks", () => {
|
|
|
971
1149
|
return originalVerify(...args)
|
|
972
1150
|
}
|
|
973
1151
|
|
|
974
|
-
vault
|
|
1152
|
+
collect(vault, app, [first, second])
|
|
1153
|
+
vault.automaticScans.queueApp(app)
|
|
975
1154
|
await waitFor(() => !vault.automaticScans.active &&
|
|
976
1155
|
!vault.automaticScans.entries.has(app))
|
|
977
1156
|
|
|
978
1157
|
assert.deepEqual(hashedPaths.sort(), [first, second].sort())
|
|
979
1158
|
assert.deepEqual(verifiedPaths.sort(), [first, second].sort())
|
|
980
1159
|
assert.deepEqual(vault.automaticScans.snapshot().rows, [])
|
|
1160
|
+
assert.equal(vault.automaticScans.changedPaths.has(app), false)
|
|
981
1161
|
await close(vault)
|
|
982
1162
|
})
|
|
983
1163
|
|
|
@@ -1192,6 +1372,58 @@ describe("automatic app checks", () => {
|
|
|
1192
1372
|
await close(vault)
|
|
1193
1373
|
})
|
|
1194
1374
|
|
|
1375
|
+
test("direct directories, symbolic links, and empty files are not candidates", async (t) => {
|
|
1376
|
+
const home = await makeHome()
|
|
1377
|
+
const app = "non-file-candidates-app"
|
|
1378
|
+
const appRoot = path.join(home, "api", app)
|
|
1379
|
+
const outside = path.join(home, "outside-directory")
|
|
1380
|
+
const directory = path.join(appRoot, "directory")
|
|
1381
|
+
const link = path.join(appRoot, "direct-link")
|
|
1382
|
+
const empty = path.join(appRoot, "empty.bin")
|
|
1383
|
+
const first = path.join(appRoot, "first.bin")
|
|
1384
|
+
const second = path.join(appRoot, "second.bin")
|
|
1385
|
+
await fs.promises.mkdir(appRoot)
|
|
1386
|
+
await fs.promises.mkdir(outside)
|
|
1387
|
+
await fs.promises.mkdir(directory)
|
|
1388
|
+
await fs.promises.writeFile(empty, "")
|
|
1389
|
+
await fs.promises.writeFile(first, "same")
|
|
1390
|
+
await fs.promises.writeFile(second, "same")
|
|
1391
|
+
await fs.promises.symlink(
|
|
1392
|
+
outside,
|
|
1393
|
+
link,
|
|
1394
|
+
process.platform === "win32" ? "junction" : "dir"
|
|
1395
|
+
)
|
|
1396
|
+
const vault = await makeVault(home)
|
|
1397
|
+
t.after(() => close(vault))
|
|
1398
|
+
vault.automaticScans.candidateThreshold = async () => 1
|
|
1399
|
+
const stagedPaths = []
|
|
1400
|
+
const stage = vault.registry.stageAutomaticPrecheckFiles.bind(
|
|
1401
|
+
vault.registry)
|
|
1402
|
+
vault.registry.stageAutomaticPrecheckFiles = async (stagedApp, entries) => {
|
|
1403
|
+
stagedPaths.push(...entries.map((entry) => entry.path))
|
|
1404
|
+
return stage(stagedApp, entries)
|
|
1405
|
+
}
|
|
1406
|
+
const hashedPaths = []
|
|
1407
|
+
const hashFile = vault.hashFile.bind(vault)
|
|
1408
|
+
vault.hashFile = async (filePath, options) => {
|
|
1409
|
+
hashedPaths.push(filePath)
|
|
1410
|
+
return hashFile(filePath, options)
|
|
1411
|
+
}
|
|
1412
|
+
|
|
1413
|
+
vault.automaticScans.queueApp(app, [
|
|
1414
|
+
directory,
|
|
1415
|
+
link,
|
|
1416
|
+
empty,
|
|
1417
|
+
first,
|
|
1418
|
+
second
|
|
1419
|
+
])
|
|
1420
|
+
await waitFor(() => !vault.automaticScans.active &&
|
|
1421
|
+
vault.automaticScans.entries.get(app)?.state === "result")
|
|
1422
|
+
|
|
1423
|
+
assert.deepEqual(new Set(stagedPaths), new Set([first, second]))
|
|
1424
|
+
assert.deepEqual(new Set(hashedPaths), new Set([first, second]))
|
|
1425
|
+
})
|
|
1426
|
+
|
|
1195
1427
|
test("an app removed during its active check loses automatic state", async () => {
|
|
1196
1428
|
const home = await makeHome()
|
|
1197
1429
|
const app = "deleted-during-check"
|
|
@@ -1459,158 +1691,334 @@ describe("automatic app checks", () => {
|
|
|
1459
1691
|
unavailable_reason: null,
|
|
1460
1692
|
updated_at: Date.now()
|
|
1461
1693
|
})
|
|
1694
|
+
const hashedPaths = []
|
|
1695
|
+
const originalHashFile = vault.hashFile.bind(vault)
|
|
1696
|
+
vault.hashFile = async (filePath, options) => {
|
|
1697
|
+
hashedPaths.push(filePath)
|
|
1698
|
+
return originalHashFile(filePath, options)
|
|
1699
|
+
}
|
|
1462
1700
|
|
|
1463
1701
|
vault.automaticScans.queueApp(app, [candidate])
|
|
1464
1702
|
await waitFor(() => !vault.automaticScans.active &&
|
|
1465
1703
|
!vault.automaticScans.entries.has(app))
|
|
1466
1704
|
|
|
1705
|
+
assert.deepEqual(hashedPaths.sort(), [candidate, peer].sort())
|
|
1467
1706
|
assert.deepEqual(vault.automaticScans.snapshot().rows, [])
|
|
1468
1707
|
await close(vault)
|
|
1469
1708
|
})
|
|
1470
1709
|
|
|
1471
|
-
test("
|
|
1710
|
+
test("a no-match precheck exhausts every candidate without a hash budget", async (t) => {
|
|
1472
1711
|
const home = await makeHome()
|
|
1473
|
-
const app = "
|
|
1712
|
+
const app = "unbounded-no-match-app"
|
|
1474
1713
|
const appRoot = path.join(home, "api", app)
|
|
1475
|
-
const
|
|
1476
|
-
const second = path.join(appRoot, "second.bin")
|
|
1714
|
+
const peerRoot = path.join(home, "indexed-large-peers")
|
|
1477
1715
|
await fs.promises.mkdir(appRoot)
|
|
1478
|
-
await fs.promises.
|
|
1479
|
-
await fs.promises.writeFile(second, "same-content")
|
|
1480
|
-
const before = await Promise.all([
|
|
1481
|
-
fs.promises.lstat(first),
|
|
1482
|
-
fs.promises.lstat(second)
|
|
1483
|
-
])
|
|
1484
|
-
assert.notEqual(before[0].ino, before[1].ino)
|
|
1716
|
+
await fs.promises.mkdir(peerRoot)
|
|
1485
1717
|
const vault = await makeVault(home)
|
|
1718
|
+
t.after(() => close(vault))
|
|
1486
1719
|
vault.automaticScans.candidateThreshold = async () => 1
|
|
1487
|
-
const
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1720
|
+
const expected = []
|
|
1721
|
+
const candidateSnapshots = new Map()
|
|
1722
|
+
let logicalBytes = 0
|
|
1723
|
+
for (let index = 0; index < 3; index++) {
|
|
1724
|
+
const size = (512 * 1024 * 1024) + index
|
|
1725
|
+
const candidate = path.join(appRoot, `candidate-${index}.bin`)
|
|
1726
|
+
const peer = path.join(peerRoot, `peer-${index}.bin`)
|
|
1727
|
+
await fs.promises.writeFile(candidate, "candidate")
|
|
1728
|
+
await fs.promises.writeFile(peer, "peer")
|
|
1729
|
+
expected.push(candidate, peer)
|
|
1730
|
+
logicalBytes += size * 2
|
|
1731
|
+
const candidateStat = await fs.promises.lstat(candidate)
|
|
1732
|
+
candidateSnapshots.set(candidate, {
|
|
1733
|
+
isFile: () => true,
|
|
1734
|
+
isSymbolicLink: () => false,
|
|
1735
|
+
size,
|
|
1736
|
+
mtimeMs: candidateStat.mtimeMs,
|
|
1737
|
+
ctimeMs: candidateStat.ctimeMs,
|
|
1738
|
+
dev: candidateStat.dev,
|
|
1739
|
+
ino: candidateStat.ino,
|
|
1740
|
+
nlink: candidateStat.nlink,
|
|
1741
|
+
mode: candidateStat.mode,
|
|
1742
|
+
uid: candidateStat.uid,
|
|
1743
|
+
gid: candidateStat.gid
|
|
1744
|
+
})
|
|
1745
|
+
const stat = await fs.promises.lstat(peer)
|
|
1746
|
+
await vault.registry.upsertFile({
|
|
1747
|
+
path: peer,
|
|
1748
|
+
hash: null,
|
|
1749
|
+
size,
|
|
1750
|
+
mtime: stat.mtimeMs,
|
|
1751
|
+
ctime: stat.ctimeMs,
|
|
1752
|
+
dev: stat.dev,
|
|
1753
|
+
ino: stat.ino,
|
|
1754
|
+
mode: stat.mode,
|
|
1755
|
+
uid: stat.uid,
|
|
1756
|
+
gid: stat.gid,
|
|
1757
|
+
source_id: "app:indexed-large-peers",
|
|
1758
|
+
app: "indexed-large-peers",
|
|
1759
|
+
status: "reference",
|
|
1760
|
+
unavailable_reason: null,
|
|
1761
|
+
updated_at: Date.now()
|
|
1762
|
+
})
|
|
1763
|
+
}
|
|
1764
|
+
vault.automaticScans.statChangedPaths = async (paths) =>
|
|
1765
|
+
paths.map((filePath) => candidateSnapshots.get(filePath) || null)
|
|
1766
|
+
vault.automaticScans.currentAutomaticEntry = async (_active, entry) => ({
|
|
1767
|
+
stat: entry,
|
|
1768
|
+
entry
|
|
1769
|
+
})
|
|
1770
|
+
const hashedPaths = []
|
|
1771
|
+
vault.scanner.hashStable = async (entry) => {
|
|
1772
|
+
hashedPaths.push(entry.path)
|
|
1773
|
+
return {
|
|
1774
|
+
result: {
|
|
1775
|
+
hash: crypto.createHash("sha256").update(entry.path).digest("hex"),
|
|
1776
|
+
size: entry.size
|
|
1777
|
+
},
|
|
1778
|
+
stable: true
|
|
1779
|
+
}
|
|
1493
1780
|
}
|
|
1494
1781
|
|
|
1495
|
-
vault.automaticScans.queueApp(
|
|
1782
|
+
vault.automaticScans.queueApp(
|
|
1783
|
+
app,
|
|
1784
|
+
expected.filter((filePath) => filePath.startsWith(appRoot))
|
|
1785
|
+
)
|
|
1496
1786
|
await waitFor(() => !vault.automaticScans.active &&
|
|
1497
1787
|
!vault.automaticScans.entries.has(app))
|
|
1498
1788
|
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
])
|
|
1503
|
-
assert.equal(after[0].ino, after[1].ino)
|
|
1789
|
+
assert.equal(hashedPaths.length, expected.length)
|
|
1790
|
+
assert.deepEqual(new Set(hashedPaths), new Set(expected))
|
|
1791
|
+
assert.ok(logicalBytes > 3 * 1024 * 1024 * 1024)
|
|
1504
1792
|
assert.deepEqual(vault.automaticScans.snapshot().rows, [])
|
|
1505
|
-
await close(vault)
|
|
1506
1793
|
})
|
|
1507
1794
|
|
|
1508
|
-
test("
|
|
1795
|
+
test("automatic verification stops at the first changed-file proof", async (t) => {
|
|
1509
1796
|
const home = await makeHome()
|
|
1510
|
-
const app = "
|
|
1797
|
+
const app = "first-changed-proof-app"
|
|
1511
1798
|
const appRoot = path.join(home, "api", app)
|
|
1512
|
-
const first = path.join(appRoot, "first.bin")
|
|
1513
|
-
const second = path.join(appRoot, "second.bin")
|
|
1514
1799
|
await fs.promises.mkdir(appRoot)
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
const
|
|
1518
|
-
fs.promises.
|
|
1519
|
-
|
|
1520
|
-
])
|
|
1521
|
-
assert.equal(before[0].ino, before[1].ino)
|
|
1800
|
+
const candidates = Array.from({ length: 8 }, (_value, index) =>
|
|
1801
|
+
path.join(appRoot, `${String(index).padStart(2, "0")}.bin`))
|
|
1802
|
+
for (const filePath of candidates) {
|
|
1803
|
+
await fs.promises.writeFile(filePath, "same")
|
|
1804
|
+
}
|
|
1522
1805
|
const vault = await makeVault(home)
|
|
1806
|
+
t.after(() => close(vault))
|
|
1523
1807
|
vault.automaticScans.candidateThreshold = async () => 1
|
|
1524
|
-
const
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1808
|
+
const hashedPaths = []
|
|
1809
|
+
const seenHashes = new Set()
|
|
1810
|
+
let firstProofAt = null
|
|
1811
|
+
let activeReads = 0
|
|
1812
|
+
let maxActiveReads = 0
|
|
1813
|
+
const originalHashFile = vault.hashFile.bind(vault)
|
|
1814
|
+
vault.hashFile = async (filePath, options) => {
|
|
1815
|
+
activeReads += 1
|
|
1816
|
+
maxActiveReads = Math.max(maxActiveReads, activeReads)
|
|
1817
|
+
try {
|
|
1818
|
+
const result = await originalHashFile(filePath, options)
|
|
1819
|
+
hashedPaths.push(filePath)
|
|
1820
|
+
if (firstProofAt === null && seenHashes.has(result.hash)) {
|
|
1821
|
+
firstProofAt = hashedPaths.length
|
|
1822
|
+
}
|
|
1823
|
+
seenHashes.add(result.hash)
|
|
1824
|
+
return result
|
|
1825
|
+
} finally {
|
|
1826
|
+
activeReads -= 1
|
|
1827
|
+
}
|
|
1828
|
+
}
|
|
1829
|
+
const rememberedPaths = []
|
|
1830
|
+
const remember = vault.registry.rememberHashCache.bind(vault.registry)
|
|
1831
|
+
vault.registry.rememberHashCache = async (entries) => {
|
|
1832
|
+
rememberedPaths.push(...entries.map((entry) => entry.path))
|
|
1833
|
+
return remember(entries)
|
|
1530
1834
|
}
|
|
1531
1835
|
|
|
1532
|
-
vault.automaticScans.queueApp(app,
|
|
1836
|
+
vault.automaticScans.queueApp(app, candidates.slice().reverse())
|
|
1533
1837
|
await waitFor(() => !vault.automaticScans.active &&
|
|
1534
1838
|
vault.automaticScans.entries.get(app)?.state === "result")
|
|
1535
1839
|
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
assert.
|
|
1541
|
-
|
|
1542
|
-
|
|
1840
|
+
assert.equal(firstProofAt, 2)
|
|
1841
|
+
assert.equal(hashedPaths.length, firstProofAt)
|
|
1842
|
+
assert.equal(maxActiveReads, 1)
|
|
1843
|
+
assert.deepEqual(rememberedPaths.sort(), hashedPaths.slice().sort())
|
|
1844
|
+
assert.deepEqual(
|
|
1845
|
+
await vault.registry.automaticPrecheckEntries(app, null, 1),
|
|
1846
|
+
{ entries: [], next_cursor: null })
|
|
1543
1847
|
})
|
|
1544
1848
|
|
|
1545
|
-
test("
|
|
1849
|
+
test("a Maestro-shaped cold check reads nothing after its first proof", async (t) => {
|
|
1546
1850
|
const home = await makeHome()
|
|
1547
|
-
const
|
|
1548
|
-
const
|
|
1851
|
+
const app = "maestro-shaped-app"
|
|
1852
|
+
const appRoot = path.join(home, "api", app)
|
|
1853
|
+
const peerRoot = path.join(home, "indexed-peers")
|
|
1549
1854
|
await fs.promises.mkdir(appRoot)
|
|
1550
|
-
await fs.promises.
|
|
1551
|
-
const
|
|
1552
|
-
await
|
|
1553
|
-
|
|
1855
|
+
await fs.promises.mkdir(peerRoot)
|
|
1856
|
+
const candidates = []
|
|
1857
|
+
const vault = await makeVault(home)
|
|
1858
|
+
t.after(() => close(vault))
|
|
1554
1859
|
vault.automaticScans.candidateThreshold = async () => 1
|
|
1555
|
-
|
|
1556
|
-
|
|
1860
|
+
for (let index = 0; index < 24; index++) {
|
|
1861
|
+
const size = index + 1
|
|
1862
|
+
const candidate = path.join(
|
|
1863
|
+
appRoot, `candidate-${String(index).padStart(2, "0")}.bin`)
|
|
1864
|
+
const peer = path.join(
|
|
1865
|
+
peerRoot, `peer-${String(index).padStart(2, "0")}.bin`)
|
|
1866
|
+
const value = (index % 200) + 1
|
|
1867
|
+
await fs.promises.writeFile(candidate, Buffer.alloc(size, value))
|
|
1868
|
+
await fs.promises.writeFile(peer, Buffer.alloc(
|
|
1869
|
+
size, index < 22 ? value : value + 1))
|
|
1870
|
+
candidates.push(candidate)
|
|
1871
|
+
const stat = await fs.promises.lstat(peer)
|
|
1872
|
+
await vault.registry.upsertFile({
|
|
1873
|
+
path: peer,
|
|
1874
|
+
hash: null,
|
|
1875
|
+
size: stat.size,
|
|
1876
|
+
mtime: stat.mtimeMs,
|
|
1877
|
+
ctime: stat.ctimeMs,
|
|
1878
|
+
dev: stat.dev,
|
|
1879
|
+
ino: stat.ino,
|
|
1880
|
+
mode: stat.mode,
|
|
1881
|
+
uid: stat.uid,
|
|
1882
|
+
gid: stat.gid,
|
|
1883
|
+
source_id: "app:indexed-peers",
|
|
1884
|
+
app: "indexed-peers",
|
|
1885
|
+
status: "reference",
|
|
1886
|
+
unavailable_reason: null,
|
|
1887
|
+
updated_at: Date.now()
|
|
1888
|
+
})
|
|
1889
|
+
}
|
|
1890
|
+
const hashedPaths = []
|
|
1891
|
+
const seenHashes = new Set()
|
|
1892
|
+
let firstProofAt = null
|
|
1893
|
+
const originalHashFile = vault.hashFile.bind(vault)
|
|
1894
|
+
vault.hashFile = async (filePath, options) => {
|
|
1895
|
+
const result = await originalHashFile(filePath, options)
|
|
1896
|
+
hashedPaths.push(filePath)
|
|
1897
|
+
if (firstProofAt === null && seenHashes.has(result.hash)) {
|
|
1898
|
+
firstProofAt = hashedPaths.length
|
|
1899
|
+
}
|
|
1900
|
+
seenHashes.add(result.hash)
|
|
1901
|
+
return result
|
|
1902
|
+
}
|
|
1903
|
+
const logs = []
|
|
1904
|
+
vault.automaticScans.log = (event, details) => {
|
|
1905
|
+
logs.push({ event, details })
|
|
1906
|
+
}
|
|
1907
|
+
let pagesAfterProof = 0
|
|
1908
|
+
const entries = vault.registry.automaticPrecheckEntries.bind(
|
|
1909
|
+
vault.registry)
|
|
1910
|
+
vault.registry.automaticPrecheckEntries = async (...args) => {
|
|
1911
|
+
if (firstProofAt !== null) pagesAfterProof += 1
|
|
1912
|
+
return entries(...args)
|
|
1913
|
+
}
|
|
1914
|
+
let validationsAfterProof = 0
|
|
1915
|
+
const currentEntry = vault.automaticScans.currentAutomaticEntry.bind(
|
|
1916
|
+
vault.automaticScans)
|
|
1917
|
+
vault.automaticScans.currentAutomaticEntry = async (...args) => {
|
|
1918
|
+
if (firstProofAt !== null) validationsAfterProof += 1
|
|
1919
|
+
return currentEntry(...args)
|
|
1920
|
+
}
|
|
1921
|
+
|
|
1922
|
+
vault.automaticScans.queueApp(app, candidates)
|
|
1923
|
+
await waitFor(() => !vault.automaticScans.active &&
|
|
1924
|
+
vault.automaticScans.entries.get(app)?.state === "result")
|
|
1925
|
+
|
|
1926
|
+
assert.ok(firstProofAt !== null)
|
|
1927
|
+
assert.equal(hashedPaths.length, firstProofAt)
|
|
1928
|
+
assert.equal(pagesAfterProof, 0)
|
|
1929
|
+
assert.equal(validationsAfterProof, 0)
|
|
1930
|
+
const verification = logs.find((record) =>
|
|
1931
|
+
record.event === "verification-started")
|
|
1932
|
+
const exit = logs.find((record) => record.event === "first-proof-exit")
|
|
1933
|
+
assert.ok(verification)
|
|
1934
|
+
assert.equal(verification.details.threshold_candidates, candidates.length)
|
|
1935
|
+
assert.ok(exit)
|
|
1936
|
+
assert.equal(exit.details.candidate_path, candidates[0])
|
|
1937
|
+
assert.equal(exit.details.peer_kind, "file")
|
|
1938
|
+
assert.equal(exit.details.hashed, firstProofAt)
|
|
1939
|
+
assert.ok(exit.details.registry_pages > 0)
|
|
1940
|
+
assert.equal(exit.details.candidate_checks, 1)
|
|
1941
|
+
assert.equal(exit.details.peer_checks, 1)
|
|
1942
|
+
assert.ok(logs.indexOf(verification) < logs.indexOf(exit))
|
|
1943
|
+
const finished = logs.find((record) => record.event === "check-finished")
|
|
1944
|
+
assert.equal(finished.details.verified_files, 1)
|
|
1945
|
+
assert.equal(finished.details.registry_pages, exit.details.registry_pages)
|
|
1946
|
+
assert.equal(finished.details.candidate_checks,
|
|
1947
|
+
exit.details.candidate_checks)
|
|
1948
|
+
assert.equal(finished.details.peer_checks, exit.details.peer_checks)
|
|
1949
|
+
})
|
|
1950
|
+
|
|
1951
|
+
test("automatic verification reads content only for current metadata candidates", async () => {
|
|
1952
|
+
const home = await makeHome()
|
|
1953
|
+
const app = "content-read-prefilter-app"
|
|
1954
|
+
const appRoot = path.join(home, "api", app)
|
|
1955
|
+
const unique = path.join(appRoot, "unique-large.bin")
|
|
1956
|
+
const duplicateA = path.join(appRoot, "duplicate-a.bin")
|
|
1957
|
+
const duplicateB = path.join(appRoot, "duplicate-b.bin")
|
|
1958
|
+
const hardlinkA = path.join(appRoot, "hardlink-a.bin")
|
|
1959
|
+
const hardlinkB = path.join(appRoot, "hardlink-b.bin")
|
|
1960
|
+
const missingCandidate = path.join(appRoot, "missing-peer-candidate.bin")
|
|
1961
|
+
const staleCandidate = path.join(appRoot, "stale-peer-candidate.bin")
|
|
1962
|
+
const anchorCandidate = path.join(appRoot, "anchor-hardlink.bin")
|
|
1963
|
+
const anchorPeer = path.join(home, "anchor-hardlink-peer.bin")
|
|
1964
|
+
await fs.promises.mkdir(appRoot)
|
|
1965
|
+
await fs.promises.writeFile(unique, "unique")
|
|
1966
|
+
await fs.promises.truncate(unique, 64 * 1024 * 1024)
|
|
1967
|
+
await fs.promises.writeFile(duplicateA, "dupe")
|
|
1968
|
+
await fs.promises.writeFile(duplicateB, "dupe")
|
|
1969
|
+
await fs.promises.writeFile(hardlinkA, "links")
|
|
1970
|
+
await fs.promises.link(hardlinkA, hardlinkB)
|
|
1971
|
+
await fs.promises.writeFile(missingCandidate, "absent")
|
|
1972
|
+
await fs.promises.writeFile(staleCandidate, "outdated")
|
|
1973
|
+
await fs.promises.writeFile(anchorCandidate, "anchorlink")
|
|
1974
|
+
await fs.promises.link(anchorCandidate, anchorPeer)
|
|
1975
|
+
const vault = await makeVault(home)
|
|
1976
|
+
vault.automaticScans.candidateThreshold = async () => 1
|
|
1977
|
+
const missingStat = await fs.promises.lstat(missingCandidate)
|
|
1978
|
+
const staleStat = await fs.promises.lstat(staleCandidate)
|
|
1979
|
+
const peer = (filePath, stat, ino, unavailableReason = null) => ({
|
|
1980
|
+
path: filePath,
|
|
1557
1981
|
hash: null,
|
|
1558
1982
|
size: stat.size,
|
|
1559
1983
|
mtime: 1,
|
|
1560
1984
|
ctime: 1,
|
|
1561
1985
|
dev: stat.dev,
|
|
1986
|
+
ino,
|
|
1562
1987
|
mode: stat.mode,
|
|
1563
1988
|
uid: stat.uid,
|
|
1564
1989
|
gid: stat.gid,
|
|
1565
1990
|
source_id: "app:peer",
|
|
1566
1991
|
app: "peer",
|
|
1567
1992
|
status: "reference",
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
assert.deepEqual(vault.automaticScans.snapshot().rows, [])
|
|
1586
|
-
await close(vault)
|
|
1587
|
-
})
|
|
1588
|
-
|
|
1589
|
-
test("a current verified store record can be a verified peer", async () => {
|
|
1590
|
-
const home = await makeHome()
|
|
1591
|
-
const appRoot = path.join(home, "api", "store-peer-app")
|
|
1592
|
-
const candidate = path.join(appRoot, "candidate.bin")
|
|
1593
|
-
await fs.promises.mkdir(appRoot)
|
|
1594
|
-
await fs.promises.writeFile(candidate, "same-size")
|
|
1595
|
-
const anchorPath = path.join(home, "anchor.bin")
|
|
1596
|
-
await fs.promises.writeFile(anchorPath, "same-size")
|
|
1597
|
-
const vault = await makeVault(home)
|
|
1598
|
-
vault.automaticScans.candidateThreshold = async () => 1
|
|
1599
|
-
const stat = await fs.promises.lstat(candidate)
|
|
1600
|
-
const anchorStat = await fs.promises.lstat(anchorPath)
|
|
1601
|
-
const hash = crypto.createHash("sha256").update("same-size").digest("hex")
|
|
1993
|
+
unavailable_reason: unavailableReason,
|
|
1994
|
+
updated_at: 1
|
|
1995
|
+
})
|
|
1996
|
+
await vault.registry.upsertFile(peer(
|
|
1997
|
+
path.join(home, "missing-peer.bin"),
|
|
1998
|
+
missingStat,
|
|
1999
|
+
missingStat.ino + 1
|
|
2000
|
+
))
|
|
2001
|
+
await vault.registry.upsertFile(peer(
|
|
2002
|
+
path.join(home, "stale-peer.bin"),
|
|
2003
|
+
staleStat,
|
|
2004
|
+
staleStat.ino + 1,
|
|
2005
|
+
"stale"
|
|
2006
|
+
))
|
|
2007
|
+
const anchorStat = await fs.promises.lstat(anchorPeer)
|
|
2008
|
+
const anchorHash = crypto.createHash("sha256")
|
|
2009
|
+
.update("anchorlink").digest("hex")
|
|
1602
2010
|
await vault.registry.upsertContent({
|
|
1603
|
-
hash,
|
|
1604
|
-
size:
|
|
1605
|
-
first_seen:
|
|
1606
|
-
verified_at:
|
|
2011
|
+
hash: anchorHash,
|
|
2012
|
+
size: anchorStat.size,
|
|
2013
|
+
first_seen: 1,
|
|
2014
|
+
verified_at: 1,
|
|
1607
2015
|
anchor_present: true
|
|
1608
2016
|
})
|
|
1609
2017
|
await vault.registry.upsertAnchor({
|
|
1610
|
-
store_id: "
|
|
1611
|
-
hash,
|
|
1612
|
-
path:
|
|
1613
|
-
verified_at:
|
|
2018
|
+
store_id: "same-inode-store",
|
|
2019
|
+
hash: anchorHash,
|
|
2020
|
+
path: anchorPeer,
|
|
2021
|
+
verified_at: 1,
|
|
1614
2022
|
dev: anchorStat.dev,
|
|
1615
2023
|
ino: anchorStat.ino,
|
|
1616
2024
|
size: anchorStat.size,
|
|
@@ -1621,22 +2029,465 @@ describe("automatic app checks", () => {
|
|
|
1621
2029
|
uid: anchorStat.uid,
|
|
1622
2030
|
gid: anchorStat.gid
|
|
1623
2031
|
})
|
|
2032
|
+
const logs = []
|
|
2033
|
+
vault.automaticScans.log = (event, details) => {
|
|
2034
|
+
logs.push({ event, details })
|
|
2035
|
+
}
|
|
2036
|
+
const hashedPaths = []
|
|
2037
|
+
const originalHashFile = vault.hashFile.bind(vault)
|
|
2038
|
+
vault.hashFile = async (filePath, options) => {
|
|
2039
|
+
hashedPaths.push(path.resolve(filePath))
|
|
2040
|
+
if (path.resolve(filePath) === path.resolve(unique)) {
|
|
2041
|
+
return { hash: "f".repeat(64), size: 64 * 1024 * 1024 }
|
|
2042
|
+
}
|
|
2043
|
+
return originalHashFile(filePath, options)
|
|
2044
|
+
}
|
|
2045
|
+
|
|
2046
|
+
vault.automaticScans.queueApp(app, [
|
|
2047
|
+
unique,
|
|
2048
|
+
duplicateA,
|
|
2049
|
+
duplicateB,
|
|
2050
|
+
hardlinkA,
|
|
2051
|
+
hardlinkB,
|
|
2052
|
+
missingCandidate,
|
|
2053
|
+
staleCandidate,
|
|
2054
|
+
anchorCandidate
|
|
2055
|
+
])
|
|
2056
|
+
await waitFor(() => !vault.automaticScans.active &&
|
|
2057
|
+
vault.automaticScans.entries.get(app)?.state === "result")
|
|
2058
|
+
|
|
2059
|
+
assert.deepEqual(hashedPaths.sort(), [duplicateA, duplicateB].sort())
|
|
2060
|
+
const finished = logs.find((record) => record.event === "check-finished")
|
|
2061
|
+
assert.equal(finished.details.candidates, 2)
|
|
2062
|
+
assert.equal(finished.details.hashed, 2)
|
|
2063
|
+
assert.equal(finished.details.hash_bytes, 8)
|
|
2064
|
+
await close(vault)
|
|
2065
|
+
})
|
|
2066
|
+
|
|
2067
|
+
test("automatic verification reads each current inode at most once", async () => {
|
|
2068
|
+
const home = await makeHome()
|
|
2069
|
+
const app = "single-inode-read-app"
|
|
2070
|
+
const appRoot = path.join(home, "api", app)
|
|
2071
|
+
const first = path.join(appRoot, "first.bin")
|
|
2072
|
+
const second = path.join(appRoot, "second.bin")
|
|
2073
|
+
const peer = path.join(home, "peer.bin")
|
|
2074
|
+
await fs.promises.mkdir(appRoot)
|
|
2075
|
+
await fs.promises.writeFile(first, "same-content")
|
|
2076
|
+
await fs.promises.link(first, second)
|
|
2077
|
+
await fs.promises.writeFile(peer, "same-content")
|
|
2078
|
+
const vault = await makeVault(home)
|
|
2079
|
+
vault.automaticScans.candidateThreshold = async () => 1
|
|
2080
|
+
const peerStat = await fs.promises.lstat(peer)
|
|
2081
|
+
await vault.registry.upsertFile({
|
|
2082
|
+
path: peer,
|
|
2083
|
+
hash: null,
|
|
2084
|
+
size: peerStat.size,
|
|
2085
|
+
mtime: peerStat.mtimeMs,
|
|
2086
|
+
ctime: peerStat.ctimeMs,
|
|
2087
|
+
dev: peerStat.dev,
|
|
2088
|
+
ino: peerStat.ino,
|
|
2089
|
+
mode: peerStat.mode,
|
|
2090
|
+
uid: peerStat.uid,
|
|
2091
|
+
gid: peerStat.gid,
|
|
2092
|
+
source_id: "app:peer",
|
|
2093
|
+
app: "peer",
|
|
2094
|
+
status: "reference",
|
|
2095
|
+
unavailable_reason: null,
|
|
2096
|
+
updated_at: 1
|
|
2097
|
+
})
|
|
2098
|
+
const hashedPaths = []
|
|
2099
|
+
const originalHashFile = vault.hashFile.bind(vault)
|
|
2100
|
+
vault.hashFile = async (filePath, options) => {
|
|
2101
|
+
hashedPaths.push(filePath)
|
|
2102
|
+
return originalHashFile(filePath, options)
|
|
2103
|
+
}
|
|
1624
2104
|
|
|
1625
|
-
vault.automaticScans.queueApp(
|
|
2105
|
+
vault.automaticScans.queueApp(app, [first, second])
|
|
1626
2106
|
await waitFor(() => !vault.automaticScans.active &&
|
|
1627
|
-
vault.automaticScans.entries.get(
|
|
2107
|
+
vault.automaticScans.entries.get(app)?.state === "result")
|
|
2108
|
+
|
|
2109
|
+
assert.equal(hashedPaths.filter((filePath) =>
|
|
2110
|
+
[first, second].includes(filePath)).length, 1)
|
|
2111
|
+
assert.equal(hashedPaths.filter((filePath) => filePath === peer).length, 1)
|
|
2112
|
+
assert.equal(hashedPaths.length, 2)
|
|
2113
|
+
await close(vault)
|
|
2114
|
+
})
|
|
2115
|
+
|
|
2116
|
+
test("peer verification rejects changed type, device, and size", async (t) => {
|
|
2117
|
+
const home = await makeHome()
|
|
2118
|
+
const automatic = makeAutomatic({
|
|
2119
|
+
enabled: true,
|
|
2120
|
+
kernel: { homedir: home, platform: "darwin" }
|
|
2121
|
+
})
|
|
2122
|
+
const active = {
|
|
2123
|
+
app: "changed-peer-app",
|
|
2124
|
+
cancelled: false
|
|
2125
|
+
}
|
|
2126
|
+
automatic.active = active
|
|
2127
|
+
automatic.log = () => {}
|
|
2128
|
+
const cases = [
|
|
2129
|
+
{
|
|
2130
|
+
name: "directory",
|
|
2131
|
+
stat: { file: false, link: false, dev: 1, size: 4 }
|
|
2132
|
+
},
|
|
2133
|
+
{
|
|
2134
|
+
name: "symbolic-link",
|
|
2135
|
+
stat: { file: true, link: true, dev: 1, size: 4 }
|
|
2136
|
+
},
|
|
2137
|
+
{
|
|
2138
|
+
name: "device",
|
|
2139
|
+
stat: { file: true, link: false, dev: 2, size: 4 }
|
|
2140
|
+
},
|
|
2141
|
+
{
|
|
2142
|
+
name: "size",
|
|
2143
|
+
stat: { file: true, link: false, dev: 1, size: 5 }
|
|
2144
|
+
}
|
|
2145
|
+
]
|
|
2146
|
+
const currentStats = new Map(cases.map(({ name, stat }) => [
|
|
2147
|
+
path.join(home, `${name}.bin`),
|
|
2148
|
+
{
|
|
2149
|
+
isFile: () => stat.file,
|
|
2150
|
+
isSymbolicLink: () => stat.link,
|
|
2151
|
+
dev: stat.dev,
|
|
2152
|
+
size: stat.size,
|
|
2153
|
+
ino: 2,
|
|
2154
|
+
mtimeMs: 1,
|
|
2155
|
+
ctimeMs: 1,
|
|
2156
|
+
nlink: 1,
|
|
2157
|
+
mode: 0,
|
|
2158
|
+
uid: 0,
|
|
2159
|
+
gid: 0
|
|
2160
|
+
}
|
|
2161
|
+
]))
|
|
2162
|
+
const lstat = fs.promises.lstat
|
|
2163
|
+
t.after(() => { fs.promises.lstat = lstat })
|
|
2164
|
+
fs.promises.lstat = async (filePath) => currentStats.get(filePath)
|
|
2165
|
+
const counts = { path_errors: 0, unstable_hashes: 0 }
|
|
2166
|
+
|
|
2167
|
+
for (const filePath of currentStats.keys()) {
|
|
2168
|
+
assert.equal(await automatic.currentAutomaticEntry(active, {
|
|
2169
|
+
kind: "file",
|
|
2170
|
+
path: filePath,
|
|
2171
|
+
dev: 1,
|
|
2172
|
+
size: 4,
|
|
2173
|
+
ino: 1,
|
|
2174
|
+
mtime: 1,
|
|
2175
|
+
ctime: 1
|
|
2176
|
+
}, counts), null)
|
|
2177
|
+
}
|
|
1628
2178
|
|
|
2179
|
+
assert.equal(counts.unstable_hashes, cases.length)
|
|
2180
|
+
assert.equal(counts.path_errors, 0)
|
|
2181
|
+
})
|
|
2182
|
+
|
|
2183
|
+
test("files that become hardlinks after staging are not duplicates", async () => {
|
|
2184
|
+
const home = await makeHome()
|
|
2185
|
+
const app = "became-hardlinks-app"
|
|
2186
|
+
const appRoot = path.join(home, "api", app)
|
|
2187
|
+
const first = path.join(appRoot, "first.bin")
|
|
2188
|
+
const second = path.join(appRoot, "second.bin")
|
|
2189
|
+
await fs.promises.mkdir(appRoot)
|
|
2190
|
+
await fs.promises.writeFile(first, "same-content")
|
|
2191
|
+
await fs.promises.writeFile(second, "same-content")
|
|
2192
|
+
const before = await Promise.all([
|
|
2193
|
+
fs.promises.lstat(first),
|
|
2194
|
+
fs.promises.lstat(second)
|
|
2195
|
+
])
|
|
2196
|
+
assert.notEqual(before[0].ino, before[1].ino)
|
|
2197
|
+
const vault = await makeVault(home)
|
|
2198
|
+
vault.automaticScans.candidateThreshold = async () => 1
|
|
2199
|
+
const stage = vault.registry.stageAutomaticPrecheckFiles.bind(
|
|
2200
|
+
vault.registry)
|
|
2201
|
+
vault.registry.stageAutomaticPrecheckFiles = async (...args) => {
|
|
2202
|
+
await stage(...args)
|
|
2203
|
+
await fs.promises.unlink(second)
|
|
2204
|
+
await fs.promises.link(first, second)
|
|
2205
|
+
}
|
|
2206
|
+
const hashedPaths = []
|
|
2207
|
+
const originalHashFile = vault.hashFile.bind(vault)
|
|
2208
|
+
vault.hashFile = async (filePath, options) => {
|
|
2209
|
+
hashedPaths.push(filePath)
|
|
2210
|
+
return originalHashFile(filePath, options)
|
|
2211
|
+
}
|
|
2212
|
+
|
|
2213
|
+
vault.automaticScans.queueApp(app, [first, second])
|
|
2214
|
+
await waitFor(() => !vault.automaticScans.active &&
|
|
2215
|
+
!vault.automaticScans.entries.has(app))
|
|
2216
|
+
|
|
2217
|
+
const after = await Promise.all([
|
|
2218
|
+
fs.promises.lstat(first),
|
|
2219
|
+
fs.promises.lstat(second)
|
|
2220
|
+
])
|
|
2221
|
+
assert.equal(after[0].ino, after[1].ino)
|
|
2222
|
+
assert.deepEqual(hashedPaths, [])
|
|
2223
|
+
assert.deepEqual(vault.automaticScans.snapshot().rows, [])
|
|
2224
|
+
await close(vault)
|
|
2225
|
+
})
|
|
2226
|
+
|
|
2227
|
+
test("hardlinks that become independent after staging are duplicates", async () => {
|
|
2228
|
+
const home = await makeHome()
|
|
2229
|
+
const app = "split-hardlinks-app"
|
|
2230
|
+
const appRoot = path.join(home, "api", app)
|
|
2231
|
+
const first = path.join(appRoot, "first.bin")
|
|
2232
|
+
const second = path.join(appRoot, "second.bin")
|
|
2233
|
+
await fs.promises.mkdir(appRoot)
|
|
2234
|
+
await fs.promises.writeFile(first, "same-content")
|
|
2235
|
+
await fs.promises.link(first, second)
|
|
2236
|
+
const before = await Promise.all([
|
|
2237
|
+
fs.promises.lstat(first),
|
|
2238
|
+
fs.promises.lstat(second)
|
|
2239
|
+
])
|
|
2240
|
+
assert.equal(before[0].ino, before[1].ino)
|
|
2241
|
+
const vault = await makeVault(home)
|
|
2242
|
+
vault.automaticScans.candidateThreshold = async () => 1
|
|
2243
|
+
const stage = vault.registry.stageAutomaticPrecheckFiles.bind(
|
|
2244
|
+
vault.registry)
|
|
2245
|
+
vault.registry.stageAutomaticPrecheckFiles = async (...args) => {
|
|
2246
|
+
await stage(...args)
|
|
2247
|
+
await fs.promises.unlink(second)
|
|
2248
|
+
await fs.promises.writeFile(second, "same-content")
|
|
2249
|
+
}
|
|
2250
|
+
const hashedPaths = []
|
|
2251
|
+
const originalHashFile = vault.hashFile.bind(vault)
|
|
2252
|
+
vault.hashFile = async (filePath, options) => {
|
|
2253
|
+
hashedPaths.push(filePath)
|
|
2254
|
+
return originalHashFile(filePath, options)
|
|
2255
|
+
}
|
|
2256
|
+
|
|
2257
|
+
vault.automaticScans.queueApp(app, [first, second])
|
|
2258
|
+
await waitFor(() => !vault.automaticScans.active &&
|
|
2259
|
+
vault.automaticScans.entries.get(app)?.state === "result")
|
|
2260
|
+
|
|
2261
|
+
const after = await Promise.all([
|
|
2262
|
+
fs.promises.lstat(first),
|
|
2263
|
+
fs.promises.lstat(second)
|
|
2264
|
+
])
|
|
2265
|
+
assert.notEqual(after[0].ino, after[1].ino)
|
|
2266
|
+
assert.deepEqual(hashedPaths.sort(), [first, second].sort())
|
|
1629
2267
|
assert.equal(vault.automaticScans.snapshot().rows[0].state, "result")
|
|
1630
2268
|
await close(vault)
|
|
1631
2269
|
})
|
|
1632
2270
|
|
|
1633
|
-
test("
|
|
2271
|
+
test("stale file rows and known identical inodes are not peers", async () => {
|
|
2272
|
+
const home = await makeHome()
|
|
2273
|
+
const appRoot = path.join(home, "api", "no-peer-app")
|
|
2274
|
+
const candidate = path.join(appRoot, "candidate.bin")
|
|
2275
|
+
await fs.promises.mkdir(appRoot)
|
|
2276
|
+
await fs.promises.writeFile(candidate, "same-size")
|
|
2277
|
+
const sameInodePeer = path.join(home, "same-inode-peer.bin")
|
|
2278
|
+
await fs.promises.link(candidate, sameInodePeer)
|
|
2279
|
+
const vault = await makeVault(home, { deferStorage: true })
|
|
2280
|
+
await vault.ensureRegistryInitialized()
|
|
2281
|
+
assert.equal(vault.initialized, false)
|
|
2282
|
+
vault.automaticScans.candidateThreshold = async () => 1
|
|
2283
|
+
const stat = await fs.promises.lstat(candidate)
|
|
2284
|
+
const base = {
|
|
2285
|
+
hash: null,
|
|
2286
|
+
size: stat.size,
|
|
2287
|
+
mtime: 1,
|
|
2288
|
+
ctime: 1,
|
|
2289
|
+
dev: stat.dev,
|
|
2290
|
+
mode: stat.mode,
|
|
2291
|
+
uid: stat.uid,
|
|
2292
|
+
gid: stat.gid,
|
|
2293
|
+
source_id: "app:peer",
|
|
2294
|
+
app: "peer",
|
|
2295
|
+
status: "reference",
|
|
2296
|
+
updated_at: Date.now()
|
|
2297
|
+
}
|
|
2298
|
+
await vault.registry.upsertFile(Object.assign({}, base, {
|
|
2299
|
+
path: path.join(home, "stale-peer.bin"),
|
|
2300
|
+
ino: stat.ino + 1,
|
|
2301
|
+
unavailable_reason: "stale"
|
|
2302
|
+
}))
|
|
2303
|
+
await vault.registry.upsertFile(Object.assign({}, base, {
|
|
2304
|
+
path: sameInodePeer,
|
|
2305
|
+
ino: stat.ino,
|
|
2306
|
+
unavailable_reason: null
|
|
2307
|
+
}))
|
|
2308
|
+
const hashedPaths = []
|
|
2309
|
+
vault.hashFile = async (filePath) => {
|
|
2310
|
+
hashedPaths.push(filePath)
|
|
2311
|
+
throw new Error("Known hardlinks must not be read for verification.")
|
|
2312
|
+
}
|
|
2313
|
+
|
|
2314
|
+
vault.automaticScans.queueApp("no-peer-app", [candidate])
|
|
2315
|
+
await waitFor(() => !vault.automaticScans.active &&
|
|
2316
|
+
!vault.automaticScans.entries.has("no-peer-app"))
|
|
2317
|
+
|
|
2318
|
+
assert.deepEqual(vault.automaticScans.snapshot().rows, [])
|
|
2319
|
+
assert.deepEqual(hashedPaths, [])
|
|
2320
|
+
await close(vault)
|
|
2321
|
+
})
|
|
2322
|
+
|
|
2323
|
+
test("an unstable comparison is skipped before a later proof", async (t) => {
|
|
2324
|
+
const home = await makeHome()
|
|
2325
|
+
const app = "unstable-then-match-app"
|
|
2326
|
+
const appRoot = path.join(home, "api", app)
|
|
2327
|
+
const peerRoot = path.join(home, "unstable-test-peers")
|
|
2328
|
+
await fs.promises.mkdir(appRoot)
|
|
2329
|
+
await fs.promises.mkdir(peerRoot)
|
|
2330
|
+
const groups = [
|
|
2331
|
+
["unstable", "aaaa", "aaaa"],
|
|
2332
|
+
["matching", "bbbbb", "bbbbb"],
|
|
2333
|
+
["later", "cccccc", "dddddd"]
|
|
2334
|
+
]
|
|
2335
|
+
const candidates = []
|
|
2336
|
+
const peers = []
|
|
2337
|
+
const vault = await makeVault(home)
|
|
2338
|
+
t.after(() => close(vault))
|
|
2339
|
+
vault.automaticScans.candidateThreshold = async () => 1
|
|
2340
|
+
for (const [name, candidateContent, peerContent] of groups) {
|
|
2341
|
+
const candidate = path.join(appRoot, `${name}.bin`)
|
|
2342
|
+
const peer = path.join(peerRoot, `${name}.bin`)
|
|
2343
|
+
await fs.promises.writeFile(candidate, candidateContent)
|
|
2344
|
+
await fs.promises.writeFile(peer, peerContent)
|
|
2345
|
+
candidates.push(candidate)
|
|
2346
|
+
peers.push(peer)
|
|
2347
|
+
const stat = await fs.promises.lstat(peer)
|
|
2348
|
+
await vault.registry.upsertFile({
|
|
2349
|
+
path: peer,
|
|
2350
|
+
hash: null,
|
|
2351
|
+
size: stat.size,
|
|
2352
|
+
mtime: stat.mtimeMs,
|
|
2353
|
+
ctime: stat.ctimeMs,
|
|
2354
|
+
dev: stat.dev,
|
|
2355
|
+
ino: stat.ino,
|
|
2356
|
+
mode: stat.mode,
|
|
2357
|
+
uid: stat.uid,
|
|
2358
|
+
gid: stat.gid,
|
|
2359
|
+
source_id: "app:unstable-test-peers",
|
|
2360
|
+
app: "unstable-test-peers",
|
|
2361
|
+
status: "reference",
|
|
2362
|
+
unavailable_reason: null,
|
|
2363
|
+
updated_at: Date.now()
|
|
2364
|
+
})
|
|
2365
|
+
}
|
|
2366
|
+
const unstablePeer = peers[0]
|
|
2367
|
+
const hashedPaths = []
|
|
2368
|
+
const hashFile = vault.hashFile.bind(vault)
|
|
2369
|
+
vault.hashFile = async (filePath, options) => {
|
|
2370
|
+
hashedPaths.push(filePath)
|
|
2371
|
+
return hashFile(filePath, options)
|
|
2372
|
+
}
|
|
2373
|
+
const hashStable = vault.scanner.hashStable.bind(vault.scanner)
|
|
2374
|
+
vault.scanner.hashStable = async (entry, options) => {
|
|
2375
|
+
const result = await hashStable(entry, options)
|
|
2376
|
+
return entry.path === unstablePeer
|
|
2377
|
+
? Object.assign({}, result, { stable: false })
|
|
2378
|
+
: result
|
|
2379
|
+
}
|
|
2380
|
+
|
|
2381
|
+
vault.automaticScans.queueApp(app, candidates.slice().reverse())
|
|
2382
|
+
await waitFor(() => !vault.automaticScans.active &&
|
|
2383
|
+
vault.automaticScans.entries.get(app)?.state === "result")
|
|
2384
|
+
|
|
2385
|
+
assert.deepEqual(hashedPaths, [
|
|
2386
|
+
candidates[0],
|
|
2387
|
+
unstablePeer,
|
|
2388
|
+
candidates[1],
|
|
2389
|
+
peers[1]
|
|
2390
|
+
])
|
|
2391
|
+
assert.equal(vault.automaticScans.snapshot().rows[0].state, "result")
|
|
2392
|
+
})
|
|
2393
|
+
|
|
2394
|
+
test("the first verified store proof stops later peer work", async (t) => {
|
|
2395
|
+
const home = await makeHome()
|
|
2396
|
+
const app = "store-peer-app"
|
|
2397
|
+
const appRoot = path.join(home, "api", app)
|
|
2398
|
+
const candidate = path.join(appRoot, "candidate.bin")
|
|
2399
|
+
const laterCandidate = path.join(appRoot, "later-candidate.bin")
|
|
2400
|
+
await fs.promises.mkdir(appRoot)
|
|
2401
|
+
await fs.promises.writeFile(candidate, "same-size")
|
|
2402
|
+
await fs.promises.writeFile(laterCandidate, "later-copy")
|
|
2403
|
+
const firstAnchor = path.join(home, "anchor-a.bin")
|
|
2404
|
+
const matchingAnchor = path.join(home, "anchor-b.bin")
|
|
2405
|
+
const laterPeer = path.join(home, "later-peer.bin")
|
|
2406
|
+
await fs.promises.writeFile(firstAnchor, "not-equal")
|
|
2407
|
+
await fs.promises.writeFile(matchingAnchor, "same-size")
|
|
2408
|
+
await fs.promises.writeFile(laterPeer, "later-copy")
|
|
2409
|
+
const vault = await makeVault(home)
|
|
2410
|
+
t.after(() => close(vault))
|
|
2411
|
+
vault.automaticScans.candidateThreshold = async () => 1
|
|
2412
|
+
for (const [index, anchorPath] of [firstAnchor, matchingAnchor].entries()) {
|
|
2413
|
+
const anchorStat = await fs.promises.lstat(anchorPath)
|
|
2414
|
+
const hash = crypto.createHash("sha256")
|
|
2415
|
+
.update(index === 0 ? "not-equal" : "same-size").digest("hex")
|
|
2416
|
+
await vault.registry.upsertContent({
|
|
2417
|
+
hash,
|
|
2418
|
+
size: anchorStat.size,
|
|
2419
|
+
first_seen: Date.now(),
|
|
2420
|
+
verified_at: Date.now(),
|
|
2421
|
+
anchor_present: true
|
|
2422
|
+
})
|
|
2423
|
+
await vault.registry.upsertAnchor({
|
|
2424
|
+
store_id: `verified-store-${index}`,
|
|
2425
|
+
hash,
|
|
2426
|
+
path: anchorPath,
|
|
2427
|
+
verified_at: Date.now(),
|
|
2428
|
+
dev: anchorStat.dev,
|
|
2429
|
+
ino: anchorStat.ino,
|
|
2430
|
+
size: anchorStat.size,
|
|
2431
|
+
mtime: anchorStat.mtimeMs,
|
|
2432
|
+
ctime: anchorStat.ctimeMs,
|
|
2433
|
+
nlink: anchorStat.nlink,
|
|
2434
|
+
mode: anchorStat.mode,
|
|
2435
|
+
uid: anchorStat.uid,
|
|
2436
|
+
gid: anchorStat.gid
|
|
2437
|
+
})
|
|
2438
|
+
}
|
|
2439
|
+
const laterStat = await fs.promises.lstat(laterPeer)
|
|
2440
|
+
await vault.registry.upsertFile({
|
|
2441
|
+
path: laterPeer,
|
|
2442
|
+
hash: null,
|
|
2443
|
+
size: laterStat.size,
|
|
2444
|
+
mtime: laterStat.mtimeMs,
|
|
2445
|
+
ctime: laterStat.ctimeMs,
|
|
2446
|
+
dev: laterStat.dev,
|
|
2447
|
+
ino: laterStat.ino,
|
|
2448
|
+
mode: laterStat.mode,
|
|
2449
|
+
uid: laterStat.uid,
|
|
2450
|
+
gid: laterStat.gid,
|
|
2451
|
+
source_id: "app:later-peer",
|
|
2452
|
+
app: "later-peer",
|
|
2453
|
+
status: "reference",
|
|
2454
|
+
unavailable_reason: null,
|
|
2455
|
+
updated_at: Date.now()
|
|
2456
|
+
})
|
|
2457
|
+
const validatedPaths = []
|
|
2458
|
+
const currentEntry = vault.automaticScans.currentAutomaticEntry.bind(
|
|
2459
|
+
vault.automaticScans)
|
|
2460
|
+
vault.automaticScans.currentAutomaticEntry = async (...args) => {
|
|
2461
|
+
validatedPaths.push(args[1].path)
|
|
2462
|
+
return currentEntry(...args)
|
|
2463
|
+
}
|
|
2464
|
+
const hashedPaths = []
|
|
2465
|
+
const hashFile = vault.hashFile.bind(vault)
|
|
2466
|
+
vault.hashFile = async (filePath, options) => {
|
|
2467
|
+
hashedPaths.push(filePath)
|
|
2468
|
+
return hashFile(filePath, options)
|
|
2469
|
+
}
|
|
2470
|
+
|
|
2471
|
+
vault.automaticScans.queueApp(app, [laterCandidate, candidate])
|
|
2472
|
+
await waitFor(() => !vault.automaticScans.active &&
|
|
2473
|
+
vault.automaticScans.entries.get(app)?.state === "result")
|
|
2474
|
+
|
|
2475
|
+
assert.equal(validatedPaths.includes(candidate), true)
|
|
2476
|
+
assert.equal(validatedPaths.includes(matchingAnchor), true)
|
|
2477
|
+
assert.equal(validatedPaths.includes(laterCandidate), false)
|
|
2478
|
+
assert.equal(validatedPaths.includes(laterPeer), false)
|
|
2479
|
+
assert.deepEqual(hashedPaths, [candidate])
|
|
2480
|
+
assert.equal(vault.automaticScans.snapshot().rows[0].state, "result")
|
|
2481
|
+
})
|
|
2482
|
+
|
|
2483
|
+
test("acknowledgement follows the verified-positive changed batch", async (t) => {
|
|
1634
2484
|
const home = await makeHome()
|
|
1635
2485
|
const appRoot = path.join(home, "api", "acknowledged-app")
|
|
1636
2486
|
await fs.promises.mkdir(appRoot)
|
|
1637
2487
|
await fs.promises.writeFile(path.join(appRoot, "first.bin"), "aaaa")
|
|
1638
2488
|
await fs.promises.writeFile(path.join(appRoot, "second.bin"), "aaaa")
|
|
1639
2489
|
const vault = await makeVault(home)
|
|
2490
|
+
t.after(() => close(vault))
|
|
1640
2491
|
vault.automaticScans.candidateThreshold = async () => 1
|
|
1641
2492
|
|
|
1642
2493
|
const acknowledgedPaths = [
|
|
@@ -1658,17 +2509,42 @@ describe("automatic app checks", () => {
|
|
|
1658
2509
|
vault.automaticScans.entries.get("acknowledged-app")?.state === "result")
|
|
1659
2510
|
assert.deepEqual(vault.automaticScans.snapshot().rows, [])
|
|
1660
2511
|
|
|
1661
|
-
|
|
2512
|
+
const third = path.join(appRoot, "third.bin")
|
|
2513
|
+
await fs.promises.writeFile(third, "bbbb")
|
|
1662
2514
|
const previousSignature = vault.automaticScans.entries.get(
|
|
1663
2515
|
"acknowledged-app").signature
|
|
2516
|
+
const hashedPaths = []
|
|
2517
|
+
const originalHashFile = vault.hashFile.bind(vault)
|
|
2518
|
+
vault.hashFile = async (filePath, options) => {
|
|
2519
|
+
hashedPaths.push(filePath)
|
|
2520
|
+
return originalHashFile(filePath, options)
|
|
2521
|
+
}
|
|
1664
2522
|
vault.automaticScans.queueApp("acknowledged-app", acknowledgedPaths.concat(
|
|
1665
|
-
|
|
2523
|
+
third))
|
|
2524
|
+
await waitFor(() => !vault.automaticScans.active &&
|
|
2525
|
+
vault.automaticScans.entries.get("acknowledged-app")?.state === "result")
|
|
2526
|
+
assert.notEqual(vault.automaticScans.entries.get(
|
|
2527
|
+
"acknowledged-app").signature, previousSignature)
|
|
2528
|
+
assert.deepEqual(hashedPaths, [])
|
|
2529
|
+
assert.equal(vault.automaticScans.snapshot().rows[0].state, "result")
|
|
2530
|
+
|
|
2531
|
+
const changedBatchSignature = vault.automaticScans.entries.get(
|
|
2532
|
+
"acknowledged-app").signature
|
|
2533
|
+
await vault.perform("automatic_acknowledge", {
|
|
2534
|
+
app: "acknowledged-app",
|
|
2535
|
+
signature: changedBatchSignature
|
|
2536
|
+
})
|
|
2537
|
+
const changed = new Date(Date.now() + 10000)
|
|
2538
|
+
await fs.promises.utimes(third, changed, changed)
|
|
2539
|
+
hashedPaths.length = 0
|
|
2540
|
+
vault.automaticScans.queueApp(
|
|
2541
|
+
"acknowledged-app", acknowledgedPaths.concat(third))
|
|
1666
2542
|
await waitFor(() => !vault.automaticScans.active &&
|
|
1667
|
-
vault.automaticScans.entries.get("acknowledged-app")?.state === "result"
|
|
1668
|
-
|
|
1669
|
-
|
|
2543
|
+
vault.automaticScans.entries.get("acknowledged-app")?.state === "result")
|
|
2544
|
+
assert.notEqual(vault.automaticScans.entries.get(
|
|
2545
|
+
"acknowledged-app").signature, changedBatchSignature)
|
|
2546
|
+
assert.deepEqual(hashedPaths, [])
|
|
1670
2547
|
assert.equal(vault.automaticScans.snapshot().rows[0].state, "result")
|
|
1671
|
-
await close(vault)
|
|
1672
2548
|
})
|
|
1673
2549
|
|
|
1674
2550
|
test("an empty replacement clears the result and its acknowledgement", async () => {
|
|
@@ -1702,7 +2578,7 @@ describe("automatic app checks", () => {
|
|
|
1702
2578
|
await close(vault)
|
|
1703
2579
|
})
|
|
1704
2580
|
|
|
1705
|
-
test("an app restart
|
|
2581
|
+
test("an app restart preserves paths for the next final stop", async () => {
|
|
1706
2582
|
const home = await makeHome()
|
|
1707
2583
|
const appRoot = path.join(home, "api", "restart-app")
|
|
1708
2584
|
const scriptPath = path.join(appRoot, "start.js")
|
|
@@ -1711,6 +2587,7 @@ describe("automatic app checks", () => {
|
|
|
1711
2587
|
await fs.promises.writeFile(path.join(appRoot, "second.bin"), "aaaa")
|
|
1712
2588
|
const vault = await makeVault(home)
|
|
1713
2589
|
vault.automaticScans.candidateThreshold = async () => 1
|
|
2590
|
+
vault.automaticScans.stopSettleMs = 0
|
|
1714
2591
|
const restartPaths = [
|
|
1715
2592
|
path.join(appRoot, "first.bin"),
|
|
1716
2593
|
path.join(appRoot, "second.bin")
|
|
@@ -1730,7 +2607,8 @@ describe("automatic app checks", () => {
|
|
|
1730
2607
|
await new Promise((resolve) => { release = resolve })
|
|
1731
2608
|
return originalStat(paths, options)
|
|
1732
2609
|
}
|
|
1733
|
-
vault
|
|
2610
|
+
collect(vault, "restart-app", restartPaths)
|
|
2611
|
+
vault.automaticScans.queueApp("restart-app")
|
|
1734
2612
|
await beganPromise
|
|
1735
2613
|
vault.automaticScans.handleStarted(scriptPath)
|
|
1736
2614
|
release()
|
|
@@ -1740,6 +2618,20 @@ describe("automatic app checks", () => {
|
|
|
1740
2618
|
const restored = vault.automaticScans.entries.get("restart-app")
|
|
1741
2619
|
assert.equal(restored.state, "result")
|
|
1742
2620
|
assert.equal(restored.signature, previous.signature)
|
|
2621
|
+
assert.deepEqual(
|
|
2622
|
+
vault.automaticScans.changedPaths.get("restart-app"),
|
|
2623
|
+
new Set(restartPaths)
|
|
2624
|
+
)
|
|
2625
|
+
|
|
2626
|
+
await vault.automaticScans.handleStopped(scriptPath)
|
|
2627
|
+
await waitFor(() => !vault.automaticScans.active &&
|
|
2628
|
+
!vault.automaticScans.pendingStops.has("restart-app") &&
|
|
2629
|
+
!vault.automaticScans.changedPaths.has("restart-app"),
|
|
2630
|
+
"restarted app final stop")
|
|
2631
|
+
assert.equal(
|
|
2632
|
+
vault.automaticScans.entries.get("restart-app").state,
|
|
2633
|
+
"result"
|
|
2634
|
+
)
|
|
1743
2635
|
await close(vault)
|
|
1744
2636
|
})
|
|
1745
2637
|
|
|
@@ -1912,14 +2804,28 @@ describe("automatic app checks", () => {
|
|
|
1912
2804
|
previous,
|
|
1913
2805
|
hidden: false
|
|
1914
2806
|
})
|
|
1915
|
-
|
|
2807
|
+
const logs = []
|
|
2808
|
+
automatic.log = (event, details) => logs.push({ event, details })
|
|
1916
2809
|
automatic.broadcast = () => {}
|
|
1917
2810
|
automatic.schedule = () => {}
|
|
1918
2811
|
|
|
1919
|
-
|
|
2812
|
+
active.startedAt = Date.now() - 10
|
|
2813
|
+
active.stage = "candidate-query"
|
|
2814
|
+
active.stagePath = "/failed/candidate.bin"
|
|
2815
|
+
active.stageKind = "changed"
|
|
2816
|
+
const failure = new Error("failed")
|
|
2817
|
+
failure.code = "EIO"
|
|
2818
|
+
await automatic.precheckFinished(active, null, failure)
|
|
1920
2819
|
|
|
1921
2820
|
assert.equal(automatic.active, null)
|
|
1922
2821
|
assert.equal(automatic.entries.get("demo").signature, previous.signature)
|
|
2822
|
+
const finished = logs.find((record) => record.event === "check-finished")
|
|
2823
|
+
assert.equal(finished.details.outcome, "failed")
|
|
2824
|
+
assert.equal(finished.details.failure_stage, "candidate-query")
|
|
2825
|
+
assert.equal(finished.details.failure_path, "/failed/candidate.bin")
|
|
2826
|
+
assert.equal(finished.details.failure_kind, "changed")
|
|
2827
|
+
assert.equal(finished.details.error_code, "EIO")
|
|
2828
|
+
assert.ok(finished.details.duration_ms >= 10)
|
|
1923
2829
|
})
|
|
1924
2830
|
|
|
1925
2831
|
test("a stale result action cannot acknowledge its replacement", async () => {
|
|
@@ -1992,6 +2898,71 @@ describe("automatic app checks", () => {
|
|
|
1992
2898
|
await close(vault)
|
|
1993
2899
|
})
|
|
1994
2900
|
|
|
2901
|
+
test("temporary precheck staging clears after cancellation and fatal failure", async (t) => {
|
|
2902
|
+
const home = await makeHome()
|
|
2903
|
+
const vault = await makeVault(home)
|
|
2904
|
+
t.after(() => close(vault))
|
|
2905
|
+
vault.automaticScans.candidateThreshold = async () => 1
|
|
2906
|
+
|
|
2907
|
+
const runFailure = async (kind) => {
|
|
2908
|
+
const app = `${kind}-staging-app`
|
|
2909
|
+
const appRoot = path.join(home, "api", app)
|
|
2910
|
+
const paths = [
|
|
2911
|
+
path.join(appRoot, "first.bin"),
|
|
2912
|
+
path.join(appRoot, "second.bin")
|
|
2913
|
+
]
|
|
2914
|
+
await fs.promises.mkdir(appRoot)
|
|
2915
|
+
await fs.promises.writeFile(paths[0], "same")
|
|
2916
|
+
await fs.promises.writeFile(paths[1], "same")
|
|
2917
|
+
const active = {
|
|
2918
|
+
app,
|
|
2919
|
+
paths,
|
|
2920
|
+
cancelled: false,
|
|
2921
|
+
reason: null,
|
|
2922
|
+
controller: new AbortController()
|
|
2923
|
+
}
|
|
2924
|
+
vault.automaticScans.active = active
|
|
2925
|
+
const stage = vault.registry.stageAutomaticPrecheckFiles.bind(
|
|
2926
|
+
vault.registry)
|
|
2927
|
+
const entries = vault.registry.automaticPrecheckEntries.bind(
|
|
2928
|
+
vault.registry)
|
|
2929
|
+
if (kind === "cancelled") {
|
|
2930
|
+
vault.registry.stageAutomaticPrecheckFiles = async (...args) => {
|
|
2931
|
+
const result = await stage(...args)
|
|
2932
|
+
active.cancelled = true
|
|
2933
|
+
return result
|
|
2934
|
+
}
|
|
2935
|
+
} else {
|
|
2936
|
+
vault.registry.automaticPrecheckEntries = async () => {
|
|
2937
|
+
throw new Error("precheck query failed")
|
|
2938
|
+
}
|
|
2939
|
+
}
|
|
2940
|
+
try {
|
|
2941
|
+
await assert.rejects(
|
|
2942
|
+
vault.automaticScans.runPrecheck(active),
|
|
2943
|
+
kind === "cancelled"
|
|
2944
|
+
? (error) => error && error.code === "EVAULTCANCELLED"
|
|
2945
|
+
: /precheck query failed/
|
|
2946
|
+
)
|
|
2947
|
+
if (kind === "fatal") {
|
|
2948
|
+
assert.equal(active.stage, "candidate-query")
|
|
2949
|
+
assert.equal(active.stagePath, null)
|
|
2950
|
+
}
|
|
2951
|
+
} finally {
|
|
2952
|
+
vault.registry.stageAutomaticPrecheckFiles = stage
|
|
2953
|
+
vault.registry.automaticPrecheckEntries = entries
|
|
2954
|
+
vault.automaticScans.active = null
|
|
2955
|
+
}
|
|
2956
|
+
assert.deepEqual(await entries(app, null, 1), {
|
|
2957
|
+
entries: [],
|
|
2958
|
+
next_cursor: null
|
|
2959
|
+
})
|
|
2960
|
+
}
|
|
2961
|
+
|
|
2962
|
+
await runFailure("cancelled")
|
|
2963
|
+
await runFailure("fatal")
|
|
2964
|
+
})
|
|
2965
|
+
|
|
1995
2966
|
test("user-started work preempts an automatic check and keeps priority", async () => {
|
|
1996
2967
|
const home = await makeHome()
|
|
1997
2968
|
const appRoot = path.join(home, "api", "priority-app")
|
|
@@ -2239,6 +3210,66 @@ describe("automatic app checks", () => {
|
|
|
2239
3210
|
assert.deepEqual([...changedHashes], ["a".repeat(64)])
|
|
2240
3211
|
})
|
|
2241
3212
|
|
|
3213
|
+
test("Automatic mode queues retained paths only when a stopped app has them", async (t) => {
|
|
3214
|
+
const home = await makeHome()
|
|
3215
|
+
const queuedApp = "stopped-with-paths"
|
|
3216
|
+
const emptyApp = "stopped-without-paths"
|
|
3217
|
+
const queuedRoot = path.join(home, "api", queuedApp)
|
|
3218
|
+
const emptyRoot = path.join(home, "api", emptyApp)
|
|
3219
|
+
const paths = [
|
|
3220
|
+
path.join(queuedRoot, "first.bin"),
|
|
3221
|
+
path.join(queuedRoot, "second.bin")
|
|
3222
|
+
]
|
|
3223
|
+
await fs.promises.mkdir(queuedRoot)
|
|
3224
|
+
await fs.promises.mkdir(emptyRoot)
|
|
3225
|
+
await fs.promises.writeFile(paths[0], "same")
|
|
3226
|
+
await fs.promises.writeFile(paths[1], "same")
|
|
3227
|
+
const vault = await makeVault(home)
|
|
3228
|
+
t.after(() => close(vault))
|
|
3229
|
+
vault.automaticScans.candidateThreshold = async () => 1
|
|
3230
|
+
await vault.perform("automatic_set_mode", {
|
|
3231
|
+
app: queuedApp,
|
|
3232
|
+
mode: "manual"
|
|
3233
|
+
})
|
|
3234
|
+
collect(vault, queuedApp, paths)
|
|
3235
|
+
const inspected = []
|
|
3236
|
+
const statChangedPaths = vault.automaticScans.statChangedPaths.bind(
|
|
3237
|
+
vault.automaticScans)
|
|
3238
|
+
vault.automaticScans.statChangedPaths = async (...args) => {
|
|
3239
|
+
inspected.push(...args[0])
|
|
3240
|
+
return statChangedPaths(...args)
|
|
3241
|
+
}
|
|
3242
|
+
|
|
3243
|
+
await vault.perform("automatic_set_mode", {
|
|
3244
|
+
app: queuedApp,
|
|
3245
|
+
mode: "automatic"
|
|
3246
|
+
})
|
|
3247
|
+
await waitFor(() => !vault.automaticScans.active &&
|
|
3248
|
+
vault.automaticScans.entries.get(queuedApp)?.state === "result")
|
|
3249
|
+
assert.deepEqual(new Set(inspected), new Set(paths))
|
|
3250
|
+
|
|
3251
|
+
await vault.perform("automatic_set_mode", {
|
|
3252
|
+
app: emptyApp,
|
|
3253
|
+
mode: "manual"
|
|
3254
|
+
})
|
|
3255
|
+
const inspectionsBefore = inspected.length
|
|
3256
|
+
vault.scanner.walk = async () => {
|
|
3257
|
+
throw new Error("An empty mode change must not walk the app.")
|
|
3258
|
+
}
|
|
3259
|
+
vault.hashFile = async () => {
|
|
3260
|
+
throw new Error("An empty mode change must not read file content.")
|
|
3261
|
+
}
|
|
3262
|
+
await vault.perform("automatic_set_mode", {
|
|
3263
|
+
app: emptyApp,
|
|
3264
|
+
mode: "automatic"
|
|
3265
|
+
})
|
|
3266
|
+
await new Promise((resolve) => setImmediate(resolve))
|
|
3267
|
+
|
|
3268
|
+
assert.equal(vault.automaticScans.entries.has(emptyApp), false)
|
|
3269
|
+
assert.equal(vault.automaticScans.pendingStops.has(emptyApp), false)
|
|
3270
|
+
assert.equal(inspected.length, inspectionsBefore)
|
|
3271
|
+
})
|
|
3272
|
+
|
|
2242
3273
|
test("a temporary Manual switch does not lose a running app's paths", async () => {
|
|
2243
3274
|
const home = await makeHome()
|
|
2244
3275
|
const appRoot = path.join(home, "api", "mode-switch-app")
|
|
@@ -2321,42 +3352,197 @@ describe("automatic app checks", () => {
|
|
|
2321
3352
|
const app = "paged-app"
|
|
2322
3353
|
const entries = []
|
|
2323
3354
|
for (let index = 0; index < 5; index++) {
|
|
2324
|
-
const
|
|
2325
|
-
|
|
2326
|
-
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
|
|
2333
|
-
|
|
2334
|
-
|
|
2335
|
-
|
|
2336
|
-
|
|
2337
|
-
|
|
2338
|
-
|
|
2339
|
-
|
|
3355
|
+
for (const copy of ["a", "b"]) {
|
|
3356
|
+
const filePath = path.join(home, "api", app, `${index}-${copy}.bin`)
|
|
3357
|
+
await fs.promises.mkdir(path.dirname(filePath), { recursive: true })
|
|
3358
|
+
await fs.promises.writeFile(filePath, Buffer.alloc(index + 1, index))
|
|
3359
|
+
const stat = await fs.promises.lstat(filePath)
|
|
3360
|
+
entries.push({
|
|
3361
|
+
path: filePath,
|
|
3362
|
+
size: stat.size,
|
|
3363
|
+
mtime: stat.mtimeMs,
|
|
3364
|
+
ctime: stat.ctimeMs,
|
|
3365
|
+
dev: stat.dev,
|
|
3366
|
+
ino: stat.ino,
|
|
3367
|
+
nlink: stat.nlink,
|
|
3368
|
+
mode: stat.mode,
|
|
3369
|
+
uid: stat.uid,
|
|
3370
|
+
gid: stat.gid
|
|
3371
|
+
})
|
|
3372
|
+
}
|
|
2340
3373
|
}
|
|
2341
3374
|
await vault.registry.beginAutomaticPrecheck(app)
|
|
2342
3375
|
await vault.registry.stageAutomaticPrecheckFiles(app, entries)
|
|
2343
3376
|
|
|
2344
|
-
const
|
|
2345
|
-
|
|
2346
|
-
|
|
2347
|
-
|
|
2348
|
-
|
|
2349
|
-
|
|
2350
|
-
|
|
2351
|
-
|
|
2352
|
-
|
|
3377
|
+
const readPages = async () => {
|
|
3378
|
+
const paths = []
|
|
3379
|
+
let cursor = null
|
|
3380
|
+
do {
|
|
3381
|
+
const page = await vault.registry.automaticPrecheckEntries(
|
|
3382
|
+
app, cursor, 2)
|
|
3383
|
+
assert.ok(page.entries.length <= 2)
|
|
3384
|
+
paths.push(...page.entries.map((entry) => entry.path))
|
|
3385
|
+
cursor = page.next_cursor
|
|
3386
|
+
} while (cursor)
|
|
3387
|
+
return paths
|
|
3388
|
+
}
|
|
3389
|
+
const paths = await readPages()
|
|
2353
3390
|
|
|
2354
3391
|
assert.equal(paths.length, entries.length)
|
|
2355
3392
|
assert.equal(new Set(paths).size, entries.length)
|
|
2356
3393
|
await vault.registry.abortAutomaticPrecheck(app)
|
|
3394
|
+
await vault.registry.beginAutomaticPrecheck(app)
|
|
3395
|
+
await vault.registry.stageAutomaticPrecheckFiles(
|
|
3396
|
+
app, entries.slice().reverse())
|
|
3397
|
+
assert.deepEqual(await readPages(), paths)
|
|
3398
|
+
await vault.registry.abortAutomaticPrecheck(app)
|
|
2357
3399
|
await close(vault)
|
|
2358
3400
|
})
|
|
2359
3401
|
|
|
3402
|
+
test("automatic precheck pages exclude groups without a possible peer path", async () => {
|
|
3403
|
+
const home = await makeHome()
|
|
3404
|
+
const registry = new RegistryCore(path.join(home, "vault"))
|
|
3405
|
+
await registry.load()
|
|
3406
|
+
const app = "metadata-prefilter-app"
|
|
3407
|
+
const candidate = (name, size, ino, dev = 1) => ({
|
|
3408
|
+
path: path.join(home, "api", app, name),
|
|
3409
|
+
size,
|
|
3410
|
+
mtime: 1,
|
|
3411
|
+
ctime: 1,
|
|
3412
|
+
dev,
|
|
3413
|
+
ino,
|
|
3414
|
+
nlink: 1,
|
|
3415
|
+
mode: 0,
|
|
3416
|
+
uid: 0,
|
|
3417
|
+
gid: 0
|
|
3418
|
+
})
|
|
3419
|
+
const entries = [
|
|
3420
|
+
candidate("unique.bin", 101, 1),
|
|
3421
|
+
candidate("same-path.bin", 102, 2),
|
|
3422
|
+
candidate("different-size.bin", 103, 3),
|
|
3423
|
+
candidate("different-device.bin", 105, 4),
|
|
3424
|
+
candidate("stale.bin", 106, 5),
|
|
3425
|
+
candidate("unverified-anchor.bin", 107, 6),
|
|
3426
|
+
candidate("hardlink-a.bin", 108, 7),
|
|
3427
|
+
candidate("hardlink-b.bin", 108, 7),
|
|
3428
|
+
candidate("file-peer.bin", 109, 8),
|
|
3429
|
+
candidate("anchor-peer.bin", 110, 9)
|
|
3430
|
+
]
|
|
3431
|
+
const file = (filePath, size, dev, ino, unavailableReason = null) => ({
|
|
3432
|
+
path: filePath,
|
|
3433
|
+
hash: null,
|
|
3434
|
+
size,
|
|
3435
|
+
mtime: 1,
|
|
3436
|
+
ctime: 1,
|
|
3437
|
+
dev,
|
|
3438
|
+
ino,
|
|
3439
|
+
mode: 0,
|
|
3440
|
+
uid: 0,
|
|
3441
|
+
gid: 0,
|
|
3442
|
+
source_id: "app:peer",
|
|
3443
|
+
app: "peer",
|
|
3444
|
+
status: "reference",
|
|
3445
|
+
unavailable_reason: unavailableReason,
|
|
3446
|
+
updated_at: 1
|
|
3447
|
+
})
|
|
3448
|
+
await registry.upsertFile(file(entries[1].path, 102, 1, 20))
|
|
3449
|
+
await registry.upsertFile(file(
|
|
3450
|
+
path.join(home, "different-size-peer.bin"), 104, 1, 21))
|
|
3451
|
+
await registry.upsertFile(file(
|
|
3452
|
+
path.join(home, "different-device-peer.bin"), 105, 2, 22))
|
|
3453
|
+
await registry.upsertFile(file(
|
|
3454
|
+
path.join(home, "stale-peer.bin"), 106, 1, 23, "stale"))
|
|
3455
|
+
await registry.upsertFile(file(
|
|
3456
|
+
path.join(home, "valid-file-peer.bin"), 109, 1, 24))
|
|
3457
|
+
|
|
3458
|
+
for (const [name, size, verifiedAt] of [
|
|
3459
|
+
["unverified", 107, null],
|
|
3460
|
+
["verified", 110, 1]
|
|
3461
|
+
]) {
|
|
3462
|
+
const hash = crypto.createHash("sha256").update(name).digest("hex")
|
|
3463
|
+
await registry.upsertContent({
|
|
3464
|
+
hash,
|
|
3465
|
+
size,
|
|
3466
|
+
first_seen: 1,
|
|
3467
|
+
verified_at: verifiedAt,
|
|
3468
|
+
anchor_present: true
|
|
3469
|
+
})
|
|
3470
|
+
await registry.upsertAnchor({
|
|
3471
|
+
store_id: `${name}-store`,
|
|
3472
|
+
hash,
|
|
3473
|
+
path: path.join(home, `${name}-anchor.bin`),
|
|
3474
|
+
verified_at: verifiedAt,
|
|
3475
|
+
dev: 1,
|
|
3476
|
+
ino: size,
|
|
3477
|
+
size,
|
|
3478
|
+
mtime: 1,
|
|
3479
|
+
ctime: 1,
|
|
3480
|
+
nlink: 1,
|
|
3481
|
+
mode: 0,
|
|
3482
|
+
uid: 0,
|
|
3483
|
+
gid: 0
|
|
3484
|
+
})
|
|
3485
|
+
}
|
|
3486
|
+
|
|
3487
|
+
await registry.beginAutomaticPrecheck(app)
|
|
3488
|
+
await registry.stageAutomaticPrecheckFiles(app, entries)
|
|
3489
|
+
const plan = registry.database.prepare(`
|
|
3490
|
+
EXPLAIN QUERY PLAN
|
|
3491
|
+
SELECT candidate.dev, candidate.size
|
|
3492
|
+
FROM automatic_precheck_files candidate
|
|
3493
|
+
WHERE candidate.app = @app
|
|
3494
|
+
AND (
|
|
3495
|
+
EXISTS (
|
|
3496
|
+
SELECT 1 FROM automatic_precheck_files peer
|
|
3497
|
+
WHERE peer.app = candidate.app
|
|
3498
|
+
AND peer.dev = candidate.dev
|
|
3499
|
+
AND peer.size = candidate.size
|
|
3500
|
+
AND pinokio_path_key(peer.path) !=
|
|
3501
|
+
pinokio_path_key(candidate.path)
|
|
3502
|
+
)
|
|
3503
|
+
OR EXISTS (
|
|
3504
|
+
SELECT 1 FROM files peer
|
|
3505
|
+
WHERE peer.dev = candidate.dev
|
|
3506
|
+
AND peer.size = candidate.size
|
|
3507
|
+
AND peer.unavailable_reason IS NOT 'stale'
|
|
3508
|
+
AND pinokio_path_key(peer.path) !=
|
|
3509
|
+
pinokio_path_key(candidate.path)
|
|
3510
|
+
)
|
|
3511
|
+
OR EXISTS (
|
|
3512
|
+
SELECT 1 FROM anchors peer
|
|
3513
|
+
WHERE peer.verified_at IS NOT NULL
|
|
3514
|
+
AND peer.dev = candidate.dev
|
|
3515
|
+
AND peer.size = candidate.size
|
|
3516
|
+
AND pinokio_path_key(peer.path) !=
|
|
3517
|
+
pinokio_path_key(candidate.path)
|
|
3518
|
+
)
|
|
3519
|
+
)
|
|
3520
|
+
GROUP BY candidate.dev, candidate.size
|
|
3521
|
+
ORDER BY candidate.dev, candidate.size
|
|
3522
|
+
LIMIT 1
|
|
3523
|
+
`).all({ app }).map((row) => row.detail).join("\n")
|
|
3524
|
+
assert.match(plan, /automatic_precheck_match_idx/)
|
|
3525
|
+
assert.match(plan, /files_size_device_path_idx/)
|
|
3526
|
+
assert.match(plan, /anchors_size_device_path_idx/)
|
|
3527
|
+
const changed = []
|
|
3528
|
+
let cursor = null
|
|
3529
|
+
do {
|
|
3530
|
+
const page = registry.automaticPrecheckEntries(app, cursor, 2)
|
|
3531
|
+
changed.push(...page.entries
|
|
3532
|
+
.filter((entry) => entry.kind === "changed")
|
|
3533
|
+
.map((entry) => entry.path))
|
|
3534
|
+
cursor = page.next_cursor
|
|
3535
|
+
} while (cursor)
|
|
3536
|
+
|
|
3537
|
+
assert.deepEqual(changed.sort(), [
|
|
3538
|
+
entries[6].path,
|
|
3539
|
+
entries[7].path,
|
|
3540
|
+
entries[8].path,
|
|
3541
|
+
entries[9].path
|
|
3542
|
+
].sort())
|
|
3543
|
+
registry.close()
|
|
3544
|
+
})
|
|
3545
|
+
|
|
2360
3546
|
test("Kernel restart waits for the complete previous Vault to dispose", async () => {
|
|
2361
3547
|
const calls = []
|
|
2362
3548
|
const fakeKernel = {
|