pinokiod 8.0.40 → 8.0.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/kernel/api/index.js +12 -3
- package/kernel/index.js +24 -0
- package/kernel/shell.js +2 -1
- package/kernel/vault/constants.js +13 -0
- package/kernel/vault/hash_worker.js +31 -0
- package/kernel/vault/index.js +2360 -0
- package/kernel/vault/registry.js +643 -0
- package/kernel/vault/snapshot.js +29 -0
- package/kernel/vault/sweeper.js +740 -0
- package/kernel/vault/walker.js +142 -0
- package/package.json +1 -1
- package/server/index.js +109 -0
- package/server/lib/privacy_filter_cache.js +4 -1
- package/server/public/storage-size.js +12 -0
- package/server/public/style.css +13 -0
- package/server/public/tab-link-popover.js +3 -0
- package/server/public/vault-nav.js +96 -0
- package/server/public/vault.css +1079 -0
- package/server/public/vault.js +1835 -0
- package/server/views/app.ejs +93 -16
- package/server/views/partials/main_sidebar.ejs +6 -0
- package/server/views/partials/vault_workspace.ejs +55 -0
- package/server/views/vault.ejs +29 -0
- package/server/views/vault_app.ejs +22 -0
- package/test/plugin-action-functions.test.js +19 -0
- package/test/privacy-filter-cache.test.js +1 -0
- package/test/vault-engine.test.js +1610 -0
- package/test/vault-sweep.test.js +1344 -0
- package/test/vault-ui.test.js +2443 -0
|
@@ -0,0 +1,643 @@
|
|
|
1
|
+
const fs = require('fs')
|
|
2
|
+
const path = require('path')
|
|
3
|
+
const crypto = require('crypto')
|
|
4
|
+
const { SHA256_RE, CANDIDATE_SIZE_OPTIONS } = require('./constants')
|
|
5
|
+
|
|
6
|
+
const isMissingError = (error) => !!(error && (error.code === "ENOENT" || error.code === "ENOTDIR"))
|
|
7
|
+
const isRecord = (value) => !!(value && typeof value === "object" && !Array.isArray(value))
|
|
8
|
+
const unsafeRegistryPath = (filePath) => {
|
|
9
|
+
const error = new Error(`Storage index path is not safe: ${filePath}`)
|
|
10
|
+
error.code = "EVAULTPATH"
|
|
11
|
+
return error
|
|
12
|
+
}
|
|
13
|
+
const lstatIfPresent = async (filePath) => {
|
|
14
|
+
try {
|
|
15
|
+
return await fs.promises.lstat(filePath)
|
|
16
|
+
} catch (error) {
|
|
17
|
+
if (isMissingError(error)) return null
|
|
18
|
+
throw error
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
// Persisted record of blobs and their known names.
|
|
23
|
+
// The registry is a cache of the filesystem, never the source of truth
|
|
24
|
+
// (spec/requirements/shared-model-store.md "Registry").
|
|
25
|
+
class Registry {
|
|
26
|
+
constructor(root) {
|
|
27
|
+
this.root = root
|
|
28
|
+
this.snapshotPath = path.resolve(root, "registry.json")
|
|
29
|
+
this.eventsPath = path.resolve(root, "events.ndjson")
|
|
30
|
+
this.reset()
|
|
31
|
+
this.persistTimer = null
|
|
32
|
+
this.persistDelay = 500
|
|
33
|
+
this.persistFailures = 0
|
|
34
|
+
this.flushPromise = Promise.resolve()
|
|
35
|
+
this.eventPromise = Promise.resolve()
|
|
36
|
+
this.persistDepth = 0
|
|
37
|
+
this.persistDirty = false
|
|
38
|
+
this.maxEvents = 2000
|
|
39
|
+
this.compactEvery = 500
|
|
40
|
+
this.maxEventBytesPerEntry = 4096
|
|
41
|
+
this.eventsSinceCompact = 0
|
|
42
|
+
}
|
|
43
|
+
reset() {
|
|
44
|
+
this.blobs = new Map() // hash -> { size, first_seen, source_urls, verified_at, orphan }
|
|
45
|
+
this.links = new Map() // path -> { hash, app, source_id, dev, ino, created, batch_id? }
|
|
46
|
+
this.scanIndex = new Map() // path -> { size, mtime, dev, ino, hash, source_id }
|
|
47
|
+
this.duplicates = new Map() // path -> { hash, size, app, source_id, discovered } pending user conversion
|
|
48
|
+
this.excluded = new Map() // path -> { ts, source_id, size } user chose "keep independent"
|
|
49
|
+
this.lastScan = null // last complete global scan
|
|
50
|
+
this.lastScanAttempt = null // last global scan attempt, complete or partial
|
|
51
|
+
this.sourceScans = new Map() // source id -> last complete scoped scan
|
|
52
|
+
this.sourceScanAttempts = new Map() // source id -> last scoped scan attempt
|
|
53
|
+
this.settings = { external_sources: [] } // durable user preferences, not derived scan state
|
|
54
|
+
this.byIno = new Map() // "dev:ino" -> hash
|
|
55
|
+
this.pathsByIno = new Map() // "dev:ino" -> Set(paths), for bounded group updates
|
|
56
|
+
this.eventsCache = null
|
|
57
|
+
this.eventError = null
|
|
58
|
+
}
|
|
59
|
+
async rootDirectory(create = false) {
|
|
60
|
+
let st = await lstatIfPresent(this.root)
|
|
61
|
+
if (!st && create) {
|
|
62
|
+
try {
|
|
63
|
+
await fs.promises.mkdir(this.root)
|
|
64
|
+
} catch (error) {
|
|
65
|
+
if (!error || error.code !== "EEXIST") throw error
|
|
66
|
+
}
|
|
67
|
+
st = await lstatIfPresent(this.root)
|
|
68
|
+
}
|
|
69
|
+
if (st && !st.isDirectory()) throw unsafeRegistryPath(this.root)
|
|
70
|
+
return st
|
|
71
|
+
}
|
|
72
|
+
async readFile(filePath) {
|
|
73
|
+
const before = await fs.promises.lstat(filePath)
|
|
74
|
+
if (!before.isFile()) throw unsafeRegistryPath(filePath)
|
|
75
|
+
const handle = await fs.promises.open(
|
|
76
|
+
filePath,
|
|
77
|
+
fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW || 0)
|
|
78
|
+
)
|
|
79
|
+
try {
|
|
80
|
+
const opened = await handle.stat()
|
|
81
|
+
if (!opened.isFile() || opened.dev !== before.dev || opened.ino !== before.ino) {
|
|
82
|
+
throw unsafeRegistryPath(filePath)
|
|
83
|
+
}
|
|
84
|
+
return await handle.readFile("utf8")
|
|
85
|
+
} finally {
|
|
86
|
+
await handle.close()
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
async readEventTail() {
|
|
90
|
+
const before = await fs.promises.lstat(this.eventsPath)
|
|
91
|
+
if (!before.isFile()) throw unsafeRegistryPath(this.eventsPath)
|
|
92
|
+
const handle = await fs.promises.open(
|
|
93
|
+
this.eventsPath,
|
|
94
|
+
fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW || 0)
|
|
95
|
+
)
|
|
96
|
+
try {
|
|
97
|
+
const opened = await handle.stat()
|
|
98
|
+
if (!opened.isFile() || opened.dev !== before.dev || opened.ino !== before.ino) {
|
|
99
|
+
throw unsafeRegistryPath(this.eventsPath)
|
|
100
|
+
}
|
|
101
|
+
const limit = Math.max(64 * 1024, this.maxEvents * this.maxEventBytesPerEntry)
|
|
102
|
+
const length = Math.min(opened.size, limit)
|
|
103
|
+
if (!length) return ""
|
|
104
|
+
const start = opened.size - length
|
|
105
|
+
const buffer = Buffer.allocUnsafe(length)
|
|
106
|
+
const { bytesRead } = await handle.read(buffer, 0, length, start)
|
|
107
|
+
let raw = buffer.subarray(0, bytesRead).toString("utf8")
|
|
108
|
+
if (start > 0) {
|
|
109
|
+
const newline = raw.indexOf("\n")
|
|
110
|
+
raw = newline === -1 ? "" : raw.slice(newline + 1)
|
|
111
|
+
}
|
|
112
|
+
return raw
|
|
113
|
+
} finally {
|
|
114
|
+
await handle.close()
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
async atomicWrite(filePath, contents) {
|
|
118
|
+
await this.rootDirectory(true)
|
|
119
|
+
const tmp = path.resolve(this.root,
|
|
120
|
+
`.${path.basename(filePath)}.${process.pid}.${crypto.randomBytes(6).toString("hex")}.tmp`)
|
|
121
|
+
let tmpStat = null
|
|
122
|
+
try {
|
|
123
|
+
await fs.promises.writeFile(tmp, contents, { flag: "wx" })
|
|
124
|
+
tmpStat = await fs.promises.lstat(tmp)
|
|
125
|
+
const current = await lstatIfPresent(tmp)
|
|
126
|
+
if (!current || current.dev !== tmpStat.dev || current.ino !== tmpStat.ino) {
|
|
127
|
+
throw unsafeRegistryPath(tmp)
|
|
128
|
+
}
|
|
129
|
+
await fs.promises.rename(tmp, filePath)
|
|
130
|
+
tmpStat = null
|
|
131
|
+
} finally {
|
|
132
|
+
if (tmpStat) {
|
|
133
|
+
const current = await lstatIfPresent(tmp).catch(() => null)
|
|
134
|
+
if (current && current.dev === tmpStat.dev && current.ino === tmpStat.ino) {
|
|
135
|
+
await fs.promises.unlink(tmp).catch(() => {})
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
async appendFile(filePath, contents) {
|
|
141
|
+
await this.rootDirectory(true)
|
|
142
|
+
let before
|
|
143
|
+
let handle
|
|
144
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
145
|
+
before = await lstatIfPresent(filePath)
|
|
146
|
+
if (before && !before.isFile()) throw unsafeRegistryPath(filePath)
|
|
147
|
+
const flags = fs.constants.O_WRONLY | fs.constants.O_APPEND |
|
|
148
|
+
(fs.constants.O_NOFOLLOW || 0) |
|
|
149
|
+
(before ? 0 : fs.constants.O_CREAT | fs.constants.O_EXCL)
|
|
150
|
+
try {
|
|
151
|
+
handle = await fs.promises.open(filePath, flags, 0o600)
|
|
152
|
+
break
|
|
153
|
+
} catch (error) {
|
|
154
|
+
if (!before && error && error.code === "EEXIST" && attempt === 0) continue
|
|
155
|
+
throw error
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
if (!handle) throw unsafeRegistryPath(filePath)
|
|
159
|
+
try {
|
|
160
|
+
const opened = await handle.stat()
|
|
161
|
+
if (!opened.isFile() || (before &&
|
|
162
|
+
(opened.dev !== before.dev || opened.ino !== before.ino))) {
|
|
163
|
+
throw unsafeRegistryPath(filePath)
|
|
164
|
+
}
|
|
165
|
+
await handle.writeFile(contents)
|
|
166
|
+
} finally {
|
|
167
|
+
await handle.close()
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
scanFor(sourceId = null) {
|
|
171
|
+
return sourceId ? (this.sourceScans.get(sourceId) || null) : this.lastScan
|
|
172
|
+
}
|
|
173
|
+
scanAttemptFor(sourceId = null) {
|
|
174
|
+
return sourceId
|
|
175
|
+
? (this.sourceScanAttempts.get(sourceId) || null)
|
|
176
|
+
: this.lastScanAttempt
|
|
177
|
+
}
|
|
178
|
+
setLastScan(meta, sourceId = null) {
|
|
179
|
+
if (sourceId) {
|
|
180
|
+
this.sourceScans.set(sourceId, meta)
|
|
181
|
+
this.sourceScanAttempts.set(sourceId, meta)
|
|
182
|
+
} else {
|
|
183
|
+
this.lastScan = meta
|
|
184
|
+
this.lastScanAttempt = meta
|
|
185
|
+
}
|
|
186
|
+
this.schedulePersist()
|
|
187
|
+
}
|
|
188
|
+
setScanAttempt(meta, sourceId = null) {
|
|
189
|
+
if (sourceId) this.sourceScanAttempts.set(sourceId, meta)
|
|
190
|
+
else this.lastScanAttempt = meta
|
|
191
|
+
this.schedulePersist()
|
|
192
|
+
}
|
|
193
|
+
inoKey(dev, ino) {
|
|
194
|
+
return `${dev}:${ino}`
|
|
195
|
+
}
|
|
196
|
+
addBlob(hash, meta) {
|
|
197
|
+
if (!this.blobs.has(hash)) {
|
|
198
|
+
this.blobs.set(hash, Object.assign({
|
|
199
|
+
size: 0,
|
|
200
|
+
first_seen: Date.now(),
|
|
201
|
+
source_urls: [],
|
|
202
|
+
verified_at: null
|
|
203
|
+
}, meta))
|
|
204
|
+
} else if (meta && meta.source_urls && meta.source_urls.length) {
|
|
205
|
+
const blob = this.blobs.get(hash)
|
|
206
|
+
for (const url of meta.source_urls) {
|
|
207
|
+
if (!blob.source_urls.includes(url)) blob.source_urls.push(url)
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
this.schedulePersist()
|
|
211
|
+
}
|
|
212
|
+
addLink(filePath, entry) {
|
|
213
|
+
const previous = this.links.get(filePath)
|
|
214
|
+
if (previous && previous.dev !== undefined && previous.ino !== undefined) {
|
|
215
|
+
this.removeIno(previous.dev, previous.ino, filePath)
|
|
216
|
+
}
|
|
217
|
+
// A path has exactly one derived classification. Registering a physical
|
|
218
|
+
// name always clears an older pending-duplicate classification first.
|
|
219
|
+
this.duplicates.delete(filePath)
|
|
220
|
+
const next = Object.assign({ created: Date.now() }, entry)
|
|
221
|
+
delete next.mode
|
|
222
|
+
const sameTrackedFile = previous && previous.hash === next.hash &&
|
|
223
|
+
previous.dev === next.dev && previous.ino === next.ino
|
|
224
|
+
if (next.batch_id === undefined && sameTrackedFile && previous.batch_id) {
|
|
225
|
+
next.batch_id = previous.batch_id
|
|
226
|
+
}
|
|
227
|
+
this.links.set(filePath, next)
|
|
228
|
+
if (entry.dev !== undefined && entry.ino !== undefined) {
|
|
229
|
+
const key = this.inoKey(entry.dev, entry.ino)
|
|
230
|
+
this.byIno.set(key, entry.hash)
|
|
231
|
+
if (!this.pathsByIno.has(key)) this.pathsByIno.set(key, new Set())
|
|
232
|
+
this.pathsByIno.get(key).add(filePath)
|
|
233
|
+
}
|
|
234
|
+
this.schedulePersist()
|
|
235
|
+
}
|
|
236
|
+
setDuplicate(filePath, entry, scanEntry = null) {
|
|
237
|
+
const previous = this.duplicates.get(filePath) || null
|
|
238
|
+
this.removeLink(filePath)
|
|
239
|
+
this.duplicates.set(filePath, entry)
|
|
240
|
+
if (scanEntry) this.scanIndex.set(filePath, scanEntry)
|
|
241
|
+
this.schedulePersist()
|
|
242
|
+
return previous
|
|
243
|
+
}
|
|
244
|
+
setScanEntry(filePath, entry) {
|
|
245
|
+
this.scanIndex.set(filePath, entry)
|
|
246
|
+
this.schedulePersist()
|
|
247
|
+
}
|
|
248
|
+
untrack(filePath) {
|
|
249
|
+
const hadLink = this.links.has(filePath)
|
|
250
|
+
const hadDuplicate = this.duplicates.delete(filePath)
|
|
251
|
+
const hadIndex = this.scanIndex.delete(filePath)
|
|
252
|
+
if (hadLink) this.removeLink(filePath)
|
|
253
|
+
if (hadDuplicate || hadIndex) this.schedulePersist()
|
|
254
|
+
}
|
|
255
|
+
exclude(filePath, entry) {
|
|
256
|
+
this.untrack(filePath)
|
|
257
|
+
this.excluded.set(filePath, entry)
|
|
258
|
+
this.schedulePersist()
|
|
259
|
+
}
|
|
260
|
+
allowSharing(filePath) {
|
|
261
|
+
const entry = this.excluded.get(filePath) || null
|
|
262
|
+
if (!this.excluded.delete(filePath)) return null
|
|
263
|
+
this.schedulePersist()
|
|
264
|
+
return entry
|
|
265
|
+
}
|
|
266
|
+
removeLink(filePath) {
|
|
267
|
+
const entry = this.links.get(filePath)
|
|
268
|
+
if (entry) {
|
|
269
|
+
this.links.delete(filePath)
|
|
270
|
+
this.removeIno(entry.dev, entry.ino, filePath)
|
|
271
|
+
this.schedulePersist()
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
removeIno(dev, ino, filePath) {
|
|
275
|
+
if (dev === undefined || ino === undefined) return
|
|
276
|
+
const key = this.inoKey(dev, ino)
|
|
277
|
+
const paths = this.pathsByIno.get(key)
|
|
278
|
+
if (paths && filePath !== undefined) paths.delete(filePath)
|
|
279
|
+
if (!paths || paths.size === 0) {
|
|
280
|
+
this.pathsByIno.delete(key)
|
|
281
|
+
this.byIno.delete(key)
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
removeBlob(hash) {
|
|
285
|
+
this.removeBlobs(new Set([hash]))
|
|
286
|
+
}
|
|
287
|
+
removeBlobs(hashes) {
|
|
288
|
+
if (!hashes || hashes.size === 0) return
|
|
289
|
+
for (const hash of hashes) this.blobs.delete(hash)
|
|
290
|
+
for (const [p, entry] of this.links) {
|
|
291
|
+
if (hashes.has(entry.hash)) {
|
|
292
|
+
this.removeLink(p)
|
|
293
|
+
this.scanIndex.delete(p)
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
// A pending review item is only actionable while its matching Vault copy
|
|
297
|
+
// exists. Clear the dependent cache records with the content group.
|
|
298
|
+
for (const [p, entry] of this.duplicates) {
|
|
299
|
+
if (hashes.has(entry.hash)) {
|
|
300
|
+
this.duplicates.delete(p)
|
|
301
|
+
this.scanIndex.delete(p)
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
this.schedulePersist()
|
|
305
|
+
}
|
|
306
|
+
snapshotState() {
|
|
307
|
+
const cloneMap = (source) => new Map([...source].map(([key, value]) => [
|
|
308
|
+
key,
|
|
309
|
+
value && typeof value === "object"
|
|
310
|
+
? Object.assign({}, value, Array.isArray(value.source_urls)
|
|
311
|
+
? { source_urls: [...value.source_urls] }
|
|
312
|
+
: {})
|
|
313
|
+
: value
|
|
314
|
+
]))
|
|
315
|
+
return {
|
|
316
|
+
blobs: cloneMap(this.blobs),
|
|
317
|
+
links: cloneMap(this.links),
|
|
318
|
+
scanIndex: cloneMap(this.scanIndex),
|
|
319
|
+
duplicates: cloneMap(this.duplicates),
|
|
320
|
+
excluded: cloneMap(this.excluded),
|
|
321
|
+
lastScan: this.lastScan ? Object.assign({}, this.lastScan) : null,
|
|
322
|
+
lastScanAttempt: this.lastScanAttempt ? Object.assign({}, this.lastScanAttempt) : null,
|
|
323
|
+
sourceScans: cloneMap(this.sourceScans),
|
|
324
|
+
sourceScanAttempts: cloneMap(this.sourceScanAttempts),
|
|
325
|
+
settings: Object.assign({}, this.settings, {
|
|
326
|
+
external_sources: [...(this.settings.external_sources || [])]
|
|
327
|
+
}),
|
|
328
|
+
byIno: new Map(this.byIno),
|
|
329
|
+
pathsByIno: new Map([...this.pathsByIno].map(([key, paths]) => [key, new Set(paths)]))
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
replaceState(state, options = {}) {
|
|
333
|
+
this.blobs = state.blobs
|
|
334
|
+
this.links = state.links
|
|
335
|
+
this.scanIndex = state.scanIndex
|
|
336
|
+
this.duplicates = state.duplicates
|
|
337
|
+
this.excluded = state.excluded
|
|
338
|
+
this.lastScan = state.lastScan
|
|
339
|
+
this.lastScanAttempt = state.lastScanAttempt || null
|
|
340
|
+
this.sourceScans = state.sourceScans
|
|
341
|
+
this.sourceScanAttempts = state.sourceScanAttempts || new Map()
|
|
342
|
+
this.settings = Object.assign({ external_sources: [] }, state.settings)
|
|
343
|
+
this.byIno = state.byIno
|
|
344
|
+
this.pathsByIno = state.pathsByIno
|
|
345
|
+
if (options.persist !== false) this.schedulePersist()
|
|
346
|
+
}
|
|
347
|
+
// Returns { corrupt: true } when the snapshot exists but cannot be parsed,
|
|
348
|
+
// so the caller can rebuild from disk instead of blocking startup.
|
|
349
|
+
async load() {
|
|
350
|
+
this.reset()
|
|
351
|
+
let raw
|
|
352
|
+
try {
|
|
353
|
+
await this.rootDirectory(false)
|
|
354
|
+
raw = await this.readFile(this.snapshotPath)
|
|
355
|
+
} catch (e) {
|
|
356
|
+
if (e && e.code === "ENOENT") return { corrupt: false, existed: false }
|
|
357
|
+
throw e
|
|
358
|
+
}
|
|
359
|
+
let json
|
|
360
|
+
try {
|
|
361
|
+
json = JSON.parse(raw)
|
|
362
|
+
} catch (e) {
|
|
363
|
+
return { corrupt: true, existed: true }
|
|
364
|
+
}
|
|
365
|
+
if (!isRecord(json)) {
|
|
366
|
+
return { corrupt: true, existed: true }
|
|
367
|
+
}
|
|
368
|
+
for (const [k, v] of Object.entries(json.blobs || {})) {
|
|
369
|
+
if (SHA256_RE.test(k) && isRecord(v)) {
|
|
370
|
+
this.blobs.set(k, Object.assign({}, v, {
|
|
371
|
+
size: Number.isFinite(v.size) && v.size >= 0 ? v.size : 0,
|
|
372
|
+
source_urls: Array.isArray(v.source_urls)
|
|
373
|
+
? v.source_urls.filter((url) => typeof url === "string")
|
|
374
|
+
: []
|
|
375
|
+
}))
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
const legacyCopies = []
|
|
379
|
+
for (const [k, v] of Object.entries(json.links || {})) {
|
|
380
|
+
if (!isRecord(v) || !SHA256_RE.test(v.hash) ||
|
|
381
|
+
!Number.isFinite(v.dev) || !Number.isFinite(v.ino)) continue
|
|
382
|
+
if (v.mode === "copy") {
|
|
383
|
+
legacyCopies.push([k, v])
|
|
384
|
+
continue
|
|
385
|
+
}
|
|
386
|
+
const link = Object.assign({}, v)
|
|
387
|
+
delete link.mode
|
|
388
|
+
this.links.set(k, link)
|
|
389
|
+
if (link.dev !== undefined && link.ino !== undefined) {
|
|
390
|
+
const key = this.inoKey(link.dev, link.ino)
|
|
391
|
+
this.byIno.set(key, link.hash)
|
|
392
|
+
if (!this.pathsByIno.has(key)) this.pathsByIno.set(key, new Set())
|
|
393
|
+
this.pathsByIno.get(key).add(k)
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
for (const [k, v] of Object.entries(json.scan_index || {})) {
|
|
397
|
+
if (isRecord(v) && (!v.hash || SHA256_RE.test(v.hash)) &&
|
|
398
|
+
[v.size, v.mtime, v.ctime, v.dev, v.ino].every(Number.isFinite)) {
|
|
399
|
+
this.scanIndex.set(k, v)
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
for (const [k, v] of Object.entries(json.duplicates || {})) {
|
|
403
|
+
if (isRecord(v) && SHA256_RE.test(v.hash) && !this.links.has(k)) {
|
|
404
|
+
this.duplicates.set(k, Object.assign({}, v, {
|
|
405
|
+
size: Number.isFinite(v.size) && v.size >= 0 ? v.size : 0
|
|
406
|
+
}))
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
// Older builds recorded independent files as mode:"copy" links. They were
|
|
410
|
+
// never shared. Migrate only their review metadata and never recreate that
|
|
411
|
+
// unsupported state.
|
|
412
|
+
for (const [filePath, link] of legacyCopies) {
|
|
413
|
+
if (this.duplicates.has(filePath) || !this.blobs.has(link.hash)) continue
|
|
414
|
+
const indexed = this.scanIndex.get(filePath)
|
|
415
|
+
if (!indexed || indexed.hash !== link.hash) continue
|
|
416
|
+
this.duplicates.set(filePath, {
|
|
417
|
+
hash: link.hash,
|
|
418
|
+
size: indexed.size,
|
|
419
|
+
app: link.app || null,
|
|
420
|
+
source_id: link.source_id || indexed.source_id || null,
|
|
421
|
+
discovered: link.created || Date.now(),
|
|
422
|
+
dev: indexed.dev,
|
|
423
|
+
ino: indexed.ino,
|
|
424
|
+
mtime: indexed.mtime,
|
|
425
|
+
ctime: indexed.ctime,
|
|
426
|
+
unavailable_reason: "unsupported_disk"
|
|
427
|
+
})
|
|
428
|
+
}
|
|
429
|
+
for (const [k, v] of Object.entries(json.excluded || {})) {
|
|
430
|
+
if (!isRecord(v)) continue
|
|
431
|
+
const linked = this.links.get(k)
|
|
432
|
+
if (linked) {
|
|
433
|
+
this.links.delete(k)
|
|
434
|
+
this.removeIno(linked.dev, linked.ino, k)
|
|
435
|
+
}
|
|
436
|
+
this.duplicates.delete(k)
|
|
437
|
+
this.scanIndex.delete(k)
|
|
438
|
+
this.excluded.set(k, Object.assign({}, v, {
|
|
439
|
+
size: Number.isFinite(v.size) && v.size >= 0 ? v.size : 0
|
|
440
|
+
}))
|
|
441
|
+
}
|
|
442
|
+
if (isRecord(json.scans)) {
|
|
443
|
+
for (const [scopeId, value] of Object.entries(json.scans)) {
|
|
444
|
+
if (!isRecord(value)) continue
|
|
445
|
+
const complete = isRecord(value.last_complete) ? value.last_complete : null
|
|
446
|
+
const attempt = isRecord(value.last_attempt) ? value.last_attempt : null
|
|
447
|
+
if (scopeId === "global") {
|
|
448
|
+
this.lastScan = complete
|
|
449
|
+
this.lastScanAttempt = attempt
|
|
450
|
+
} else {
|
|
451
|
+
if (complete) this.sourceScans.set(scopeId, complete)
|
|
452
|
+
if (attempt) this.sourceScanAttempts.set(scopeId, attempt)
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
} else {
|
|
456
|
+
if (isRecord(json.last_scan)) {
|
|
457
|
+
this.lastScan = json.last_scan
|
|
458
|
+
this.lastScanAttempt = json.last_scan
|
|
459
|
+
}
|
|
460
|
+
for (const [sourceId, scan] of Object.entries(json.source_scans || {})) {
|
|
461
|
+
if (!isRecord(scan)) continue
|
|
462
|
+
this.sourceScans.set(sourceId, scan)
|
|
463
|
+
this.sourceScanAttempts.set(sourceId, scan)
|
|
464
|
+
}
|
|
465
|
+
}
|
|
466
|
+
if (isRecord(json.settings)) {
|
|
467
|
+
if (CANDIDATE_SIZE_OPTIONS.includes(json.settings.candidate_min_bytes)) {
|
|
468
|
+
this.settings.candidate_min_bytes = json.settings.candidate_min_bytes
|
|
469
|
+
}
|
|
470
|
+
if (Array.isArray(json.settings.external_sources)) {
|
|
471
|
+
const seen = new Set()
|
|
472
|
+
for (const value of json.settings.external_sources) {
|
|
473
|
+
if (typeof value !== "string" || !path.isAbsolute(value)) continue
|
|
474
|
+
const resolved = path.resolve(value)
|
|
475
|
+
const key = process.platform === "win32" ? resolved.toLowerCase() : resolved
|
|
476
|
+
if (seen.has(key)) continue
|
|
477
|
+
seen.add(key)
|
|
478
|
+
this.settings.external_sources.push(resolved)
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
return { corrupt: false, existed: true }
|
|
483
|
+
}
|
|
484
|
+
beginBatch() {
|
|
485
|
+
if (this.persistDepth === 0 && this.persistTimer) {
|
|
486
|
+
clearTimeout(this.persistTimer)
|
|
487
|
+
this.persistTimer = null
|
|
488
|
+
this.persistDirty = true
|
|
489
|
+
}
|
|
490
|
+
this.persistDepth += 1
|
|
491
|
+
}
|
|
492
|
+
async endBatch(options = {}) {
|
|
493
|
+
if (this.persistDepth > 0) this.persistDepth -= 1
|
|
494
|
+
if (this.persistDepth) return
|
|
495
|
+
if (options.discard) {
|
|
496
|
+
this.persistDirty = false
|
|
497
|
+
return
|
|
498
|
+
}
|
|
499
|
+
if (!this.persistDirty) return
|
|
500
|
+
if (options.flush === false) this.schedulePersist()
|
|
501
|
+
else await this.flush()
|
|
502
|
+
}
|
|
503
|
+
schedulePersist(delay = this.persistDelay) {
|
|
504
|
+
this.persistDirty = true
|
|
505
|
+
if (this.persistDepth > 0) {
|
|
506
|
+
return
|
|
507
|
+
}
|
|
508
|
+
if (this.persistTimer) return
|
|
509
|
+
this.persistTimer = setTimeout(() => {
|
|
510
|
+
this.persistTimer = null
|
|
511
|
+
this.flush().catch(() => {})
|
|
512
|
+
}, delay)
|
|
513
|
+
if (this.persistTimer.unref) this.persistTimer.unref()
|
|
514
|
+
}
|
|
515
|
+
async flush() {
|
|
516
|
+
if (this.persistTimer) {
|
|
517
|
+
clearTimeout(this.persistTimer)
|
|
518
|
+
this.persistTimer = null
|
|
519
|
+
}
|
|
520
|
+
this.persistDirty = false
|
|
521
|
+
const scans = {
|
|
522
|
+
global: {
|
|
523
|
+
last_complete: this.lastScan,
|
|
524
|
+
last_attempt: this.lastScanAttempt
|
|
525
|
+
}
|
|
526
|
+
}
|
|
527
|
+
for (const sourceId of new Set([
|
|
528
|
+
...this.sourceScans.keys(),
|
|
529
|
+
...this.sourceScanAttempts.keys()
|
|
530
|
+
])) {
|
|
531
|
+
scans[sourceId] = {
|
|
532
|
+
last_complete: this.sourceScans.get(sourceId) || null,
|
|
533
|
+
last_attempt: this.sourceScanAttempts.get(sourceId) || null
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
const json = {
|
|
537
|
+
version: 1,
|
|
538
|
+
blobs: Object.fromEntries(this.blobs),
|
|
539
|
+
links: Object.fromEntries(this.links),
|
|
540
|
+
scan_index: Object.fromEntries(this.scanIndex),
|
|
541
|
+
duplicates: Object.fromEntries(this.duplicates),
|
|
542
|
+
excluded: Object.fromEntries(this.excluded),
|
|
543
|
+
scans,
|
|
544
|
+
settings: this.settings
|
|
545
|
+
}
|
|
546
|
+
const write = async () => {
|
|
547
|
+
await this.atomicWrite(this.snapshotPath, JSON.stringify(json))
|
|
548
|
+
}
|
|
549
|
+
const pending = this.flushPromise.then(write, write)
|
|
550
|
+
this.flushPromise = pending.catch(() => {})
|
|
551
|
+
try {
|
|
552
|
+
await pending
|
|
553
|
+
this.persistFailures = 0
|
|
554
|
+
} catch (error) {
|
|
555
|
+
// The in-memory mutation remains authoritative after a failed write.
|
|
556
|
+
// Keep it dirty and retry instead of silently losing it on restart.
|
|
557
|
+
const retryDelay = Math.min(this.persistDelay * (2 ** this.persistFailures), 30000)
|
|
558
|
+
this.persistFailures += 1
|
|
559
|
+
this.schedulePersist(retryDelay)
|
|
560
|
+
throw error
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
async appendEvent(event) {
|
|
564
|
+
return this.appendEvents([event])
|
|
565
|
+
}
|
|
566
|
+
async appendEvents(events) {
|
|
567
|
+
if (!events.length) return
|
|
568
|
+
const entries = events.map((event) => Object.assign({ ts: Date.now() }, event))
|
|
569
|
+
const contents = entries.map((event) => JSON.stringify(event)).join("\n") + "\n"
|
|
570
|
+
const append = async () => {
|
|
571
|
+
await this.appendFile(this.eventsPath, contents)
|
|
572
|
+
if (this.eventsCache) this.eventsCache.push(...entries)
|
|
573
|
+
this.eventsSinceCompact += entries.length
|
|
574
|
+
if ((this.eventsCache && this.eventsCache.length > this.maxEvents + this.compactEvery) ||
|
|
575
|
+
this.eventsSinceCompact >= this.compactEvery) {
|
|
576
|
+
await this.compactEventsNow()
|
|
577
|
+
}
|
|
578
|
+
}
|
|
579
|
+
const pending = this.eventPromise.then(append, append)
|
|
580
|
+
// One ordered writer keeps action and scan batches ordered. Keep it usable
|
|
581
|
+
// after a failure while returning the real append result so the dashboard
|
|
582
|
+
// can expose missing activity without misreporting the action.
|
|
583
|
+
this.eventPromise = pending.then(
|
|
584
|
+
() => { this.eventError = null },
|
|
585
|
+
(error) => {
|
|
586
|
+
this.eventError = error && error.message ? error.message : String(error)
|
|
587
|
+
}
|
|
588
|
+
)
|
|
589
|
+
return pending
|
|
590
|
+
}
|
|
591
|
+
async compactEventsNow(events = null) {
|
|
592
|
+
if (!events) {
|
|
593
|
+
let raw = ""
|
|
594
|
+
try {
|
|
595
|
+
raw = await this.readEventTail()
|
|
596
|
+
} catch (error) {
|
|
597
|
+
if (!error || error.code !== "ENOENT") throw error
|
|
598
|
+
}
|
|
599
|
+
events = this.parseEvents(raw)
|
|
600
|
+
}
|
|
601
|
+
const recent = events.slice(-this.maxEvents)
|
|
602
|
+
await this.atomicWrite(this.eventsPath,
|
|
603
|
+
recent.map((event) => JSON.stringify(event)).join("\n") + (recent.length ? "\n" : ""))
|
|
604
|
+
this.eventsCache = recent
|
|
605
|
+
this.eventsSinceCompact = 0
|
|
606
|
+
}
|
|
607
|
+
parseEvents(raw) {
|
|
608
|
+
const events = []
|
|
609
|
+
for (const line of String(raw || "").split("\n")) {
|
|
610
|
+
if (!line.trim()) continue
|
|
611
|
+
try {
|
|
612
|
+
const event = JSON.parse(line)
|
|
613
|
+
if (isRecord(event)) events.push(event)
|
|
614
|
+
} catch (error) {}
|
|
615
|
+
}
|
|
616
|
+
return events
|
|
617
|
+
}
|
|
618
|
+
// Torn final line (crash mid-append) is discarded, per persistence rules.
|
|
619
|
+
async readEvents() {
|
|
620
|
+
const read = async () => {
|
|
621
|
+
if (this.eventsCache) {
|
|
622
|
+
return this.eventsCache.slice(-this.maxEvents)
|
|
623
|
+
}
|
|
624
|
+
let raw
|
|
625
|
+
try {
|
|
626
|
+
await this.rootDirectory(false)
|
|
627
|
+
raw = await this.readEventTail()
|
|
628
|
+
} catch (error) {
|
|
629
|
+
if (!error || error.code !== "ENOENT") throw error
|
|
630
|
+
this.eventsCache = []
|
|
631
|
+
return []
|
|
632
|
+
}
|
|
633
|
+
const events = this.parseEvents(raw)
|
|
634
|
+
this.eventsCache = events.slice(-this.maxEvents)
|
|
635
|
+
return this.eventsCache.slice()
|
|
636
|
+
}
|
|
637
|
+
const pending = this.eventPromise.then(read, read)
|
|
638
|
+
this.eventPromise = pending.then(() => {}, () => {})
|
|
639
|
+
return pending
|
|
640
|
+
}
|
|
641
|
+
}
|
|
642
|
+
|
|
643
|
+
module.exports = Registry
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
const fileSnapshot = (st) => ({
|
|
2
|
+
dev: st.dev,
|
|
3
|
+
ino: st.ino,
|
|
4
|
+
size: st.size,
|
|
5
|
+
mtime: st.mtimeMs,
|
|
6
|
+
ctime: st.ctimeMs
|
|
7
|
+
})
|
|
8
|
+
|
|
9
|
+
const sameSnapshot = (snapshot, st) => !!(
|
|
10
|
+
snapshot && st &&
|
|
11
|
+
snapshot.dev === st.dev &&
|
|
12
|
+
snapshot.ino === st.ino &&
|
|
13
|
+
snapshot.size === st.size &&
|
|
14
|
+
snapshot.mtime === st.mtimeMs &&
|
|
15
|
+
snapshot.ctime === st.ctimeMs
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
// Adding or removing a hardlink legitimately changes ctime without changing
|
|
19
|
+
// the file identity or bytes. Use this only across Vault's own link updates;
|
|
20
|
+
// hashing and externally visible mutations still require sameSnapshot().
|
|
21
|
+
const sameContentState = (snapshot, st) => !!(
|
|
22
|
+
snapshot && st &&
|
|
23
|
+
snapshot.dev === st.dev &&
|
|
24
|
+
snapshot.ino === st.ino &&
|
|
25
|
+
snapshot.size === st.size &&
|
|
26
|
+
snapshot.mtime === st.mtimeMs
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
module.exports = { fileSnapshot, sameSnapshot, sameContentState }
|