pinokiod 8.0.40 → 8.0.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/kernel/api/index.js +12 -3
- package/kernel/index.js +24 -0
- package/kernel/shell.js +2 -1
- package/kernel/vault/constants.js +13 -0
- package/kernel/vault/hash_worker.js +31 -0
- package/kernel/vault/index.js +2360 -0
- package/kernel/vault/registry.js +643 -0
- package/kernel/vault/snapshot.js +29 -0
- package/kernel/vault/sweeper.js +740 -0
- package/kernel/vault/walker.js +142 -0
- package/package.json +1 -1
- package/server/index.js +109 -0
- package/server/lib/privacy_filter_cache.js +4 -1
- package/server/public/storage-size.js +12 -0
- package/server/public/style.css +13 -0
- package/server/public/tab-link-popover.js +3 -0
- package/server/public/vault-nav.js +96 -0
- package/server/public/vault.css +1079 -0
- package/server/public/vault.js +1835 -0
- package/server/views/app.ejs +93 -16
- package/server/views/partials/main_sidebar.ejs +6 -0
- package/server/views/partials/vault_workspace.ejs +55 -0
- package/server/views/vault.ejs +29 -0
- package/server/views/vault_app.ejs +22 -0
- package/test/plugin-action-functions.test.js +19 -0
- package/test/privacy-filter-cache.test.js +1 -0
- package/test/vault-engine.test.js +1610 -0
- package/test/vault-sweep.test.js +1344 -0
- package/test/vault-ui.test.js +2443 -0
|
@@ -0,0 +1,2360 @@
|
|
|
1
|
+
const fs = require('fs')
|
|
2
|
+
const path = require('path')
|
|
3
|
+
const crypto = require('crypto')
|
|
4
|
+
const { Worker } = require('worker_threads')
|
|
5
|
+
const Registry = require('./registry')
|
|
6
|
+
const Sweeper = require('./sweeper')
|
|
7
|
+
const { walkBatches, statMany } = require('./walker')
|
|
8
|
+
const { fileSnapshot, sameSnapshot, sameContentState } = require('./snapshot')
|
|
9
|
+
const {
|
|
10
|
+
SIZE_THRESHOLD, CANDIDATE_SIZE_OPTIONS, TMP_SUFFIX, SHA256_RE,
|
|
11
|
+
DIR_CONCURRENCY, STAT_CONCURRENCY
|
|
12
|
+
} = require('./constants')
|
|
13
|
+
|
|
14
|
+
// Shared model store engine (spec/requirements/shared-model-store.md).
|
|
15
|
+
// Store + registry + volume probe + hashing + adopt/convert/verify, plus the
|
|
16
|
+
// user-facing operations behind the vault dashboard (detach, undo, reclaim).
|
|
17
|
+
// Nothing here runs automatically except startup verify; discovery happens
|
|
18
|
+
// only through the user's manual scan.
|
|
19
|
+
|
|
20
|
+
const NO_LINK_CODES = new Set(["EXDEV", "ENOTSUP", "ENOSYS"])
|
|
21
|
+
const LOCK_CODES = new Set(["EBUSY", "EPERM", "EACCES"])
|
|
22
|
+
const COPY_PROGRESS_CHUNK_SIZE = 8 * 1024 * 1024
|
|
23
|
+
const isMissingError = (error) => !!(error && (error.code === "ENOENT" || error.code === "ENOTDIR"))
|
|
24
|
+
const isAccessError = (error) => !!(error && (error.code === "EACCES" || error.code === "EPERM"))
|
|
25
|
+
const lstatIfPresent = async (filePath) => {
|
|
26
|
+
try {
|
|
27
|
+
return await fs.promises.lstat(filePath)
|
|
28
|
+
} catch (error) {
|
|
29
|
+
if (isMissingError(error)) return null
|
|
30
|
+
throw error
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
const sameIdentity = (left, right) => !!(
|
|
34
|
+
left && right && left.dev === right.dev && left.ino === right.ino
|
|
35
|
+
)
|
|
36
|
+
const unlinkIfSame = async (filePath, expected) => {
|
|
37
|
+
if (!expected) return false
|
|
38
|
+
const current = await lstatIfPresent(filePath)
|
|
39
|
+
if (!sameIdentity(current, expected)) return false
|
|
40
|
+
try {
|
|
41
|
+
await fs.promises.unlink(filePath)
|
|
42
|
+
return true
|
|
43
|
+
} catch (error) {
|
|
44
|
+
if (isMissingError(error)) return false
|
|
45
|
+
throw error
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
const unsafeStoragePath = (filePath) => {
|
|
49
|
+
const error = new Error(`Storage path is not a real directory: ${filePath}`)
|
|
50
|
+
error.code = "EVAULTPATH"
|
|
51
|
+
return error
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
const isPathWithin = (root, target) => {
|
|
55
|
+
const rel = path.relative(path.resolve(root), path.resolve(target))
|
|
56
|
+
return rel === "" || (!rel.startsWith(`..${path.sep}`) && rel !== ".." && !path.isAbsolute(rel))
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
const samePath = (left, right) => {
|
|
60
|
+
const a = path.resolve(left)
|
|
61
|
+
const b = path.resolve(right)
|
|
62
|
+
return process.platform === "win32" ? a.toLowerCase() === b.toLowerCase() : a === b
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
const sameFileMetadata = (left, right) => {
|
|
66
|
+
if (!left || !right) return false
|
|
67
|
+
if ((left.mode & 0o7777) !== (right.mode & 0o7777)) return false
|
|
68
|
+
if (left.uid !== undefined && right.uid !== undefined && left.uid !== right.uid) return false
|
|
69
|
+
if (left.gid !== undefined && right.gid !== undefined && left.gid !== right.gid) return false
|
|
70
|
+
return true
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
const sourceId = (kind, name) => `${kind}:${encodeURIComponent(name)}`
|
|
74
|
+
const externalSourceId = (root) => {
|
|
75
|
+
const value = process.platform === "win32" ? path.resolve(root).toLowerCase() : path.resolve(root)
|
|
76
|
+
return `external:${crypto.createHash("sha256").update(value).digest("hex")}`
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
class Vault {
|
|
80
|
+
constructor(kernel) {
|
|
81
|
+
this.kernel = kernel
|
|
82
|
+
this.enabled = false
|
|
83
|
+
this.initialized = false
|
|
84
|
+
this.mode = null // 'link' | 'unsupported' for the vault's volume
|
|
85
|
+
this.volumeModes = new Map() // dev -> 'link' | 'unsupported'
|
|
86
|
+
this.registry = null
|
|
87
|
+
this.worker = null
|
|
88
|
+
this.workerJobs = new Map()
|
|
89
|
+
this.workerSeq = 0
|
|
90
|
+
this.workerIdleTimer = null
|
|
91
|
+
this.workerIdleMs = 750
|
|
92
|
+
this.statConcurrency = STAT_CONCURRENCY
|
|
93
|
+
this.dirConcurrency = DIR_CONCURRENCY
|
|
94
|
+
this.sizeThreshold = SIZE_THRESHOLD
|
|
95
|
+
this._sources = []
|
|
96
|
+
this._sourceBases = new Map()
|
|
97
|
+
this.operationTail = Promise.resolve()
|
|
98
|
+
this.initializationPromise = null
|
|
99
|
+
this.verificationPending = false
|
|
100
|
+
this.scanError = null
|
|
101
|
+
this.scanPersistenceWarning = false
|
|
102
|
+
this.scanScopeId = null
|
|
103
|
+
this.fileAction = null
|
|
104
|
+
this.fileActions = new Map()
|
|
105
|
+
this.actionSequence = 0
|
|
106
|
+
this.scanEvents = null
|
|
107
|
+
this.scanCreatedStores = null
|
|
108
|
+
this.scanRetiredStores = null
|
|
109
|
+
this.scanRestoredStores = null
|
|
110
|
+
}
|
|
111
|
+
get root() {
|
|
112
|
+
return path.resolve(this.kernel.homedir, "vault")
|
|
113
|
+
}
|
|
114
|
+
get blobRoot() {
|
|
115
|
+
return path.resolve(this.root, "sha256")
|
|
116
|
+
}
|
|
117
|
+
storePathFor(hash) {
|
|
118
|
+
if (typeof hash !== "string" || !SHA256_RE.test(hash)) {
|
|
119
|
+
throw new TypeError("Invalid vault content identifier.")
|
|
120
|
+
}
|
|
121
|
+
return path.resolve(this.blobRoot, hash.slice(0, 2), hash)
|
|
122
|
+
}
|
|
123
|
+
async directoryIfSafe(directory) {
|
|
124
|
+
const st = await lstatIfPresent(directory)
|
|
125
|
+
if (st && !st.isDirectory()) throw unsafeStoragePath(directory)
|
|
126
|
+
return st
|
|
127
|
+
}
|
|
128
|
+
async ensureDirectory(directory) {
|
|
129
|
+
let st = await this.directoryIfSafe(directory)
|
|
130
|
+
if (st) return st
|
|
131
|
+
try {
|
|
132
|
+
await fs.promises.mkdir(directory)
|
|
133
|
+
} catch (error) {
|
|
134
|
+
if (!error || error.code !== "EEXIST") throw error
|
|
135
|
+
}
|
|
136
|
+
st = await this.directoryIfSafe(directory)
|
|
137
|
+
if (!st) throw unsafeStoragePath(directory)
|
|
138
|
+
return st
|
|
139
|
+
}
|
|
140
|
+
async storeStatIfPresent(storePath, options = {}) {
|
|
141
|
+
if (!isPathWithin(this.blobRoot, storePath)) throw unsafeStoragePath(storePath)
|
|
142
|
+
if (!await this.directoryIfSafe(this.root) || !await this.directoryIfSafe(this.blobRoot)) {
|
|
143
|
+
throw unsafeStoragePath(this.blobRoot)
|
|
144
|
+
}
|
|
145
|
+
const shard = path.dirname(storePath)
|
|
146
|
+
let shardStat = await this.directoryIfSafe(shard)
|
|
147
|
+
if (!shardStat && options.createParent) shardStat = await this.ensureDirectory(shard)
|
|
148
|
+
if (!shardStat) return null
|
|
149
|
+
return lstatIfPresent(storePath)
|
|
150
|
+
}
|
|
151
|
+
// All dashboard mutations share one queue. Status reads remain concurrent,
|
|
152
|
+
// while scans, conversions, repair, undo, detach, and reclaim can never
|
|
153
|
+
// publish interleaved filesystem/registry state.
|
|
154
|
+
runExclusive(operation) {
|
|
155
|
+
const pending = this.operationTail.then(operation, operation)
|
|
156
|
+
this.operationTail = pending.catch(() => {})
|
|
157
|
+
return pending
|
|
158
|
+
}
|
|
159
|
+
// A filesystem mutation is committed before its registry snapshot is
|
|
160
|
+
// flushed. Keep that distinction in the response: failed persistence is
|
|
161
|
+
// retried by Registry and must not make an already-completed action look as
|
|
162
|
+
// though it never happened.
|
|
163
|
+
runMutation(operation) {
|
|
164
|
+
return this.runExclusive(async () => {
|
|
165
|
+
const result = await operation()
|
|
166
|
+
try {
|
|
167
|
+
if (this.registry) await this.registry.flush()
|
|
168
|
+
} catch (error) {
|
|
169
|
+
return Object.assign({}, result, { persistence_warning: true })
|
|
170
|
+
}
|
|
171
|
+
return result
|
|
172
|
+
})
|
|
173
|
+
}
|
|
174
|
+
queueFileMutation(kind, filePath, sourceId, bytesTotal, operation, progress = {}) {
|
|
175
|
+
const id = `action-${crypto.randomUUID()}`
|
|
176
|
+
const action = {
|
|
177
|
+
id,
|
|
178
|
+
kind,
|
|
179
|
+
path: filePath || null,
|
|
180
|
+
source_id: sourceId || null,
|
|
181
|
+
phase: "queued",
|
|
182
|
+
bytes_copied: 0,
|
|
183
|
+
bytes_total: Math.max(0, Number(bytesTotal) || 0),
|
|
184
|
+
files_completed: Math.max(0, Number(progress.files_completed) || 0),
|
|
185
|
+
files_total: Math.max(0, Number(progress.files_total) || 0)
|
|
186
|
+
}
|
|
187
|
+
this.fileActions.set(id, action)
|
|
188
|
+
this.actionSequence += 1
|
|
189
|
+
return this.runExclusive(async () => {
|
|
190
|
+
action.phase = kind.startsWith("deduplicate") ? "deduplicating" : "copying"
|
|
191
|
+
this.fileAction = action
|
|
192
|
+
this.actionSequence += 1
|
|
193
|
+
try {
|
|
194
|
+
const result = await operation(action)
|
|
195
|
+
try {
|
|
196
|
+
if (this.registry) await this.registry.flush()
|
|
197
|
+
} catch (error) {
|
|
198
|
+
return Object.assign({}, result, { persistence_warning: true })
|
|
199
|
+
}
|
|
200
|
+
return result
|
|
201
|
+
} finally {
|
|
202
|
+
if (this.fileAction === action) this.fileAction = null
|
|
203
|
+
this.fileActions.delete(id)
|
|
204
|
+
this.actionSequence += 1
|
|
205
|
+
}
|
|
206
|
+
})
|
|
207
|
+
}
|
|
208
|
+
beginScanDraft() {
|
|
209
|
+
this.scanEvents = []
|
|
210
|
+
this.scanCreatedStores = []
|
|
211
|
+
this.scanRetiredStores = []
|
|
212
|
+
this.scanRestoredStores = []
|
|
213
|
+
}
|
|
214
|
+
rememberScanStore(storePath, stat) {
|
|
215
|
+
if (this.scanCreatedStores) this.scanCreatedStores.push({ storePath, stat })
|
|
216
|
+
}
|
|
217
|
+
rememberRetiredScanStore(storePath, stat, sourcePaths = []) {
|
|
218
|
+
if (this.scanRetiredStores) {
|
|
219
|
+
this.scanRetiredStores.push({ storePath, stat, sourcePaths: [...new Set(sourcePaths)] })
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
async restoreRetiredScanStore(item) {
|
|
223
|
+
const current = await this.storeStatIfPresent(item.storePath)
|
|
224
|
+
if (current) return sameIdentity(current, item.stat) ? current : null
|
|
225
|
+
const key = this.registry.inoKey(item.stat.dev, item.stat.ino)
|
|
226
|
+
const sourcePaths = new Set([
|
|
227
|
+
...(item.sourcePaths || []),
|
|
228
|
+
...(this.registry.pathsByIno.get(key) || [])
|
|
229
|
+
])
|
|
230
|
+
for (const filePath of sourcePaths) {
|
|
231
|
+
const st = await lstatIfPresent(filePath)
|
|
232
|
+
if (!sameIdentity(st, item.stat)) continue
|
|
233
|
+
await fs.promises.link(filePath, item.storePath)
|
|
234
|
+
const restored = await this.storeStatIfPresent(item.storePath)
|
|
235
|
+
return sameIdentity(restored, item.stat) ? restored : null
|
|
236
|
+
}
|
|
237
|
+
return null
|
|
238
|
+
}
|
|
239
|
+
async commitScanDraft() {
|
|
240
|
+
const events = (this.scanEvents || []).slice(-this.registry.maxEvents)
|
|
241
|
+
const retired = this.scanRetiredStores || []
|
|
242
|
+
const removed = []
|
|
243
|
+
try {
|
|
244
|
+
for (const item of retired) {
|
|
245
|
+
if (!await unlinkIfSame(item.storePath, item.stat)) continue
|
|
246
|
+
removed.push(item)
|
|
247
|
+
const hash = this.registry.byIno.get(this.registry.inoKey(item.stat.dev, item.stat.ino))
|
|
248
|
+
if (hash) await this.refreshLinkSnapshots(hash, item.stat.dev, item.stat.ino)
|
|
249
|
+
}
|
|
250
|
+
} catch (error) {
|
|
251
|
+
for (const item of removed.reverse()) {
|
|
252
|
+
const restored = await this.restoreRetiredScanStore(item).catch(() => null)
|
|
253
|
+
if (restored) this.scanRestoredStores.push({ item, stat: restored })
|
|
254
|
+
}
|
|
255
|
+
throw error
|
|
256
|
+
}
|
|
257
|
+
this.scanEvents = null
|
|
258
|
+
this.scanCreatedStores = null
|
|
259
|
+
this.scanRetiredStores = null
|
|
260
|
+
this.scanRestoredStores = null
|
|
261
|
+
try {
|
|
262
|
+
await this.registry.appendEvents(events)
|
|
263
|
+
return true
|
|
264
|
+
} catch (error) {
|
|
265
|
+
return false
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
async abortScanDraft(snapshot) {
|
|
269
|
+
const created = this.scanCreatedStores || []
|
|
270
|
+
const restored = this.scanRestoredStores || []
|
|
271
|
+
this.scanEvents = null
|
|
272
|
+
this.scanCreatedStores = null
|
|
273
|
+
this.scanRetiredStores = null
|
|
274
|
+
this.scanRestoredStores = null
|
|
275
|
+
for (const item of created.reverse()) {
|
|
276
|
+
await unlinkIfSame(item.storePath, item.stat).catch(() => {})
|
|
277
|
+
}
|
|
278
|
+
this.registry.replaceState(snapshot, { persist: false })
|
|
279
|
+
for (const item of restored) {
|
|
280
|
+
const hash = this.registry.byIno.get(
|
|
281
|
+
this.registry.inoKey(item.stat.dev, item.stat.ino))
|
|
282
|
+
if (hash) await this.refreshLinkSnapshots(hash, item.stat.dev, item.stat.ino)
|
|
283
|
+
}
|
|
284
|
+
return restored.length > 0
|
|
285
|
+
}
|
|
286
|
+
startScan(scopeId = null) {
|
|
287
|
+
if (!this.enabled || !this.sweeper) return { started: false, disabled: true }
|
|
288
|
+
if (this.scanPromise) return { started: false, already_running: true }
|
|
289
|
+
this.scanError = null
|
|
290
|
+
this.scanPersistenceWarning = false
|
|
291
|
+
this.scanScopeId = scopeId
|
|
292
|
+
this.scanPromise = this.runExclusive(() => this.sweeper.scan(scopeId))
|
|
293
|
+
.catch((error) => {
|
|
294
|
+
this.scanError = error && error.message ? error.message : String(error)
|
|
295
|
+
throw error
|
|
296
|
+
})
|
|
297
|
+
.finally(() => {
|
|
298
|
+
this.scanPromise = null
|
|
299
|
+
this.scanScopeId = null
|
|
300
|
+
})
|
|
301
|
+
this.scanPromise.catch(() => {})
|
|
302
|
+
return { started: true }
|
|
303
|
+
}
|
|
304
|
+
async perform(action, payload = {}) {
|
|
305
|
+
if (!this.enabled) return { error: "Save space is disabled." }
|
|
306
|
+
await this.ensureInitialized()
|
|
307
|
+
switch (action) {
|
|
308
|
+
case "add_source": {
|
|
309
|
+
const result = await this.runExclusive(() => this.addExternalSource(payload.path))
|
|
310
|
+
return {
|
|
311
|
+
created: result.created,
|
|
312
|
+
source: {
|
|
313
|
+
id: result.source.id,
|
|
314
|
+
label: result.source.label,
|
|
315
|
+
target_path: result.source.root,
|
|
316
|
+
shareable: !!result.source.shareable
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
case "remove_source":
|
|
321
|
+
return this.runMutation(() => this.removeExternalSource(payload.source_id))
|
|
322
|
+
case "scan":
|
|
323
|
+
if (payload.scope_id && !this.scanSource(payload.scope_id)) {
|
|
324
|
+
return { error: "That scan location is no longer available." }
|
|
325
|
+
}
|
|
326
|
+
if (payload.candidate_min_bytes !== undefined) {
|
|
327
|
+
if (!CANDIDATE_SIZE_OPTIONS.includes(payload.candidate_min_bytes)) {
|
|
328
|
+
return { error: "Choose one of the available minimum file sizes." }
|
|
329
|
+
}
|
|
330
|
+
if (this.scanPromise || (this.sweeper && this.sweeper.state.active)) {
|
|
331
|
+
return { error: "Wait for the current scan to finish before changing the minimum file size." }
|
|
332
|
+
}
|
|
333
|
+
this.sizeThreshold = payload.candidate_min_bytes
|
|
334
|
+
this.registry.settings.candidate_min_bytes = this.sizeThreshold
|
|
335
|
+
this.registry.schedulePersist()
|
|
336
|
+
}
|
|
337
|
+
return this.startScan(payload.scope_id || null)
|
|
338
|
+
case "deduplicate":
|
|
339
|
+
if (typeof payload.scope_id !== "string" || !payload.scope_id) {
|
|
340
|
+
return { error: "Choose a location to deduplicate." }
|
|
341
|
+
}
|
|
342
|
+
{
|
|
343
|
+
const paths = Array.isArray(payload.paths) ? payload.paths : null
|
|
344
|
+
const filesTotal = paths
|
|
345
|
+
? new Set(paths.filter((filePath) => typeof filePath === "string" && filePath)).size
|
|
346
|
+
: this.registry.duplicates.size
|
|
347
|
+
return this.queueFileMutation(
|
|
348
|
+
"deduplicate",
|
|
349
|
+
null,
|
|
350
|
+
payload.scope_id,
|
|
351
|
+
0,
|
|
352
|
+
(progress) => this.deduplicateScope(payload.scope_id, {
|
|
353
|
+
batch_id: `batch-${crypto.randomUUID()}`,
|
|
354
|
+
paths,
|
|
355
|
+
progress
|
|
356
|
+
}),
|
|
357
|
+
{ files_total: filesTotal }
|
|
358
|
+
)
|
|
359
|
+
}
|
|
360
|
+
case "deduplicate_all": {
|
|
361
|
+
const seenPaths = new Set()
|
|
362
|
+
const pathsByScope = new Map()
|
|
363
|
+
for (const group of Array.isArray(payload.groups) ? payload.groups : []) {
|
|
364
|
+
if (!group || typeof group.scope_id !== "string" || !group.scope_id) continue
|
|
365
|
+
if (!pathsByScope.has(group.scope_id)) pathsByScope.set(group.scope_id, [])
|
|
366
|
+
for (const filePath of Array.isArray(group.paths) ? group.paths : []) {
|
|
367
|
+
if (typeof filePath !== "string" || !filePath || seenPaths.has(filePath)) continue
|
|
368
|
+
seenPaths.add(filePath)
|
|
369
|
+
pathsByScope.get(group.scope_id).push(filePath)
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
const groups = [...pathsByScope]
|
|
373
|
+
.map(([scope_id, paths]) => ({ scope_id, paths }))
|
|
374
|
+
.filter((group) => group.paths.length)
|
|
375
|
+
if (!groups.length) return { error: "No displayed files were selected." }
|
|
376
|
+
return this.queueFileMutation(
|
|
377
|
+
"deduplicate_all",
|
|
378
|
+
null,
|
|
379
|
+
null,
|
|
380
|
+
0,
|
|
381
|
+
(progress) => this.deduplicateAll(groups, progress),
|
|
382
|
+
{ files_total: seenPaths.size }
|
|
383
|
+
)
|
|
384
|
+
}
|
|
385
|
+
case "reclaim":
|
|
386
|
+
return this.runMutation(() => this.reclaim(payload.hash))
|
|
387
|
+
case "reclaim_all":
|
|
388
|
+
return this.runMutation(() => this.reclaimAll())
|
|
389
|
+
case "repair":
|
|
390
|
+
case "rebuild":
|
|
391
|
+
if (this.scanPromise || (this.sweeper && this.sweeper.state.active)) {
|
|
392
|
+
return { error: "Wait for the current scan to finish before repairing the index." }
|
|
393
|
+
}
|
|
394
|
+
return this.runMutation(async () => {
|
|
395
|
+
await this.rebuild(undefined, { flush: false })
|
|
396
|
+
return { done: true }
|
|
397
|
+
})
|
|
398
|
+
case "undo":
|
|
399
|
+
return this.queueFileMutation("undo", null, null, 0,
|
|
400
|
+
(action) => this.undoBatch(payload.batch_id, action))
|
|
401
|
+
case "detach": {
|
|
402
|
+
const entry = this.registry.links.get(payload.path)
|
|
403
|
+
return this.queueFileMutation(
|
|
404
|
+
"detach",
|
|
405
|
+
payload.path,
|
|
406
|
+
entry && entry.source_id,
|
|
407
|
+
entry && this.registry.blobs.get(entry.hash)
|
|
408
|
+
? this.registry.blobs.get(entry.hash).size
|
|
409
|
+
: 0,
|
|
410
|
+
(action) => this.detach(payload.path, action)
|
|
411
|
+
)
|
|
412
|
+
}
|
|
413
|
+
case "skip":
|
|
414
|
+
return this.runMutation(() => this.skip(payload.path))
|
|
415
|
+
case "reshare":
|
|
416
|
+
return this.runMutation(() => this.reshare(payload.path))
|
|
417
|
+
default:
|
|
418
|
+
return { error: "unknown action" }
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
async refreshSources() {
|
|
422
|
+
const home = path.resolve(this.kernel.homedir)
|
|
423
|
+
const apiRoot = path.resolve(home, "api")
|
|
424
|
+
const sources = [
|
|
425
|
+
{ id: "pinokio", kind: "pinokio", label: "Pinokio", root: home, parent_id: null },
|
|
426
|
+
{ id: "apps", kind: "virtual", label: "Apps", root: apiRoot, parent_id: "pinokio" },
|
|
427
|
+
{ id: "external", kind: "virtual", label: "External folders", root: null, parent_id: null }
|
|
428
|
+
]
|
|
429
|
+
let storeDev = null
|
|
430
|
+
try {
|
|
431
|
+
storeDev = (await fs.promises.stat(this.root)).dev
|
|
432
|
+
} catch (error) {
|
|
433
|
+
if (!isMissingError(error)) throw error
|
|
434
|
+
}
|
|
435
|
+
const decorate = async (source) => {
|
|
436
|
+
if (!source.root) return source
|
|
437
|
+
try {
|
|
438
|
+
const st = await fs.promises.lstat(source.root)
|
|
439
|
+
source.dev = st.dev
|
|
440
|
+
source.available = st.isDirectory() && !st.isSymbolicLink()
|
|
441
|
+
source.shareable = source.available && storeDev !== null && st.dev === storeDev && this.mode === "link"
|
|
442
|
+
} catch (error) {
|
|
443
|
+
if (!isMissingError(error) && !isAccessError(error)) throw error
|
|
444
|
+
source.available = false
|
|
445
|
+
source.shareable = false
|
|
446
|
+
source.error_code = error && error.code ? error.code : null
|
|
447
|
+
}
|
|
448
|
+
return source
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
const externalPaths = this.registry && Array.isArray(this.registry.settings.external_sources)
|
|
452
|
+
? this.registry.settings.external_sources
|
|
453
|
+
: []
|
|
454
|
+
for (const externalPath of externalPaths) {
|
|
455
|
+
const canonical = path.resolve(externalPath)
|
|
456
|
+
sources.push(await decorate({
|
|
457
|
+
id: externalSourceId(canonical),
|
|
458
|
+
kind: "external",
|
|
459
|
+
label: path.basename(canonical) || canonical,
|
|
460
|
+
root: canonical,
|
|
461
|
+
parent_id: "external"
|
|
462
|
+
}))
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
let apiEntries = []
|
|
466
|
+
try {
|
|
467
|
+
apiEntries = await fs.promises.readdir(apiRoot, { withFileTypes: true })
|
|
468
|
+
} catch (error) {
|
|
469
|
+
if (!isMissingError(error)) throw error
|
|
470
|
+
}
|
|
471
|
+
for (const entry of apiEntries) {
|
|
472
|
+
const mountPath = path.resolve(apiRoot, entry.name)
|
|
473
|
+
if (entry.isDirectory()) {
|
|
474
|
+
sources.push(await decorate({
|
|
475
|
+
id: sourceId("app", entry.name), kind: "app", label: entry.name,
|
|
476
|
+
app: entry.name, root: mountPath, parent_id: "apps"
|
|
477
|
+
}))
|
|
478
|
+
}
|
|
479
|
+
}
|
|
480
|
+
let homeEntries = []
|
|
481
|
+
try {
|
|
482
|
+
homeEntries = await fs.promises.readdir(home, { withFileTypes: true })
|
|
483
|
+
} catch (error) {
|
|
484
|
+
if (!isMissingError(error)) throw error
|
|
485
|
+
}
|
|
486
|
+
for (const entry of homeEntries) {
|
|
487
|
+
if (!entry.isDirectory() || entry.isSymbolicLink() || entry.name === "api" || entry.name === "vault") continue
|
|
488
|
+
sources.push(await decorate({
|
|
489
|
+
id: sourceId("folder", entry.name), kind: "folder", label: entry.name,
|
|
490
|
+
root: path.resolve(home, entry.name), parent_id: "pinokio"
|
|
491
|
+
}))
|
|
492
|
+
}
|
|
493
|
+
const homeSource = sources.find((source) => source.id === "pinokio")
|
|
494
|
+
if (homeSource) await decorate(homeSource)
|
|
495
|
+
this._sources = sources
|
|
496
|
+
this._sourceBases = new Map()
|
|
497
|
+
for (const source of sources) {
|
|
498
|
+
if (!source.root || source.kind === "virtual") continue
|
|
499
|
+
const resolved = path.resolve(source.root)
|
|
500
|
+
const key = process.platform === "win32" ? resolved.toLowerCase() : resolved
|
|
501
|
+
if (!this._sourceBases.has(key)) this._sourceBases.set(key, [])
|
|
502
|
+
this._sourceBases.get(key).push(source)
|
|
503
|
+
}
|
|
504
|
+
return sources
|
|
505
|
+
}
|
|
506
|
+
sources() {
|
|
507
|
+
return this._sources
|
|
508
|
+
}
|
|
509
|
+
async addExternalSource(folderPath) {
|
|
510
|
+
if (!this.enabled) throw new Error("Save space is disabled.")
|
|
511
|
+
if (typeof folderPath !== "string" || !path.isAbsolute(folderPath.trim())) {
|
|
512
|
+
throw new Error("Choose a valid folder.")
|
|
513
|
+
}
|
|
514
|
+
const requested = folderPath.trim()
|
|
515
|
+
let canonical
|
|
516
|
+
let stats
|
|
517
|
+
try {
|
|
518
|
+
canonical = path.resolve(await fs.promises.realpath(requested))
|
|
519
|
+
stats = await fs.promises.stat(canonical)
|
|
520
|
+
} catch (error) {
|
|
521
|
+
throw new Error("That folder is no longer available.")
|
|
522
|
+
}
|
|
523
|
+
if (!stats.isDirectory()) throw new Error("Choose a folder, not a file.")
|
|
524
|
+
|
|
525
|
+
const home = path.resolve(this.kernel.homedir)
|
|
526
|
+
const canonicalHome = path.resolve(await fs.promises.realpath(home))
|
|
527
|
+
if (isPathWithin(canonicalHome, canonical) || isPathWithin(canonical, canonicalHome)) {
|
|
528
|
+
throw new Error("That folder is already inside Pinokio and is included in scans.")
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
await this.refreshSources()
|
|
532
|
+
const existing = this._sources.find((source) => source.kind === "external" && source.root && samePath(source.root, canonical))
|
|
533
|
+
if (existing) {
|
|
534
|
+
return { created: false, source: existing }
|
|
535
|
+
}
|
|
536
|
+
const configured = [...this.registry.settings.external_sources]
|
|
537
|
+
if (configured.some((root) => isPathWithin(root, canonical) || isPathWithin(canonical, root))) {
|
|
538
|
+
throw new Error("Choose a folder that does not contain or overlap another scan location.")
|
|
539
|
+
}
|
|
540
|
+
const previousScan = this.registry.lastScan
|
|
541
|
+
const previousAttempt = this.registry.lastScanAttempt
|
|
542
|
+
this.registry.settings.external_sources = [...configured, canonical]
|
|
543
|
+
this.registry.lastScan = null
|
|
544
|
+
this.registry.lastScanAttempt = null
|
|
545
|
+
this.registry.schedulePersist()
|
|
546
|
+
try {
|
|
547
|
+
await this.registry.flush()
|
|
548
|
+
await this.refreshSources()
|
|
549
|
+
const source = this._sources.find((item) =>
|
|
550
|
+
item.kind === "external" && item.root && samePath(item.root, canonical))
|
|
551
|
+
if (!source) throw new Error("The folder could not be added.")
|
|
552
|
+
return { created: true, source }
|
|
553
|
+
} catch (error) {
|
|
554
|
+
this.registry.settings.external_sources = configured
|
|
555
|
+
this.registry.lastScan = previousScan
|
|
556
|
+
this.registry.lastScanAttempt = previousAttempt
|
|
557
|
+
this.registry.schedulePersist()
|
|
558
|
+
await this.registry.flush().catch(() => {})
|
|
559
|
+
await this.refreshSources().catch(() => {})
|
|
560
|
+
throw error
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
async removeExternalSource(sourceId) {
|
|
564
|
+
if (typeof sourceId !== "string" || !sourceId) {
|
|
565
|
+
throw new Error("Choose an external folder to remove.")
|
|
566
|
+
}
|
|
567
|
+
await this.refreshSources()
|
|
568
|
+
const source = this._sources.find((item) =>
|
|
569
|
+
item.id === sourceId && item.kind === "external" && item.root)
|
|
570
|
+
if (!source) throw new Error("That external folder is no longer configured.")
|
|
571
|
+
|
|
572
|
+
let canonical
|
|
573
|
+
let rootStat
|
|
574
|
+
try {
|
|
575
|
+
canonical = path.resolve(await fs.promises.realpath(source.root))
|
|
576
|
+
rootStat = await fs.promises.stat(canonical)
|
|
577
|
+
} catch (error) {
|
|
578
|
+
throw new Error("Reconnect this folder before removing it.")
|
|
579
|
+
}
|
|
580
|
+
if (!rootStat.isDirectory() || !samePath(canonical, source.root)) {
|
|
581
|
+
throw new Error("Reconnect this folder before removing it.")
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
const storesToRemove = new Map()
|
|
585
|
+
for (const [filePath, entry] of this.registry.links) {
|
|
586
|
+
if (entry.source_id !== source.id) continue
|
|
587
|
+
let fileStat
|
|
588
|
+
try {
|
|
589
|
+
fileStat = await fs.promises.lstat(filePath)
|
|
590
|
+
} catch (error) {
|
|
591
|
+
if (isMissingError(error)) continue
|
|
592
|
+
throw new Error("Some files in this folder could not be checked.")
|
|
593
|
+
}
|
|
594
|
+
if (!fileStat.isFile()) continue
|
|
595
|
+
const storePath = this.storePathFor(entry.hash)
|
|
596
|
+
const storeStat = await this.storeStatIfPresent(storePath)
|
|
597
|
+
const visibleNames = fileStat.nlink - (sameIdentity(fileStat, storeStat) ? 1 : 0)
|
|
598
|
+
if (visibleNames > 1) {
|
|
599
|
+
throw new Error("Turn sharing off for this folder before removing it.")
|
|
600
|
+
}
|
|
601
|
+
if (visibleNames === 1 && sameIdentity(fileStat, storeStat)) {
|
|
602
|
+
storesToRemove.set(entry.hash, {
|
|
603
|
+
path: storePath,
|
|
604
|
+
stat: storeStat
|
|
605
|
+
})
|
|
606
|
+
}
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
for (const [hash, store] of storesToRemove) {
|
|
610
|
+
if (await unlinkIfSame(store.path, store.stat)) {
|
|
611
|
+
this.registry.removeBlob(hash)
|
|
612
|
+
}
|
|
613
|
+
}
|
|
614
|
+
|
|
615
|
+
for (const [filePath, entry] of [...this.registry.links]) {
|
|
616
|
+
if (entry.source_id === source.id) this.registry.untrack(filePath)
|
|
617
|
+
}
|
|
618
|
+
for (const [filePath, entry] of [...this.registry.duplicates]) {
|
|
619
|
+
if (entry.source_id === source.id) this.registry.untrack(filePath)
|
|
620
|
+
}
|
|
621
|
+
for (const [filePath, entry] of [...this.registry.scanIndex]) {
|
|
622
|
+
if (entry.source_id === source.id) this.registry.untrack(filePath)
|
|
623
|
+
}
|
|
624
|
+
for (const [filePath, entry] of [...this.registry.excluded]) {
|
|
625
|
+
if (entry.source_id === source.id) this.registry.excluded.delete(filePath)
|
|
626
|
+
}
|
|
627
|
+
this.registry.sourceScans.delete(source.id)
|
|
628
|
+
this.registry.sourceScanAttempts.delete(source.id)
|
|
629
|
+
this.registry.settings.external_sources = this.registry.settings.external_sources
|
|
630
|
+
.filter((root) => !samePath(root, source.root))
|
|
631
|
+
this.registry.lastScan = null
|
|
632
|
+
this.registry.lastScanAttempt = null
|
|
633
|
+
this.registry.schedulePersist()
|
|
634
|
+
await this.refreshSources()
|
|
635
|
+
return { removed: true, source_id: source.id }
|
|
636
|
+
}
|
|
637
|
+
sourceForPath(filePath, preferredId) {
|
|
638
|
+
let cursor = path.resolve(filePath)
|
|
639
|
+
while (true) {
|
|
640
|
+
const key = process.platform === "win32" ? cursor.toLowerCase() : cursor
|
|
641
|
+
const matches = this._sourceBases.get(key)
|
|
642
|
+
if (matches && matches.length) {
|
|
643
|
+
return matches.find((source) => source.id === preferredId) || matches[0]
|
|
644
|
+
}
|
|
645
|
+
const parent = path.dirname(cursor)
|
|
646
|
+
if (parent === cursor) return null
|
|
647
|
+
cursor = parent
|
|
648
|
+
}
|
|
649
|
+
}
|
|
650
|
+
async canonicalPathIsWithinSource(filePath, source, options = {}) {
|
|
651
|
+
if (!source || !source.root) return false
|
|
652
|
+
try {
|
|
653
|
+
const [canonicalRoot, canonicalFile] = await Promise.all([
|
|
654
|
+
fs.promises.realpath(source.root),
|
|
655
|
+
fs.promises.realpath(filePath)
|
|
656
|
+
])
|
|
657
|
+
return isPathWithin(canonicalRoot, canonicalFile)
|
|
658
|
+
} catch (error) {
|
|
659
|
+
if (options.strictErrors && !isMissingError(error)) throw error
|
|
660
|
+
return false
|
|
661
|
+
}
|
|
662
|
+
}
|
|
663
|
+
locationForPath(filePath, preferredId) {
|
|
664
|
+
const source = this.sourceForPath(filePath, preferredId)
|
|
665
|
+
if (!source) return { source_id: null, relative_path: path.basename(filePath) }
|
|
666
|
+
const absolute = path.resolve(filePath)
|
|
667
|
+
const relative = path.relative(source.root, absolute).split(path.sep).join("/") || path.basename(absolute)
|
|
668
|
+
return {
|
|
669
|
+
source_id: source.id,
|
|
670
|
+
source_kind: source.kind,
|
|
671
|
+
source_label: source.label,
|
|
672
|
+
relative_path: relative
|
|
673
|
+
}
|
|
674
|
+
}
|
|
675
|
+
async recordEvent(event) {
|
|
676
|
+
let entry = event
|
|
677
|
+
if (event.path) {
|
|
678
|
+
const location = this.locationForPath(event.path, event.source_id)
|
|
679
|
+
const context = location.source_id ? location : { relative_path: event.path }
|
|
680
|
+
entry = Object.assign({}, event, context)
|
|
681
|
+
}
|
|
682
|
+
if (this.scanEvents) {
|
|
683
|
+
this.scanEvents.push(entry)
|
|
684
|
+
if (this.scanEvents.length > this.registry.maxEvents * 2) {
|
|
685
|
+
this.scanEvents.splice(0, this.registry.maxEvents)
|
|
686
|
+
}
|
|
687
|
+
return true
|
|
688
|
+
}
|
|
689
|
+
try {
|
|
690
|
+
await this.registry.appendEvent(entry)
|
|
691
|
+
return true
|
|
692
|
+
} catch (error) {
|
|
693
|
+
return false
|
|
694
|
+
}
|
|
695
|
+
}
|
|
696
|
+
scanSource(scopeId) {
|
|
697
|
+
if (!scopeId) return null
|
|
698
|
+
return this._sources.find((source) =>
|
|
699
|
+
source.id === scopeId && source.kind !== "virtual" && source.available && source.root) || null
|
|
700
|
+
}
|
|
701
|
+
scanRoots(scopeId = null) {
|
|
702
|
+
if (scopeId) {
|
|
703
|
+
const source = this.scanSource(scopeId)
|
|
704
|
+
return source ? [{ root: path.resolve(source.root), source_id: source.id }] : []
|
|
705
|
+
}
|
|
706
|
+
const home = path.resolve(this.kernel.homedir)
|
|
707
|
+
const candidates = [{ root: home, source_id: "pinokio" }]
|
|
708
|
+
for (const source of this._sources) {
|
|
709
|
+
if (source.kind !== "external" || !source.available || !source.root) continue
|
|
710
|
+
const canonical = path.resolve(source.root)
|
|
711
|
+
if (isPathWithin(home, canonical) || isPathWithin(canonical, home)) continue
|
|
712
|
+
candidates.push({ root: canonical, source_id: source.id })
|
|
713
|
+
}
|
|
714
|
+
candidates.sort((left, right) => left.root.length - right.root.length)
|
|
715
|
+
const roots = []
|
|
716
|
+
for (const candidate of candidates) {
|
|
717
|
+
if (roots.some((existing) => isPathWithin(existing.root, candidate.root))) continue
|
|
718
|
+
roots.push(candidate)
|
|
719
|
+
}
|
|
720
|
+
return roots
|
|
721
|
+
}
|
|
722
|
+
reconcileConfiguredSources() {
|
|
723
|
+
if (!this.registry) return
|
|
724
|
+
const configured = (filePath, sourceId) => !!this.sourceForPath(filePath, sourceId)
|
|
725
|
+
for (const [filePath, entry] of [...this.registry.links]) {
|
|
726
|
+
if (!configured(filePath, entry.source_id)) this.registry.untrack(filePath)
|
|
727
|
+
}
|
|
728
|
+
for (const [filePath, entry] of [...this.registry.duplicates]) {
|
|
729
|
+
if (!configured(filePath, entry.source_id)) this.registry.untrack(filePath)
|
|
730
|
+
}
|
|
731
|
+
for (const [filePath, entry] of [...this.registry.scanIndex]) {
|
|
732
|
+
if (!configured(filePath, entry.source_id)) this.registry.untrack(filePath)
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
// Kill switch: system ENVIRONMENT variable PINOKIO_VAULT, default unset =
|
|
736
|
+
// enabled. Read directly (single key) to avoid loading the environment
|
|
737
|
+
// module chain before the kernel is fully up.
|
|
738
|
+
async isEnabled() {
|
|
739
|
+
let value = process.env.PINOKIO_VAULT
|
|
740
|
+
if (value === undefined && this.kernel.homedir) {
|
|
741
|
+
try {
|
|
742
|
+
const raw = await fs.promises.readFile(path.resolve(this.kernel.homedir, "ENVIRONMENT"), "utf8")
|
|
743
|
+
for (const line of raw.split("\n")) {
|
|
744
|
+
const m = line.match(/^\s*PINOKIO_VAULT\s*=\s*(.*)\s*$/)
|
|
745
|
+
if (m) value = m[1].trim()
|
|
746
|
+
}
|
|
747
|
+
} catch (error) {
|
|
748
|
+
if (!isMissingError(error)) throw error
|
|
749
|
+
}
|
|
750
|
+
}
|
|
751
|
+
return String(value).toLowerCase() !== "false"
|
|
752
|
+
}
|
|
753
|
+
// Disabled ⇒ do nothing observable: no directory, no registry, no probes.
|
|
754
|
+
async init(options = {}) {
|
|
755
|
+
this.enabled = await this.isEnabled()
|
|
756
|
+
if (!this.enabled) return { enabled: false }
|
|
757
|
+
if (options.existingOnly) {
|
|
758
|
+
try {
|
|
759
|
+
await fs.promises.stat(this.root)
|
|
760
|
+
} catch (error) {
|
|
761
|
+
if (error && error.code === "ENOENT") return { enabled: true, fresh: true }
|
|
762
|
+
throw error
|
|
763
|
+
}
|
|
764
|
+
}
|
|
765
|
+
return this.initializeStorage()
|
|
766
|
+
}
|
|
767
|
+
async initializeStorage() {
|
|
768
|
+
if (this.initialized) return { enabled: true, mode: this.mode }
|
|
769
|
+
await this.ensureDirectory(this.root)
|
|
770
|
+
await this.ensureDirectory(this.blobRoot)
|
|
771
|
+
this.registry = new Registry(this.root)
|
|
772
|
+
this.mode = await this.probe(this.root)
|
|
773
|
+
const loaded = await this.registry.load()
|
|
774
|
+
await this.refreshSources()
|
|
775
|
+
const missingWithBlobs = !loaded.existed && await this.hasStoredBlobs()
|
|
776
|
+
if (loaded.corrupt || missingWithBlobs) {
|
|
777
|
+
// Recovery may inspect Vault's own hash shards, but it must not discover
|
|
778
|
+
// or walk configured sources. A later explicit Scan or Repair rebuilds
|
|
779
|
+
// visible-path state.
|
|
780
|
+
await this.rebuild([], { preserveState: false })
|
|
781
|
+
}
|
|
782
|
+
this.sizeThreshold = this.registry.settings.candidate_min_bytes || SIZE_THRESHOLD
|
|
783
|
+
this.sweeper = new Sweeper(this)
|
|
784
|
+
this.initialized = true
|
|
785
|
+
return {
|
|
786
|
+
enabled: true,
|
|
787
|
+
mode: this.mode,
|
|
788
|
+
corrupt_recovered: !!loaded.corrupt,
|
|
789
|
+
missing_recovered: missingWithBlobs
|
|
790
|
+
}
|
|
791
|
+
}
|
|
792
|
+
async ensureInitialized() {
|
|
793
|
+
if (!this.enabled) return { enabled: false }
|
|
794
|
+
if (this.initialized && !this.verificationPending) return { enabled: true, mode: this.mode }
|
|
795
|
+
if (!this.initializationPromise) {
|
|
796
|
+
this.initializationPromise = (async () => {
|
|
797
|
+
const result = this.initialized
|
|
798
|
+
? { enabled: true, mode: this.mode }
|
|
799
|
+
: await this.initializeStorage()
|
|
800
|
+
await this.runExclusive(async () => {
|
|
801
|
+
await this.verify()
|
|
802
|
+
await this.registry.flush()
|
|
803
|
+
})
|
|
804
|
+
this.verificationPending = false
|
|
805
|
+
return result
|
|
806
|
+
})().catch((error) => {
|
|
807
|
+
if (this.initialized) this.verificationPending = true
|
|
808
|
+
throw error
|
|
809
|
+
}).finally(() => {
|
|
810
|
+
this.initializationPromise = null
|
|
811
|
+
})
|
|
812
|
+
}
|
|
813
|
+
return this.initializationPromise
|
|
814
|
+
}
|
|
815
|
+
async hasStoredBlobs() {
|
|
816
|
+
let shards = []
|
|
817
|
+
try {
|
|
818
|
+
shards = await fs.promises.readdir(this.blobRoot, { withFileTypes: true })
|
|
819
|
+
} catch (error) {
|
|
820
|
+
if (isMissingError(error)) return false
|
|
821
|
+
throw error
|
|
822
|
+
}
|
|
823
|
+
for (const shard of shards) {
|
|
824
|
+
if (!shard.isDirectory() || !/^[0-9a-f]{2}$/.test(shard.name)) continue
|
|
825
|
+
const shardPath = path.resolve(this.blobRoot, shard.name)
|
|
826
|
+
const shardStat = await this.directoryIfSafe(shardPath)
|
|
827
|
+
if (!shardStat) continue
|
|
828
|
+
let names = []
|
|
829
|
+
try {
|
|
830
|
+
names = await fs.promises.readdir(shardPath)
|
|
831
|
+
} catch (error) {
|
|
832
|
+
if (isMissingError(error)) continue
|
|
833
|
+
throw error
|
|
834
|
+
}
|
|
835
|
+
if (names.some((name) => SHA256_RE.test(name) && name.startsWith(shard.name))) return true
|
|
836
|
+
}
|
|
837
|
+
return false
|
|
838
|
+
}
|
|
839
|
+
// Capability probe: temp file + node:fs.link + nlink check. Once per volume.
|
|
840
|
+
async probe(dir) {
|
|
841
|
+
const dev = (await fs.promises.stat(dir)).dev
|
|
842
|
+
if (this.volumeModes.has(dev)) return this.volumeModes.get(dev)
|
|
843
|
+
const a = path.resolve(dir, `.pinokio-probe-${crypto.randomBytes(6).toString("hex")}`)
|
|
844
|
+
const b = a + "-link"
|
|
845
|
+
let mode = "unsupported"
|
|
846
|
+
let aStat = null
|
|
847
|
+
let bStat = null
|
|
848
|
+
let failure = null
|
|
849
|
+
try {
|
|
850
|
+
await fs.promises.writeFile(a, "probe", { flag: "wx" })
|
|
851
|
+
aStat = await fs.promises.lstat(a)
|
|
852
|
+
await fs.promises.link(a, b)
|
|
853
|
+
bStat = await fs.promises.lstat(b)
|
|
854
|
+
if (sameIdentity(bStat, aStat) && bStat.nlink === 2) mode = "link"
|
|
855
|
+
} catch (error) {
|
|
856
|
+
if (!NO_LINK_CODES.has(error && error.code)) failure = error
|
|
857
|
+
} finally {
|
|
858
|
+
try {
|
|
859
|
+
await unlinkIfSame(b, bStat)
|
|
860
|
+
await unlinkIfSame(a, aStat)
|
|
861
|
+
} catch (error) {
|
|
862
|
+
if (!failure) failure = error
|
|
863
|
+
}
|
|
864
|
+
}
|
|
865
|
+
if (failure) throw failure
|
|
866
|
+
this.volumeModes.set(dev, mode)
|
|
867
|
+
return mode
|
|
868
|
+
}
|
|
869
|
+
async hashFile(filePath) {
|
|
870
|
+
if (this.workerIdleTimer) {
|
|
871
|
+
clearTimeout(this.workerIdleTimer)
|
|
872
|
+
this.workerIdleTimer = null
|
|
873
|
+
}
|
|
874
|
+
if (!this.worker) {
|
|
875
|
+
const worker = new Worker(path.resolve(__dirname, "hash_worker.js"))
|
|
876
|
+
this.worker = worker
|
|
877
|
+
worker.unref()
|
|
878
|
+
worker.on("message", ({ id, hash, size, error }) => {
|
|
879
|
+
const job = this.workerJobs.get(id)
|
|
880
|
+
if (!job) return
|
|
881
|
+
this.workerJobs.delete(id)
|
|
882
|
+
if (error) {
|
|
883
|
+
const failure = new Error(error.message || String(error))
|
|
884
|
+
if (error.code) failure.code = error.code
|
|
885
|
+
job.reject(failure)
|
|
886
|
+
}
|
|
887
|
+
else job.resolve({ hash, size })
|
|
888
|
+
// Keep the worker warm across the scan queue. Starting one worker per
|
|
889
|
+
// file costs ~30ms and adds up quickly on a first scan.
|
|
890
|
+
if (this.workerJobs.size === 0 && this.worker === worker) {
|
|
891
|
+
this.workerIdleTimer = setTimeout(() => {
|
|
892
|
+
this.workerIdleTimer = null
|
|
893
|
+
if (this.workerJobs.size === 0 && this.worker === worker) {
|
|
894
|
+
this.worker = null
|
|
895
|
+
worker.terminate().catch(() => {})
|
|
896
|
+
}
|
|
897
|
+
}, this.workerIdleMs)
|
|
898
|
+
if (this.workerIdleTimer.unref) this.workerIdleTimer.unref()
|
|
899
|
+
}
|
|
900
|
+
})
|
|
901
|
+
worker.on("error", (error) => {
|
|
902
|
+
if (this.worker !== worker) return
|
|
903
|
+
if (this.workerIdleTimer) clearTimeout(this.workerIdleTimer)
|
|
904
|
+
this.workerIdleTimer = null
|
|
905
|
+
for (const job of this.workerJobs.values()) job.reject(error)
|
|
906
|
+
this.workerJobs.clear()
|
|
907
|
+
this.worker = null
|
|
908
|
+
})
|
|
909
|
+
worker.on("exit", (code) => {
|
|
910
|
+
if (this.worker !== worker) return
|
|
911
|
+
if (this.workerIdleTimer) clearTimeout(this.workerIdleTimer)
|
|
912
|
+
this.workerIdleTimer = null
|
|
913
|
+
for (const job of this.workerJobs.values()) {
|
|
914
|
+
job.reject(new Error(`hash worker exited with code ${code}`))
|
|
915
|
+
}
|
|
916
|
+
this.workerJobs.clear()
|
|
917
|
+
this.worker = null
|
|
918
|
+
})
|
|
919
|
+
}
|
|
920
|
+
const id = ++this.workerSeq
|
|
921
|
+
return new Promise((resolve, reject) => {
|
|
922
|
+
this.workerJobs.set(id, { resolve, reject })
|
|
923
|
+
this.worker.postMessage({ id, filePath })
|
|
924
|
+
})
|
|
925
|
+
}
|
|
926
|
+
async refreshLinkSnapshots(hash, dev, ino) {
|
|
927
|
+
if (!this.registry) return
|
|
928
|
+
const key = this.registry.inoKey(dev, ino)
|
|
929
|
+
const paths = [...(this.registry.pathsByIno.get(key) || [])]
|
|
930
|
+
for (const linkPath of paths) {
|
|
931
|
+
const entry = this.registry.links.get(linkPath)
|
|
932
|
+
if (!entry || entry.hash !== hash) continue
|
|
933
|
+
try {
|
|
934
|
+
const source = this.sourceForPath(linkPath, entry.source_id)
|
|
935
|
+
if (!source || !await this.canonicalPathIsWithinSource(linkPath, source)) continue
|
|
936
|
+
const st = await fs.promises.lstat(linkPath)
|
|
937
|
+
if (st.dev !== dev || st.ino !== ino) continue
|
|
938
|
+
this.registry.setScanEntry(linkPath, Object.assign(fileSnapshot(st), {
|
|
939
|
+
hash,
|
|
940
|
+
source_id: entry.source_id || null
|
|
941
|
+
}))
|
|
942
|
+
} catch (error) {
|
|
943
|
+
this.registry.scanIndex.delete(linkPath)
|
|
944
|
+
entry.unverified = true
|
|
945
|
+
}
|
|
946
|
+
}
|
|
947
|
+
this.registry.schedulePersist()
|
|
948
|
+
}
|
|
949
|
+
async verifyStoreContent(hash, storePath, storeStat) {
|
|
950
|
+
if (!storeStat || !storeStat.isFile()) return { valid: false }
|
|
951
|
+
const key = this.registry.inoKey(storeStat.dev, storeStat.ino)
|
|
952
|
+
for (const linkPath of this.registry.pathsByIno.get(key) || []) {
|
|
953
|
+
const entry = this.registry.links.get(linkPath)
|
|
954
|
+
if (!entry) continue
|
|
955
|
+
if (entry.hash !== hash ||
|
|
956
|
+
entry.dev !== storeStat.dev || entry.ino !== storeStat.ino) continue
|
|
957
|
+
const expected = this.registry.scanIndex.get(linkPath)
|
|
958
|
+
if (expected && expected.hash === hash && sameSnapshot(expected, storeStat)) {
|
|
959
|
+
return { valid: true, snapshot: fileSnapshot(storeStat) }
|
|
960
|
+
}
|
|
961
|
+
}
|
|
962
|
+
|
|
963
|
+
const before = fileSnapshot(storeStat)
|
|
964
|
+
let hashed
|
|
965
|
+
try {
|
|
966
|
+
hashed = await this.hashFile(storePath)
|
|
967
|
+
} catch (error) {
|
|
968
|
+
if (isMissingError(error)) return { valid: false }
|
|
969
|
+
throw error
|
|
970
|
+
}
|
|
971
|
+
let after
|
|
972
|
+
try {
|
|
973
|
+
after = await this.storeStatIfPresent(storePath)
|
|
974
|
+
} catch (error) {
|
|
975
|
+
if (isMissingError(error)) return { valid: false }
|
|
976
|
+
throw error
|
|
977
|
+
}
|
|
978
|
+
if (!sameSnapshot(before, after) || hashed.hash !== hash || hashed.size !== after.size) {
|
|
979
|
+
return { valid: false }
|
|
980
|
+
}
|
|
981
|
+
await this.refreshLinkSnapshots(hash, after.dev, after.ino)
|
|
982
|
+
return { valid: true, snapshot: fileSnapshot(after) }
|
|
983
|
+
}
|
|
984
|
+
// adopt: give an existing file a store name. Metadata-only, never copies.
|
|
985
|
+
async adopt(filePath, hash, meta = {}) {
|
|
986
|
+
if (!this.enabled) return { status: "disabled" }
|
|
987
|
+
const storePath = this.storePathFor(hash)
|
|
988
|
+
const st = await fs.promises.lstat(filePath)
|
|
989
|
+
if (!st.isFile()) return { status: "stale" }
|
|
990
|
+
if (meta.expected && !sameSnapshot(meta.expected, st)) return { status: "stale" }
|
|
991
|
+
const before = fileSnapshot(st)
|
|
992
|
+
const linkEntry = {
|
|
993
|
+
hash,
|
|
994
|
+
app: meta.app || null,
|
|
995
|
+
source_id: meta.source_id || null,
|
|
996
|
+
dev: st.dev,
|
|
997
|
+
ino: st.ino,
|
|
998
|
+
}
|
|
999
|
+
const storeStat = await this.storeStatIfPresent(storePath, {
|
|
1000
|
+
createParent: this.mode === "link"
|
|
1001
|
+
})
|
|
1002
|
+
if (storeStat) {
|
|
1003
|
+
if (storeStat.dev === st.dev && storeStat.ino === st.ino) {
|
|
1004
|
+
const current = await lstatIfPresent(filePath)
|
|
1005
|
+
if (!sameSnapshot(before, current)) return { status: "stale" }
|
|
1006
|
+
this.registry.addBlob(hash, { size: st.size, source_urls: meta.source_urls || [] })
|
|
1007
|
+
this.registry.addLink(filePath, linkEntry)
|
|
1008
|
+
await this.refreshLinkSnapshots(hash, st.dev, st.ino)
|
|
1009
|
+
return { status: "already" }
|
|
1010
|
+
}
|
|
1011
|
+
const current = await lstatIfPresent(filePath)
|
|
1012
|
+
if (!sameSnapshot(before, current)) return { status: "stale" }
|
|
1013
|
+
return { status: "duplicate", storePath }
|
|
1014
|
+
}
|
|
1015
|
+
if (this.mode !== "link") return { status: "unavailable", code: "ENOTSUP" }
|
|
1016
|
+
try {
|
|
1017
|
+
await fs.promises.link(filePath, storePath)
|
|
1018
|
+
} catch (e) {
|
|
1019
|
+
if (NO_LINK_CODES.has(e.code)) {
|
|
1020
|
+
return { status: "unavailable", code: e.code }
|
|
1021
|
+
}
|
|
1022
|
+
throw e
|
|
1023
|
+
}
|
|
1024
|
+
const [current, currentStore] = await Promise.all([
|
|
1025
|
+
lstatIfPresent(filePath),
|
|
1026
|
+
this.storeStatIfPresent(storePath)
|
|
1027
|
+
])
|
|
1028
|
+
if (!sameContentState(before, current) || !currentStore ||
|
|
1029
|
+
currentStore.dev !== st.dev || currentStore.ino !== st.ino) {
|
|
1030
|
+
// Remove only a store name that still points either to the inode we
|
|
1031
|
+
// intended to link or to the current source inode that won the race.
|
|
1032
|
+
// An unrelated replacement at the reserved store path is preserved.
|
|
1033
|
+
const ownsStoreName = currentStore && (
|
|
1034
|
+
(currentStore.dev === st.dev && currentStore.ino === st.ino) ||
|
|
1035
|
+
(current && currentStore.dev === current.dev && currentStore.ino === current.ino)
|
|
1036
|
+
)
|
|
1037
|
+
if (ownsStoreName) {
|
|
1038
|
+
try {
|
|
1039
|
+
await fs.promises.unlink(storePath)
|
|
1040
|
+
} catch (error) {
|
|
1041
|
+
if (!isMissingError(error)) throw error
|
|
1042
|
+
}
|
|
1043
|
+
}
|
|
1044
|
+
return { status: "stale" }
|
|
1045
|
+
}
|
|
1046
|
+
this.registry.addBlob(hash, { size: st.size, source_urls: meta.source_urls || [] })
|
|
1047
|
+
this.registry.addLink(filePath, linkEntry)
|
|
1048
|
+
this.rememberScanStore(storePath, currentStore)
|
|
1049
|
+
await this.refreshLinkSnapshots(hash, st.dev, st.ino)
|
|
1050
|
+
await this.recordEvent({ kind: "adopt", hash, path: filePath, app: meta.app || null, source_id: meta.source_id || null, bytes_saved: 0 })
|
|
1051
|
+
return { status: "adopted" }
|
|
1052
|
+
}
|
|
1053
|
+
// convert: replace a byte-identical duplicate with a link to an existing
|
|
1054
|
+
// blob. Atomic link+rename; target is never missing at any instant.
|
|
1055
|
+
async convert(targetPath, hash, meta = {}) {
|
|
1056
|
+
if (!this.enabled) return { status: "disabled" }
|
|
1057
|
+
const storePath = this.storePathFor(hash)
|
|
1058
|
+
const storeStat = await this.storeStatIfPresent(storePath)
|
|
1059
|
+
if (!storeStat) return { status: "no-blob" }
|
|
1060
|
+
let targetStat
|
|
1061
|
+
try {
|
|
1062
|
+
targetStat = await fs.promises.lstat(targetPath)
|
|
1063
|
+
} catch (error) {
|
|
1064
|
+
if (LOCK_CODES.has(error && error.code)) return { status: "locked" }
|
|
1065
|
+
if (isMissingError(error)) return { status: "stale" }
|
|
1066
|
+
throw error
|
|
1067
|
+
}
|
|
1068
|
+
if (!storeStat.isFile() || !targetStat.isFile()) return { status: "stale" }
|
|
1069
|
+
if (meta.expected && !sameSnapshot(meta.expected, targetStat)) {
|
|
1070
|
+
return { status: "stale" }
|
|
1071
|
+
}
|
|
1072
|
+
if (storeStat.dev !== targetStat.dev) {
|
|
1073
|
+
return { status: "unavailable", code: "EXDEV" }
|
|
1074
|
+
}
|
|
1075
|
+
if (storeStat.dev === targetStat.dev && storeStat.ino === targetStat.ino) {
|
|
1076
|
+
this.registry.addLink(targetPath, {
|
|
1077
|
+
hash, app: meta.app || null, source_id: meta.source_id || null,
|
|
1078
|
+
dev: storeStat.dev, ino: storeStat.ino, unverified: false
|
|
1079
|
+
})
|
|
1080
|
+
await this.refreshLinkSnapshots(hash, storeStat.dev, storeStat.ino)
|
|
1081
|
+
return { status: "already" }
|
|
1082
|
+
}
|
|
1083
|
+
if (storeStat.size !== targetStat.size) {
|
|
1084
|
+
return { status: "size-mismatch" }
|
|
1085
|
+
}
|
|
1086
|
+
if (!sameFileMetadata(storeStat, targetStat)) {
|
|
1087
|
+
return { status: "metadata-mismatch" }
|
|
1088
|
+
}
|
|
1089
|
+
const verifiedStore = await this.verifyStoreContent(hash, storePath, storeStat)
|
|
1090
|
+
if (!verifiedStore.valid) return { status: "stale-blob" }
|
|
1091
|
+
if (meta.source && this.sourceAppIsRunning(meta.source)) return { status: "locked" }
|
|
1092
|
+
const tmp = targetPath + TMP_SUFFIX
|
|
1093
|
+
let ownsTmp = false
|
|
1094
|
+
try {
|
|
1095
|
+
try {
|
|
1096
|
+
await fs.promises.link(storePath, tmp)
|
|
1097
|
+
ownsTmp = true
|
|
1098
|
+
} catch (e) {
|
|
1099
|
+
if (e.code === "EEXIST") {
|
|
1100
|
+
let tmpStat = null
|
|
1101
|
+
try { tmpStat = await fs.promises.lstat(tmp) } catch (error) {}
|
|
1102
|
+
if (!tmpStat || tmpStat.dev !== storeStat.dev || tmpStat.ino !== storeStat.ino) {
|
|
1103
|
+
return { status: "conflict" }
|
|
1104
|
+
}
|
|
1105
|
+
} else {
|
|
1106
|
+
throw e
|
|
1107
|
+
}
|
|
1108
|
+
}
|
|
1109
|
+
// Narrow the portable stat/rename race as far as Node permits. The app
|
|
1110
|
+
// guard prevents known Pinokio writers; arbitrary external writers in the
|
|
1111
|
+
// final syscall-sized interval are the documented residual limitation.
|
|
1112
|
+
if (meta.source && this.sourceAppIsRunning(meta.source)) {
|
|
1113
|
+
if (ownsTmp) await unlinkIfSame(tmp, storeStat).catch(() => {})
|
|
1114
|
+
return { status: "locked" }
|
|
1115
|
+
}
|
|
1116
|
+
const currentTmpStat = await lstatIfPresent(tmp)
|
|
1117
|
+
if (!currentTmpStat || currentTmpStat.dev !== storeStat.dev || currentTmpStat.ino !== storeStat.ino) {
|
|
1118
|
+
if (ownsTmp) await unlinkIfSame(tmp, storeStat).catch(() => {})
|
|
1119
|
+
return { status: "conflict" }
|
|
1120
|
+
}
|
|
1121
|
+
const currentTargetStat = await fs.promises.lstat(targetPath)
|
|
1122
|
+
if (!sameSnapshot(fileSnapshot(targetStat), currentTargetStat)) {
|
|
1123
|
+
if (ownsTmp) await unlinkIfSame(tmp, storeStat).catch(() => {})
|
|
1124
|
+
return { status: "stale" }
|
|
1125
|
+
}
|
|
1126
|
+
const currentStoreStat = await this.storeStatIfPresent(storePath)
|
|
1127
|
+
// Creating our temporary hardlink changes ctime on the shared inode.
|
|
1128
|
+
// Identity, size, and mtime still detect replacement or content writes.
|
|
1129
|
+
if (!currentStoreStat || !sameContentState(verifiedStore.snapshot, currentStoreStat) ||
|
|
1130
|
+
!sameFileMetadata(currentStoreStat, currentTargetStat)) {
|
|
1131
|
+
if (ownsTmp) await unlinkIfSame(tmp, storeStat).catch(() => {})
|
|
1132
|
+
return { status: "stale-blob" }
|
|
1133
|
+
}
|
|
1134
|
+
await fs.promises.rename(tmp, targetPath)
|
|
1135
|
+
ownsTmp = false
|
|
1136
|
+
} catch (e) {
|
|
1137
|
+
if (ownsTmp) await unlinkIfSame(tmp, storeStat).catch(() => {})
|
|
1138
|
+
if (LOCK_CODES.has(e.code)) {
|
|
1139
|
+
return { status: "locked" }
|
|
1140
|
+
}
|
|
1141
|
+
if (NO_LINK_CODES.has(e.code)) {
|
|
1142
|
+
return { status: "unavailable", code: e.code }
|
|
1143
|
+
}
|
|
1144
|
+
throw e
|
|
1145
|
+
}
|
|
1146
|
+
let st
|
|
1147
|
+
try {
|
|
1148
|
+
st = await fs.promises.lstat(targetPath)
|
|
1149
|
+
} catch (error) {
|
|
1150
|
+
if (isMissingError(error)) return { status: "stale" }
|
|
1151
|
+
// rename() is the commit point: it succeeded with a temporary name
|
|
1152
|
+
// whose identity was already verified. A later metadata read failure
|
|
1153
|
+
// must not report the completed replacement as failed.
|
|
1154
|
+
st = storeStat
|
|
1155
|
+
}
|
|
1156
|
+
if (st.dev !== storeStat.dev || st.ino !== storeStat.ino) {
|
|
1157
|
+
return { status: "stale" }
|
|
1158
|
+
}
|
|
1159
|
+
this.registry.addLink(targetPath, {
|
|
1160
|
+
hash, app: meta.app || null, source_id: meta.source_id || null,
|
|
1161
|
+
dev: st.dev, ino: st.ino, batch_id: meta.batch_id || null,
|
|
1162
|
+
unverified: false
|
|
1163
|
+
})
|
|
1164
|
+
await this.refreshLinkSnapshots(hash, st.dev, st.ino)
|
|
1165
|
+
await this.recordEvent({
|
|
1166
|
+
kind: "convert", hash, path: targetPath, app: meta.app || null, source_id: meta.source_id || null,
|
|
1167
|
+
bytes_saved: storeStat.size, batch_id: meta.batch_id || null
|
|
1168
|
+
})
|
|
1169
|
+
return { status: "converted", bytes_saved: storeStat.size }
|
|
1170
|
+
}
|
|
1171
|
+
sourceAppIsRunning(source) {
|
|
1172
|
+
if (!source || source.kind !== "app") return false
|
|
1173
|
+
const appRoot = path.resolve(this.kernel.homedir, "api", source.app)
|
|
1174
|
+
const api = this.kernel.api || {}
|
|
1175
|
+
const running = api.running || {}
|
|
1176
|
+
const runningPaths = api.running_paths || {}
|
|
1177
|
+
return Object.keys(running).some((runningId) => {
|
|
1178
|
+
if (!running[runningId]) return false
|
|
1179
|
+
const runningPath = runningPaths[runningId] || (path.isAbsolute(runningId) ? runningId.split("?")[0] : null)
|
|
1180
|
+
return !!(runningPath && isPathWithin(appRoot, runningPath))
|
|
1181
|
+
})
|
|
1182
|
+
}
|
|
1183
|
+
async deduplicateScope(scopeId, options = {}) {
|
|
1184
|
+
await this.refreshSources()
|
|
1185
|
+
const source = this._sources.find((item) => item.id === scopeId && item.kind !== "virtual")
|
|
1186
|
+
if (!source) return { error: "That location is no longer available. Scan again to refresh it." }
|
|
1187
|
+
if (!source.shareable) return { error: "This location is on a disk that cannot share space with this vault." }
|
|
1188
|
+
if (this.sourceAppIsRunning(source)) {
|
|
1189
|
+
return { error: "This app has a running script. Stop it first, then deduplicate." }
|
|
1190
|
+
}
|
|
1191
|
+
|
|
1192
|
+
const registry = this.registry
|
|
1193
|
+
const batch = options.batch_id || `batch-${crypto.randomUUID()}`
|
|
1194
|
+
const requestedPaths = options.paths === undefined
|
|
1195
|
+
? new Set([...registry.duplicates.keys()])
|
|
1196
|
+
: new Set((options.paths || [])
|
|
1197
|
+
.filter((filePath) => typeof filePath === "string" && filePath))
|
|
1198
|
+
if (!requestedPaths.size) return { error: "No displayed files were selected." }
|
|
1199
|
+
const candidates = [...registry.duplicates].filter(([filePath, entry]) => {
|
|
1200
|
+
if (!requestedPaths.has(filePath)) return false
|
|
1201
|
+
const currentSource = this.sourceForPath(filePath, entry.source_id)
|
|
1202
|
+
return !!(currentSource && currentSource.id === scopeId && currentSource.shareable)
|
|
1203
|
+
})
|
|
1204
|
+
const progress = options.progress || null
|
|
1205
|
+
if (progress) {
|
|
1206
|
+
progress.files_completed = 0
|
|
1207
|
+
progress.files_total = candidates.length
|
|
1208
|
+
this.actionSequence += 1
|
|
1209
|
+
}
|
|
1210
|
+
const summary = { converted: 0, bytes_saved: 0, locked: 0, stale: 0, incompatible: 0, unavailable: 0, failed: 0 }
|
|
1211
|
+
const staleHashes = new Set()
|
|
1212
|
+
for (const [filePath, entry] of candidates) {
|
|
1213
|
+
try {
|
|
1214
|
+
const currentSource = this.sourceForPath(filePath, entry.source_id)
|
|
1215
|
+
if (!currentSource || currentSource.id !== scopeId || !currentSource.shareable) continue
|
|
1216
|
+
if (entry.unverified) {
|
|
1217
|
+
summary.unavailable += 1
|
|
1218
|
+
continue
|
|
1219
|
+
}
|
|
1220
|
+
if (!await this.canonicalPathIsWithinSource(filePath, currentSource)) {
|
|
1221
|
+
summary.stale += 1
|
|
1222
|
+
continue
|
|
1223
|
+
}
|
|
1224
|
+
const indexed = registry.scanIndex.get(filePath)
|
|
1225
|
+
const expected = indexed && indexed.hash === entry.hash
|
|
1226
|
+
? indexed
|
|
1227
|
+
: (entry.dev !== undefined && entry.ino !== undefined &&
|
|
1228
|
+
entry.mtime !== undefined && entry.ctime !== undefined ? entry : null)
|
|
1229
|
+
if (!expected) {
|
|
1230
|
+
summary.stale += 1
|
|
1231
|
+
continue
|
|
1232
|
+
}
|
|
1233
|
+
if (staleHashes.has(entry.hash)) {
|
|
1234
|
+
summary.stale += 1
|
|
1235
|
+
continue
|
|
1236
|
+
}
|
|
1237
|
+
try {
|
|
1238
|
+
const result = await this.convert(filePath, entry.hash, {
|
|
1239
|
+
app: source.kind === "app" ? source.app : (entry.app || null),
|
|
1240
|
+
source_id: scopeId,
|
|
1241
|
+
batch_id: batch,
|
|
1242
|
+
source: currentSource,
|
|
1243
|
+
expected
|
|
1244
|
+
})
|
|
1245
|
+
if (result.status === "converted" || result.status === "already") {
|
|
1246
|
+
summary.converted += 1
|
|
1247
|
+
summary.bytes_saved += result.bytes_saved || 0
|
|
1248
|
+
} else if (result.status === "locked") {
|
|
1249
|
+
summary.locked += 1
|
|
1250
|
+
} else if (result.status === "stale") {
|
|
1251
|
+
summary.stale += 1
|
|
1252
|
+
} else if (result.status === "stale-blob") {
|
|
1253
|
+
staleHashes.add(entry.hash)
|
|
1254
|
+
summary.stale += 1
|
|
1255
|
+
} else if (result.status === "metadata-mismatch") {
|
|
1256
|
+
entry.unavailable_reason = "metadata"
|
|
1257
|
+
summary.incompatible += 1
|
|
1258
|
+
} else if (result.status === "unavailable") {
|
|
1259
|
+
summary.unavailable += 1
|
|
1260
|
+
} else {
|
|
1261
|
+
summary.failed += 1
|
|
1262
|
+
}
|
|
1263
|
+
} catch (error) {
|
|
1264
|
+
summary.failed += 1
|
|
1265
|
+
}
|
|
1266
|
+
} finally {
|
|
1267
|
+
if (progress) {
|
|
1268
|
+
progress.files_completed += 1
|
|
1269
|
+
this.actionSequence += 1
|
|
1270
|
+
}
|
|
1271
|
+
if (options.onFileComplete) options.onFileComplete()
|
|
1272
|
+
}
|
|
1273
|
+
}
|
|
1274
|
+
registry.schedulePersist()
|
|
1275
|
+
return summary
|
|
1276
|
+
}
|
|
1277
|
+
async deduplicateAll(groups, progress) {
|
|
1278
|
+
const summary = {
|
|
1279
|
+
converted: 0,
|
|
1280
|
+
bytes_saved: 0,
|
|
1281
|
+
locked: 0,
|
|
1282
|
+
stale: 0,
|
|
1283
|
+
incompatible: 0,
|
|
1284
|
+
unavailable: 0,
|
|
1285
|
+
failed: 0,
|
|
1286
|
+
locations_skipped: 0
|
|
1287
|
+
}
|
|
1288
|
+
const fields = ["converted", "bytes_saved", "locked", "stale", "incompatible", "unavailable", "failed"]
|
|
1289
|
+
progress.files_completed = 0
|
|
1290
|
+
progress.files_total = groups.reduce((sum, group) => sum + group.paths.length, 0)
|
|
1291
|
+
this.actionSequence += 1
|
|
1292
|
+
for (const group of groups) {
|
|
1293
|
+
let completed = 0
|
|
1294
|
+
const result = await this.deduplicateScope(group.scope_id, {
|
|
1295
|
+
batch_id: `batch-${crypto.randomUUID()}`,
|
|
1296
|
+
paths: group.paths,
|
|
1297
|
+
onFileComplete: () => {
|
|
1298
|
+
completed += 1
|
|
1299
|
+
progress.files_completed += 1
|
|
1300
|
+
this.actionSequence += 1
|
|
1301
|
+
}
|
|
1302
|
+
})
|
|
1303
|
+
if (result.error) {
|
|
1304
|
+
summary.unavailable += group.paths.length
|
|
1305
|
+
summary.locations_skipped += 1
|
|
1306
|
+
} else {
|
|
1307
|
+
for (const field of fields) summary[field] += Number(result[field]) || 0
|
|
1308
|
+
}
|
|
1309
|
+
const unprocessed = Math.max(0, group.paths.length - completed)
|
|
1310
|
+
if (unprocessed) {
|
|
1311
|
+
progress.files_completed += unprocessed
|
|
1312
|
+
this.actionSequence += 1
|
|
1313
|
+
}
|
|
1314
|
+
}
|
|
1315
|
+
return summary
|
|
1316
|
+
}
|
|
1317
|
+
// reclaim: delete an orphan's store name. Refuses when any app name remains.
|
|
1318
|
+
async reclaim(hash, options = {}) {
|
|
1319
|
+
if (!this.enabled) return { status: "disabled" }
|
|
1320
|
+
const storePath = this.storePathFor(hash)
|
|
1321
|
+
if (!this.registry.blobs.has(hash)) return { status: "not-found" }
|
|
1322
|
+
const st = await this.storeStatIfPresent(storePath)
|
|
1323
|
+
if (!st) {
|
|
1324
|
+
if (!options.deferRegistry) this.registry.removeBlob(hash)
|
|
1325
|
+
return options.deferRegistry ? { status: "gone", remove_hash: true } : { status: "gone" }
|
|
1326
|
+
}
|
|
1327
|
+
if (!st.isFile()) {
|
|
1328
|
+
if (!options.deferRegistry) this.registry.removeBlob(hash)
|
|
1329
|
+
return options.deferRegistry
|
|
1330
|
+
? { status: "invalid", remove_hash: true }
|
|
1331
|
+
: { status: "invalid" }
|
|
1332
|
+
}
|
|
1333
|
+
if (st.nlink > 1) return { status: "in-use" }
|
|
1334
|
+
await fs.promises.unlink(storePath)
|
|
1335
|
+
if (!options.deferRegistry) this.registry.removeBlob(hash)
|
|
1336
|
+
await this.recordEvent({ kind: "reclaim", hash, bytes_saved: st.size })
|
|
1337
|
+
return options.deferRegistry
|
|
1338
|
+
? { status: "reclaimed", bytes_freed: st.size, remove_hash: true }
|
|
1339
|
+
: { status: "reclaimed", bytes_freed: st.size }
|
|
1340
|
+
}
|
|
1341
|
+
// Startup registry verification (trigger 5): bounded by registry size,
|
|
1342
|
+
// never a discovery walk. Prunes dead links, flags orphans, removes stray
|
|
1343
|
+
// conversion tmp files, re-adopts blobs whose store name was deleted.
|
|
1344
|
+
async verify(options = {}) {
|
|
1345
|
+
if (!this.enabled || !this.registry) return
|
|
1346
|
+
const preservePath = options.preservePath || (() => false)
|
|
1347
|
+
const includePath = options.includePath || (() => true)
|
|
1348
|
+
const preserveReadError = (error, filePath, entry) => {
|
|
1349
|
+
if (isMissingError(error)) return false
|
|
1350
|
+
if (options.onAccessError) {
|
|
1351
|
+
if (!isAccessError(error) || !options.onAccessError(error, filePath)) return false
|
|
1352
|
+
}
|
|
1353
|
+
if (entry) entry.unverified = true
|
|
1354
|
+
return true
|
|
1355
|
+
}
|
|
1356
|
+
if (options.reconcile !== false) this.reconcileConfiguredSources()
|
|
1357
|
+
for (const [linkPath, entry] of [...this.registry.links]) {
|
|
1358
|
+
if (!includePath(linkPath)) continue
|
|
1359
|
+
if (preservePath(linkPath)) continue
|
|
1360
|
+
try {
|
|
1361
|
+
const source = this.sourceForPath(linkPath, entry.source_id)
|
|
1362
|
+
if (source && source.available === false) {
|
|
1363
|
+
entry.unverified = true
|
|
1364
|
+
continue
|
|
1365
|
+
}
|
|
1366
|
+
if (!source) {
|
|
1367
|
+
this.registry.untrack(linkPath)
|
|
1368
|
+
continue
|
|
1369
|
+
}
|
|
1370
|
+
const st = await lstatIfPresent(linkPath)
|
|
1371
|
+
if (!st) {
|
|
1372
|
+
this.registry.untrack(linkPath)
|
|
1373
|
+
continue
|
|
1374
|
+
}
|
|
1375
|
+
const managedPath = await this.canonicalPathIsWithinSource(
|
|
1376
|
+
linkPath, source, { strictErrors: true })
|
|
1377
|
+
const expected = this.registry.scanIndex.get(linkPath)
|
|
1378
|
+
entry.unverified = !managedPath || !st.isFile() ||
|
|
1379
|
+
st.ino !== entry.ino || st.dev !== entry.dev ||
|
|
1380
|
+
!expected || expected.hash !== entry.hash || !sameSnapshot(expected, st)
|
|
1381
|
+
} catch (error) {
|
|
1382
|
+
if (!preserveReadError(error, linkPath, entry)) throw error
|
|
1383
|
+
}
|
|
1384
|
+
}
|
|
1385
|
+
for (const [dupPath, entry] of [...this.registry.duplicates]) {
|
|
1386
|
+
if (!includePath(dupPath)) continue
|
|
1387
|
+
if (preservePath(dupPath)) continue
|
|
1388
|
+
let valid = false
|
|
1389
|
+
try {
|
|
1390
|
+
const source = this.sourceForPath(dupPath, entry.source_id)
|
|
1391
|
+
if (source && source.available === false) {
|
|
1392
|
+
entry.unverified = true
|
|
1393
|
+
continue
|
|
1394
|
+
}
|
|
1395
|
+
const st = await lstatIfPresent(dupPath)
|
|
1396
|
+
if (!st) {
|
|
1397
|
+
this.registry.untrack(dupPath)
|
|
1398
|
+
continue
|
|
1399
|
+
}
|
|
1400
|
+
const expected = this.registry.scanIndex.get(dupPath)
|
|
1401
|
+
valid = !!(source && st.isFile() &&
|
|
1402
|
+
await this.canonicalPathIsWithinSource(dupPath, source, { strictErrors: true }) &&
|
|
1403
|
+
expected && expected.hash === entry.hash && sameSnapshot(expected, st))
|
|
1404
|
+
} catch (error) {
|
|
1405
|
+
if (preserveReadError(error, dupPath, entry)) continue
|
|
1406
|
+
throw error
|
|
1407
|
+
}
|
|
1408
|
+
if (!valid) {
|
|
1409
|
+
entry.unverified = true
|
|
1410
|
+
continue
|
|
1411
|
+
}
|
|
1412
|
+
entry.unverified = false
|
|
1413
|
+
const storeStat = await this.storeStatIfPresent(this.storePathFor(entry.hash))
|
|
1414
|
+
if (storeStat && storeStat.isFile()) {
|
|
1415
|
+
await unlinkIfSame(dupPath + TMP_SUFFIX, storeStat)
|
|
1416
|
+
}
|
|
1417
|
+
}
|
|
1418
|
+
if (options.verifyBlobs === false) {
|
|
1419
|
+
this.registry.schedulePersist()
|
|
1420
|
+
return
|
|
1421
|
+
}
|
|
1422
|
+
const linksByHash = new Map()
|
|
1423
|
+
for (const [linkPath, entry] of this.registry.links) {
|
|
1424
|
+
if (!linksByHash.has(entry.hash)) linksByHash.set(entry.hash, [])
|
|
1425
|
+
linksByHash.get(entry.hash).push([linkPath, entry])
|
|
1426
|
+
}
|
|
1427
|
+
const invalidBlobs = new Set()
|
|
1428
|
+
for (const [hash, blob] of [...this.registry.blobs]) {
|
|
1429
|
+
const storePath = this.storePathFor(hash)
|
|
1430
|
+
const st = await this.storeStatIfPresent(storePath)
|
|
1431
|
+
if (!st) {
|
|
1432
|
+
const names = linksByHash.get(hash) || []
|
|
1433
|
+
// Deleting a name changes ctime, so trusted re-adoption compares the
|
|
1434
|
+
// preserved identity, size, and mtime. Changed or legacy-unsnapshotted
|
|
1435
|
+
// content waits for the next explicit scan instead of being assigned a
|
|
1436
|
+
// stale hash filename during startup.
|
|
1437
|
+
let readopted = false
|
|
1438
|
+
let inaccessibleName = false
|
|
1439
|
+
for (const [linkPath, entry] of names) {
|
|
1440
|
+
if (preservePath(linkPath)) {
|
|
1441
|
+
inaccessibleName = true
|
|
1442
|
+
continue
|
|
1443
|
+
}
|
|
1444
|
+
let source
|
|
1445
|
+
let linkStat
|
|
1446
|
+
try {
|
|
1447
|
+
source = this.sourceForPath(linkPath, entry.source_id)
|
|
1448
|
+
if (!source || !await this.canonicalPathIsWithinSource(linkPath, source, { strictErrors: true })) continue
|
|
1449
|
+
linkStat = await lstatIfPresent(linkPath)
|
|
1450
|
+
} catch (error) {
|
|
1451
|
+
if (!preserveReadError(error, linkPath, entry)) throw error
|
|
1452
|
+
inaccessibleName = true
|
|
1453
|
+
continue
|
|
1454
|
+
}
|
|
1455
|
+
const expected = this.registry.scanIndex.get(linkPath)
|
|
1456
|
+
if (!linkStat || !expected || expected.hash !== hash ||
|
|
1457
|
+
entry.dev !== linkStat.dev || entry.ino !== linkStat.ino ||
|
|
1458
|
+
!sameContentState(expected, linkStat)) continue
|
|
1459
|
+
const before = fileSnapshot(linkStat)
|
|
1460
|
+
let created = false
|
|
1461
|
+
try {
|
|
1462
|
+
await this.storeStatIfPresent(storePath, { createParent: true })
|
|
1463
|
+
await fs.promises.link(linkPath, storePath)
|
|
1464
|
+
created = true
|
|
1465
|
+
} catch (error) {
|
|
1466
|
+
if (!error || error.code !== "EEXIST") throw error
|
|
1467
|
+
const currentStore = await this.storeStatIfPresent(storePath)
|
|
1468
|
+
const currentLink = await lstatIfPresent(linkPath)
|
|
1469
|
+
if (!currentStore || !sameContentState(before, currentLink) ||
|
|
1470
|
+
currentStore.dev !== linkStat.dev || currentStore.ino !== linkStat.ino) continue
|
|
1471
|
+
}
|
|
1472
|
+
const [currentLink, currentStore] = await Promise.all([
|
|
1473
|
+
lstatIfPresent(linkPath),
|
|
1474
|
+
this.storeStatIfPresent(storePath)
|
|
1475
|
+
])
|
|
1476
|
+
if (!sameContentState(before, currentLink) || !currentStore ||
|
|
1477
|
+
currentStore.dev !== linkStat.dev || currentStore.ino !== linkStat.ino) {
|
|
1478
|
+
const ownsStoreName = created && currentStore && (
|
|
1479
|
+
(currentStore.dev === linkStat.dev && currentStore.ino === linkStat.ino) ||
|
|
1480
|
+
(currentLink && currentStore.dev === currentLink.dev && currentStore.ino === currentLink.ino)
|
|
1481
|
+
)
|
|
1482
|
+
if (ownsStoreName) await fs.promises.unlink(storePath)
|
|
1483
|
+
continue
|
|
1484
|
+
}
|
|
1485
|
+
await this.refreshLinkSnapshots(hash, linkStat.dev, linkStat.ino)
|
|
1486
|
+
readopted = true
|
|
1487
|
+
break
|
|
1488
|
+
}
|
|
1489
|
+
if (readopted || inaccessibleName) {
|
|
1490
|
+
blob.orphan = false
|
|
1491
|
+
blob.verified_at = inaccessibleName && !readopted ? null : Date.now()
|
|
1492
|
+
} else {
|
|
1493
|
+
invalidBlobs.add(hash)
|
|
1494
|
+
}
|
|
1495
|
+
continue
|
|
1496
|
+
}
|
|
1497
|
+
if (!st.isFile()) {
|
|
1498
|
+
invalidBlobs.add(hash)
|
|
1499
|
+
continue
|
|
1500
|
+
}
|
|
1501
|
+
blob.orphan = st.nlink === 1
|
|
1502
|
+
blob.verified_at = Date.now()
|
|
1503
|
+
}
|
|
1504
|
+
this.registry.removeBlobs(invalidBlobs)
|
|
1505
|
+
this.registry.schedulePersist()
|
|
1506
|
+
}
|
|
1507
|
+
// Rebuild derived state from disk: store filenames are hashes, stat gives
|
|
1508
|
+
// nlink, and (dev, ino) matching re-associates app paths. The replacement
|
|
1509
|
+
// maps are assembled off to the side and swapped in only after the walk so
|
|
1510
|
+
// a long repair never persists a half-reset registry.
|
|
1511
|
+
async rebuild(roots, options = {}) {
|
|
1512
|
+
if (!this.enabled) return
|
|
1513
|
+
const registry = this.registry
|
|
1514
|
+
const preserveState = options.preserveState !== false
|
|
1515
|
+
const preserved = preserveState ? {
|
|
1516
|
+
blobs: new Map(registry.blobs),
|
|
1517
|
+
links: new Map(registry.links),
|
|
1518
|
+
excluded: new Map(registry.excluded),
|
|
1519
|
+
lastScan: registry.lastScan ? Object.assign({}, registry.lastScan) : null,
|
|
1520
|
+
lastScanAttempt: registry.lastScanAttempt
|
|
1521
|
+
? Object.assign({}, registry.lastScanAttempt)
|
|
1522
|
+
: null,
|
|
1523
|
+
sourceScans: new Map([...registry.sourceScans].map(([id, scan]) => [id, Object.assign({}, scan)])),
|
|
1524
|
+
sourceScanAttempts: new Map([...registry.sourceScanAttempts]
|
|
1525
|
+
.map(([id, scan]) => [id, Object.assign({}, scan)])),
|
|
1526
|
+
settings: Object.assign({}, registry.settings),
|
|
1527
|
+
duplicates: new Map(registry.duplicates),
|
|
1528
|
+
scanIndex: new Map(registry.scanIndex)
|
|
1529
|
+
} : null
|
|
1530
|
+
const rebuiltBlobs = new Map()
|
|
1531
|
+
const rebuiltLinks = new Map()
|
|
1532
|
+
const storeInoToHash = new Map()
|
|
1533
|
+
let shards = []
|
|
1534
|
+
try {
|
|
1535
|
+
shards = await fs.promises.readdir(this.blobRoot, { withFileTypes: true })
|
|
1536
|
+
} catch (error) {
|
|
1537
|
+
if (!isMissingError(error)) throw error
|
|
1538
|
+
}
|
|
1539
|
+
for (const shardEntry of shards) {
|
|
1540
|
+
if (!shardEntry.isDirectory() || shardEntry.isSymbolicLink() ||
|
|
1541
|
+
!/^[0-9a-f]{2}$/.test(shardEntry.name)) continue
|
|
1542
|
+
const shard = shardEntry.name
|
|
1543
|
+
const shardPath = path.resolve(this.blobRoot, shard)
|
|
1544
|
+
if (!await this.directoryIfSafe(shardPath)) continue
|
|
1545
|
+
let names = []
|
|
1546
|
+
try {
|
|
1547
|
+
names = await fs.promises.readdir(shardPath)
|
|
1548
|
+
} catch (error) {
|
|
1549
|
+
if (isMissingError(error)) continue
|
|
1550
|
+
throw error
|
|
1551
|
+
}
|
|
1552
|
+
for (const name of names) {
|
|
1553
|
+
if (!SHA256_RE.test(name) || !name.startsWith(shard)) continue
|
|
1554
|
+
try {
|
|
1555
|
+
const st = await fs.promises.lstat(path.resolve(this.blobRoot, shard, name))
|
|
1556
|
+
if (!st.isFile()) continue
|
|
1557
|
+
const previous = registry.blobs.get(name)
|
|
1558
|
+
rebuiltBlobs.set(name, {
|
|
1559
|
+
size: st.size,
|
|
1560
|
+
first_seen: previous && previous.first_seen ? previous.first_seen : Date.now(),
|
|
1561
|
+
source_urls: previous && Array.isArray(previous.source_urls) ? [...previous.source_urls] : [],
|
|
1562
|
+
verified_at: Date.now(),
|
|
1563
|
+
orphan: st.nlink === 1
|
|
1564
|
+
})
|
|
1565
|
+
storeInoToHash.set(`${st.dev}:${st.ino}`, name)
|
|
1566
|
+
} catch (error) {
|
|
1567
|
+
if (!isMissingError(error)) throw error
|
|
1568
|
+
}
|
|
1569
|
+
}
|
|
1570
|
+
}
|
|
1571
|
+
await this.refreshSources()
|
|
1572
|
+
const walkRoots = roots
|
|
1573
|
+
? roots.map((root) => typeof root === "string" ? { root, source_id: null } : root)
|
|
1574
|
+
: [
|
|
1575
|
+
{ root: path.resolve(this.kernel.homedir), source_id: "pinokio" },
|
|
1576
|
+
...this._sources
|
|
1577
|
+
.filter((source) => source.kind === "external" && source.root)
|
|
1578
|
+
.map((source) => ({ root: source.root, source_id: source.id }))
|
|
1579
|
+
]
|
|
1580
|
+
for (const entry of walkRoots) {
|
|
1581
|
+
const rootStat = await fs.promises.lstat(entry.root)
|
|
1582
|
+
if (!rootStat.isDirectory() || rootStat.isSymbolicLink()) {
|
|
1583
|
+
throw new Error(`Repair source is not a real directory: ${entry.root}`)
|
|
1584
|
+
}
|
|
1585
|
+
}
|
|
1586
|
+
for (const entry of walkRoots) {
|
|
1587
|
+
await this.walkForInoMatch(
|
|
1588
|
+
entry.root,
|
|
1589
|
+
storeInoToHash,
|
|
1590
|
+
entry.source_id === "pinokio" ? null : entry.source_id,
|
|
1591
|
+
rebuiltLinks
|
|
1592
|
+
)
|
|
1593
|
+
}
|
|
1594
|
+
const rebuiltDuplicates = new Map()
|
|
1595
|
+
const rebuiltScanIndex = new Map()
|
|
1596
|
+
const trustedHashes = new Set()
|
|
1597
|
+
if (preserved) {
|
|
1598
|
+
// A store filename is only a trustworthy content hash when at least one
|
|
1599
|
+
// surviving name still matches the snapshot captured when that content
|
|
1600
|
+
// was hashed. Repair itself never hashes, so unverified names are left
|
|
1601
|
+
// without a scan-index entry and the next manual scan hashes them once.
|
|
1602
|
+
for (const [linkPath, link] of rebuiltLinks) {
|
|
1603
|
+
const expected = preserved.scanIndex.get(linkPath)
|
|
1604
|
+
if (!expected || expected.hash !== link.hash) continue
|
|
1605
|
+
try {
|
|
1606
|
+
const st = await fs.promises.lstat(linkPath)
|
|
1607
|
+
if (sameSnapshot(expected, st)) trustedHashes.add(link.hash)
|
|
1608
|
+
} catch (error) {
|
|
1609
|
+
if (!isMissingError(error)) throw error
|
|
1610
|
+
}
|
|
1611
|
+
}
|
|
1612
|
+
for (const [linkPath, link] of rebuiltLinks) {
|
|
1613
|
+
if (!trustedHashes.has(link.hash)) continue
|
|
1614
|
+
try {
|
|
1615
|
+
const st = await fs.promises.lstat(linkPath)
|
|
1616
|
+
rebuiltScanIndex.set(linkPath, {
|
|
1617
|
+
hash: link.hash, size: st.size, dev: st.dev, ino: st.ino,
|
|
1618
|
+
mtime: st.mtimeMs, ctime: st.ctimeMs,
|
|
1619
|
+
source_id: link.source_id || null
|
|
1620
|
+
})
|
|
1621
|
+
} catch (error) {
|
|
1622
|
+
if (!isMissingError(error)) throw error
|
|
1623
|
+
}
|
|
1624
|
+
}
|
|
1625
|
+
for (const [duplicatePath, duplicate] of preserved.duplicates) {
|
|
1626
|
+
if (preserved.excluded.has(duplicatePath) || rebuiltLinks.has(duplicatePath) ||
|
|
1627
|
+
!rebuiltBlobs.has(duplicate.hash) || !trustedHashes.has(duplicate.hash) ||
|
|
1628
|
+
!this.sourceForPath(duplicatePath, duplicate.source_id)) continue
|
|
1629
|
+
try {
|
|
1630
|
+
const source = this.sourceForPath(duplicatePath, duplicate.source_id)
|
|
1631
|
+
if (!await this.canonicalPathIsWithinSource(duplicatePath, source, { strictErrors: true })) continue
|
|
1632
|
+
const st = await fs.promises.lstat(duplicatePath)
|
|
1633
|
+
const indexed = preserved.scanIndex.get(duplicatePath)
|
|
1634
|
+
const expected = indexed && indexed.hash === duplicate.hash ? indexed : duplicate
|
|
1635
|
+
if (st.isFile() && sameSnapshot(expected, st)) {
|
|
1636
|
+
rebuiltDuplicates.set(duplicatePath, Object.assign({}, duplicate, {
|
|
1637
|
+
size: st.size, dev: st.dev, ino: st.ino, mtime: st.mtimeMs, ctime: st.ctimeMs
|
|
1638
|
+
}))
|
|
1639
|
+
rebuiltScanIndex.set(duplicatePath, {
|
|
1640
|
+
hash: duplicate.hash, size: st.size, dev: st.dev, ino: st.ino,
|
|
1641
|
+
mtime: st.mtimeMs, ctime: st.ctimeMs,
|
|
1642
|
+
source_id: duplicate.source_id || expected.source_id || null
|
|
1643
|
+
})
|
|
1644
|
+
}
|
|
1645
|
+
} catch (error) {
|
|
1646
|
+
if (!isMissingError(error)) throw error
|
|
1647
|
+
}
|
|
1648
|
+
}
|
|
1649
|
+
for (const excludedPath of preserved.excluded.keys()) {
|
|
1650
|
+
rebuiltLinks.delete(excludedPath)
|
|
1651
|
+
rebuiltScanIndex.delete(excludedPath)
|
|
1652
|
+
}
|
|
1653
|
+
}
|
|
1654
|
+
const rebuiltByIno = new Map()
|
|
1655
|
+
const rebuiltPathsByIno = new Map()
|
|
1656
|
+
for (const [linkPath, entry] of rebuiltLinks) {
|
|
1657
|
+
const key = registry.inoKey(entry.dev, entry.ino)
|
|
1658
|
+
rebuiltByIno.set(key, entry.hash)
|
|
1659
|
+
if (!rebuiltPathsByIno.has(key)) rebuiltPathsByIno.set(key, new Set())
|
|
1660
|
+
rebuiltPathsByIno.get(key).add(linkPath)
|
|
1661
|
+
}
|
|
1662
|
+
registry.replaceState({
|
|
1663
|
+
blobs: rebuiltBlobs,
|
|
1664
|
+
links: rebuiltLinks,
|
|
1665
|
+
scanIndex: rebuiltScanIndex,
|
|
1666
|
+
duplicates: rebuiltDuplicates,
|
|
1667
|
+
excluded: preserved ? preserved.excluded : new Map(),
|
|
1668
|
+
lastScan: preserved ? preserved.lastScan : null,
|
|
1669
|
+
lastScanAttempt: preserved ? preserved.lastScanAttempt : null,
|
|
1670
|
+
sourceScans: preserved ? preserved.sourceScans : new Map(),
|
|
1671
|
+
sourceScanAttempts: preserved ? preserved.sourceScanAttempts : new Map(),
|
|
1672
|
+
settings: preserved ? preserved.settings : {},
|
|
1673
|
+
byIno: rebuiltByIno,
|
|
1674
|
+
pathsByIno: rebuiltPathsByIno
|
|
1675
|
+
})
|
|
1676
|
+
if (options.flush !== false) await registry.flush()
|
|
1677
|
+
}
|
|
1678
|
+
async walkForInoMatch(root, storeInoToHash, preferredSourceId = null, outputLinks = null) {
|
|
1679
|
+
const vaultRoot = path.resolve(this.root)
|
|
1680
|
+
for await (const batch of walkBatches(root, {
|
|
1681
|
+
concurrency: this.dirConcurrency,
|
|
1682
|
+
skipDirectory: (full) => full === vaultRoot,
|
|
1683
|
+
strictErrors: true,
|
|
1684
|
+
onRootMissing: (error) => { throw error }
|
|
1685
|
+
})) {
|
|
1686
|
+
const files = batch.flatMap((group) => group.files.map((file) => file.path))
|
|
1687
|
+
const stats = await statMany(files, this.statConcurrency, null, {
|
|
1688
|
+
strictErrors: true,
|
|
1689
|
+
followSymlinks: false
|
|
1690
|
+
})
|
|
1691
|
+
for (let index = 0; index < files.length; index++) {
|
|
1692
|
+
const st = stats[index]
|
|
1693
|
+
if (!st || !st.isFile()) continue
|
|
1694
|
+
const hash = storeInoToHash.get(`${st.dev}:${st.ino}`)
|
|
1695
|
+
if (hash) {
|
|
1696
|
+
const source = this.sourceForPath(files[index], preferredSourceId)
|
|
1697
|
+
const previous = this.registry.links.get(files[index])
|
|
1698
|
+
const batchId = previous && previous.hash === hash &&
|
|
1699
|
+
previous.dev === st.dev && previous.ino === st.ino
|
|
1700
|
+
? previous.batch_id || null
|
|
1701
|
+
: null
|
|
1702
|
+
const link = {
|
|
1703
|
+
hash, app: source && source.kind === "app" ? source.app : null,
|
|
1704
|
+
source_id: source ? source.id : null, dev: st.dev, ino: st.ino,
|
|
1705
|
+
created: Date.now(),
|
|
1706
|
+
batch_id: batchId
|
|
1707
|
+
}
|
|
1708
|
+
if (outputLinks) outputLinks.set(files[index], link)
|
|
1709
|
+
else this.registry.addLink(files[index], link)
|
|
1710
|
+
}
|
|
1711
|
+
}
|
|
1712
|
+
}
|
|
1713
|
+
}
|
|
1714
|
+
cloudSyncProvider() {
|
|
1715
|
+
const home = path.resolve(this.kernel.homedir || "").replace(/\\/g, "/").toLowerCase()
|
|
1716
|
+
if (home.includes("/onedrive") || home.includes("/one drive")) return "OneDrive"
|
|
1717
|
+
if (home.includes("/dropbox")) return "Dropbox"
|
|
1718
|
+
if (home.includes("/library/mobile documents/") || home.includes("/icloud drive/")) return "iCloud Drive"
|
|
1719
|
+
return null
|
|
1720
|
+
}
|
|
1721
|
+
// Global state for the vault page and the health endpoint. Answered from
|
|
1722
|
+
// memory + one stat per blob and occupied shard; never a walk.
|
|
1723
|
+
async status(scopeId = null) {
|
|
1724
|
+
if (!this.enabled || !this.registry) return { enabled: false }
|
|
1725
|
+
const responseActionSequence = this.actionSequence
|
|
1726
|
+
const registry = this.registry
|
|
1727
|
+
const scope = scopeId ? this.scanSource(scopeId) : null
|
|
1728
|
+
if (scopeId && !scope) throw new Error("That location is no longer available.")
|
|
1729
|
+
const scopeHashes = scopeId ? new Set() : null
|
|
1730
|
+
if (scopeHashes) {
|
|
1731
|
+
for (const entry of registry.links.values()) {
|
|
1732
|
+
if (entry.source_id === scopeId) scopeHashes.add(entry.hash)
|
|
1733
|
+
}
|
|
1734
|
+
for (const entry of registry.duplicates.values()) {
|
|
1735
|
+
if (entry.source_id === scopeId) scopeHashes.add(entry.hash)
|
|
1736
|
+
}
|
|
1737
|
+
}
|
|
1738
|
+
const blobs = []
|
|
1739
|
+
const blobByHash = new Map()
|
|
1740
|
+
const storeStats = new Map()
|
|
1741
|
+
const namesByHash = new Map()
|
|
1742
|
+
const undoBatchMap = new Map()
|
|
1743
|
+
for (const [linkPath, entry] of registry.links) {
|
|
1744
|
+
if (scopeHashes && !scopeHashes.has(entry.hash)) continue
|
|
1745
|
+
if (!namesByHash.has(entry.hash)) namesByHash.set(entry.hash, [])
|
|
1746
|
+
const location = this.locationForPath(linkPath, entry.source_id)
|
|
1747
|
+
namesByHash.get(entry.hash).push(Object.assign({
|
|
1748
|
+
path: linkPath, app: entry.app || null, mode: "link",
|
|
1749
|
+
unverified: !!entry.unverified
|
|
1750
|
+
}, location))
|
|
1751
|
+
if (entry.batch_id && (!scopeId || entry.source_id === scopeId)) {
|
|
1752
|
+
if (!undoBatchMap.has(entry.batch_id)) {
|
|
1753
|
+
undoBatchMap.set(entry.batch_id, { batch_id: entry.batch_id, files: 0, bytes: 0, ts: null })
|
|
1754
|
+
}
|
|
1755
|
+
const batch = undoBatchMap.get(entry.batch_id)
|
|
1756
|
+
const blob = registry.blobs.get(entry.hash)
|
|
1757
|
+
batch.files += 1
|
|
1758
|
+
batch.bytes += blob && Number.isFinite(blob.size) ? blob.size : 0
|
|
1759
|
+
}
|
|
1760
|
+
}
|
|
1761
|
+
const blobEntries = [...registry.blobs].filter(([hash]) => !scopeHashes || scopeHashes.has(hash))
|
|
1762
|
+
if (!await this.directoryIfSafe(this.root) || !await this.directoryIfSafe(this.blobRoot)) {
|
|
1763
|
+
throw unsafeStoragePath(this.blobRoot)
|
|
1764
|
+
}
|
|
1765
|
+
const storePaths = blobEntries.map(([hash]) => this.storePathFor(hash))
|
|
1766
|
+
await Promise.all([...new Set(storePaths.map((storePath) => path.dirname(storePath)))]
|
|
1767
|
+
.map((shard) => this.directoryIfSafe(shard)))
|
|
1768
|
+
const blobStats = await statMany(
|
|
1769
|
+
storePaths,
|
|
1770
|
+
this.statConcurrency,
|
|
1771
|
+
null,
|
|
1772
|
+
{ strictErrors: true, followSymlinks: false }
|
|
1773
|
+
)
|
|
1774
|
+
let savedBySharing = 0
|
|
1775
|
+
for (let index = 0; index < blobEntries.length; index++) {
|
|
1776
|
+
const [hash, blob] = blobEntries[index]
|
|
1777
|
+
const names = namesByHash.get(hash) || []
|
|
1778
|
+
const apps = new Set(names.map((name) => name.app).filter(Boolean))
|
|
1779
|
+
const storeStat = blobStats[index] && blobStats[index].isFile() ? blobStats[index] : null
|
|
1780
|
+
const nlink = storeStat ? storeStat.nlink : null
|
|
1781
|
+
storeStats.set(hash, storeStat)
|
|
1782
|
+
const size = blob.size || 0
|
|
1783
|
+
const liveNames = storeStat ? Math.max(0, storeStat.nlink - 1) : 0
|
|
1784
|
+
if (liveNames >= 2) {
|
|
1785
|
+
for (const name of names) {
|
|
1786
|
+
if (name.unverified || (scopeId && name.source_id !== scopeId)) continue
|
|
1787
|
+
savedBySharing += size - (size / liveNames)
|
|
1788
|
+
}
|
|
1789
|
+
}
|
|
1790
|
+
const orphan = !!(storeStat && storeStat.nlink === 1)
|
|
1791
|
+
const publicBlob = {
|
|
1792
|
+
hash, size: blob.size || 0, orphan, nlink,
|
|
1793
|
+
sharing_locations: liveNames,
|
|
1794
|
+
names, apps: [...apps], source_urls: blob.source_urls || []
|
|
1795
|
+
}
|
|
1796
|
+
blobs.push(publicBlob)
|
|
1797
|
+
blobByHash.set(hash, publicBlob)
|
|
1798
|
+
}
|
|
1799
|
+
const duplicates = []
|
|
1800
|
+
for (const [p, entry] of registry.duplicates) {
|
|
1801
|
+
if (scopeId && entry.source_id !== scopeId) continue
|
|
1802
|
+
const location = this.locationForPath(p, entry.source_id)
|
|
1803
|
+
const source = this.sourceForPath(p, location.source_id)
|
|
1804
|
+
const index = registry.scanIndex.get(p)
|
|
1805
|
+
const storeStat = storeStats.get(entry.hash)
|
|
1806
|
+
const shareable = !!(source && source.shareable && storeStat && !entry.unverified &&
|
|
1807
|
+
!entry.unavailable_reason &&
|
|
1808
|
+
(!index || index.dev === undefined || index.dev === storeStat.dev))
|
|
1809
|
+
let unavailableReason = entry.unavailable_reason || null
|
|
1810
|
+
if (!shareable && !unavailableReason) {
|
|
1811
|
+
unavailableReason = source && storeStat && index && index.dev !== undefined && index.dev !== storeStat.dev
|
|
1812
|
+
? "different_disk"
|
|
1813
|
+
: "unsupported_disk"
|
|
1814
|
+
}
|
|
1815
|
+
const blob = blobByHash.get(entry.hash)
|
|
1816
|
+
const match = blob ? blob.names.find((name) => name.path !== p) || null : null
|
|
1817
|
+
duplicates.push(Object.assign({
|
|
1818
|
+
path: p, hash: entry.hash, size: entry.size || 0, app: entry.app || null,
|
|
1819
|
+
shareable, unverified: !!entry.unverified,
|
|
1820
|
+
unavailable_reason: entry.unverified ? "unavailable" : unavailableReason, match
|
|
1821
|
+
}, location))
|
|
1822
|
+
}
|
|
1823
|
+
const events = (await registry.readEvents()).reverse()
|
|
1824
|
+
.filter((event) => !scopeId || event.source_id === scopeId)
|
|
1825
|
+
.map((event) => {
|
|
1826
|
+
const currentLocation = event.path ? this.locationForPath(event.path, event.source_id) : null
|
|
1827
|
+
const fallbackLocation = currentLocation && currentLocation.source_id
|
|
1828
|
+
? currentLocation
|
|
1829
|
+
: (event.path ? { relative_path: event.path } : {})
|
|
1830
|
+
const publicEvent = Object.assign({}, fallbackLocation, event)
|
|
1831
|
+
if (event.kind === "convert" && event.batch_id && event.path) {
|
|
1832
|
+
const current = registry.links.get(event.path)
|
|
1833
|
+
publicEvent.undoable = !!(current && (
|
|
1834
|
+
current.batch_id === event.batch_id ||
|
|
1835
|
+
(!current.batch_id && current.hash === event.hash)
|
|
1836
|
+
))
|
|
1837
|
+
const batch = undoBatchMap.get(event.batch_id)
|
|
1838
|
+
if (batch && Number.isFinite(event.ts)) {
|
|
1839
|
+
batch.ts = Math.max(batch.ts || 0, event.ts)
|
|
1840
|
+
}
|
|
1841
|
+
}
|
|
1842
|
+
return publicEvent
|
|
1843
|
+
})
|
|
1844
|
+
const undoBatches = [...undoBatchMap.values()].sort((a, b) =>
|
|
1845
|
+
(b.ts || 0) - (a.ts || 0) || a.batch_id.localeCompare(b.batch_id))
|
|
1846
|
+
const scan = this.scanStatus()
|
|
1847
|
+
const excluded = []
|
|
1848
|
+
for (const [p, meta] of registry.excluded) {
|
|
1849
|
+
if (scopeId && meta.source_id !== scopeId) continue
|
|
1850
|
+
excluded.push(Object.assign({
|
|
1851
|
+
path: p,
|
|
1852
|
+
ts: meta.ts || null,
|
|
1853
|
+
size: Number(meta.size) || 0,
|
|
1854
|
+
unverified: !!meta.unverified
|
|
1855
|
+
}, this.locationForPath(p, meta.source_id)))
|
|
1856
|
+
}
|
|
1857
|
+
const publicSources = this._sources.filter((source) => !scopeId || source.id === scopeId).map((source) => ({
|
|
1858
|
+
id: source.id, kind: source.kind, label: source.label, root: source.root,
|
|
1859
|
+
display_path: source.root,
|
|
1860
|
+
target_path: source.kind === "external" ? source.root : null,
|
|
1861
|
+
parent_id: scopeId ? null : source.parent_id, app: source.app || null,
|
|
1862
|
+
available: source.available !== false, shareable: source.kind === "virtual" ? null : !!source.shareable
|
|
1863
|
+
}))
|
|
1864
|
+
const lastComplete = registry.scanFor(scopeId)
|
|
1865
|
+
const lastAttempt = registry.scanAttemptFor(scopeId)
|
|
1866
|
+
const inScope = (entry) => !scopeId || entry.source_id === scopeId
|
|
1867
|
+
const hasUnavailableSource = !scopeId && this._sources.some((source) =>
|
|
1868
|
+
source.kind === "external" && source.available === false)
|
|
1869
|
+
const hasUnverified = hasUnavailableSource || [...registry.links.values()].some((entry) =>
|
|
1870
|
+
inScope(entry) && entry.unverified) ||
|
|
1871
|
+
[...registry.duplicates.values()].some((entry) =>
|
|
1872
|
+
inScope(entry) && entry.unverified) ||
|
|
1873
|
+
[...registry.excluded.values()].some((entry) =>
|
|
1874
|
+
inScope(entry) && entry.unverified)
|
|
1875
|
+
const incomplete = hasUnverified || !!(
|
|
1876
|
+
lastAttempt && lastAttempt.complete === false &&
|
|
1877
|
+
(!lastComplete || (lastAttempt.ts || 0) >= (lastComplete.ts || 0))
|
|
1878
|
+
)
|
|
1879
|
+
const metricsExact = !!lastComplete && !incomplete
|
|
1880
|
+
const beforeBytes = metricsExact && Number.isFinite(lastComplete.bytes_total)
|
|
1881
|
+
? Math.max(0, lastComplete.bytes_total)
|
|
1882
|
+
: null
|
|
1883
|
+
const afterBytes = beforeBytes === null
|
|
1884
|
+
? null
|
|
1885
|
+
: Math.max(0, beforeBytes - savedBySharing)
|
|
1886
|
+
const result = {
|
|
1887
|
+
enabled: true,
|
|
1888
|
+
mode: this.mode,
|
|
1889
|
+
candidate_min_bytes: this.sizeThreshold,
|
|
1890
|
+
candidate_size_options: CANDIDATE_SIZE_OPTIONS,
|
|
1891
|
+
scan,
|
|
1892
|
+
file_action: this.fileActionStatus(scopeId),
|
|
1893
|
+
file_actions: this.fileActionStatuses(scopeId),
|
|
1894
|
+
action_sequence: responseActionSequence,
|
|
1895
|
+
last_scan: lastComplete,
|
|
1896
|
+
last_attempt: lastAttempt,
|
|
1897
|
+
incomplete,
|
|
1898
|
+
metrics_exact: metricsExact,
|
|
1899
|
+
bytes_on_disk: afterBytes,
|
|
1900
|
+
bytes_without_sharing: beforeBytes,
|
|
1901
|
+
saved_by_sharing: savedBySharing,
|
|
1902
|
+
reclaimable: blobs.filter((b) => b.orphan).reduce((sum, b) => sum + b.size, 0),
|
|
1903
|
+
pending_bytes: duplicates.filter((d) => d.shareable).reduce((sum, d) => sum + d.size, 0),
|
|
1904
|
+
activity_error: registry.eventError,
|
|
1905
|
+
cloud_sync_warning: this.cloudSyncProvider(),
|
|
1906
|
+
sources: publicSources, blobs, duplicates, excluded, events,
|
|
1907
|
+
undo_batches: undoBatches
|
|
1908
|
+
}
|
|
1909
|
+
if (scopeId) {
|
|
1910
|
+
result.scope_id = scopeId
|
|
1911
|
+
result.reclaimable = 0
|
|
1912
|
+
}
|
|
1913
|
+
return result
|
|
1914
|
+
}
|
|
1915
|
+
scanStatus() {
|
|
1916
|
+
const sweeper = this.sweeper
|
|
1917
|
+
if (!sweeper) return null
|
|
1918
|
+
const pending = !!this.scanPromise && !sweeper.state.active
|
|
1919
|
+
const state = pending
|
|
1920
|
+
? Object.assign(sweeper.idleState(), { phase: "queued", scope_id: this.scanScopeId })
|
|
1921
|
+
: sweeper.state
|
|
1922
|
+
return Object.assign({}, state, {
|
|
1923
|
+
current_file: sweeper.currentHash ? path.basename(sweeper.currentHash.path) : null,
|
|
1924
|
+
pending,
|
|
1925
|
+
error: this.scanError,
|
|
1926
|
+
persistence_warning: this.scanPersistenceWarning &&
|
|
1927
|
+
!!(this.registry && this.registry.persistDirty)
|
|
1928
|
+
})
|
|
1929
|
+
}
|
|
1930
|
+
progressStatus(scopeId = null) {
|
|
1931
|
+
return {
|
|
1932
|
+
enabled: !!this.enabled,
|
|
1933
|
+
candidate_min_bytes: this.sizeThreshold,
|
|
1934
|
+
candidate_size_options: CANDIDATE_SIZE_OPTIONS,
|
|
1935
|
+
scan: this.scanStatus(),
|
|
1936
|
+
file_action: this.fileActionStatus(scopeId),
|
|
1937
|
+
file_actions: this.fileActionStatuses(scopeId),
|
|
1938
|
+
action_sequence: this.actionSequence,
|
|
1939
|
+
last_scan: this.registry ? this.registry.scanFor(scopeId) : null,
|
|
1940
|
+
last_attempt: this.registry ? this.registry.scanAttemptFor(scopeId) : null
|
|
1941
|
+
}
|
|
1942
|
+
}
|
|
1943
|
+
fileActionStatus(scopeId = null) {
|
|
1944
|
+
const action = this.fileAction
|
|
1945
|
+
if (!action || (scopeId && action.source_id !== scopeId)) return null
|
|
1946
|
+
return {
|
|
1947
|
+
id: action.id,
|
|
1948
|
+
kind: action.kind,
|
|
1949
|
+
phase: action.phase,
|
|
1950
|
+
path: action.path,
|
|
1951
|
+
source_id: action.source_id,
|
|
1952
|
+
bytes_copied: action.bytes_copied,
|
|
1953
|
+
bytes_total: action.bytes_total,
|
|
1954
|
+
files_completed: action.files_completed,
|
|
1955
|
+
files_total: action.files_total
|
|
1956
|
+
}
|
|
1957
|
+
}
|
|
1958
|
+
fileActionStatuses(scopeId = null) {
|
|
1959
|
+
return [...this.fileActions.values()]
|
|
1960
|
+
.filter((action) => !scopeId || action.source_id === scopeId)
|
|
1961
|
+
.map((action) => ({
|
|
1962
|
+
id: action.id,
|
|
1963
|
+
kind: action.kind,
|
|
1964
|
+
phase: action.phase,
|
|
1965
|
+
path: action.path,
|
|
1966
|
+
source_id: action.source_id,
|
|
1967
|
+
bytes_copied: action.bytes_copied,
|
|
1968
|
+
bytes_total: action.bytes_total,
|
|
1969
|
+
files_completed: action.files_completed,
|
|
1970
|
+
files_total: action.files_total
|
|
1971
|
+
}))
|
|
1972
|
+
}
|
|
1973
|
+
navigationStatus(scopeId = null) {
|
|
1974
|
+
if (!this.enabled || !this.initialized || !this.registry) {
|
|
1975
|
+
return {
|
|
1976
|
+
any_scan: false,
|
|
1977
|
+
scanned: false,
|
|
1978
|
+
scanning: false,
|
|
1979
|
+
pending_bytes: 0
|
|
1980
|
+
}
|
|
1981
|
+
}
|
|
1982
|
+
const scan = this.scanStatus()
|
|
1983
|
+
const active = !!(scan && (scan.pending || scan.active || scan.queued > 0))
|
|
1984
|
+
const sources = new Map(this._sources.map((source) => [source.id, source]))
|
|
1985
|
+
let pendingBytes = 0
|
|
1986
|
+
for (const entry of this.registry.duplicates.values()) {
|
|
1987
|
+
if (scopeId && entry.source_id !== scopeId) continue
|
|
1988
|
+
const source = sources.get(entry.source_id)
|
|
1989
|
+
if (!source || source.available === false || !source.shareable ||
|
|
1990
|
+
entry.unverified || entry.unavailable_reason ||
|
|
1991
|
+
!this.registry.blobs.has(entry.hash)) continue
|
|
1992
|
+
pendingBytes += Math.max(0, Number(entry.size) || 0)
|
|
1993
|
+
}
|
|
1994
|
+
const lastComplete = this.registry.scanFor(scopeId)
|
|
1995
|
+
const lastAttempt = this.registry.scanAttemptFor(scopeId)
|
|
1996
|
+
const hasUnavailableSource = !scopeId && this._sources.some((source) =>
|
|
1997
|
+
source.kind === "external" && source.available === false)
|
|
1998
|
+
const hasUnverified = hasUnavailableSource || [...this.registry.links.values()].some((entry) =>
|
|
1999
|
+
(!scopeId || entry.source_id === scopeId) && entry.unverified) ||
|
|
2000
|
+
[...this.registry.duplicates.values()].some((entry) =>
|
|
2001
|
+
(!scopeId || entry.source_id === scopeId) && entry.unverified) ||
|
|
2002
|
+
[...this.registry.excluded.values()].some((entry) =>
|
|
2003
|
+
(!scopeId || entry.source_id === scopeId) && entry.unverified)
|
|
2004
|
+
const incomplete = hasUnverified || !!(
|
|
2005
|
+
lastAttempt && lastAttempt.complete === false &&
|
|
2006
|
+
(!lastComplete || (lastAttempt.ts || 0) >= (lastComplete.ts || 0))
|
|
2007
|
+
)
|
|
2008
|
+
return {
|
|
2009
|
+
any_scan: !!this.registry.lastScanAttempt || this.registry.sourceScanAttempts.size > 0,
|
|
2010
|
+
scanned: !!lastAttempt,
|
|
2011
|
+
scanning: active && (!scopeId || scan.scope_id === scopeId),
|
|
2012
|
+
pending_bytes: pendingBytes,
|
|
2013
|
+
incomplete
|
|
2014
|
+
}
|
|
2015
|
+
}
|
|
2016
|
+
navigationStatusForApp(appName) {
|
|
2017
|
+
return this.navigationStatus(sourceId("app", appName))
|
|
2018
|
+
}
|
|
2019
|
+
async preflightCopyCapacity(targets) {
|
|
2020
|
+
if (!targets.length) return { ok: true }
|
|
2021
|
+
if (!fs.promises.statfs) return { ok: false, status: "capacity-unavailable" }
|
|
2022
|
+
const byDevice = new Map()
|
|
2023
|
+
try {
|
|
2024
|
+
for (const target of targets) {
|
|
2025
|
+
const st = await fs.promises.lstat(target.path)
|
|
2026
|
+
const key = String(st.dev)
|
|
2027
|
+
if (!byDevice.has(key)) {
|
|
2028
|
+
byDevice.set(key, { path: path.dirname(target.path), bytes: 0 })
|
|
2029
|
+
}
|
|
2030
|
+
byDevice.get(key).bytes += Math.max(0, Number(target.bytes) || 0)
|
|
2031
|
+
}
|
|
2032
|
+
for (const group of byDevice.values()) {
|
|
2033
|
+
const stat = await fs.promises.statfs(group.path)
|
|
2034
|
+
const blockSize = BigInt(stat.bsize)
|
|
2035
|
+
const availableBlocks = BigInt(stat.bavail)
|
|
2036
|
+
const available = blockSize * availableBlocks
|
|
2037
|
+
if (available < BigInt(Math.ceil(group.bytes))) {
|
|
2038
|
+
return { ok: false, status: "insufficient-space" }
|
|
2039
|
+
}
|
|
2040
|
+
}
|
|
2041
|
+
return { ok: true }
|
|
2042
|
+
} catch (error) {
|
|
2043
|
+
return { ok: false, status: "capacity-unavailable" }
|
|
2044
|
+
}
|
|
2045
|
+
}
|
|
2046
|
+
async copyOut(filePath, sourcePath, expectedTarget, expectedSource = expectedTarget, options = {}) {
|
|
2047
|
+
const tmp = filePath + TMP_SUFFIX
|
|
2048
|
+
let copiedStat = null
|
|
2049
|
+
try {
|
|
2050
|
+
if (options.onProgress) {
|
|
2051
|
+
let sourceHandle = null
|
|
2052
|
+
let targetHandle = null
|
|
2053
|
+
try {
|
|
2054
|
+
sourceHandle = await fs.promises.open(sourcePath, "r")
|
|
2055
|
+
const sourceStat = await sourceHandle.stat()
|
|
2056
|
+
targetHandle = await fs.promises.open(tmp,
|
|
2057
|
+
fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL,
|
|
2058
|
+
sourceStat.mode & 0o7777)
|
|
2059
|
+
copiedStat = await targetHandle.stat()
|
|
2060
|
+
const buffer = Buffer.allocUnsafe(COPY_PROGRESS_CHUNK_SIZE)
|
|
2061
|
+
let offset = 0
|
|
2062
|
+
options.onProgress(0, sourceStat.size)
|
|
2063
|
+
while (offset < sourceStat.size) {
|
|
2064
|
+
const length = Math.min(buffer.length, sourceStat.size - offset)
|
|
2065
|
+
const { bytesRead } = await sourceHandle.read(buffer, 0, length, offset)
|
|
2066
|
+
if (!bytesRead) {
|
|
2067
|
+
const error = new Error("Source file ended while copying.")
|
|
2068
|
+
error.code = "EIO"
|
|
2069
|
+
throw error
|
|
2070
|
+
}
|
|
2071
|
+
let written = 0
|
|
2072
|
+
while (written < bytesRead) {
|
|
2073
|
+
const result = await targetHandle.write(
|
|
2074
|
+
buffer, written, bytesRead - written, offset + written)
|
|
2075
|
+
if (!result.bytesWritten) {
|
|
2076
|
+
const error = new Error("Could not write the independent copy.")
|
|
2077
|
+
error.code = "EIO"
|
|
2078
|
+
throw error
|
|
2079
|
+
}
|
|
2080
|
+
written += result.bytesWritten
|
|
2081
|
+
}
|
|
2082
|
+
offset += bytesRead
|
|
2083
|
+
options.onProgress(offset, sourceStat.size)
|
|
2084
|
+
}
|
|
2085
|
+
await targetHandle.chmod(sourceStat.mode & 0o7777)
|
|
2086
|
+
} finally {
|
|
2087
|
+
await Promise.all([
|
|
2088
|
+
sourceHandle && sourceHandle.close(),
|
|
2089
|
+
targetHandle && targetHandle.close()
|
|
2090
|
+
].filter(Boolean))
|
|
2091
|
+
}
|
|
2092
|
+
} else {
|
|
2093
|
+
await fs.promises.copyFile(sourcePath, tmp, fs.constants.COPYFILE_EXCL)
|
|
2094
|
+
}
|
|
2095
|
+
copiedStat = await fs.promises.lstat(tmp)
|
|
2096
|
+
if (options.canReplace && !options.canReplace()) {
|
|
2097
|
+
await unlinkIfSame(tmp, copiedStat)
|
|
2098
|
+
return { status: "locked" }
|
|
2099
|
+
}
|
|
2100
|
+
const currentTarget = await lstatIfPresent(filePath)
|
|
2101
|
+
const currentSource = sourcePath === filePath ? currentTarget : await lstatIfPresent(sourcePath)
|
|
2102
|
+
const currentTmp = await lstatIfPresent(tmp)
|
|
2103
|
+
if (!currentTarget || !currentSource || !sameSnapshot(expectedTarget, currentTarget) ||
|
|
2104
|
+
!sameSnapshot(expectedSource, currentSource) || !sameIdentity(currentTmp, copiedStat)) {
|
|
2105
|
+
await unlinkIfSame(tmp, copiedStat)
|
|
2106
|
+
return { status: "stale" }
|
|
2107
|
+
}
|
|
2108
|
+
await fs.promises.rename(tmp, filePath)
|
|
2109
|
+
let finalStat
|
|
2110
|
+
try {
|
|
2111
|
+
finalStat = await lstatIfPresent(filePath)
|
|
2112
|
+
} catch (error) {
|
|
2113
|
+
// rename() committed the independent copy, but its pre-rename ctime is
|
|
2114
|
+
// not final metadata. Keep only stable identity fields and require a
|
|
2115
|
+
// later scan to verify the name.
|
|
2116
|
+
const committedStat = copiedStat
|
|
2117
|
+
copiedStat = null
|
|
2118
|
+
return { status: "copied", stat: committedStat, unverified: true }
|
|
2119
|
+
}
|
|
2120
|
+
if (!finalStat || finalStat.dev !== copiedStat.dev || finalStat.ino !== copiedStat.ino) {
|
|
2121
|
+
return { status: "stale" }
|
|
2122
|
+
}
|
|
2123
|
+
copiedStat = null
|
|
2124
|
+
return { status: "copied", stat: finalStat }
|
|
2125
|
+
} catch (error) {
|
|
2126
|
+
await unlinkIfSame(tmp, copiedStat).catch(() => {})
|
|
2127
|
+
if (error && error.code === "EEXIST") return { status: "conflict" }
|
|
2128
|
+
if (isMissingError(error)) return { status: "stale" }
|
|
2129
|
+
throw error
|
|
2130
|
+
}
|
|
2131
|
+
}
|
|
2132
|
+
// detach: undo sharing for one linked name and restore it as a pending
|
|
2133
|
+
// duplicate.
|
|
2134
|
+
async detach(filePath, queuedAction = null) {
|
|
2135
|
+
if (!this.enabled) return { status: "disabled" }
|
|
2136
|
+
await this.refreshSources()
|
|
2137
|
+
const entry = this.registry.links.get(filePath)
|
|
2138
|
+
if (entry) {
|
|
2139
|
+
if (entry.unverified) return { status: "unavailable" }
|
|
2140
|
+
const source = this.sourceForPath(filePath, entry.source_id)
|
|
2141
|
+
if (!source || !await this.canonicalPathIsWithinSource(filePath, source)) {
|
|
2142
|
+
return { status: "stale" }
|
|
2143
|
+
}
|
|
2144
|
+
if (this.sourceAppIsRunning(source)) return { status: "locked" }
|
|
2145
|
+
const before = await lstatIfPresent(filePath)
|
|
2146
|
+
if (!before || !before.isFile() || before.dev !== entry.dev || before.ino !== entry.ino) return { status: "stale" }
|
|
2147
|
+
const capacity = await this.preflightCopyCapacity([{ path: filePath, bytes: before.size }])
|
|
2148
|
+
if (!capacity.ok) return { status: capacity.status }
|
|
2149
|
+
const action = queuedAction || {
|
|
2150
|
+
kind: "detach", path: filePath, source_id: entry.source_id || null,
|
|
2151
|
+
bytes_copied: 0, bytes_total: before.size
|
|
2152
|
+
}
|
|
2153
|
+
action.bytes_total = before.size
|
|
2154
|
+
const ownsAction = !queuedAction
|
|
2155
|
+
if (ownsAction) this.fileAction = action
|
|
2156
|
+
try {
|
|
2157
|
+
const copied = await this.copyOut(filePath, filePath, fileSnapshot(before), fileSnapshot(before), {
|
|
2158
|
+
canReplace: () => !this.sourceAppIsRunning(source),
|
|
2159
|
+
onProgress: (bytesCopied, bytesTotal) => {
|
|
2160
|
+
action.bytes_copied = bytesCopied
|
|
2161
|
+
action.bytes_total = bytesTotal
|
|
2162
|
+
}
|
|
2163
|
+
})
|
|
2164
|
+
if (copied.status !== "copied") return copied
|
|
2165
|
+
const st = copied.stat
|
|
2166
|
+
const baseEntry = {
|
|
2167
|
+
hash: entry.hash, size: st.size, source_id: entry.source_id || null,
|
|
2168
|
+
dev: st.dev, ino: st.ino
|
|
2169
|
+
}
|
|
2170
|
+
const scanEntry = copied.unverified ? null : Object.assign({}, baseEntry, {
|
|
2171
|
+
mtime: st.mtimeMs, ctime: st.ctimeMs
|
|
2172
|
+
})
|
|
2173
|
+
this.registry.setDuplicate(filePath, Object.assign({
|
|
2174
|
+
app: entry.app || null, discovered: Date.now(),
|
|
2175
|
+
unverified: !!copied.unverified
|
|
2176
|
+
}, scanEntry || baseEntry), scanEntry)
|
|
2177
|
+
if (copied.unverified) this.registry.scanIndex.delete(filePath)
|
|
2178
|
+
await this.refreshLinkSnapshots(entry.hash, entry.dev, entry.ino)
|
|
2179
|
+
await this.recordEvent({
|
|
2180
|
+
kind: "detach", hash: entry.hash, path: filePath,
|
|
2181
|
+
app: entry.app || null, source_id: entry.source_id || null,
|
|
2182
|
+
size: st.size, bytes_saved: st.size
|
|
2183
|
+
})
|
|
2184
|
+
return { status: "detached" }
|
|
2185
|
+
} finally {
|
|
2186
|
+
if (ownsAction && this.fileAction === action) this.fileAction = null
|
|
2187
|
+
}
|
|
2188
|
+
}
|
|
2189
|
+
return { status: "not-found" }
|
|
2190
|
+
}
|
|
2191
|
+
async skip(filePath) {
|
|
2192
|
+
if (!this.enabled) return { status: "disabled" }
|
|
2193
|
+
const duplicate = this.registry.duplicates.get(filePath)
|
|
2194
|
+
if (!duplicate) return { status: "not-found" }
|
|
2195
|
+
if (duplicate.unverified) return { status: "unavailable" }
|
|
2196
|
+
this.registry.exclude(filePath, {
|
|
2197
|
+
ts: Date.now(),
|
|
2198
|
+
source_id: duplicate.source_id || null,
|
|
2199
|
+
size: duplicate.size || 0
|
|
2200
|
+
})
|
|
2201
|
+
await this.recordEvent({
|
|
2202
|
+
kind: "skip",
|
|
2203
|
+
hash: duplicate.hash,
|
|
2204
|
+
path: filePath,
|
|
2205
|
+
app: duplicate.app || null,
|
|
2206
|
+
source_id: duplicate.source_id || null,
|
|
2207
|
+
size: duplicate.size || 0
|
|
2208
|
+
})
|
|
2209
|
+
return { status: "ignored" }
|
|
2210
|
+
}
|
|
2211
|
+
// reshare: un-pin a skipped path; the next scan may list it again.
|
|
2212
|
+
async reshare(filePath) {
|
|
2213
|
+
if (!this.enabled) return { status: "disabled" }
|
|
2214
|
+
const current = this.registry.excluded.get(filePath)
|
|
2215
|
+
if (current && current.unverified) return { status: "unavailable" }
|
|
2216
|
+
const excluded = this.registry.allowSharing(filePath)
|
|
2217
|
+
if (excluded) {
|
|
2218
|
+
await this.recordEvent({ kind: "reshare", path: filePath, source_id: excluded && excluded.source_id ? excluded.source_id : null })
|
|
2219
|
+
return { status: "resharable" }
|
|
2220
|
+
}
|
|
2221
|
+
return { status: "not-found" }
|
|
2222
|
+
}
|
|
2223
|
+
// Undo a conversion batch: replace each converted name with an independent
|
|
2224
|
+
// copy of the bytes (spec, UX contract surface 3).
|
|
2225
|
+
async undoBatch(batchId, queuedAction = null) {
|
|
2226
|
+
if (!this.enabled || !batchId) return { undone: 0 }
|
|
2227
|
+
await this.refreshSources()
|
|
2228
|
+
const events = await this.registry.readEvents()
|
|
2229
|
+
const conversionEvents = new Map(events
|
|
2230
|
+
.filter((event) => event.kind === "convert" && event.batch_id === batchId && event.path)
|
|
2231
|
+
.map((event) => [event.path, event]))
|
|
2232
|
+
const targets = new Set(conversionEvents.keys())
|
|
2233
|
+
for (const [filePath, entry] of this.registry.links) {
|
|
2234
|
+
if (entry.batch_id === batchId) targets.add(filePath)
|
|
2235
|
+
}
|
|
2236
|
+
const summary = { undone: 0, bytes: 0, failed: 0 }
|
|
2237
|
+
const prepared = []
|
|
2238
|
+
for (const filePath of targets) {
|
|
2239
|
+
const event = conversionEvents.get(filePath) || {}
|
|
2240
|
+
const entry = this.registry.links.get(filePath)
|
|
2241
|
+
if (!entry || (entry.batch_id && entry.batch_id !== batchId) ||
|
|
2242
|
+
(event.hash && entry.hash !== event.hash)) continue
|
|
2243
|
+
if (entry.unverified) {
|
|
2244
|
+
summary.failed += 1
|
|
2245
|
+
continue
|
|
2246
|
+
}
|
|
2247
|
+
const storePath = this.storePathFor(entry.hash)
|
|
2248
|
+
try {
|
|
2249
|
+
const source = this.sourceForPath(filePath, entry.source_id || event.source_id)
|
|
2250
|
+
if (!source || !await this.canonicalPathIsWithinSource(filePath, source)) {
|
|
2251
|
+
summary.failed += 1
|
|
2252
|
+
continue
|
|
2253
|
+
}
|
|
2254
|
+
if (this.sourceAppIsRunning(source)) {
|
|
2255
|
+
summary.failed += 1
|
|
2256
|
+
continue
|
|
2257
|
+
}
|
|
2258
|
+
const st = await fs.promises.lstat(filePath)
|
|
2259
|
+
const storeStat = await this.storeStatIfPresent(storePath)
|
|
2260
|
+
if (!st.isFile() || !storeStat || !storeStat.isFile()) {
|
|
2261
|
+
summary.failed += 1
|
|
2262
|
+
continue
|
|
2263
|
+
}
|
|
2264
|
+
if (st.ino !== storeStat.ino || st.dev !== storeStat.dev ||
|
|
2265
|
+
st.dev !== entry.dev || st.ino !== entry.ino) {
|
|
2266
|
+
summary.failed += 1
|
|
2267
|
+
continue
|
|
2268
|
+
}
|
|
2269
|
+
prepared.push({ filePath, event, entry, source, storePath, st, storeStat })
|
|
2270
|
+
} catch (e) {
|
|
2271
|
+
summary.failed += 1
|
|
2272
|
+
}
|
|
2273
|
+
}
|
|
2274
|
+
const capacity = await this.preflightCopyCapacity(prepared.map((target) => ({
|
|
2275
|
+
path: target.filePath,
|
|
2276
|
+
bytes: target.storeStat.size
|
|
2277
|
+
})))
|
|
2278
|
+
if (!capacity.ok) {
|
|
2279
|
+
return Object.assign(summary, { status: capacity.status })
|
|
2280
|
+
}
|
|
2281
|
+
const action = queuedAction || {
|
|
2282
|
+
kind: "undo", path: null, source_id: null,
|
|
2283
|
+
bytes_copied: 0, bytes_total: 0
|
|
2284
|
+
}
|
|
2285
|
+
action.bytes_total = prepared.reduce((sum, target) => sum + target.storeStat.size, 0)
|
|
2286
|
+
let completedBytes = 0
|
|
2287
|
+
for (const target of prepared) {
|
|
2288
|
+
const { filePath, event, entry, source, storePath, st, storeStat } = target
|
|
2289
|
+
try {
|
|
2290
|
+
action.path = filePath
|
|
2291
|
+
action.source_id = entry.source_id || event.source_id || null
|
|
2292
|
+
const copied = await this.copyOut(filePath, storePath, fileSnapshot(st), fileSnapshot(storeStat), {
|
|
2293
|
+
canReplace: () => !this.sourceAppIsRunning(source),
|
|
2294
|
+
onProgress: (bytesCopied) => {
|
|
2295
|
+
action.bytes_copied = completedBytes + bytesCopied
|
|
2296
|
+
}
|
|
2297
|
+
})
|
|
2298
|
+
if (copied.status !== "copied") {
|
|
2299
|
+
summary.failed += 1
|
|
2300
|
+
continue
|
|
2301
|
+
}
|
|
2302
|
+
const independentStat = copied.stat
|
|
2303
|
+
const duplicateEntry = {
|
|
2304
|
+
hash: entry.hash, size: storeStat.size, app: entry.app || null,
|
|
2305
|
+
source_id: entry.source_id || event.source_id || null, discovered: Date.now(),
|
|
2306
|
+
dev: independentStat.dev, ino: independentStat.ino,
|
|
2307
|
+
unverified: !!copied.unverified
|
|
2308
|
+
}
|
|
2309
|
+
const scanEntry = copied.unverified ? null : {
|
|
2310
|
+
hash: entry.hash, size: independentStat.size,
|
|
2311
|
+
dev: independentStat.dev, ino: independentStat.ino,
|
|
2312
|
+
mtime: independentStat.mtimeMs, ctime: independentStat.ctimeMs,
|
|
2313
|
+
source_id: entry.source_id || event.source_id || null
|
|
2314
|
+
}
|
|
2315
|
+
this.registry.setDuplicate(filePath, duplicateEntry, scanEntry)
|
|
2316
|
+
if (copied.unverified) this.registry.scanIndex.delete(filePath)
|
|
2317
|
+
await this.refreshLinkSnapshots(entry.hash, entry.dev, entry.ino)
|
|
2318
|
+
const bytesSaved = event.bytes_saved || storeStat.size
|
|
2319
|
+
await this.recordEvent({
|
|
2320
|
+
kind: "undo", hash: entry.hash, path: filePath,
|
|
2321
|
+
source_id: entry.source_id || event.source_id || null, batch_id: batchId,
|
|
2322
|
+
bytes_saved: bytesSaved
|
|
2323
|
+
})
|
|
2324
|
+
summary.undone += 1
|
|
2325
|
+
summary.bytes += storeStat.size
|
|
2326
|
+
completedBytes += storeStat.size
|
|
2327
|
+
action.bytes_copied = completedBytes
|
|
2328
|
+
} catch (e) {
|
|
2329
|
+
summary.failed += 1
|
|
2330
|
+
}
|
|
2331
|
+
}
|
|
2332
|
+
this.registry.schedulePersist()
|
|
2333
|
+
return summary
|
|
2334
|
+
}
|
|
2335
|
+
async reclaimAll() {
|
|
2336
|
+
if (!this.enabled) return { reclaimed: 0, bytes_freed: 0 }
|
|
2337
|
+
const summary = { reclaimed: 0, bytes_freed: 0, failed: 0 }
|
|
2338
|
+
const removedHashes = new Set()
|
|
2339
|
+
try {
|
|
2340
|
+
for (const [hash] of [...this.registry.blobs]) {
|
|
2341
|
+
const result = await this.reclaim(hash, {
|
|
2342
|
+
deferRegistry: true
|
|
2343
|
+
})
|
|
2344
|
+
if (result.remove_hash) removedHashes.add(hash)
|
|
2345
|
+
if (result.status === "reclaimed") {
|
|
2346
|
+
summary.reclaimed += 1
|
|
2347
|
+
summary.bytes_freed += result.bytes_freed || 0
|
|
2348
|
+
} else if (!["gone", "in-use", "unavailable"].includes(result.status)) summary.failed += 1
|
|
2349
|
+
}
|
|
2350
|
+
} finally {
|
|
2351
|
+
this.registry.removeBlobs(removedHashes)
|
|
2352
|
+
}
|
|
2353
|
+
return summary
|
|
2354
|
+
}
|
|
2355
|
+
}
|
|
2356
|
+
|
|
2357
|
+
Vault.SIZE_THRESHOLD = SIZE_THRESHOLD
|
|
2358
|
+
Vault.TMP_SUFFIX = TMP_SUFFIX
|
|
2359
|
+
|
|
2360
|
+
module.exports = Vault
|