pinokiod 8.0.42 → 8.0.45
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/kernel/vault/constants.js +1 -5
- package/kernel/vault/hash_worker.js +1 -7
- package/kernel/vault/index.js +532 -874
- package/kernel/vault/registry.js +35 -157
- package/kernel/vault/sweeper.js +106 -181
- package/kernel/vault/walker.js +1 -8
- package/package.json +1 -1
- package/server/index.js +3 -20
- package/server/public/storage-size.js +3 -1
- package/server/public/style.css +0 -13
- package/server/public/vault.css +61 -174
- package/server/public/vault.js +401 -639
- package/server/views/app.ejs +6 -85
- package/server/views/partials/main_sidebar.ejs +1 -4
- package/server/views/partials/vault_workspace.ejs +2 -5
- package/server/views/vault.ejs +1 -1
- package/server/views/vault_app.ejs +1 -1
- package/test/vault-engine.test.js +229 -212
- package/test/vault-sweep.test.js +163 -292
- package/test/vault-ui.test.js +703 -1016
- package/server/public/vault-nav.js +0 -96
package/kernel/vault/index.js
CHANGED
|
@@ -6,10 +6,7 @@ const Registry = require('./registry')
|
|
|
6
6
|
const Sweeper = require('./sweeper')
|
|
7
7
|
const { walkBatches, statMany } = require('./walker')
|
|
8
8
|
const { fileSnapshot, sameSnapshot, sameContentState } = require('./snapshot')
|
|
9
|
-
const {
|
|
10
|
-
SIZE_THRESHOLD, CANDIDATE_SIZE_OPTIONS, TMP_SUFFIX, SHA256_RE,
|
|
11
|
-
DIR_CONCURRENCY, STAT_CONCURRENCY
|
|
12
|
-
} = require('./constants')
|
|
9
|
+
const { SIZE_THRESHOLD, TMP_SUFFIX, SHA256_RE, DIR_CONCURRENCY, STAT_CONCURRENCY } = require('./constants')
|
|
13
10
|
|
|
14
11
|
// Shared model store engine (spec/requirements/shared-model-store.md).
|
|
15
12
|
// Store + registry + volume probe + hashing + adopt/convert/verify, plus the
|
|
@@ -19,7 +16,6 @@ const {
|
|
|
19
16
|
|
|
20
17
|
const NO_LINK_CODES = new Set(["EXDEV", "ENOTSUP", "ENOSYS"])
|
|
21
18
|
const LOCK_CODES = new Set(["EBUSY", "EPERM", "EACCES"])
|
|
22
|
-
const COPY_PROGRESS_CHUNK_SIZE = 8 * 1024 * 1024
|
|
23
19
|
const isMissingError = (error) => !!(error && (error.code === "ENOENT" || error.code === "ENOTDIR"))
|
|
24
20
|
const isAccessError = (error) => !!(error && (error.code === "EACCES" || error.code === "EPERM"))
|
|
25
21
|
const lstatIfPresent = async (filePath) => {
|
|
@@ -71,18 +67,14 @@ const sameFileMetadata = (left, right) => {
|
|
|
71
67
|
}
|
|
72
68
|
|
|
73
69
|
const sourceId = (kind, name) => `${kind}:${encodeURIComponent(name)}`
|
|
74
|
-
const externalSourceId = (root) => {
|
|
75
|
-
const value = process.platform === "win32" ? path.resolve(root).toLowerCase() : path.resolve(root)
|
|
76
|
-
return `external:${crypto.createHash("sha256").update(value).digest("hex")}`
|
|
77
|
-
}
|
|
78
70
|
|
|
79
71
|
class Vault {
|
|
80
72
|
constructor(kernel) {
|
|
81
73
|
this.kernel = kernel
|
|
82
74
|
this.enabled = false
|
|
83
75
|
this.initialized = false
|
|
84
|
-
this.mode = null // 'link' | '
|
|
85
|
-
this.volumeModes = new Map() // dev -> 'link' | '
|
|
76
|
+
this.mode = null // 'link' | 'copy' for the vault's own volume
|
|
77
|
+
this.volumeModes = new Map() // dev -> 'link' | 'copy'
|
|
86
78
|
this.registry = null
|
|
87
79
|
this.worker = null
|
|
88
80
|
this.workerJobs = new Map()
|
|
@@ -98,15 +90,8 @@ class Vault {
|
|
|
98
90
|
this.initializationPromise = null
|
|
99
91
|
this.verificationPending = false
|
|
100
92
|
this.scanError = null
|
|
101
|
-
this.scanPersistenceWarning = false
|
|
102
93
|
this.scanScopeId = null
|
|
103
|
-
this.
|
|
104
|
-
this.fileActions = new Map()
|
|
105
|
-
this.actionSequence = 0
|
|
106
|
-
this.scanEvents = null
|
|
107
|
-
this.scanCreatedStores = null
|
|
108
|
-
this.scanRetiredStores = null
|
|
109
|
-
this.scanRestoredStores = null
|
|
94
|
+
this.fileActionProgress = null
|
|
110
95
|
}
|
|
111
96
|
get root() {
|
|
112
97
|
return path.resolve(this.kernel.homedir, "vault")
|
|
@@ -114,6 +99,9 @@ class Vault {
|
|
|
114
99
|
get blobRoot() {
|
|
115
100
|
return path.resolve(this.root, "sha256")
|
|
116
101
|
}
|
|
102
|
+
get sourceRoot() {
|
|
103
|
+
return path.resolve(this.root, "sources")
|
|
104
|
+
}
|
|
117
105
|
storePathFor(hash) {
|
|
118
106
|
if (typeof hash !== "string" || !SHA256_RE.test(hash)) {
|
|
119
107
|
throw new TypeError("Invalid vault content identifier.")
|
|
@@ -171,123 +159,18 @@ class Vault {
|
|
|
171
159
|
return result
|
|
172
160
|
})
|
|
173
161
|
}
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
const action = {
|
|
177
|
-
id,
|
|
178
|
-
kind,
|
|
179
|
-
path: filePath || null,
|
|
180
|
-
source_id: sourceId || null,
|
|
181
|
-
phase: "queued",
|
|
182
|
-
bytes_copied: 0,
|
|
183
|
-
bytes_total: Math.max(0, Number(bytesTotal) || 0),
|
|
184
|
-
files_completed: Math.max(0, Number(progress.files_completed) || 0),
|
|
185
|
-
files_total: Math.max(0, Number(progress.files_total) || 0)
|
|
186
|
-
}
|
|
187
|
-
this.fileActions.set(id, action)
|
|
188
|
-
this.actionSequence += 1
|
|
189
|
-
return this.runExclusive(async () => {
|
|
190
|
-
action.phase = kind.startsWith("deduplicate") ? "deduplicating" : "copying"
|
|
191
|
-
this.fileAction = action
|
|
192
|
-
this.actionSequence += 1
|
|
193
|
-
try {
|
|
194
|
-
const result = await operation(action)
|
|
195
|
-
try {
|
|
196
|
-
if (this.registry) await this.registry.flush()
|
|
197
|
-
} catch (error) {
|
|
198
|
-
return Object.assign({}, result, { persistence_warning: true })
|
|
199
|
-
}
|
|
200
|
-
return result
|
|
201
|
-
} finally {
|
|
202
|
-
if (this.fileAction === action) this.fileAction = null
|
|
203
|
-
this.fileActions.delete(id)
|
|
204
|
-
this.actionSequence += 1
|
|
205
|
-
}
|
|
206
|
-
})
|
|
207
|
-
}
|
|
208
|
-
beginScanDraft() {
|
|
209
|
-
this.scanEvents = []
|
|
210
|
-
this.scanCreatedStores = []
|
|
211
|
-
this.scanRetiredStores = []
|
|
212
|
-
this.scanRestoredStores = []
|
|
213
|
-
}
|
|
214
|
-
rememberScanStore(storePath, stat) {
|
|
215
|
-
if (this.scanCreatedStores) this.scanCreatedStores.push({ storePath, stat })
|
|
216
|
-
}
|
|
217
|
-
rememberRetiredScanStore(storePath, stat, sourcePaths = []) {
|
|
218
|
-
if (this.scanRetiredStores) {
|
|
219
|
-
this.scanRetiredStores.push({ storePath, stat, sourcePaths: [...new Set(sourcePaths)] })
|
|
220
|
-
}
|
|
221
|
-
}
|
|
222
|
-
async restoreRetiredScanStore(item) {
|
|
223
|
-
const current = await this.storeStatIfPresent(item.storePath)
|
|
224
|
-
if (current) return sameIdentity(current, item.stat) ? current : null
|
|
225
|
-
const key = this.registry.inoKey(item.stat.dev, item.stat.ino)
|
|
226
|
-
const sourcePaths = new Set([
|
|
227
|
-
...(item.sourcePaths || []),
|
|
228
|
-
...(this.registry.pathsByIno.get(key) || [])
|
|
229
|
-
])
|
|
230
|
-
for (const filePath of sourcePaths) {
|
|
231
|
-
const st = await lstatIfPresent(filePath)
|
|
232
|
-
if (!sameIdentity(st, item.stat)) continue
|
|
233
|
-
await fs.promises.link(filePath, item.storePath)
|
|
234
|
-
const restored = await this.storeStatIfPresent(item.storePath)
|
|
235
|
-
return sameIdentity(restored, item.stat) ? restored : null
|
|
236
|
-
}
|
|
237
|
-
return null
|
|
238
|
-
}
|
|
239
|
-
async commitScanDraft() {
|
|
240
|
-
const events = (this.scanEvents || []).slice(-this.registry.maxEvents)
|
|
241
|
-
const retired = this.scanRetiredStores || []
|
|
242
|
-
const removed = []
|
|
162
|
+
async runFileAction(progress, operation) {
|
|
163
|
+
this.fileActionProgress = progress
|
|
243
164
|
try {
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
const hash = this.registry.byIno.get(this.registry.inoKey(item.stat.dev, item.stat.ino))
|
|
248
|
-
if (hash) await this.refreshLinkSnapshots(hash, item.stat.dev, item.stat.ino)
|
|
249
|
-
}
|
|
250
|
-
} catch (error) {
|
|
251
|
-
for (const item of removed.reverse()) {
|
|
252
|
-
const restored = await this.restoreRetiredScanStore(item).catch(() => null)
|
|
253
|
-
if (restored) this.scanRestoredStores.push({ item, stat: restored })
|
|
254
|
-
}
|
|
255
|
-
throw error
|
|
256
|
-
}
|
|
257
|
-
this.scanEvents = null
|
|
258
|
-
this.scanCreatedStores = null
|
|
259
|
-
this.scanRetiredStores = null
|
|
260
|
-
this.scanRestoredStores = null
|
|
261
|
-
try {
|
|
262
|
-
await this.registry.appendEvents(events)
|
|
263
|
-
return true
|
|
264
|
-
} catch (error) {
|
|
265
|
-
return false
|
|
165
|
+
return await operation(progress)
|
|
166
|
+
} finally {
|
|
167
|
+
if (this.fileActionProgress === progress) this.fileActionProgress = null
|
|
266
168
|
}
|
|
267
169
|
}
|
|
268
|
-
async abortScanDraft(snapshot) {
|
|
269
|
-
const created = this.scanCreatedStores || []
|
|
270
|
-
const restored = this.scanRestoredStores || []
|
|
271
|
-
this.scanEvents = null
|
|
272
|
-
this.scanCreatedStores = null
|
|
273
|
-
this.scanRetiredStores = null
|
|
274
|
-
this.scanRestoredStores = null
|
|
275
|
-
for (const item of created.reverse()) {
|
|
276
|
-
await unlinkIfSame(item.storePath, item.stat).catch(() => {})
|
|
277
|
-
}
|
|
278
|
-
this.registry.replaceState(snapshot, { persist: false })
|
|
279
|
-
for (const item of restored) {
|
|
280
|
-
const hash = this.registry.byIno.get(
|
|
281
|
-
this.registry.inoKey(item.stat.dev, item.stat.ino))
|
|
282
|
-
if (hash) await this.refreshLinkSnapshots(hash, item.stat.dev, item.stat.ino)
|
|
283
|
-
}
|
|
284
|
-
return restored.length > 0
|
|
285
|
-
}
|
|
286
170
|
startScan(scopeId = null) {
|
|
287
171
|
if (!this.enabled || !this.sweeper) return { started: false, disabled: true }
|
|
288
172
|
if (this.scanPromise) return { started: false, already_running: true }
|
|
289
173
|
this.scanError = null
|
|
290
|
-
this.scanPersistenceWarning = false
|
|
291
174
|
this.scanScopeId = scopeId
|
|
292
175
|
this.scanPromise = this.runExclusive(() => this.sweeper.scan(scopeId))
|
|
293
176
|
.catch((error) => {
|
|
@@ -317,71 +200,49 @@ class Vault {
|
|
|
317
200
|
}
|
|
318
201
|
}
|
|
319
202
|
}
|
|
320
|
-
case "remove_source":
|
|
321
|
-
return this.runMutation(() => this.removeExternalSource(payload.source_id))
|
|
322
203
|
case "scan":
|
|
323
204
|
if (payload.scope_id && !this.scanSource(payload.scope_id)) {
|
|
324
205
|
return { error: "That scan location is no longer available." }
|
|
325
206
|
}
|
|
326
|
-
if (payload.candidate_min_bytes !== undefined) {
|
|
327
|
-
if (!CANDIDATE_SIZE_OPTIONS.includes(payload.candidate_min_bytes)) {
|
|
328
|
-
return { error: "Choose one of the available minimum file sizes." }
|
|
329
|
-
}
|
|
330
|
-
if (this.scanPromise || (this.sweeper && this.sweeper.state.active)) {
|
|
331
|
-
return { error: "Wait for the current scan to finish before changing the minimum file size." }
|
|
332
|
-
}
|
|
333
|
-
this.sizeThreshold = payload.candidate_min_bytes
|
|
334
|
-
this.registry.settings.candidate_min_bytes = this.sizeThreshold
|
|
335
|
-
this.registry.schedulePersist()
|
|
336
|
-
}
|
|
337
207
|
return this.startScan(payload.scope_id || null)
|
|
338
208
|
case "deduplicate":
|
|
339
|
-
if (typeof payload.
|
|
340
|
-
return
|
|
209
|
+
if (typeof payload.path === "string" && payload.path) {
|
|
210
|
+
return this.runMutation(() => this.runFileAction({
|
|
211
|
+
kind: "deduplicate-file",
|
|
212
|
+
path: path.resolve(payload.path)
|
|
213
|
+
}, () => this.deduplicateFile(payload.path, {
|
|
214
|
+
batch_id: `batch-${crypto.randomUUID()}`
|
|
215
|
+
})))
|
|
341
216
|
}
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
? new Set(paths.filter((filePath) => typeof filePath === "string" && filePath)).size
|
|
346
|
-
: this.registry.duplicates.size
|
|
347
|
-
return this.queueFileMutation(
|
|
348
|
-
"deduplicate",
|
|
349
|
-
null,
|
|
350
|
-
payload.scope_id,
|
|
351
|
-
0,
|
|
352
|
-
(progress) => this.deduplicateScope(payload.scope_id, {
|
|
353
|
-
batch_id: `batch-${crypto.randomUUID()}`,
|
|
354
|
-
paths,
|
|
355
|
-
progress
|
|
356
|
-
}),
|
|
357
|
-
{ files_total: filesTotal }
|
|
358
|
-
)
|
|
217
|
+
const selection = payload.selection || null
|
|
218
|
+
if (selection && selection !== "duplicates" && selection !== "kept-separate") {
|
|
219
|
+
return { error: "Choose files to deduplicate." }
|
|
359
220
|
}
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
for (const group of Array.isArray(payload.groups) ? payload.groups : []) {
|
|
364
|
-
if (!group || typeof group.scope_id !== "string" || !group.scope_id) continue
|
|
365
|
-
if (!pathsByScope.has(group.scope_id)) pathsByScope.set(group.scope_id, [])
|
|
366
|
-
for (const filePath of Array.isArray(group.paths) ? group.paths : []) {
|
|
367
|
-
if (typeof filePath !== "string" || !filePath || seenPaths.has(filePath)) continue
|
|
368
|
-
seenPaths.add(filePath)
|
|
369
|
-
pathsByScope.get(group.scope_id).push(filePath)
|
|
370
|
-
}
|
|
221
|
+
if (payload.scope_id != null &&
|
|
222
|
+
(typeof payload.scope_id !== "string" || !payload.scope_id)) {
|
|
223
|
+
return { error: "Choose a valid location to deduplicate." }
|
|
371
224
|
}
|
|
372
|
-
const
|
|
373
|
-
.
|
|
374
|
-
|
|
375
|
-
if (!
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
225
|
+
const scopeId = typeof payload.scope_id === "string" && payload.scope_id
|
|
226
|
+
? payload.scope_id
|
|
227
|
+
: null
|
|
228
|
+
if (!selection && !scopeId) {
|
|
229
|
+
return { error: "Choose a location to deduplicate." }
|
|
230
|
+
}
|
|
231
|
+
return this.runMutation(() => this.runFileAction({
|
|
232
|
+
kind: "deduplicate",
|
|
233
|
+
scope_id: scopeId,
|
|
234
|
+
selection: selection || "duplicates",
|
|
235
|
+
files_total: 0,
|
|
236
|
+
files_completed: 0
|
|
237
|
+
}, async (progress) => {
|
|
238
|
+
const options = {
|
|
239
|
+
batch_id: `batch-${crypto.randomUUID()}`,
|
|
240
|
+
progress
|
|
241
|
+
}
|
|
242
|
+
return selection
|
|
243
|
+
? this.deduplicateSelection(selection, scopeId, options)
|
|
244
|
+
: this.deduplicateScope(scopeId, options)
|
|
245
|
+
}))
|
|
385
246
|
case "reclaim":
|
|
386
247
|
return this.runMutation(() => this.reclaim(payload.hash))
|
|
387
248
|
case "reclaim_all":
|
|
@@ -396,24 +257,16 @@ class Vault {
|
|
|
396
257
|
return { done: true }
|
|
397
258
|
})
|
|
398
259
|
case "undo":
|
|
399
|
-
return this.
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
: 0,
|
|
410
|
-
(action) => this.detach(payload.path, action)
|
|
411
|
-
)
|
|
412
|
-
}
|
|
413
|
-
case "skip":
|
|
414
|
-
return this.runMutation(() => this.skip(payload.path))
|
|
415
|
-
case "reshare":
|
|
416
|
-
return this.runMutation(() => this.reshare(payload.path))
|
|
260
|
+
return this.runMutation(() => this.undoBatch(payload.batch_id))
|
|
261
|
+
case "detach":
|
|
262
|
+
if (typeof payload.path !== "string" || !payload.path) return { status: "not-found" }
|
|
263
|
+
return this.runMutation(() => {
|
|
264
|
+
const targetPath = path.resolve(payload.path)
|
|
265
|
+
const kind = this.registry.duplicates.has(targetPath)
|
|
266
|
+
? "keep-separate"
|
|
267
|
+
: "make-separate"
|
|
268
|
+
return this.runFileAction({ kind, path: targetPath }, () => this.detach(targetPath))
|
|
269
|
+
})
|
|
417
270
|
default:
|
|
418
271
|
return { error: "unknown action" }
|
|
419
272
|
}
|
|
@@ -435,31 +288,44 @@ class Vault {
|
|
|
435
288
|
const decorate = async (source) => {
|
|
436
289
|
if (!source.root) return source
|
|
437
290
|
try {
|
|
438
|
-
const st = await fs.promises.
|
|
291
|
+
const st = await fs.promises.stat(source.root)
|
|
439
292
|
source.dev = st.dev
|
|
440
|
-
source.available = st.isDirectory()
|
|
293
|
+
source.available = st.isDirectory()
|
|
441
294
|
source.shareable = source.available && storeDev !== null && st.dev === storeDev && this.mode === "link"
|
|
442
295
|
} catch (error) {
|
|
443
|
-
if (!isMissingError(error)
|
|
296
|
+
if (!isMissingError(error)) throw error
|
|
444
297
|
source.available = false
|
|
445
298
|
source.shareable = false
|
|
446
|
-
source.error_code = error && error.code ? error.code : null
|
|
447
299
|
}
|
|
448
300
|
return source
|
|
449
301
|
}
|
|
450
302
|
|
|
451
|
-
const
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
303
|
+
const seenExternalRoots = new Set()
|
|
304
|
+
const addExternalEntries = async (importRoot, idPrefix = "") => {
|
|
305
|
+
let entries = []
|
|
306
|
+
try {
|
|
307
|
+
entries = await fs.promises.readdir(importRoot, { withFileTypes: true })
|
|
308
|
+
} catch (error) {
|
|
309
|
+
if (!isMissingError(error)) throw error
|
|
310
|
+
}
|
|
311
|
+
for (const entry of entries) {
|
|
312
|
+
if (!entry.isSymbolicLink()) continue
|
|
313
|
+
const mountPath = path.resolve(importRoot, entry.name)
|
|
314
|
+
try {
|
|
315
|
+
const root = await fs.promises.realpath(mountPath)
|
|
316
|
+
const st = await fs.promises.stat(root)
|
|
317
|
+
if (!st.isDirectory()) continue
|
|
318
|
+
const canonical = path.resolve(root)
|
|
319
|
+
if (seenExternalRoots.has(canonical)) continue
|
|
320
|
+
seenExternalRoots.add(canonical)
|
|
321
|
+
sources.push(await decorate({
|
|
322
|
+
id: sourceId("external", `${idPrefix}${entry.name}`), kind: "external", label: entry.name,
|
|
323
|
+
root: canonical, mount_path: mountPath, parent_id: "external"
|
|
324
|
+
}))
|
|
325
|
+
} catch (error) {
|
|
326
|
+
if (!isMissingError(error)) throw error
|
|
327
|
+
}
|
|
328
|
+
}
|
|
463
329
|
}
|
|
464
330
|
|
|
465
331
|
let apiEntries = []
|
|
@@ -477,6 +343,11 @@ class Vault {
|
|
|
477
343
|
}))
|
|
478
344
|
}
|
|
479
345
|
}
|
|
346
|
+
await addExternalEntries(apiRoot)
|
|
347
|
+
if (await this.directoryIfSafe(this.sourceRoot)) {
|
|
348
|
+
await addExternalEntries(this.sourceRoot, "vault:")
|
|
349
|
+
}
|
|
350
|
+
|
|
480
351
|
let homeEntries = []
|
|
481
352
|
try {
|
|
482
353
|
homeEntries = await fs.promises.readdir(home, { withFileTypes: true })
|
|
@@ -496,10 +367,13 @@ class Vault {
|
|
|
496
367
|
this._sourceBases = new Map()
|
|
497
368
|
for (const source of sources) {
|
|
498
369
|
if (!source.root || source.kind === "virtual") continue
|
|
499
|
-
const
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
370
|
+
for (const base of [source.root, source.mount_path]) {
|
|
371
|
+
if (!base) continue
|
|
372
|
+
const resolved = path.resolve(base)
|
|
373
|
+
const key = process.platform === "win32" ? resolved.toLowerCase() : resolved
|
|
374
|
+
if (!this._sourceBases.has(key)) this._sourceBases.set(key, [])
|
|
375
|
+
this._sourceBases.get(key).push(source)
|
|
376
|
+
}
|
|
503
377
|
}
|
|
504
378
|
return sources
|
|
505
379
|
}
|
|
@@ -533,106 +407,50 @@ class Vault {
|
|
|
533
407
|
if (existing) {
|
|
534
408
|
return { created: false, source: existing }
|
|
535
409
|
}
|
|
536
|
-
const configured = [...this.registry.settings.external_sources]
|
|
537
|
-
if (configured.some((root) => isPathWithin(root, canonical) || isPathWithin(canonical, root))) {
|
|
538
|
-
throw new Error("Choose a folder that does not contain or overlap another scan location.")
|
|
539
|
-
}
|
|
540
|
-
const previousScan = this.registry.lastScan
|
|
541
|
-
const previousAttempt = this.registry.lastScanAttempt
|
|
542
|
-
this.registry.settings.external_sources = [...configured, canonical]
|
|
543
|
-
this.registry.lastScan = null
|
|
544
|
-
this.registry.lastScanAttempt = null
|
|
545
|
-
this.registry.schedulePersist()
|
|
546
|
-
try {
|
|
547
|
-
await this.registry.flush()
|
|
548
|
-
await this.refreshSources()
|
|
549
|
-
const source = this._sources.find((item) =>
|
|
550
|
-
item.kind === "external" && item.root && samePath(item.root, canonical))
|
|
551
|
-
if (!source) throw new Error("The folder could not be added.")
|
|
552
|
-
return { created: true, source }
|
|
553
|
-
} catch (error) {
|
|
554
|
-
this.registry.settings.external_sources = configured
|
|
555
|
-
this.registry.lastScan = previousScan
|
|
556
|
-
this.registry.lastScanAttempt = previousAttempt
|
|
557
|
-
this.registry.schedulePersist()
|
|
558
|
-
await this.registry.flush().catch(() => {})
|
|
559
|
-
await this.refreshSources().catch(() => {})
|
|
560
|
-
throw error
|
|
561
|
-
}
|
|
562
|
-
}
|
|
563
|
-
async removeExternalSource(sourceId) {
|
|
564
|
-
if (typeof sourceId !== "string" || !sourceId) {
|
|
565
|
-
throw new Error("Choose an external folder to remove.")
|
|
566
|
-
}
|
|
567
|
-
await this.refreshSources()
|
|
568
|
-
const source = this._sources.find((item) =>
|
|
569
|
-
item.id === sourceId && item.kind === "external" && item.root)
|
|
570
|
-
if (!source) throw new Error("That external folder is no longer configured.")
|
|
571
410
|
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
}
|
|
580
|
-
if (!rootStat.isDirectory() || !samePath(canonical, source.root)) {
|
|
581
|
-
throw new Error("Reconnect this folder before removing it.")
|
|
582
|
-
}
|
|
583
|
-
|
|
584
|
-
const storesToRemove = new Map()
|
|
585
|
-
for (const [filePath, entry] of this.registry.links) {
|
|
586
|
-
if (entry.source_id !== source.id) continue
|
|
587
|
-
let fileStat
|
|
411
|
+
const importRoot = this.sourceRoot
|
|
412
|
+
await this.ensureDirectory(importRoot)
|
|
413
|
+
const baseLabel = path.basename(canonical) || "external-folder"
|
|
414
|
+
let mountPath = null
|
|
415
|
+
for (let index = 1; index < 10000; index++) {
|
|
416
|
+
const label = index === 1 ? baseLabel : `${baseLabel}-${index}`
|
|
417
|
+
const candidate = path.resolve(importRoot, label)
|
|
588
418
|
try {
|
|
589
|
-
|
|
419
|
+
await fs.promises.symlink(canonical, candidate, this.kernel.platform === "win32" ? "junction" : "dir")
|
|
420
|
+
mountPath = candidate
|
|
421
|
+
break
|
|
590
422
|
} catch (error) {
|
|
591
|
-
if (
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
const storeStat = await this.storeStatIfPresent(storePath)
|
|
597
|
-
const visibleNames = fileStat.nlink - (sameIdentity(fileStat, storeStat) ? 1 : 0)
|
|
598
|
-
if (visibleNames > 1) {
|
|
599
|
-
throw new Error("Turn sharing off for this folder before removing it.")
|
|
600
|
-
}
|
|
601
|
-
if (visibleNames === 1 && sameIdentity(fileStat, storeStat)) {
|
|
602
|
-
storesToRemove.set(entry.hash, {
|
|
603
|
-
path: storePath,
|
|
604
|
-
stat: storeStat
|
|
605
|
-
})
|
|
423
|
+
if (error && error.code === "EEXIST") continue
|
|
424
|
+
if (error && (error.code === "EACCES" || error.code === "EPERM")) {
|
|
425
|
+
throw new Error("Pinokio does not have permission to add that folder.")
|
|
426
|
+
}
|
|
427
|
+
throw new Error("Pinokio could not add that folder.")
|
|
606
428
|
}
|
|
607
429
|
}
|
|
430
|
+
if (!mountPath) throw new Error("Pinokio could not create a unique name for that folder.")
|
|
608
431
|
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
432
|
+
try {
|
|
433
|
+
await this.refreshSources()
|
|
434
|
+
const source = this._sources.find((item) => item.kind === "external" && item.mount_path && samePath(item.mount_path, mountPath))
|
|
435
|
+
if (!source) throw new Error("The folder could not be added. Restart Pinokio and try again.")
|
|
436
|
+
return { created: true, source }
|
|
437
|
+
} catch (error) {
|
|
438
|
+
// Adding a source is transactional. Roll back only the exact directory
|
|
439
|
+
// link created above; preserve any path an external writer replaced.
|
|
440
|
+
try {
|
|
441
|
+
const [mounted, target] = await Promise.all([
|
|
442
|
+
fs.promises.lstat(mountPath),
|
|
443
|
+
fs.promises.realpath(mountPath)
|
|
444
|
+
])
|
|
445
|
+
if (mounted.isSymbolicLink() && samePath(target, canonical)) {
|
|
446
|
+
await fs.promises.unlink(mountPath)
|
|
447
|
+
}
|
|
448
|
+
} catch {
|
|
449
|
+
// The original error remains authoritative; an unverified path is
|
|
450
|
+
// deliberately preserved rather than deleted.
|
|
612
451
|
}
|
|
452
|
+
throw error
|
|
613
453
|
}
|
|
614
|
-
|
|
615
|
-
for (const [filePath, entry] of [...this.registry.links]) {
|
|
616
|
-
if (entry.source_id === source.id) this.registry.untrack(filePath)
|
|
617
|
-
}
|
|
618
|
-
for (const [filePath, entry] of [...this.registry.duplicates]) {
|
|
619
|
-
if (entry.source_id === source.id) this.registry.untrack(filePath)
|
|
620
|
-
}
|
|
621
|
-
for (const [filePath, entry] of [...this.registry.scanIndex]) {
|
|
622
|
-
if (entry.source_id === source.id) this.registry.untrack(filePath)
|
|
623
|
-
}
|
|
624
|
-
for (const [filePath, entry] of [...this.registry.excluded]) {
|
|
625
|
-
if (entry.source_id === source.id) this.registry.excluded.delete(filePath)
|
|
626
|
-
}
|
|
627
|
-
this.registry.sourceScans.delete(source.id)
|
|
628
|
-
this.registry.sourceScanAttempts.delete(source.id)
|
|
629
|
-
this.registry.settings.external_sources = this.registry.settings.external_sources
|
|
630
|
-
.filter((root) => !samePath(root, source.root))
|
|
631
|
-
this.registry.lastScan = null
|
|
632
|
-
this.registry.lastScanAttempt = null
|
|
633
|
-
this.registry.schedulePersist()
|
|
634
|
-
await this.refreshSources()
|
|
635
|
-
return { removed: true, source_id: source.id }
|
|
636
454
|
}
|
|
637
455
|
sourceForPath(filePath, preferredId) {
|
|
638
456
|
let cursor = path.resolve(filePath)
|
|
@@ -664,7 +482,9 @@ class Vault {
|
|
|
664
482
|
const source = this.sourceForPath(filePath, preferredId)
|
|
665
483
|
if (!source) return { source_id: null, relative_path: path.basename(filePath) }
|
|
666
484
|
const absolute = path.resolve(filePath)
|
|
667
|
-
|
|
485
|
+
let base = source.root
|
|
486
|
+
if (source.mount_path && isPathWithin(source.mount_path, absolute)) base = source.mount_path
|
|
487
|
+
const relative = path.relative(base, absolute).split(path.sep).join("/") || path.basename(absolute)
|
|
668
488
|
return {
|
|
669
489
|
source_id: source.id,
|
|
670
490
|
source_kind: source.kind,
|
|
@@ -679,13 +499,6 @@ class Vault {
|
|
|
679
499
|
const context = location.source_id ? location : { relative_path: event.path }
|
|
680
500
|
entry = Object.assign({}, event, context)
|
|
681
501
|
}
|
|
682
|
-
if (this.scanEvents) {
|
|
683
|
-
this.scanEvents.push(entry)
|
|
684
|
-
if (this.scanEvents.length > this.registry.maxEvents * 2) {
|
|
685
|
-
this.scanEvents.splice(0, this.registry.maxEvents)
|
|
686
|
-
}
|
|
687
|
-
return true
|
|
688
|
-
}
|
|
689
502
|
try {
|
|
690
503
|
await this.registry.appendEvent(entry)
|
|
691
504
|
return true
|
|
@@ -770,16 +583,12 @@ class Vault {
|
|
|
770
583
|
await this.ensureDirectory(this.blobRoot)
|
|
771
584
|
this.registry = new Registry(this.root)
|
|
772
585
|
this.mode = await this.probe(this.root)
|
|
773
|
-
const loaded = await this.registry.load()
|
|
774
586
|
await this.refreshSources()
|
|
587
|
+
const loaded = await this.registry.load()
|
|
775
588
|
const missingWithBlobs = !loaded.existed && await this.hasStoredBlobs()
|
|
776
589
|
if (loaded.corrupt || missingWithBlobs) {
|
|
777
|
-
|
|
778
|
-
// or walk configured sources. A later explicit Scan or Repair rebuilds
|
|
779
|
-
// visible-path state.
|
|
780
|
-
await this.rebuild([], { preserveState: false })
|
|
590
|
+
await this.rebuild(undefined, { preserveState: false })
|
|
781
591
|
}
|
|
782
|
-
this.sizeThreshold = this.registry.settings.candidate_min_bytes || SIZE_THRESHOLD
|
|
783
592
|
this.sweeper = new Sweeper(this)
|
|
784
593
|
this.initialized = true
|
|
785
594
|
return {
|
|
@@ -842,7 +651,7 @@ class Vault {
|
|
|
842
651
|
if (this.volumeModes.has(dev)) return this.volumeModes.get(dev)
|
|
843
652
|
const a = path.resolve(dir, `.pinokio-probe-${crypto.randomBytes(6).toString("hex")}`)
|
|
844
653
|
const b = a + "-link"
|
|
845
|
-
let mode = "
|
|
654
|
+
let mode = "copy"
|
|
846
655
|
let aStat = null
|
|
847
656
|
let bStat = null
|
|
848
657
|
let failure = null
|
|
@@ -879,11 +688,7 @@ class Vault {
|
|
|
879
688
|
const job = this.workerJobs.get(id)
|
|
880
689
|
if (!job) return
|
|
881
690
|
this.workerJobs.delete(id)
|
|
882
|
-
if (error)
|
|
883
|
-
const failure = new Error(error.message || String(error))
|
|
884
|
-
if (error.code) failure.code = error.code
|
|
885
|
-
job.reject(failure)
|
|
886
|
-
}
|
|
691
|
+
if (error) job.reject(new Error(error))
|
|
887
692
|
else job.resolve({ hash, size })
|
|
888
693
|
// Keep the worker warm across the scan queue. Starting one worker per
|
|
889
694
|
// file costs ~30ms and adds up quickly on a first scan.
|
|
@@ -939,10 +744,7 @@ class Vault {
|
|
|
939
744
|
hash,
|
|
940
745
|
source_id: entry.source_id || null
|
|
941
746
|
}))
|
|
942
|
-
} catch (error) {
|
|
943
|
-
this.registry.scanIndex.delete(linkPath)
|
|
944
|
-
entry.unverified = true
|
|
945
|
-
}
|
|
747
|
+
} catch (error) {}
|
|
946
748
|
}
|
|
947
749
|
this.registry.schedulePersist()
|
|
948
750
|
}
|
|
@@ -952,7 +754,7 @@ class Vault {
|
|
|
952
754
|
for (const linkPath of this.registry.pathsByIno.get(key) || []) {
|
|
953
755
|
const entry = this.registry.links.get(linkPath)
|
|
954
756
|
if (!entry) continue
|
|
955
|
-
if (entry.hash !== hash ||
|
|
757
|
+
if (entry.hash !== hash || entry.mode !== "link" ||
|
|
956
758
|
entry.dev !== storeStat.dev || entry.ino !== storeStat.ino) continue
|
|
957
759
|
const expected = this.registry.scanIndex.get(linkPath)
|
|
958
760
|
if (expected && expected.hash === hash && sameSnapshot(expected, storeStat)) {
|
|
@@ -965,15 +767,13 @@ class Vault {
|
|
|
965
767
|
try {
|
|
966
768
|
hashed = await this.hashFile(storePath)
|
|
967
769
|
} catch (error) {
|
|
968
|
-
|
|
969
|
-
throw error
|
|
770
|
+
return { valid: false }
|
|
970
771
|
}
|
|
971
772
|
let after
|
|
972
773
|
try {
|
|
973
774
|
after = await this.storeStatIfPresent(storePath)
|
|
974
775
|
} catch (error) {
|
|
975
|
-
|
|
976
|
-
throw error
|
|
776
|
+
return { valid: false }
|
|
977
777
|
}
|
|
978
778
|
if (!sameSnapshot(before, after) || hashed.hash !== hash || hashed.size !== after.size) {
|
|
979
779
|
return { valid: false }
|
|
@@ -995,6 +795,7 @@ class Vault {
|
|
|
995
795
|
source_id: meta.source_id || null,
|
|
996
796
|
dev: st.dev,
|
|
997
797
|
ino: st.ino,
|
|
798
|
+
mode: "link"
|
|
998
799
|
}
|
|
999
800
|
const storeStat = await this.storeStatIfPresent(storePath, {
|
|
1000
801
|
createParent: this.mode === "link"
|
|
@@ -1012,12 +813,25 @@ class Vault {
|
|
|
1012
813
|
if (!sameSnapshot(before, current)) return { status: "stale" }
|
|
1013
814
|
return { status: "duplicate", storePath }
|
|
1014
815
|
}
|
|
1015
|
-
|
|
816
|
+
const registerCopy = async () => {
|
|
817
|
+
const current = await lstatIfPresent(filePath)
|
|
818
|
+
if (!sameSnapshot(before, current)) return { status: "stale" }
|
|
819
|
+
linkEntry.mode = "copy"
|
|
820
|
+
this.registry.addBlob(hash, { size: st.size, source_urls: meta.source_urls || [] })
|
|
821
|
+
this.registry.addLink(filePath, linkEntry)
|
|
822
|
+
await this.refreshLinkSnapshots(hash, st.dev, st.ino)
|
|
823
|
+
await this.recordEvent({
|
|
824
|
+
kind: "adopt", hash, path: filePath, app: meta.app || null,
|
|
825
|
+
source_id: meta.source_id || null, bytes_saved: 0, mode: "copy"
|
|
826
|
+
})
|
|
827
|
+
return { status: "copy-mode" }
|
|
828
|
+
}
|
|
829
|
+
if (this.mode === "copy") return registerCopy()
|
|
1016
830
|
try {
|
|
1017
831
|
await fs.promises.link(filePath, storePath)
|
|
1018
832
|
} catch (e) {
|
|
1019
833
|
if (NO_LINK_CODES.has(e.code)) {
|
|
1020
|
-
return
|
|
834
|
+
return registerCopy()
|
|
1021
835
|
}
|
|
1022
836
|
throw e
|
|
1023
837
|
}
|
|
@@ -1045,7 +859,6 @@ class Vault {
|
|
|
1045
859
|
}
|
|
1046
860
|
this.registry.addBlob(hash, { size: st.size, source_urls: meta.source_urls || [] })
|
|
1047
861
|
this.registry.addLink(filePath, linkEntry)
|
|
1048
|
-
this.rememberScanStore(storePath, currentStore)
|
|
1049
862
|
await this.refreshLinkSnapshots(hash, st.dev, st.ino)
|
|
1050
863
|
await this.recordEvent({ kind: "adopt", hash, path: filePath, app: meta.app || null, source_id: meta.source_id || null, bytes_saved: 0 })
|
|
1051
864
|
return { status: "adopted" }
|
|
@@ -1055,28 +868,39 @@ class Vault {
|
|
|
1055
868
|
async convert(targetPath, hash, meta = {}) {
|
|
1056
869
|
if (!this.enabled) return { status: "disabled" }
|
|
1057
870
|
const storePath = this.storePathFor(hash)
|
|
1058
|
-
|
|
1059
|
-
if (!storeStat) return { status: "no-blob" }
|
|
1060
|
-
let targetStat
|
|
871
|
+
let storeStat
|
|
1061
872
|
try {
|
|
1062
|
-
|
|
1063
|
-
} catch (
|
|
1064
|
-
|
|
1065
|
-
if (isMissingError(error)) return { status: "stale" }
|
|
1066
|
-
throw error
|
|
873
|
+
storeStat = await this.storeStatIfPresent(storePath)
|
|
874
|
+
} catch (e) {
|
|
875
|
+
return { status: "no-blob" }
|
|
1067
876
|
}
|
|
877
|
+
if (!storeStat) return { status: "no-blob" }
|
|
878
|
+
let targetStat = await fs.promises.lstat(targetPath)
|
|
1068
879
|
if (!storeStat.isFile() || !targetStat.isFile()) return { status: "stale" }
|
|
1069
880
|
if (meta.expected && !sameSnapshot(meta.expected, targetStat)) {
|
|
1070
|
-
|
|
881
|
+
// Windows metadata operations can change ctime without changing the
|
|
882
|
+
// file's identity or bytes. Recover only that isolated mismatch by
|
|
883
|
+
// hashing the target again, then retain the exact pre-rename check below.
|
|
884
|
+
if (!sameContentState(meta.expected, targetStat)) return { status: "stale" }
|
|
885
|
+
const before = fileSnapshot(targetStat)
|
|
886
|
+
let verifiedTarget
|
|
887
|
+
try {
|
|
888
|
+
verifiedTarget = await this.hashFile(targetPath)
|
|
889
|
+
} catch (error) {
|
|
890
|
+
return { status: "stale" }
|
|
891
|
+
}
|
|
892
|
+
const after = await lstatIfPresent(targetPath)
|
|
893
|
+
if (!sameSnapshot(before, after) ||
|
|
894
|
+
verifiedTarget.hash !== hash || verifiedTarget.size !== after.size) {
|
|
895
|
+
return { status: "stale" }
|
|
896
|
+
}
|
|
897
|
+
targetStat = after
|
|
1071
898
|
}
|
|
1072
899
|
if (storeStat.dev !== targetStat.dev) {
|
|
1073
900
|
return { status: "unavailable", code: "EXDEV" }
|
|
1074
901
|
}
|
|
1075
902
|
if (storeStat.dev === targetStat.dev && storeStat.ino === targetStat.ino) {
|
|
1076
|
-
this.registry.addLink(targetPath, {
|
|
1077
|
-
hash, app: meta.app || null, source_id: meta.source_id || null,
|
|
1078
|
-
dev: storeStat.dev, ino: storeStat.ino, unverified: false
|
|
1079
|
-
})
|
|
903
|
+
this.registry.addLink(targetPath, { hash, app: meta.app || null, source_id: meta.source_id || null, dev: storeStat.dev, ino: storeStat.ino, mode: "link" })
|
|
1080
904
|
await this.refreshLinkSnapshots(hash, storeStat.dev, storeStat.ino)
|
|
1081
905
|
return { status: "already" }
|
|
1082
906
|
}
|
|
@@ -1158,10 +982,10 @@ class Vault {
|
|
|
1158
982
|
}
|
|
1159
983
|
this.registry.addLink(targetPath, {
|
|
1160
984
|
hash, app: meta.app || null, source_id: meta.source_id || null,
|
|
1161
|
-
dev: st.dev, ino: st.ino, batch_id: meta.batch_id || null
|
|
1162
|
-
unverified: false
|
|
985
|
+
dev: st.dev, ino: st.ino, mode: "link", batch_id: meta.batch_id || null
|
|
1163
986
|
})
|
|
1164
987
|
await this.refreshLinkSnapshots(hash, st.dev, st.ino)
|
|
988
|
+
this.registry.addSaved(storeStat.size)
|
|
1165
989
|
await this.recordEvent({
|
|
1166
990
|
kind: "convert", hash, path: targetPath, app: meta.app || null, source_id: meta.source_id || null,
|
|
1167
991
|
bytes_saved: storeStat.size, batch_id: meta.batch_id || null
|
|
@@ -1180,138 +1004,196 @@ class Vault {
|
|
|
1180
1004
|
return !!(runningPath && isPathWithin(appRoot, runningPath))
|
|
1181
1005
|
})
|
|
1182
1006
|
}
|
|
1183
|
-
async
|
|
1007
|
+
async deduplicateFile(filePath, options = {}) {
|
|
1008
|
+
if (!this.enabled || typeof filePath !== "string" || !filePath) {
|
|
1009
|
+
return { status: this.enabled ? "not-found" : "disabled" }
|
|
1010
|
+
}
|
|
1184
1011
|
await this.refreshSources()
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1012
|
+
return this.deduplicateExcludedFile(filePath, options)
|
|
1013
|
+
}
|
|
1014
|
+
async deduplicateExcludedFile(filePath, options = {}) {
|
|
1015
|
+
const targetPath = path.resolve(filePath)
|
|
1016
|
+
const excluded = this.registry.excluded.get(targetPath)
|
|
1017
|
+
if (!excluded) return { status: "not-found" }
|
|
1018
|
+
const source = this.sourceForPath(targetPath, excluded.source_id)
|
|
1019
|
+
if (!source || !await this.canonicalPathIsWithinSource(targetPath, source)) {
|
|
1020
|
+
return { status: "stale" }
|
|
1190
1021
|
}
|
|
1022
|
+
if (!source.shareable) return { status: "unavailable" }
|
|
1023
|
+
if (this.sourceAppIsRunning(source)) return { status: "locked" }
|
|
1191
1024
|
|
|
1192
|
-
const
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
return
|
|
1025
|
+
const before = await lstatIfPresent(targetPath)
|
|
1026
|
+
if (!before || !before.isFile()) return { status: "stale" }
|
|
1027
|
+
let hashed
|
|
1028
|
+
try {
|
|
1029
|
+
hashed = await this.hashFile(targetPath)
|
|
1030
|
+
} catch (error) {
|
|
1031
|
+
return { status: "stale" }
|
|
1032
|
+
}
|
|
1033
|
+
const after = await lstatIfPresent(targetPath)
|
|
1034
|
+
if (!sameSnapshot(fileSnapshot(before), after) || hashed.size !== after.size) {
|
|
1035
|
+
return { status: "stale" }
|
|
1036
|
+
}
|
|
1037
|
+
if (!this.registry.blobs.has(hashed.hash)) return { status: "no-match" }
|
|
1038
|
+
if (!await this.canonicalPathIsWithinSource(targetPath, source)) return { status: "stale" }
|
|
1039
|
+
const current = await lstatIfPresent(targetPath)
|
|
1040
|
+
if (!sameSnapshot(fileSnapshot(after), current)) return { status: "stale" }
|
|
1041
|
+
|
|
1042
|
+
const result = await this.convert(targetPath, hashed.hash, {
|
|
1043
|
+
app: source.kind === "app" ? source.app : null,
|
|
1044
|
+
source_id: source.id,
|
|
1045
|
+
batch_id: options.batch_id || `batch-${crypto.randomUUID()}`,
|
|
1046
|
+
source,
|
|
1047
|
+
expected: fileSnapshot(current)
|
|
1203
1048
|
})
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
progress.files_completed = 0
|
|
1207
|
-
progress.files_total = candidates.length
|
|
1208
|
-
this.actionSequence += 1
|
|
1049
|
+
if (result.status === "converted" || result.status === "already") {
|
|
1050
|
+
this.registry.allowSharing(targetPath)
|
|
1209
1051
|
}
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1052
|
+
return result
|
|
1053
|
+
}
|
|
1054
|
+
sourceIsWithinScope(source, scopeId) {
|
|
1055
|
+
if (!scopeId) return true
|
|
1056
|
+
const seen = new Set()
|
|
1057
|
+
let current = source
|
|
1058
|
+
while (current && !seen.has(current.id)) {
|
|
1059
|
+
if (current.id === scopeId) return true
|
|
1060
|
+
seen.add(current.id)
|
|
1061
|
+
current = this._sources.find((item) => item.id === current.parent_id)
|
|
1062
|
+
}
|
|
1063
|
+
return false
|
|
1064
|
+
}
|
|
1065
|
+
async deduplicateSelection(selection, scopeId, options = {}) {
|
|
1066
|
+
if (selection === "duplicates") {
|
|
1067
|
+
return this.deduplicateScope(scopeId, Object.assign({}, options, { includeDescendants: true }))
|
|
1068
|
+
}
|
|
1069
|
+
await this.refreshSources()
|
|
1070
|
+
if (scopeId && !this._sources.some((source) => source.id === scopeId)) {
|
|
1071
|
+
return { error: "That location is no longer available. Scan again to refresh it." }
|
|
1072
|
+
}
|
|
1073
|
+
const paths = [...this.registry.excluded].filter(([filePath, entry]) => {
|
|
1074
|
+
const source = this.sourceForPath(filePath, entry.source_id)
|
|
1075
|
+
return source ? this.sourceIsWithinScope(source, scopeId) : !scopeId
|
|
1076
|
+
}).map(([filePath]) => filePath)
|
|
1077
|
+
if (options.progress) options.progress.files_total = paths.length
|
|
1078
|
+
const summary = {
|
|
1079
|
+
converted: 0, bytes_saved: 0, locked: 0, stale: 0, unmatched: 0,
|
|
1080
|
+
incompatible: 0, unavailable: 0, failed: 0
|
|
1081
|
+
}
|
|
1082
|
+
const batch = options.batch_id || `batch-${crypto.randomUUID()}`
|
|
1083
|
+
for (const filePath of paths) {
|
|
1213
1084
|
try {
|
|
1214
|
-
const
|
|
1215
|
-
if (
|
|
1216
|
-
|
|
1217
|
-
summary.
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
if (
|
|
1221
|
-
summary.stale += 1
|
|
1222
|
-
continue
|
|
1223
|
-
}
|
|
1224
|
-
const indexed = registry.scanIndex.get(filePath)
|
|
1225
|
-
const expected = indexed && indexed.hash === entry.hash
|
|
1226
|
-
? indexed
|
|
1227
|
-
: (entry.dev !== undefined && entry.ino !== undefined &&
|
|
1228
|
-
entry.mtime !== undefined && entry.ctime !== undefined ? entry : null)
|
|
1229
|
-
if (!expected) {
|
|
1085
|
+
const result = await this.deduplicateExcludedFile(filePath, { batch_id: batch })
|
|
1086
|
+
if (result.status === "converted" || result.status === "already") {
|
|
1087
|
+
summary.converted += 1
|
|
1088
|
+
summary.bytes_saved += result.bytes_saved || 0
|
|
1089
|
+
} else if (result.status === "locked") {
|
|
1090
|
+
summary.locked += 1
|
|
1091
|
+
} else if (result.status === "stale") {
|
|
1230
1092
|
summary.stale += 1
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
}
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
app: source.kind === "app" ? source.app : (entry.app || null),
|
|
1240
|
-
source_id: scopeId,
|
|
1241
|
-
batch_id: batch,
|
|
1242
|
-
source: currentSource,
|
|
1243
|
-
expected
|
|
1244
|
-
})
|
|
1245
|
-
if (result.status === "converted" || result.status === "already") {
|
|
1246
|
-
summary.converted += 1
|
|
1247
|
-
summary.bytes_saved += result.bytes_saved || 0
|
|
1248
|
-
} else if (result.status === "locked") {
|
|
1249
|
-
summary.locked += 1
|
|
1250
|
-
} else if (result.status === "stale") {
|
|
1251
|
-
summary.stale += 1
|
|
1252
|
-
} else if (result.status === "stale-blob") {
|
|
1253
|
-
staleHashes.add(entry.hash)
|
|
1254
|
-
summary.stale += 1
|
|
1255
|
-
} else if (result.status === "metadata-mismatch") {
|
|
1256
|
-
entry.unavailable_reason = "metadata"
|
|
1257
|
-
summary.incompatible += 1
|
|
1258
|
-
} else if (result.status === "unavailable") {
|
|
1259
|
-
summary.unavailable += 1
|
|
1260
|
-
} else {
|
|
1261
|
-
summary.failed += 1
|
|
1262
|
-
}
|
|
1263
|
-
} catch (error) {
|
|
1093
|
+
} else if (result.status === "no-match" || result.status === "no-blob" ||
|
|
1094
|
+
result.status === "size-mismatch") {
|
|
1095
|
+
summary.unmatched += 1
|
|
1096
|
+
} else if (result.status === "metadata-mismatch") {
|
|
1097
|
+
summary.incompatible += 1
|
|
1098
|
+
} else if (result.status === "unavailable" || result.status === "copy-mode") {
|
|
1099
|
+
summary.unavailable += 1
|
|
1100
|
+
} else {
|
|
1264
1101
|
summary.failed += 1
|
|
1265
1102
|
}
|
|
1266
|
-
}
|
|
1267
|
-
|
|
1268
|
-
progress.files_completed += 1
|
|
1269
|
-
this.actionSequence += 1
|
|
1270
|
-
}
|
|
1271
|
-
if (options.onFileComplete) options.onFileComplete()
|
|
1103
|
+
} catch (error) {
|
|
1104
|
+
summary.failed += 1
|
|
1272
1105
|
}
|
|
1106
|
+
if (options.progress) options.progress.files_completed += 1
|
|
1273
1107
|
}
|
|
1274
|
-
registry.schedulePersist()
|
|
1275
1108
|
return summary
|
|
1276
1109
|
}
|
|
1277
|
-
async
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1110
|
+
async deduplicateScope(scopeId, options = {}) {
|
|
1111
|
+
await this.refreshSources()
|
|
1112
|
+
const includeDescendants = options.includeDescendants === true
|
|
1113
|
+
const source = scopeId
|
|
1114
|
+
? this._sources.find((item) => item.id === scopeId && (includeDescendants || item.kind !== "virtual"))
|
|
1115
|
+
: null
|
|
1116
|
+
if ((!includeDescendants || scopeId) && !source) {
|
|
1117
|
+
return { error: "That location is no longer available. Scan again to refresh it." }
|
|
1118
|
+
}
|
|
1119
|
+
if (!includeDescendants) {
|
|
1120
|
+
if (!source.shareable) return { error: "This location is on a disk that cannot share space with this vault." }
|
|
1121
|
+
if (this.sourceAppIsRunning(source)) {
|
|
1122
|
+
return { error: "This app has a running script. Stop it first, then deduplicate." }
|
|
1123
|
+
}
|
|
1124
|
+
}
|
|
1125
|
+
|
|
1126
|
+
const registry = this.registry
|
|
1127
|
+
const batch = options.batch_id || `batch-${crypto.randomUUID()}`
|
|
1128
|
+
const summary = { converted: 0, bytes_saved: 0, locked: 0, stale: 0, incompatible: 0, unavailable: 0, failed: 0 }
|
|
1129
|
+
const staleHashes = new Set()
|
|
1130
|
+
const entries = [...registry.duplicates]
|
|
1131
|
+
const matchesScope = (currentSource) => currentSource && currentSource.shareable &&
|
|
1132
|
+
(includeDescendants ? this.sourceIsWithinScope(currentSource, scopeId) : currentSource.id === scopeId)
|
|
1133
|
+
if (options.progress) {
|
|
1134
|
+
options.progress.files_total = entries.reduce((total, [filePath, entry]) => {
|
|
1135
|
+
const currentSource = this.sourceForPath(filePath, entry.source_id)
|
|
1136
|
+
return total + (matchesScope(currentSource) ? 1 : 0)
|
|
1137
|
+
}, 0)
|
|
1138
|
+
}
|
|
1139
|
+
const completeProgress = () => {
|
|
1140
|
+
if (options.progress) options.progress.files_completed += 1
|
|
1141
|
+
}
|
|
1142
|
+
for (const [filePath, entry] of entries) {
|
|
1143
|
+
const currentSource = this.sourceForPath(filePath, entry.source_id)
|
|
1144
|
+
if (!matchesScope(currentSource)) continue
|
|
1145
|
+
if (!await this.canonicalPathIsWithinSource(filePath, currentSource)) {
|
|
1146
|
+
summary.stale += 1
|
|
1147
|
+
completeProgress()
|
|
1148
|
+
continue
|
|
1308
1149
|
}
|
|
1309
|
-
const
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1150
|
+
const indexed = registry.scanIndex.get(filePath)
|
|
1151
|
+
const expected = indexed && indexed.hash === entry.hash
|
|
1152
|
+
? indexed
|
|
1153
|
+
: (entry.dev !== undefined && entry.ino !== undefined &&
|
|
1154
|
+
entry.mtime !== undefined && entry.ctime !== undefined ? entry : null)
|
|
1155
|
+
if (!expected) {
|
|
1156
|
+
summary.stale += 1
|
|
1157
|
+
completeProgress()
|
|
1158
|
+
continue
|
|
1159
|
+
}
|
|
1160
|
+
if (staleHashes.has(entry.hash)) {
|
|
1161
|
+
summary.stale += 1
|
|
1162
|
+
completeProgress()
|
|
1163
|
+
continue
|
|
1164
|
+
}
|
|
1165
|
+
try {
|
|
1166
|
+
const result = await this.convert(filePath, entry.hash, {
|
|
1167
|
+
app: currentSource.kind === "app" ? currentSource.app : (entry.app || null),
|
|
1168
|
+
source_id: currentSource.id,
|
|
1169
|
+
batch_id: batch,
|
|
1170
|
+
source: currentSource,
|
|
1171
|
+
expected
|
|
1172
|
+
})
|
|
1173
|
+
if (result.status === "converted" || result.status === "already") {
|
|
1174
|
+
summary.converted += 1
|
|
1175
|
+
summary.bytes_saved += result.bytes_saved || 0
|
|
1176
|
+
} else if (result.status === "locked") {
|
|
1177
|
+
summary.locked += 1
|
|
1178
|
+
} else if (result.status === "stale") {
|
|
1179
|
+
summary.stale += 1
|
|
1180
|
+
} else if (result.status === "stale-blob") {
|
|
1181
|
+
staleHashes.add(entry.hash)
|
|
1182
|
+
summary.stale += 1
|
|
1183
|
+
} else if (result.status === "metadata-mismatch") {
|
|
1184
|
+
entry.unavailable_reason = "metadata"
|
|
1185
|
+
summary.incompatible += 1
|
|
1186
|
+
} else if (result.status === "unavailable" || result.status === "copy-mode") {
|
|
1187
|
+
summary.unavailable += 1
|
|
1188
|
+
} else {
|
|
1189
|
+
summary.failed += 1
|
|
1190
|
+
}
|
|
1191
|
+
} catch (error) {
|
|
1192
|
+
summary.failed += 1
|
|
1313
1193
|
}
|
|
1194
|
+
completeProgress()
|
|
1314
1195
|
}
|
|
1196
|
+
registry.schedulePersist()
|
|
1315
1197
|
return summary
|
|
1316
1198
|
}
|
|
1317
1199
|
// reclaim: delete an orphan's store name. Refuses when any app name remains.
|
|
@@ -1321,6 +1203,10 @@ class Vault {
|
|
|
1321
1203
|
if (!this.registry.blobs.has(hash)) return { status: "not-found" }
|
|
1322
1204
|
const st = await this.storeStatIfPresent(storePath)
|
|
1323
1205
|
if (!st) {
|
|
1206
|
+
const hasCopyNames = options.hasCopyNames === undefined
|
|
1207
|
+
? [...this.registry.links.values()].some((entry) => entry.hash === hash && entry.mode === "copy")
|
|
1208
|
+
: options.hasCopyNames
|
|
1209
|
+
if (hasCopyNames) return { status: "unavailable" }
|
|
1324
1210
|
if (!options.deferRegistry) this.registry.removeBlob(hash)
|
|
1325
1211
|
return options.deferRegistry ? { status: "gone", remove_hash: true } : { status: "gone" }
|
|
1326
1212
|
}
|
|
@@ -1345,41 +1231,26 @@ class Vault {
|
|
|
1345
1231
|
if (!this.enabled || !this.registry) return
|
|
1346
1232
|
const preservePath = options.preservePath || (() => false)
|
|
1347
1233
|
const includePath = options.includePath || (() => true)
|
|
1348
|
-
const
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
if (!isAccessError(error) || !options.onAccessError(error, filePath)) return false
|
|
1352
|
-
}
|
|
1353
|
-
if (entry) entry.unverified = true
|
|
1354
|
-
return true
|
|
1355
|
-
}
|
|
1234
|
+
const handleAccessError = (error, filePath) => !!(
|
|
1235
|
+
isAccessError(error) && options.onAccessError && options.onAccessError(error, filePath)
|
|
1236
|
+
)
|
|
1356
1237
|
if (options.reconcile !== false) this.reconcileConfiguredSources()
|
|
1357
1238
|
for (const [linkPath, entry] of [...this.registry.links]) {
|
|
1358
1239
|
if (!includePath(linkPath)) continue
|
|
1359
1240
|
if (preservePath(linkPath)) continue
|
|
1360
1241
|
try {
|
|
1361
1242
|
const source = this.sourceForPath(linkPath, entry.source_id)
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
continue
|
|
1365
|
-
}
|
|
1366
|
-
if (!source) {
|
|
1243
|
+
const managedPath = source && await this.canonicalPathIsWithinSource(linkPath, source, { strictErrors: true })
|
|
1244
|
+
if (!managedPath) {
|
|
1367
1245
|
this.registry.untrack(linkPath)
|
|
1368
1246
|
continue
|
|
1369
1247
|
}
|
|
1370
1248
|
const st = await lstatIfPresent(linkPath)
|
|
1371
|
-
if (!st) {
|
|
1249
|
+
if (!st || !st.isFile() || st.ino !== entry.ino || st.dev !== entry.dev) {
|
|
1372
1250
|
this.registry.untrack(linkPath)
|
|
1373
|
-
continue
|
|
1374
1251
|
}
|
|
1375
|
-
const managedPath = await this.canonicalPathIsWithinSource(
|
|
1376
|
-
linkPath, source, { strictErrors: true })
|
|
1377
|
-
const expected = this.registry.scanIndex.get(linkPath)
|
|
1378
|
-
entry.unverified = !managedPath || !st.isFile() ||
|
|
1379
|
-
st.ino !== entry.ino || st.dev !== entry.dev ||
|
|
1380
|
-
!expected || expected.hash !== entry.hash || !sameSnapshot(expected, st)
|
|
1381
1252
|
} catch (error) {
|
|
1382
|
-
if (!
|
|
1253
|
+
if (!handleAccessError(error, linkPath)) throw error
|
|
1383
1254
|
}
|
|
1384
1255
|
}
|
|
1385
1256
|
for (const [dupPath, entry] of [...this.registry.duplicates]) {
|
|
@@ -1388,28 +1259,17 @@ class Vault {
|
|
|
1388
1259
|
let valid = false
|
|
1389
1260
|
try {
|
|
1390
1261
|
const source = this.sourceForPath(dupPath, entry.source_id)
|
|
1391
|
-
if (source && source.available === false) {
|
|
1392
|
-
entry.unverified = true
|
|
1393
|
-
continue
|
|
1394
|
-
}
|
|
1395
1262
|
const st = await lstatIfPresent(dupPath)
|
|
1396
|
-
|
|
1397
|
-
this.
|
|
1398
|
-
continue
|
|
1399
|
-
}
|
|
1400
|
-
const expected = this.registry.scanIndex.get(dupPath)
|
|
1401
|
-
valid = !!(source && st.isFile() &&
|
|
1402
|
-
await this.canonicalPathIsWithinSource(dupPath, source, { strictErrors: true }) &&
|
|
1403
|
-
expected && expected.hash === entry.hash && sameSnapshot(expected, st))
|
|
1263
|
+
valid = !!(source && st && st.isFile() &&
|
|
1264
|
+
await this.canonicalPathIsWithinSource(dupPath, source, { strictErrors: true }))
|
|
1404
1265
|
} catch (error) {
|
|
1405
|
-
if (
|
|
1266
|
+
if (handleAccessError(error, dupPath)) continue
|
|
1406
1267
|
throw error
|
|
1407
1268
|
}
|
|
1408
1269
|
if (!valid) {
|
|
1409
|
-
|
|
1270
|
+
this.registry.untrack(dupPath)
|
|
1410
1271
|
continue
|
|
1411
1272
|
}
|
|
1412
|
-
entry.unverified = false
|
|
1413
1273
|
const storeStat = await this.storeStatIfPresent(this.storePathFor(entry.hash))
|
|
1414
1274
|
if (storeStat && storeStat.isFile()) {
|
|
1415
1275
|
await unlinkIfSame(dupPath + TMP_SUFFIX, storeStat)
|
|
@@ -1430,6 +1290,7 @@ class Vault {
|
|
|
1430
1290
|
const st = await this.storeStatIfPresent(storePath)
|
|
1431
1291
|
if (!st) {
|
|
1432
1292
|
const names = linksByHash.get(hash) || []
|
|
1293
|
+
const copyNames = names.filter(([, entry]) => entry.mode === "copy")
|
|
1433
1294
|
// Deleting a name changes ctime, so trusted re-adoption compares the
|
|
1434
1295
|
// preserved identity, size, and mtime. Changed or legacy-unsnapshotted
|
|
1435
1296
|
// content waits for the next explicit scan instead of being assigned a
|
|
@@ -1437,6 +1298,7 @@ class Vault {
|
|
|
1437
1298
|
let readopted = false
|
|
1438
1299
|
let inaccessibleName = false
|
|
1439
1300
|
for (const [linkPath, entry] of names) {
|
|
1301
|
+
if (entry.mode !== "link") continue
|
|
1440
1302
|
if (preservePath(linkPath)) {
|
|
1441
1303
|
inaccessibleName = true
|
|
1442
1304
|
continue
|
|
@@ -1448,7 +1310,7 @@ class Vault {
|
|
|
1448
1310
|
if (!source || !await this.canonicalPathIsWithinSource(linkPath, source, { strictErrors: true })) continue
|
|
1449
1311
|
linkStat = await lstatIfPresent(linkPath)
|
|
1450
1312
|
} catch (error) {
|
|
1451
|
-
if (!
|
|
1313
|
+
if (!handleAccessError(error, linkPath)) throw error
|
|
1452
1314
|
inaccessibleName = true
|
|
1453
1315
|
continue
|
|
1454
1316
|
}
|
|
@@ -1486,7 +1348,10 @@ class Vault {
|
|
|
1486
1348
|
readopted = true
|
|
1487
1349
|
break
|
|
1488
1350
|
}
|
|
1489
|
-
|
|
1351
|
+
// On a volume without file sharing support, copy-mode names are the
|
|
1352
|
+
// durable content group. They remain useful for duplicate discovery
|
|
1353
|
+
// even though there is no canonical file in the vault tree.
|
|
1354
|
+
if (readopted || copyNames.length || inaccessibleName) {
|
|
1490
1355
|
blob.orphan = false
|
|
1491
1356
|
blob.verified_at = inaccessibleName && !readopted ? null : Date.now()
|
|
1492
1357
|
} else {
|
|
@@ -1516,14 +1381,9 @@ class Vault {
|
|
|
1516
1381
|
blobs: new Map(registry.blobs),
|
|
1517
1382
|
links: new Map(registry.links),
|
|
1518
1383
|
excluded: new Map(registry.excluded),
|
|
1384
|
+
totals: Object.assign({}, registry.totals),
|
|
1519
1385
|
lastScan: registry.lastScan ? Object.assign({}, registry.lastScan) : null,
|
|
1520
|
-
lastScanAttempt: registry.lastScanAttempt
|
|
1521
|
-
? Object.assign({}, registry.lastScanAttempt)
|
|
1522
|
-
: null,
|
|
1523
1386
|
sourceScans: new Map([...registry.sourceScans].map(([id, scan]) => [id, Object.assign({}, scan)])),
|
|
1524
|
-
sourceScanAttempts: new Map([...registry.sourceScanAttempts]
|
|
1525
|
-
.map(([id, scan]) => [id, Object.assign({}, scan)])),
|
|
1526
|
-
settings: Object.assign({}, registry.settings),
|
|
1527
1387
|
duplicates: new Map(registry.duplicates),
|
|
1528
1388
|
scanIndex: new Map(registry.scanIndex)
|
|
1529
1389
|
} : null
|
|
@@ -1571,18 +1431,7 @@ class Vault {
|
|
|
1571
1431
|
await this.refreshSources()
|
|
1572
1432
|
const walkRoots = roots
|
|
1573
1433
|
? roots.map((root) => typeof root === "string" ? { root, source_id: null } : root)
|
|
1574
|
-
:
|
|
1575
|
-
{ root: path.resolve(this.kernel.homedir), source_id: "pinokio" },
|
|
1576
|
-
...this._sources
|
|
1577
|
-
.filter((source) => source.kind === "external" && source.root)
|
|
1578
|
-
.map((source) => ({ root: source.root, source_id: source.id }))
|
|
1579
|
-
]
|
|
1580
|
-
for (const entry of walkRoots) {
|
|
1581
|
-
const rootStat = await fs.promises.lstat(entry.root)
|
|
1582
|
-
if (!rootStat.isDirectory() || rootStat.isSymbolicLink()) {
|
|
1583
|
-
throw new Error(`Repair source is not a real directory: ${entry.root}`)
|
|
1584
|
-
}
|
|
1585
|
-
}
|
|
1434
|
+
: this.scanRoots()
|
|
1586
1435
|
for (const entry of walkRoots) {
|
|
1587
1436
|
await this.walkForInoMatch(
|
|
1588
1437
|
entry.root,
|
|
@@ -1595,6 +1444,38 @@ class Vault {
|
|
|
1595
1444
|
const rebuiltScanIndex = new Map()
|
|
1596
1445
|
const trustedHashes = new Set()
|
|
1597
1446
|
if (preserved) {
|
|
1447
|
+
// Copy-mode content has no store inode to rediscover. Preserve only names
|
|
1448
|
+
// whose exact scan snapshot still proves the recorded content group.
|
|
1449
|
+
for (const [linkPath, link] of preserved.links) {
|
|
1450
|
+
if (link.mode !== "copy" || preserved.excluded.has(linkPath)) continue
|
|
1451
|
+
const blob = preserved.blobs.get(link.hash)
|
|
1452
|
+
const expected = preserved.scanIndex.get(linkPath)
|
|
1453
|
+
const source = this.sourceForPath(linkPath, link.source_id)
|
|
1454
|
+
if (!blob || !expected || expected.hash !== link.hash || !source ||
|
|
1455
|
+
!await this.canonicalPathIsWithinSource(linkPath, source, { strictErrors: true })) continue
|
|
1456
|
+
try {
|
|
1457
|
+
const st = await fs.promises.lstat(linkPath)
|
|
1458
|
+
if (!st.isFile() || st.size < this.sizeThreshold ||
|
|
1459
|
+
link.dev !== st.dev || link.ino !== st.ino ||
|
|
1460
|
+
!sameSnapshot(expected, st)) continue
|
|
1461
|
+
if (!rebuiltBlobs.has(link.hash)) {
|
|
1462
|
+
rebuiltBlobs.set(link.hash, Object.assign({}, blob, {
|
|
1463
|
+
size: st.size,
|
|
1464
|
+
verified_at: Date.now(),
|
|
1465
|
+
orphan: false
|
|
1466
|
+
}))
|
|
1467
|
+
}
|
|
1468
|
+
rebuiltLinks.set(linkPath, Object.assign({}, link, {
|
|
1469
|
+
app: source.kind === "app" ? source.app : (link.app || null),
|
|
1470
|
+
source_id: source.id,
|
|
1471
|
+
dev: st.dev,
|
|
1472
|
+
ino: st.ino,
|
|
1473
|
+
mode: "copy"
|
|
1474
|
+
}))
|
|
1475
|
+
} catch (error) {
|
|
1476
|
+
if (!isMissingError(error)) throw error
|
|
1477
|
+
}
|
|
1478
|
+
}
|
|
1598
1479
|
// A store filename is only a trustworthy content hash when at least one
|
|
1599
1480
|
// surviving name still matches the snapshot captured when that content
|
|
1600
1481
|
// was hashed. Repair itself never hashes, so unverified names are left
|
|
@@ -1666,10 +1547,8 @@ class Vault {
|
|
|
1666
1547
|
duplicates: rebuiltDuplicates,
|
|
1667
1548
|
excluded: preserved ? preserved.excluded : new Map(),
|
|
1668
1549
|
lastScan: preserved ? preserved.lastScan : null,
|
|
1669
|
-
lastScanAttempt: preserved ? preserved.lastScanAttempt : null,
|
|
1670
1550
|
sourceScans: preserved ? preserved.sourceScans : new Map(),
|
|
1671
|
-
|
|
1672
|
-
settings: preserved ? preserved.settings : {},
|
|
1551
|
+
totals: preserved ? preserved.totals : { lifetime_bytes_saved: 0 },
|
|
1673
1552
|
byIno: rebuiltByIno,
|
|
1674
1553
|
pathsByIno: rebuiltPathsByIno
|
|
1675
1554
|
})
|
|
@@ -1680,8 +1559,7 @@ class Vault {
|
|
|
1680
1559
|
for await (const batch of walkBatches(root, {
|
|
1681
1560
|
concurrency: this.dirConcurrency,
|
|
1682
1561
|
skipDirectory: (full) => full === vaultRoot,
|
|
1683
|
-
strictErrors: true
|
|
1684
|
-
onRootMissing: (error) => { throw error }
|
|
1562
|
+
strictErrors: true
|
|
1685
1563
|
})) {
|
|
1686
1564
|
const files = batch.flatMap((group) => group.files.map((file) => file.path))
|
|
1687
1565
|
const stats = await statMany(files, this.statConcurrency, null, {
|
|
@@ -1690,19 +1568,19 @@ class Vault {
|
|
|
1690
1568
|
})
|
|
1691
1569
|
for (let index = 0; index < files.length; index++) {
|
|
1692
1570
|
const st = stats[index]
|
|
1693
|
-
if (!st || !st.isFile()) continue
|
|
1571
|
+
if (!st || !st.isFile() || st.size < this.sizeThreshold) continue
|
|
1694
1572
|
const hash = storeInoToHash.get(`${st.dev}:${st.ino}`)
|
|
1695
1573
|
if (hash) {
|
|
1696
1574
|
const source = this.sourceForPath(files[index], preferredSourceId)
|
|
1697
1575
|
const previous = this.registry.links.get(files[index])
|
|
1698
|
-
const batchId = previous && previous.hash === hash &&
|
|
1576
|
+
const batchId = previous && previous.hash === hash && previous.mode === "link" &&
|
|
1699
1577
|
previous.dev === st.dev && previous.ino === st.ino
|
|
1700
1578
|
? previous.batch_id || null
|
|
1701
1579
|
: null
|
|
1702
1580
|
const link = {
|
|
1703
1581
|
hash, app: source && source.kind === "app" ? source.app : null,
|
|
1704
1582
|
source_id: source ? source.id : null, dev: st.dev, ino: st.ino,
|
|
1705
|
-
created: Date.now(),
|
|
1583
|
+
mode: "link", created: Date.now(),
|
|
1706
1584
|
batch_id: batchId
|
|
1707
1585
|
}
|
|
1708
1586
|
if (outputLinks) outputLinks.set(files[index], link)
|
|
@@ -1722,7 +1600,6 @@ class Vault {
|
|
|
1722
1600
|
// memory + one stat per blob and occupied shard; never a walk.
|
|
1723
1601
|
async status(scopeId = null) {
|
|
1724
1602
|
if (!this.enabled || !this.registry) return { enabled: false }
|
|
1725
|
-
const responseActionSequence = this.actionSequence
|
|
1726
1603
|
const registry = this.registry
|
|
1727
1604
|
const scope = scopeId ? this.scanSource(scopeId) : null
|
|
1728
1605
|
if (scopeId && !scope) throw new Error("That location is no longer available.")
|
|
@@ -1745,8 +1622,7 @@ class Vault {
|
|
|
1745
1622
|
if (!namesByHash.has(entry.hash)) namesByHash.set(entry.hash, [])
|
|
1746
1623
|
const location = this.locationForPath(linkPath, entry.source_id)
|
|
1747
1624
|
namesByHash.get(entry.hash).push(Object.assign({
|
|
1748
|
-
path: linkPath, app: entry.app || null, mode:
|
|
1749
|
-
unverified: !!entry.unverified
|
|
1625
|
+
path: linkPath, app: entry.app || null, mode: entry.mode
|
|
1750
1626
|
}, location))
|
|
1751
1627
|
if (entry.batch_id && (!scopeId || entry.source_id === scopeId)) {
|
|
1752
1628
|
if (!undoBatchMap.has(entry.batch_id)) {
|
|
@@ -1758,6 +1634,11 @@ class Vault {
|
|
|
1758
1634
|
batch.bytes += blob && Number.isFinite(blob.size) ? blob.size : 0
|
|
1759
1635
|
}
|
|
1760
1636
|
}
|
|
1637
|
+
const pendingBytesByHash = new Map()
|
|
1638
|
+
for (const entry of registry.duplicates.values()) {
|
|
1639
|
+
if (scopeId && entry.source_id !== scopeId) continue
|
|
1640
|
+
pendingBytesByHash.set(entry.hash, (pendingBytesByHash.get(entry.hash) || 0) + (entry.size || 0))
|
|
1641
|
+
}
|
|
1761
1642
|
const blobEntries = [...registry.blobs].filter(([hash]) => !scopeHashes || scopeHashes.has(hash))
|
|
1762
1643
|
if (!await this.directoryIfSafe(this.root) || !await this.directoryIfSafe(this.blobRoot)) {
|
|
1763
1644
|
throw unsafeStoragePath(this.blobRoot)
|
|
@@ -1771,7 +1652,8 @@ class Vault {
|
|
|
1771
1652
|
null,
|
|
1772
1653
|
{ strictErrors: true, followSymlinks: false }
|
|
1773
1654
|
)
|
|
1774
|
-
let
|
|
1655
|
+
let bytesOnDisk = 0
|
|
1656
|
+
let wouldBe = 0
|
|
1775
1657
|
for (let index = 0; index < blobEntries.length; index++) {
|
|
1776
1658
|
const [hash, blob] = blobEntries[index]
|
|
1777
1659
|
const names = namesByHash.get(hash) || []
|
|
@@ -1780,17 +1662,14 @@ class Vault {
|
|
|
1780
1662
|
const nlink = storeStat ? storeStat.nlink : null
|
|
1781
1663
|
storeStats.set(hash, storeStat)
|
|
1782
1664
|
const size = blob.size || 0
|
|
1783
|
-
const
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
1788
|
-
}
|
|
1789
|
-
}
|
|
1665
|
+
const linkedCopy = storeStat || names.some((name) => name.mode === "link") ? 1 : 0
|
|
1666
|
+
const physicalCopies = linkedCopy + names.filter((name) => name.mode === "copy").length
|
|
1667
|
+
const pendingBytes = pendingBytesByHash.get(hash) || 0
|
|
1668
|
+
bytesOnDisk += (size * physicalCopies) + pendingBytes
|
|
1669
|
+
wouldBe += (size * Math.max(physicalCopies, names.length)) + pendingBytes
|
|
1790
1670
|
const orphan = !!(storeStat && storeStat.nlink === 1)
|
|
1791
1671
|
const publicBlob = {
|
|
1792
1672
|
hash, size: blob.size || 0, orphan, nlink,
|
|
1793
|
-
sharing_locations: liveNames,
|
|
1794
1673
|
names, apps: [...apps], source_urls: blob.source_urls || []
|
|
1795
1674
|
}
|
|
1796
1675
|
blobs.push(publicBlob)
|
|
@@ -1803,8 +1682,7 @@ class Vault {
|
|
|
1803
1682
|
const source = this.sourceForPath(p, location.source_id)
|
|
1804
1683
|
const index = registry.scanIndex.get(p)
|
|
1805
1684
|
const storeStat = storeStats.get(entry.hash)
|
|
1806
|
-
const shareable = !!(source && source.shareable && storeStat && !entry.
|
|
1807
|
-
!entry.unavailable_reason &&
|
|
1685
|
+
const shareable = !!(source && source.shareable && storeStat && !entry.unavailable_reason &&
|
|
1808
1686
|
(!index || index.dev === undefined || index.dev === storeStat.dev))
|
|
1809
1687
|
let unavailableReason = entry.unavailable_reason || null
|
|
1810
1688
|
if (!shareable && !unavailableReason) {
|
|
@@ -1816,8 +1694,7 @@ class Vault {
|
|
|
1816
1694
|
const match = blob ? blob.names.find((name) => name.path !== p) || null : null
|
|
1817
1695
|
duplicates.push(Object.assign({
|
|
1818
1696
|
path: p, hash: entry.hash, size: entry.size || 0, app: entry.app || null,
|
|
1819
|
-
shareable,
|
|
1820
|
-
unavailable_reason: entry.unverified ? "unavailable" : unavailableReason, match
|
|
1697
|
+
shareable, unavailable_reason: unavailableReason, match
|
|
1821
1698
|
}, location))
|
|
1822
1699
|
}
|
|
1823
1700
|
const events = (await registry.readEvents()).reverse()
|
|
@@ -1850,64 +1727,64 @@ class Vault {
|
|
|
1850
1727
|
excluded.push(Object.assign({
|
|
1851
1728
|
path: p,
|
|
1852
1729
|
ts: meta.ts || null,
|
|
1853
|
-
size: Number(meta.size) || 0
|
|
1854
|
-
unverified: !!meta.unverified
|
|
1730
|
+
size: Number(meta.size) || 0
|
|
1855
1731
|
}, this.locationForPath(p, meta.source_id)))
|
|
1856
1732
|
}
|
|
1857
1733
|
const publicSources = this._sources.filter((source) => !scopeId || source.id === scopeId).map((source) => ({
|
|
1858
1734
|
id: source.id, kind: source.kind, label: source.label, root: source.root,
|
|
1859
|
-
display_path: source.root,
|
|
1735
|
+
display_path: source.kind === "pinokio" ? source.root : (source.mount_path || source.root),
|
|
1860
1736
|
target_path: source.kind === "external" ? source.root : null,
|
|
1861
1737
|
parent_id: scopeId ? null : source.parent_id, app: source.app || null,
|
|
1862
1738
|
available: source.available !== false, shareable: source.kind === "virtual" ? null : !!source.shareable
|
|
1863
1739
|
}))
|
|
1864
|
-
const lastComplete = registry.scanFor(scopeId)
|
|
1865
|
-
const lastAttempt = registry.scanAttemptFor(scopeId)
|
|
1866
|
-
const inScope = (entry) => !scopeId || entry.source_id === scopeId
|
|
1867
|
-
const hasUnavailableSource = !scopeId && this._sources.some((source) =>
|
|
1868
|
-
source.kind === "external" && source.available === false)
|
|
1869
|
-
const hasUnverified = hasUnavailableSource || [...registry.links.values()].some((entry) =>
|
|
1870
|
-
inScope(entry) && entry.unverified) ||
|
|
1871
|
-
[...registry.duplicates.values()].some((entry) =>
|
|
1872
|
-
inScope(entry) && entry.unverified) ||
|
|
1873
|
-
[...registry.excluded.values()].some((entry) =>
|
|
1874
|
-
inScope(entry) && entry.unverified)
|
|
1875
|
-
const incomplete = hasUnverified || !!(
|
|
1876
|
-
lastAttempt && lastAttempt.complete === false &&
|
|
1877
|
-
(!lastComplete || (lastAttempt.ts || 0) >= (lastComplete.ts || 0))
|
|
1878
|
-
)
|
|
1879
|
-
const metricsExact = !!lastComplete && !incomplete
|
|
1880
|
-
const beforeBytes = metricsExact && Number.isFinite(lastComplete.bytes_total)
|
|
1881
|
-
? Math.max(0, lastComplete.bytes_total)
|
|
1882
|
-
: null
|
|
1883
|
-
const afterBytes = beforeBytes === null
|
|
1884
|
-
? null
|
|
1885
|
-
: Math.max(0, beforeBytes - savedBySharing)
|
|
1886
1740
|
const result = {
|
|
1887
1741
|
enabled: true,
|
|
1888
1742
|
mode: this.mode,
|
|
1889
|
-
candidate_min_bytes: this.sizeThreshold,
|
|
1890
|
-
candidate_size_options: CANDIDATE_SIZE_OPTIONS,
|
|
1891
1743
|
scan,
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
incomplete,
|
|
1898
|
-
metrics_exact: metricsExact,
|
|
1899
|
-
bytes_on_disk: afterBytes,
|
|
1900
|
-
bytes_without_sharing: beforeBytes,
|
|
1901
|
-
saved_by_sharing: savedBySharing,
|
|
1744
|
+
last_scan: registry.scanFor(scopeId),
|
|
1745
|
+
bytes_on_disk: bytesOnDisk,
|
|
1746
|
+
bytes_without_sharing: wouldBe,
|
|
1747
|
+
saved_by_sharing: wouldBe - bytesOnDisk,
|
|
1748
|
+
lifetime_bytes_saved: registry.totals.lifetime_bytes_saved,
|
|
1902
1749
|
reclaimable: blobs.filter((b) => b.orphan).reduce((sum, b) => sum + b.size, 0),
|
|
1903
1750
|
pending_bytes: duplicates.filter((d) => d.shareable).reduce((sum, d) => sum + d.size, 0),
|
|
1751
|
+
file_action: this.fileActionStatus(scopeId),
|
|
1904
1752
|
activity_error: registry.eventError,
|
|
1905
1753
|
cloud_sync_warning: this.cloudSyncProvider(),
|
|
1906
1754
|
sources: publicSources, blobs, duplicates, excluded, events,
|
|
1907
1755
|
undo_batches: undoBatches
|
|
1908
1756
|
}
|
|
1909
1757
|
if (scopeId) {
|
|
1758
|
+
const duplicateBytes = duplicates.reduce((sum, item) => sum + item.size, 0)
|
|
1759
|
+
const excludedBytes = excluded.reduce((sum, item) => sum + item.size, 0)
|
|
1760
|
+
let linkedBytes = 0
|
|
1761
|
+
let effectiveLinkedBytes = 0
|
|
1762
|
+
let sharedBytes = 0
|
|
1763
|
+
for (const blob of blobs) {
|
|
1764
|
+
const scopedNames = blob.names.filter((name) => name.source_id === scopeId)
|
|
1765
|
+
const scopedLinks = scopedNames.filter((name) => name.mode === "link").length
|
|
1766
|
+
const scopedCopies = scopedNames.length - scopedLinks
|
|
1767
|
+
const registeredLinks = blob.names.filter((name) => name.mode === "link").length
|
|
1768
|
+
// The store name is one hardlink but not a user-visible location.
|
|
1769
|
+
// Registry count prevents stale filesystem metadata from over-attributing the inode.
|
|
1770
|
+
const filesystemLinks = Number.isFinite(blob.nlink) ? Math.max(0, blob.nlink - 1) : 0
|
|
1771
|
+
const sharingLocations = Math.max(1, registeredLinks, filesystemLinks)
|
|
1772
|
+
linkedBytes += blob.size * scopedNames.length
|
|
1773
|
+
effectiveLinkedBytes += (blob.size * scopedCopies) +
|
|
1774
|
+
(blob.size * scopedLinks / sharingLocations)
|
|
1775
|
+
if (registeredLinks >= 2) sharedBytes += blob.size * scopedLinks
|
|
1776
|
+
}
|
|
1777
|
+
const trackedBytes = linkedBytes + duplicateBytes + excludedBytes
|
|
1778
|
+
const effectiveTrackedBytes = effectiveLinkedBytes + duplicateBytes + excludedBytes
|
|
1779
|
+
const folderBytes = result.last_scan && Number.isFinite(result.last_scan.bytes_total)
|
|
1780
|
+
? result.last_scan.bytes_total
|
|
1781
|
+
: null
|
|
1910
1782
|
result.scope_id = scopeId
|
|
1783
|
+
result.tracked_bytes = trackedBytes
|
|
1784
|
+
result.effective_bytes = folderBytes === null
|
|
1785
|
+
? null
|
|
1786
|
+
: Math.max(0, folderBytes - trackedBytes) + effectiveTrackedBytes
|
|
1787
|
+
result.shared_bytes = sharedBytes
|
|
1911
1788
|
result.reclaimable = 0
|
|
1912
1789
|
}
|
|
1913
1790
|
return result
|
|
@@ -1922,176 +1799,34 @@ class Vault {
|
|
|
1922
1799
|
return Object.assign({}, state, {
|
|
1923
1800
|
current_file: sweeper.currentHash ? path.basename(sweeper.currentHash.path) : null,
|
|
1924
1801
|
pending,
|
|
1925
|
-
error: this.scanError
|
|
1926
|
-
persistence_warning: this.scanPersistenceWarning &&
|
|
1927
|
-
!!(this.registry && this.registry.persistDirty)
|
|
1802
|
+
error: this.scanError
|
|
1928
1803
|
})
|
|
1929
1804
|
}
|
|
1805
|
+
fileActionStatus(scopeId = null) {
|
|
1806
|
+
const progress = this.fileActionProgress
|
|
1807
|
+
if (!progress) return null
|
|
1808
|
+
if (scopeId) {
|
|
1809
|
+
if (progress.scope_id && progress.scope_id !== scopeId) return null
|
|
1810
|
+
if (progress.path) {
|
|
1811
|
+
const source = this.sourceForPath(progress.path)
|
|
1812
|
+
if (!source || source.id !== scopeId) return null
|
|
1813
|
+
}
|
|
1814
|
+
}
|
|
1815
|
+
return Object.assign({}, progress)
|
|
1816
|
+
}
|
|
1930
1817
|
progressStatus(scopeId = null) {
|
|
1931
1818
|
return {
|
|
1932
1819
|
enabled: !!this.enabled,
|
|
1933
|
-
candidate_min_bytes: this.sizeThreshold,
|
|
1934
|
-
candidate_size_options: CANDIDATE_SIZE_OPTIONS,
|
|
1935
1820
|
scan: this.scanStatus(),
|
|
1936
1821
|
file_action: this.fileActionStatus(scopeId),
|
|
1937
|
-
|
|
1938
|
-
action_sequence: this.actionSequence,
|
|
1939
|
-
last_scan: this.registry ? this.registry.scanFor(scopeId) : null,
|
|
1940
|
-
last_attempt: this.registry ? this.registry.scanAttemptFor(scopeId) : null
|
|
1941
|
-
}
|
|
1942
|
-
}
|
|
1943
|
-
fileActionStatus(scopeId = null) {
|
|
1944
|
-
const action = this.fileAction
|
|
1945
|
-
if (!action || (scopeId && action.source_id !== scopeId)) return null
|
|
1946
|
-
return {
|
|
1947
|
-
id: action.id,
|
|
1948
|
-
kind: action.kind,
|
|
1949
|
-
phase: action.phase,
|
|
1950
|
-
path: action.path,
|
|
1951
|
-
source_id: action.source_id,
|
|
1952
|
-
bytes_copied: action.bytes_copied,
|
|
1953
|
-
bytes_total: action.bytes_total,
|
|
1954
|
-
files_completed: action.files_completed,
|
|
1955
|
-
files_total: action.files_total
|
|
1956
|
-
}
|
|
1957
|
-
}
|
|
1958
|
-
fileActionStatuses(scopeId = null) {
|
|
1959
|
-
return [...this.fileActions.values()]
|
|
1960
|
-
.filter((action) => !scopeId || action.source_id === scopeId)
|
|
1961
|
-
.map((action) => ({
|
|
1962
|
-
id: action.id,
|
|
1963
|
-
kind: action.kind,
|
|
1964
|
-
phase: action.phase,
|
|
1965
|
-
path: action.path,
|
|
1966
|
-
source_id: action.source_id,
|
|
1967
|
-
bytes_copied: action.bytes_copied,
|
|
1968
|
-
bytes_total: action.bytes_total,
|
|
1969
|
-
files_completed: action.files_completed,
|
|
1970
|
-
files_total: action.files_total
|
|
1971
|
-
}))
|
|
1972
|
-
}
|
|
1973
|
-
navigationStatus(scopeId = null) {
|
|
1974
|
-
if (!this.enabled || !this.initialized || !this.registry) {
|
|
1975
|
-
return {
|
|
1976
|
-
any_scan: false,
|
|
1977
|
-
scanned: false,
|
|
1978
|
-
scanning: false,
|
|
1979
|
-
pending_bytes: 0
|
|
1980
|
-
}
|
|
1981
|
-
}
|
|
1982
|
-
const scan = this.scanStatus()
|
|
1983
|
-
const active = !!(scan && (scan.pending || scan.active || scan.queued > 0))
|
|
1984
|
-
const sources = new Map(this._sources.map((source) => [source.id, source]))
|
|
1985
|
-
let pendingBytes = 0
|
|
1986
|
-
for (const entry of this.registry.duplicates.values()) {
|
|
1987
|
-
if (scopeId && entry.source_id !== scopeId) continue
|
|
1988
|
-
const source = sources.get(entry.source_id)
|
|
1989
|
-
if (!source || source.available === false || !source.shareable ||
|
|
1990
|
-
entry.unverified || entry.unavailable_reason ||
|
|
1991
|
-
!this.registry.blobs.has(entry.hash)) continue
|
|
1992
|
-
pendingBytes += Math.max(0, Number(entry.size) || 0)
|
|
1993
|
-
}
|
|
1994
|
-
const lastComplete = this.registry.scanFor(scopeId)
|
|
1995
|
-
const lastAttempt = this.registry.scanAttemptFor(scopeId)
|
|
1996
|
-
const hasUnavailableSource = !scopeId && this._sources.some((source) =>
|
|
1997
|
-
source.kind === "external" && source.available === false)
|
|
1998
|
-
const hasUnverified = hasUnavailableSource || [...this.registry.links.values()].some((entry) =>
|
|
1999
|
-
(!scopeId || entry.source_id === scopeId) && entry.unverified) ||
|
|
2000
|
-
[...this.registry.duplicates.values()].some((entry) =>
|
|
2001
|
-
(!scopeId || entry.source_id === scopeId) && entry.unverified) ||
|
|
2002
|
-
[...this.registry.excluded.values()].some((entry) =>
|
|
2003
|
-
(!scopeId || entry.source_id === scopeId) && entry.unverified)
|
|
2004
|
-
const incomplete = hasUnverified || !!(
|
|
2005
|
-
lastAttempt && lastAttempt.complete === false &&
|
|
2006
|
-
(!lastComplete || (lastAttempt.ts || 0) >= (lastComplete.ts || 0))
|
|
2007
|
-
)
|
|
2008
|
-
return {
|
|
2009
|
-
any_scan: !!this.registry.lastScanAttempt || this.registry.sourceScanAttempts.size > 0,
|
|
2010
|
-
scanned: !!lastAttempt,
|
|
2011
|
-
scanning: active && (!scopeId || scan.scope_id === scopeId),
|
|
2012
|
-
pending_bytes: pendingBytes,
|
|
2013
|
-
incomplete
|
|
2014
|
-
}
|
|
2015
|
-
}
|
|
2016
|
-
navigationStatusForApp(appName) {
|
|
2017
|
-
return this.navigationStatus(sourceId("app", appName))
|
|
2018
|
-
}
|
|
2019
|
-
async preflightCopyCapacity(targets) {
|
|
2020
|
-
if (!targets.length) return { ok: true }
|
|
2021
|
-
if (!fs.promises.statfs) return { ok: false, status: "capacity-unavailable" }
|
|
2022
|
-
const byDevice = new Map()
|
|
2023
|
-
try {
|
|
2024
|
-
for (const target of targets) {
|
|
2025
|
-
const st = await fs.promises.lstat(target.path)
|
|
2026
|
-
const key = String(st.dev)
|
|
2027
|
-
if (!byDevice.has(key)) {
|
|
2028
|
-
byDevice.set(key, { path: path.dirname(target.path), bytes: 0 })
|
|
2029
|
-
}
|
|
2030
|
-
byDevice.get(key).bytes += Math.max(0, Number(target.bytes) || 0)
|
|
2031
|
-
}
|
|
2032
|
-
for (const group of byDevice.values()) {
|
|
2033
|
-
const stat = await fs.promises.statfs(group.path)
|
|
2034
|
-
const blockSize = BigInt(stat.bsize)
|
|
2035
|
-
const availableBlocks = BigInt(stat.bavail)
|
|
2036
|
-
const available = blockSize * availableBlocks
|
|
2037
|
-
if (available < BigInt(Math.ceil(group.bytes))) {
|
|
2038
|
-
return { ok: false, status: "insufficient-space" }
|
|
2039
|
-
}
|
|
2040
|
-
}
|
|
2041
|
-
return { ok: true }
|
|
2042
|
-
} catch (error) {
|
|
2043
|
-
return { ok: false, status: "capacity-unavailable" }
|
|
1822
|
+
last_scan: this.registry ? this.registry.scanFor(scopeId) : null
|
|
2044
1823
|
}
|
|
2045
1824
|
}
|
|
2046
1825
|
async copyOut(filePath, sourcePath, expectedTarget, expectedSource = expectedTarget, options = {}) {
|
|
2047
1826
|
const tmp = filePath + TMP_SUFFIX
|
|
2048
1827
|
let copiedStat = null
|
|
2049
1828
|
try {
|
|
2050
|
-
|
|
2051
|
-
let sourceHandle = null
|
|
2052
|
-
let targetHandle = null
|
|
2053
|
-
try {
|
|
2054
|
-
sourceHandle = await fs.promises.open(sourcePath, "r")
|
|
2055
|
-
const sourceStat = await sourceHandle.stat()
|
|
2056
|
-
targetHandle = await fs.promises.open(tmp,
|
|
2057
|
-
fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL,
|
|
2058
|
-
sourceStat.mode & 0o7777)
|
|
2059
|
-
copiedStat = await targetHandle.stat()
|
|
2060
|
-
const buffer = Buffer.allocUnsafe(COPY_PROGRESS_CHUNK_SIZE)
|
|
2061
|
-
let offset = 0
|
|
2062
|
-
options.onProgress(0, sourceStat.size)
|
|
2063
|
-
while (offset < sourceStat.size) {
|
|
2064
|
-
const length = Math.min(buffer.length, sourceStat.size - offset)
|
|
2065
|
-
const { bytesRead } = await sourceHandle.read(buffer, 0, length, offset)
|
|
2066
|
-
if (!bytesRead) {
|
|
2067
|
-
const error = new Error("Source file ended while copying.")
|
|
2068
|
-
error.code = "EIO"
|
|
2069
|
-
throw error
|
|
2070
|
-
}
|
|
2071
|
-
let written = 0
|
|
2072
|
-
while (written < bytesRead) {
|
|
2073
|
-
const result = await targetHandle.write(
|
|
2074
|
-
buffer, written, bytesRead - written, offset + written)
|
|
2075
|
-
if (!result.bytesWritten) {
|
|
2076
|
-
const error = new Error("Could not write the independent copy.")
|
|
2077
|
-
error.code = "EIO"
|
|
2078
|
-
throw error
|
|
2079
|
-
}
|
|
2080
|
-
written += result.bytesWritten
|
|
2081
|
-
}
|
|
2082
|
-
offset += bytesRead
|
|
2083
|
-
options.onProgress(offset, sourceStat.size)
|
|
2084
|
-
}
|
|
2085
|
-
await targetHandle.chmod(sourceStat.mode & 0o7777)
|
|
2086
|
-
} finally {
|
|
2087
|
-
await Promise.all([
|
|
2088
|
-
sourceHandle && sourceHandle.close(),
|
|
2089
|
-
targetHandle && targetHandle.close()
|
|
2090
|
-
].filter(Boolean))
|
|
2091
|
-
}
|
|
2092
|
-
} else {
|
|
2093
|
-
await fs.promises.copyFile(sourcePath, tmp, fs.constants.COPYFILE_EXCL)
|
|
2094
|
-
}
|
|
1829
|
+
await fs.promises.copyFile(sourcePath, tmp, fs.constants.COPYFILE_EXCL)
|
|
2095
1830
|
copiedStat = await fs.promises.lstat(tmp)
|
|
2096
1831
|
if (options.canReplace && !options.canReplace()) {
|
|
2097
1832
|
await unlinkIfSame(tmp, copiedStat)
|
|
@@ -2110,12 +1845,9 @@ class Vault {
|
|
|
2110
1845
|
try {
|
|
2111
1846
|
finalStat = await lstatIfPresent(filePath)
|
|
2112
1847
|
} catch (error) {
|
|
2113
|
-
//
|
|
2114
|
-
//
|
|
2115
|
-
|
|
2116
|
-
const committedStat = copiedStat
|
|
2117
|
-
copiedStat = null
|
|
2118
|
-
return { status: "copied", stat: committedStat, unverified: true }
|
|
1848
|
+
// The independent copy was committed by rename(). Preserve that
|
|
1849
|
+
// completed action when only the post-commit metadata read failed.
|
|
1850
|
+
finalStat = copiedStat
|
|
2119
1851
|
}
|
|
2120
1852
|
if (!finalStat || finalStat.dev !== copiedStat.dev || finalStat.ino !== copiedStat.ino) {
|
|
2121
1853
|
return { status: "stale" }
|
|
@@ -2129,100 +1861,54 @@ class Vault {
|
|
|
2129
1861
|
throw error
|
|
2130
1862
|
}
|
|
2131
1863
|
}
|
|
2132
|
-
// detach:
|
|
2133
|
-
//
|
|
2134
|
-
|
|
1864
|
+
// detach: make one name an independent copy again ("go back"), and pin it
|
|
1865
|
+
// so future scans neither re-list nor re-share it. Works on linked names
|
|
1866
|
+
// (copies bytes out) and on pending duplicates (just ignores them).
|
|
1867
|
+
async detach(filePath) {
|
|
2135
1868
|
if (!this.enabled) return { status: "disabled" }
|
|
2136
1869
|
await this.refreshSources()
|
|
2137
1870
|
const entry = this.registry.links.get(filePath)
|
|
2138
1871
|
if (entry) {
|
|
2139
|
-
if (entry.unverified) return { status: "unavailable" }
|
|
2140
1872
|
const source = this.sourceForPath(filePath, entry.source_id)
|
|
2141
1873
|
if (!source || !await this.canonicalPathIsWithinSource(filePath, source)) {
|
|
2142
1874
|
return { status: "stale" }
|
|
2143
1875
|
}
|
|
2144
|
-
if (
|
|
2145
|
-
|
|
2146
|
-
|
|
2147
|
-
|
|
2148
|
-
|
|
2149
|
-
const action = queuedAction || {
|
|
2150
|
-
kind: "detach", path: filePath, source_id: entry.source_id || null,
|
|
2151
|
-
bytes_copied: 0, bytes_total: before.size
|
|
2152
|
-
}
|
|
2153
|
-
action.bytes_total = before.size
|
|
2154
|
-
const ownsAction = !queuedAction
|
|
2155
|
-
if (ownsAction) this.fileAction = action
|
|
2156
|
-
try {
|
|
2157
|
-
const copied = await this.copyOut(filePath, filePath, fileSnapshot(before), fileSnapshot(before), {
|
|
2158
|
-
canReplace: () => !this.sourceAppIsRunning(source),
|
|
2159
|
-
onProgress: (bytesCopied, bytesTotal) => {
|
|
2160
|
-
action.bytes_copied = bytesCopied
|
|
2161
|
-
action.bytes_total = bytesTotal
|
|
2162
|
-
}
|
|
2163
|
-
})
|
|
2164
|
-
if (copied.status !== "copied") return copied
|
|
2165
|
-
const st = copied.stat
|
|
2166
|
-
const baseEntry = {
|
|
2167
|
-
hash: entry.hash, size: st.size, source_id: entry.source_id || null,
|
|
2168
|
-
dev: st.dev, ino: st.ino
|
|
2169
|
-
}
|
|
2170
|
-
const scanEntry = copied.unverified ? null : Object.assign({}, baseEntry, {
|
|
2171
|
-
mtime: st.mtimeMs, ctime: st.ctimeMs
|
|
1876
|
+
if (entry.mode === "copy") {
|
|
1877
|
+
const st = await lstatIfPresent(filePath)
|
|
1878
|
+
if (!st || !st.isFile() || st.dev !== entry.dev || st.ino !== entry.ino) return { status: "stale" }
|
|
1879
|
+
this.registry.exclude(filePath, {
|
|
1880
|
+
ts: Date.now(), source_id: entry.source_id || null, size: st.size
|
|
2172
1881
|
})
|
|
2173
|
-
this.registry.setDuplicate(filePath, Object.assign({
|
|
2174
|
-
app: entry.app || null, discovered: Date.now(),
|
|
2175
|
-
unverified: !!copied.unverified
|
|
2176
|
-
}, scanEntry || baseEntry), scanEntry)
|
|
2177
|
-
if (copied.unverified) this.registry.scanIndex.delete(filePath)
|
|
2178
|
-
await this.refreshLinkSnapshots(entry.hash, entry.dev, entry.ino)
|
|
2179
1882
|
await this.recordEvent({
|
|
2180
|
-
kind: "
|
|
2181
|
-
app: entry.app || null, source_id: entry.source_id || null,
|
|
2182
|
-
size: st.size, bytes_saved: st.size
|
|
1883
|
+
kind: "skip", hash: entry.hash, path: filePath,
|
|
1884
|
+
app: entry.app || null, source_id: entry.source_id || null, size: st.size
|
|
2183
1885
|
})
|
|
2184
|
-
return { status: "
|
|
2185
|
-
} finally {
|
|
2186
|
-
if (ownsAction && this.fileAction === action) this.fileAction = null
|
|
1886
|
+
return { status: "ignored" }
|
|
2187
1887
|
}
|
|
2188
|
-
|
|
2189
|
-
|
|
2190
|
-
|
|
2191
|
-
|
|
2192
|
-
|
|
2193
|
-
|
|
2194
|
-
|
|
2195
|
-
|
|
2196
|
-
|
|
2197
|
-
|
|
2198
|
-
source_id:
|
|
2199
|
-
|
|
2200
|
-
}
|
|
2201
|
-
|
|
2202
|
-
|
|
2203
|
-
|
|
2204
|
-
path: filePath,
|
|
2205
|
-
|
|
2206
|
-
source_id: duplicate.source_id || null,
|
|
2207
|
-
size: duplicate.size || 0
|
|
2208
|
-
})
|
|
2209
|
-
return { status: "ignored" }
|
|
2210
|
-
}
|
|
2211
|
-
// reshare: un-pin a skipped path; the next scan may list it again.
|
|
2212
|
-
async reshare(filePath) {
|
|
2213
|
-
if (!this.enabled) return { status: "disabled" }
|
|
2214
|
-
const current = this.registry.excluded.get(filePath)
|
|
2215
|
-
if (current && current.unverified) return { status: "unavailable" }
|
|
2216
|
-
const excluded = this.registry.allowSharing(filePath)
|
|
2217
|
-
if (excluded) {
|
|
2218
|
-
await this.recordEvent({ kind: "reshare", path: filePath, source_id: excluded && excluded.source_id ? excluded.source_id : null })
|
|
2219
|
-
return { status: "resharable" }
|
|
1888
|
+
if (this.sourceAppIsRunning(source)) return { status: "locked" }
|
|
1889
|
+
const before = await lstatIfPresent(filePath)
|
|
1890
|
+
if (!before || !before.isFile() || before.dev !== entry.dev || before.ino !== entry.ino) return { status: "stale" }
|
|
1891
|
+
const copied = await this.copyOut(filePath, filePath, fileSnapshot(before), fileSnapshot(before), {
|
|
1892
|
+
canReplace: () => !this.sourceAppIsRunning(source)
|
|
1893
|
+
})
|
|
1894
|
+
if (copied.status !== "copied") return copied
|
|
1895
|
+
const st = copied.stat
|
|
1896
|
+
this.registry.exclude(filePath, { ts: Date.now(), source_id: entry.source_id || null, size: st.size })
|
|
1897
|
+
await this.refreshLinkSnapshots(entry.hash, entry.dev, entry.ino)
|
|
1898
|
+
await this.recordEvent({ kind: "detach", hash: entry.hash, path: filePath, app: entry.app || null, source_id: entry.source_id || null, size: st.size })
|
|
1899
|
+
return { status: "detached" }
|
|
1900
|
+
}
|
|
1901
|
+
if (this.registry.duplicates.has(filePath)) {
|
|
1902
|
+
const dup = this.registry.duplicates.get(filePath)
|
|
1903
|
+
this.registry.exclude(filePath, { ts: Date.now(), source_id: dup.source_id || null, size: dup.size || 0 })
|
|
1904
|
+
await this.recordEvent({ kind: "skip", hash: dup.hash, path: filePath, app: dup.app || null, source_id: dup.source_id || null, size: dup.size || 0 })
|
|
1905
|
+
return { status: "ignored" }
|
|
2220
1906
|
}
|
|
2221
1907
|
return { status: "not-found" }
|
|
2222
1908
|
}
|
|
2223
1909
|
// Undo a conversion batch: replace each converted name with an independent
|
|
2224
1910
|
// copy of the bytes (spec, UX contract surface 3).
|
|
2225
|
-
async undoBatch(batchId
|
|
1911
|
+
async undoBatch(batchId) {
|
|
2226
1912
|
if (!this.enabled || !batchId) return { undone: 0 }
|
|
2227
1913
|
await this.refreshSources()
|
|
2228
1914
|
const events = await this.registry.readEvents()
|
|
@@ -2234,16 +1920,11 @@ class Vault {
|
|
|
2234
1920
|
if (entry.batch_id === batchId) targets.add(filePath)
|
|
2235
1921
|
}
|
|
2236
1922
|
const summary = { undone: 0, bytes: 0, failed: 0 }
|
|
2237
|
-
const prepared = []
|
|
2238
1923
|
for (const filePath of targets) {
|
|
2239
1924
|
const event = conversionEvents.get(filePath) || {}
|
|
2240
1925
|
const entry = this.registry.links.get(filePath)
|
|
2241
1926
|
if (!entry || (entry.batch_id && entry.batch_id !== batchId) ||
|
|
2242
1927
|
(event.hash && entry.hash !== event.hash)) continue
|
|
2243
|
-
if (entry.unverified) {
|
|
2244
|
-
summary.failed += 1
|
|
2245
|
-
continue
|
|
2246
|
-
}
|
|
2247
1928
|
const storePath = this.storePathFor(entry.hash)
|
|
2248
1929
|
try {
|
|
2249
1930
|
const source = this.sourceForPath(filePath, entry.source_id || event.source_id)
|
|
@@ -2266,34 +1947,8 @@ class Vault {
|
|
|
2266
1947
|
summary.failed += 1
|
|
2267
1948
|
continue
|
|
2268
1949
|
}
|
|
2269
|
-
prepared.push({ filePath, event, entry, source, storePath, st, storeStat })
|
|
2270
|
-
} catch (e) {
|
|
2271
|
-
summary.failed += 1
|
|
2272
|
-
}
|
|
2273
|
-
}
|
|
2274
|
-
const capacity = await this.preflightCopyCapacity(prepared.map((target) => ({
|
|
2275
|
-
path: target.filePath,
|
|
2276
|
-
bytes: target.storeStat.size
|
|
2277
|
-
})))
|
|
2278
|
-
if (!capacity.ok) {
|
|
2279
|
-
return Object.assign(summary, { status: capacity.status })
|
|
2280
|
-
}
|
|
2281
|
-
const action = queuedAction || {
|
|
2282
|
-
kind: "undo", path: null, source_id: null,
|
|
2283
|
-
bytes_copied: 0, bytes_total: 0
|
|
2284
|
-
}
|
|
2285
|
-
action.bytes_total = prepared.reduce((sum, target) => sum + target.storeStat.size, 0)
|
|
2286
|
-
let completedBytes = 0
|
|
2287
|
-
for (const target of prepared) {
|
|
2288
|
-
const { filePath, event, entry, source, storePath, st, storeStat } = target
|
|
2289
|
-
try {
|
|
2290
|
-
action.path = filePath
|
|
2291
|
-
action.source_id = entry.source_id || event.source_id || null
|
|
2292
1950
|
const copied = await this.copyOut(filePath, storePath, fileSnapshot(st), fileSnapshot(storeStat), {
|
|
2293
|
-
canReplace: () => !this.sourceAppIsRunning(source)
|
|
2294
|
-
onProgress: (bytesCopied) => {
|
|
2295
|
-
action.bytes_copied = completedBytes + bytesCopied
|
|
2296
|
-
}
|
|
1951
|
+
canReplace: () => !this.sourceAppIsRunning(source)
|
|
2297
1952
|
})
|
|
2298
1953
|
if (copied.status !== "copied") {
|
|
2299
1954
|
summary.failed += 1
|
|
@@ -2304,18 +1959,19 @@ class Vault {
|
|
|
2304
1959
|
hash: entry.hash, size: storeStat.size, app: entry.app || null,
|
|
2305
1960
|
source_id: entry.source_id || event.source_id || null, discovered: Date.now(),
|
|
2306
1961
|
dev: independentStat.dev, ino: independentStat.ino,
|
|
2307
|
-
|
|
1962
|
+
mtime: independentStat.mtimeMs, ctime: independentStat.ctimeMs
|
|
2308
1963
|
}
|
|
2309
|
-
const scanEntry =
|
|
1964
|
+
const scanEntry = {
|
|
2310
1965
|
hash: entry.hash, size: independentStat.size,
|
|
2311
1966
|
dev: independentStat.dev, ino: independentStat.ino,
|
|
2312
1967
|
mtime: independentStat.mtimeMs, ctime: independentStat.ctimeMs,
|
|
2313
1968
|
source_id: entry.source_id || event.source_id || null
|
|
2314
1969
|
}
|
|
2315
1970
|
this.registry.setDuplicate(filePath, duplicateEntry, scanEntry)
|
|
2316
|
-
if (copied.unverified) this.registry.scanIndex.delete(filePath)
|
|
2317
1971
|
await this.refreshLinkSnapshots(entry.hash, entry.dev, entry.ino)
|
|
2318
1972
|
const bytesSaved = event.bytes_saved || storeStat.size
|
|
1973
|
+
this.registry.totals.lifetime_bytes_saved = Math.max(0,
|
|
1974
|
+
this.registry.totals.lifetime_bytes_saved - bytesSaved)
|
|
2319
1975
|
await this.recordEvent({
|
|
2320
1976
|
kind: "undo", hash: entry.hash, path: filePath,
|
|
2321
1977
|
source_id: entry.source_id || event.source_id || null, batch_id: batchId,
|
|
@@ -2323,8 +1979,6 @@ class Vault {
|
|
|
2323
1979
|
})
|
|
2324
1980
|
summary.undone += 1
|
|
2325
1981
|
summary.bytes += storeStat.size
|
|
2326
|
-
completedBytes += storeStat.size
|
|
2327
|
-
action.bytes_copied = completedBytes
|
|
2328
1982
|
} catch (e) {
|
|
2329
1983
|
summary.failed += 1
|
|
2330
1984
|
}
|
|
@@ -2336,10 +1990,14 @@ class Vault {
|
|
|
2336
1990
|
if (!this.enabled) return { reclaimed: 0, bytes_freed: 0 }
|
|
2337
1991
|
const summary = { reclaimed: 0, bytes_freed: 0, failed: 0 }
|
|
2338
1992
|
const removedHashes = new Set()
|
|
1993
|
+
const copyHashes = new Set([...this.registry.links.values()]
|
|
1994
|
+
.filter((entry) => entry.mode === "copy")
|
|
1995
|
+
.map((entry) => entry.hash))
|
|
2339
1996
|
try {
|
|
2340
1997
|
for (const [hash] of [...this.registry.blobs]) {
|
|
2341
1998
|
const result = await this.reclaim(hash, {
|
|
2342
|
-
deferRegistry: true
|
|
1999
|
+
deferRegistry: true,
|
|
2000
|
+
hasCopyNames: copyHashes.has(hash)
|
|
2343
2001
|
})
|
|
2344
2002
|
if (result.remove_hash) removedHashes.add(hash)
|
|
2345
2003
|
if (result.status === "reclaimed") {
|