pinokiod 8.0.42 → 8.0.44
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/kernel/vault/constants.js +1 -5
- package/kernel/vault/hash_worker.js +1 -7
- package/kernel/vault/index.js +515 -873
- package/kernel/vault/registry.js +35 -157
- package/kernel/vault/sweeper.js +106 -181
- package/kernel/vault/walker.js +1 -8
- package/package.json +1 -1
- package/server/index.js +3 -20
- package/server/public/storage-size.js +3 -1
- package/server/public/style.css +0 -13
- package/server/public/vault.css +60 -173
- package/server/public/vault.js +399 -638
- package/server/views/app.ejs +6 -85
- package/server/views/partials/main_sidebar.ejs +1 -4
- package/server/views/partials/vault_workspace.ejs +2 -5
- package/server/views/vault.ejs +1 -1
- package/server/views/vault_app.ejs +1 -1
- package/test/vault-engine.test.js +123 -212
- package/test/vault-sweep.test.js +163 -292
- package/test/vault-ui.test.js +695 -1015
- package/server/public/vault-nav.js +0 -96
package/kernel/vault/index.js
CHANGED
|
@@ -6,10 +6,7 @@ const Registry = require('./registry')
|
|
|
6
6
|
const Sweeper = require('./sweeper')
|
|
7
7
|
const { walkBatches, statMany } = require('./walker')
|
|
8
8
|
const { fileSnapshot, sameSnapshot, sameContentState } = require('./snapshot')
|
|
9
|
-
const {
|
|
10
|
-
SIZE_THRESHOLD, CANDIDATE_SIZE_OPTIONS, TMP_SUFFIX, SHA256_RE,
|
|
11
|
-
DIR_CONCURRENCY, STAT_CONCURRENCY
|
|
12
|
-
} = require('./constants')
|
|
9
|
+
const { SIZE_THRESHOLD, TMP_SUFFIX, SHA256_RE, DIR_CONCURRENCY, STAT_CONCURRENCY } = require('./constants')
|
|
13
10
|
|
|
14
11
|
// Shared model store engine (spec/requirements/shared-model-store.md).
|
|
15
12
|
// Store + registry + volume probe + hashing + adopt/convert/verify, plus the
|
|
@@ -19,7 +16,6 @@ const {
|
|
|
19
16
|
|
|
20
17
|
const NO_LINK_CODES = new Set(["EXDEV", "ENOTSUP", "ENOSYS"])
|
|
21
18
|
const LOCK_CODES = new Set(["EBUSY", "EPERM", "EACCES"])
|
|
22
|
-
const COPY_PROGRESS_CHUNK_SIZE = 8 * 1024 * 1024
|
|
23
19
|
const isMissingError = (error) => !!(error && (error.code === "ENOENT" || error.code === "ENOTDIR"))
|
|
24
20
|
const isAccessError = (error) => !!(error && (error.code === "EACCES" || error.code === "EPERM"))
|
|
25
21
|
const lstatIfPresent = async (filePath) => {
|
|
@@ -71,18 +67,14 @@ const sameFileMetadata = (left, right) => {
|
|
|
71
67
|
}
|
|
72
68
|
|
|
73
69
|
const sourceId = (kind, name) => `${kind}:${encodeURIComponent(name)}`
|
|
74
|
-
const externalSourceId = (root) => {
|
|
75
|
-
const value = process.platform === "win32" ? path.resolve(root).toLowerCase() : path.resolve(root)
|
|
76
|
-
return `external:${crypto.createHash("sha256").update(value).digest("hex")}`
|
|
77
|
-
}
|
|
78
70
|
|
|
79
71
|
class Vault {
|
|
80
72
|
constructor(kernel) {
|
|
81
73
|
this.kernel = kernel
|
|
82
74
|
this.enabled = false
|
|
83
75
|
this.initialized = false
|
|
84
|
-
this.mode = null // 'link' | '
|
|
85
|
-
this.volumeModes = new Map() // dev -> 'link' | '
|
|
76
|
+
this.mode = null // 'link' | 'copy' for the vault's own volume
|
|
77
|
+
this.volumeModes = new Map() // dev -> 'link' | 'copy'
|
|
86
78
|
this.registry = null
|
|
87
79
|
this.worker = null
|
|
88
80
|
this.workerJobs = new Map()
|
|
@@ -98,15 +90,8 @@ class Vault {
|
|
|
98
90
|
this.initializationPromise = null
|
|
99
91
|
this.verificationPending = false
|
|
100
92
|
this.scanError = null
|
|
101
|
-
this.scanPersistenceWarning = false
|
|
102
93
|
this.scanScopeId = null
|
|
103
|
-
this.
|
|
104
|
-
this.fileActions = new Map()
|
|
105
|
-
this.actionSequence = 0
|
|
106
|
-
this.scanEvents = null
|
|
107
|
-
this.scanCreatedStores = null
|
|
108
|
-
this.scanRetiredStores = null
|
|
109
|
-
this.scanRestoredStores = null
|
|
94
|
+
this.fileActionProgress = null
|
|
110
95
|
}
|
|
111
96
|
get root() {
|
|
112
97
|
return path.resolve(this.kernel.homedir, "vault")
|
|
@@ -114,6 +99,9 @@ class Vault {
|
|
|
114
99
|
get blobRoot() {
|
|
115
100
|
return path.resolve(this.root, "sha256")
|
|
116
101
|
}
|
|
102
|
+
get sourceRoot() {
|
|
103
|
+
return path.resolve(this.root, "sources")
|
|
104
|
+
}
|
|
117
105
|
storePathFor(hash) {
|
|
118
106
|
if (typeof hash !== "string" || !SHA256_RE.test(hash)) {
|
|
119
107
|
throw new TypeError("Invalid vault content identifier.")
|
|
@@ -171,123 +159,18 @@ class Vault {
|
|
|
171
159
|
return result
|
|
172
160
|
})
|
|
173
161
|
}
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
const action = {
|
|
177
|
-
id,
|
|
178
|
-
kind,
|
|
179
|
-
path: filePath || null,
|
|
180
|
-
source_id: sourceId || null,
|
|
181
|
-
phase: "queued",
|
|
182
|
-
bytes_copied: 0,
|
|
183
|
-
bytes_total: Math.max(0, Number(bytesTotal) || 0),
|
|
184
|
-
files_completed: Math.max(0, Number(progress.files_completed) || 0),
|
|
185
|
-
files_total: Math.max(0, Number(progress.files_total) || 0)
|
|
186
|
-
}
|
|
187
|
-
this.fileActions.set(id, action)
|
|
188
|
-
this.actionSequence += 1
|
|
189
|
-
return this.runExclusive(async () => {
|
|
190
|
-
action.phase = kind.startsWith("deduplicate") ? "deduplicating" : "copying"
|
|
191
|
-
this.fileAction = action
|
|
192
|
-
this.actionSequence += 1
|
|
193
|
-
try {
|
|
194
|
-
const result = await operation(action)
|
|
195
|
-
try {
|
|
196
|
-
if (this.registry) await this.registry.flush()
|
|
197
|
-
} catch (error) {
|
|
198
|
-
return Object.assign({}, result, { persistence_warning: true })
|
|
199
|
-
}
|
|
200
|
-
return result
|
|
201
|
-
} finally {
|
|
202
|
-
if (this.fileAction === action) this.fileAction = null
|
|
203
|
-
this.fileActions.delete(id)
|
|
204
|
-
this.actionSequence += 1
|
|
205
|
-
}
|
|
206
|
-
})
|
|
207
|
-
}
|
|
208
|
-
beginScanDraft() {
|
|
209
|
-
this.scanEvents = []
|
|
210
|
-
this.scanCreatedStores = []
|
|
211
|
-
this.scanRetiredStores = []
|
|
212
|
-
this.scanRestoredStores = []
|
|
213
|
-
}
|
|
214
|
-
rememberScanStore(storePath, stat) {
|
|
215
|
-
if (this.scanCreatedStores) this.scanCreatedStores.push({ storePath, stat })
|
|
216
|
-
}
|
|
217
|
-
rememberRetiredScanStore(storePath, stat, sourcePaths = []) {
|
|
218
|
-
if (this.scanRetiredStores) {
|
|
219
|
-
this.scanRetiredStores.push({ storePath, stat, sourcePaths: [...new Set(sourcePaths)] })
|
|
220
|
-
}
|
|
221
|
-
}
|
|
222
|
-
async restoreRetiredScanStore(item) {
|
|
223
|
-
const current = await this.storeStatIfPresent(item.storePath)
|
|
224
|
-
if (current) return sameIdentity(current, item.stat) ? current : null
|
|
225
|
-
const key = this.registry.inoKey(item.stat.dev, item.stat.ino)
|
|
226
|
-
const sourcePaths = new Set([
|
|
227
|
-
...(item.sourcePaths || []),
|
|
228
|
-
...(this.registry.pathsByIno.get(key) || [])
|
|
229
|
-
])
|
|
230
|
-
for (const filePath of sourcePaths) {
|
|
231
|
-
const st = await lstatIfPresent(filePath)
|
|
232
|
-
if (!sameIdentity(st, item.stat)) continue
|
|
233
|
-
await fs.promises.link(filePath, item.storePath)
|
|
234
|
-
const restored = await this.storeStatIfPresent(item.storePath)
|
|
235
|
-
return sameIdentity(restored, item.stat) ? restored : null
|
|
236
|
-
}
|
|
237
|
-
return null
|
|
238
|
-
}
|
|
239
|
-
async commitScanDraft() {
|
|
240
|
-
const events = (this.scanEvents || []).slice(-this.registry.maxEvents)
|
|
241
|
-
const retired = this.scanRetiredStores || []
|
|
242
|
-
const removed = []
|
|
162
|
+
async runFileAction(progress, operation) {
|
|
163
|
+
this.fileActionProgress = progress
|
|
243
164
|
try {
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
const hash = this.registry.byIno.get(this.registry.inoKey(item.stat.dev, item.stat.ino))
|
|
248
|
-
if (hash) await this.refreshLinkSnapshots(hash, item.stat.dev, item.stat.ino)
|
|
249
|
-
}
|
|
250
|
-
} catch (error) {
|
|
251
|
-
for (const item of removed.reverse()) {
|
|
252
|
-
const restored = await this.restoreRetiredScanStore(item).catch(() => null)
|
|
253
|
-
if (restored) this.scanRestoredStores.push({ item, stat: restored })
|
|
254
|
-
}
|
|
255
|
-
throw error
|
|
256
|
-
}
|
|
257
|
-
this.scanEvents = null
|
|
258
|
-
this.scanCreatedStores = null
|
|
259
|
-
this.scanRetiredStores = null
|
|
260
|
-
this.scanRestoredStores = null
|
|
261
|
-
try {
|
|
262
|
-
await this.registry.appendEvents(events)
|
|
263
|
-
return true
|
|
264
|
-
} catch (error) {
|
|
265
|
-
return false
|
|
165
|
+
return await operation(progress)
|
|
166
|
+
} finally {
|
|
167
|
+
if (this.fileActionProgress === progress) this.fileActionProgress = null
|
|
266
168
|
}
|
|
267
169
|
}
|
|
268
|
-
async abortScanDraft(snapshot) {
|
|
269
|
-
const created = this.scanCreatedStores || []
|
|
270
|
-
const restored = this.scanRestoredStores || []
|
|
271
|
-
this.scanEvents = null
|
|
272
|
-
this.scanCreatedStores = null
|
|
273
|
-
this.scanRetiredStores = null
|
|
274
|
-
this.scanRestoredStores = null
|
|
275
|
-
for (const item of created.reverse()) {
|
|
276
|
-
await unlinkIfSame(item.storePath, item.stat).catch(() => {})
|
|
277
|
-
}
|
|
278
|
-
this.registry.replaceState(snapshot, { persist: false })
|
|
279
|
-
for (const item of restored) {
|
|
280
|
-
const hash = this.registry.byIno.get(
|
|
281
|
-
this.registry.inoKey(item.stat.dev, item.stat.ino))
|
|
282
|
-
if (hash) await this.refreshLinkSnapshots(hash, item.stat.dev, item.stat.ino)
|
|
283
|
-
}
|
|
284
|
-
return restored.length > 0
|
|
285
|
-
}
|
|
286
170
|
startScan(scopeId = null) {
|
|
287
171
|
if (!this.enabled || !this.sweeper) return { started: false, disabled: true }
|
|
288
172
|
if (this.scanPromise) return { started: false, already_running: true }
|
|
289
173
|
this.scanError = null
|
|
290
|
-
this.scanPersistenceWarning = false
|
|
291
174
|
this.scanScopeId = scopeId
|
|
292
175
|
this.scanPromise = this.runExclusive(() => this.sweeper.scan(scopeId))
|
|
293
176
|
.catch((error) => {
|
|
@@ -317,71 +200,49 @@ class Vault {
|
|
|
317
200
|
}
|
|
318
201
|
}
|
|
319
202
|
}
|
|
320
|
-
case "remove_source":
|
|
321
|
-
return this.runMutation(() => this.removeExternalSource(payload.source_id))
|
|
322
203
|
case "scan":
|
|
323
204
|
if (payload.scope_id && !this.scanSource(payload.scope_id)) {
|
|
324
205
|
return { error: "That scan location is no longer available." }
|
|
325
206
|
}
|
|
326
|
-
if (payload.candidate_min_bytes !== undefined) {
|
|
327
|
-
if (!CANDIDATE_SIZE_OPTIONS.includes(payload.candidate_min_bytes)) {
|
|
328
|
-
return { error: "Choose one of the available minimum file sizes." }
|
|
329
|
-
}
|
|
330
|
-
if (this.scanPromise || (this.sweeper && this.sweeper.state.active)) {
|
|
331
|
-
return { error: "Wait for the current scan to finish before changing the minimum file size." }
|
|
332
|
-
}
|
|
333
|
-
this.sizeThreshold = payload.candidate_min_bytes
|
|
334
|
-
this.registry.settings.candidate_min_bytes = this.sizeThreshold
|
|
335
|
-
this.registry.schedulePersist()
|
|
336
|
-
}
|
|
337
207
|
return this.startScan(payload.scope_id || null)
|
|
338
208
|
case "deduplicate":
|
|
339
|
-
if (typeof payload.
|
|
340
|
-
return
|
|
209
|
+
if (typeof payload.path === "string" && payload.path) {
|
|
210
|
+
return this.runMutation(() => this.runFileAction({
|
|
211
|
+
kind: "deduplicate-file",
|
|
212
|
+
path: path.resolve(payload.path)
|
|
213
|
+
}, () => this.deduplicateFile(payload.path, {
|
|
214
|
+
batch_id: `batch-${crypto.randomUUID()}`
|
|
215
|
+
})))
|
|
341
216
|
}
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
? new Set(paths.filter((filePath) => typeof filePath === "string" && filePath)).size
|
|
346
|
-
: this.registry.duplicates.size
|
|
347
|
-
return this.queueFileMutation(
|
|
348
|
-
"deduplicate",
|
|
349
|
-
null,
|
|
350
|
-
payload.scope_id,
|
|
351
|
-
0,
|
|
352
|
-
(progress) => this.deduplicateScope(payload.scope_id, {
|
|
353
|
-
batch_id: `batch-${crypto.randomUUID()}`,
|
|
354
|
-
paths,
|
|
355
|
-
progress
|
|
356
|
-
}),
|
|
357
|
-
{ files_total: filesTotal }
|
|
358
|
-
)
|
|
217
|
+
const selection = payload.selection || null
|
|
218
|
+
if (selection && selection !== "duplicates" && selection !== "kept-separate") {
|
|
219
|
+
return { error: "Choose files to deduplicate." }
|
|
359
220
|
}
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
for (const group of Array.isArray(payload.groups) ? payload.groups : []) {
|
|
364
|
-
if (!group || typeof group.scope_id !== "string" || !group.scope_id) continue
|
|
365
|
-
if (!pathsByScope.has(group.scope_id)) pathsByScope.set(group.scope_id, [])
|
|
366
|
-
for (const filePath of Array.isArray(group.paths) ? group.paths : []) {
|
|
367
|
-
if (typeof filePath !== "string" || !filePath || seenPaths.has(filePath)) continue
|
|
368
|
-
seenPaths.add(filePath)
|
|
369
|
-
pathsByScope.get(group.scope_id).push(filePath)
|
|
370
|
-
}
|
|
221
|
+
if (payload.scope_id != null &&
|
|
222
|
+
(typeof payload.scope_id !== "string" || !payload.scope_id)) {
|
|
223
|
+
return { error: "Choose a valid location to deduplicate." }
|
|
371
224
|
}
|
|
372
|
-
const
|
|
373
|
-
.
|
|
374
|
-
|
|
375
|
-
if (!
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
225
|
+
const scopeId = typeof payload.scope_id === "string" && payload.scope_id
|
|
226
|
+
? payload.scope_id
|
|
227
|
+
: null
|
|
228
|
+
if (!selection && !scopeId) {
|
|
229
|
+
return { error: "Choose a location to deduplicate." }
|
|
230
|
+
}
|
|
231
|
+
return this.runMutation(() => this.runFileAction({
|
|
232
|
+
kind: "deduplicate",
|
|
233
|
+
scope_id: scopeId,
|
|
234
|
+
selection: selection || "duplicates",
|
|
235
|
+
files_total: 0,
|
|
236
|
+
files_completed: 0
|
|
237
|
+
}, async (progress) => {
|
|
238
|
+
const options = {
|
|
239
|
+
batch_id: `batch-${crypto.randomUUID()}`,
|
|
240
|
+
progress
|
|
241
|
+
}
|
|
242
|
+
return selection
|
|
243
|
+
? this.deduplicateSelection(selection, scopeId, options)
|
|
244
|
+
: this.deduplicateScope(scopeId, options)
|
|
245
|
+
}))
|
|
385
246
|
case "reclaim":
|
|
386
247
|
return this.runMutation(() => this.reclaim(payload.hash))
|
|
387
248
|
case "reclaim_all":
|
|
@@ -396,24 +257,16 @@ class Vault {
|
|
|
396
257
|
return { done: true }
|
|
397
258
|
})
|
|
398
259
|
case "undo":
|
|
399
|
-
return this.
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
: 0,
|
|
410
|
-
(action) => this.detach(payload.path, action)
|
|
411
|
-
)
|
|
412
|
-
}
|
|
413
|
-
case "skip":
|
|
414
|
-
return this.runMutation(() => this.skip(payload.path))
|
|
415
|
-
case "reshare":
|
|
416
|
-
return this.runMutation(() => this.reshare(payload.path))
|
|
260
|
+
return this.runMutation(() => this.undoBatch(payload.batch_id))
|
|
261
|
+
case "detach":
|
|
262
|
+
if (typeof payload.path !== "string" || !payload.path) return { status: "not-found" }
|
|
263
|
+
return this.runMutation(() => {
|
|
264
|
+
const targetPath = path.resolve(payload.path)
|
|
265
|
+
const kind = this.registry.duplicates.has(targetPath)
|
|
266
|
+
? "keep-separate"
|
|
267
|
+
: "make-separate"
|
|
268
|
+
return this.runFileAction({ kind, path: targetPath }, () => this.detach(targetPath))
|
|
269
|
+
})
|
|
417
270
|
default:
|
|
418
271
|
return { error: "unknown action" }
|
|
419
272
|
}
|
|
@@ -435,31 +288,44 @@ class Vault {
|
|
|
435
288
|
const decorate = async (source) => {
|
|
436
289
|
if (!source.root) return source
|
|
437
290
|
try {
|
|
438
|
-
const st = await fs.promises.
|
|
291
|
+
const st = await fs.promises.stat(source.root)
|
|
439
292
|
source.dev = st.dev
|
|
440
|
-
source.available = st.isDirectory()
|
|
293
|
+
source.available = st.isDirectory()
|
|
441
294
|
source.shareable = source.available && storeDev !== null && st.dev === storeDev && this.mode === "link"
|
|
442
295
|
} catch (error) {
|
|
443
|
-
if (!isMissingError(error)
|
|
296
|
+
if (!isMissingError(error)) throw error
|
|
444
297
|
source.available = false
|
|
445
298
|
source.shareable = false
|
|
446
|
-
source.error_code = error && error.code ? error.code : null
|
|
447
299
|
}
|
|
448
300
|
return source
|
|
449
301
|
}
|
|
450
302
|
|
|
451
|
-
const
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
303
|
+
const seenExternalRoots = new Set()
|
|
304
|
+
const addExternalEntries = async (importRoot, idPrefix = "") => {
|
|
305
|
+
let entries = []
|
|
306
|
+
try {
|
|
307
|
+
entries = await fs.promises.readdir(importRoot, { withFileTypes: true })
|
|
308
|
+
} catch (error) {
|
|
309
|
+
if (!isMissingError(error)) throw error
|
|
310
|
+
}
|
|
311
|
+
for (const entry of entries) {
|
|
312
|
+
if (!entry.isSymbolicLink()) continue
|
|
313
|
+
const mountPath = path.resolve(importRoot, entry.name)
|
|
314
|
+
try {
|
|
315
|
+
const root = await fs.promises.realpath(mountPath)
|
|
316
|
+
const st = await fs.promises.stat(root)
|
|
317
|
+
if (!st.isDirectory()) continue
|
|
318
|
+
const canonical = path.resolve(root)
|
|
319
|
+
if (seenExternalRoots.has(canonical)) continue
|
|
320
|
+
seenExternalRoots.add(canonical)
|
|
321
|
+
sources.push(await decorate({
|
|
322
|
+
id: sourceId("external", `${idPrefix}${entry.name}`), kind: "external", label: entry.name,
|
|
323
|
+
root: canonical, mount_path: mountPath, parent_id: "external"
|
|
324
|
+
}))
|
|
325
|
+
} catch (error) {
|
|
326
|
+
if (!isMissingError(error)) throw error
|
|
327
|
+
}
|
|
328
|
+
}
|
|
463
329
|
}
|
|
464
330
|
|
|
465
331
|
let apiEntries = []
|
|
@@ -477,6 +343,11 @@ class Vault {
|
|
|
477
343
|
}))
|
|
478
344
|
}
|
|
479
345
|
}
|
|
346
|
+
await addExternalEntries(apiRoot)
|
|
347
|
+
if (await this.directoryIfSafe(this.sourceRoot)) {
|
|
348
|
+
await addExternalEntries(this.sourceRoot, "vault:")
|
|
349
|
+
}
|
|
350
|
+
|
|
480
351
|
let homeEntries = []
|
|
481
352
|
try {
|
|
482
353
|
homeEntries = await fs.promises.readdir(home, { withFileTypes: true })
|
|
@@ -496,10 +367,13 @@ class Vault {
|
|
|
496
367
|
this._sourceBases = new Map()
|
|
497
368
|
for (const source of sources) {
|
|
498
369
|
if (!source.root || source.kind === "virtual") continue
|
|
499
|
-
const
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
370
|
+
for (const base of [source.root, source.mount_path]) {
|
|
371
|
+
if (!base) continue
|
|
372
|
+
const resolved = path.resolve(base)
|
|
373
|
+
const key = process.platform === "win32" ? resolved.toLowerCase() : resolved
|
|
374
|
+
if (!this._sourceBases.has(key)) this._sourceBases.set(key, [])
|
|
375
|
+
this._sourceBases.get(key).push(source)
|
|
376
|
+
}
|
|
503
377
|
}
|
|
504
378
|
return sources
|
|
505
379
|
}
|
|
@@ -533,106 +407,50 @@ class Vault {
|
|
|
533
407
|
if (existing) {
|
|
534
408
|
return { created: false, source: existing }
|
|
535
409
|
}
|
|
536
|
-
const configured = [...this.registry.settings.external_sources]
|
|
537
|
-
if (configured.some((root) => isPathWithin(root, canonical) || isPathWithin(canonical, root))) {
|
|
538
|
-
throw new Error("Choose a folder that does not contain or overlap another scan location.")
|
|
539
|
-
}
|
|
540
|
-
const previousScan = this.registry.lastScan
|
|
541
|
-
const previousAttempt = this.registry.lastScanAttempt
|
|
542
|
-
this.registry.settings.external_sources = [...configured, canonical]
|
|
543
|
-
this.registry.lastScan = null
|
|
544
|
-
this.registry.lastScanAttempt = null
|
|
545
|
-
this.registry.schedulePersist()
|
|
546
|
-
try {
|
|
547
|
-
await this.registry.flush()
|
|
548
|
-
await this.refreshSources()
|
|
549
|
-
const source = this._sources.find((item) =>
|
|
550
|
-
item.kind === "external" && item.root && samePath(item.root, canonical))
|
|
551
|
-
if (!source) throw new Error("The folder could not be added.")
|
|
552
|
-
return { created: true, source }
|
|
553
|
-
} catch (error) {
|
|
554
|
-
this.registry.settings.external_sources = configured
|
|
555
|
-
this.registry.lastScan = previousScan
|
|
556
|
-
this.registry.lastScanAttempt = previousAttempt
|
|
557
|
-
this.registry.schedulePersist()
|
|
558
|
-
await this.registry.flush().catch(() => {})
|
|
559
|
-
await this.refreshSources().catch(() => {})
|
|
560
|
-
throw error
|
|
561
|
-
}
|
|
562
|
-
}
|
|
563
|
-
async removeExternalSource(sourceId) {
|
|
564
|
-
if (typeof sourceId !== "string" || !sourceId) {
|
|
565
|
-
throw new Error("Choose an external folder to remove.")
|
|
566
|
-
}
|
|
567
|
-
await this.refreshSources()
|
|
568
|
-
const source = this._sources.find((item) =>
|
|
569
|
-
item.id === sourceId && item.kind === "external" && item.root)
|
|
570
|
-
if (!source) throw new Error("That external folder is no longer configured.")
|
|
571
410
|
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
}
|
|
580
|
-
if (!rootStat.isDirectory() || !samePath(canonical, source.root)) {
|
|
581
|
-
throw new Error("Reconnect this folder before removing it.")
|
|
582
|
-
}
|
|
583
|
-
|
|
584
|
-
const storesToRemove = new Map()
|
|
585
|
-
for (const [filePath, entry] of this.registry.links) {
|
|
586
|
-
if (entry.source_id !== source.id) continue
|
|
587
|
-
let fileStat
|
|
411
|
+
const importRoot = this.sourceRoot
|
|
412
|
+
await this.ensureDirectory(importRoot)
|
|
413
|
+
const baseLabel = path.basename(canonical) || "external-folder"
|
|
414
|
+
let mountPath = null
|
|
415
|
+
for (let index = 1; index < 10000; index++) {
|
|
416
|
+
const label = index === 1 ? baseLabel : `${baseLabel}-${index}`
|
|
417
|
+
const candidate = path.resolve(importRoot, label)
|
|
588
418
|
try {
|
|
589
|
-
|
|
419
|
+
await fs.promises.symlink(canonical, candidate, this.kernel.platform === "win32" ? "junction" : "dir")
|
|
420
|
+
mountPath = candidate
|
|
421
|
+
break
|
|
590
422
|
} catch (error) {
|
|
591
|
-
if (
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
const storeStat = await this.storeStatIfPresent(storePath)
|
|
597
|
-
const visibleNames = fileStat.nlink - (sameIdentity(fileStat, storeStat) ? 1 : 0)
|
|
598
|
-
if (visibleNames > 1) {
|
|
599
|
-
throw new Error("Turn sharing off for this folder before removing it.")
|
|
600
|
-
}
|
|
601
|
-
if (visibleNames === 1 && sameIdentity(fileStat, storeStat)) {
|
|
602
|
-
storesToRemove.set(entry.hash, {
|
|
603
|
-
path: storePath,
|
|
604
|
-
stat: storeStat
|
|
605
|
-
})
|
|
423
|
+
if (error && error.code === "EEXIST") continue
|
|
424
|
+
if (error && (error.code === "EACCES" || error.code === "EPERM")) {
|
|
425
|
+
throw new Error("Pinokio does not have permission to add that folder.")
|
|
426
|
+
}
|
|
427
|
+
throw new Error("Pinokio could not add that folder.")
|
|
606
428
|
}
|
|
607
429
|
}
|
|
430
|
+
if (!mountPath) throw new Error("Pinokio could not create a unique name for that folder.")
|
|
608
431
|
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
432
|
+
try {
|
|
433
|
+
await this.refreshSources()
|
|
434
|
+
const source = this._sources.find((item) => item.kind === "external" && item.mount_path && samePath(item.mount_path, mountPath))
|
|
435
|
+
if (!source) throw new Error("The folder could not be added. Restart Pinokio and try again.")
|
|
436
|
+
return { created: true, source }
|
|
437
|
+
} catch (error) {
|
|
438
|
+
// Adding a source is transactional. Roll back only the exact directory
|
|
439
|
+
// link created above; preserve any path an external writer replaced.
|
|
440
|
+
try {
|
|
441
|
+
const [mounted, target] = await Promise.all([
|
|
442
|
+
fs.promises.lstat(mountPath),
|
|
443
|
+
fs.promises.realpath(mountPath)
|
|
444
|
+
])
|
|
445
|
+
if (mounted.isSymbolicLink() && samePath(target, canonical)) {
|
|
446
|
+
await fs.promises.unlink(mountPath)
|
|
447
|
+
}
|
|
448
|
+
} catch {
|
|
449
|
+
// The original error remains authoritative; an unverified path is
|
|
450
|
+
// deliberately preserved rather than deleted.
|
|
612
451
|
}
|
|
452
|
+
throw error
|
|
613
453
|
}
|
|
614
|
-
|
|
615
|
-
for (const [filePath, entry] of [...this.registry.links]) {
|
|
616
|
-
if (entry.source_id === source.id) this.registry.untrack(filePath)
|
|
617
|
-
}
|
|
618
|
-
for (const [filePath, entry] of [...this.registry.duplicates]) {
|
|
619
|
-
if (entry.source_id === source.id) this.registry.untrack(filePath)
|
|
620
|
-
}
|
|
621
|
-
for (const [filePath, entry] of [...this.registry.scanIndex]) {
|
|
622
|
-
if (entry.source_id === source.id) this.registry.untrack(filePath)
|
|
623
|
-
}
|
|
624
|
-
for (const [filePath, entry] of [...this.registry.excluded]) {
|
|
625
|
-
if (entry.source_id === source.id) this.registry.excluded.delete(filePath)
|
|
626
|
-
}
|
|
627
|
-
this.registry.sourceScans.delete(source.id)
|
|
628
|
-
this.registry.sourceScanAttempts.delete(source.id)
|
|
629
|
-
this.registry.settings.external_sources = this.registry.settings.external_sources
|
|
630
|
-
.filter((root) => !samePath(root, source.root))
|
|
631
|
-
this.registry.lastScan = null
|
|
632
|
-
this.registry.lastScanAttempt = null
|
|
633
|
-
this.registry.schedulePersist()
|
|
634
|
-
await this.refreshSources()
|
|
635
|
-
return { removed: true, source_id: source.id }
|
|
636
454
|
}
|
|
637
455
|
sourceForPath(filePath, preferredId) {
|
|
638
456
|
let cursor = path.resolve(filePath)
|
|
@@ -664,7 +482,9 @@ class Vault {
|
|
|
664
482
|
const source = this.sourceForPath(filePath, preferredId)
|
|
665
483
|
if (!source) return { source_id: null, relative_path: path.basename(filePath) }
|
|
666
484
|
const absolute = path.resolve(filePath)
|
|
667
|
-
|
|
485
|
+
let base = source.root
|
|
486
|
+
if (source.mount_path && isPathWithin(source.mount_path, absolute)) base = source.mount_path
|
|
487
|
+
const relative = path.relative(base, absolute).split(path.sep).join("/") || path.basename(absolute)
|
|
668
488
|
return {
|
|
669
489
|
source_id: source.id,
|
|
670
490
|
source_kind: source.kind,
|
|
@@ -679,13 +499,6 @@ class Vault {
|
|
|
679
499
|
const context = location.source_id ? location : { relative_path: event.path }
|
|
680
500
|
entry = Object.assign({}, event, context)
|
|
681
501
|
}
|
|
682
|
-
if (this.scanEvents) {
|
|
683
|
-
this.scanEvents.push(entry)
|
|
684
|
-
if (this.scanEvents.length > this.registry.maxEvents * 2) {
|
|
685
|
-
this.scanEvents.splice(0, this.registry.maxEvents)
|
|
686
|
-
}
|
|
687
|
-
return true
|
|
688
|
-
}
|
|
689
502
|
try {
|
|
690
503
|
await this.registry.appendEvent(entry)
|
|
691
504
|
return true
|
|
@@ -770,16 +583,12 @@ class Vault {
|
|
|
770
583
|
await this.ensureDirectory(this.blobRoot)
|
|
771
584
|
this.registry = new Registry(this.root)
|
|
772
585
|
this.mode = await this.probe(this.root)
|
|
773
|
-
const loaded = await this.registry.load()
|
|
774
586
|
await this.refreshSources()
|
|
587
|
+
const loaded = await this.registry.load()
|
|
775
588
|
const missingWithBlobs = !loaded.existed && await this.hasStoredBlobs()
|
|
776
589
|
if (loaded.corrupt || missingWithBlobs) {
|
|
777
|
-
|
|
778
|
-
// or walk configured sources. A later explicit Scan or Repair rebuilds
|
|
779
|
-
// visible-path state.
|
|
780
|
-
await this.rebuild([], { preserveState: false })
|
|
590
|
+
await this.rebuild(undefined, { preserveState: false })
|
|
781
591
|
}
|
|
782
|
-
this.sizeThreshold = this.registry.settings.candidate_min_bytes || SIZE_THRESHOLD
|
|
783
592
|
this.sweeper = new Sweeper(this)
|
|
784
593
|
this.initialized = true
|
|
785
594
|
return {
|
|
@@ -842,7 +651,7 @@ class Vault {
|
|
|
842
651
|
if (this.volumeModes.has(dev)) return this.volumeModes.get(dev)
|
|
843
652
|
const a = path.resolve(dir, `.pinokio-probe-${crypto.randomBytes(6).toString("hex")}`)
|
|
844
653
|
const b = a + "-link"
|
|
845
|
-
let mode = "
|
|
654
|
+
let mode = "copy"
|
|
846
655
|
let aStat = null
|
|
847
656
|
let bStat = null
|
|
848
657
|
let failure = null
|
|
@@ -879,11 +688,7 @@ class Vault {
|
|
|
879
688
|
const job = this.workerJobs.get(id)
|
|
880
689
|
if (!job) return
|
|
881
690
|
this.workerJobs.delete(id)
|
|
882
|
-
if (error)
|
|
883
|
-
const failure = new Error(error.message || String(error))
|
|
884
|
-
if (error.code) failure.code = error.code
|
|
885
|
-
job.reject(failure)
|
|
886
|
-
}
|
|
691
|
+
if (error) job.reject(new Error(error))
|
|
887
692
|
else job.resolve({ hash, size })
|
|
888
693
|
// Keep the worker warm across the scan queue. Starting one worker per
|
|
889
694
|
// file costs ~30ms and adds up quickly on a first scan.
|
|
@@ -939,10 +744,7 @@ class Vault {
|
|
|
939
744
|
hash,
|
|
940
745
|
source_id: entry.source_id || null
|
|
941
746
|
}))
|
|
942
|
-
} catch (error) {
|
|
943
|
-
this.registry.scanIndex.delete(linkPath)
|
|
944
|
-
entry.unverified = true
|
|
945
|
-
}
|
|
747
|
+
} catch (error) {}
|
|
946
748
|
}
|
|
947
749
|
this.registry.schedulePersist()
|
|
948
750
|
}
|
|
@@ -952,7 +754,7 @@ class Vault {
|
|
|
952
754
|
for (const linkPath of this.registry.pathsByIno.get(key) || []) {
|
|
953
755
|
const entry = this.registry.links.get(linkPath)
|
|
954
756
|
if (!entry) continue
|
|
955
|
-
if (entry.hash !== hash ||
|
|
757
|
+
if (entry.hash !== hash || entry.mode !== "link" ||
|
|
956
758
|
entry.dev !== storeStat.dev || entry.ino !== storeStat.ino) continue
|
|
957
759
|
const expected = this.registry.scanIndex.get(linkPath)
|
|
958
760
|
if (expected && expected.hash === hash && sameSnapshot(expected, storeStat)) {
|
|
@@ -965,15 +767,13 @@ class Vault {
|
|
|
965
767
|
try {
|
|
966
768
|
hashed = await this.hashFile(storePath)
|
|
967
769
|
} catch (error) {
|
|
968
|
-
|
|
969
|
-
throw error
|
|
770
|
+
return { valid: false }
|
|
970
771
|
}
|
|
971
772
|
let after
|
|
972
773
|
try {
|
|
973
774
|
after = await this.storeStatIfPresent(storePath)
|
|
974
775
|
} catch (error) {
|
|
975
|
-
|
|
976
|
-
throw error
|
|
776
|
+
return { valid: false }
|
|
977
777
|
}
|
|
978
778
|
if (!sameSnapshot(before, after) || hashed.hash !== hash || hashed.size !== after.size) {
|
|
979
779
|
return { valid: false }
|
|
@@ -995,6 +795,7 @@ class Vault {
|
|
|
995
795
|
source_id: meta.source_id || null,
|
|
996
796
|
dev: st.dev,
|
|
997
797
|
ino: st.ino,
|
|
798
|
+
mode: "link"
|
|
998
799
|
}
|
|
999
800
|
const storeStat = await this.storeStatIfPresent(storePath, {
|
|
1000
801
|
createParent: this.mode === "link"
|
|
@@ -1012,12 +813,25 @@ class Vault {
|
|
|
1012
813
|
if (!sameSnapshot(before, current)) return { status: "stale" }
|
|
1013
814
|
return { status: "duplicate", storePath }
|
|
1014
815
|
}
|
|
1015
|
-
|
|
816
|
+
const registerCopy = async () => {
|
|
817
|
+
const current = await lstatIfPresent(filePath)
|
|
818
|
+
if (!sameSnapshot(before, current)) return { status: "stale" }
|
|
819
|
+
linkEntry.mode = "copy"
|
|
820
|
+
this.registry.addBlob(hash, { size: st.size, source_urls: meta.source_urls || [] })
|
|
821
|
+
this.registry.addLink(filePath, linkEntry)
|
|
822
|
+
await this.refreshLinkSnapshots(hash, st.dev, st.ino)
|
|
823
|
+
await this.recordEvent({
|
|
824
|
+
kind: "adopt", hash, path: filePath, app: meta.app || null,
|
|
825
|
+
source_id: meta.source_id || null, bytes_saved: 0, mode: "copy"
|
|
826
|
+
})
|
|
827
|
+
return { status: "copy-mode" }
|
|
828
|
+
}
|
|
829
|
+
if (this.mode === "copy") return registerCopy()
|
|
1016
830
|
try {
|
|
1017
831
|
await fs.promises.link(filePath, storePath)
|
|
1018
832
|
} catch (e) {
|
|
1019
833
|
if (NO_LINK_CODES.has(e.code)) {
|
|
1020
|
-
return
|
|
834
|
+
return registerCopy()
|
|
1021
835
|
}
|
|
1022
836
|
throw e
|
|
1023
837
|
}
|
|
@@ -1045,7 +859,6 @@ class Vault {
|
|
|
1045
859
|
}
|
|
1046
860
|
this.registry.addBlob(hash, { size: st.size, source_urls: meta.source_urls || [] })
|
|
1047
861
|
this.registry.addLink(filePath, linkEntry)
|
|
1048
|
-
this.rememberScanStore(storePath, currentStore)
|
|
1049
862
|
await this.refreshLinkSnapshots(hash, st.dev, st.ino)
|
|
1050
863
|
await this.recordEvent({ kind: "adopt", hash, path: filePath, app: meta.app || null, source_id: meta.source_id || null, bytes_saved: 0 })
|
|
1051
864
|
return { status: "adopted" }
|
|
@@ -1055,16 +868,14 @@ class Vault {
|
|
|
1055
868
|
async convert(targetPath, hash, meta = {}) {
|
|
1056
869
|
if (!this.enabled) return { status: "disabled" }
|
|
1057
870
|
const storePath = this.storePathFor(hash)
|
|
1058
|
-
|
|
1059
|
-
if (!storeStat) return { status: "no-blob" }
|
|
1060
|
-
let targetStat
|
|
871
|
+
let storeStat
|
|
1061
872
|
try {
|
|
1062
|
-
|
|
1063
|
-
} catch (
|
|
1064
|
-
|
|
1065
|
-
if (isMissingError(error)) return { status: "stale" }
|
|
1066
|
-
throw error
|
|
873
|
+
storeStat = await this.storeStatIfPresent(storePath)
|
|
874
|
+
} catch (e) {
|
|
875
|
+
return { status: "no-blob" }
|
|
1067
876
|
}
|
|
877
|
+
if (!storeStat) return { status: "no-blob" }
|
|
878
|
+
const targetStat = await fs.promises.lstat(targetPath)
|
|
1068
879
|
if (!storeStat.isFile() || !targetStat.isFile()) return { status: "stale" }
|
|
1069
880
|
if (meta.expected && !sameSnapshot(meta.expected, targetStat)) {
|
|
1070
881
|
return { status: "stale" }
|
|
@@ -1073,10 +884,7 @@ class Vault {
|
|
|
1073
884
|
return { status: "unavailable", code: "EXDEV" }
|
|
1074
885
|
}
|
|
1075
886
|
if (storeStat.dev === targetStat.dev && storeStat.ino === targetStat.ino) {
|
|
1076
|
-
this.registry.addLink(targetPath, {
|
|
1077
|
-
hash, app: meta.app || null, source_id: meta.source_id || null,
|
|
1078
|
-
dev: storeStat.dev, ino: storeStat.ino, unverified: false
|
|
1079
|
-
})
|
|
887
|
+
this.registry.addLink(targetPath, { hash, app: meta.app || null, source_id: meta.source_id || null, dev: storeStat.dev, ino: storeStat.ino, mode: "link" })
|
|
1080
888
|
await this.refreshLinkSnapshots(hash, storeStat.dev, storeStat.ino)
|
|
1081
889
|
return { status: "already" }
|
|
1082
890
|
}
|
|
@@ -1158,10 +966,10 @@ class Vault {
|
|
|
1158
966
|
}
|
|
1159
967
|
this.registry.addLink(targetPath, {
|
|
1160
968
|
hash, app: meta.app || null, source_id: meta.source_id || null,
|
|
1161
|
-
dev: st.dev, ino: st.ino, batch_id: meta.batch_id || null
|
|
1162
|
-
unverified: false
|
|
969
|
+
dev: st.dev, ino: st.ino, mode: "link", batch_id: meta.batch_id || null
|
|
1163
970
|
})
|
|
1164
971
|
await this.refreshLinkSnapshots(hash, st.dev, st.ino)
|
|
972
|
+
this.registry.addSaved(storeStat.size)
|
|
1165
973
|
await this.recordEvent({
|
|
1166
974
|
kind: "convert", hash, path: targetPath, app: meta.app || null, source_id: meta.source_id || null,
|
|
1167
975
|
bytes_saved: storeStat.size, batch_id: meta.batch_id || null
|
|
@@ -1180,138 +988,196 @@ class Vault {
|
|
|
1180
988
|
return !!(runningPath && isPathWithin(appRoot, runningPath))
|
|
1181
989
|
})
|
|
1182
990
|
}
|
|
1183
|
-
async
|
|
991
|
+
async deduplicateFile(filePath, options = {}) {
|
|
992
|
+
if (!this.enabled || typeof filePath !== "string" || !filePath) {
|
|
993
|
+
return { status: this.enabled ? "not-found" : "disabled" }
|
|
994
|
+
}
|
|
1184
995
|
await this.refreshSources()
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
996
|
+
return this.deduplicateExcludedFile(filePath, options)
|
|
997
|
+
}
|
|
998
|
+
async deduplicateExcludedFile(filePath, options = {}) {
|
|
999
|
+
const targetPath = path.resolve(filePath)
|
|
1000
|
+
const excluded = this.registry.excluded.get(targetPath)
|
|
1001
|
+
if (!excluded) return { status: "not-found" }
|
|
1002
|
+
const source = this.sourceForPath(targetPath, excluded.source_id)
|
|
1003
|
+
if (!source || !await this.canonicalPathIsWithinSource(targetPath, source)) {
|
|
1004
|
+
return { status: "stale" }
|
|
1190
1005
|
}
|
|
1006
|
+
if (!source.shareable) return { status: "unavailable" }
|
|
1007
|
+
if (this.sourceAppIsRunning(source)) return { status: "locked" }
|
|
1191
1008
|
|
|
1192
|
-
const
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
return
|
|
1009
|
+
const before = await lstatIfPresent(targetPath)
|
|
1010
|
+
if (!before || !before.isFile()) return { status: "stale" }
|
|
1011
|
+
let hashed
|
|
1012
|
+
try {
|
|
1013
|
+
hashed = await this.hashFile(targetPath)
|
|
1014
|
+
} catch (error) {
|
|
1015
|
+
return { status: "stale" }
|
|
1016
|
+
}
|
|
1017
|
+
const after = await lstatIfPresent(targetPath)
|
|
1018
|
+
if (!sameSnapshot(fileSnapshot(before), after) || hashed.size !== after.size) {
|
|
1019
|
+
return { status: "stale" }
|
|
1020
|
+
}
|
|
1021
|
+
if (!this.registry.blobs.has(hashed.hash)) return { status: "no-match" }
|
|
1022
|
+
if (!await this.canonicalPathIsWithinSource(targetPath, source)) return { status: "stale" }
|
|
1023
|
+
const current = await lstatIfPresent(targetPath)
|
|
1024
|
+
if (!sameSnapshot(fileSnapshot(after), current)) return { status: "stale" }
|
|
1025
|
+
|
|
1026
|
+
const result = await this.convert(targetPath, hashed.hash, {
|
|
1027
|
+
app: source.kind === "app" ? source.app : null,
|
|
1028
|
+
source_id: source.id,
|
|
1029
|
+
batch_id: options.batch_id || `batch-${crypto.randomUUID()}`,
|
|
1030
|
+
source,
|
|
1031
|
+
expected: fileSnapshot(current)
|
|
1203
1032
|
})
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
progress.files_completed = 0
|
|
1207
|
-
progress.files_total = candidates.length
|
|
1208
|
-
this.actionSequence += 1
|
|
1033
|
+
if (result.status === "converted" || result.status === "already") {
|
|
1034
|
+
this.registry.allowSharing(targetPath)
|
|
1209
1035
|
}
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1036
|
+
return result
|
|
1037
|
+
}
|
|
1038
|
+
sourceIsWithinScope(source, scopeId) {
|
|
1039
|
+
if (!scopeId) return true
|
|
1040
|
+
const seen = new Set()
|
|
1041
|
+
let current = source
|
|
1042
|
+
while (current && !seen.has(current.id)) {
|
|
1043
|
+
if (current.id === scopeId) return true
|
|
1044
|
+
seen.add(current.id)
|
|
1045
|
+
current = this._sources.find((item) => item.id === current.parent_id)
|
|
1046
|
+
}
|
|
1047
|
+
return false
|
|
1048
|
+
}
|
|
1049
|
+
async deduplicateSelection(selection, scopeId, options = {}) {
|
|
1050
|
+
if (selection === "duplicates") {
|
|
1051
|
+
return this.deduplicateScope(scopeId, Object.assign({}, options, { includeDescendants: true }))
|
|
1052
|
+
}
|
|
1053
|
+
await this.refreshSources()
|
|
1054
|
+
if (scopeId && !this._sources.some((source) => source.id === scopeId)) {
|
|
1055
|
+
return { error: "That location is no longer available. Scan again to refresh it." }
|
|
1056
|
+
}
|
|
1057
|
+
const paths = [...this.registry.excluded].filter(([filePath, entry]) => {
|
|
1058
|
+
const source = this.sourceForPath(filePath, entry.source_id)
|
|
1059
|
+
return source ? this.sourceIsWithinScope(source, scopeId) : !scopeId
|
|
1060
|
+
}).map(([filePath]) => filePath)
|
|
1061
|
+
if (options.progress) options.progress.files_total = paths.length
|
|
1062
|
+
const summary = {
|
|
1063
|
+
converted: 0, bytes_saved: 0, locked: 0, stale: 0, unmatched: 0,
|
|
1064
|
+
incompatible: 0, unavailable: 0, failed: 0
|
|
1065
|
+
}
|
|
1066
|
+
const batch = options.batch_id || `batch-${crypto.randomUUID()}`
|
|
1067
|
+
for (const filePath of paths) {
|
|
1213
1068
|
try {
|
|
1214
|
-
const
|
|
1215
|
-
if (
|
|
1216
|
-
|
|
1217
|
-
summary.
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
if (
|
|
1221
|
-
summary.stale += 1
|
|
1222
|
-
continue
|
|
1223
|
-
}
|
|
1224
|
-
const indexed = registry.scanIndex.get(filePath)
|
|
1225
|
-
const expected = indexed && indexed.hash === entry.hash
|
|
1226
|
-
? indexed
|
|
1227
|
-
: (entry.dev !== undefined && entry.ino !== undefined &&
|
|
1228
|
-
entry.mtime !== undefined && entry.ctime !== undefined ? entry : null)
|
|
1229
|
-
if (!expected) {
|
|
1069
|
+
const result = await this.deduplicateExcludedFile(filePath, { batch_id: batch })
|
|
1070
|
+
if (result.status === "converted" || result.status === "already") {
|
|
1071
|
+
summary.converted += 1
|
|
1072
|
+
summary.bytes_saved += result.bytes_saved || 0
|
|
1073
|
+
} else if (result.status === "locked") {
|
|
1074
|
+
summary.locked += 1
|
|
1075
|
+
} else if (result.status === "stale") {
|
|
1230
1076
|
summary.stale += 1
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
}
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
app: source.kind === "app" ? source.app : (entry.app || null),
|
|
1240
|
-
source_id: scopeId,
|
|
1241
|
-
batch_id: batch,
|
|
1242
|
-
source: currentSource,
|
|
1243
|
-
expected
|
|
1244
|
-
})
|
|
1245
|
-
if (result.status === "converted" || result.status === "already") {
|
|
1246
|
-
summary.converted += 1
|
|
1247
|
-
summary.bytes_saved += result.bytes_saved || 0
|
|
1248
|
-
} else if (result.status === "locked") {
|
|
1249
|
-
summary.locked += 1
|
|
1250
|
-
} else if (result.status === "stale") {
|
|
1251
|
-
summary.stale += 1
|
|
1252
|
-
} else if (result.status === "stale-blob") {
|
|
1253
|
-
staleHashes.add(entry.hash)
|
|
1254
|
-
summary.stale += 1
|
|
1255
|
-
} else if (result.status === "metadata-mismatch") {
|
|
1256
|
-
entry.unavailable_reason = "metadata"
|
|
1257
|
-
summary.incompatible += 1
|
|
1258
|
-
} else if (result.status === "unavailable") {
|
|
1259
|
-
summary.unavailable += 1
|
|
1260
|
-
} else {
|
|
1261
|
-
summary.failed += 1
|
|
1262
|
-
}
|
|
1263
|
-
} catch (error) {
|
|
1077
|
+
} else if (result.status === "no-match" || result.status === "no-blob" ||
|
|
1078
|
+
result.status === "size-mismatch") {
|
|
1079
|
+
summary.unmatched += 1
|
|
1080
|
+
} else if (result.status === "metadata-mismatch") {
|
|
1081
|
+
summary.incompatible += 1
|
|
1082
|
+
} else if (result.status === "unavailable" || result.status === "copy-mode") {
|
|
1083
|
+
summary.unavailable += 1
|
|
1084
|
+
} else {
|
|
1264
1085
|
summary.failed += 1
|
|
1265
1086
|
}
|
|
1266
|
-
}
|
|
1267
|
-
|
|
1268
|
-
progress.files_completed += 1
|
|
1269
|
-
this.actionSequence += 1
|
|
1270
|
-
}
|
|
1271
|
-
if (options.onFileComplete) options.onFileComplete()
|
|
1087
|
+
} catch (error) {
|
|
1088
|
+
summary.failed += 1
|
|
1272
1089
|
}
|
|
1090
|
+
if (options.progress) options.progress.files_completed += 1
|
|
1273
1091
|
}
|
|
1274
|
-
registry.schedulePersist()
|
|
1275
1092
|
return summary
|
|
1276
1093
|
}
|
|
1277
|
-
async
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1094
|
+
async deduplicateScope(scopeId, options = {}) {
|
|
1095
|
+
await this.refreshSources()
|
|
1096
|
+
const includeDescendants = options.includeDescendants === true
|
|
1097
|
+
const source = scopeId
|
|
1098
|
+
? this._sources.find((item) => item.id === scopeId && (includeDescendants || item.kind !== "virtual"))
|
|
1099
|
+
: null
|
|
1100
|
+
if ((!includeDescendants || scopeId) && !source) {
|
|
1101
|
+
return { error: "That location is no longer available. Scan again to refresh it." }
|
|
1102
|
+
}
|
|
1103
|
+
if (!includeDescendants) {
|
|
1104
|
+
if (!source.shareable) return { error: "This location is on a disk that cannot share space with this vault." }
|
|
1105
|
+
if (this.sourceAppIsRunning(source)) {
|
|
1106
|
+
return { error: "This app has a running script. Stop it first, then deduplicate." }
|
|
1107
|
+
}
|
|
1108
|
+
}
|
|
1109
|
+
|
|
1110
|
+
const registry = this.registry
|
|
1111
|
+
const batch = options.batch_id || `batch-${crypto.randomUUID()}`
|
|
1112
|
+
const summary = { converted: 0, bytes_saved: 0, locked: 0, stale: 0, incompatible: 0, unavailable: 0, failed: 0 }
|
|
1113
|
+
const staleHashes = new Set()
|
|
1114
|
+
const entries = [...registry.duplicates]
|
|
1115
|
+
const matchesScope = (currentSource) => currentSource && currentSource.shareable &&
|
|
1116
|
+
(includeDescendants ? this.sourceIsWithinScope(currentSource, scopeId) : currentSource.id === scopeId)
|
|
1117
|
+
if (options.progress) {
|
|
1118
|
+
options.progress.files_total = entries.reduce((total, [filePath, entry]) => {
|
|
1119
|
+
const currentSource = this.sourceForPath(filePath, entry.source_id)
|
|
1120
|
+
return total + (matchesScope(currentSource) ? 1 : 0)
|
|
1121
|
+
}, 0)
|
|
1122
|
+
}
|
|
1123
|
+
const completeProgress = () => {
|
|
1124
|
+
if (options.progress) options.progress.files_completed += 1
|
|
1125
|
+
}
|
|
1126
|
+
for (const [filePath, entry] of entries) {
|
|
1127
|
+
const currentSource = this.sourceForPath(filePath, entry.source_id)
|
|
1128
|
+
if (!matchesScope(currentSource)) continue
|
|
1129
|
+
if (!await this.canonicalPathIsWithinSource(filePath, currentSource)) {
|
|
1130
|
+
summary.stale += 1
|
|
1131
|
+
completeProgress()
|
|
1132
|
+
continue
|
|
1308
1133
|
}
|
|
1309
|
-
const
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1134
|
+
const indexed = registry.scanIndex.get(filePath)
|
|
1135
|
+
const expected = indexed && indexed.hash === entry.hash
|
|
1136
|
+
? indexed
|
|
1137
|
+
: (entry.dev !== undefined && entry.ino !== undefined &&
|
|
1138
|
+
entry.mtime !== undefined && entry.ctime !== undefined ? entry : null)
|
|
1139
|
+
if (!expected) {
|
|
1140
|
+
summary.stale += 1
|
|
1141
|
+
completeProgress()
|
|
1142
|
+
continue
|
|
1143
|
+
}
|
|
1144
|
+
if (staleHashes.has(entry.hash)) {
|
|
1145
|
+
summary.stale += 1
|
|
1146
|
+
completeProgress()
|
|
1147
|
+
continue
|
|
1313
1148
|
}
|
|
1149
|
+
try {
|
|
1150
|
+
const result = await this.convert(filePath, entry.hash, {
|
|
1151
|
+
app: currentSource.kind === "app" ? currentSource.app : (entry.app || null),
|
|
1152
|
+
source_id: currentSource.id,
|
|
1153
|
+
batch_id: batch,
|
|
1154
|
+
source: currentSource,
|
|
1155
|
+
expected
|
|
1156
|
+
})
|
|
1157
|
+
if (result.status === "converted" || result.status === "already") {
|
|
1158
|
+
summary.converted += 1
|
|
1159
|
+
summary.bytes_saved += result.bytes_saved || 0
|
|
1160
|
+
} else if (result.status === "locked") {
|
|
1161
|
+
summary.locked += 1
|
|
1162
|
+
} else if (result.status === "stale") {
|
|
1163
|
+
summary.stale += 1
|
|
1164
|
+
} else if (result.status === "stale-blob") {
|
|
1165
|
+
staleHashes.add(entry.hash)
|
|
1166
|
+
summary.stale += 1
|
|
1167
|
+
} else if (result.status === "metadata-mismatch") {
|
|
1168
|
+
entry.unavailable_reason = "metadata"
|
|
1169
|
+
summary.incompatible += 1
|
|
1170
|
+
} else if (result.status === "unavailable" || result.status === "copy-mode") {
|
|
1171
|
+
summary.unavailable += 1
|
|
1172
|
+
} else {
|
|
1173
|
+
summary.failed += 1
|
|
1174
|
+
}
|
|
1175
|
+
} catch (error) {
|
|
1176
|
+
summary.failed += 1
|
|
1177
|
+
}
|
|
1178
|
+
completeProgress()
|
|
1314
1179
|
}
|
|
1180
|
+
registry.schedulePersist()
|
|
1315
1181
|
return summary
|
|
1316
1182
|
}
|
|
1317
1183
|
// reclaim: delete an orphan's store name. Refuses when any app name remains.
|
|
@@ -1321,6 +1187,10 @@ class Vault {
|
|
|
1321
1187
|
if (!this.registry.blobs.has(hash)) return { status: "not-found" }
|
|
1322
1188
|
const st = await this.storeStatIfPresent(storePath)
|
|
1323
1189
|
if (!st) {
|
|
1190
|
+
const hasCopyNames = options.hasCopyNames === undefined
|
|
1191
|
+
? [...this.registry.links.values()].some((entry) => entry.hash === hash && entry.mode === "copy")
|
|
1192
|
+
: options.hasCopyNames
|
|
1193
|
+
if (hasCopyNames) return { status: "unavailable" }
|
|
1324
1194
|
if (!options.deferRegistry) this.registry.removeBlob(hash)
|
|
1325
1195
|
return options.deferRegistry ? { status: "gone", remove_hash: true } : { status: "gone" }
|
|
1326
1196
|
}
|
|
@@ -1345,41 +1215,26 @@ class Vault {
|
|
|
1345
1215
|
if (!this.enabled || !this.registry) return
|
|
1346
1216
|
const preservePath = options.preservePath || (() => false)
|
|
1347
1217
|
const includePath = options.includePath || (() => true)
|
|
1348
|
-
const
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
if (!isAccessError(error) || !options.onAccessError(error, filePath)) return false
|
|
1352
|
-
}
|
|
1353
|
-
if (entry) entry.unverified = true
|
|
1354
|
-
return true
|
|
1355
|
-
}
|
|
1218
|
+
const handleAccessError = (error, filePath) => !!(
|
|
1219
|
+
isAccessError(error) && options.onAccessError && options.onAccessError(error, filePath)
|
|
1220
|
+
)
|
|
1356
1221
|
if (options.reconcile !== false) this.reconcileConfiguredSources()
|
|
1357
1222
|
for (const [linkPath, entry] of [...this.registry.links]) {
|
|
1358
1223
|
if (!includePath(linkPath)) continue
|
|
1359
1224
|
if (preservePath(linkPath)) continue
|
|
1360
1225
|
try {
|
|
1361
1226
|
const source = this.sourceForPath(linkPath, entry.source_id)
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
continue
|
|
1365
|
-
}
|
|
1366
|
-
if (!source) {
|
|
1227
|
+
const managedPath = source && await this.canonicalPathIsWithinSource(linkPath, source, { strictErrors: true })
|
|
1228
|
+
if (!managedPath) {
|
|
1367
1229
|
this.registry.untrack(linkPath)
|
|
1368
1230
|
continue
|
|
1369
1231
|
}
|
|
1370
1232
|
const st = await lstatIfPresent(linkPath)
|
|
1371
|
-
if (!st) {
|
|
1233
|
+
if (!st || !st.isFile() || st.ino !== entry.ino || st.dev !== entry.dev) {
|
|
1372
1234
|
this.registry.untrack(linkPath)
|
|
1373
|
-
continue
|
|
1374
1235
|
}
|
|
1375
|
-
const managedPath = await this.canonicalPathIsWithinSource(
|
|
1376
|
-
linkPath, source, { strictErrors: true })
|
|
1377
|
-
const expected = this.registry.scanIndex.get(linkPath)
|
|
1378
|
-
entry.unverified = !managedPath || !st.isFile() ||
|
|
1379
|
-
st.ino !== entry.ino || st.dev !== entry.dev ||
|
|
1380
|
-
!expected || expected.hash !== entry.hash || !sameSnapshot(expected, st)
|
|
1381
1236
|
} catch (error) {
|
|
1382
|
-
if (!
|
|
1237
|
+
if (!handleAccessError(error, linkPath)) throw error
|
|
1383
1238
|
}
|
|
1384
1239
|
}
|
|
1385
1240
|
for (const [dupPath, entry] of [...this.registry.duplicates]) {
|
|
@@ -1388,28 +1243,17 @@ class Vault {
|
|
|
1388
1243
|
let valid = false
|
|
1389
1244
|
try {
|
|
1390
1245
|
const source = this.sourceForPath(dupPath, entry.source_id)
|
|
1391
|
-
if (source && source.available === false) {
|
|
1392
|
-
entry.unverified = true
|
|
1393
|
-
continue
|
|
1394
|
-
}
|
|
1395
1246
|
const st = await lstatIfPresent(dupPath)
|
|
1396
|
-
|
|
1397
|
-
this.
|
|
1398
|
-
continue
|
|
1399
|
-
}
|
|
1400
|
-
const expected = this.registry.scanIndex.get(dupPath)
|
|
1401
|
-
valid = !!(source && st.isFile() &&
|
|
1402
|
-
await this.canonicalPathIsWithinSource(dupPath, source, { strictErrors: true }) &&
|
|
1403
|
-
expected && expected.hash === entry.hash && sameSnapshot(expected, st))
|
|
1247
|
+
valid = !!(source && st && st.isFile() &&
|
|
1248
|
+
await this.canonicalPathIsWithinSource(dupPath, source, { strictErrors: true }))
|
|
1404
1249
|
} catch (error) {
|
|
1405
|
-
if (
|
|
1250
|
+
if (handleAccessError(error, dupPath)) continue
|
|
1406
1251
|
throw error
|
|
1407
1252
|
}
|
|
1408
1253
|
if (!valid) {
|
|
1409
|
-
|
|
1254
|
+
this.registry.untrack(dupPath)
|
|
1410
1255
|
continue
|
|
1411
1256
|
}
|
|
1412
|
-
entry.unverified = false
|
|
1413
1257
|
const storeStat = await this.storeStatIfPresent(this.storePathFor(entry.hash))
|
|
1414
1258
|
if (storeStat && storeStat.isFile()) {
|
|
1415
1259
|
await unlinkIfSame(dupPath + TMP_SUFFIX, storeStat)
|
|
@@ -1430,6 +1274,7 @@ class Vault {
|
|
|
1430
1274
|
const st = await this.storeStatIfPresent(storePath)
|
|
1431
1275
|
if (!st) {
|
|
1432
1276
|
const names = linksByHash.get(hash) || []
|
|
1277
|
+
const copyNames = names.filter(([, entry]) => entry.mode === "copy")
|
|
1433
1278
|
// Deleting a name changes ctime, so trusted re-adoption compares the
|
|
1434
1279
|
// preserved identity, size, and mtime. Changed or legacy-unsnapshotted
|
|
1435
1280
|
// content waits for the next explicit scan instead of being assigned a
|
|
@@ -1437,6 +1282,7 @@ class Vault {
|
|
|
1437
1282
|
let readopted = false
|
|
1438
1283
|
let inaccessibleName = false
|
|
1439
1284
|
for (const [linkPath, entry] of names) {
|
|
1285
|
+
if (entry.mode !== "link") continue
|
|
1440
1286
|
if (preservePath(linkPath)) {
|
|
1441
1287
|
inaccessibleName = true
|
|
1442
1288
|
continue
|
|
@@ -1448,7 +1294,7 @@ class Vault {
|
|
|
1448
1294
|
if (!source || !await this.canonicalPathIsWithinSource(linkPath, source, { strictErrors: true })) continue
|
|
1449
1295
|
linkStat = await lstatIfPresent(linkPath)
|
|
1450
1296
|
} catch (error) {
|
|
1451
|
-
if (!
|
|
1297
|
+
if (!handleAccessError(error, linkPath)) throw error
|
|
1452
1298
|
inaccessibleName = true
|
|
1453
1299
|
continue
|
|
1454
1300
|
}
|
|
@@ -1486,7 +1332,10 @@ class Vault {
|
|
|
1486
1332
|
readopted = true
|
|
1487
1333
|
break
|
|
1488
1334
|
}
|
|
1489
|
-
|
|
1335
|
+
// On a volume without file sharing support, copy-mode names are the
|
|
1336
|
+
// durable content group. They remain useful for duplicate discovery
|
|
1337
|
+
// even though there is no canonical file in the vault tree.
|
|
1338
|
+
if (readopted || copyNames.length || inaccessibleName) {
|
|
1490
1339
|
blob.orphan = false
|
|
1491
1340
|
blob.verified_at = inaccessibleName && !readopted ? null : Date.now()
|
|
1492
1341
|
} else {
|
|
@@ -1516,14 +1365,9 @@ class Vault {
|
|
|
1516
1365
|
blobs: new Map(registry.blobs),
|
|
1517
1366
|
links: new Map(registry.links),
|
|
1518
1367
|
excluded: new Map(registry.excluded),
|
|
1368
|
+
totals: Object.assign({}, registry.totals),
|
|
1519
1369
|
lastScan: registry.lastScan ? Object.assign({}, registry.lastScan) : null,
|
|
1520
|
-
lastScanAttempt: registry.lastScanAttempt
|
|
1521
|
-
? Object.assign({}, registry.lastScanAttempt)
|
|
1522
|
-
: null,
|
|
1523
1370
|
sourceScans: new Map([...registry.sourceScans].map(([id, scan]) => [id, Object.assign({}, scan)])),
|
|
1524
|
-
sourceScanAttempts: new Map([...registry.sourceScanAttempts]
|
|
1525
|
-
.map(([id, scan]) => [id, Object.assign({}, scan)])),
|
|
1526
|
-
settings: Object.assign({}, registry.settings),
|
|
1527
1371
|
duplicates: new Map(registry.duplicates),
|
|
1528
1372
|
scanIndex: new Map(registry.scanIndex)
|
|
1529
1373
|
} : null
|
|
@@ -1571,18 +1415,7 @@ class Vault {
|
|
|
1571
1415
|
await this.refreshSources()
|
|
1572
1416
|
const walkRoots = roots
|
|
1573
1417
|
? roots.map((root) => typeof root === "string" ? { root, source_id: null } : root)
|
|
1574
|
-
:
|
|
1575
|
-
{ root: path.resolve(this.kernel.homedir), source_id: "pinokio" },
|
|
1576
|
-
...this._sources
|
|
1577
|
-
.filter((source) => source.kind === "external" && source.root)
|
|
1578
|
-
.map((source) => ({ root: source.root, source_id: source.id }))
|
|
1579
|
-
]
|
|
1580
|
-
for (const entry of walkRoots) {
|
|
1581
|
-
const rootStat = await fs.promises.lstat(entry.root)
|
|
1582
|
-
if (!rootStat.isDirectory() || rootStat.isSymbolicLink()) {
|
|
1583
|
-
throw new Error(`Repair source is not a real directory: ${entry.root}`)
|
|
1584
|
-
}
|
|
1585
|
-
}
|
|
1418
|
+
: this.scanRoots()
|
|
1586
1419
|
for (const entry of walkRoots) {
|
|
1587
1420
|
await this.walkForInoMatch(
|
|
1588
1421
|
entry.root,
|
|
@@ -1595,6 +1428,38 @@ class Vault {
|
|
|
1595
1428
|
const rebuiltScanIndex = new Map()
|
|
1596
1429
|
const trustedHashes = new Set()
|
|
1597
1430
|
if (preserved) {
|
|
1431
|
+
// Copy-mode content has no store inode to rediscover. Preserve only names
|
|
1432
|
+
// whose exact scan snapshot still proves the recorded content group.
|
|
1433
|
+
for (const [linkPath, link] of preserved.links) {
|
|
1434
|
+
if (link.mode !== "copy" || preserved.excluded.has(linkPath)) continue
|
|
1435
|
+
const blob = preserved.blobs.get(link.hash)
|
|
1436
|
+
const expected = preserved.scanIndex.get(linkPath)
|
|
1437
|
+
const source = this.sourceForPath(linkPath, link.source_id)
|
|
1438
|
+
if (!blob || !expected || expected.hash !== link.hash || !source ||
|
|
1439
|
+
!await this.canonicalPathIsWithinSource(linkPath, source, { strictErrors: true })) continue
|
|
1440
|
+
try {
|
|
1441
|
+
const st = await fs.promises.lstat(linkPath)
|
|
1442
|
+
if (!st.isFile() || st.size < this.sizeThreshold ||
|
|
1443
|
+
link.dev !== st.dev || link.ino !== st.ino ||
|
|
1444
|
+
!sameSnapshot(expected, st)) continue
|
|
1445
|
+
if (!rebuiltBlobs.has(link.hash)) {
|
|
1446
|
+
rebuiltBlobs.set(link.hash, Object.assign({}, blob, {
|
|
1447
|
+
size: st.size,
|
|
1448
|
+
verified_at: Date.now(),
|
|
1449
|
+
orphan: false
|
|
1450
|
+
}))
|
|
1451
|
+
}
|
|
1452
|
+
rebuiltLinks.set(linkPath, Object.assign({}, link, {
|
|
1453
|
+
app: source.kind === "app" ? source.app : (link.app || null),
|
|
1454
|
+
source_id: source.id,
|
|
1455
|
+
dev: st.dev,
|
|
1456
|
+
ino: st.ino,
|
|
1457
|
+
mode: "copy"
|
|
1458
|
+
}))
|
|
1459
|
+
} catch (error) {
|
|
1460
|
+
if (!isMissingError(error)) throw error
|
|
1461
|
+
}
|
|
1462
|
+
}
|
|
1598
1463
|
// A store filename is only a trustworthy content hash when at least one
|
|
1599
1464
|
// surviving name still matches the snapshot captured when that content
|
|
1600
1465
|
// was hashed. Repair itself never hashes, so unverified names are left
|
|
@@ -1666,10 +1531,8 @@ class Vault {
|
|
|
1666
1531
|
duplicates: rebuiltDuplicates,
|
|
1667
1532
|
excluded: preserved ? preserved.excluded : new Map(),
|
|
1668
1533
|
lastScan: preserved ? preserved.lastScan : null,
|
|
1669
|
-
lastScanAttempt: preserved ? preserved.lastScanAttempt : null,
|
|
1670
1534
|
sourceScans: preserved ? preserved.sourceScans : new Map(),
|
|
1671
|
-
|
|
1672
|
-
settings: preserved ? preserved.settings : {},
|
|
1535
|
+
totals: preserved ? preserved.totals : { lifetime_bytes_saved: 0 },
|
|
1673
1536
|
byIno: rebuiltByIno,
|
|
1674
1537
|
pathsByIno: rebuiltPathsByIno
|
|
1675
1538
|
})
|
|
@@ -1680,8 +1543,7 @@ class Vault {
|
|
|
1680
1543
|
for await (const batch of walkBatches(root, {
|
|
1681
1544
|
concurrency: this.dirConcurrency,
|
|
1682
1545
|
skipDirectory: (full) => full === vaultRoot,
|
|
1683
|
-
strictErrors: true
|
|
1684
|
-
onRootMissing: (error) => { throw error }
|
|
1546
|
+
strictErrors: true
|
|
1685
1547
|
})) {
|
|
1686
1548
|
const files = batch.flatMap((group) => group.files.map((file) => file.path))
|
|
1687
1549
|
const stats = await statMany(files, this.statConcurrency, null, {
|
|
@@ -1690,19 +1552,19 @@ class Vault {
|
|
|
1690
1552
|
})
|
|
1691
1553
|
for (let index = 0; index < files.length; index++) {
|
|
1692
1554
|
const st = stats[index]
|
|
1693
|
-
if (!st || !st.isFile()) continue
|
|
1555
|
+
if (!st || !st.isFile() || st.size < this.sizeThreshold) continue
|
|
1694
1556
|
const hash = storeInoToHash.get(`${st.dev}:${st.ino}`)
|
|
1695
1557
|
if (hash) {
|
|
1696
1558
|
const source = this.sourceForPath(files[index], preferredSourceId)
|
|
1697
1559
|
const previous = this.registry.links.get(files[index])
|
|
1698
|
-
const batchId = previous && previous.hash === hash &&
|
|
1560
|
+
const batchId = previous && previous.hash === hash && previous.mode === "link" &&
|
|
1699
1561
|
previous.dev === st.dev && previous.ino === st.ino
|
|
1700
1562
|
? previous.batch_id || null
|
|
1701
1563
|
: null
|
|
1702
1564
|
const link = {
|
|
1703
1565
|
hash, app: source && source.kind === "app" ? source.app : null,
|
|
1704
1566
|
source_id: source ? source.id : null, dev: st.dev, ino: st.ino,
|
|
1705
|
-
created: Date.now(),
|
|
1567
|
+
mode: "link", created: Date.now(),
|
|
1706
1568
|
batch_id: batchId
|
|
1707
1569
|
}
|
|
1708
1570
|
if (outputLinks) outputLinks.set(files[index], link)
|
|
@@ -1722,7 +1584,6 @@ class Vault {
|
|
|
1722
1584
|
// memory + one stat per blob and occupied shard; never a walk.
|
|
1723
1585
|
async status(scopeId = null) {
|
|
1724
1586
|
if (!this.enabled || !this.registry) return { enabled: false }
|
|
1725
|
-
const responseActionSequence = this.actionSequence
|
|
1726
1587
|
const registry = this.registry
|
|
1727
1588
|
const scope = scopeId ? this.scanSource(scopeId) : null
|
|
1728
1589
|
if (scopeId && !scope) throw new Error("That location is no longer available.")
|
|
@@ -1745,8 +1606,7 @@ class Vault {
|
|
|
1745
1606
|
if (!namesByHash.has(entry.hash)) namesByHash.set(entry.hash, [])
|
|
1746
1607
|
const location = this.locationForPath(linkPath, entry.source_id)
|
|
1747
1608
|
namesByHash.get(entry.hash).push(Object.assign({
|
|
1748
|
-
path: linkPath, app: entry.app || null, mode:
|
|
1749
|
-
unverified: !!entry.unverified
|
|
1609
|
+
path: linkPath, app: entry.app || null, mode: entry.mode
|
|
1750
1610
|
}, location))
|
|
1751
1611
|
if (entry.batch_id && (!scopeId || entry.source_id === scopeId)) {
|
|
1752
1612
|
if (!undoBatchMap.has(entry.batch_id)) {
|
|
@@ -1758,6 +1618,11 @@ class Vault {
|
|
|
1758
1618
|
batch.bytes += blob && Number.isFinite(blob.size) ? blob.size : 0
|
|
1759
1619
|
}
|
|
1760
1620
|
}
|
|
1621
|
+
const pendingBytesByHash = new Map()
|
|
1622
|
+
for (const entry of registry.duplicates.values()) {
|
|
1623
|
+
if (scopeId && entry.source_id !== scopeId) continue
|
|
1624
|
+
pendingBytesByHash.set(entry.hash, (pendingBytesByHash.get(entry.hash) || 0) + (entry.size || 0))
|
|
1625
|
+
}
|
|
1761
1626
|
const blobEntries = [...registry.blobs].filter(([hash]) => !scopeHashes || scopeHashes.has(hash))
|
|
1762
1627
|
if (!await this.directoryIfSafe(this.root) || !await this.directoryIfSafe(this.blobRoot)) {
|
|
1763
1628
|
throw unsafeStoragePath(this.blobRoot)
|
|
@@ -1771,7 +1636,8 @@ class Vault {
|
|
|
1771
1636
|
null,
|
|
1772
1637
|
{ strictErrors: true, followSymlinks: false }
|
|
1773
1638
|
)
|
|
1774
|
-
let
|
|
1639
|
+
let bytesOnDisk = 0
|
|
1640
|
+
let wouldBe = 0
|
|
1775
1641
|
for (let index = 0; index < blobEntries.length; index++) {
|
|
1776
1642
|
const [hash, blob] = blobEntries[index]
|
|
1777
1643
|
const names = namesByHash.get(hash) || []
|
|
@@ -1780,17 +1646,14 @@ class Vault {
|
|
|
1780
1646
|
const nlink = storeStat ? storeStat.nlink : null
|
|
1781
1647
|
storeStats.set(hash, storeStat)
|
|
1782
1648
|
const size = blob.size || 0
|
|
1783
|
-
const
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
1788
|
-
}
|
|
1789
|
-
}
|
|
1649
|
+
const linkedCopy = storeStat || names.some((name) => name.mode === "link") ? 1 : 0
|
|
1650
|
+
const physicalCopies = linkedCopy + names.filter((name) => name.mode === "copy").length
|
|
1651
|
+
const pendingBytes = pendingBytesByHash.get(hash) || 0
|
|
1652
|
+
bytesOnDisk += (size * physicalCopies) + pendingBytes
|
|
1653
|
+
wouldBe += (size * Math.max(physicalCopies, names.length)) + pendingBytes
|
|
1790
1654
|
const orphan = !!(storeStat && storeStat.nlink === 1)
|
|
1791
1655
|
const publicBlob = {
|
|
1792
1656
|
hash, size: blob.size || 0, orphan, nlink,
|
|
1793
|
-
sharing_locations: liveNames,
|
|
1794
1657
|
names, apps: [...apps], source_urls: blob.source_urls || []
|
|
1795
1658
|
}
|
|
1796
1659
|
blobs.push(publicBlob)
|
|
@@ -1803,8 +1666,7 @@ class Vault {
|
|
|
1803
1666
|
const source = this.sourceForPath(p, location.source_id)
|
|
1804
1667
|
const index = registry.scanIndex.get(p)
|
|
1805
1668
|
const storeStat = storeStats.get(entry.hash)
|
|
1806
|
-
const shareable = !!(source && source.shareable && storeStat && !entry.
|
|
1807
|
-
!entry.unavailable_reason &&
|
|
1669
|
+
const shareable = !!(source && source.shareable && storeStat && !entry.unavailable_reason &&
|
|
1808
1670
|
(!index || index.dev === undefined || index.dev === storeStat.dev))
|
|
1809
1671
|
let unavailableReason = entry.unavailable_reason || null
|
|
1810
1672
|
if (!shareable && !unavailableReason) {
|
|
@@ -1816,8 +1678,7 @@ class Vault {
|
|
|
1816
1678
|
const match = blob ? blob.names.find((name) => name.path !== p) || null : null
|
|
1817
1679
|
duplicates.push(Object.assign({
|
|
1818
1680
|
path: p, hash: entry.hash, size: entry.size || 0, app: entry.app || null,
|
|
1819
|
-
shareable,
|
|
1820
|
-
unavailable_reason: entry.unverified ? "unavailable" : unavailableReason, match
|
|
1681
|
+
shareable, unavailable_reason: unavailableReason, match
|
|
1821
1682
|
}, location))
|
|
1822
1683
|
}
|
|
1823
1684
|
const events = (await registry.readEvents()).reverse()
|
|
@@ -1850,64 +1711,64 @@ class Vault {
|
|
|
1850
1711
|
excluded.push(Object.assign({
|
|
1851
1712
|
path: p,
|
|
1852
1713
|
ts: meta.ts || null,
|
|
1853
|
-
size: Number(meta.size) || 0
|
|
1854
|
-
unverified: !!meta.unverified
|
|
1714
|
+
size: Number(meta.size) || 0
|
|
1855
1715
|
}, this.locationForPath(p, meta.source_id)))
|
|
1856
1716
|
}
|
|
1857
1717
|
const publicSources = this._sources.filter((source) => !scopeId || source.id === scopeId).map((source) => ({
|
|
1858
1718
|
id: source.id, kind: source.kind, label: source.label, root: source.root,
|
|
1859
|
-
display_path: source.root,
|
|
1719
|
+
display_path: source.kind === "pinokio" ? source.root : (source.mount_path || source.root),
|
|
1860
1720
|
target_path: source.kind === "external" ? source.root : null,
|
|
1861
1721
|
parent_id: scopeId ? null : source.parent_id, app: source.app || null,
|
|
1862
1722
|
available: source.available !== false, shareable: source.kind === "virtual" ? null : !!source.shareable
|
|
1863
1723
|
}))
|
|
1864
|
-
const lastComplete = registry.scanFor(scopeId)
|
|
1865
|
-
const lastAttempt = registry.scanAttemptFor(scopeId)
|
|
1866
|
-
const inScope = (entry) => !scopeId || entry.source_id === scopeId
|
|
1867
|
-
const hasUnavailableSource = !scopeId && this._sources.some((source) =>
|
|
1868
|
-
source.kind === "external" && source.available === false)
|
|
1869
|
-
const hasUnverified = hasUnavailableSource || [...registry.links.values()].some((entry) =>
|
|
1870
|
-
inScope(entry) && entry.unverified) ||
|
|
1871
|
-
[...registry.duplicates.values()].some((entry) =>
|
|
1872
|
-
inScope(entry) && entry.unverified) ||
|
|
1873
|
-
[...registry.excluded.values()].some((entry) =>
|
|
1874
|
-
inScope(entry) && entry.unverified)
|
|
1875
|
-
const incomplete = hasUnverified || !!(
|
|
1876
|
-
lastAttempt && lastAttempt.complete === false &&
|
|
1877
|
-
(!lastComplete || (lastAttempt.ts || 0) >= (lastComplete.ts || 0))
|
|
1878
|
-
)
|
|
1879
|
-
const metricsExact = !!lastComplete && !incomplete
|
|
1880
|
-
const beforeBytes = metricsExact && Number.isFinite(lastComplete.bytes_total)
|
|
1881
|
-
? Math.max(0, lastComplete.bytes_total)
|
|
1882
|
-
: null
|
|
1883
|
-
const afterBytes = beforeBytes === null
|
|
1884
|
-
? null
|
|
1885
|
-
: Math.max(0, beforeBytes - savedBySharing)
|
|
1886
1724
|
const result = {
|
|
1887
1725
|
enabled: true,
|
|
1888
1726
|
mode: this.mode,
|
|
1889
|
-
candidate_min_bytes: this.sizeThreshold,
|
|
1890
|
-
candidate_size_options: CANDIDATE_SIZE_OPTIONS,
|
|
1891
1727
|
scan,
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
incomplete,
|
|
1898
|
-
metrics_exact: metricsExact,
|
|
1899
|
-
bytes_on_disk: afterBytes,
|
|
1900
|
-
bytes_without_sharing: beforeBytes,
|
|
1901
|
-
saved_by_sharing: savedBySharing,
|
|
1728
|
+
last_scan: registry.scanFor(scopeId),
|
|
1729
|
+
bytes_on_disk: bytesOnDisk,
|
|
1730
|
+
bytes_without_sharing: wouldBe,
|
|
1731
|
+
saved_by_sharing: wouldBe - bytesOnDisk,
|
|
1732
|
+
lifetime_bytes_saved: registry.totals.lifetime_bytes_saved,
|
|
1902
1733
|
reclaimable: blobs.filter((b) => b.orphan).reduce((sum, b) => sum + b.size, 0),
|
|
1903
1734
|
pending_bytes: duplicates.filter((d) => d.shareable).reduce((sum, d) => sum + d.size, 0),
|
|
1735
|
+
file_action: this.fileActionStatus(scopeId),
|
|
1904
1736
|
activity_error: registry.eventError,
|
|
1905
1737
|
cloud_sync_warning: this.cloudSyncProvider(),
|
|
1906
1738
|
sources: publicSources, blobs, duplicates, excluded, events,
|
|
1907
1739
|
undo_batches: undoBatches
|
|
1908
1740
|
}
|
|
1909
1741
|
if (scopeId) {
|
|
1742
|
+
const duplicateBytes = duplicates.reduce((sum, item) => sum + item.size, 0)
|
|
1743
|
+
const excludedBytes = excluded.reduce((sum, item) => sum + item.size, 0)
|
|
1744
|
+
let linkedBytes = 0
|
|
1745
|
+
let effectiveLinkedBytes = 0
|
|
1746
|
+
let sharedBytes = 0
|
|
1747
|
+
for (const blob of blobs) {
|
|
1748
|
+
const scopedNames = blob.names.filter((name) => name.source_id === scopeId)
|
|
1749
|
+
const scopedLinks = scopedNames.filter((name) => name.mode === "link").length
|
|
1750
|
+
const scopedCopies = scopedNames.length - scopedLinks
|
|
1751
|
+
const registeredLinks = blob.names.filter((name) => name.mode === "link").length
|
|
1752
|
+
// The store name is one hardlink but not a user-visible location.
|
|
1753
|
+
// Registry count prevents stale filesystem metadata from over-attributing the inode.
|
|
1754
|
+
const filesystemLinks = Number.isFinite(blob.nlink) ? Math.max(0, blob.nlink - 1) : 0
|
|
1755
|
+
const sharingLocations = Math.max(1, registeredLinks, filesystemLinks)
|
|
1756
|
+
linkedBytes += blob.size * scopedNames.length
|
|
1757
|
+
effectiveLinkedBytes += (blob.size * scopedCopies) +
|
|
1758
|
+
(blob.size * scopedLinks / sharingLocations)
|
|
1759
|
+
if (registeredLinks >= 2) sharedBytes += blob.size * scopedLinks
|
|
1760
|
+
}
|
|
1761
|
+
const trackedBytes = linkedBytes + duplicateBytes + excludedBytes
|
|
1762
|
+
const effectiveTrackedBytes = effectiveLinkedBytes + duplicateBytes + excludedBytes
|
|
1763
|
+
const folderBytes = result.last_scan && Number.isFinite(result.last_scan.bytes_total)
|
|
1764
|
+
? result.last_scan.bytes_total
|
|
1765
|
+
: null
|
|
1910
1766
|
result.scope_id = scopeId
|
|
1767
|
+
result.tracked_bytes = trackedBytes
|
|
1768
|
+
result.effective_bytes = folderBytes === null
|
|
1769
|
+
? null
|
|
1770
|
+
: Math.max(0, folderBytes - trackedBytes) + effectiveTrackedBytes
|
|
1771
|
+
result.shared_bytes = sharedBytes
|
|
1911
1772
|
result.reclaimable = 0
|
|
1912
1773
|
}
|
|
1913
1774
|
return result
|
|
@@ -1922,176 +1783,34 @@ class Vault {
|
|
|
1922
1783
|
return Object.assign({}, state, {
|
|
1923
1784
|
current_file: sweeper.currentHash ? path.basename(sweeper.currentHash.path) : null,
|
|
1924
1785
|
pending,
|
|
1925
|
-
error: this.scanError
|
|
1926
|
-
persistence_warning: this.scanPersistenceWarning &&
|
|
1927
|
-
!!(this.registry && this.registry.persistDirty)
|
|
1786
|
+
error: this.scanError
|
|
1928
1787
|
})
|
|
1929
1788
|
}
|
|
1789
|
+
fileActionStatus(scopeId = null) {
|
|
1790
|
+
const progress = this.fileActionProgress
|
|
1791
|
+
if (!progress) return null
|
|
1792
|
+
if (scopeId) {
|
|
1793
|
+
if (progress.scope_id && progress.scope_id !== scopeId) return null
|
|
1794
|
+
if (progress.path) {
|
|
1795
|
+
const source = this.sourceForPath(progress.path)
|
|
1796
|
+
if (!source || source.id !== scopeId) return null
|
|
1797
|
+
}
|
|
1798
|
+
}
|
|
1799
|
+
return Object.assign({}, progress)
|
|
1800
|
+
}
|
|
1930
1801
|
progressStatus(scopeId = null) {
|
|
1931
1802
|
return {
|
|
1932
1803
|
enabled: !!this.enabled,
|
|
1933
|
-
candidate_min_bytes: this.sizeThreshold,
|
|
1934
|
-
candidate_size_options: CANDIDATE_SIZE_OPTIONS,
|
|
1935
1804
|
scan: this.scanStatus(),
|
|
1936
1805
|
file_action: this.fileActionStatus(scopeId),
|
|
1937
|
-
|
|
1938
|
-
action_sequence: this.actionSequence,
|
|
1939
|
-
last_scan: this.registry ? this.registry.scanFor(scopeId) : null,
|
|
1940
|
-
last_attempt: this.registry ? this.registry.scanAttemptFor(scopeId) : null
|
|
1941
|
-
}
|
|
1942
|
-
}
|
|
1943
|
-
fileActionStatus(scopeId = null) {
|
|
1944
|
-
const action = this.fileAction
|
|
1945
|
-
if (!action || (scopeId && action.source_id !== scopeId)) return null
|
|
1946
|
-
return {
|
|
1947
|
-
id: action.id,
|
|
1948
|
-
kind: action.kind,
|
|
1949
|
-
phase: action.phase,
|
|
1950
|
-
path: action.path,
|
|
1951
|
-
source_id: action.source_id,
|
|
1952
|
-
bytes_copied: action.bytes_copied,
|
|
1953
|
-
bytes_total: action.bytes_total,
|
|
1954
|
-
files_completed: action.files_completed,
|
|
1955
|
-
files_total: action.files_total
|
|
1956
|
-
}
|
|
1957
|
-
}
|
|
1958
|
-
fileActionStatuses(scopeId = null) {
|
|
1959
|
-
return [...this.fileActions.values()]
|
|
1960
|
-
.filter((action) => !scopeId || action.source_id === scopeId)
|
|
1961
|
-
.map((action) => ({
|
|
1962
|
-
id: action.id,
|
|
1963
|
-
kind: action.kind,
|
|
1964
|
-
phase: action.phase,
|
|
1965
|
-
path: action.path,
|
|
1966
|
-
source_id: action.source_id,
|
|
1967
|
-
bytes_copied: action.bytes_copied,
|
|
1968
|
-
bytes_total: action.bytes_total,
|
|
1969
|
-
files_completed: action.files_completed,
|
|
1970
|
-
files_total: action.files_total
|
|
1971
|
-
}))
|
|
1972
|
-
}
|
|
1973
|
-
navigationStatus(scopeId = null) {
|
|
1974
|
-
if (!this.enabled || !this.initialized || !this.registry) {
|
|
1975
|
-
return {
|
|
1976
|
-
any_scan: false,
|
|
1977
|
-
scanned: false,
|
|
1978
|
-
scanning: false,
|
|
1979
|
-
pending_bytes: 0
|
|
1980
|
-
}
|
|
1981
|
-
}
|
|
1982
|
-
const scan = this.scanStatus()
|
|
1983
|
-
const active = !!(scan && (scan.pending || scan.active || scan.queued > 0))
|
|
1984
|
-
const sources = new Map(this._sources.map((source) => [source.id, source]))
|
|
1985
|
-
let pendingBytes = 0
|
|
1986
|
-
for (const entry of this.registry.duplicates.values()) {
|
|
1987
|
-
if (scopeId && entry.source_id !== scopeId) continue
|
|
1988
|
-
const source = sources.get(entry.source_id)
|
|
1989
|
-
if (!source || source.available === false || !source.shareable ||
|
|
1990
|
-
entry.unverified || entry.unavailable_reason ||
|
|
1991
|
-
!this.registry.blobs.has(entry.hash)) continue
|
|
1992
|
-
pendingBytes += Math.max(0, Number(entry.size) || 0)
|
|
1993
|
-
}
|
|
1994
|
-
const lastComplete = this.registry.scanFor(scopeId)
|
|
1995
|
-
const lastAttempt = this.registry.scanAttemptFor(scopeId)
|
|
1996
|
-
const hasUnavailableSource = !scopeId && this._sources.some((source) =>
|
|
1997
|
-
source.kind === "external" && source.available === false)
|
|
1998
|
-
const hasUnverified = hasUnavailableSource || [...this.registry.links.values()].some((entry) =>
|
|
1999
|
-
(!scopeId || entry.source_id === scopeId) && entry.unverified) ||
|
|
2000
|
-
[...this.registry.duplicates.values()].some((entry) =>
|
|
2001
|
-
(!scopeId || entry.source_id === scopeId) && entry.unverified) ||
|
|
2002
|
-
[...this.registry.excluded.values()].some((entry) =>
|
|
2003
|
-
(!scopeId || entry.source_id === scopeId) && entry.unverified)
|
|
2004
|
-
const incomplete = hasUnverified || !!(
|
|
2005
|
-
lastAttempt && lastAttempt.complete === false &&
|
|
2006
|
-
(!lastComplete || (lastAttempt.ts || 0) >= (lastComplete.ts || 0))
|
|
2007
|
-
)
|
|
2008
|
-
return {
|
|
2009
|
-
any_scan: !!this.registry.lastScanAttempt || this.registry.sourceScanAttempts.size > 0,
|
|
2010
|
-
scanned: !!lastAttempt,
|
|
2011
|
-
scanning: active && (!scopeId || scan.scope_id === scopeId),
|
|
2012
|
-
pending_bytes: pendingBytes,
|
|
2013
|
-
incomplete
|
|
2014
|
-
}
|
|
2015
|
-
}
|
|
2016
|
-
navigationStatusForApp(appName) {
|
|
2017
|
-
return this.navigationStatus(sourceId("app", appName))
|
|
2018
|
-
}
|
|
2019
|
-
async preflightCopyCapacity(targets) {
|
|
2020
|
-
if (!targets.length) return { ok: true }
|
|
2021
|
-
if (!fs.promises.statfs) return { ok: false, status: "capacity-unavailable" }
|
|
2022
|
-
const byDevice = new Map()
|
|
2023
|
-
try {
|
|
2024
|
-
for (const target of targets) {
|
|
2025
|
-
const st = await fs.promises.lstat(target.path)
|
|
2026
|
-
const key = String(st.dev)
|
|
2027
|
-
if (!byDevice.has(key)) {
|
|
2028
|
-
byDevice.set(key, { path: path.dirname(target.path), bytes: 0 })
|
|
2029
|
-
}
|
|
2030
|
-
byDevice.get(key).bytes += Math.max(0, Number(target.bytes) || 0)
|
|
2031
|
-
}
|
|
2032
|
-
for (const group of byDevice.values()) {
|
|
2033
|
-
const stat = await fs.promises.statfs(group.path)
|
|
2034
|
-
const blockSize = BigInt(stat.bsize)
|
|
2035
|
-
const availableBlocks = BigInt(stat.bavail)
|
|
2036
|
-
const available = blockSize * availableBlocks
|
|
2037
|
-
if (available < BigInt(Math.ceil(group.bytes))) {
|
|
2038
|
-
return { ok: false, status: "insufficient-space" }
|
|
2039
|
-
}
|
|
2040
|
-
}
|
|
2041
|
-
return { ok: true }
|
|
2042
|
-
} catch (error) {
|
|
2043
|
-
return { ok: false, status: "capacity-unavailable" }
|
|
1806
|
+
last_scan: this.registry ? this.registry.scanFor(scopeId) : null
|
|
2044
1807
|
}
|
|
2045
1808
|
}
|
|
2046
1809
|
async copyOut(filePath, sourcePath, expectedTarget, expectedSource = expectedTarget, options = {}) {
|
|
2047
1810
|
const tmp = filePath + TMP_SUFFIX
|
|
2048
1811
|
let copiedStat = null
|
|
2049
1812
|
try {
|
|
2050
|
-
|
|
2051
|
-
let sourceHandle = null
|
|
2052
|
-
let targetHandle = null
|
|
2053
|
-
try {
|
|
2054
|
-
sourceHandle = await fs.promises.open(sourcePath, "r")
|
|
2055
|
-
const sourceStat = await sourceHandle.stat()
|
|
2056
|
-
targetHandle = await fs.promises.open(tmp,
|
|
2057
|
-
fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL,
|
|
2058
|
-
sourceStat.mode & 0o7777)
|
|
2059
|
-
copiedStat = await targetHandle.stat()
|
|
2060
|
-
const buffer = Buffer.allocUnsafe(COPY_PROGRESS_CHUNK_SIZE)
|
|
2061
|
-
let offset = 0
|
|
2062
|
-
options.onProgress(0, sourceStat.size)
|
|
2063
|
-
while (offset < sourceStat.size) {
|
|
2064
|
-
const length = Math.min(buffer.length, sourceStat.size - offset)
|
|
2065
|
-
const { bytesRead } = await sourceHandle.read(buffer, 0, length, offset)
|
|
2066
|
-
if (!bytesRead) {
|
|
2067
|
-
const error = new Error("Source file ended while copying.")
|
|
2068
|
-
error.code = "EIO"
|
|
2069
|
-
throw error
|
|
2070
|
-
}
|
|
2071
|
-
let written = 0
|
|
2072
|
-
while (written < bytesRead) {
|
|
2073
|
-
const result = await targetHandle.write(
|
|
2074
|
-
buffer, written, bytesRead - written, offset + written)
|
|
2075
|
-
if (!result.bytesWritten) {
|
|
2076
|
-
const error = new Error("Could not write the independent copy.")
|
|
2077
|
-
error.code = "EIO"
|
|
2078
|
-
throw error
|
|
2079
|
-
}
|
|
2080
|
-
written += result.bytesWritten
|
|
2081
|
-
}
|
|
2082
|
-
offset += bytesRead
|
|
2083
|
-
options.onProgress(offset, sourceStat.size)
|
|
2084
|
-
}
|
|
2085
|
-
await targetHandle.chmod(sourceStat.mode & 0o7777)
|
|
2086
|
-
} finally {
|
|
2087
|
-
await Promise.all([
|
|
2088
|
-
sourceHandle && sourceHandle.close(),
|
|
2089
|
-
targetHandle && targetHandle.close()
|
|
2090
|
-
].filter(Boolean))
|
|
2091
|
-
}
|
|
2092
|
-
} else {
|
|
2093
|
-
await fs.promises.copyFile(sourcePath, tmp, fs.constants.COPYFILE_EXCL)
|
|
2094
|
-
}
|
|
1813
|
+
await fs.promises.copyFile(sourcePath, tmp, fs.constants.COPYFILE_EXCL)
|
|
2095
1814
|
copiedStat = await fs.promises.lstat(tmp)
|
|
2096
1815
|
if (options.canReplace && !options.canReplace()) {
|
|
2097
1816
|
await unlinkIfSame(tmp, copiedStat)
|
|
@@ -2110,12 +1829,9 @@ class Vault {
|
|
|
2110
1829
|
try {
|
|
2111
1830
|
finalStat = await lstatIfPresent(filePath)
|
|
2112
1831
|
} catch (error) {
|
|
2113
|
-
//
|
|
2114
|
-
//
|
|
2115
|
-
|
|
2116
|
-
const committedStat = copiedStat
|
|
2117
|
-
copiedStat = null
|
|
2118
|
-
return { status: "copied", stat: committedStat, unverified: true }
|
|
1832
|
+
// The independent copy was committed by rename(). Preserve that
|
|
1833
|
+
// completed action when only the post-commit metadata read failed.
|
|
1834
|
+
finalStat = copiedStat
|
|
2119
1835
|
}
|
|
2120
1836
|
if (!finalStat || finalStat.dev !== copiedStat.dev || finalStat.ino !== copiedStat.ino) {
|
|
2121
1837
|
return { status: "stale" }
|
|
@@ -2129,100 +1845,54 @@ class Vault {
|
|
|
2129
1845
|
throw error
|
|
2130
1846
|
}
|
|
2131
1847
|
}
|
|
2132
|
-
// detach:
|
|
2133
|
-
//
|
|
2134
|
-
|
|
1848
|
+
// detach: make one name an independent copy again ("go back"), and pin it
|
|
1849
|
+
// so future scans neither re-list nor re-share it. Works on linked names
|
|
1850
|
+
// (copies bytes out) and on pending duplicates (just ignores them).
|
|
1851
|
+
async detach(filePath) {
|
|
2135
1852
|
if (!this.enabled) return { status: "disabled" }
|
|
2136
1853
|
await this.refreshSources()
|
|
2137
1854
|
const entry = this.registry.links.get(filePath)
|
|
2138
1855
|
if (entry) {
|
|
2139
|
-
if (entry.unverified) return { status: "unavailable" }
|
|
2140
1856
|
const source = this.sourceForPath(filePath, entry.source_id)
|
|
2141
1857
|
if (!source || !await this.canonicalPathIsWithinSource(filePath, source)) {
|
|
2142
1858
|
return { status: "stale" }
|
|
2143
1859
|
}
|
|
2144
|
-
if (
|
|
2145
|
-
|
|
2146
|
-
|
|
2147
|
-
|
|
2148
|
-
|
|
2149
|
-
const action = queuedAction || {
|
|
2150
|
-
kind: "detach", path: filePath, source_id: entry.source_id || null,
|
|
2151
|
-
bytes_copied: 0, bytes_total: before.size
|
|
2152
|
-
}
|
|
2153
|
-
action.bytes_total = before.size
|
|
2154
|
-
const ownsAction = !queuedAction
|
|
2155
|
-
if (ownsAction) this.fileAction = action
|
|
2156
|
-
try {
|
|
2157
|
-
const copied = await this.copyOut(filePath, filePath, fileSnapshot(before), fileSnapshot(before), {
|
|
2158
|
-
canReplace: () => !this.sourceAppIsRunning(source),
|
|
2159
|
-
onProgress: (bytesCopied, bytesTotal) => {
|
|
2160
|
-
action.bytes_copied = bytesCopied
|
|
2161
|
-
action.bytes_total = bytesTotal
|
|
2162
|
-
}
|
|
2163
|
-
})
|
|
2164
|
-
if (copied.status !== "copied") return copied
|
|
2165
|
-
const st = copied.stat
|
|
2166
|
-
const baseEntry = {
|
|
2167
|
-
hash: entry.hash, size: st.size, source_id: entry.source_id || null,
|
|
2168
|
-
dev: st.dev, ino: st.ino
|
|
2169
|
-
}
|
|
2170
|
-
const scanEntry = copied.unverified ? null : Object.assign({}, baseEntry, {
|
|
2171
|
-
mtime: st.mtimeMs, ctime: st.ctimeMs
|
|
1860
|
+
if (entry.mode === "copy") {
|
|
1861
|
+
const st = await lstatIfPresent(filePath)
|
|
1862
|
+
if (!st || !st.isFile() || st.dev !== entry.dev || st.ino !== entry.ino) return { status: "stale" }
|
|
1863
|
+
this.registry.exclude(filePath, {
|
|
1864
|
+
ts: Date.now(), source_id: entry.source_id || null, size: st.size
|
|
2172
1865
|
})
|
|
2173
|
-
this.registry.setDuplicate(filePath, Object.assign({
|
|
2174
|
-
app: entry.app || null, discovered: Date.now(),
|
|
2175
|
-
unverified: !!copied.unverified
|
|
2176
|
-
}, scanEntry || baseEntry), scanEntry)
|
|
2177
|
-
if (copied.unverified) this.registry.scanIndex.delete(filePath)
|
|
2178
|
-
await this.refreshLinkSnapshots(entry.hash, entry.dev, entry.ino)
|
|
2179
1866
|
await this.recordEvent({
|
|
2180
|
-
kind: "
|
|
2181
|
-
app: entry.app || null, source_id: entry.source_id || null,
|
|
2182
|
-
size: st.size, bytes_saved: st.size
|
|
1867
|
+
kind: "skip", hash: entry.hash, path: filePath,
|
|
1868
|
+
app: entry.app || null, source_id: entry.source_id || null, size: st.size
|
|
2183
1869
|
})
|
|
2184
|
-
return { status: "
|
|
2185
|
-
} finally {
|
|
2186
|
-
if (ownsAction && this.fileAction === action) this.fileAction = null
|
|
1870
|
+
return { status: "ignored" }
|
|
2187
1871
|
}
|
|
2188
|
-
|
|
2189
|
-
|
|
2190
|
-
|
|
2191
|
-
|
|
2192
|
-
|
|
2193
|
-
|
|
2194
|
-
|
|
2195
|
-
|
|
2196
|
-
|
|
2197
|
-
|
|
2198
|
-
source_id:
|
|
2199
|
-
|
|
2200
|
-
}
|
|
2201
|
-
|
|
2202
|
-
|
|
2203
|
-
|
|
2204
|
-
path: filePath,
|
|
2205
|
-
|
|
2206
|
-
source_id: duplicate.source_id || null,
|
|
2207
|
-
size: duplicate.size || 0
|
|
2208
|
-
})
|
|
2209
|
-
return { status: "ignored" }
|
|
2210
|
-
}
|
|
2211
|
-
// reshare: un-pin a skipped path; the next scan may list it again.
|
|
2212
|
-
async reshare(filePath) {
|
|
2213
|
-
if (!this.enabled) return { status: "disabled" }
|
|
2214
|
-
const current = this.registry.excluded.get(filePath)
|
|
2215
|
-
if (current && current.unverified) return { status: "unavailable" }
|
|
2216
|
-
const excluded = this.registry.allowSharing(filePath)
|
|
2217
|
-
if (excluded) {
|
|
2218
|
-
await this.recordEvent({ kind: "reshare", path: filePath, source_id: excluded && excluded.source_id ? excluded.source_id : null })
|
|
2219
|
-
return { status: "resharable" }
|
|
1872
|
+
if (this.sourceAppIsRunning(source)) return { status: "locked" }
|
|
1873
|
+
const before = await lstatIfPresent(filePath)
|
|
1874
|
+
if (!before || !before.isFile() || before.dev !== entry.dev || before.ino !== entry.ino) return { status: "stale" }
|
|
1875
|
+
const copied = await this.copyOut(filePath, filePath, fileSnapshot(before), fileSnapshot(before), {
|
|
1876
|
+
canReplace: () => !this.sourceAppIsRunning(source)
|
|
1877
|
+
})
|
|
1878
|
+
if (copied.status !== "copied") return copied
|
|
1879
|
+
const st = copied.stat
|
|
1880
|
+
this.registry.exclude(filePath, { ts: Date.now(), source_id: entry.source_id || null, size: st.size })
|
|
1881
|
+
await this.refreshLinkSnapshots(entry.hash, entry.dev, entry.ino)
|
|
1882
|
+
await this.recordEvent({ kind: "detach", hash: entry.hash, path: filePath, app: entry.app || null, source_id: entry.source_id || null, size: st.size })
|
|
1883
|
+
return { status: "detached" }
|
|
1884
|
+
}
|
|
1885
|
+
if (this.registry.duplicates.has(filePath)) {
|
|
1886
|
+
const dup = this.registry.duplicates.get(filePath)
|
|
1887
|
+
this.registry.exclude(filePath, { ts: Date.now(), source_id: dup.source_id || null, size: dup.size || 0 })
|
|
1888
|
+
await this.recordEvent({ kind: "skip", hash: dup.hash, path: filePath, app: dup.app || null, source_id: dup.source_id || null, size: dup.size || 0 })
|
|
1889
|
+
return { status: "ignored" }
|
|
2220
1890
|
}
|
|
2221
1891
|
return { status: "not-found" }
|
|
2222
1892
|
}
|
|
2223
1893
|
// Undo a conversion batch: replace each converted name with an independent
|
|
2224
1894
|
// copy of the bytes (spec, UX contract surface 3).
|
|
2225
|
-
async undoBatch(batchId
|
|
1895
|
+
async undoBatch(batchId) {
|
|
2226
1896
|
if (!this.enabled || !batchId) return { undone: 0 }
|
|
2227
1897
|
await this.refreshSources()
|
|
2228
1898
|
const events = await this.registry.readEvents()
|
|
@@ -2234,16 +1904,11 @@ class Vault {
|
|
|
2234
1904
|
if (entry.batch_id === batchId) targets.add(filePath)
|
|
2235
1905
|
}
|
|
2236
1906
|
const summary = { undone: 0, bytes: 0, failed: 0 }
|
|
2237
|
-
const prepared = []
|
|
2238
1907
|
for (const filePath of targets) {
|
|
2239
1908
|
const event = conversionEvents.get(filePath) || {}
|
|
2240
1909
|
const entry = this.registry.links.get(filePath)
|
|
2241
1910
|
if (!entry || (entry.batch_id && entry.batch_id !== batchId) ||
|
|
2242
1911
|
(event.hash && entry.hash !== event.hash)) continue
|
|
2243
|
-
if (entry.unverified) {
|
|
2244
|
-
summary.failed += 1
|
|
2245
|
-
continue
|
|
2246
|
-
}
|
|
2247
1912
|
const storePath = this.storePathFor(entry.hash)
|
|
2248
1913
|
try {
|
|
2249
1914
|
const source = this.sourceForPath(filePath, entry.source_id || event.source_id)
|
|
@@ -2266,34 +1931,8 @@ class Vault {
|
|
|
2266
1931
|
summary.failed += 1
|
|
2267
1932
|
continue
|
|
2268
1933
|
}
|
|
2269
|
-
prepared.push({ filePath, event, entry, source, storePath, st, storeStat })
|
|
2270
|
-
} catch (e) {
|
|
2271
|
-
summary.failed += 1
|
|
2272
|
-
}
|
|
2273
|
-
}
|
|
2274
|
-
const capacity = await this.preflightCopyCapacity(prepared.map((target) => ({
|
|
2275
|
-
path: target.filePath,
|
|
2276
|
-
bytes: target.storeStat.size
|
|
2277
|
-
})))
|
|
2278
|
-
if (!capacity.ok) {
|
|
2279
|
-
return Object.assign(summary, { status: capacity.status })
|
|
2280
|
-
}
|
|
2281
|
-
const action = queuedAction || {
|
|
2282
|
-
kind: "undo", path: null, source_id: null,
|
|
2283
|
-
bytes_copied: 0, bytes_total: 0
|
|
2284
|
-
}
|
|
2285
|
-
action.bytes_total = prepared.reduce((sum, target) => sum + target.storeStat.size, 0)
|
|
2286
|
-
let completedBytes = 0
|
|
2287
|
-
for (const target of prepared) {
|
|
2288
|
-
const { filePath, event, entry, source, storePath, st, storeStat } = target
|
|
2289
|
-
try {
|
|
2290
|
-
action.path = filePath
|
|
2291
|
-
action.source_id = entry.source_id || event.source_id || null
|
|
2292
1934
|
const copied = await this.copyOut(filePath, storePath, fileSnapshot(st), fileSnapshot(storeStat), {
|
|
2293
|
-
canReplace: () => !this.sourceAppIsRunning(source)
|
|
2294
|
-
onProgress: (bytesCopied) => {
|
|
2295
|
-
action.bytes_copied = completedBytes + bytesCopied
|
|
2296
|
-
}
|
|
1935
|
+
canReplace: () => !this.sourceAppIsRunning(source)
|
|
2297
1936
|
})
|
|
2298
1937
|
if (copied.status !== "copied") {
|
|
2299
1938
|
summary.failed += 1
|
|
@@ -2304,18 +1943,19 @@ class Vault {
|
|
|
2304
1943
|
hash: entry.hash, size: storeStat.size, app: entry.app || null,
|
|
2305
1944
|
source_id: entry.source_id || event.source_id || null, discovered: Date.now(),
|
|
2306
1945
|
dev: independentStat.dev, ino: independentStat.ino,
|
|
2307
|
-
|
|
1946
|
+
mtime: independentStat.mtimeMs, ctime: independentStat.ctimeMs
|
|
2308
1947
|
}
|
|
2309
|
-
const scanEntry =
|
|
1948
|
+
const scanEntry = {
|
|
2310
1949
|
hash: entry.hash, size: independentStat.size,
|
|
2311
1950
|
dev: independentStat.dev, ino: independentStat.ino,
|
|
2312
1951
|
mtime: independentStat.mtimeMs, ctime: independentStat.ctimeMs,
|
|
2313
1952
|
source_id: entry.source_id || event.source_id || null
|
|
2314
1953
|
}
|
|
2315
1954
|
this.registry.setDuplicate(filePath, duplicateEntry, scanEntry)
|
|
2316
|
-
if (copied.unverified) this.registry.scanIndex.delete(filePath)
|
|
2317
1955
|
await this.refreshLinkSnapshots(entry.hash, entry.dev, entry.ino)
|
|
2318
1956
|
const bytesSaved = event.bytes_saved || storeStat.size
|
|
1957
|
+
this.registry.totals.lifetime_bytes_saved = Math.max(0,
|
|
1958
|
+
this.registry.totals.lifetime_bytes_saved - bytesSaved)
|
|
2319
1959
|
await this.recordEvent({
|
|
2320
1960
|
kind: "undo", hash: entry.hash, path: filePath,
|
|
2321
1961
|
source_id: entry.source_id || event.source_id || null, batch_id: batchId,
|
|
@@ -2323,8 +1963,6 @@ class Vault {
|
|
|
2323
1963
|
})
|
|
2324
1964
|
summary.undone += 1
|
|
2325
1965
|
summary.bytes += storeStat.size
|
|
2326
|
-
completedBytes += storeStat.size
|
|
2327
|
-
action.bytes_copied = completedBytes
|
|
2328
1966
|
} catch (e) {
|
|
2329
1967
|
summary.failed += 1
|
|
2330
1968
|
}
|
|
@@ -2336,10 +1974,14 @@ class Vault {
|
|
|
2336
1974
|
if (!this.enabled) return { reclaimed: 0, bytes_freed: 0 }
|
|
2337
1975
|
const summary = { reclaimed: 0, bytes_freed: 0, failed: 0 }
|
|
2338
1976
|
const removedHashes = new Set()
|
|
1977
|
+
const copyHashes = new Set([...this.registry.links.values()]
|
|
1978
|
+
.filter((entry) => entry.mode === "copy")
|
|
1979
|
+
.map((entry) => entry.hash))
|
|
2339
1980
|
try {
|
|
2340
1981
|
for (const [hash] of [...this.registry.blobs]) {
|
|
2341
1982
|
const result = await this.reclaim(hash, {
|
|
2342
|
-
deferRegistry: true
|
|
1983
|
+
deferRegistry: true,
|
|
1984
|
+
hasCopyNames: copyHashes.has(hash)
|
|
2343
1985
|
})
|
|
2344
1986
|
if (result.remove_hash) removedHashes.add(hash)
|
|
2345
1987
|
if (result.status === "reclaimed") {
|