pinokiod 8.0.42 → 8.0.44

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -6,10 +6,7 @@ const Registry = require('./registry')
6
6
  const Sweeper = require('./sweeper')
7
7
  const { walkBatches, statMany } = require('./walker')
8
8
  const { fileSnapshot, sameSnapshot, sameContentState } = require('./snapshot')
9
- const {
10
- SIZE_THRESHOLD, CANDIDATE_SIZE_OPTIONS, TMP_SUFFIX, SHA256_RE,
11
- DIR_CONCURRENCY, STAT_CONCURRENCY
12
- } = require('./constants')
9
+ const { SIZE_THRESHOLD, TMP_SUFFIX, SHA256_RE, DIR_CONCURRENCY, STAT_CONCURRENCY } = require('./constants')
13
10
 
14
11
  // Shared model store engine (spec/requirements/shared-model-store.md).
15
12
  // Store + registry + volume probe + hashing + adopt/convert/verify, plus the
@@ -19,7 +16,6 @@ const {
19
16
 
20
17
  const NO_LINK_CODES = new Set(["EXDEV", "ENOTSUP", "ENOSYS"])
21
18
  const LOCK_CODES = new Set(["EBUSY", "EPERM", "EACCES"])
22
- const COPY_PROGRESS_CHUNK_SIZE = 8 * 1024 * 1024
23
19
  const isMissingError = (error) => !!(error && (error.code === "ENOENT" || error.code === "ENOTDIR"))
24
20
  const isAccessError = (error) => !!(error && (error.code === "EACCES" || error.code === "EPERM"))
25
21
  const lstatIfPresent = async (filePath) => {
@@ -71,18 +67,14 @@ const sameFileMetadata = (left, right) => {
71
67
  }
72
68
 
73
69
  const sourceId = (kind, name) => `${kind}:${encodeURIComponent(name)}`
74
- const externalSourceId = (root) => {
75
- const value = process.platform === "win32" ? path.resolve(root).toLowerCase() : path.resolve(root)
76
- return `external:${crypto.createHash("sha256").update(value).digest("hex")}`
77
- }
78
70
 
79
71
  class Vault {
80
72
  constructor(kernel) {
81
73
  this.kernel = kernel
82
74
  this.enabled = false
83
75
  this.initialized = false
84
- this.mode = null // 'link' | 'unsupported' for the vault's volume
85
- this.volumeModes = new Map() // dev -> 'link' | 'unsupported'
76
+ this.mode = null // 'link' | 'copy' for the vault's own volume
77
+ this.volumeModes = new Map() // dev -> 'link' | 'copy'
86
78
  this.registry = null
87
79
  this.worker = null
88
80
  this.workerJobs = new Map()
@@ -98,15 +90,8 @@ class Vault {
98
90
  this.initializationPromise = null
99
91
  this.verificationPending = false
100
92
  this.scanError = null
101
- this.scanPersistenceWarning = false
102
93
  this.scanScopeId = null
103
- this.fileAction = null
104
- this.fileActions = new Map()
105
- this.actionSequence = 0
106
- this.scanEvents = null
107
- this.scanCreatedStores = null
108
- this.scanRetiredStores = null
109
- this.scanRestoredStores = null
94
+ this.fileActionProgress = null
110
95
  }
111
96
  get root() {
112
97
  return path.resolve(this.kernel.homedir, "vault")
@@ -114,6 +99,9 @@ class Vault {
114
99
  get blobRoot() {
115
100
  return path.resolve(this.root, "sha256")
116
101
  }
102
+ get sourceRoot() {
103
+ return path.resolve(this.root, "sources")
104
+ }
117
105
  storePathFor(hash) {
118
106
  if (typeof hash !== "string" || !SHA256_RE.test(hash)) {
119
107
  throw new TypeError("Invalid vault content identifier.")
@@ -171,123 +159,18 @@ class Vault {
171
159
  return result
172
160
  })
173
161
  }
174
- queueFileMutation(kind, filePath, sourceId, bytesTotal, operation, progress = {}) {
175
- const id = `action-${crypto.randomUUID()}`
176
- const action = {
177
- id,
178
- kind,
179
- path: filePath || null,
180
- source_id: sourceId || null,
181
- phase: "queued",
182
- bytes_copied: 0,
183
- bytes_total: Math.max(0, Number(bytesTotal) || 0),
184
- files_completed: Math.max(0, Number(progress.files_completed) || 0),
185
- files_total: Math.max(0, Number(progress.files_total) || 0)
186
- }
187
- this.fileActions.set(id, action)
188
- this.actionSequence += 1
189
- return this.runExclusive(async () => {
190
- action.phase = kind.startsWith("deduplicate") ? "deduplicating" : "copying"
191
- this.fileAction = action
192
- this.actionSequence += 1
193
- try {
194
- const result = await operation(action)
195
- try {
196
- if (this.registry) await this.registry.flush()
197
- } catch (error) {
198
- return Object.assign({}, result, { persistence_warning: true })
199
- }
200
- return result
201
- } finally {
202
- if (this.fileAction === action) this.fileAction = null
203
- this.fileActions.delete(id)
204
- this.actionSequence += 1
205
- }
206
- })
207
- }
208
- beginScanDraft() {
209
- this.scanEvents = []
210
- this.scanCreatedStores = []
211
- this.scanRetiredStores = []
212
- this.scanRestoredStores = []
213
- }
214
- rememberScanStore(storePath, stat) {
215
- if (this.scanCreatedStores) this.scanCreatedStores.push({ storePath, stat })
216
- }
217
- rememberRetiredScanStore(storePath, stat, sourcePaths = []) {
218
- if (this.scanRetiredStores) {
219
- this.scanRetiredStores.push({ storePath, stat, sourcePaths: [...new Set(sourcePaths)] })
220
- }
221
- }
222
- async restoreRetiredScanStore(item) {
223
- const current = await this.storeStatIfPresent(item.storePath)
224
- if (current) return sameIdentity(current, item.stat) ? current : null
225
- const key = this.registry.inoKey(item.stat.dev, item.stat.ino)
226
- const sourcePaths = new Set([
227
- ...(item.sourcePaths || []),
228
- ...(this.registry.pathsByIno.get(key) || [])
229
- ])
230
- for (const filePath of sourcePaths) {
231
- const st = await lstatIfPresent(filePath)
232
- if (!sameIdentity(st, item.stat)) continue
233
- await fs.promises.link(filePath, item.storePath)
234
- const restored = await this.storeStatIfPresent(item.storePath)
235
- return sameIdentity(restored, item.stat) ? restored : null
236
- }
237
- return null
238
- }
239
- async commitScanDraft() {
240
- const events = (this.scanEvents || []).slice(-this.registry.maxEvents)
241
- const retired = this.scanRetiredStores || []
242
- const removed = []
162
+ async runFileAction(progress, operation) {
163
+ this.fileActionProgress = progress
243
164
  try {
244
- for (const item of retired) {
245
- if (!await unlinkIfSame(item.storePath, item.stat)) continue
246
- removed.push(item)
247
- const hash = this.registry.byIno.get(this.registry.inoKey(item.stat.dev, item.stat.ino))
248
- if (hash) await this.refreshLinkSnapshots(hash, item.stat.dev, item.stat.ino)
249
- }
250
- } catch (error) {
251
- for (const item of removed.reverse()) {
252
- const restored = await this.restoreRetiredScanStore(item).catch(() => null)
253
- if (restored) this.scanRestoredStores.push({ item, stat: restored })
254
- }
255
- throw error
256
- }
257
- this.scanEvents = null
258
- this.scanCreatedStores = null
259
- this.scanRetiredStores = null
260
- this.scanRestoredStores = null
261
- try {
262
- await this.registry.appendEvents(events)
263
- return true
264
- } catch (error) {
265
- return false
165
+ return await operation(progress)
166
+ } finally {
167
+ if (this.fileActionProgress === progress) this.fileActionProgress = null
266
168
  }
267
169
  }
268
- async abortScanDraft(snapshot) {
269
- const created = this.scanCreatedStores || []
270
- const restored = this.scanRestoredStores || []
271
- this.scanEvents = null
272
- this.scanCreatedStores = null
273
- this.scanRetiredStores = null
274
- this.scanRestoredStores = null
275
- for (const item of created.reverse()) {
276
- await unlinkIfSame(item.storePath, item.stat).catch(() => {})
277
- }
278
- this.registry.replaceState(snapshot, { persist: false })
279
- for (const item of restored) {
280
- const hash = this.registry.byIno.get(
281
- this.registry.inoKey(item.stat.dev, item.stat.ino))
282
- if (hash) await this.refreshLinkSnapshots(hash, item.stat.dev, item.stat.ino)
283
- }
284
- return restored.length > 0
285
- }
286
170
  startScan(scopeId = null) {
287
171
  if (!this.enabled || !this.sweeper) return { started: false, disabled: true }
288
172
  if (this.scanPromise) return { started: false, already_running: true }
289
173
  this.scanError = null
290
- this.scanPersistenceWarning = false
291
174
  this.scanScopeId = scopeId
292
175
  this.scanPromise = this.runExclusive(() => this.sweeper.scan(scopeId))
293
176
  .catch((error) => {
@@ -317,71 +200,49 @@ class Vault {
317
200
  }
318
201
  }
319
202
  }
320
- case "remove_source":
321
- return this.runMutation(() => this.removeExternalSource(payload.source_id))
322
203
  case "scan":
323
204
  if (payload.scope_id && !this.scanSource(payload.scope_id)) {
324
205
  return { error: "That scan location is no longer available." }
325
206
  }
326
- if (payload.candidate_min_bytes !== undefined) {
327
- if (!CANDIDATE_SIZE_OPTIONS.includes(payload.candidate_min_bytes)) {
328
- return { error: "Choose one of the available minimum file sizes." }
329
- }
330
- if (this.scanPromise || (this.sweeper && this.sweeper.state.active)) {
331
- return { error: "Wait for the current scan to finish before changing the minimum file size." }
332
- }
333
- this.sizeThreshold = payload.candidate_min_bytes
334
- this.registry.settings.candidate_min_bytes = this.sizeThreshold
335
- this.registry.schedulePersist()
336
- }
337
207
  return this.startScan(payload.scope_id || null)
338
208
  case "deduplicate":
339
- if (typeof payload.scope_id !== "string" || !payload.scope_id) {
340
- return { error: "Choose a location to deduplicate." }
209
+ if (typeof payload.path === "string" && payload.path) {
210
+ return this.runMutation(() => this.runFileAction({
211
+ kind: "deduplicate-file",
212
+ path: path.resolve(payload.path)
213
+ }, () => this.deduplicateFile(payload.path, {
214
+ batch_id: `batch-${crypto.randomUUID()}`
215
+ })))
341
216
  }
342
- {
343
- const paths = Array.isArray(payload.paths) ? payload.paths : null
344
- const filesTotal = paths
345
- ? new Set(paths.filter((filePath) => typeof filePath === "string" && filePath)).size
346
- : this.registry.duplicates.size
347
- return this.queueFileMutation(
348
- "deduplicate",
349
- null,
350
- payload.scope_id,
351
- 0,
352
- (progress) => this.deduplicateScope(payload.scope_id, {
353
- batch_id: `batch-${crypto.randomUUID()}`,
354
- paths,
355
- progress
356
- }),
357
- { files_total: filesTotal }
358
- )
217
+ const selection = payload.selection || null
218
+ if (selection && selection !== "duplicates" && selection !== "kept-separate") {
219
+ return { error: "Choose files to deduplicate." }
359
220
  }
360
- case "deduplicate_all": {
361
- const seenPaths = new Set()
362
- const pathsByScope = new Map()
363
- for (const group of Array.isArray(payload.groups) ? payload.groups : []) {
364
- if (!group || typeof group.scope_id !== "string" || !group.scope_id) continue
365
- if (!pathsByScope.has(group.scope_id)) pathsByScope.set(group.scope_id, [])
366
- for (const filePath of Array.isArray(group.paths) ? group.paths : []) {
367
- if (typeof filePath !== "string" || !filePath || seenPaths.has(filePath)) continue
368
- seenPaths.add(filePath)
369
- pathsByScope.get(group.scope_id).push(filePath)
370
- }
221
+ if (payload.scope_id != null &&
222
+ (typeof payload.scope_id !== "string" || !payload.scope_id)) {
223
+ return { error: "Choose a valid location to deduplicate." }
371
224
  }
372
- const groups = [...pathsByScope]
373
- .map(([scope_id, paths]) => ({ scope_id, paths }))
374
- .filter((group) => group.paths.length)
375
- if (!groups.length) return { error: "No displayed files were selected." }
376
- return this.queueFileMutation(
377
- "deduplicate_all",
378
- null,
379
- null,
380
- 0,
381
- (progress) => this.deduplicateAll(groups, progress),
382
- { files_total: seenPaths.size }
383
- )
384
- }
225
+ const scopeId = typeof payload.scope_id === "string" && payload.scope_id
226
+ ? payload.scope_id
227
+ : null
228
+ if (!selection && !scopeId) {
229
+ return { error: "Choose a location to deduplicate." }
230
+ }
231
+ return this.runMutation(() => this.runFileAction({
232
+ kind: "deduplicate",
233
+ scope_id: scopeId,
234
+ selection: selection || "duplicates",
235
+ files_total: 0,
236
+ files_completed: 0
237
+ }, async (progress) => {
238
+ const options = {
239
+ batch_id: `batch-${crypto.randomUUID()}`,
240
+ progress
241
+ }
242
+ return selection
243
+ ? this.deduplicateSelection(selection, scopeId, options)
244
+ : this.deduplicateScope(scopeId, options)
245
+ }))
385
246
  case "reclaim":
386
247
  return this.runMutation(() => this.reclaim(payload.hash))
387
248
  case "reclaim_all":
@@ -396,24 +257,16 @@ class Vault {
396
257
  return { done: true }
397
258
  })
398
259
  case "undo":
399
- return this.queueFileMutation("undo", null, null, 0,
400
- (action) => this.undoBatch(payload.batch_id, action))
401
- case "detach": {
402
- const entry = this.registry.links.get(payload.path)
403
- return this.queueFileMutation(
404
- "detach",
405
- payload.path,
406
- entry && entry.source_id,
407
- entry && this.registry.blobs.get(entry.hash)
408
- ? this.registry.blobs.get(entry.hash).size
409
- : 0,
410
- (action) => this.detach(payload.path, action)
411
- )
412
- }
413
- case "skip":
414
- return this.runMutation(() => this.skip(payload.path))
415
- case "reshare":
416
- return this.runMutation(() => this.reshare(payload.path))
260
+ return this.runMutation(() => this.undoBatch(payload.batch_id))
261
+ case "detach":
262
+ if (typeof payload.path !== "string" || !payload.path) return { status: "not-found" }
263
+ return this.runMutation(() => {
264
+ const targetPath = path.resolve(payload.path)
265
+ const kind = this.registry.duplicates.has(targetPath)
266
+ ? "keep-separate"
267
+ : "make-separate"
268
+ return this.runFileAction({ kind, path: targetPath }, () => this.detach(targetPath))
269
+ })
417
270
  default:
418
271
  return { error: "unknown action" }
419
272
  }
@@ -435,31 +288,44 @@ class Vault {
435
288
  const decorate = async (source) => {
436
289
  if (!source.root) return source
437
290
  try {
438
- const st = await fs.promises.lstat(source.root)
291
+ const st = await fs.promises.stat(source.root)
439
292
  source.dev = st.dev
440
- source.available = st.isDirectory() && !st.isSymbolicLink()
293
+ source.available = st.isDirectory()
441
294
  source.shareable = source.available && storeDev !== null && st.dev === storeDev && this.mode === "link"
442
295
  } catch (error) {
443
- if (!isMissingError(error) && !isAccessError(error)) throw error
296
+ if (!isMissingError(error)) throw error
444
297
  source.available = false
445
298
  source.shareable = false
446
- source.error_code = error && error.code ? error.code : null
447
299
  }
448
300
  return source
449
301
  }
450
302
 
451
- const externalPaths = this.registry && Array.isArray(this.registry.settings.external_sources)
452
- ? this.registry.settings.external_sources
453
- : []
454
- for (const externalPath of externalPaths) {
455
- const canonical = path.resolve(externalPath)
456
- sources.push(await decorate({
457
- id: externalSourceId(canonical),
458
- kind: "external",
459
- label: path.basename(canonical) || canonical,
460
- root: canonical,
461
- parent_id: "external"
462
- }))
303
+ const seenExternalRoots = new Set()
304
+ const addExternalEntries = async (importRoot, idPrefix = "") => {
305
+ let entries = []
306
+ try {
307
+ entries = await fs.promises.readdir(importRoot, { withFileTypes: true })
308
+ } catch (error) {
309
+ if (!isMissingError(error)) throw error
310
+ }
311
+ for (const entry of entries) {
312
+ if (!entry.isSymbolicLink()) continue
313
+ const mountPath = path.resolve(importRoot, entry.name)
314
+ try {
315
+ const root = await fs.promises.realpath(mountPath)
316
+ const st = await fs.promises.stat(root)
317
+ if (!st.isDirectory()) continue
318
+ const canonical = path.resolve(root)
319
+ if (seenExternalRoots.has(canonical)) continue
320
+ seenExternalRoots.add(canonical)
321
+ sources.push(await decorate({
322
+ id: sourceId("external", `${idPrefix}${entry.name}`), kind: "external", label: entry.name,
323
+ root: canonical, mount_path: mountPath, parent_id: "external"
324
+ }))
325
+ } catch (error) {
326
+ if (!isMissingError(error)) throw error
327
+ }
328
+ }
463
329
  }
464
330
 
465
331
  let apiEntries = []
@@ -477,6 +343,11 @@ class Vault {
477
343
  }))
478
344
  }
479
345
  }
346
+ await addExternalEntries(apiRoot)
347
+ if (await this.directoryIfSafe(this.sourceRoot)) {
348
+ await addExternalEntries(this.sourceRoot, "vault:")
349
+ }
350
+
480
351
  let homeEntries = []
481
352
  try {
482
353
  homeEntries = await fs.promises.readdir(home, { withFileTypes: true })
@@ -496,10 +367,13 @@ class Vault {
496
367
  this._sourceBases = new Map()
497
368
  for (const source of sources) {
498
369
  if (!source.root || source.kind === "virtual") continue
499
- const resolved = path.resolve(source.root)
500
- const key = process.platform === "win32" ? resolved.toLowerCase() : resolved
501
- if (!this._sourceBases.has(key)) this._sourceBases.set(key, [])
502
- this._sourceBases.get(key).push(source)
370
+ for (const base of [source.root, source.mount_path]) {
371
+ if (!base) continue
372
+ const resolved = path.resolve(base)
373
+ const key = process.platform === "win32" ? resolved.toLowerCase() : resolved
374
+ if (!this._sourceBases.has(key)) this._sourceBases.set(key, [])
375
+ this._sourceBases.get(key).push(source)
376
+ }
503
377
  }
504
378
  return sources
505
379
  }
@@ -533,106 +407,50 @@ class Vault {
533
407
  if (existing) {
534
408
  return { created: false, source: existing }
535
409
  }
536
- const configured = [...this.registry.settings.external_sources]
537
- if (configured.some((root) => isPathWithin(root, canonical) || isPathWithin(canonical, root))) {
538
- throw new Error("Choose a folder that does not contain or overlap another scan location.")
539
- }
540
- const previousScan = this.registry.lastScan
541
- const previousAttempt = this.registry.lastScanAttempt
542
- this.registry.settings.external_sources = [...configured, canonical]
543
- this.registry.lastScan = null
544
- this.registry.lastScanAttempt = null
545
- this.registry.schedulePersist()
546
- try {
547
- await this.registry.flush()
548
- await this.refreshSources()
549
- const source = this._sources.find((item) =>
550
- item.kind === "external" && item.root && samePath(item.root, canonical))
551
- if (!source) throw new Error("The folder could not be added.")
552
- return { created: true, source }
553
- } catch (error) {
554
- this.registry.settings.external_sources = configured
555
- this.registry.lastScan = previousScan
556
- this.registry.lastScanAttempt = previousAttempt
557
- this.registry.schedulePersist()
558
- await this.registry.flush().catch(() => {})
559
- await this.refreshSources().catch(() => {})
560
- throw error
561
- }
562
- }
563
- async removeExternalSource(sourceId) {
564
- if (typeof sourceId !== "string" || !sourceId) {
565
- throw new Error("Choose an external folder to remove.")
566
- }
567
- await this.refreshSources()
568
- const source = this._sources.find((item) =>
569
- item.id === sourceId && item.kind === "external" && item.root)
570
- if (!source) throw new Error("That external folder is no longer configured.")
571
410
 
572
- let canonical
573
- let rootStat
574
- try {
575
- canonical = path.resolve(await fs.promises.realpath(source.root))
576
- rootStat = await fs.promises.stat(canonical)
577
- } catch (error) {
578
- throw new Error("Reconnect this folder before removing it.")
579
- }
580
- if (!rootStat.isDirectory() || !samePath(canonical, source.root)) {
581
- throw new Error("Reconnect this folder before removing it.")
582
- }
583
-
584
- const storesToRemove = new Map()
585
- for (const [filePath, entry] of this.registry.links) {
586
- if (entry.source_id !== source.id) continue
587
- let fileStat
411
+ const importRoot = this.sourceRoot
412
+ await this.ensureDirectory(importRoot)
413
+ const baseLabel = path.basename(canonical) || "external-folder"
414
+ let mountPath = null
415
+ for (let index = 1; index < 10000; index++) {
416
+ const label = index === 1 ? baseLabel : `${baseLabel}-${index}`
417
+ const candidate = path.resolve(importRoot, label)
588
418
  try {
589
- fileStat = await fs.promises.lstat(filePath)
419
+ await fs.promises.symlink(canonical, candidate, this.kernel.platform === "win32" ? "junction" : "dir")
420
+ mountPath = candidate
421
+ break
590
422
  } catch (error) {
591
- if (isMissingError(error)) continue
592
- throw new Error("Some files in this folder could not be checked.")
593
- }
594
- if (!fileStat.isFile()) continue
595
- const storePath = this.storePathFor(entry.hash)
596
- const storeStat = await this.storeStatIfPresent(storePath)
597
- const visibleNames = fileStat.nlink - (sameIdentity(fileStat, storeStat) ? 1 : 0)
598
- if (visibleNames > 1) {
599
- throw new Error("Turn sharing off for this folder before removing it.")
600
- }
601
- if (visibleNames === 1 && sameIdentity(fileStat, storeStat)) {
602
- storesToRemove.set(entry.hash, {
603
- path: storePath,
604
- stat: storeStat
605
- })
423
+ if (error && error.code === "EEXIST") continue
424
+ if (error && (error.code === "EACCES" || error.code === "EPERM")) {
425
+ throw new Error("Pinokio does not have permission to add that folder.")
426
+ }
427
+ throw new Error("Pinokio could not add that folder.")
606
428
  }
607
429
  }
430
+ if (!mountPath) throw new Error("Pinokio could not create a unique name for that folder.")
608
431
 
609
- for (const [hash, store] of storesToRemove) {
610
- if (await unlinkIfSame(store.path, store.stat)) {
611
- this.registry.removeBlob(hash)
432
+ try {
433
+ await this.refreshSources()
434
+ const source = this._sources.find((item) => item.kind === "external" && item.mount_path && samePath(item.mount_path, mountPath))
435
+ if (!source) throw new Error("The folder could not be added. Restart Pinokio and try again.")
436
+ return { created: true, source }
437
+ } catch (error) {
438
+ // Adding a source is transactional. Roll back only the exact directory
439
+ // link created above; preserve any path an external writer replaced.
440
+ try {
441
+ const [mounted, target] = await Promise.all([
442
+ fs.promises.lstat(mountPath),
443
+ fs.promises.realpath(mountPath)
444
+ ])
445
+ if (mounted.isSymbolicLink() && samePath(target, canonical)) {
446
+ await fs.promises.unlink(mountPath)
447
+ }
448
+ } catch {
449
+ // The original error remains authoritative; an unverified path is
450
+ // deliberately preserved rather than deleted.
612
451
  }
452
+ throw error
613
453
  }
614
-
615
- for (const [filePath, entry] of [...this.registry.links]) {
616
- if (entry.source_id === source.id) this.registry.untrack(filePath)
617
- }
618
- for (const [filePath, entry] of [...this.registry.duplicates]) {
619
- if (entry.source_id === source.id) this.registry.untrack(filePath)
620
- }
621
- for (const [filePath, entry] of [...this.registry.scanIndex]) {
622
- if (entry.source_id === source.id) this.registry.untrack(filePath)
623
- }
624
- for (const [filePath, entry] of [...this.registry.excluded]) {
625
- if (entry.source_id === source.id) this.registry.excluded.delete(filePath)
626
- }
627
- this.registry.sourceScans.delete(source.id)
628
- this.registry.sourceScanAttempts.delete(source.id)
629
- this.registry.settings.external_sources = this.registry.settings.external_sources
630
- .filter((root) => !samePath(root, source.root))
631
- this.registry.lastScan = null
632
- this.registry.lastScanAttempt = null
633
- this.registry.schedulePersist()
634
- await this.refreshSources()
635
- return { removed: true, source_id: source.id }
636
454
  }
637
455
  sourceForPath(filePath, preferredId) {
638
456
  let cursor = path.resolve(filePath)
@@ -664,7 +482,9 @@ class Vault {
664
482
  const source = this.sourceForPath(filePath, preferredId)
665
483
  if (!source) return { source_id: null, relative_path: path.basename(filePath) }
666
484
  const absolute = path.resolve(filePath)
667
- const relative = path.relative(source.root, absolute).split(path.sep).join("/") || path.basename(absolute)
485
+ let base = source.root
486
+ if (source.mount_path && isPathWithin(source.mount_path, absolute)) base = source.mount_path
487
+ const relative = path.relative(base, absolute).split(path.sep).join("/") || path.basename(absolute)
668
488
  return {
669
489
  source_id: source.id,
670
490
  source_kind: source.kind,
@@ -679,13 +499,6 @@ class Vault {
679
499
  const context = location.source_id ? location : { relative_path: event.path }
680
500
  entry = Object.assign({}, event, context)
681
501
  }
682
- if (this.scanEvents) {
683
- this.scanEvents.push(entry)
684
- if (this.scanEvents.length > this.registry.maxEvents * 2) {
685
- this.scanEvents.splice(0, this.registry.maxEvents)
686
- }
687
- return true
688
- }
689
502
  try {
690
503
  await this.registry.appendEvent(entry)
691
504
  return true
@@ -770,16 +583,12 @@ class Vault {
770
583
  await this.ensureDirectory(this.blobRoot)
771
584
  this.registry = new Registry(this.root)
772
585
  this.mode = await this.probe(this.root)
773
- const loaded = await this.registry.load()
774
586
  await this.refreshSources()
587
+ const loaded = await this.registry.load()
775
588
  const missingWithBlobs = !loaded.existed && await this.hasStoredBlobs()
776
589
  if (loaded.corrupt || missingWithBlobs) {
777
- // Recovery may inspect Vault's own hash shards, but it must not discover
778
- // or walk configured sources. A later explicit Scan or Repair rebuilds
779
- // visible-path state.
780
- await this.rebuild([], { preserveState: false })
590
+ await this.rebuild(undefined, { preserveState: false })
781
591
  }
782
- this.sizeThreshold = this.registry.settings.candidate_min_bytes || SIZE_THRESHOLD
783
592
  this.sweeper = new Sweeper(this)
784
593
  this.initialized = true
785
594
  return {
@@ -842,7 +651,7 @@ class Vault {
842
651
  if (this.volumeModes.has(dev)) return this.volumeModes.get(dev)
843
652
  const a = path.resolve(dir, `.pinokio-probe-${crypto.randomBytes(6).toString("hex")}`)
844
653
  const b = a + "-link"
845
- let mode = "unsupported"
654
+ let mode = "copy"
846
655
  let aStat = null
847
656
  let bStat = null
848
657
  let failure = null
@@ -879,11 +688,7 @@ class Vault {
879
688
  const job = this.workerJobs.get(id)
880
689
  if (!job) return
881
690
  this.workerJobs.delete(id)
882
- if (error) {
883
- const failure = new Error(error.message || String(error))
884
- if (error.code) failure.code = error.code
885
- job.reject(failure)
886
- }
691
+ if (error) job.reject(new Error(error))
887
692
  else job.resolve({ hash, size })
888
693
  // Keep the worker warm across the scan queue. Starting one worker per
889
694
  // file costs ~30ms and adds up quickly on a first scan.
@@ -939,10 +744,7 @@ class Vault {
939
744
  hash,
940
745
  source_id: entry.source_id || null
941
746
  }))
942
- } catch (error) {
943
- this.registry.scanIndex.delete(linkPath)
944
- entry.unverified = true
945
- }
747
+ } catch (error) {}
946
748
  }
947
749
  this.registry.schedulePersist()
948
750
  }
@@ -952,7 +754,7 @@ class Vault {
952
754
  for (const linkPath of this.registry.pathsByIno.get(key) || []) {
953
755
  const entry = this.registry.links.get(linkPath)
954
756
  if (!entry) continue
955
- if (entry.hash !== hash ||
757
+ if (entry.hash !== hash || entry.mode !== "link" ||
956
758
  entry.dev !== storeStat.dev || entry.ino !== storeStat.ino) continue
957
759
  const expected = this.registry.scanIndex.get(linkPath)
958
760
  if (expected && expected.hash === hash && sameSnapshot(expected, storeStat)) {
@@ -965,15 +767,13 @@ class Vault {
965
767
  try {
966
768
  hashed = await this.hashFile(storePath)
967
769
  } catch (error) {
968
- if (isMissingError(error)) return { valid: false }
969
- throw error
770
+ return { valid: false }
970
771
  }
971
772
  let after
972
773
  try {
973
774
  after = await this.storeStatIfPresent(storePath)
974
775
  } catch (error) {
975
- if (isMissingError(error)) return { valid: false }
976
- throw error
776
+ return { valid: false }
977
777
  }
978
778
  if (!sameSnapshot(before, after) || hashed.hash !== hash || hashed.size !== after.size) {
979
779
  return { valid: false }
@@ -995,6 +795,7 @@ class Vault {
995
795
  source_id: meta.source_id || null,
996
796
  dev: st.dev,
997
797
  ino: st.ino,
798
+ mode: "link"
998
799
  }
999
800
  const storeStat = await this.storeStatIfPresent(storePath, {
1000
801
  createParent: this.mode === "link"
@@ -1012,12 +813,25 @@ class Vault {
1012
813
  if (!sameSnapshot(before, current)) return { status: "stale" }
1013
814
  return { status: "duplicate", storePath }
1014
815
  }
1015
- if (this.mode !== "link") return { status: "unavailable", code: "ENOTSUP" }
816
+ const registerCopy = async () => {
817
+ const current = await lstatIfPresent(filePath)
818
+ if (!sameSnapshot(before, current)) return { status: "stale" }
819
+ linkEntry.mode = "copy"
820
+ this.registry.addBlob(hash, { size: st.size, source_urls: meta.source_urls || [] })
821
+ this.registry.addLink(filePath, linkEntry)
822
+ await this.refreshLinkSnapshots(hash, st.dev, st.ino)
823
+ await this.recordEvent({
824
+ kind: "adopt", hash, path: filePath, app: meta.app || null,
825
+ source_id: meta.source_id || null, bytes_saved: 0, mode: "copy"
826
+ })
827
+ return { status: "copy-mode" }
828
+ }
829
+ if (this.mode === "copy") return registerCopy()
1016
830
  try {
1017
831
  await fs.promises.link(filePath, storePath)
1018
832
  } catch (e) {
1019
833
  if (NO_LINK_CODES.has(e.code)) {
1020
- return { status: "unavailable", code: e.code }
834
+ return registerCopy()
1021
835
  }
1022
836
  throw e
1023
837
  }
@@ -1045,7 +859,6 @@ class Vault {
1045
859
  }
1046
860
  this.registry.addBlob(hash, { size: st.size, source_urls: meta.source_urls || [] })
1047
861
  this.registry.addLink(filePath, linkEntry)
1048
- this.rememberScanStore(storePath, currentStore)
1049
862
  await this.refreshLinkSnapshots(hash, st.dev, st.ino)
1050
863
  await this.recordEvent({ kind: "adopt", hash, path: filePath, app: meta.app || null, source_id: meta.source_id || null, bytes_saved: 0 })
1051
864
  return { status: "adopted" }
@@ -1055,16 +868,14 @@ class Vault {
1055
868
  async convert(targetPath, hash, meta = {}) {
1056
869
  if (!this.enabled) return { status: "disabled" }
1057
870
  const storePath = this.storePathFor(hash)
1058
- const storeStat = await this.storeStatIfPresent(storePath)
1059
- if (!storeStat) return { status: "no-blob" }
1060
- let targetStat
871
+ let storeStat
1061
872
  try {
1062
- targetStat = await fs.promises.lstat(targetPath)
1063
- } catch (error) {
1064
- if (LOCK_CODES.has(error && error.code)) return { status: "locked" }
1065
- if (isMissingError(error)) return { status: "stale" }
1066
- throw error
873
+ storeStat = await this.storeStatIfPresent(storePath)
874
+ } catch (e) {
875
+ return { status: "no-blob" }
1067
876
  }
877
+ if (!storeStat) return { status: "no-blob" }
878
+ const targetStat = await fs.promises.lstat(targetPath)
1068
879
  if (!storeStat.isFile() || !targetStat.isFile()) return { status: "stale" }
1069
880
  if (meta.expected && !sameSnapshot(meta.expected, targetStat)) {
1070
881
  return { status: "stale" }
@@ -1073,10 +884,7 @@ class Vault {
1073
884
  return { status: "unavailable", code: "EXDEV" }
1074
885
  }
1075
886
  if (storeStat.dev === targetStat.dev && storeStat.ino === targetStat.ino) {
1076
- this.registry.addLink(targetPath, {
1077
- hash, app: meta.app || null, source_id: meta.source_id || null,
1078
- dev: storeStat.dev, ino: storeStat.ino, unverified: false
1079
- })
887
+ this.registry.addLink(targetPath, { hash, app: meta.app || null, source_id: meta.source_id || null, dev: storeStat.dev, ino: storeStat.ino, mode: "link" })
1080
888
  await this.refreshLinkSnapshots(hash, storeStat.dev, storeStat.ino)
1081
889
  return { status: "already" }
1082
890
  }
@@ -1158,10 +966,10 @@ class Vault {
1158
966
  }
1159
967
  this.registry.addLink(targetPath, {
1160
968
  hash, app: meta.app || null, source_id: meta.source_id || null,
1161
- dev: st.dev, ino: st.ino, batch_id: meta.batch_id || null,
1162
- unverified: false
969
+ dev: st.dev, ino: st.ino, mode: "link", batch_id: meta.batch_id || null
1163
970
  })
1164
971
  await this.refreshLinkSnapshots(hash, st.dev, st.ino)
972
+ this.registry.addSaved(storeStat.size)
1165
973
  await this.recordEvent({
1166
974
  kind: "convert", hash, path: targetPath, app: meta.app || null, source_id: meta.source_id || null,
1167
975
  bytes_saved: storeStat.size, batch_id: meta.batch_id || null
@@ -1180,138 +988,196 @@ class Vault {
1180
988
  return !!(runningPath && isPathWithin(appRoot, runningPath))
1181
989
  })
1182
990
  }
1183
- async deduplicateScope(scopeId, options = {}) {
991
+ async deduplicateFile(filePath, options = {}) {
992
+ if (!this.enabled || typeof filePath !== "string" || !filePath) {
993
+ return { status: this.enabled ? "not-found" : "disabled" }
994
+ }
1184
995
  await this.refreshSources()
1185
- const source = this._sources.find((item) => item.id === scopeId && item.kind !== "virtual")
1186
- if (!source) return { error: "That location is no longer available. Scan again to refresh it." }
1187
- if (!source.shareable) return { error: "This location is on a disk that cannot share space with this vault." }
1188
- if (this.sourceAppIsRunning(source)) {
1189
- return { error: "This app has a running script. Stop it first, then deduplicate." }
996
+ return this.deduplicateExcludedFile(filePath, options)
997
+ }
998
+ async deduplicateExcludedFile(filePath, options = {}) {
999
+ const targetPath = path.resolve(filePath)
1000
+ const excluded = this.registry.excluded.get(targetPath)
1001
+ if (!excluded) return { status: "not-found" }
1002
+ const source = this.sourceForPath(targetPath, excluded.source_id)
1003
+ if (!source || !await this.canonicalPathIsWithinSource(targetPath, source)) {
1004
+ return { status: "stale" }
1190
1005
  }
1006
+ if (!source.shareable) return { status: "unavailable" }
1007
+ if (this.sourceAppIsRunning(source)) return { status: "locked" }
1191
1008
 
1192
- const registry = this.registry
1193
- const batch = options.batch_id || `batch-${crypto.randomUUID()}`
1194
- const requestedPaths = options.paths === undefined
1195
- ? new Set([...registry.duplicates.keys()])
1196
- : new Set((options.paths || [])
1197
- .filter((filePath) => typeof filePath === "string" && filePath))
1198
- if (!requestedPaths.size) return { error: "No displayed files were selected." }
1199
- const candidates = [...registry.duplicates].filter(([filePath, entry]) => {
1200
- if (!requestedPaths.has(filePath)) return false
1201
- const currentSource = this.sourceForPath(filePath, entry.source_id)
1202
- return !!(currentSource && currentSource.id === scopeId && currentSource.shareable)
1009
+ const before = await lstatIfPresent(targetPath)
1010
+ if (!before || !before.isFile()) return { status: "stale" }
1011
+ let hashed
1012
+ try {
1013
+ hashed = await this.hashFile(targetPath)
1014
+ } catch (error) {
1015
+ return { status: "stale" }
1016
+ }
1017
+ const after = await lstatIfPresent(targetPath)
1018
+ if (!sameSnapshot(fileSnapshot(before), after) || hashed.size !== after.size) {
1019
+ return { status: "stale" }
1020
+ }
1021
+ if (!this.registry.blobs.has(hashed.hash)) return { status: "no-match" }
1022
+ if (!await this.canonicalPathIsWithinSource(targetPath, source)) return { status: "stale" }
1023
+ const current = await lstatIfPresent(targetPath)
1024
+ if (!sameSnapshot(fileSnapshot(after), current)) return { status: "stale" }
1025
+
1026
+ const result = await this.convert(targetPath, hashed.hash, {
1027
+ app: source.kind === "app" ? source.app : null,
1028
+ source_id: source.id,
1029
+ batch_id: options.batch_id || `batch-${crypto.randomUUID()}`,
1030
+ source,
1031
+ expected: fileSnapshot(current)
1203
1032
  })
1204
- const progress = options.progress || null
1205
- if (progress) {
1206
- progress.files_completed = 0
1207
- progress.files_total = candidates.length
1208
- this.actionSequence += 1
1033
+ if (result.status === "converted" || result.status === "already") {
1034
+ this.registry.allowSharing(targetPath)
1209
1035
  }
1210
- const summary = { converted: 0, bytes_saved: 0, locked: 0, stale: 0, incompatible: 0, unavailable: 0, failed: 0 }
1211
- const staleHashes = new Set()
1212
- for (const [filePath, entry] of candidates) {
1036
+ return result
1037
+ }
1038
+ sourceIsWithinScope(source, scopeId) {
1039
+ if (!scopeId) return true
1040
+ const seen = new Set()
1041
+ let current = source
1042
+ while (current && !seen.has(current.id)) {
1043
+ if (current.id === scopeId) return true
1044
+ seen.add(current.id)
1045
+ current = this._sources.find((item) => item.id === current.parent_id)
1046
+ }
1047
+ return false
1048
+ }
1049
+ async deduplicateSelection(selection, scopeId, options = {}) {
1050
+ if (selection === "duplicates") {
1051
+ return this.deduplicateScope(scopeId, Object.assign({}, options, { includeDescendants: true }))
1052
+ }
1053
+ await this.refreshSources()
1054
+ if (scopeId && !this._sources.some((source) => source.id === scopeId)) {
1055
+ return { error: "That location is no longer available. Scan again to refresh it." }
1056
+ }
1057
+ const paths = [...this.registry.excluded].filter(([filePath, entry]) => {
1058
+ const source = this.sourceForPath(filePath, entry.source_id)
1059
+ return source ? this.sourceIsWithinScope(source, scopeId) : !scopeId
1060
+ }).map(([filePath]) => filePath)
1061
+ if (options.progress) options.progress.files_total = paths.length
1062
+ const summary = {
1063
+ converted: 0, bytes_saved: 0, locked: 0, stale: 0, unmatched: 0,
1064
+ incompatible: 0, unavailable: 0, failed: 0
1065
+ }
1066
+ const batch = options.batch_id || `batch-${crypto.randomUUID()}`
1067
+ for (const filePath of paths) {
1213
1068
  try {
1214
- const currentSource = this.sourceForPath(filePath, entry.source_id)
1215
- if (!currentSource || currentSource.id !== scopeId || !currentSource.shareable) continue
1216
- if (entry.unverified) {
1217
- summary.unavailable += 1
1218
- continue
1219
- }
1220
- if (!await this.canonicalPathIsWithinSource(filePath, currentSource)) {
1221
- summary.stale += 1
1222
- continue
1223
- }
1224
- const indexed = registry.scanIndex.get(filePath)
1225
- const expected = indexed && indexed.hash === entry.hash
1226
- ? indexed
1227
- : (entry.dev !== undefined && entry.ino !== undefined &&
1228
- entry.mtime !== undefined && entry.ctime !== undefined ? entry : null)
1229
- if (!expected) {
1069
+ const result = await this.deduplicateExcludedFile(filePath, { batch_id: batch })
1070
+ if (result.status === "converted" || result.status === "already") {
1071
+ summary.converted += 1
1072
+ summary.bytes_saved += result.bytes_saved || 0
1073
+ } else if (result.status === "locked") {
1074
+ summary.locked += 1
1075
+ } else if (result.status === "stale") {
1230
1076
  summary.stale += 1
1231
- continue
1232
- }
1233
- if (staleHashes.has(entry.hash)) {
1234
- summary.stale += 1
1235
- continue
1236
- }
1237
- try {
1238
- const result = await this.convert(filePath, entry.hash, {
1239
- app: source.kind === "app" ? source.app : (entry.app || null),
1240
- source_id: scopeId,
1241
- batch_id: batch,
1242
- source: currentSource,
1243
- expected
1244
- })
1245
- if (result.status === "converted" || result.status === "already") {
1246
- summary.converted += 1
1247
- summary.bytes_saved += result.bytes_saved || 0
1248
- } else if (result.status === "locked") {
1249
- summary.locked += 1
1250
- } else if (result.status === "stale") {
1251
- summary.stale += 1
1252
- } else if (result.status === "stale-blob") {
1253
- staleHashes.add(entry.hash)
1254
- summary.stale += 1
1255
- } else if (result.status === "metadata-mismatch") {
1256
- entry.unavailable_reason = "metadata"
1257
- summary.incompatible += 1
1258
- } else if (result.status === "unavailable") {
1259
- summary.unavailable += 1
1260
- } else {
1261
- summary.failed += 1
1262
- }
1263
- } catch (error) {
1077
+ } else if (result.status === "no-match" || result.status === "no-blob" ||
1078
+ result.status === "size-mismatch") {
1079
+ summary.unmatched += 1
1080
+ } else if (result.status === "metadata-mismatch") {
1081
+ summary.incompatible += 1
1082
+ } else if (result.status === "unavailable" || result.status === "copy-mode") {
1083
+ summary.unavailable += 1
1084
+ } else {
1264
1085
  summary.failed += 1
1265
1086
  }
1266
- } finally {
1267
- if (progress) {
1268
- progress.files_completed += 1
1269
- this.actionSequence += 1
1270
- }
1271
- if (options.onFileComplete) options.onFileComplete()
1087
+ } catch (error) {
1088
+ summary.failed += 1
1272
1089
  }
1090
+ if (options.progress) options.progress.files_completed += 1
1273
1091
  }
1274
- registry.schedulePersist()
1275
1092
  return summary
1276
1093
  }
1277
- async deduplicateAll(groups, progress) {
1278
- const summary = {
1279
- converted: 0,
1280
- bytes_saved: 0,
1281
- locked: 0,
1282
- stale: 0,
1283
- incompatible: 0,
1284
- unavailable: 0,
1285
- failed: 0,
1286
- locations_skipped: 0
1287
- }
1288
- const fields = ["converted", "bytes_saved", "locked", "stale", "incompatible", "unavailable", "failed"]
1289
- progress.files_completed = 0
1290
- progress.files_total = groups.reduce((sum, group) => sum + group.paths.length, 0)
1291
- this.actionSequence += 1
1292
- for (const group of groups) {
1293
- let completed = 0
1294
- const result = await this.deduplicateScope(group.scope_id, {
1295
- batch_id: `batch-${crypto.randomUUID()}`,
1296
- paths: group.paths,
1297
- onFileComplete: () => {
1298
- completed += 1
1299
- progress.files_completed += 1
1300
- this.actionSequence += 1
1301
- }
1302
- })
1303
- if (result.error) {
1304
- summary.unavailable += group.paths.length
1305
- summary.locations_skipped += 1
1306
- } else {
1307
- for (const field of fields) summary[field] += Number(result[field]) || 0
1094
+ async deduplicateScope(scopeId, options = {}) {
1095
+ await this.refreshSources()
1096
+ const includeDescendants = options.includeDescendants === true
1097
+ const source = scopeId
1098
+ ? this._sources.find((item) => item.id === scopeId && (includeDescendants || item.kind !== "virtual"))
1099
+ : null
1100
+ if ((!includeDescendants || scopeId) && !source) {
1101
+ return { error: "That location is no longer available. Scan again to refresh it." }
1102
+ }
1103
+ if (!includeDescendants) {
1104
+ if (!source.shareable) return { error: "This location is on a disk that cannot share space with this vault." }
1105
+ if (this.sourceAppIsRunning(source)) {
1106
+ return { error: "This app has a running script. Stop it first, then deduplicate." }
1107
+ }
1108
+ }
1109
+
1110
+ const registry = this.registry
1111
+ const batch = options.batch_id || `batch-${crypto.randomUUID()}`
1112
+ const summary = { converted: 0, bytes_saved: 0, locked: 0, stale: 0, incompatible: 0, unavailable: 0, failed: 0 }
1113
+ const staleHashes = new Set()
1114
+ const entries = [...registry.duplicates]
1115
+ const matchesScope = (currentSource) => currentSource && currentSource.shareable &&
1116
+ (includeDescendants ? this.sourceIsWithinScope(currentSource, scopeId) : currentSource.id === scopeId)
1117
+ if (options.progress) {
1118
+ options.progress.files_total = entries.reduce((total, [filePath, entry]) => {
1119
+ const currentSource = this.sourceForPath(filePath, entry.source_id)
1120
+ return total + (matchesScope(currentSource) ? 1 : 0)
1121
+ }, 0)
1122
+ }
1123
+ const completeProgress = () => {
1124
+ if (options.progress) options.progress.files_completed += 1
1125
+ }
1126
+ for (const [filePath, entry] of entries) {
1127
+ const currentSource = this.sourceForPath(filePath, entry.source_id)
1128
+ if (!matchesScope(currentSource)) continue
1129
+ if (!await this.canonicalPathIsWithinSource(filePath, currentSource)) {
1130
+ summary.stale += 1
1131
+ completeProgress()
1132
+ continue
1308
1133
  }
1309
- const unprocessed = Math.max(0, group.paths.length - completed)
1310
- if (unprocessed) {
1311
- progress.files_completed += unprocessed
1312
- this.actionSequence += 1
1134
+ const indexed = registry.scanIndex.get(filePath)
1135
+ const expected = indexed && indexed.hash === entry.hash
1136
+ ? indexed
1137
+ : (entry.dev !== undefined && entry.ino !== undefined &&
1138
+ entry.mtime !== undefined && entry.ctime !== undefined ? entry : null)
1139
+ if (!expected) {
1140
+ summary.stale += 1
1141
+ completeProgress()
1142
+ continue
1143
+ }
1144
+ if (staleHashes.has(entry.hash)) {
1145
+ summary.stale += 1
1146
+ completeProgress()
1147
+ continue
1313
1148
  }
1149
+ try {
1150
+ const result = await this.convert(filePath, entry.hash, {
1151
+ app: currentSource.kind === "app" ? currentSource.app : (entry.app || null),
1152
+ source_id: currentSource.id,
1153
+ batch_id: batch,
1154
+ source: currentSource,
1155
+ expected
1156
+ })
1157
+ if (result.status === "converted" || result.status === "already") {
1158
+ summary.converted += 1
1159
+ summary.bytes_saved += result.bytes_saved || 0
1160
+ } else if (result.status === "locked") {
1161
+ summary.locked += 1
1162
+ } else if (result.status === "stale") {
1163
+ summary.stale += 1
1164
+ } else if (result.status === "stale-blob") {
1165
+ staleHashes.add(entry.hash)
1166
+ summary.stale += 1
1167
+ } else if (result.status === "metadata-mismatch") {
1168
+ entry.unavailable_reason = "metadata"
1169
+ summary.incompatible += 1
1170
+ } else if (result.status === "unavailable" || result.status === "copy-mode") {
1171
+ summary.unavailable += 1
1172
+ } else {
1173
+ summary.failed += 1
1174
+ }
1175
+ } catch (error) {
1176
+ summary.failed += 1
1177
+ }
1178
+ completeProgress()
1314
1179
  }
1180
+ registry.schedulePersist()
1315
1181
  return summary
1316
1182
  }
1317
1183
  // reclaim: delete an orphan's store name. Refuses when any app name remains.
@@ -1321,6 +1187,10 @@ class Vault {
1321
1187
  if (!this.registry.blobs.has(hash)) return { status: "not-found" }
1322
1188
  const st = await this.storeStatIfPresent(storePath)
1323
1189
  if (!st) {
1190
+ const hasCopyNames = options.hasCopyNames === undefined
1191
+ ? [...this.registry.links.values()].some((entry) => entry.hash === hash && entry.mode === "copy")
1192
+ : options.hasCopyNames
1193
+ if (hasCopyNames) return { status: "unavailable" }
1324
1194
  if (!options.deferRegistry) this.registry.removeBlob(hash)
1325
1195
  return options.deferRegistry ? { status: "gone", remove_hash: true } : { status: "gone" }
1326
1196
  }
@@ -1345,41 +1215,26 @@ class Vault {
1345
1215
  if (!this.enabled || !this.registry) return
1346
1216
  const preservePath = options.preservePath || (() => false)
1347
1217
  const includePath = options.includePath || (() => true)
1348
- const preserveReadError = (error, filePath, entry) => {
1349
- if (isMissingError(error)) return false
1350
- if (options.onAccessError) {
1351
- if (!isAccessError(error) || !options.onAccessError(error, filePath)) return false
1352
- }
1353
- if (entry) entry.unverified = true
1354
- return true
1355
- }
1218
+ const handleAccessError = (error, filePath) => !!(
1219
+ isAccessError(error) && options.onAccessError && options.onAccessError(error, filePath)
1220
+ )
1356
1221
  if (options.reconcile !== false) this.reconcileConfiguredSources()
1357
1222
  for (const [linkPath, entry] of [...this.registry.links]) {
1358
1223
  if (!includePath(linkPath)) continue
1359
1224
  if (preservePath(linkPath)) continue
1360
1225
  try {
1361
1226
  const source = this.sourceForPath(linkPath, entry.source_id)
1362
- if (source && source.available === false) {
1363
- entry.unverified = true
1364
- continue
1365
- }
1366
- if (!source) {
1227
+ const managedPath = source && await this.canonicalPathIsWithinSource(linkPath, source, { strictErrors: true })
1228
+ if (!managedPath) {
1367
1229
  this.registry.untrack(linkPath)
1368
1230
  continue
1369
1231
  }
1370
1232
  const st = await lstatIfPresent(linkPath)
1371
- if (!st) {
1233
+ if (!st || !st.isFile() || st.ino !== entry.ino || st.dev !== entry.dev) {
1372
1234
  this.registry.untrack(linkPath)
1373
- continue
1374
1235
  }
1375
- const managedPath = await this.canonicalPathIsWithinSource(
1376
- linkPath, source, { strictErrors: true })
1377
- const expected = this.registry.scanIndex.get(linkPath)
1378
- entry.unverified = !managedPath || !st.isFile() ||
1379
- st.ino !== entry.ino || st.dev !== entry.dev ||
1380
- !expected || expected.hash !== entry.hash || !sameSnapshot(expected, st)
1381
1236
  } catch (error) {
1382
- if (!preserveReadError(error, linkPath, entry)) throw error
1237
+ if (!handleAccessError(error, linkPath)) throw error
1383
1238
  }
1384
1239
  }
1385
1240
  for (const [dupPath, entry] of [...this.registry.duplicates]) {
@@ -1388,28 +1243,17 @@ class Vault {
1388
1243
  let valid = false
1389
1244
  try {
1390
1245
  const source = this.sourceForPath(dupPath, entry.source_id)
1391
- if (source && source.available === false) {
1392
- entry.unverified = true
1393
- continue
1394
- }
1395
1246
  const st = await lstatIfPresent(dupPath)
1396
- if (!st) {
1397
- this.registry.untrack(dupPath)
1398
- continue
1399
- }
1400
- const expected = this.registry.scanIndex.get(dupPath)
1401
- valid = !!(source && st.isFile() &&
1402
- await this.canonicalPathIsWithinSource(dupPath, source, { strictErrors: true }) &&
1403
- expected && expected.hash === entry.hash && sameSnapshot(expected, st))
1247
+ valid = !!(source && st && st.isFile() &&
1248
+ await this.canonicalPathIsWithinSource(dupPath, source, { strictErrors: true }))
1404
1249
  } catch (error) {
1405
- if (preserveReadError(error, dupPath, entry)) continue
1250
+ if (handleAccessError(error, dupPath)) continue
1406
1251
  throw error
1407
1252
  }
1408
1253
  if (!valid) {
1409
- entry.unverified = true
1254
+ this.registry.untrack(dupPath)
1410
1255
  continue
1411
1256
  }
1412
- entry.unverified = false
1413
1257
  const storeStat = await this.storeStatIfPresent(this.storePathFor(entry.hash))
1414
1258
  if (storeStat && storeStat.isFile()) {
1415
1259
  await unlinkIfSame(dupPath + TMP_SUFFIX, storeStat)
@@ -1430,6 +1274,7 @@ class Vault {
1430
1274
  const st = await this.storeStatIfPresent(storePath)
1431
1275
  if (!st) {
1432
1276
  const names = linksByHash.get(hash) || []
1277
+ const copyNames = names.filter(([, entry]) => entry.mode === "copy")
1433
1278
  // Deleting a name changes ctime, so trusted re-adoption compares the
1434
1279
  // preserved identity, size, and mtime. Changed or legacy-unsnapshotted
1435
1280
  // content waits for the next explicit scan instead of being assigned a
@@ -1437,6 +1282,7 @@ class Vault {
1437
1282
  let readopted = false
1438
1283
  let inaccessibleName = false
1439
1284
  for (const [linkPath, entry] of names) {
1285
+ if (entry.mode !== "link") continue
1440
1286
  if (preservePath(linkPath)) {
1441
1287
  inaccessibleName = true
1442
1288
  continue
@@ -1448,7 +1294,7 @@ class Vault {
1448
1294
  if (!source || !await this.canonicalPathIsWithinSource(linkPath, source, { strictErrors: true })) continue
1449
1295
  linkStat = await lstatIfPresent(linkPath)
1450
1296
  } catch (error) {
1451
- if (!preserveReadError(error, linkPath, entry)) throw error
1297
+ if (!handleAccessError(error, linkPath)) throw error
1452
1298
  inaccessibleName = true
1453
1299
  continue
1454
1300
  }
@@ -1486,7 +1332,10 @@ class Vault {
1486
1332
  readopted = true
1487
1333
  break
1488
1334
  }
1489
- if (readopted || inaccessibleName) {
1335
+ // On a volume without file sharing support, copy-mode names are the
1336
+ // durable content group. They remain useful for duplicate discovery
1337
+ // even though there is no canonical file in the vault tree.
1338
+ if (readopted || copyNames.length || inaccessibleName) {
1490
1339
  blob.orphan = false
1491
1340
  blob.verified_at = inaccessibleName && !readopted ? null : Date.now()
1492
1341
  } else {
@@ -1516,14 +1365,9 @@ class Vault {
1516
1365
  blobs: new Map(registry.blobs),
1517
1366
  links: new Map(registry.links),
1518
1367
  excluded: new Map(registry.excluded),
1368
+ totals: Object.assign({}, registry.totals),
1519
1369
  lastScan: registry.lastScan ? Object.assign({}, registry.lastScan) : null,
1520
- lastScanAttempt: registry.lastScanAttempt
1521
- ? Object.assign({}, registry.lastScanAttempt)
1522
- : null,
1523
1370
  sourceScans: new Map([...registry.sourceScans].map(([id, scan]) => [id, Object.assign({}, scan)])),
1524
- sourceScanAttempts: new Map([...registry.sourceScanAttempts]
1525
- .map(([id, scan]) => [id, Object.assign({}, scan)])),
1526
- settings: Object.assign({}, registry.settings),
1527
1371
  duplicates: new Map(registry.duplicates),
1528
1372
  scanIndex: new Map(registry.scanIndex)
1529
1373
  } : null
@@ -1571,18 +1415,7 @@ class Vault {
1571
1415
  await this.refreshSources()
1572
1416
  const walkRoots = roots
1573
1417
  ? roots.map((root) => typeof root === "string" ? { root, source_id: null } : root)
1574
- : [
1575
- { root: path.resolve(this.kernel.homedir), source_id: "pinokio" },
1576
- ...this._sources
1577
- .filter((source) => source.kind === "external" && source.root)
1578
- .map((source) => ({ root: source.root, source_id: source.id }))
1579
- ]
1580
- for (const entry of walkRoots) {
1581
- const rootStat = await fs.promises.lstat(entry.root)
1582
- if (!rootStat.isDirectory() || rootStat.isSymbolicLink()) {
1583
- throw new Error(`Repair source is not a real directory: ${entry.root}`)
1584
- }
1585
- }
1418
+ : this.scanRoots()
1586
1419
  for (const entry of walkRoots) {
1587
1420
  await this.walkForInoMatch(
1588
1421
  entry.root,
@@ -1595,6 +1428,38 @@ class Vault {
1595
1428
  const rebuiltScanIndex = new Map()
1596
1429
  const trustedHashes = new Set()
1597
1430
  if (preserved) {
1431
+ // Copy-mode content has no store inode to rediscover. Preserve only names
1432
+ // whose exact scan snapshot still proves the recorded content group.
1433
+ for (const [linkPath, link] of preserved.links) {
1434
+ if (link.mode !== "copy" || preserved.excluded.has(linkPath)) continue
1435
+ const blob = preserved.blobs.get(link.hash)
1436
+ const expected = preserved.scanIndex.get(linkPath)
1437
+ const source = this.sourceForPath(linkPath, link.source_id)
1438
+ if (!blob || !expected || expected.hash !== link.hash || !source ||
1439
+ !await this.canonicalPathIsWithinSource(linkPath, source, { strictErrors: true })) continue
1440
+ try {
1441
+ const st = await fs.promises.lstat(linkPath)
1442
+ if (!st.isFile() || st.size < this.sizeThreshold ||
1443
+ link.dev !== st.dev || link.ino !== st.ino ||
1444
+ !sameSnapshot(expected, st)) continue
1445
+ if (!rebuiltBlobs.has(link.hash)) {
1446
+ rebuiltBlobs.set(link.hash, Object.assign({}, blob, {
1447
+ size: st.size,
1448
+ verified_at: Date.now(),
1449
+ orphan: false
1450
+ }))
1451
+ }
1452
+ rebuiltLinks.set(linkPath, Object.assign({}, link, {
1453
+ app: source.kind === "app" ? source.app : (link.app || null),
1454
+ source_id: source.id,
1455
+ dev: st.dev,
1456
+ ino: st.ino,
1457
+ mode: "copy"
1458
+ }))
1459
+ } catch (error) {
1460
+ if (!isMissingError(error)) throw error
1461
+ }
1462
+ }
1598
1463
  // A store filename is only a trustworthy content hash when at least one
1599
1464
  // surviving name still matches the snapshot captured when that content
1600
1465
  // was hashed. Repair itself never hashes, so unverified names are left
@@ -1666,10 +1531,8 @@ class Vault {
1666
1531
  duplicates: rebuiltDuplicates,
1667
1532
  excluded: preserved ? preserved.excluded : new Map(),
1668
1533
  lastScan: preserved ? preserved.lastScan : null,
1669
- lastScanAttempt: preserved ? preserved.lastScanAttempt : null,
1670
1534
  sourceScans: preserved ? preserved.sourceScans : new Map(),
1671
- sourceScanAttempts: preserved ? preserved.sourceScanAttempts : new Map(),
1672
- settings: preserved ? preserved.settings : {},
1535
+ totals: preserved ? preserved.totals : { lifetime_bytes_saved: 0 },
1673
1536
  byIno: rebuiltByIno,
1674
1537
  pathsByIno: rebuiltPathsByIno
1675
1538
  })
@@ -1680,8 +1543,7 @@ class Vault {
1680
1543
  for await (const batch of walkBatches(root, {
1681
1544
  concurrency: this.dirConcurrency,
1682
1545
  skipDirectory: (full) => full === vaultRoot,
1683
- strictErrors: true,
1684
- onRootMissing: (error) => { throw error }
1546
+ strictErrors: true
1685
1547
  })) {
1686
1548
  const files = batch.flatMap((group) => group.files.map((file) => file.path))
1687
1549
  const stats = await statMany(files, this.statConcurrency, null, {
@@ -1690,19 +1552,19 @@ class Vault {
1690
1552
  })
1691
1553
  for (let index = 0; index < files.length; index++) {
1692
1554
  const st = stats[index]
1693
- if (!st || !st.isFile()) continue
1555
+ if (!st || !st.isFile() || st.size < this.sizeThreshold) continue
1694
1556
  const hash = storeInoToHash.get(`${st.dev}:${st.ino}`)
1695
1557
  if (hash) {
1696
1558
  const source = this.sourceForPath(files[index], preferredSourceId)
1697
1559
  const previous = this.registry.links.get(files[index])
1698
- const batchId = previous && previous.hash === hash &&
1560
+ const batchId = previous && previous.hash === hash && previous.mode === "link" &&
1699
1561
  previous.dev === st.dev && previous.ino === st.ino
1700
1562
  ? previous.batch_id || null
1701
1563
  : null
1702
1564
  const link = {
1703
1565
  hash, app: source && source.kind === "app" ? source.app : null,
1704
1566
  source_id: source ? source.id : null, dev: st.dev, ino: st.ino,
1705
- created: Date.now(),
1567
+ mode: "link", created: Date.now(),
1706
1568
  batch_id: batchId
1707
1569
  }
1708
1570
  if (outputLinks) outputLinks.set(files[index], link)
@@ -1722,7 +1584,6 @@ class Vault {
1722
1584
  // memory + one stat per blob and occupied shard; never a walk.
1723
1585
  async status(scopeId = null) {
1724
1586
  if (!this.enabled || !this.registry) return { enabled: false }
1725
- const responseActionSequence = this.actionSequence
1726
1587
  const registry = this.registry
1727
1588
  const scope = scopeId ? this.scanSource(scopeId) : null
1728
1589
  if (scopeId && !scope) throw new Error("That location is no longer available.")
@@ -1745,8 +1606,7 @@ class Vault {
1745
1606
  if (!namesByHash.has(entry.hash)) namesByHash.set(entry.hash, [])
1746
1607
  const location = this.locationForPath(linkPath, entry.source_id)
1747
1608
  namesByHash.get(entry.hash).push(Object.assign({
1748
- path: linkPath, app: entry.app || null, mode: "link",
1749
- unverified: !!entry.unverified
1609
+ path: linkPath, app: entry.app || null, mode: entry.mode
1750
1610
  }, location))
1751
1611
  if (entry.batch_id && (!scopeId || entry.source_id === scopeId)) {
1752
1612
  if (!undoBatchMap.has(entry.batch_id)) {
@@ -1758,6 +1618,11 @@ class Vault {
1758
1618
  batch.bytes += blob && Number.isFinite(blob.size) ? blob.size : 0
1759
1619
  }
1760
1620
  }
1621
+ const pendingBytesByHash = new Map()
1622
+ for (const entry of registry.duplicates.values()) {
1623
+ if (scopeId && entry.source_id !== scopeId) continue
1624
+ pendingBytesByHash.set(entry.hash, (pendingBytesByHash.get(entry.hash) || 0) + (entry.size || 0))
1625
+ }
1761
1626
  const blobEntries = [...registry.blobs].filter(([hash]) => !scopeHashes || scopeHashes.has(hash))
1762
1627
  if (!await this.directoryIfSafe(this.root) || !await this.directoryIfSafe(this.blobRoot)) {
1763
1628
  throw unsafeStoragePath(this.blobRoot)
@@ -1771,7 +1636,8 @@ class Vault {
1771
1636
  null,
1772
1637
  { strictErrors: true, followSymlinks: false }
1773
1638
  )
1774
- let savedBySharing = 0
1639
+ let bytesOnDisk = 0
1640
+ let wouldBe = 0
1775
1641
  for (let index = 0; index < blobEntries.length; index++) {
1776
1642
  const [hash, blob] = blobEntries[index]
1777
1643
  const names = namesByHash.get(hash) || []
@@ -1780,17 +1646,14 @@ class Vault {
1780
1646
  const nlink = storeStat ? storeStat.nlink : null
1781
1647
  storeStats.set(hash, storeStat)
1782
1648
  const size = blob.size || 0
1783
- const liveNames = storeStat ? Math.max(0, storeStat.nlink - 1) : 0
1784
- if (liveNames >= 2) {
1785
- for (const name of names) {
1786
- if (name.unverified || (scopeId && name.source_id !== scopeId)) continue
1787
- savedBySharing += size - (size / liveNames)
1788
- }
1789
- }
1649
+ const linkedCopy = storeStat || names.some((name) => name.mode === "link") ? 1 : 0
1650
+ const physicalCopies = linkedCopy + names.filter((name) => name.mode === "copy").length
1651
+ const pendingBytes = pendingBytesByHash.get(hash) || 0
1652
+ bytesOnDisk += (size * physicalCopies) + pendingBytes
1653
+ wouldBe += (size * Math.max(physicalCopies, names.length)) + pendingBytes
1790
1654
  const orphan = !!(storeStat && storeStat.nlink === 1)
1791
1655
  const publicBlob = {
1792
1656
  hash, size: blob.size || 0, orphan, nlink,
1793
- sharing_locations: liveNames,
1794
1657
  names, apps: [...apps], source_urls: blob.source_urls || []
1795
1658
  }
1796
1659
  blobs.push(publicBlob)
@@ -1803,8 +1666,7 @@ class Vault {
1803
1666
  const source = this.sourceForPath(p, location.source_id)
1804
1667
  const index = registry.scanIndex.get(p)
1805
1668
  const storeStat = storeStats.get(entry.hash)
1806
- const shareable = !!(source && source.shareable && storeStat && !entry.unverified &&
1807
- !entry.unavailable_reason &&
1669
+ const shareable = !!(source && source.shareable && storeStat && !entry.unavailable_reason &&
1808
1670
  (!index || index.dev === undefined || index.dev === storeStat.dev))
1809
1671
  let unavailableReason = entry.unavailable_reason || null
1810
1672
  if (!shareable && !unavailableReason) {
@@ -1816,8 +1678,7 @@ class Vault {
1816
1678
  const match = blob ? blob.names.find((name) => name.path !== p) || null : null
1817
1679
  duplicates.push(Object.assign({
1818
1680
  path: p, hash: entry.hash, size: entry.size || 0, app: entry.app || null,
1819
- shareable, unverified: !!entry.unverified,
1820
- unavailable_reason: entry.unverified ? "unavailable" : unavailableReason, match
1681
+ shareable, unavailable_reason: unavailableReason, match
1821
1682
  }, location))
1822
1683
  }
1823
1684
  const events = (await registry.readEvents()).reverse()
@@ -1850,64 +1711,64 @@ class Vault {
1850
1711
  excluded.push(Object.assign({
1851
1712
  path: p,
1852
1713
  ts: meta.ts || null,
1853
- size: Number(meta.size) || 0,
1854
- unverified: !!meta.unverified
1714
+ size: Number(meta.size) || 0
1855
1715
  }, this.locationForPath(p, meta.source_id)))
1856
1716
  }
1857
1717
  const publicSources = this._sources.filter((source) => !scopeId || source.id === scopeId).map((source) => ({
1858
1718
  id: source.id, kind: source.kind, label: source.label, root: source.root,
1859
- display_path: source.root,
1719
+ display_path: source.kind === "pinokio" ? source.root : (source.mount_path || source.root),
1860
1720
  target_path: source.kind === "external" ? source.root : null,
1861
1721
  parent_id: scopeId ? null : source.parent_id, app: source.app || null,
1862
1722
  available: source.available !== false, shareable: source.kind === "virtual" ? null : !!source.shareable
1863
1723
  }))
1864
- const lastComplete = registry.scanFor(scopeId)
1865
- const lastAttempt = registry.scanAttemptFor(scopeId)
1866
- const inScope = (entry) => !scopeId || entry.source_id === scopeId
1867
- const hasUnavailableSource = !scopeId && this._sources.some((source) =>
1868
- source.kind === "external" && source.available === false)
1869
- const hasUnverified = hasUnavailableSource || [...registry.links.values()].some((entry) =>
1870
- inScope(entry) && entry.unverified) ||
1871
- [...registry.duplicates.values()].some((entry) =>
1872
- inScope(entry) && entry.unverified) ||
1873
- [...registry.excluded.values()].some((entry) =>
1874
- inScope(entry) && entry.unverified)
1875
- const incomplete = hasUnverified || !!(
1876
- lastAttempt && lastAttempt.complete === false &&
1877
- (!lastComplete || (lastAttempt.ts || 0) >= (lastComplete.ts || 0))
1878
- )
1879
- const metricsExact = !!lastComplete && !incomplete
1880
- const beforeBytes = metricsExact && Number.isFinite(lastComplete.bytes_total)
1881
- ? Math.max(0, lastComplete.bytes_total)
1882
- : null
1883
- const afterBytes = beforeBytes === null
1884
- ? null
1885
- : Math.max(0, beforeBytes - savedBySharing)
1886
1724
  const result = {
1887
1725
  enabled: true,
1888
1726
  mode: this.mode,
1889
- candidate_min_bytes: this.sizeThreshold,
1890
- candidate_size_options: CANDIDATE_SIZE_OPTIONS,
1891
1727
  scan,
1892
- file_action: this.fileActionStatus(scopeId),
1893
- file_actions: this.fileActionStatuses(scopeId),
1894
- action_sequence: responseActionSequence,
1895
- last_scan: lastComplete,
1896
- last_attempt: lastAttempt,
1897
- incomplete,
1898
- metrics_exact: metricsExact,
1899
- bytes_on_disk: afterBytes,
1900
- bytes_without_sharing: beforeBytes,
1901
- saved_by_sharing: savedBySharing,
1728
+ last_scan: registry.scanFor(scopeId),
1729
+ bytes_on_disk: bytesOnDisk,
1730
+ bytes_without_sharing: wouldBe,
1731
+ saved_by_sharing: wouldBe - bytesOnDisk,
1732
+ lifetime_bytes_saved: registry.totals.lifetime_bytes_saved,
1902
1733
  reclaimable: blobs.filter((b) => b.orphan).reduce((sum, b) => sum + b.size, 0),
1903
1734
  pending_bytes: duplicates.filter((d) => d.shareable).reduce((sum, d) => sum + d.size, 0),
1735
+ file_action: this.fileActionStatus(scopeId),
1904
1736
  activity_error: registry.eventError,
1905
1737
  cloud_sync_warning: this.cloudSyncProvider(),
1906
1738
  sources: publicSources, blobs, duplicates, excluded, events,
1907
1739
  undo_batches: undoBatches
1908
1740
  }
1909
1741
  if (scopeId) {
1742
+ const duplicateBytes = duplicates.reduce((sum, item) => sum + item.size, 0)
1743
+ const excludedBytes = excluded.reduce((sum, item) => sum + item.size, 0)
1744
+ let linkedBytes = 0
1745
+ let effectiveLinkedBytes = 0
1746
+ let sharedBytes = 0
1747
+ for (const blob of blobs) {
1748
+ const scopedNames = blob.names.filter((name) => name.source_id === scopeId)
1749
+ const scopedLinks = scopedNames.filter((name) => name.mode === "link").length
1750
+ const scopedCopies = scopedNames.length - scopedLinks
1751
+ const registeredLinks = blob.names.filter((name) => name.mode === "link").length
1752
+ // The store name is one hardlink but not a user-visible location.
1753
+ // Registry count prevents stale filesystem metadata from over-attributing the inode.
1754
+ const filesystemLinks = Number.isFinite(blob.nlink) ? Math.max(0, blob.nlink - 1) : 0
1755
+ const sharingLocations = Math.max(1, registeredLinks, filesystemLinks)
1756
+ linkedBytes += blob.size * scopedNames.length
1757
+ effectiveLinkedBytes += (blob.size * scopedCopies) +
1758
+ (blob.size * scopedLinks / sharingLocations)
1759
+ if (registeredLinks >= 2) sharedBytes += blob.size * scopedLinks
1760
+ }
1761
+ const trackedBytes = linkedBytes + duplicateBytes + excludedBytes
1762
+ const effectiveTrackedBytes = effectiveLinkedBytes + duplicateBytes + excludedBytes
1763
+ const folderBytes = result.last_scan && Number.isFinite(result.last_scan.bytes_total)
1764
+ ? result.last_scan.bytes_total
1765
+ : null
1910
1766
  result.scope_id = scopeId
1767
+ result.tracked_bytes = trackedBytes
1768
+ result.effective_bytes = folderBytes === null
1769
+ ? null
1770
+ : Math.max(0, folderBytes - trackedBytes) + effectiveTrackedBytes
1771
+ result.shared_bytes = sharedBytes
1911
1772
  result.reclaimable = 0
1912
1773
  }
1913
1774
  return result
@@ -1922,176 +1783,34 @@ class Vault {
1922
1783
  return Object.assign({}, state, {
1923
1784
  current_file: sweeper.currentHash ? path.basename(sweeper.currentHash.path) : null,
1924
1785
  pending,
1925
- error: this.scanError,
1926
- persistence_warning: this.scanPersistenceWarning &&
1927
- !!(this.registry && this.registry.persistDirty)
1786
+ error: this.scanError
1928
1787
  })
1929
1788
  }
1789
+ fileActionStatus(scopeId = null) {
1790
+ const progress = this.fileActionProgress
1791
+ if (!progress) return null
1792
+ if (scopeId) {
1793
+ if (progress.scope_id && progress.scope_id !== scopeId) return null
1794
+ if (progress.path) {
1795
+ const source = this.sourceForPath(progress.path)
1796
+ if (!source || source.id !== scopeId) return null
1797
+ }
1798
+ }
1799
+ return Object.assign({}, progress)
1800
+ }
1930
1801
  progressStatus(scopeId = null) {
1931
1802
  return {
1932
1803
  enabled: !!this.enabled,
1933
- candidate_min_bytes: this.sizeThreshold,
1934
- candidate_size_options: CANDIDATE_SIZE_OPTIONS,
1935
1804
  scan: this.scanStatus(),
1936
1805
  file_action: this.fileActionStatus(scopeId),
1937
- file_actions: this.fileActionStatuses(scopeId),
1938
- action_sequence: this.actionSequence,
1939
- last_scan: this.registry ? this.registry.scanFor(scopeId) : null,
1940
- last_attempt: this.registry ? this.registry.scanAttemptFor(scopeId) : null
1941
- }
1942
- }
1943
- fileActionStatus(scopeId = null) {
1944
- const action = this.fileAction
1945
- if (!action || (scopeId && action.source_id !== scopeId)) return null
1946
- return {
1947
- id: action.id,
1948
- kind: action.kind,
1949
- phase: action.phase,
1950
- path: action.path,
1951
- source_id: action.source_id,
1952
- bytes_copied: action.bytes_copied,
1953
- bytes_total: action.bytes_total,
1954
- files_completed: action.files_completed,
1955
- files_total: action.files_total
1956
- }
1957
- }
1958
- fileActionStatuses(scopeId = null) {
1959
- return [...this.fileActions.values()]
1960
- .filter((action) => !scopeId || action.source_id === scopeId)
1961
- .map((action) => ({
1962
- id: action.id,
1963
- kind: action.kind,
1964
- phase: action.phase,
1965
- path: action.path,
1966
- source_id: action.source_id,
1967
- bytes_copied: action.bytes_copied,
1968
- bytes_total: action.bytes_total,
1969
- files_completed: action.files_completed,
1970
- files_total: action.files_total
1971
- }))
1972
- }
1973
- navigationStatus(scopeId = null) {
1974
- if (!this.enabled || !this.initialized || !this.registry) {
1975
- return {
1976
- any_scan: false,
1977
- scanned: false,
1978
- scanning: false,
1979
- pending_bytes: 0
1980
- }
1981
- }
1982
- const scan = this.scanStatus()
1983
- const active = !!(scan && (scan.pending || scan.active || scan.queued > 0))
1984
- const sources = new Map(this._sources.map((source) => [source.id, source]))
1985
- let pendingBytes = 0
1986
- for (const entry of this.registry.duplicates.values()) {
1987
- if (scopeId && entry.source_id !== scopeId) continue
1988
- const source = sources.get(entry.source_id)
1989
- if (!source || source.available === false || !source.shareable ||
1990
- entry.unverified || entry.unavailable_reason ||
1991
- !this.registry.blobs.has(entry.hash)) continue
1992
- pendingBytes += Math.max(0, Number(entry.size) || 0)
1993
- }
1994
- const lastComplete = this.registry.scanFor(scopeId)
1995
- const lastAttempt = this.registry.scanAttemptFor(scopeId)
1996
- const hasUnavailableSource = !scopeId && this._sources.some((source) =>
1997
- source.kind === "external" && source.available === false)
1998
- const hasUnverified = hasUnavailableSource || [...this.registry.links.values()].some((entry) =>
1999
- (!scopeId || entry.source_id === scopeId) && entry.unverified) ||
2000
- [...this.registry.duplicates.values()].some((entry) =>
2001
- (!scopeId || entry.source_id === scopeId) && entry.unverified) ||
2002
- [...this.registry.excluded.values()].some((entry) =>
2003
- (!scopeId || entry.source_id === scopeId) && entry.unverified)
2004
- const incomplete = hasUnverified || !!(
2005
- lastAttempt && lastAttempt.complete === false &&
2006
- (!lastComplete || (lastAttempt.ts || 0) >= (lastComplete.ts || 0))
2007
- )
2008
- return {
2009
- any_scan: !!this.registry.lastScanAttempt || this.registry.sourceScanAttempts.size > 0,
2010
- scanned: !!lastAttempt,
2011
- scanning: active && (!scopeId || scan.scope_id === scopeId),
2012
- pending_bytes: pendingBytes,
2013
- incomplete
2014
- }
2015
- }
2016
- navigationStatusForApp(appName) {
2017
- return this.navigationStatus(sourceId("app", appName))
2018
- }
2019
- async preflightCopyCapacity(targets) {
2020
- if (!targets.length) return { ok: true }
2021
- if (!fs.promises.statfs) return { ok: false, status: "capacity-unavailable" }
2022
- const byDevice = new Map()
2023
- try {
2024
- for (const target of targets) {
2025
- const st = await fs.promises.lstat(target.path)
2026
- const key = String(st.dev)
2027
- if (!byDevice.has(key)) {
2028
- byDevice.set(key, { path: path.dirname(target.path), bytes: 0 })
2029
- }
2030
- byDevice.get(key).bytes += Math.max(0, Number(target.bytes) || 0)
2031
- }
2032
- for (const group of byDevice.values()) {
2033
- const stat = await fs.promises.statfs(group.path)
2034
- const blockSize = BigInt(stat.bsize)
2035
- const availableBlocks = BigInt(stat.bavail)
2036
- const available = blockSize * availableBlocks
2037
- if (available < BigInt(Math.ceil(group.bytes))) {
2038
- return { ok: false, status: "insufficient-space" }
2039
- }
2040
- }
2041
- return { ok: true }
2042
- } catch (error) {
2043
- return { ok: false, status: "capacity-unavailable" }
1806
+ last_scan: this.registry ? this.registry.scanFor(scopeId) : null
2044
1807
  }
2045
1808
  }
2046
1809
  async copyOut(filePath, sourcePath, expectedTarget, expectedSource = expectedTarget, options = {}) {
2047
1810
  const tmp = filePath + TMP_SUFFIX
2048
1811
  let copiedStat = null
2049
1812
  try {
2050
- if (options.onProgress) {
2051
- let sourceHandle = null
2052
- let targetHandle = null
2053
- try {
2054
- sourceHandle = await fs.promises.open(sourcePath, "r")
2055
- const sourceStat = await sourceHandle.stat()
2056
- targetHandle = await fs.promises.open(tmp,
2057
- fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL,
2058
- sourceStat.mode & 0o7777)
2059
- copiedStat = await targetHandle.stat()
2060
- const buffer = Buffer.allocUnsafe(COPY_PROGRESS_CHUNK_SIZE)
2061
- let offset = 0
2062
- options.onProgress(0, sourceStat.size)
2063
- while (offset < sourceStat.size) {
2064
- const length = Math.min(buffer.length, sourceStat.size - offset)
2065
- const { bytesRead } = await sourceHandle.read(buffer, 0, length, offset)
2066
- if (!bytesRead) {
2067
- const error = new Error("Source file ended while copying.")
2068
- error.code = "EIO"
2069
- throw error
2070
- }
2071
- let written = 0
2072
- while (written < bytesRead) {
2073
- const result = await targetHandle.write(
2074
- buffer, written, bytesRead - written, offset + written)
2075
- if (!result.bytesWritten) {
2076
- const error = new Error("Could not write the independent copy.")
2077
- error.code = "EIO"
2078
- throw error
2079
- }
2080
- written += result.bytesWritten
2081
- }
2082
- offset += bytesRead
2083
- options.onProgress(offset, sourceStat.size)
2084
- }
2085
- await targetHandle.chmod(sourceStat.mode & 0o7777)
2086
- } finally {
2087
- await Promise.all([
2088
- sourceHandle && sourceHandle.close(),
2089
- targetHandle && targetHandle.close()
2090
- ].filter(Boolean))
2091
- }
2092
- } else {
2093
- await fs.promises.copyFile(sourcePath, tmp, fs.constants.COPYFILE_EXCL)
2094
- }
1813
+ await fs.promises.copyFile(sourcePath, tmp, fs.constants.COPYFILE_EXCL)
2095
1814
  copiedStat = await fs.promises.lstat(tmp)
2096
1815
  if (options.canReplace && !options.canReplace()) {
2097
1816
  await unlinkIfSame(tmp, copiedStat)
@@ -2110,12 +1829,9 @@ class Vault {
2110
1829
  try {
2111
1830
  finalStat = await lstatIfPresent(filePath)
2112
1831
  } catch (error) {
2113
- // rename() committed the independent copy, but its pre-rename ctime is
2114
- // not final metadata. Keep only stable identity fields and require a
2115
- // later scan to verify the name.
2116
- const committedStat = copiedStat
2117
- copiedStat = null
2118
- return { status: "copied", stat: committedStat, unverified: true }
1832
+ // The independent copy was committed by rename(). Preserve that
1833
+ // completed action when only the post-commit metadata read failed.
1834
+ finalStat = copiedStat
2119
1835
  }
2120
1836
  if (!finalStat || finalStat.dev !== copiedStat.dev || finalStat.ino !== copiedStat.ino) {
2121
1837
  return { status: "stale" }
@@ -2129,100 +1845,54 @@ class Vault {
2129
1845
  throw error
2130
1846
  }
2131
1847
  }
2132
- // detach: undo sharing for one linked name and restore it as a pending
2133
- // duplicate.
2134
- async detach(filePath, queuedAction = null) {
1848
+ // detach: make one name an independent copy again ("go back"), and pin it
1849
+ // so future scans neither re-list nor re-share it. Works on linked names
1850
+ // (copies bytes out) and on pending duplicates (just ignores them).
1851
+ async detach(filePath) {
2135
1852
  if (!this.enabled) return { status: "disabled" }
2136
1853
  await this.refreshSources()
2137
1854
  const entry = this.registry.links.get(filePath)
2138
1855
  if (entry) {
2139
- if (entry.unverified) return { status: "unavailable" }
2140
1856
  const source = this.sourceForPath(filePath, entry.source_id)
2141
1857
  if (!source || !await this.canonicalPathIsWithinSource(filePath, source)) {
2142
1858
  return { status: "stale" }
2143
1859
  }
2144
- if (this.sourceAppIsRunning(source)) return { status: "locked" }
2145
- const before = await lstatIfPresent(filePath)
2146
- if (!before || !before.isFile() || before.dev !== entry.dev || before.ino !== entry.ino) return { status: "stale" }
2147
- const capacity = await this.preflightCopyCapacity([{ path: filePath, bytes: before.size }])
2148
- if (!capacity.ok) return { status: capacity.status }
2149
- const action = queuedAction || {
2150
- kind: "detach", path: filePath, source_id: entry.source_id || null,
2151
- bytes_copied: 0, bytes_total: before.size
2152
- }
2153
- action.bytes_total = before.size
2154
- const ownsAction = !queuedAction
2155
- if (ownsAction) this.fileAction = action
2156
- try {
2157
- const copied = await this.copyOut(filePath, filePath, fileSnapshot(before), fileSnapshot(before), {
2158
- canReplace: () => !this.sourceAppIsRunning(source),
2159
- onProgress: (bytesCopied, bytesTotal) => {
2160
- action.bytes_copied = bytesCopied
2161
- action.bytes_total = bytesTotal
2162
- }
2163
- })
2164
- if (copied.status !== "copied") return copied
2165
- const st = copied.stat
2166
- const baseEntry = {
2167
- hash: entry.hash, size: st.size, source_id: entry.source_id || null,
2168
- dev: st.dev, ino: st.ino
2169
- }
2170
- const scanEntry = copied.unverified ? null : Object.assign({}, baseEntry, {
2171
- mtime: st.mtimeMs, ctime: st.ctimeMs
1860
+ if (entry.mode === "copy") {
1861
+ const st = await lstatIfPresent(filePath)
1862
+ if (!st || !st.isFile() || st.dev !== entry.dev || st.ino !== entry.ino) return { status: "stale" }
1863
+ this.registry.exclude(filePath, {
1864
+ ts: Date.now(), source_id: entry.source_id || null, size: st.size
2172
1865
  })
2173
- this.registry.setDuplicate(filePath, Object.assign({
2174
- app: entry.app || null, discovered: Date.now(),
2175
- unverified: !!copied.unverified
2176
- }, scanEntry || baseEntry), scanEntry)
2177
- if (copied.unverified) this.registry.scanIndex.delete(filePath)
2178
- await this.refreshLinkSnapshots(entry.hash, entry.dev, entry.ino)
2179
1866
  await this.recordEvent({
2180
- kind: "detach", hash: entry.hash, path: filePath,
2181
- app: entry.app || null, source_id: entry.source_id || null,
2182
- size: st.size, bytes_saved: st.size
1867
+ kind: "skip", hash: entry.hash, path: filePath,
1868
+ app: entry.app || null, source_id: entry.source_id || null, size: st.size
2183
1869
  })
2184
- return { status: "detached" }
2185
- } finally {
2186
- if (ownsAction && this.fileAction === action) this.fileAction = null
1870
+ return { status: "ignored" }
2187
1871
  }
2188
- }
2189
- return { status: "not-found" }
2190
- }
2191
- async skip(filePath) {
2192
- if (!this.enabled) return { status: "disabled" }
2193
- const duplicate = this.registry.duplicates.get(filePath)
2194
- if (!duplicate) return { status: "not-found" }
2195
- if (duplicate.unverified) return { status: "unavailable" }
2196
- this.registry.exclude(filePath, {
2197
- ts: Date.now(),
2198
- source_id: duplicate.source_id || null,
2199
- size: duplicate.size || 0
2200
- })
2201
- await this.recordEvent({
2202
- kind: "skip",
2203
- hash: duplicate.hash,
2204
- path: filePath,
2205
- app: duplicate.app || null,
2206
- source_id: duplicate.source_id || null,
2207
- size: duplicate.size || 0
2208
- })
2209
- return { status: "ignored" }
2210
- }
2211
- // reshare: un-pin a skipped path; the next scan may list it again.
2212
- async reshare(filePath) {
2213
- if (!this.enabled) return { status: "disabled" }
2214
- const current = this.registry.excluded.get(filePath)
2215
- if (current && current.unverified) return { status: "unavailable" }
2216
- const excluded = this.registry.allowSharing(filePath)
2217
- if (excluded) {
2218
- await this.recordEvent({ kind: "reshare", path: filePath, source_id: excluded && excluded.source_id ? excluded.source_id : null })
2219
- return { status: "resharable" }
1872
+ if (this.sourceAppIsRunning(source)) return { status: "locked" }
1873
+ const before = await lstatIfPresent(filePath)
1874
+ if (!before || !before.isFile() || before.dev !== entry.dev || before.ino !== entry.ino) return { status: "stale" }
1875
+ const copied = await this.copyOut(filePath, filePath, fileSnapshot(before), fileSnapshot(before), {
1876
+ canReplace: () => !this.sourceAppIsRunning(source)
1877
+ })
1878
+ if (copied.status !== "copied") return copied
1879
+ const st = copied.stat
1880
+ this.registry.exclude(filePath, { ts: Date.now(), source_id: entry.source_id || null, size: st.size })
1881
+ await this.refreshLinkSnapshots(entry.hash, entry.dev, entry.ino)
1882
+ await this.recordEvent({ kind: "detach", hash: entry.hash, path: filePath, app: entry.app || null, source_id: entry.source_id || null, size: st.size })
1883
+ return { status: "detached" }
1884
+ }
1885
+ if (this.registry.duplicates.has(filePath)) {
1886
+ const dup = this.registry.duplicates.get(filePath)
1887
+ this.registry.exclude(filePath, { ts: Date.now(), source_id: dup.source_id || null, size: dup.size || 0 })
1888
+ await this.recordEvent({ kind: "skip", hash: dup.hash, path: filePath, app: dup.app || null, source_id: dup.source_id || null, size: dup.size || 0 })
1889
+ return { status: "ignored" }
2220
1890
  }
2221
1891
  return { status: "not-found" }
2222
1892
  }
2223
1893
  // Undo a conversion batch: replace each converted name with an independent
2224
1894
  // copy of the bytes (spec, UX contract surface 3).
2225
- async undoBatch(batchId, queuedAction = null) {
1895
+ async undoBatch(batchId) {
2226
1896
  if (!this.enabled || !batchId) return { undone: 0 }
2227
1897
  await this.refreshSources()
2228
1898
  const events = await this.registry.readEvents()
@@ -2234,16 +1904,11 @@ class Vault {
2234
1904
  if (entry.batch_id === batchId) targets.add(filePath)
2235
1905
  }
2236
1906
  const summary = { undone: 0, bytes: 0, failed: 0 }
2237
- const prepared = []
2238
1907
  for (const filePath of targets) {
2239
1908
  const event = conversionEvents.get(filePath) || {}
2240
1909
  const entry = this.registry.links.get(filePath)
2241
1910
  if (!entry || (entry.batch_id && entry.batch_id !== batchId) ||
2242
1911
  (event.hash && entry.hash !== event.hash)) continue
2243
- if (entry.unverified) {
2244
- summary.failed += 1
2245
- continue
2246
- }
2247
1912
  const storePath = this.storePathFor(entry.hash)
2248
1913
  try {
2249
1914
  const source = this.sourceForPath(filePath, entry.source_id || event.source_id)
@@ -2266,34 +1931,8 @@ class Vault {
2266
1931
  summary.failed += 1
2267
1932
  continue
2268
1933
  }
2269
- prepared.push({ filePath, event, entry, source, storePath, st, storeStat })
2270
- } catch (e) {
2271
- summary.failed += 1
2272
- }
2273
- }
2274
- const capacity = await this.preflightCopyCapacity(prepared.map((target) => ({
2275
- path: target.filePath,
2276
- bytes: target.storeStat.size
2277
- })))
2278
- if (!capacity.ok) {
2279
- return Object.assign(summary, { status: capacity.status })
2280
- }
2281
- const action = queuedAction || {
2282
- kind: "undo", path: null, source_id: null,
2283
- bytes_copied: 0, bytes_total: 0
2284
- }
2285
- action.bytes_total = prepared.reduce((sum, target) => sum + target.storeStat.size, 0)
2286
- let completedBytes = 0
2287
- for (const target of prepared) {
2288
- const { filePath, event, entry, source, storePath, st, storeStat } = target
2289
- try {
2290
- action.path = filePath
2291
- action.source_id = entry.source_id || event.source_id || null
2292
1934
  const copied = await this.copyOut(filePath, storePath, fileSnapshot(st), fileSnapshot(storeStat), {
2293
- canReplace: () => !this.sourceAppIsRunning(source),
2294
- onProgress: (bytesCopied) => {
2295
- action.bytes_copied = completedBytes + bytesCopied
2296
- }
1935
+ canReplace: () => !this.sourceAppIsRunning(source)
2297
1936
  })
2298
1937
  if (copied.status !== "copied") {
2299
1938
  summary.failed += 1
@@ -2304,18 +1943,19 @@ class Vault {
2304
1943
  hash: entry.hash, size: storeStat.size, app: entry.app || null,
2305
1944
  source_id: entry.source_id || event.source_id || null, discovered: Date.now(),
2306
1945
  dev: independentStat.dev, ino: independentStat.ino,
2307
- unverified: !!copied.unverified
1946
+ mtime: independentStat.mtimeMs, ctime: independentStat.ctimeMs
2308
1947
  }
2309
- const scanEntry = copied.unverified ? null : {
1948
+ const scanEntry = {
2310
1949
  hash: entry.hash, size: independentStat.size,
2311
1950
  dev: independentStat.dev, ino: independentStat.ino,
2312
1951
  mtime: independentStat.mtimeMs, ctime: independentStat.ctimeMs,
2313
1952
  source_id: entry.source_id || event.source_id || null
2314
1953
  }
2315
1954
  this.registry.setDuplicate(filePath, duplicateEntry, scanEntry)
2316
- if (copied.unverified) this.registry.scanIndex.delete(filePath)
2317
1955
  await this.refreshLinkSnapshots(entry.hash, entry.dev, entry.ino)
2318
1956
  const bytesSaved = event.bytes_saved || storeStat.size
1957
+ this.registry.totals.lifetime_bytes_saved = Math.max(0,
1958
+ this.registry.totals.lifetime_bytes_saved - bytesSaved)
2319
1959
  await this.recordEvent({
2320
1960
  kind: "undo", hash: entry.hash, path: filePath,
2321
1961
  source_id: entry.source_id || event.source_id || null, batch_id: batchId,
@@ -2323,8 +1963,6 @@ class Vault {
2323
1963
  })
2324
1964
  summary.undone += 1
2325
1965
  summary.bytes += storeStat.size
2326
- completedBytes += storeStat.size
2327
- action.bytes_copied = completedBytes
2328
1966
  } catch (e) {
2329
1967
  summary.failed += 1
2330
1968
  }
@@ -2336,10 +1974,14 @@ class Vault {
2336
1974
  if (!this.enabled) return { reclaimed: 0, bytes_freed: 0 }
2337
1975
  const summary = { reclaimed: 0, bytes_freed: 0, failed: 0 }
2338
1976
  const removedHashes = new Set()
1977
+ const copyHashes = new Set([...this.registry.links.values()]
1978
+ .filter((entry) => entry.mode === "copy")
1979
+ .map((entry) => entry.hash))
2339
1980
  try {
2340
1981
  for (const [hash] of [...this.registry.blobs]) {
2341
1982
  const result = await this.reclaim(hash, {
2342
- deferRegistry: true
1983
+ deferRegistry: true,
1984
+ hasCopyNames: copyHashes.has(hash)
2343
1985
  })
2344
1986
  if (result.remove_hash) removedHashes.add(hash)
2345
1987
  if (result.status === "reclaimed") {