pinokiod 8.0.50 → 8.0.53

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,26 +1,37 @@
1
- const fs = require('fs')
2
- const path = require('path')
3
- const crypto = require('crypto')
4
- const { Worker } = require('worker_threads')
5
- const Registry = require('./registry')
6
- const Sweeper = require('./sweeper')
7
- const { walkBatches, statMany } = require('./walker')
8
- const { fileSnapshot, sameSnapshot, sameContentState } = require('./snapshot')
1
+ const fs = require("fs")
2
+ const path = require("path")
3
+ const crypto = require("crypto")
4
+ const { execFile } = require("child_process")
5
+ const { Worker } = require("worker_threads")
6
+ const Registry = require("./registry")
7
+ const Sweeper = require("./sweeper")
8
+ const { fileSnapshot, sameSnapshot, sameContentState } = require("./snapshot")
9
9
  const {
10
- SIZE_THRESHOLD, CANDIDATE_SIZE_OPTIONS, TMP_SUFFIX, SHA256_RE,
11
- DIR_CONCURRENCY, STAT_CONCURRENCY, HASH_INACTIVITY_MS
12
- } = require('./constants')
10
+ SIZE_THRESHOLD,
11
+ CANDIDATE_SIZE_OPTIONS,
12
+ TMP_SUFFIX,
13
+ SHA256_RE,
14
+ DIR_CONCURRENCY,
15
+ STAT_CONCURRENCY,
16
+ HASH_INACTIVITY_MS
17
+ } = require("./constants")
13
18
 
14
- // Shared model store engine (spec/requirements/shared-model-store.md).
15
- // Store + registry + volume probe + hashing + adopt/convert/verify, plus the
16
- // user-facing operations behind the vault dashboard (detach, undo, reclaim).
17
- // Nothing here runs automatically except startup verify; discovery happens
18
- // only through the user's manual scan.
19
-
20
- const NO_LINK_CODES = new Set(["EXDEV", "ENOTSUP", "ENOSYS"])
19
+ const NO_LINK_CODES = new Set([
20
+ "EXDEV", "ENOTSUP", "EOPNOTSUPP", "ENOSYS", "EPERM", "EACCES"
21
+ ])
21
22
  const LOCK_CODES = new Set(["EBUSY", "EPERM", "EACCES"])
22
- const isMissingError = (error) => !!(error && (error.code === "ENOENT" || error.code === "ENOTDIR"))
23
- const isAccessError = (error) => !!(error && (error.code === "EACCES" || error.code === "EPERM"))
23
+ const STATUS_PAGE_SIZE = 500
24
+ const MAX_BULK_FILE_ACTIONS = 500
25
+ const STATUS_VIEWS = new Set([
26
+ "all", "duplicates", "shared", "tracked", "reclaimable", "activity"
27
+ ])
28
+ const STATUS_FILTERS = new Set([
29
+ "all", "duplicate", "shared", "tracked"
30
+ ])
31
+
32
+ const isMissingError = (error) => !!(error &&
33
+ (error.code === "ENOENT" || error.code === "ENOTDIR"))
34
+
24
35
  const lstatIfPresent = async (filePath) => {
25
36
  try {
26
37
  return await fs.promises.lstat(filePath)
@@ -29,9 +40,11 @@ const lstatIfPresent = async (filePath) => {
29
40
  throw error
30
41
  }
31
42
  }
43
+
32
44
  const sameIdentity = (left, right) => !!(
33
45
  left && right && left.dev === right.dev && left.ino === right.ino
34
46
  )
47
+
35
48
  const unlinkIfSame = async (filePath, expected) => {
36
49
  if (!expected) return false
37
50
  const current = await lstatIfPresent(filePath)
@@ -44,41 +57,92 @@ const unlinkIfSame = async (filePath, expected) => {
44
57
  throw error
45
58
  }
46
59
  }
60
+
47
61
  const unsafeStoragePath = (filePath) => {
48
62
  const error = new Error(`Storage path is not a real directory: ${filePath}`)
49
63
  error.code = "EVAULTPATH"
50
64
  return error
51
65
  }
52
66
 
67
+ const unavailableStorage = (message) => {
68
+ const error = new Error(message)
69
+ error.code = "EVAULTUNAVAILABLE"
70
+ return error
71
+ }
72
+
53
73
  const isPathWithin = (root, target) => {
54
- const rel = path.relative(path.resolve(root), path.resolve(target))
55
- return rel === "" || (!rel.startsWith(`..${path.sep}`) && rel !== ".." && !path.isAbsolute(rel))
74
+ const relative = path.relative(path.resolve(root), path.resolve(target))
75
+ return relative === "" || (
76
+ !relative.startsWith(`..${path.sep}`) &&
77
+ relative !== ".." &&
78
+ !path.isAbsolute(relative)
79
+ )
56
80
  }
57
81
 
58
82
  const samePath = (left, right) => {
59
- const a = path.resolve(left)
60
- const b = path.resolve(right)
61
- return process.platform === "win32" ? a.toLowerCase() === b.toLowerCase() : a === b
83
+ const first = path.resolve(left)
84
+ const second = path.resolve(right)
85
+ return process.platform === "win32"
86
+ ? first.toLowerCase() === second.toLowerCase()
87
+ : first === second
62
88
  }
63
89
 
64
90
  const sameFileMetadata = (left, right) => {
65
91
  if (!left || !right) return false
66
92
  if ((left.mode & 0o7777) !== (right.mode & 0o7777)) return false
67
- if (left.uid !== undefined && right.uid !== undefined && left.uid !== right.uid) return false
68
- if (left.gid !== undefined && right.gid !== undefined && left.gid !== right.gid) return false
93
+ if (left.uid !== undefined && right.uid !== undefined &&
94
+ left.uid !== right.uid) return false
95
+ if (left.gid !== undefined && right.gid !== undefined &&
96
+ left.gid !== right.gid) return false
69
97
  return true
70
98
  }
71
99
 
72
100
  const sourceId = (kind, name) => `${kind}:${encodeURIComponent(name)}`
101
+ const boundedInteger = (value, fallback, minimum, maximum) => {
102
+ const parsed = Number(value)
103
+ if (!Number.isSafeInteger(parsed)) return fallback
104
+ return Math.max(minimum, Math.min(maximum, parsed))
105
+ }
106
+ const rowSnapshot = (row) => ({
107
+ size: row.size,
108
+ mtime: row.mtime,
109
+ ctime: row.ctime,
110
+ dev: row.dev,
111
+ ino: row.ino
112
+ })
113
+
114
+ const revealInFileManager = (filePath, platform = process.platform) =>
115
+ new Promise((resolve, reject) => {
116
+ const command = platform === "darwin"
117
+ ? "open"
118
+ : platform === "win32"
119
+ ? "explorer.exe"
120
+ : "xdg-open"
121
+ const args = platform === "darwin"
122
+ ? ["-R", filePath]
123
+ : platform === "win32"
124
+ ? [`/select,${filePath}`]
125
+ : [path.dirname(filePath)]
126
+ execFile(command, args, {
127
+ timeout: 10000,
128
+ windowsHide: true
129
+ }, (error) => error ? reject(error) : resolve())
130
+ })
73
131
 
74
132
  class Vault {
75
133
  constructor(kernel) {
76
134
  this.kernel = kernel
135
+ this.fileManagerLauncher = (filePath) =>
136
+ revealInFileManager(
137
+ filePath,
138
+ this.kernel.platform || process.platform
139
+ )
77
140
  this.enabled = false
78
141
  this.initialized = false
79
- this.mode = null // 'link' | 'copy' for the vault's own volume
80
- this.volumeModes = new Map() // dev -> 'link' | 'copy'
142
+ this.mode = null
143
+ this.volumeModes = new Map()
81
144
  this.registry = null
145
+ this.sweeper = null
82
146
  this.worker = null
83
147
  this.workerJobs = new Map()
84
148
  this.workerSeq = 0
@@ -89,597 +153,575 @@ class Vault {
89
153
  this.dirConcurrency = DIR_CONCURRENCY
90
154
  this.sizeThreshold = SIZE_THRESHOLD
91
155
  this._sources = []
156
+ this._sourcesById = new Map()
92
157
  this._sourceBases = new Map()
158
+ this._anchorStores = []
159
+ this._anchorStoresById = new Map()
160
+ this._anchorStoresByDevice = new Map()
161
+ this._fallbackConfig = null
93
162
  this.operationTail = Promise.resolve()
94
163
  this.initializationPromise = null
95
- this.verificationPending = false
164
+ this.scanPromise = null
96
165
  this.scanError = null
97
166
  this.scanScopeId = null
167
+ this.scanCancelRequested = false
168
+ this.lastScanCache = new Map()
98
169
  this.fileActionProgress = null
170
+ this.fileActionCancelRequested = false
99
171
  }
172
+
100
173
  get root() {
101
174
  return path.resolve(this.kernel.homedir, "vault")
102
175
  }
176
+
103
177
  get blobRoot() {
104
- return path.resolve(this.root, "sha256")
178
+ const store = this.defaultAnchorStore()
179
+ return store
180
+ ? path.resolve(store.root, "sha256")
181
+ : path.resolve(this.globalConfigRoot(), "vault", "sha256")
105
182
  }
106
- get sourceRoot() {
107
- return path.resolve(this.root, "sources")
108
- }
109
- storePathFor(hash) {
183
+
184
+ storePathFor(hash, storeId = null) {
110
185
  if (typeof hash !== "string" || !SHA256_RE.test(hash)) {
111
186
  throw new TypeError("Invalid vault content identifier.")
112
187
  }
113
- return path.resolve(this.blobRoot, hash.slice(0, 2), hash)
188
+ const store = storeId
189
+ ? this._anchorStoresById.get(storeId)
190
+ : this.defaultAnchorStore()
191
+ if (!store) throw new Error("No anchor store is configured.")
192
+ return path.resolve(
193
+ store.root, "sha256", hash.slice(0, 2), hash)
194
+ }
195
+
196
+ globalConfigRoot() {
197
+ return this.kernel.store && typeof this.kernel.store.root === "string"
198
+ ? path.resolve(this.kernel.store.root)
199
+ : path.resolve(this.kernel.homedir, ".pinokio")
200
+ }
201
+
202
+ readConfig() {
203
+ const stored = this.kernel.store &&
204
+ typeof this.kernel.store.get === "function"
205
+ ? this.kernel.store.get("vault")
206
+ : this._fallbackConfig
207
+ const config = stored && typeof stored === "object" ? stored : {}
208
+ return {
209
+ locations: Array.isArray(config.locations)
210
+ ? config.locations.filter((item) => typeof item === "string")
211
+ : [],
212
+ anchor_stores: Array.isArray(config.anchor_stores)
213
+ ? config.anchor_stores.filter((item) =>
214
+ item &&
215
+ typeof item === "object" &&
216
+ typeof item.id === "string" &&
217
+ item.id &&
218
+ typeof item.root === "string" &&
219
+ typeof item.probe_path === "string")
220
+ : []
221
+ }
222
+ }
223
+
224
+ writeConfig(config) {
225
+ const value = {
226
+ locations: [...new Set((config.locations || [])
227
+ .filter((item) => typeof item === "string")
228
+ .map((item) => path.resolve(item)))],
229
+ anchor_stores: (config.anchor_stores || []).map((store) => ({
230
+ id: store.id,
231
+ root: path.resolve(store.root),
232
+ probe_path: path.resolve(store.probe_path)
233
+ }))
234
+ }
235
+ if (this.kernel.store && typeof this.kernel.store.set === "function") {
236
+ this.kernel.store.set("vault", value)
237
+ } else {
238
+ this._fallbackConfig = value
239
+ }
240
+ return value
241
+ }
242
+
243
+ configuredLocations() {
244
+ return this.readConfig().locations
245
+ }
246
+
247
+ defaultAnchorStore() {
248
+ const home = path.resolve(this.kernel.homedir)
249
+ return this._anchorStores.find((store) =>
250
+ store.available && isPathWithin(store.probe_path, home)) ||
251
+ this._anchorStores[0] ||
252
+ null
114
253
  }
254
+
115
255
  async directoryIfSafe(directory) {
116
- const st = await lstatIfPresent(directory)
117
- if (st && !st.isDirectory()) throw unsafeStoragePath(directory)
118
- return st
256
+ const stat = await lstatIfPresent(directory)
257
+ if (stat && (!stat.isDirectory() || stat.isSymbolicLink())) {
258
+ throw unsafeStoragePath(directory)
259
+ }
260
+ return stat
119
261
  }
262
+
120
263
  async ensureDirectory(directory) {
121
- let st = await this.directoryIfSafe(directory)
122
- if (st) return st
264
+ let stat = await this.directoryIfSafe(directory)
265
+ if (stat) return stat
123
266
  try {
124
- await fs.promises.mkdir(directory)
267
+ await fs.promises.mkdir(directory, { mode: 0o700 })
125
268
  } catch (error) {
126
269
  if (!error || error.code !== "EEXIST") throw error
127
270
  }
128
- st = await this.directoryIfSafe(directory)
129
- if (!st) throw unsafeStoragePath(directory)
130
- return st
271
+ stat = await this.directoryIfSafe(directory)
272
+ if (!stat) throw unsafeStoragePath(directory)
273
+ return stat
131
274
  }
275
+
132
276
  async storeStatIfPresent(storePath, options = {}) {
133
- if (!isPathWithin(this.blobRoot, storePath)) throw unsafeStoragePath(storePath)
134
- if (!await this.directoryIfSafe(this.root) || !await this.directoryIfSafe(this.blobRoot)) {
135
- throw unsafeStoragePath(this.blobRoot)
277
+ const store = options.store_id
278
+ ? this._anchorStoresById.get(options.store_id)
279
+ : this._anchorStores.find((candidate) =>
280
+ isPathWithin(path.resolve(candidate.root, "sha256"), storePath))
281
+ if (!store) {
282
+ throw unsafeStoragePath(storePath)
283
+ }
284
+ const blobRoot = path.resolve(store.root, "sha256")
285
+ if (!isPathWithin(blobRoot, storePath)) {
286
+ throw unsafeStoragePath(storePath)
287
+ }
288
+ if (options.createParent) {
289
+ await this.ensureAnchorStore(store, options.dev)
290
+ }
291
+ if (!await this.directoryIfSafe(store.root) ||
292
+ !await this.directoryIfSafe(blobRoot)) {
293
+ return null
136
294
  }
137
295
  const shard = path.dirname(storePath)
138
296
  let shardStat = await this.directoryIfSafe(shard)
139
- if (!shardStat && options.createParent) shardStat = await this.ensureDirectory(shard)
297
+ if (!shardStat && options.createParent) {
298
+ shardStat = await this.ensureDirectory(shard)
299
+ }
140
300
  if (!shardStat) return null
141
- return lstatIfPresent(storePath)
301
+ const stat = await lstatIfPresent(storePath)
302
+ if (stat && (!stat.isFile() || stat.isSymbolicLink())) {
303
+ throw unsafeStoragePath(storePath)
304
+ }
305
+ return stat
306
+ }
307
+
308
+ async ensureAnchorStore(store, expectedDev = null) {
309
+ if (!store) throw new Error("No anchor store is configured.")
310
+ const managementParent = path.dirname(store.root)
311
+ const parentStat = await lstatIfPresent(managementParent)
312
+ if (!parentStat) {
313
+ await fs.promises.mkdir(managementParent, {
314
+ recursive: true,
315
+ mode: 0o700
316
+ })
317
+ } else if (!parentStat.isDirectory() || parentStat.isSymbolicLink()) {
318
+ throw unsafeStoragePath(managementParent)
319
+ }
320
+ await this.ensureDirectory(store.root)
321
+ const markerPath = path.resolve(store.root, "store.json")
322
+ const markerStat = await lstatIfPresent(markerPath)
323
+ if (markerStat && (!markerStat.isFile() || markerStat.isSymbolicLink())) {
324
+ throw unsafeStoragePath(markerPath)
325
+ }
326
+ if (markerStat) {
327
+ let marker
328
+ try {
329
+ marker = JSON.parse(await fs.promises.readFile(markerPath, "utf8"))
330
+ } catch (error) {
331
+ throw unsafeStoragePath(markerPath)
332
+ }
333
+ if (!marker || marker.id !== store.id || marker.version !== 1) {
334
+ throw unsafeStoragePath(markerPath)
335
+ }
336
+ } else {
337
+ await fs.promises.writeFile(
338
+ markerPath,
339
+ `${JSON.stringify({ id: store.id, version: 1 }, null, 2)}\n`,
340
+ { flag: "wx", mode: 0o600 }
341
+ )
342
+ }
343
+ const rootStat = await fs.promises.stat(store.root)
344
+ if (expectedDev !== null && rootStat.dev !== expectedDev) {
345
+ throw unavailableStorage(
346
+ "The configured anchor store is on a different filesystem.")
347
+ }
348
+ const mode = await this.probe(store.root)
349
+ store.available = true
350
+ store.dev = rootStat.dev
351
+ store.mode = mode
352
+ this._anchorStoresByDevice.set(rootStat.dev, store)
353
+ if (mode !== "link") {
354
+ throw unavailableStorage(
355
+ "This filesystem does not support hardlinks.")
356
+ }
357
+ await this.ensureDirectory(path.resolve(store.root, "sha256"))
358
+ return store
359
+ }
360
+
361
+ async validateAnchorStore(store, expectedDev = null) {
362
+ if (!store) throw new Error("No anchor store is configured.")
363
+ const markerPath = path.resolve(store.root, "store.json")
364
+ const [rootStat, markerStat] = await Promise.all([
365
+ this.directoryIfSafe(store.root),
366
+ lstatIfPresent(markerPath)
367
+ ])
368
+ if (!rootStat ||
369
+ !markerStat ||
370
+ !markerStat.isFile() ||
371
+ markerStat.isSymbolicLink()) {
372
+ throw unsafeStoragePath(store.root)
373
+ }
374
+ let marker
375
+ try {
376
+ marker = JSON.parse(await fs.promises.readFile(markerPath, "utf8"))
377
+ } catch (error) {
378
+ throw unsafeStoragePath(markerPath)
379
+ }
380
+ if (!marker || marker.id !== store.id || marker.version !== 1) {
381
+ throw unsafeStoragePath(markerPath)
382
+ }
383
+ const current = await fs.promises.stat(store.root)
384
+ if (expectedDev !== null && current.dev !== expectedDev) {
385
+ throw unavailableStorage(
386
+ "The configured anchor store is on a different filesystem.")
387
+ }
388
+ store.available = true
389
+ store.dev = current.dev
390
+ return current
142
391
  }
143
- // All dashboard mutations share one queue. Status reads remain concurrent,
144
- // while scans, conversions, repair, undo, detach, and reclaim can never
145
- // publish interleaved filesystem/registry state.
392
+
146
393
  runExclusive(operation) {
147
394
  const pending = this.operationTail.then(operation, operation)
148
395
  this.operationTail = pending.catch(() => {})
149
396
  return pending
150
397
  }
151
- // A filesystem mutation is committed before its registry snapshot is
152
- // flushed. Keep that distinction in the response: failed persistence is
153
- // retried by Registry and must not make an already-completed action look as
154
- // though it never happened.
398
+
155
399
  runMutation(operation) {
156
- return this.runExclusive(async () => {
157
- const result = await operation()
158
- try {
159
- if (this.registry) await this.registry.flush()
160
- } catch (error) {
161
- return Object.assign({}, result, { persistence_warning: true })
162
- }
163
- return result
164
- })
400
+ return this.runExclusive(operation)
165
401
  }
402
+
166
403
  async runFileAction(progress, operation) {
404
+ this.fileActionCancelRequested = false
167
405
  this.fileActionProgress = progress
168
406
  try {
169
407
  return await operation(progress)
170
408
  } finally {
171
- if (this.fileActionProgress === progress) this.fileActionProgress = null
409
+ if (this.fileActionProgress === progress) {
410
+ this.fileActionProgress = null
411
+ }
412
+ this.fileActionCancelRequested = false
172
413
  }
173
414
  }
174
- startScan(scopeId = null, sizeThreshold = this.sizeThreshold) {
175
- if (!this.enabled || !this.sweeper) return { started: false, disabled: true }
176
- if (this.scanPromise) return { started: false, already_running: true }
177
- this.sizeThreshold = sizeThreshold
178
- this.scanError = null
179
- this.scanScopeId = scopeId
180
- this.scanPromise = this.runExclusive(() => this.sweeper.scan(scopeId))
181
- .catch((error) => {
182
- this.scanError = error && error.message ? error.message : String(error)
183
- throw error
184
- })
185
- .finally(() => {
186
- this.scanPromise = null
187
- this.scanScopeId = null
188
- })
189
- this.scanPromise.catch(() => {})
190
- return { started: true }
415
+
416
+ cancelFileAction() {
417
+ if (!this.fileActionProgress ||
418
+ !this.fileActionProgress.cancelable) {
419
+ return { cancel_requested: false }
420
+ }
421
+ this.fileActionCancelRequested = true
422
+ this.fileActionProgress.cancel_requested = true
423
+ return { cancel_requested: true }
191
424
  }
192
- async perform(action, payload = {}) {
193
- if (!this.enabled) return { error: "Save space is disabled." }
194
- await this.ensureInitialized()
195
- switch (action) {
196
- case "add_source": {
197
- const result = await this.runExclusive(() => this.addExternalSource(payload.path))
198
- return {
199
- created: result.created,
200
- source: {
201
- id: result.source.id,
202
- label: result.source.label,
203
- target_path: result.source.root,
204
- shareable: !!result.source.shareable
205
- }
206
- }
207
- }
208
- case "scan": {
209
- if (payload.scope_id && !this.scanSource(payload.scope_id)) {
210
- return { error: "That scan location is no longer available." }
211
- }
212
- let sizeThreshold = this.sizeThreshold
213
- if (payload.candidate_size != null) {
214
- if (!CANDIDATE_SIZE_OPTIONS.includes(payload.candidate_size)) {
215
- return { error: "Choose a valid minimum file size." }
216
- }
217
- sizeThreshold = payload.candidate_size
425
+
426
+ async isEnabled() {
427
+ let value = process.env.PINOKIO_VAULT
428
+ if (value === undefined && this.kernel.homedir) {
429
+ try {
430
+ const raw = await fs.promises.readFile(
431
+ path.resolve(this.kernel.homedir, "ENVIRONMENT"), "utf8")
432
+ for (const line of raw.split("\n")) {
433
+ const match = line.match(/^\s*PINOKIO_VAULT\s*=\s*(.*)\s*$/)
434
+ if (match) value = match[1].trim()
218
435
  }
219
- return this.startScan(payload.scope_id || null, sizeThreshold)
436
+ } catch (error) {
437
+ if (!isMissingError(error)) throw error
220
438
  }
221
- case "deduplicate":
222
- if (typeof payload.path === "string" && payload.path) {
223
- return this.runMutation(() => this.runFileAction({
224
- kind: "deduplicate-file",
225
- path: path.resolve(payload.path)
226
- }, () => this.deduplicateFile(payload.path, {
227
- batch_id: `batch-${crypto.randomUUID()}`
228
- })))
229
- }
230
- const selection = payload.selection || null
231
- if (selection && selection !== "duplicates" && selection !== "kept-separate") {
232
- return { error: "Choose files to deduplicate." }
233
- }
234
- if (payload.scope_id != null &&
235
- (typeof payload.scope_id !== "string" || !payload.scope_id)) {
236
- return { error: "Choose a valid location to deduplicate." }
237
- }
238
- const scopeId = typeof payload.scope_id === "string" && payload.scope_id
239
- ? payload.scope_id
240
- : null
241
- if (!selection && !scopeId) {
242
- return { error: "Choose a location to deduplicate." }
243
- }
244
- return this.runMutation(() => this.runFileAction({
245
- kind: "deduplicate",
246
- scope_id: scopeId,
247
- selection: selection || "duplicates",
248
- files_total: 0,
249
- files_completed: 0
250
- }, async (progress) => {
251
- const options = {
252
- batch_id: `batch-${crypto.randomUUID()}`,
253
- progress
254
- }
255
- return selection
256
- ? this.deduplicateSelection(selection, scopeId, options)
257
- : this.deduplicateScope(scopeId, options)
258
- }))
259
- case "reclaim":
260
- return this.runMutation(() => this.reclaim(payload.hash))
261
- case "reclaim_all":
262
- return this.runMutation(() => this.reclaimAll())
263
- case "repair":
264
- case "rebuild":
265
- if (this.scanPromise || (this.sweeper && this.sweeper.state.active)) {
266
- return { error: "Wait for the current scan to finish before repairing the index." }
267
- }
268
- return this.runMutation(async () => {
269
- await this.rebuild(undefined, { flush: false })
270
- return { done: true }
271
- })
272
- case "undo":
273
- return this.runMutation(() => this.undoBatch(payload.batch_id))
274
- case "detach":
275
- if (typeof payload.path !== "string" || !payload.path) return { status: "not-found" }
276
- return this.runMutation(() => {
277
- const targetPath = path.resolve(payload.path)
278
- const kind = this.registry.duplicates.has(targetPath)
279
- ? "keep-separate"
280
- : "make-separate"
281
- return this.runFileAction({ kind, path: targetPath }, () => this.detach(targetPath))
282
- })
283
- default:
284
- return { error: "unknown action" }
285
439
  }
440
+ return String(value).toLowerCase() !== "false"
286
441
  }
287
- async refreshSources() {
288
- const home = path.resolve(this.kernel.homedir)
289
- const apiRoot = path.resolve(home, "api")
290
- const sources = [
291
- { id: "pinokio", kind: "pinokio", label: "Pinokio", root: home, parent_id: null },
292
- { id: "apps", kind: "virtual", label: "Apps", root: apiRoot, parent_id: "pinokio" },
293
- { id: "external", kind: "virtual", label: "External folders", root: null, parent_id: null }
294
- ]
295
- let storeDev = null
442
+
443
+ async init(options = {}) {
444
+ this.enabled = await this.isEnabled()
445
+ if (!this.enabled) return { enabled: false }
446
+ if (options.deferStorage) return { enabled: true }
447
+ return this.initializeStorage()
448
+ }
449
+
450
+ async initializeStorage() {
451
+ if (this.initialized) return { enabled: true, mode: this.mode }
452
+ await this.ensureDirectory(this.root)
453
+ this.registry = new Registry(this.root)
454
+ await this.registry.load()
455
+ await this.importLegacyLocationConfig()
456
+ await this.refreshAnchorStores()
457
+ await this.migrateLegacyAnchorStore()
458
+ this.mode = this.defaultAnchorStore() &&
459
+ this.defaultAnchorStore().mode === "copy"
460
+ ? "copy"
461
+ : "link"
462
+ await this.refreshSources()
463
+ this.sweeper = new Sweeper(this)
464
+ this.initialized = true
465
+ return { enabled: true, mode: this.mode }
466
+ }
467
+
468
+ async ensureInitialized() {
469
+ if (!this.enabled) return { enabled: false }
470
+ if (this.initialized) return { enabled: true, mode: this.mode }
471
+ if (!this.initializationPromise) {
472
+ this.initializationPromise = this.initializeStorage().finally(() => {
473
+ this.initializationPromise = null
474
+ })
475
+ }
476
+ return this.initializationPromise
477
+ }
478
+
479
+ async openWorkspace() {
480
+ if (!this.initialized) return this.ensureInitialized()
481
+ await this.refreshAnchorStores()
482
+ await this.refreshSources()
483
+ return { enabled: true, mode: this.mode }
484
+ }
485
+
486
+ async importLegacyLocationConfig() {
487
+ const config = this.readConfig()
488
+ const legacy = await this.registry.externalSources()
489
+ const locations = [...new Set([
490
+ ...config.locations.map((item) => path.resolve(item)),
491
+ ...legacy.map((item) => path.resolve(item))
492
+ ])]
493
+ if (locations.length !== config.locations.length ||
494
+ locations.some((item, index) =>
495
+ item !== path.resolve(config.locations[index] || ""))) {
496
+ this.writeConfig(Object.assign({}, config, { locations }))
497
+ }
498
+ }
499
+
500
+ async migrateLegacyAnchorStore() {
501
+ const legacyRoot = path.resolve(this.root, "sha256")
502
+ const legacyStat = await lstatIfPresent(legacyRoot)
503
+ if (!legacyStat) return false
504
+ if (!legacyStat.isDirectory() || legacyStat.isSymbolicLink()) {
505
+ throw unsafeStoragePath(legacyRoot)
506
+ }
507
+ if (!(await fs.promises.readdir(legacyRoot)).length) {
508
+ await fs.promises.rmdir(legacyRoot)
509
+ return true
510
+ }
511
+ const store = this.anchorStoreForDevice(legacyStat.dev)
512
+ if (!store) {
513
+ throw unavailableStorage(
514
+ "The legacy anchor store has no configured store on its filesystem.")
515
+ }
516
+ const targetRoot = path.resolve(store.root, "sha256")
517
+ if (samePath(legacyRoot, targetRoot)) return false
518
+
519
+ await this.ensureAnchorStore(store, legacyStat.dev)
520
+ const targetEntries = await fs.promises.readdir(targetRoot)
521
+ if (targetEntries.length) {
522
+ throw unavailableStorage(
523
+ "Both the legacy and configured anchor stores contain files.")
524
+ }
525
+ await fs.promises.rmdir(targetRoot)
296
526
  try {
297
- storeDev = (await fs.promises.stat(this.root)).dev
527
+ await fs.promises.rename(legacyRoot, targetRoot)
298
528
  } catch (error) {
299
- if (!isMissingError(error)) throw error
529
+ await this.ensureDirectory(targetRoot).catch(() => {})
530
+ throw error
300
531
  }
301
- const decorate = async (source) => {
302
- if (!source.root) return source
532
+ return true
533
+ }
534
+
535
+ async refreshAnchorStores() {
536
+ let config = this.readConfig()
537
+ const candidates = [
538
+ {
539
+ probe_path: path.resolve(this.kernel.homedir),
540
+ preferred_root: path.resolve(this.globalConfigRoot(), "vault"),
541
+ home: true
542
+ },
543
+ ...config.locations.map((location) => ({
544
+ probe_path: path.resolve(location),
545
+ home: false
546
+ }))
547
+ ]
548
+ const configured = []
549
+ for (const entry of config.anchor_stores) {
550
+ if (typeof entry.id !== "string" ||
551
+ !entry.id ||
552
+ typeof entry.root !== "string" ||
553
+ typeof entry.probe_path !== "string") continue
554
+ configured.push({
555
+ id: entry.id,
556
+ root: path.resolve(entry.root),
557
+ probe_path: path.resolve(entry.probe_path)
558
+ })
559
+ }
560
+
561
+ const observed = []
562
+ for (const store of configured) {
303
563
  try {
304
- const st = await fs.promises.stat(source.root)
305
- source.dev = st.dev
306
- source.available = st.isDirectory()
307
- source.shareable = source.available && storeDev !== null && st.dev === storeDev && this.mode === "link"
564
+ const stat = await fs.promises.stat(store.probe_path)
565
+ if (stat.isDirectory()) observed.push({ store, dev: stat.dev })
308
566
  } catch (error) {
309
567
  if (!isMissingError(error)) throw error
310
- source.available = false
311
- source.shareable = false
312
568
  }
313
- return source
569
+ }
570
+ let changed = configured.length !== config.anchor_stores.length
571
+ for (const candidate of candidates) {
572
+ let stat
573
+ try {
574
+ stat = await fs.promises.stat(candidate.probe_path)
575
+ } catch (error) {
576
+ if (isMissingError(error)) continue
577
+ throw error
578
+ }
579
+ if (!stat.isDirectory() ||
580
+ observed.some((item) => item.dev === stat.dev)) continue
581
+ const filesystemAnchor = candidate.home
582
+ ? null
583
+ : await this.filesystemAnchor(candidate.probe_path, stat.dev)
584
+ const store = {
585
+ id: crypto.randomUUID(),
586
+ root: candidate.home
587
+ ? candidate.preferred_root
588
+ : path.resolve(filesystemAnchor, ".pinokio", "vault"),
589
+ probe_path: candidate.probe_path
590
+ }
591
+ configured.push(store)
592
+ observed.push({ store, dev: stat.dev })
593
+ changed = true
594
+ }
595
+ if (changed) {
596
+ config = this.writeConfig(Object.assign({}, config, {
597
+ anchor_stores: configured
598
+ }))
314
599
  }
315
600
 
316
- const seenExternalRoots = new Set()
317
- const addExternalEntries = async (importRoot, idPrefix = "") => {
318
- let entries = []
601
+ const stores = []
602
+ for (const entry of config.anchor_stores) {
603
+ const store = {
604
+ id: entry.id,
605
+ root: path.resolve(entry.root),
606
+ probe_path: path.resolve(entry.probe_path),
607
+ available: false,
608
+ dev: null,
609
+ mode: null
610
+ }
319
611
  try {
320
- entries = await fs.promises.readdir(importRoot, { withFileTypes: true })
612
+ const probeStat = await fs.promises.stat(store.probe_path)
613
+ if (!probeStat.isDirectory()) {
614
+ stores.push(store)
615
+ continue
616
+ }
617
+ const rootStat = await lstatIfPresent(store.root)
618
+ if (rootStat &&
619
+ (!rootStat.isDirectory() || rootStat.isSymbolicLink())) {
620
+ throw unsafeStoragePath(store.root)
621
+ }
622
+ store.available = true
623
+ store.dev = probeStat.dev
624
+ store.mode = this.volumeModes.get(store.dev) || null
321
625
  } catch (error) {
322
626
  if (!isMissingError(error)) throw error
323
627
  }
324
- for (const entry of entries) {
325
- if (!entry.isSymbolicLink()) continue
326
- const mountPath = path.resolve(importRoot, entry.name)
327
- try {
328
- const root = await fs.promises.realpath(mountPath)
329
- const st = await fs.promises.stat(root)
330
- if (!st.isDirectory()) continue
331
- const canonical = path.resolve(root)
332
- if (seenExternalRoots.has(canonical)) continue
333
- seenExternalRoots.add(canonical)
334
- sources.push(await decorate({
335
- id: sourceId("external", `${idPrefix}${entry.name}`), kind: "external", label: entry.name,
336
- root: canonical, mount_path: mountPath, parent_id: "external"
337
- }))
338
- } catch (error) {
339
- if (!isMissingError(error)) throw error
340
- }
628
+ stores.push(store)
629
+ }
630
+ this._anchorStores = stores
631
+ this._anchorStoresById = new Map(
632
+ stores.map((store) => [store.id, store]))
633
+ this._anchorStoresByDevice = new Map()
634
+ for (const store of stores) {
635
+ if (!store.available || !Number.isFinite(store.dev)) continue
636
+ if (this._anchorStoresByDevice.has(store.dev)) {
637
+ store.available = false
638
+ store.error = "duplicate_device"
639
+ continue
341
640
  }
641
+ this._anchorStoresByDevice.set(store.dev, store)
342
642
  }
643
+ return stores
644
+ }
343
645
 
344
- let apiEntries = []
345
- try {
346
- apiEntries = await fs.promises.readdir(apiRoot, { withFileTypes: true })
347
- } catch (error) {
348
- if (!isMissingError(error)) throw error
646
+ async filesystemAnchor(directory, expectedDev = null) {
647
+ let current = path.resolve(directory)
648
+ const first = await fs.promises.stat(current)
649
+ const dev = expectedDev === null ? first.dev : expectedDev
650
+ if (!first.isDirectory() || first.dev !== dev) {
651
+ throw new Error("The configured location is not on the expected filesystem.")
349
652
  }
350
- for (const entry of apiEntries) {
351
- const mountPath = path.resolve(apiRoot, entry.name)
352
- if (entry.isDirectory()) {
353
- sources.push(await decorate({
354
- id: sourceId("app", entry.name), kind: "app", label: entry.name,
355
- app: entry.name, root: mountPath, parent_id: "apps"
356
- }))
653
+ while (true) {
654
+ const parent = path.dirname(current)
655
+ if (parent === current) return current
656
+ let parentStat
657
+ try {
658
+ parentStat = await fs.promises.stat(parent)
659
+ } catch (error) {
660
+ if (isMissingError(error)) return current
661
+ throw error
357
662
  }
663
+ if (parentStat.dev !== dev) return current
664
+ current = parent
358
665
  }
359
- await addExternalEntries(apiRoot)
360
- if (await this.directoryIfSafe(this.sourceRoot)) {
361
- await addExternalEntries(this.sourceRoot, "vault:")
362
- }
666
+ }
363
667
 
364
- let homeEntries = []
365
- try {
366
- homeEntries = await fs.promises.readdir(home, { withFileTypes: true })
367
- } catch (error) {
368
- if (!isMissingError(error)) throw error
369
- }
370
- for (const entry of homeEntries) {
371
- if (!entry.isDirectory() || entry.isSymbolicLink() || entry.name === "api" || entry.name === "vault") continue
372
- sources.push(await decorate({
373
- id: sourceId("folder", entry.name), kind: "folder", label: entry.name,
374
- root: path.resolve(home, entry.name), parent_id: "pinokio"
375
- }))
376
- }
377
- const homeSource = sources.find((source) => source.id === "pinokio")
378
- if (homeSource) await decorate(homeSource)
379
- this._sources = sources
380
- this._sourceBases = new Map()
381
- for (const source of sources) {
382
- if (!source.root || source.kind === "virtual") continue
383
- for (const base of [source.root, source.mount_path]) {
384
- if (!base) continue
385
- const resolved = path.resolve(base)
386
- const key = process.platform === "win32" ? resolved.toLowerCase() : resolved
387
- if (!this._sourceBases.has(key)) this._sourceBases.set(key, [])
388
- this._sourceBases.get(key).push(source)
389
- }
390
- }
391
- return sources
668
+ anchorStoreForDevice(dev) {
669
+ return this._anchorStoresByDevice.get(dev) || null
392
670
  }
393
- sources() {
394
- return this._sources
671
+
672
+ anchorStores() {
673
+ return this._anchorStores
395
674
  }
396
- async addExternalSource(folderPath) {
397
- if (!this.enabled) throw new Error("Save space is disabled.")
398
- if (typeof folderPath !== "string" || !path.isAbsolute(folderPath.trim())) {
399
- throw new Error("Choose a valid folder.")
400
- }
401
- const requested = folderPath.trim()
402
- let canonical
403
- let stats
404
- try {
405
- canonical = path.resolve(await fs.promises.realpath(requested))
406
- stats = await fs.promises.stat(canonical)
407
- } catch (error) {
408
- throw new Error("That folder is no longer available.")
409
- }
410
- if (!stats.isDirectory()) throw new Error("Choose a folder, not a file.")
411
675
 
412
- const home = path.resolve(this.kernel.homedir)
413
- const canonicalHome = path.resolve(await fs.promises.realpath(home))
414
- if (isPathWithin(canonicalHome, canonical) || isPathWithin(canonical, canonicalHome)) {
415
- throw new Error("That folder is already inside Pinokio and is included in scans.")
416
- }
676
+ anchorStoreForPath(filePath) {
677
+ return this._anchorStores.find((store) =>
678
+ isPathWithin(path.resolve(store.root, "sha256"), filePath)) || null
679
+ }
417
680
 
418
- await this.refreshSources()
419
- const existing = this._sources.find((source) => source.kind === "external" && source.root && samePath(source.root, canonical))
420
- if (existing) {
421
- return { created: false, source: existing }
422
- }
423
-
424
- const importRoot = this.sourceRoot
425
- await this.ensureDirectory(importRoot)
426
- const baseLabel = path.basename(canonical) || "external-folder"
427
- let mountPath = null
428
- for (let index = 1; index < 10000; index++) {
429
- const label = index === 1 ? baseLabel : `${baseLabel}-${index}`
430
- const candidate = path.resolve(importRoot, label)
681
+ storageRoots() {
682
+ const roots = [
683
+ this.root,
684
+ ...this._anchorStores.map((store) => store.root)
685
+ ].map((item) => path.resolve(item))
686
+ for (const root of [...roots]) {
431
687
  try {
432
- await fs.promises.symlink(canonical, candidate, this.kernel.platform === "win32" ? "junction" : "dir")
433
- mountPath = candidate
434
- break
688
+ roots.push(path.resolve(fs.realpathSync(root)))
435
689
  } catch (error) {
436
- if (error && error.code === "EEXIST") continue
437
- if (error && (error.code === "EACCES" || error.code === "EPERM")) {
438
- throw new Error("Pinokio does not have permission to add that folder.")
439
- }
440
- throw new Error("Pinokio could not add that folder.")
690
+ if (!isMissingError(error)) throw error
441
691
  }
442
692
  }
443
- if (!mountPath) throw new Error("Pinokio could not create a unique name for that folder.")
693
+ return [...new Set(roots)]
694
+ }
444
695
 
445
- try {
446
- await this.refreshSources()
447
- const source = this._sources.find((item) => item.kind === "external" && item.mount_path && samePath(item.mount_path, mountPath))
448
- if (!source) throw new Error("The folder could not be added. Restart Pinokio and try again.")
449
- return { created: true, source }
450
- } catch (error) {
451
- // Adding a source is transactional. Roll back only the exact directory
452
- // link created above; preserve any path an external writer replaced.
453
- try {
454
- const [mounted, target] = await Promise.all([
455
- fs.promises.lstat(mountPath),
456
- fs.promises.realpath(mountPath)
457
- ])
458
- if (mounted.isSymbolicLink() && samePath(target, canonical)) {
459
- await fs.promises.unlink(mountPath)
460
- }
461
- } catch {
462
- // The original error remains authoritative; an unverified path is
463
- // deliberately preserved rather than deleted.
464
- }
465
- throw error
466
- }
467
- }
468
- sourceForPath(filePath, preferredId) {
469
- let cursor = path.resolve(filePath)
470
- while (true) {
471
- const key = process.platform === "win32" ? cursor.toLowerCase() : cursor
472
- const matches = this._sourceBases.get(key)
473
- if (matches && matches.length) {
474
- return matches.find((source) => source.id === preferredId) || matches[0]
475
- }
476
- const parent = path.dirname(cursor)
477
- if (parent === cursor) return null
478
- cursor = parent
479
- }
480
- }
481
- async canonicalPathIsWithinSource(filePath, source, options = {}) {
482
- if (!source || !source.root) return false
483
- try {
484
- const [canonicalRoot, canonicalFile] = await Promise.all([
485
- fs.promises.realpath(source.root),
486
- fs.promises.realpath(filePath)
487
- ])
488
- return isPathWithin(canonicalRoot, canonicalFile)
489
- } catch (error) {
490
- if (options.strictErrors && !isMissingError(error)) throw error
491
- return false
492
- }
493
- }
494
- locationForPath(filePath, preferredId) {
495
- const source = this.sourceForPath(filePath, preferredId)
496
- if (!source) return { source_id: null, relative_path: path.basename(filePath) }
497
- const absolute = path.resolve(filePath)
498
- let base = source.root
499
- if (source.mount_path && isPathWithin(source.mount_path, absolute)) base = source.mount_path
500
- const relative = path.relative(base, absolute).split(path.sep).join("/") || path.basename(absolute)
501
- return {
502
- source_id: source.id,
503
- source_kind: source.kind,
504
- source_label: source.label,
505
- relative_path: relative
506
- }
507
- }
508
- async recordEvent(event) {
509
- let entry = event
510
- if (event.path) {
511
- const location = this.locationForPath(event.path, event.source_id)
512
- const context = location.source_id ? location : { relative_path: event.path }
513
- entry = Object.assign({}, event, context)
514
- }
515
- try {
516
- await this.registry.appendEvent(entry)
517
- return true
518
- } catch (error) {
519
- return false
520
- }
521
- }
522
- scanSource(scopeId) {
523
- if (!scopeId) return null
524
- return this._sources.find((source) =>
525
- source.id === scopeId && source.kind !== "virtual" && source.available && source.root) || null
526
- }
527
- scanRoots(scopeId = null) {
528
- if (scopeId) {
529
- const source = this.scanSource(scopeId)
530
- return source ? [{ root: path.resolve(source.root), source_id: source.id }] : []
531
- }
532
- const home = path.resolve(this.kernel.homedir)
533
- const candidates = [{ root: home, source_id: "pinokio" }]
534
- for (const source of this._sources) {
535
- if (source.kind !== "external" || !source.available || !source.root) continue
536
- const canonical = path.resolve(source.root)
537
- if (isPathWithin(home, canonical) || isPathWithin(canonical, home)) continue
538
- candidates.push({ root: canonical, source_id: source.id })
539
- }
540
- candidates.sort((left, right) => left.root.length - right.root.length)
541
- const roots = []
542
- for (const candidate of candidates) {
543
- if (roots.some((existing) => isPathWithin(existing.root, candidate.root))) continue
544
- roots.push(candidate)
545
- }
546
- return roots
547
- }
548
- reconcileConfiguredSources() {
549
- if (!this.registry) return
550
- const configured = (filePath, sourceId) => !!this.sourceForPath(filePath, sourceId)
551
- for (const [filePath, entry] of [...this.registry.links]) {
552
- if (!configured(filePath, entry.source_id)) this.registry.untrack(filePath)
553
- }
554
- for (const [filePath, entry] of [...this.registry.duplicates]) {
555
- if (!configured(filePath, entry.source_id)) this.registry.untrack(filePath)
556
- }
557
- for (const [filePath, entry] of [...this.registry.scanIndex]) {
558
- if (!configured(filePath, entry.source_id)) this.registry.untrack(filePath)
559
- }
560
- }
561
- // Kill switch: system ENVIRONMENT variable PINOKIO_VAULT, default unset =
562
- // enabled. Read directly (single key) to avoid loading the environment
563
- // module chain before the kernel is fully up.
564
- async isEnabled() {
565
- let value = process.env.PINOKIO_VAULT
566
- if (value === undefined && this.kernel.homedir) {
567
- try {
568
- const raw = await fs.promises.readFile(path.resolve(this.kernel.homedir, "ENVIRONMENT"), "utf8")
569
- for (const line of raw.split("\n")) {
570
- const m = line.match(/^\s*PINOKIO_VAULT\s*=\s*(.*)\s*$/)
571
- if (m) value = m[1].trim()
572
- }
573
- } catch (error) {
574
- if (!isMissingError(error)) throw error
575
- }
576
- }
577
- return String(value).toLowerCase() !== "false"
578
- }
579
- // Disabled ⇒ do nothing observable: no directory, no registry, no probes.
580
- async init(options = {}) {
581
- this.enabled = await this.isEnabled()
582
- if (!this.enabled) return { enabled: false }
583
- if (options.existingOnly) {
584
- try {
585
- await fs.promises.stat(this.root)
586
- } catch (error) {
587
- if (error && error.code === "ENOENT") return { enabled: true, fresh: true }
588
- throw error
589
- }
590
- }
591
- return this.initializeStorage()
592
- }
593
- async initializeStorage() {
594
- if (this.initialized) return { enabled: true, mode: this.mode }
595
- await this.ensureDirectory(this.root)
596
- await this.ensureDirectory(this.blobRoot)
597
- this.registry = new Registry(this.root)
598
- this.mode = await this.probe(this.root)
599
- await this.refreshSources()
600
- const loaded = await this.registry.load()
601
- const missingWithBlobs = !loaded.existed && await this.hasStoredBlobs()
602
- if (loaded.corrupt || missingWithBlobs) {
603
- await this.rebuild(undefined, { preserveState: false })
604
- }
605
- this.sweeper = new Sweeper(this)
606
- this.initialized = true
607
- return {
608
- enabled: true,
609
- mode: this.mode,
610
- corrupt_recovered: !!loaded.corrupt,
611
- missing_recovered: missingWithBlobs
612
- }
696
+ isStorageRoot(directory) {
697
+ const target = path.resolve(directory)
698
+ return this.storageRoots().some((root) => samePath(root, target))
613
699
  }
614
- async ensureInitialized() {
615
- if (!this.enabled) return { enabled: false }
616
- if (this.initialized && !this.verificationPending) return { enabled: true, mode: this.mode }
617
- if (!this.initializationPromise) {
618
- this.initializationPromise = (async () => {
619
- const result = this.initialized
620
- ? { enabled: true, mode: this.mode }
621
- : await this.initializeStorage()
622
- await this.runExclusive(async () => {
623
- await this.verify()
624
- await this.registry.flush()
625
- })
626
- this.verificationPending = false
627
- return result
628
- })().catch((error) => {
629
- if (this.initialized) this.verificationPending = true
630
- throw error
631
- }).finally(() => {
632
- this.initializationPromise = null
633
- })
634
- }
635
- return this.initializationPromise
636
- }
637
- async hasStoredBlobs() {
638
- let shards = []
639
- try {
640
- shards = await fs.promises.readdir(this.blobRoot, { withFileTypes: true })
641
- } catch (error) {
642
- if (isMissingError(error)) return false
643
- throw error
644
- }
645
- for (const shard of shards) {
646
- if (!shard.isDirectory() || !/^[0-9a-f]{2}$/.test(shard.name)) continue
647
- const shardPath = path.resolve(this.blobRoot, shard.name)
648
- const shardStat = await this.directoryIfSafe(shardPath)
649
- if (!shardStat) continue
650
- let names = []
651
- try {
652
- names = await fs.promises.readdir(shardPath)
653
- } catch (error) {
654
- if (isMissingError(error)) continue
655
- throw error
656
- }
657
- if (names.some((name) => SHA256_RE.test(name) && name.startsWith(shard.name))) return true
658
- }
659
- return false
660
- }
661
- // Capability probe: temp file + node:fs.link + nlink check. Once per volume.
662
- async probe(dir) {
663
- const dev = (await fs.promises.stat(dir)).dev
700
+
701
+ async probe(directory) {
702
+ const dev = (await fs.promises.stat(directory)).dev
664
703
  if (this.volumeModes.has(dev)) return this.volumeModes.get(dev)
665
- const a = path.resolve(dir, `.pinokio-probe-${crypto.randomBytes(6).toString("hex")}`)
666
- const b = a + "-link"
704
+ const first = path.resolve(
705
+ directory, `.pinokio-probe-${crypto.randomBytes(6).toString("hex")}`)
706
+ const second = `${first}-link`
667
707
  let mode = "copy"
668
- let aStat = null
669
- let bStat = null
708
+ let firstStat = null
709
+ let secondStat = null
670
710
  let failure = null
671
711
  try {
672
- await fs.promises.writeFile(a, "probe", { flag: "wx" })
673
- aStat = await fs.promises.lstat(a)
674
- await fs.promises.link(a, b)
675
- bStat = await fs.promises.lstat(b)
676
- if (sameIdentity(bStat, aStat) && bStat.nlink === 2) mode = "link"
712
+ await fs.promises.writeFile(first, "probe", { flag: "wx" })
713
+ firstStat = await fs.promises.lstat(first)
714
+ await fs.promises.link(first, second)
715
+ secondStat = await fs.promises.lstat(second)
716
+ if (sameIdentity(firstStat, secondStat) && secondStat.nlink === 2) {
717
+ mode = "link"
718
+ }
677
719
  } catch (error) {
678
720
  if (!NO_LINK_CODES.has(error && error.code)) failure = error
679
721
  } finally {
680
722
  try {
681
- await unlinkIfSame(b, bStat)
682
- await unlinkIfSame(a, aStat)
723
+ await unlinkIfSame(second, secondStat)
724
+ await unlinkIfSame(first, firstStat)
683
725
  } catch (error) {
684
726
  if (!failure) failure = error
685
727
  }
@@ -688,6 +730,7 @@ class Vault {
688
730
  this.volumeModes.set(dev, mode)
689
731
  return mode
690
732
  }
733
+
691
734
  failHashWorker(worker, error, terminate = false) {
692
735
  if (this.worker === worker) {
693
736
  if (this.workerIdleTimer) clearTimeout(this.workerIdleTimer)
@@ -702,6 +745,7 @@ class Vault {
702
745
  }
703
746
  if (terminate) worker.terminate().catch(() => {})
704
747
  }
748
+
705
749
  async hashFile(filePath, options = {}) {
706
750
  if (this.workerIdleTimer) {
707
751
  clearTimeout(this.workerIdleTimer)
@@ -711,7 +755,9 @@ class Vault {
711
755
  const worker = new Worker(path.resolve(__dirname, "hash_worker.js"))
712
756
  this.worker = worker
713
757
  worker.unref()
714
- worker.on("message", ({ id, hash, size, bytes_read: bytesRead, error, code }) => {
758
+ worker.on("message", ({
759
+ id, hash, size, bytes_read: bytesRead, error, code
760
+ }) => {
715
761
  const job = this.workerJobs.get(id)
716
762
  if (!job || job.worker !== worker) return
717
763
  if (Number.isFinite(bytesRead)) {
@@ -729,8 +775,6 @@ class Vault {
729
775
  job.reportProgress(size)
730
776
  job.resolve({ hash, size })
731
777
  }
732
- // Keep the worker warm across the scan queue. Starting one worker per
733
- // file costs ~30ms and adds up quickly on a first scan.
734
778
  if (this.workerJobs.size === 0 && this.worker === worker) {
735
779
  this.workerIdleTimer = setTimeout(() => {
736
780
  this.workerIdleTimer = null
@@ -744,37 +788,49 @@ class Vault {
744
788
  })
745
789
  worker.on("error", (error) => {
746
790
  if (this.worker !== worker) return
747
- this.failHashWorker(worker, error)
791
+ const failure = new Error(error && error.message
792
+ ? error.message
793
+ : "Hash worker failed.")
794
+ failure.code = "EVAULTHASHWORKER"
795
+ this.failHashWorker(worker, failure)
748
796
  })
749
797
  worker.on("exit", (code) => {
750
- if (this.worker !== worker) return
751
- this.failHashWorker(worker, new Error(`hash worker exited with code ${code}`))
798
+ if (this.worker === worker) {
799
+ const failure = new Error(`hash worker exited with code ${code}`)
800
+ failure.code = "EVAULTHASHWORKER"
801
+ this.failHashWorker(worker, failure)
802
+ }
752
803
  })
753
804
  }
754
805
  const worker = this.worker
755
806
  const id = ++this.workerSeq
756
807
  return new Promise((resolve, reject) => {
757
- const onProgress = typeof options.onProgress === "function" ? options.onProgress : null
808
+ const onProgress = typeof options.onProgress === "function"
809
+ ? options.onProgress
810
+ : null
758
811
  const job = {
759
812
  worker,
760
813
  resolve,
761
814
  reject,
762
815
  reportProgress: (bytes) => {
763
816
  if (!onProgress) return
764
- // Progress is observational and must never fail a content hash.
765
- try { onProgress(bytes) } catch (_) {}
817
+ try {
818
+ onProgress(bytes)
819
+ } catch (error) {}
766
820
  },
767
821
  inactivityTimer: null,
768
822
  resetInactivity: null
769
823
  }
770
824
  job.resetInactivity = () => {
771
825
  clearTimeout(job.inactivityTimer)
772
- const inactivityMs = Math.max(1, Number(this.hashInactivityMs) || HASH_INACTIVITY_MS)
826
+ const timeout = Math.max(
827
+ 1, Number(this.hashInactivityMs) || HASH_INACTIVITY_MS)
773
828
  job.inactivityTimer = setTimeout(() => {
774
- const failure = new Error(`Timed out while reading ${path.basename(filePath)}`)
829
+ const failure = new Error(
830
+ `Timed out while reading ${path.basename(filePath)}`)
775
831
  failure.code = "ETIMEDOUT"
776
832
  this.failHashWorker(worker, failure, true)
777
- }, inactivityMs)
833
+ }, timeout)
778
834
  if (job.inactivityTimer.unref) job.inactivityTimer.unref()
779
835
  }
780
836
  this.workerJobs.set(id, job)
@@ -782,1315 +838,1893 @@ class Vault {
782
838
  try {
783
839
  worker.postMessage({ id, filePath })
784
840
  } catch (error) {
785
- this.failHashWorker(worker, error, true)
841
+ const failure = new Error(error && error.message
842
+ ? error.message
843
+ : "Hash worker could not accept work.")
844
+ failure.code = "EVAULTHASHWORKER"
845
+ this.failHashWorker(worker, failure, true)
786
846
  }
787
847
  })
788
848
  }
789
- async refreshLinkSnapshots(hash, dev, ino) {
790
- if (!this.registry) return
791
- const key = this.registry.inoKey(dev, ino)
792
- const paths = [...(this.registry.pathsByIno.get(key) || [])]
793
- for (const linkPath of paths) {
794
- const entry = this.registry.links.get(linkPath)
795
- if (!entry || entry.hash !== hash) continue
849
+
850
+ async refreshSources() {
851
+ const home = path.resolve(this.kernel.homedir)
852
+ const apiRoot = path.resolve(home, "api")
853
+ const sources = [
854
+ {
855
+ id: "pinokio", kind: "pinokio", label: "Pinokio",
856
+ root: home, parent_id: null
857
+ },
858
+ {
859
+ id: "apps", kind: "virtual", label: "Apps",
860
+ root: apiRoot, parent_id: "pinokio"
861
+ },
862
+ {
863
+ id: "external", kind: "virtual", label: "External folders",
864
+ root: null, parent_id: null
865
+ }
866
+ ]
867
+ const decorate = async (source) => {
868
+ if (!source.root) return source
796
869
  try {
797
- const source = this.sourceForPath(linkPath, entry.source_id)
798
- if (!source || !await this.canonicalPathIsWithinSource(linkPath, source)) continue
799
- const st = await fs.promises.lstat(linkPath)
800
- if (st.dev !== dev || st.ino !== ino) continue
801
- this.registry.setScanEntry(linkPath, Object.assign(fileSnapshot(st), {
802
- hash,
803
- source_id: entry.source_id || null
804
- }))
805
- } catch (error) {}
806
- }
807
- this.registry.schedulePersist()
808
- }
809
- async verifyStoreContent(hash, storePath, storeStat) {
810
- if (!storeStat || !storeStat.isFile()) return { valid: false }
811
- const key = this.registry.inoKey(storeStat.dev, storeStat.ino)
812
- for (const linkPath of this.registry.pathsByIno.get(key) || []) {
813
- const entry = this.registry.links.get(linkPath)
814
- if (!entry) continue
815
- if (entry.hash !== hash || entry.mode !== "link" ||
816
- entry.dev !== storeStat.dev || entry.ino !== storeStat.ino) continue
817
- const expected = this.registry.scanIndex.get(linkPath)
818
- if (expected && expected.hash === hash && sameSnapshot(expected, storeStat)) {
819
- return { valid: true, snapshot: fileSnapshot(storeStat) }
870
+ const [stat, realRoot] = await Promise.all([
871
+ fs.promises.lstat(source.root),
872
+ fs.promises.realpath(source.root)
873
+ ])
874
+ source.canonical_root = path.resolve(realRoot)
875
+ source.dev = stat.dev
876
+ source.available = stat.isDirectory() &&
877
+ !stat.isSymbolicLink() &&
878
+ (source.kind !== "external" || samePath(realRoot, source.root))
879
+ const store = this.anchorStoreForDevice(stat.dev)
880
+ source.shareable = source.available &&
881
+ !!store &&
882
+ store.mode !== "copy"
883
+ source.store_id = store ? store.id : null
884
+ } catch (error) {
885
+ if (!isMissingError(error)) throw error
886
+ source.available = false
887
+ source.shareable = false
820
888
  }
889
+ return source
821
890
  }
822
891
 
823
- const before = fileSnapshot(storeStat)
824
- let hashed
892
+ let apps = []
825
893
  try {
826
- hashed = await this.hashFile(storePath)
894
+ apps = await fs.promises.readdir(apiRoot, { withFileTypes: true })
827
895
  } catch (error) {
828
- return { valid: false }
896
+ if (!isMissingError(error)) throw error
897
+ }
898
+ for (const entry of apps) {
899
+ if (!entry.isDirectory() || entry.isSymbolicLink()) continue
900
+ sources.push(await decorate({
901
+ id: sourceId("app", entry.name),
902
+ kind: "app",
903
+ label: entry.name,
904
+ app: entry.name,
905
+ root: path.resolve(apiRoot, entry.name),
906
+ parent_id: "apps"
907
+ }))
908
+ }
909
+
910
+ for (const configuredPath of this.configuredLocations()) {
911
+ sources.push(await decorate({
912
+ id: sourceId("external", configuredPath),
913
+ kind: "external",
914
+ label: path.basename(configuredPath) || configuredPath,
915
+ root: configuredPath,
916
+ parent_id: "external",
917
+ configured: true
918
+ }))
829
919
  }
830
- let after
920
+
921
+ let homeEntries = []
831
922
  try {
832
- after = await this.storeStatIfPresent(storePath)
923
+ homeEntries = await fs.promises.readdir(home, { withFileTypes: true })
833
924
  } catch (error) {
834
- return { valid: false }
925
+ if (!isMissingError(error)) throw error
835
926
  }
836
- if (!sameSnapshot(before, after) || hashed.hash !== hash || hashed.size !== after.size) {
837
- return { valid: false }
927
+ for (const entry of homeEntries) {
928
+ if (!entry.isDirectory() || entry.isSymbolicLink() ||
929
+ entry.name === "api" || entry.name === "vault") continue
930
+ sources.push(await decorate({
931
+ id: sourceId("folder", entry.name),
932
+ kind: "folder",
933
+ label: entry.name,
934
+ root: path.resolve(home, entry.name),
935
+ parent_id: "pinokio"
936
+ }))
838
937
  }
839
- await this.refreshLinkSnapshots(hash, after.dev, after.ino)
840
- return { valid: true, snapshot: fileSnapshot(after) }
841
- }
842
- // adopt: give an existing file a store name. Metadata-only, never copies.
843
- async adopt(filePath, hash, meta = {}) {
844
- if (!this.enabled) return { status: "disabled" }
845
- const storePath = this.storePathFor(hash)
846
- const st = await fs.promises.lstat(filePath)
847
- if (!st.isFile()) return { status: "stale" }
848
- if (meta.expected && !sameSnapshot(meta.expected, st)) return { status: "stale" }
849
- const before = fileSnapshot(st)
850
- const linkEntry = {
851
- hash,
852
- app: meta.app || null,
853
- source_id: meta.source_id || null,
854
- dev: st.dev,
855
- ino: st.ino,
856
- mode: "link"
857
- }
858
- const storeStat = await this.storeStatIfPresent(storePath, {
859
- createParent: this.mode === "link"
860
- })
861
- if (storeStat) {
862
- if (storeStat.dev === st.dev && storeStat.ino === st.ino) {
863
- const current = await lstatIfPresent(filePath)
864
- if (!sameSnapshot(before, current)) return { status: "stale" }
865
- this.registry.addBlob(hash, { size: st.size, source_urls: meta.source_urls || [] })
866
- this.registry.addLink(filePath, linkEntry)
867
- await this.refreshLinkSnapshots(hash, st.dev, st.ino)
868
- return { status: "already" }
869
- }
870
- const current = await lstatIfPresent(filePath)
871
- if (!sameSnapshot(before, current)) return { status: "stale" }
872
- return { status: "duplicate", storePath }
873
- }
874
- const registerCopy = async () => {
875
- const current = await lstatIfPresent(filePath)
876
- if (!sameSnapshot(before, current)) return { status: "stale" }
877
- linkEntry.mode = "copy"
878
- this.registry.addBlob(hash, { size: st.size, source_urls: meta.source_urls || [] })
879
- this.registry.addLink(filePath, linkEntry)
880
- await this.refreshLinkSnapshots(hash, st.dev, st.ino)
881
- await this.recordEvent({
882
- kind: "adopt", hash, path: filePath, app: meta.app || null,
883
- source_id: meta.source_id || null, bytes_saved: 0, mode: "copy"
884
- })
885
- return { status: "copy-mode" }
938
+
939
+ const pinokio = sources.find((source) => source.id === "pinokio")
940
+ if (pinokio) await decorate(pinokio)
941
+ this._sources = sources
942
+ this._sourcesById = new Map(
943
+ sources.map((source) => [source.id, source])
944
+ )
945
+ this._sourceBases = new Map()
946
+ for (const source of sources) {
947
+ if (!source.root || source.kind === "virtual") continue
948
+ const bases = new Set([
949
+ path.resolve(source.root),
950
+ source.canonical_root && path.resolve(source.canonical_root)
951
+ ].filter(Boolean))
952
+ for (const resolved of bases) {
953
+ const key = process.platform === "win32"
954
+ ? resolved.toLowerCase()
955
+ : resolved
956
+ if (!this._sourceBases.has(key)) this._sourceBases.set(key, [])
957
+ this._sourceBases.get(key).push(source)
958
+ }
886
959
  }
887
- if (this.mode === "copy") return registerCopy()
888
- try {
889
- await fs.promises.link(filePath, storePath)
890
- } catch (e) {
891
- if (NO_LINK_CODES.has(e.code)) {
892
- return registerCopy()
960
+ return sources
961
+ }
962
+
963
+ sources() {
964
+ return this._sources
965
+ }
966
+
967
+ sourceForPath(filePath, preferredId = null) {
968
+ let cursor = path.resolve(filePath)
969
+ while (true) {
970
+ const key = process.platform === "win32"
971
+ ? cursor.toLowerCase()
972
+ : cursor
973
+ const matches = this._sourceBases.get(key)
974
+ if (matches && matches.length) {
975
+ return matches.find((source) => source.id === preferredId) || matches[0]
893
976
  }
894
- throw e
977
+ const parent = path.dirname(cursor)
978
+ if (parent === cursor) return null
979
+ cursor = parent
895
980
  }
896
- const [current, currentStore] = await Promise.all([
897
- lstatIfPresent(filePath),
898
- this.storeStatIfPresent(storePath)
899
- ])
900
- if (!sameContentState(before, current) || !currentStore ||
901
- currentStore.dev !== st.dev || currentStore.ino !== st.ino) {
902
- // Remove only a store name that still points either to the inode we
903
- // intended to link or to the current source inode that won the race.
904
- // An unrelated replacement at the reserved store path is preserved.
905
- const ownsStoreName = currentStore && (
906
- (currentStore.dev === st.dev && currentStore.ino === st.ino) ||
907
- (current && currentStore.dev === current.dev && currentStore.ino === current.ino)
908
- )
909
- if (ownsStoreName) {
910
- try {
911
- await fs.promises.unlink(storePath)
912
- } catch (error) {
913
- if (!isMissingError(error)) throw error
981
+ }
982
+
983
+ sourceIsWithinScope(source, scopeId) {
984
+ if (!scopeId) return true
985
+ const seen = new Set()
986
+ let current = source
987
+ while (current && !seen.has(current.id)) {
988
+ if (current.id === scopeId) return true
989
+ seen.add(current.id)
990
+ current = this._sourcesById.get(current.parent_id)
991
+ }
992
+ return false
993
+ }
994
+
995
+ scanSource(scopeId) {
996
+ if (!scopeId) return null
997
+ const source = this._sourcesById.get(scopeId)
998
+ return source &&
999
+ source.kind !== "virtual" &&
1000
+ source.available &&
1001
+ source.root
1002
+ ? source
1003
+ : null
1004
+ }
1005
+
1006
+ scanRoots(scopeId = null) {
1007
+ if (scopeId) {
1008
+ const source = this.scanSource(scopeId)
1009
+ return source
1010
+ ? [{ root: path.resolve(source.root), source_id: source.id }]
1011
+ : []
1012
+ }
1013
+ const pinokio = this._sourcesById.get("pinokio")
1014
+ const candidates = [
1015
+ {
1016
+ root: path.resolve(pinokio && pinokio.root || this.kernel.homedir),
1017
+ canonical_root: path.resolve(
1018
+ pinokio && pinokio.canonical_root ||
1019
+ pinokio && pinokio.root ||
1020
+ this.kernel.homedir
1021
+ ),
1022
+ source_id: "pinokio"
1023
+ },
1024
+ ...this._sources
1025
+ .filter((source) =>
1026
+ source.kind === "external" && source.available && source.root)
1027
+ .map((source) => ({
1028
+ root: path.resolve(source.root),
1029
+ canonical_root: path.resolve(source.canonical_root || source.root),
1030
+ source_id: source.id
1031
+ }))
1032
+ ].sort((left, right) =>
1033
+ left.canonical_root.length - right.canonical_root.length ||
1034
+ left.canonical_root.localeCompare(right.canonical_root))
1035
+ const roots = []
1036
+ for (const candidate of candidates) {
1037
+ const canonical = candidate.canonical_root
1038
+ if (roots.some((root) =>
1039
+ isPathWithin(root.canonical_root, canonical))) continue
1040
+ for (let index = roots.length - 1; index >= 0; index--) {
1041
+ if (isPathWithin(canonical, roots[index].canonical_root)) {
1042
+ roots.splice(index, 1)
914
1043
  }
915
1044
  }
916
- return { status: "stale" }
1045
+ roots.push(candidate)
917
1046
  }
918
- this.registry.addBlob(hash, { size: st.size, source_urls: meta.source_urls || [] })
919
- this.registry.addLink(filePath, linkEntry)
920
- await this.refreshLinkSnapshots(hash, st.dev, st.ino)
921
- await this.recordEvent({ kind: "adopt", hash, path: filePath, app: meta.app || null, source_id: meta.source_id || null, bytes_saved: 0 })
922
- return { status: "adopted" }
923
- }
924
- // convert: replace a byte-identical duplicate with a link to an existing
925
- // blob. Atomic link+rename; target is never missing at any instant.
926
- async convert(targetPath, hash, meta = {}) {
927
- if (!this.enabled) return { status: "disabled" }
928
- const storePath = this.storePathFor(hash)
929
- let storeStat
1047
+ return roots.map(({ root, source_id: sourceIdValue }) => ({
1048
+ root,
1049
+ source_id: sourceIdValue
1050
+ }))
1051
+ }
1052
+
1053
+ scopeSourceIds(scopeId = null, locationId = null) {
1054
+ return this._sources
1055
+ .filter((source) => source.kind !== "virtual")
1056
+ .filter((source) => !scopeId || this.sourceIsWithinScope(source, scopeId))
1057
+ .filter((source) =>
1058
+ !locationId || this.sourceIsWithinScope(source, locationId))
1059
+ .map((source) => source.id)
1060
+ }
1061
+
1062
+ async canonicalPathIsWithinSource(filePath, source) {
1063
+ if (!source || !source.root) return false
930
1064
  try {
931
- storeStat = await this.storeStatIfPresent(storePath)
932
- } catch (e) {
933
- return { status: "no-blob" }
934
- }
935
- if (!storeStat) return { status: "no-blob" }
936
- let targetStat = await fs.promises.lstat(targetPath)
937
- if (!storeStat.isFile() || !targetStat.isFile()) return { status: "stale" }
938
- if (meta.expected && !sameSnapshot(meta.expected, targetStat)) {
939
- // Windows metadata operations can change ctime without changing the
940
- // file's identity or bytes. Recover only that isolated mismatch by
941
- // hashing the target again, then retain the exact pre-rename check below.
942
- if (!sameContentState(meta.expected, targetStat)) return { status: "stale" }
943
- const before = fileSnapshot(targetStat)
944
- let verifiedTarget
945
- try {
946
- verifiedTarget = await this.hashFile(targetPath)
947
- } catch (error) {
948
- return { status: "stale" }
949
- }
950
- const after = await lstatIfPresent(targetPath)
951
- if (!sameSnapshot(before, after) ||
952
- verifiedTarget.hash !== hash || verifiedTarget.size !== after.size) {
953
- return { status: "stale" }
1065
+ const rootStat = await fs.promises.lstat(source.root)
1066
+ if (!rootStat.isDirectory() || rootStat.isSymbolicLink()) return false
1067
+ const [root, target] = await Promise.all([
1068
+ fs.promises.realpath(source.root),
1069
+ fs.promises.realpath(filePath)
1070
+ ])
1071
+ if (source.kind === "external" && !samePath(root, source.root)) {
1072
+ return false
954
1073
  }
955
- targetStat = after
1074
+ return isPathWithin(root, target)
1075
+ } catch (error) {
1076
+ if (!isMissingError(error)) throw error
1077
+ return false
956
1078
  }
957
- if (storeStat.dev !== targetStat.dev) {
958
- return { status: "unavailable", code: "EXDEV" }
1079
+ }
1080
+
1081
+ async revealFile(scopeId, filePath) {
1082
+ if (typeof filePath !== "string" ||
1083
+ !path.isAbsolute(filePath)) {
1084
+ return { error: "Choose a valid file." }
959
1085
  }
960
- if (storeStat.dev === targetStat.dev && storeStat.ino === targetStat.ino) {
961
- this.registry.addLink(targetPath, { hash, app: meta.app || null, source_id: meta.source_id || null, dev: storeStat.dev, ino: storeStat.ino, mode: "link" })
962
- await this.refreshLinkSnapshots(hash, storeStat.dev, storeStat.ino)
963
- return { status: "already" }
1086
+ if (scopeId && !this._sourcesById.has(scopeId)) {
1087
+ return { error: "That location is no longer available." }
964
1088
  }
965
- if (storeStat.size !== targetStat.size) {
966
- return { status: "size-mismatch" }
1089
+ const entry = await this.registry.getFile(path.resolve(filePath))
1090
+ if (!entry || entry.unavailable_reason === "stale") {
1091
+ return { error: "This file is no longer tracked. Scan again to refresh this view." }
967
1092
  }
968
- if (!sameFileMetadata(storeStat, targetStat)) {
969
- return { status: "metadata-mismatch" }
1093
+ const source = this.sourceForPath(entry.path, entry.source_id)
1094
+ if (!source || !this.sourceIsWithinScope(source, scopeId)) {
1095
+ return { error: "This file is outside the current location." }
1096
+ }
1097
+ if (!await this.canonicalPathIsWithinSource(entry.path, source)) {
1098
+ return { error: "This file is no longer available at the scanned location." }
1099
+ }
1100
+ const current = await lstatIfPresent(entry.path)
1101
+ if (!current || !current.isFile()) {
1102
+ return { error: "This file is no longer available at the scanned location." }
970
1103
  }
971
- const verifiedStore = await this.verifyStoreContent(hash, storePath, storeStat)
972
- if (!verifiedStore.valid) return { status: "stale-blob" }
973
- if (meta.source && this.sourceAppIsRunning(meta.source)) return { status: "locked" }
974
- const tmp = targetPath + TMP_SUFFIX
975
- let ownsTmp = false
976
1104
  try {
977
- try {
978
- await fs.promises.link(storePath, tmp)
979
- ownsTmp = true
980
- } catch (e) {
981
- if (e.code === "EEXIST") {
982
- let tmpStat = null
983
- try { tmpStat = await fs.promises.lstat(tmp) } catch (error) {}
984
- if (!tmpStat || tmpStat.dev !== storeStat.dev || tmpStat.ino !== storeStat.ino) {
985
- return { status: "conflict" }
986
- }
987
- } else {
988
- throw e
1105
+ await this.fileManagerLauncher(entry.path)
1106
+ } catch (error) {
1107
+ return { error: "The file manager could not open this file." }
1108
+ }
1109
+ return { revealed: true }
1110
+ }
1111
+
1112
+ locationForPath(filePath, preferredId = null) {
1113
+ const source = this.sourceForPath(filePath, preferredId)
1114
+ if (!source) {
1115
+ return {
1116
+ source_id: null,
1117
+ relative_path: path.basename(filePath)
1118
+ }
1119
+ }
1120
+ const resolvedPath = path.resolve(filePath)
1121
+ const sourceBase = source.canonical_root &&
1122
+ isPathWithin(source.canonical_root, resolvedPath)
1123
+ ? source.canonical_root
1124
+ : source.root
1125
+ const relative = path.relative(sourceBase, resolvedPath)
1126
+ .split(path.sep).join("/") || path.basename(filePath)
1127
+ return {
1128
+ source_id: source.id,
1129
+ source_kind: source.kind,
1130
+ source_label: source.label,
1131
+ relative_path: relative
1132
+ }
1133
+ }
1134
+
1135
+ async addExternalSource(folderPath) {
1136
+ if (typeof folderPath !== "string" ||
1137
+ !path.isAbsolute(folderPath.trim())) {
1138
+ throw new Error("Choose a valid folder.")
1139
+ }
1140
+ const canonical = path.resolve(
1141
+ await fs.promises.realpath(folderPath.trim()))
1142
+ const stat = await fs.promises.stat(canonical)
1143
+ if (!stat.isDirectory()) throw new Error("Choose a folder, not a file.")
1144
+
1145
+ const home = path.resolve(await fs.promises.realpath(this.kernel.homedir))
1146
+ if (isPathWithin(home, canonical)) {
1147
+ throw new Error(
1148
+ "That folder is already inside Pinokio and is included in scans.")
1149
+ }
1150
+ await this.refreshSources()
1151
+ const existing = this._sources.find((source) =>
1152
+ source.kind === "external" &&
1153
+ source.root &&
1154
+ samePath(source.root, canonical))
1155
+ if (existing) return { created: false, source: existing }
1156
+
1157
+ const config = this.readConfig()
1158
+ this.writeConfig(Object.assign({}, config, {
1159
+ locations: [...config.locations, canonical]
1160
+ }))
1161
+ await this.refreshAnchorStores()
1162
+ await this.refreshSources()
1163
+ const source = this._sources.find((candidate) =>
1164
+ candidate.kind === "external" &&
1165
+ samePath(candidate.root, canonical))
1166
+ return { created: true, source }
1167
+ }
1168
+
1169
+ async removeExternalSource(sourceIdToRemove) {
1170
+ await this.refreshSources()
1171
+ const source = this._sources.find((candidate) =>
1172
+ candidate.id === sourceIdToRemove &&
1173
+ candidate.kind === "external" &&
1174
+ candidate.configured)
1175
+ if (!source) {
1176
+ return { error: "That external folder is no longer configured." }
1177
+ }
1178
+ const config = this.readConfig()
1179
+ this.writeConfig(Object.assign({}, config, {
1180
+ locations: config.locations.filter((item) =>
1181
+ !samePath(item, source.root))
1182
+ }))
1183
+ await this.registry.removeExternalSourceState(source.root, source.id)
1184
+ await this.refreshAnchorStores()
1185
+ await this.refreshSources()
1186
+ return {
1187
+ removed: true,
1188
+ source_id: source.id,
1189
+ label: source.label
1190
+ }
1191
+ }
1192
+
1193
+ async recordEvent(event) {
1194
+ try {
1195
+ await this.registry.addEvent(event)
1196
+ return true
1197
+ } catch (error) {
1198
+ return false
1199
+ }
1200
+ }
1201
+
1202
+ startScan(scopeId = null, sizeThreshold = this.sizeThreshold) {
1203
+ if (!this.enabled || !this.sweeper) {
1204
+ return { started: false, disabled: true }
1205
+ }
1206
+ if (this.scanPromise) return { started: false, already_running: true }
1207
+ this.sizeThreshold = sizeThreshold
1208
+ this.scanError = null
1209
+ this.scanScopeId = scopeId
1210
+ this.scanCancelRequested = false
1211
+ this.scanPromise = this.runExclusive(() => {
1212
+ if (this.scanCancelRequested) {
1213
+ this.sweeper.state = Object.assign(this.sweeper.idleState(), {
1214
+ phase: "cancelled",
1215
+ scope_id: scopeId
1216
+ })
1217
+ return { cancelled: true }
1218
+ }
1219
+ return this.sweeper.scan(scopeId)
1220
+ }).catch((error) => {
1221
+ this.scanError = error && error.message
1222
+ ? error.message
1223
+ : String(error)
1224
+ }).finally(() => {
1225
+ this.scanPromise = null
1226
+ this.scanScopeId = null
1227
+ this.scanCancelRequested = false
1228
+ })
1229
+ return { started: true }
1230
+ }
1231
+
1232
+ cancelScan() {
1233
+ if (!this.scanPromise || !this.sweeper) {
1234
+ return { cancel_requested: false }
1235
+ }
1236
+ this.scanCancelRequested = true
1237
+ if (this.sweeper.state.active && this.sweeper.currentHash &&
1238
+ this.worker) {
1239
+ const error = new Error("Scan cancelled.")
1240
+ error.code = "EVAULTCANCELLED"
1241
+ this.failHashWorker(this.worker, error, true)
1242
+ }
1243
+ return {
1244
+ cancel_requested: this.sweeper.state.active
1245
+ ? this.sweeper.cancel()
1246
+ : true
1247
+ }
1248
+ }
1249
+
1250
+ async perform(action, payload = {}) {
1251
+ if (!this.enabled) return { error: "Save space is disabled." }
1252
+ await this.ensureInitialized()
1253
+ switch (action) {
1254
+ case "add_source": {
1255
+ const result = await this.runExclusive(() =>
1256
+ this.addExternalSource(payload.path))
1257
+ return {
1258
+ created: result.created,
1259
+ source: {
1260
+ id: result.source.id,
1261
+ label: result.source.label,
1262
+ target_path: result.source.root,
1263
+ shareable: !!result.source.shareable
1264
+ }
989
1265
  }
990
1266
  }
991
- // Narrow the portable stat/rename race as far as Node permits. The app
992
- // guard prevents known Pinokio writers; arbitrary external writers in the
993
- // final syscall-sized interval are the documented residual limitation.
994
- if (meta.source && this.sourceAppIsRunning(meta.source)) {
995
- if (ownsTmp) await unlinkIfSame(tmp, storeStat).catch(() => {})
996
- return { status: "locked" }
1267
+ case "remove_source":
1268
+ if (typeof payload.source_id !== "string" || !payload.source_id) {
1269
+ return { error: "Choose an external folder to remove." }
1270
+ }
1271
+ return this.runMutation(() =>
1272
+ this.removeExternalSource(payload.source_id))
1273
+ case "scan": {
1274
+ if (payload.scope_id && !this.scanSource(payload.scope_id)) {
1275
+ return { error: "That scan location is no longer available." }
1276
+ }
1277
+ let threshold = this.sizeThreshold
1278
+ if (payload.candidate_size != null) {
1279
+ if (!CANDIDATE_SIZE_OPTIONS.includes(payload.candidate_size)) {
1280
+ return { error: "Choose a valid minimum file size." }
1281
+ }
1282
+ threshold = payload.candidate_size
1283
+ }
1284
+ return this.startScan(payload.scope_id || null, threshold)
1285
+ }
1286
+ case "cancel_scan":
1287
+ return this.cancelScan()
1288
+ case "cancel_file_action":
1289
+ return this.cancelFileAction()
1290
+ case "reveal":
1291
+ return this.revealFile(
1292
+ typeof payload.scope_id === "string" && payload.scope_id
1293
+ ? payload.scope_id
1294
+ : null,
1295
+ payload.path
1296
+ )
1297
+ case "deduplicate": {
1298
+ const scopeId = typeof payload.scope_id === "string" &&
1299
+ payload.scope_id
1300
+ ? payload.scope_id
1301
+ : null
1302
+ if (scopeId && !this._sources.some((source) =>
1303
+ source.id === scopeId)) {
1304
+ return { error: "That location is no longer available." }
1305
+ }
1306
+ if (typeof payload.path === "string" && payload.path) {
1307
+ return this.runMutation(() => this.runFileAction({
1308
+ kind: "deduplicate-file",
1309
+ path: path.resolve(payload.path)
1310
+ }, () => this.deduplicateFile(payload.path)))
1311
+ }
1312
+ const filesTotal = await this.countActionFiles("duplicate", scopeId)
1313
+ return this.runMutation(() => this.runFileAction({
1314
+ kind: "deduplicate",
1315
+ scope_id: scopeId,
1316
+ files_total: filesTotal,
1317
+ files_completed: 0
1318
+ }, (progress) => this.deduplicateScope(scopeId, { progress })))
1319
+ }
1320
+ case "deduplicate_files": {
1321
+ if (!Array.isArray(payload.paths) ||
1322
+ payload.paths.length === 0 ||
1323
+ payload.paths.length > MAX_BULK_FILE_ACTIONS ||
1324
+ payload.paths.some((item) =>
1325
+ typeof item !== "string" || !item)) {
1326
+ return { error: "Choose valid duplicate files to deduplicate." }
1327
+ }
1328
+ const paths = [...new Set(payload.paths.map((item) =>
1329
+ path.resolve(item)))]
1330
+ return this.runMutation(() => this.runFileAction({
1331
+ kind: "deduplicate-files",
1332
+ files_total: paths.length,
1333
+ files_completed: 0
1334
+ }, (progress) => this.deduplicateFiles(paths, progress)))
997
1335
  }
998
- const currentTmpStat = await lstatIfPresent(tmp)
999
- if (!currentTmpStat || currentTmpStat.dev !== storeStat.dev || currentTmpStat.ino !== storeStat.ino) {
1000
- if (ownsTmp) await unlinkIfSame(tmp, storeStat).catch(() => {})
1001
- return { status: "conflict" }
1336
+ case "detach":
1337
+ if (typeof payload.path !== "string" || !payload.path) {
1338
+ return { status: "not-found" }
1339
+ }
1340
+ return this.runMutation(() => this.runFileAction({
1341
+ kind: "make-separate",
1342
+ path: path.resolve(payload.path)
1343
+ }, () => this.separate(payload.path)))
1344
+ case "separate_files": {
1345
+ if (!Array.isArray(payload.paths) ||
1346
+ payload.paths.length === 0 ||
1347
+ payload.paths.length > MAX_BULK_FILE_ACTIONS ||
1348
+ payload.paths.some((item) =>
1349
+ typeof item !== "string" || !item)) {
1350
+ return { error: "Choose valid deduplicated files to separate." }
1351
+ }
1352
+ return this.runMutation(() => this.runFileAction({
1353
+ kind: "separate-files",
1354
+ files_total: new Set(payload.paths.map((item) =>
1355
+ path.resolve(item))).size,
1356
+ files_completed: 0
1357
+ }, (progress) => this.separateFiles(payload.paths, progress)))
1358
+ }
1359
+ case "separate_all": {
1360
+ const selection = this.separateSelection(payload)
1361
+ if (selection.error) return { error: selection.error }
1362
+ const total = await this.registry.matchingFileSummary(
1363
+ "linked", selection.sourceIds, selection.query)
1364
+ if (!total.count) {
1365
+ return { error: "No matching deduplicated files remain." }
1366
+ }
1367
+ return this.runMutation(() => this.runFileAction({
1368
+ kind: "separate-files",
1369
+ files_total: total.count,
1370
+ files_completed: 0,
1371
+ all_matching: true,
1372
+ cancelable: true,
1373
+ cancel_requested: false
1374
+ }, (progress) =>
1375
+ this.separateMatchingFiles(selection, progress)))
1002
1376
  }
1003
- const currentTargetStat = await fs.promises.lstat(targetPath)
1004
- if (!sameSnapshot(fileSnapshot(targetStat), currentTargetStat)) {
1005
- if (ownsTmp) await unlinkIfSame(tmp, storeStat).catch(() => {})
1006
- return { status: "stale" }
1377
+ case "reclaim":
1378
+ return this.runMutation(() =>
1379
+ this.reclaim(payload.hash, payload.store_id))
1380
+ case "reclaim_all":
1381
+ return this.runMutation(() => this.reclaimAll())
1382
+ default:
1383
+ return { error: "unknown action" }
1384
+ }
1385
+ }
1386
+
1387
+ sourceAppIsRunning(source) {
1388
+ if (!source || source.kind !== "app") return false
1389
+ const appRoot = path.resolve(this.kernel.homedir, "api", source.app)
1390
+ const api = this.kernel.api || {}
1391
+ const running = api.running || {}
1392
+ const runningPaths = api.running_paths || {}
1393
+ return Object.keys(running).some((id) => {
1394
+ if (!running[id]) return false
1395
+ const runningPath = runningPaths[id] ||
1396
+ (path.isAbsolute(id) ? id.split("?")[0] : null)
1397
+ return !!(runningPath && isPathWithin(appRoot, runningPath))
1398
+ })
1399
+ }
1400
+
1401
+ async refreshInodeSnapshots(hash, dev, ino, storeId = null) {
1402
+ const store = storeId
1403
+ ? this._anchorStoresById.get(storeId)
1404
+ : this.anchorStoreForDevice(dev)
1405
+ const storePath = store ? this.storePathFor(hash, store.id) : null
1406
+ const storeStat = storePath
1407
+ ? await this.storeStatIfPresent(storePath, { store_id: store.id })
1408
+ : null
1409
+ let inodeStat = storeStat &&
1410
+ storeStat.dev === dev &&
1411
+ storeStat.ino === ino
1412
+ ? storeStat
1413
+ : null
1414
+ if (!inodeStat) {
1415
+ const row = await this.registry.firstFileForInode(hash, dev, ino)
1416
+ if (row) {
1417
+ try {
1418
+ const stat = await fs.promises.lstat(row.path)
1419
+ if (stat.isFile() && stat.dev === dev && stat.ino === ino) {
1420
+ inodeStat = stat
1421
+ }
1422
+ } catch (error) {}
1007
1423
  }
1008
- const currentStoreStat = await this.storeStatIfPresent(storePath)
1009
- // Creating our temporary hardlink changes ctime on the shared inode.
1010
- // Identity, size, and mtime still detect replacement or content writes.
1011
- if (!currentStoreStat || !sameContentState(verifiedStore.snapshot, currentStoreStat) ||
1012
- !sameFileMetadata(currentStoreStat, currentTargetStat)) {
1013
- if (ownsTmp) await unlinkIfSame(tmp, storeStat).catch(() => {})
1014
- return { status: "stale-blob" }
1015
- }
1016
- await fs.promises.rename(tmp, targetPath)
1017
- ownsTmp = false
1018
- } catch (e) {
1019
- if (ownsTmp) await unlinkIfSame(tmp, storeStat).catch(() => {})
1020
- if (LOCK_CODES.has(e.code)) {
1021
- return { status: "locked" }
1424
+ }
1425
+ if (inodeStat) {
1426
+ await this.registry.updateInodeSnapshots(
1427
+ dev, ino, fileSnapshot(inodeStat))
1428
+ }
1429
+ const content = await this.registry.getContent(hash)
1430
+ if (content) {
1431
+ await this.registry.upsertContent(Object.assign({}, content, {
1432
+ hash,
1433
+ size: storeStat ? storeStat.size : content.size
1434
+ }))
1435
+ }
1436
+ if (store && storeStat) {
1437
+ await this.registry.upsertAnchor({
1438
+ store_id: store.id,
1439
+ hash,
1440
+ path: storePath,
1441
+ verified_at: Date.now(),
1442
+ dev: storeStat.dev,
1443
+ ino: storeStat.ino,
1444
+ size: storeStat.size,
1445
+ mtime: storeStat.mtimeMs,
1446
+ ctime: storeStat.ctimeMs,
1447
+ nlink: storeStat.nlink,
1448
+ mode: storeStat.mode,
1449
+ uid: storeStat.uid,
1450
+ gid: storeStat.gid
1451
+ })
1452
+ } else if (store) {
1453
+ await this.registry.removeAnchor(store.id, hash)
1454
+ }
1455
+ }
1456
+
1457
+ async reclassifyHashes(hashes) {
1458
+ const stores = this.anchorStores()
1459
+ .filter((store) =>
1460
+ store.available && Number.isFinite(store.dev))
1461
+ .map((store) => ({
1462
+ store_id: store.id,
1463
+ dev: store.dev,
1464
+ can_link: store.mode !== "copy"
1465
+ }))
1466
+ for (const hash of new Set(hashes)) {
1467
+ const anchorSnapshots = []
1468
+ for (const store of this.anchorStores()) {
1469
+ const storePath = this.storePathFor(hash, store.id)
1470
+ const stat = await this.storeStatIfPresent(storePath, {
1471
+ store_id: store.id
1472
+ })
1473
+ if (stat) anchorSnapshots.push(fileSnapshot(stat))
1022
1474
  }
1023
- if (NO_LINK_CODES.has(e.code)) {
1024
- return { status: "unavailable", code: e.code }
1475
+ await this.registry.reclassifyHash(
1476
+ hash,
1477
+ stores,
1478
+ anchorSnapshots
1479
+ )
1480
+ }
1481
+ }
1482
+
1483
+ async verifyStoreContent(hash, storePath, storeStat, storeId = null) {
1484
+ if (!storeStat || !storeStat.isFile()) return { valid: false }
1485
+ const store = storeId
1486
+ ? this._anchorStoresById.get(storeId)
1487
+ : this.anchorStoreForPath(storePath)
1488
+ if (!store) return { valid: false }
1489
+ const content = await this.registry.getContent(hash)
1490
+ const anchor = await this.registry.getAnchor(store.id, hash)
1491
+ if (anchor &&
1492
+ anchor.verified_at &&
1493
+ anchor.dev === storeStat.dev &&
1494
+ anchor.ino === storeStat.ino &&
1495
+ anchor.size === storeStat.size &&
1496
+ anchor.mtime === storeStat.mtimeMs &&
1497
+ anchor.ctime === storeStat.ctimeMs) {
1498
+ return { valid: true, snapshot: fileSnapshot(storeStat) }
1499
+ }
1500
+ const before = fileSnapshot(storeStat)
1501
+ let hashed
1502
+ try {
1503
+ hashed = await this.hashFile(storePath)
1504
+ } catch (error) {
1505
+ return { valid: false }
1506
+ }
1507
+ const after = await this.storeStatIfPresent(storePath)
1508
+ if (!after ||
1509
+ !sameSnapshot(before, after) ||
1510
+ hashed.hash !== hash ||
1511
+ hashed.size !== after.size) {
1512
+ return { valid: false }
1513
+ }
1514
+ await this.registry.upsertContent({
1515
+ hash,
1516
+ size: after.size,
1517
+ first_seen: content && content.first_seen,
1518
+ verified_at: Date.now()
1519
+ })
1520
+ await this.registry.upsertAnchor({
1521
+ store_id: store.id,
1522
+ hash,
1523
+ path: storePath,
1524
+ verified_at: Date.now(),
1525
+ dev: after.dev,
1526
+ ino: after.ino,
1527
+ size: after.size,
1528
+ mtime: after.mtimeMs,
1529
+ ctime: after.ctimeMs,
1530
+ nlink: after.nlink,
1531
+ mode: after.mode,
1532
+ uid: after.uid,
1533
+ gid: after.gid
1534
+ })
1535
+ return { valid: true, snapshot: fileSnapshot(after) }
1536
+ }
1537
+
1538
+ async adopt(filePath, hash, expected, selectedStore = null) {
1539
+ const current = await lstatIfPresent(filePath)
1540
+ if (!current ||
1541
+ !current.isFile() ||
1542
+ !sameSnapshot(expected, current)) {
1543
+ return { status: "stale" }
1544
+ }
1545
+ const store = selectedStore || this.anchorStoreForDevice(current.dev)
1546
+ if (!store) return { status: "unavailable" }
1547
+ const storePath = this.storePathFor(hash, store.id)
1548
+ let storeStat
1549
+ try {
1550
+ storeStat = await this.storeStatIfPresent(storePath, {
1551
+ createParent: true,
1552
+ dev: current.dev,
1553
+ store_id: store.id
1554
+ })
1555
+ } catch (error) {
1556
+ if (error && (
1557
+ error.code === "EVAULTUNAVAILABLE" ||
1558
+ NO_LINK_CODES.has(error.code) ||
1559
+ LOCK_CODES.has(error.code)
1560
+ )) {
1561
+ return { status: "unavailable" }
1025
1562
  }
1026
- throw e
1563
+ throw error
1564
+ }
1565
+ if (storeStat) {
1566
+ if (sameIdentity(storeStat, current)) return { status: "ready" }
1567
+ return { status: "exists" }
1027
1568
  }
1028
- let st
1029
1569
  try {
1030
- st = await fs.promises.lstat(targetPath)
1570
+ await fs.promises.link(filePath, storePath)
1031
1571
  } catch (error) {
1032
- if (isMissingError(error)) return { status: "stale" }
1033
- // rename() is the commit point: it succeeded with a temporary name
1034
- // whose identity was already verified. A later metadata read failure
1035
- // must not report the completed replacement as failed.
1036
- st = storeStat
1572
+ if (NO_LINK_CODES.has(error && error.code)) {
1573
+ return { status: "unavailable" }
1574
+ }
1575
+ throw error
1037
1576
  }
1038
- if (st.dev !== storeStat.dev || st.ino !== storeStat.ino) {
1577
+ storeStat = await this.storeStatIfPresent(storePath, {
1578
+ store_id: store.id
1579
+ })
1580
+ const after = await lstatIfPresent(filePath)
1581
+ if (!storeStat ||
1582
+ !after ||
1583
+ !sameIdentity(storeStat, after) ||
1584
+ !sameContentState(expected, after)) {
1585
+ if (storeStat) await unlinkIfSame(storePath, storeStat)
1039
1586
  return { status: "stale" }
1040
1587
  }
1041
- this.registry.addLink(targetPath, {
1042
- hash, app: meta.app || null, source_id: meta.source_id || null,
1043
- dev: st.dev, ino: st.ino, mode: "link", batch_id: meta.batch_id || null
1588
+ const existing = await this.registry.getFile(filePath)
1589
+ await this.registry.upsertContent({
1590
+ hash,
1591
+ size: storeStat.size,
1592
+ verified_at: Date.now(),
1593
+ first_seen: Date.now()
1044
1594
  })
1045
- await this.refreshLinkSnapshots(hash, st.dev, st.ino)
1046
- this.registry.addSaved(storeStat.size)
1047
- await this.recordEvent({
1048
- kind: "convert", hash, path: targetPath, app: meta.app || null, source_id: meta.source_id || null,
1049
- bytes_saved: storeStat.size, batch_id: meta.batch_id || null
1595
+ await this.registry.upsertAnchor({
1596
+ store_id: store.id,
1597
+ hash,
1598
+ path: storePath,
1599
+ verified_at: Date.now(),
1600
+ dev: storeStat.dev,
1601
+ ino: storeStat.ino,
1602
+ size: storeStat.size,
1603
+ mtime: storeStat.mtimeMs,
1604
+ ctime: storeStat.ctimeMs,
1605
+ nlink: storeStat.nlink,
1606
+ mode: storeStat.mode,
1607
+ uid: storeStat.uid,
1608
+ gid: storeStat.gid
1050
1609
  })
1051
- return { status: "converted", bytes_saved: storeStat.size }
1610
+ await this.registry.upsertFile(Object.assign({}, existing, {
1611
+ path: filePath,
1612
+ hash,
1613
+ size: after.size,
1614
+ mtime: after.mtimeMs,
1615
+ ctime: after.ctimeMs,
1616
+ dev: after.dev,
1617
+ ino: after.ino,
1618
+ source_id: existing && existing.source_id,
1619
+ app: existing && existing.app,
1620
+ status: "linked"
1621
+ }))
1622
+ await this.refreshInodeSnapshots(hash, after.dev, after.ino, store.id)
1623
+ return { status: "ready", store_id: store.id }
1052
1624
  }
1053
- sourceAppIsRunning(source) {
1054
- if (!source || source.kind !== "app") return false
1055
- const appRoot = path.resolve(this.kernel.homedir, "api", source.app)
1056
- const api = this.kernel.api || {}
1057
- const running = api.running || {}
1058
- const runningPaths = api.running_paths || {}
1059
- return Object.keys(running).some((runningId) => {
1060
- if (!running[runningId]) return false
1061
- const runningPath = runningPaths[runningId] || (path.isAbsolute(runningId) ? runningId.split("?")[0] : null)
1062
- return !!(runningPath && isPathWithin(appRoot, runningPath))
1063
- })
1625
+
1626
+ async ensureAnchorForHash(hash, targetStat) {
1627
+ const dev = targetStat.dev
1628
+ const store = this.anchorStoreForDevice(dev)
1629
+ if (!store) return { status: "unavailable" }
1630
+ const storePath = this.storePathFor(hash, store.id)
1631
+ let existingStore
1632
+ try {
1633
+ existingStore = await this.storeStatIfPresent(storePath, {
1634
+ store_id: store.id
1635
+ })
1636
+ if (existingStore) await this.validateAnchorStore(store, dev)
1637
+ } catch (error) {
1638
+ if (error && (
1639
+ error.code === "EVAULTPATH" ||
1640
+ error.code === "EVAULTUNAVAILABLE" ||
1641
+ NO_LINK_CODES.has(error.code) ||
1642
+ LOCK_CODES.has(error.code)
1643
+ )) {
1644
+ return { status: "unavailable" }
1645
+ }
1646
+ throw error
1647
+ }
1648
+ if (existingStore) {
1649
+ if (existingStore.dev !== dev) return { status: "unavailable" }
1650
+ const verified = await this.verifyStoreContent(
1651
+ hash, storePath, existingStore, store.id)
1652
+ return verified.valid
1653
+ ? { status: "ready", stat: existingStore, store_id: store.id }
1654
+ : { status: "stale" }
1655
+ }
1656
+
1657
+ const candidate = await this.registry.anchorCandidate(
1658
+ hash,
1659
+ dev,
1660
+ targetStat.mode & 0o7777,
1661
+ targetStat.uid,
1662
+ targetStat.gid
1663
+ )
1664
+ if (!candidate) return { status: "no-source" }
1665
+ const source = this.sourceForPath(candidate.path, candidate.source_id)
1666
+ if (!source ||
1667
+ !await this.canonicalPathIsWithinSource(candidate.path, source)) {
1668
+ return { status: "stale" }
1669
+ }
1670
+ if (this.sourceAppIsRunning(source)) return { status: "locked" }
1671
+ const before = await lstatIfPresent(candidate.path)
1672
+ if (!before ||
1673
+ !before.isFile() ||
1674
+ !sameSnapshot(rowSnapshot(candidate), before)) {
1675
+ return { status: "stale" }
1676
+ }
1677
+ const hashed = await this.hashFile(candidate.path)
1678
+ const after = await lstatIfPresent(candidate.path)
1679
+ if (!after ||
1680
+ !sameSnapshot(fileSnapshot(before), after) ||
1681
+ hashed.hash !== hash) {
1682
+ return { status: "stale" }
1683
+ }
1684
+ const adopted = await this.adopt(
1685
+ candidate.path, hash, fileSnapshot(after), store)
1686
+ if (adopted.status !== "ready") return adopted
1687
+ return {
1688
+ status: "ready",
1689
+ store_id: store.id,
1690
+ stat: await this.storeStatIfPresent(storePath, {
1691
+ store_id: store.id
1692
+ })
1693
+ }
1064
1694
  }
1065
- async deduplicateFile(filePath, options = {}) {
1066
- if (!this.enabled || typeof filePath !== "string" || !filePath) {
1067
- return { status: this.enabled ? "not-found" : "disabled" }
1695
+
1696
+ async convert(targetPath) {
1697
+ const target = await this.registry.getFile(targetPath)
1698
+ if (!target ||
1699
+ target.unavailable_reason === "stale" ||
1700
+ target.status !== "duplicate" ||
1701
+ !target.hash) {
1702
+ return { status: "not-found" }
1703
+ }
1704
+ const source = this.sourceForPath(target.path, target.source_id)
1705
+ if (!source ||
1706
+ !await this.canonicalPathIsWithinSource(target.path, source)) {
1707
+ return { status: "stale" }
1068
1708
  }
1069
- await this.refreshSources()
1070
- return this.deduplicateExcludedFile(filePath, options)
1071
- }
1072
- async deduplicateExcludedFile(filePath, options = {}) {
1073
- const targetPath = path.resolve(filePath)
1074
- const excluded = this.registry.excluded.get(targetPath)
1075
- if (!excluded) return { status: "not-found" }
1076
- const source = this.sourceForPath(targetPath, excluded.source_id)
1077
- if (!source || !await this.canonicalPathIsWithinSource(targetPath, source)) {
1709
+ if (this.sourceAppIsRunning(source)) return { status: "locked" }
1710
+
1711
+ let targetStat = await lstatIfPresent(target.path)
1712
+ if (!targetStat ||
1713
+ !targetStat.isFile() ||
1714
+ !sameSnapshot(rowSnapshot(target), targetStat)) {
1078
1715
  return { status: "stale" }
1079
1716
  }
1080
- if (!source.shareable) return { status: "unavailable" }
1081
- if (this.sourceAppIsRunning(source)) return { status: "locked" }
1717
+ const prepared = await this.ensureAnchorForHash(target.hash, targetStat)
1718
+ if (prepared.status !== "ready") {
1719
+ if (prepared.status === "unavailable") {
1720
+ await this.reclassifyHashes([target.hash])
1721
+ }
1722
+ return prepared
1723
+ }
1724
+ const storePath = this.storePathFor(target.hash, prepared.store_id)
1725
+ let storeStat = await this.storeStatIfPresent(
1726
+ storePath, { store_id: prepared.store_id })
1727
+ if (!storeStat) return { status: "stale" }
1728
+ if (storeStat.dev !== targetStat.dev) {
1729
+ return { status: "unavailable" }
1730
+ }
1731
+ if (sameIdentity(storeStat, targetStat)) {
1732
+ await this.registry.upsertFile(Object.assign({}, target, {
1733
+ status: "linked"
1734
+ }))
1735
+ return { status: "already", bytes_saved: 0 }
1736
+ }
1737
+ if (storeStat.size !== targetStat.size) {
1738
+ return { status: "size-mismatch" }
1739
+ }
1740
+ if (!sameFileMetadata(storeStat, targetStat)) {
1741
+ return { status: "metadata-mismatch" }
1742
+ }
1743
+ const verified = await this.verifyStoreContent(
1744
+ target.hash, storePath, storeStat, prepared.store_id)
1745
+ if (!verified.valid) return { status: "stale" }
1746
+
1747
+ const temporary = `${target.path}${TMP_SUFFIX}`
1748
+ let temporaryStat = null
1749
+ let committedStat = null
1750
+ try {
1751
+ await fs.promises.link(storePath, temporary)
1752
+ temporaryStat = await fs.promises.lstat(temporary)
1753
+ if (this.sourceAppIsRunning(source)) {
1754
+ await unlinkIfSame(temporary, temporaryStat)
1755
+ return { status: "locked" }
1756
+ }
1757
+ const [currentTarget, currentStore, currentTemporary] =
1758
+ await Promise.all([
1759
+ lstatIfPresent(target.path),
1760
+ this.storeStatIfPresent(storePath, {
1761
+ store_id: prepared.store_id
1762
+ }),
1763
+ lstatIfPresent(temporary)
1764
+ ])
1765
+ if (!currentTarget ||
1766
+ !sameSnapshot(fileSnapshot(targetStat), currentTarget) ||
1767
+ !currentStore ||
1768
+ !currentTemporary ||
1769
+ !sameIdentity(currentStore, currentTemporary) ||
1770
+ !sameContentState(verified.snapshot, currentStore)) {
1771
+ await unlinkIfSame(temporary, temporaryStat)
1772
+ return { status: "stale" }
1773
+ }
1774
+ committedStat = temporaryStat
1775
+ await fs.promises.rename(temporary, target.path)
1776
+ temporaryStat = null
1777
+ } catch (error) {
1778
+ if (temporaryStat) {
1779
+ await unlinkIfSame(temporary, temporaryStat).catch(() => {})
1780
+ }
1781
+ if (LOCK_CODES.has(error && error.code)) return { status: "locked" }
1782
+ if (NO_LINK_CODES.has(error && error.code)) {
1783
+ return { status: "unavailable" }
1784
+ }
1785
+ if (error && error.code === "EEXIST") return { status: "conflict" }
1786
+ throw error
1787
+ }
1788
+
1789
+ let finalStat
1790
+ try {
1791
+ finalStat = await fs.promises.lstat(target.path)
1792
+ } catch (error) {
1793
+ if (!committedStat) throw error
1794
+ finalStat = committedStat
1795
+ }
1796
+ storeStat = await this.storeStatIfPresent(storePath, {
1797
+ store_id: prepared.store_id
1798
+ })
1799
+ if (!storeStat || !sameIdentity(finalStat, storeStat)) {
1800
+ return { status: "stale" }
1801
+ }
1802
+ await this.registry.upsertFile(Object.assign({}, target, {
1803
+ size: finalStat.size,
1804
+ mtime: finalStat.mtimeMs,
1805
+ ctime: finalStat.ctimeMs,
1806
+ dev: finalStat.dev,
1807
+ ino: finalStat.ino,
1808
+ status: "linked"
1809
+ }))
1810
+ await this.refreshInodeSnapshots(
1811
+ target.hash, finalStat.dev, finalStat.ino, prepared.store_id)
1812
+ return {
1813
+ status: "converted",
1814
+ bytes_saved: finalStat.size,
1815
+ hash: target.hash,
1816
+ path: target.path,
1817
+ app: target.app,
1818
+ source_id: target.source_id
1819
+ }
1820
+ }
1821
+
1822
+ async countActionFiles(status, scopeId = null) {
1823
+ const sourceIds = this.scopeSourceIds(scopeId)
1824
+ if (scopeId && !sourceIds.length) return 0
1825
+ return this.registry.countActionFiles(status, sourceIds)
1826
+ }
1827
+
1828
+ separateSelection(payload = {}) {
1829
+ const scopeId = typeof payload.scope_id === "string" &&
1830
+ payload.scope_id
1831
+ ? payload.scope_id
1832
+ : null
1833
+ const locationId = typeof payload.location_id === "string" &&
1834
+ payload.location_id
1835
+ ? payload.location_id
1836
+ : null
1837
+ if (scopeId && !this._sourcesById.has(scopeId)) {
1838
+ return { error: "That location is no longer available." }
1839
+ }
1840
+ if (locationId && !this._sourcesById.has(locationId)) {
1841
+ return { error: "That location is no longer available." }
1842
+ }
1843
+ const view = STATUS_VIEWS.has(payload.view) ? payload.view : "all"
1844
+ const statusFilter = STATUS_FILTERS.has(payload.status_filter)
1845
+ ? payload.status_filter
1846
+ : "all"
1847
+ if (view !== "shared" &&
1848
+ !(view === "all" &&
1849
+ (statusFilter === "all" || statusFilter === "shared"))) {
1850
+ return {
1851
+ error: "The current view has no deduplicated files to separate."
1852
+ }
1853
+ }
1854
+ const sourceIds = this.scopeSourceIds(scopeId, locationId)
1855
+ if (!sourceIds.length) {
1856
+ return { error: "That location is no longer available." }
1857
+ }
1858
+ return {
1859
+ sourceIds,
1860
+ query: String(payload.query || "").slice(0, 500).trim()
1861
+ }
1862
+ }
1863
+
1864
+ async deduplicateFile(filePath) {
1865
+ const result = await this.convert(path.resolve(filePath))
1866
+ if (result.status === "converted") {
1867
+ if (!await this.recordEvent({
1868
+ kind: "convert",
1869
+ hash: result.hash,
1870
+ app: result.app,
1871
+ source_id: result.source_id,
1872
+ bytes: result.bytes_saved,
1873
+ files: 1
1874
+ })) {
1875
+ result.activity_warning = true
1876
+ }
1877
+ }
1878
+ return result
1879
+ }
1880
+
1881
+ deduplicationSummary() {
1882
+ return {
1883
+ converted: 0,
1884
+ bytes_saved: 0,
1885
+ locked: 0,
1886
+ stale: 0,
1887
+ incompatible: 0,
1888
+ unavailable: 0,
1889
+ failed: 0
1890
+ }
1891
+ }
1892
+
1893
+ addDeduplicationResult(summary, result) {
1894
+ if (result.status === "converted") {
1895
+ summary.converted += 1
1896
+ summary.bytes_saved += result.bytes_saved || 0
1897
+ } else if (result.status === "already") {
1898
+ // Another selected path already caused this inode to be shared.
1899
+ } else if (result.status === "locked") {
1900
+ summary.locked += 1
1901
+ } else if (result.status === "metadata-mismatch") {
1902
+ summary.incompatible += 1
1903
+ } else if (result.status === "unavailable") {
1904
+ summary.unavailable += 1
1905
+ } else {
1906
+ summary.stale += 1
1907
+ }
1908
+ }
1909
+
1910
+ async recordDeduplicationSummary(
1911
+ summary,
1912
+ representative = null,
1913
+ sourceId = null
1914
+ ) {
1915
+ if (!summary.converted) return
1916
+ const event = {
1917
+ kind: "convert",
1918
+ bytes: summary.bytes_saved,
1919
+ files: summary.converted
1920
+ }
1921
+ if (summary.converted === 1 && representative) {
1922
+ event.hash = representative.hash
1923
+ event.path = representative.path
1924
+ event.app = representative.app
1925
+ event.source_id = representative.source_id
1926
+ } else if (sourceId) {
1927
+ event.source_id = sourceId
1928
+ }
1929
+ if (!await this.recordEvent(event)) summary.activity_warning = true
1930
+ }
1931
+
1932
+ async deduplicateFiles(filePaths, progress = null) {
1933
+ const summary = Object.assign(this.deduplicationSummary(), {
1934
+ results: []
1935
+ })
1936
+ let representative = null
1937
+ let commonSourceId
1938
+ for (const filePath of new Set(filePaths.map((item) =>
1939
+ path.resolve(item)))) {
1940
+ try {
1941
+ const result = await this.convert(filePath)
1942
+ this.addDeduplicationResult(summary, result)
1943
+ if (result.status === "converted") {
1944
+ if (!representative) representative = result
1945
+ const sourceId = result.source_id || null
1946
+ commonSourceId = commonSourceId === undefined
1947
+ ? sourceId
1948
+ : commonSourceId === sourceId ? commonSourceId : null
1949
+ }
1950
+ summary.results.push({ path: filePath, status: result.status })
1951
+ } catch (error) {
1952
+ summary.failed += 1
1953
+ summary.results.push({ path: filePath, status: "failed" })
1954
+ } finally {
1955
+ if (progress) progress.files_completed += 1
1956
+ }
1957
+ }
1958
+ await this.recordDeduplicationSummary(
1959
+ summary,
1960
+ representative,
1961
+ commonSourceId
1962
+ )
1963
+ return summary
1964
+ }
1965
+
1966
+ async deduplicateScope(scopeId = null, options = {}) {
1967
+ const sourceIds = this.scopeSourceIds(scopeId)
1968
+ const summary = this.deduplicationSummary()
1969
+ if (scopeId && !sourceIds.length) return summary
1970
+ let cursor = ""
1971
+ while (true) {
1972
+ const rows = await this.registry.fileBatch(
1973
+ "duplicate", sourceIds, cursor, 100)
1974
+ if (!rows.length) break
1975
+ for (const row of rows) {
1976
+ cursor = row.path
1977
+ try {
1978
+ const result = await this.convert(row.path)
1979
+ this.addDeduplicationResult(summary, result)
1980
+ } catch (error) {
1981
+ summary.failed += 1
1982
+ }
1983
+ if (options.progress) options.progress.files_completed += 1
1984
+ }
1985
+ }
1986
+ if (summary.converted) {
1987
+ const scopedSource = scopeId
1988
+ ? this._sources.find((source) => source.id === scopeId)
1989
+ : null
1990
+ await this.recordDeduplicationSummary(
1991
+ summary,
1992
+ null,
1993
+ scopedSource && scopedSource.kind !== "virtual"
1994
+ ? scopeId
1995
+ : null
1996
+ )
1997
+ }
1998
+ return summary
1999
+ }
2000
+
2001
+ async copyOut(filePath, expected, source) {
2002
+ const temporary = `${filePath}${TMP_SUFFIX}`
2003
+ let temporaryStat = null
2004
+ try {
2005
+ await fs.promises.copyFile(
2006
+ filePath, temporary, fs.constants.COPYFILE_EXCL)
2007
+ if (Number.isFinite(expected.atimeMs) &&
2008
+ Number.isFinite(expected.mtimeMs)) {
2009
+ await fs.promises.utimes(
2010
+ temporary,
2011
+ new Date(expected.atimeMs),
2012
+ new Date(expected.mtimeMs)
2013
+ )
2014
+ }
2015
+ temporaryStat = await fs.promises.lstat(temporary)
2016
+ if (this.sourceAppIsRunning(source)) {
2017
+ await unlinkIfSame(temporary, temporaryStat)
2018
+ return { status: "locked" }
2019
+ }
2020
+ const [current, currentTemporary] = await Promise.all([
2021
+ lstatIfPresent(filePath),
2022
+ lstatIfPresent(temporary)
2023
+ ])
2024
+ if (!current ||
2025
+ !sameSnapshot(expected, current) ||
2026
+ !sameIdentity(temporaryStat, currentTemporary)) {
2027
+ await unlinkIfSame(temporary, temporaryStat)
2028
+ return { status: "stale" }
2029
+ }
2030
+ await fs.promises.rename(temporary, filePath)
2031
+ let finalStat
2032
+ try {
2033
+ finalStat = await fs.promises.lstat(filePath)
2034
+ } catch (error) {
2035
+ // rename() already committed the separate copy. Preserve that
2036
+ // successful action if only the post-commit metadata read failed.
2037
+ finalStat = temporaryStat
2038
+ }
2039
+ if (!sameIdentity(finalStat, temporaryStat)) {
2040
+ return { status: "stale" }
2041
+ }
2042
+ temporaryStat = null
2043
+ return {
2044
+ status: "copied",
2045
+ stat: finalStat
2046
+ }
2047
+ } catch (error) {
2048
+ if (temporaryStat) {
2049
+ await unlinkIfSame(temporary, temporaryStat).catch(() => {})
2050
+ }
2051
+ if (error && error.code === "EEXIST") return { status: "conflict" }
2052
+ if (isMissingError(error)) return { status: "stale" }
2053
+ if (LOCK_CODES.has(error && error.code)) return { status: "locked" }
2054
+ throw error
2055
+ }
2056
+ }
1082
2057
 
1083
- const before = await lstatIfPresent(targetPath)
1084
- if (!before || !before.isFile()) return { status: "stale" }
1085
- let hashed
1086
- try {
1087
- hashed = await this.hashFile(targetPath)
1088
- } catch (error) {
2058
+ async separate(filePath, options = {}) {
2059
+ const entry = await this.registry.getFile(filePath)
2060
+ if (!entry || entry.unavailable_reason === "stale") {
2061
+ return { status: "not-found" }
2062
+ }
2063
+ const source = this.sourceForPath(entry.path, entry.source_id)
2064
+ if (!source ||
2065
+ !await this.canonicalPathIsWithinSource(entry.path, source)) {
1089
2066
  return { status: "stale" }
1090
2067
  }
1091
- const after = await lstatIfPresent(targetPath)
1092
- if (!sameSnapshot(fileSnapshot(before), after) || hashed.size !== after.size) {
2068
+ const current = await lstatIfPresent(entry.path)
2069
+ if (!current ||
2070
+ !current.isFile() ||
2071
+ !sameSnapshot(rowSnapshot(entry), current)) {
1093
2072
  return { status: "stale" }
1094
2073
  }
1095
- if (!this.registry.blobs.has(hashed.hash)) return { status: "no-match" }
1096
- if (!await this.canonicalPathIsWithinSource(targetPath, source)) return { status: "stale" }
1097
- const current = await lstatIfPresent(targetPath)
1098
- if (!sameSnapshot(fileSnapshot(after), current)) return { status: "stale" }
1099
-
1100
- const result = await this.convert(targetPath, hashed.hash, {
1101
- app: source.kind === "app" ? source.app : null,
1102
- source_id: source.id,
1103
- batch_id: options.batch_id || `batch-${crypto.randomUUID()}`,
1104
- source,
1105
- expected: fileSnapshot(current)
1106
- })
1107
- if (result.status === "converted" || result.status === "already") {
1108
- this.registry.allowSharing(targetPath)
2074
+ if (entry.status !== "linked") return { status: "not-found" }
2075
+ if (this.sourceAppIsRunning(source)) return { status: "locked" }
2076
+ const copied = await this.copyOut(
2077
+ entry.path, current, source)
2078
+ if (copied.status !== "copied") return copied
2079
+ await this.registry.upsertFile(Object.assign(
2080
+ {},
2081
+ entry,
2082
+ fileSnapshot(copied.stat),
2083
+ {
2084
+ status: "linked",
2085
+ unavailable_reason: null
2086
+ }
2087
+ ))
2088
+ await this.refreshInodeSnapshots(entry.hash, entry.dev, entry.ino)
2089
+ if (options.reclassify !== false) {
2090
+ await this.reclassifyHashes([entry.hash])
1109
2091
  }
2092
+ const result = {
2093
+ status: "detached",
2094
+ bytes: entry.size,
2095
+ hash: entry.hash
2096
+ }
2097
+ if (options.recordActivity !== false && !await this.recordEvent({
2098
+ kind: "detach",
2099
+ hash: entry.hash,
2100
+ path: entry.path,
2101
+ app: entry.app,
2102
+ source_id: entry.source_id,
2103
+ bytes: entry.size
2104
+ })) result.activity_warning = true
1110
2105
  return result
1111
2106
  }
1112
- sourceIsWithinScope(source, scopeId) {
1113
- if (!scopeId) return true
1114
- const seen = new Set()
1115
- let current = source
1116
- while (current && !seen.has(current.id)) {
1117
- if (current.id === scopeId) return true
1118
- seen.add(current.id)
1119
- current = this._sources.find((item) => item.id === current.parent_id)
1120
- }
1121
- return false
1122
- }
1123
- async deduplicateSelection(selection, scopeId, options = {}) {
1124
- if (selection === "duplicates") {
1125
- return this.deduplicateScope(scopeId, Object.assign({}, options, { includeDescendants: true }))
1126
- }
1127
- await this.refreshSources()
1128
- if (scopeId && !this._sources.some((source) => source.id === scopeId)) {
1129
- return { error: "That location is no longer available. Scan again to refresh it." }
1130
- }
1131
- const paths = [...this.registry.excluded].filter(([filePath, entry]) => {
1132
- const source = this.sourceForPath(filePath, entry.source_id)
1133
- return source ? this.sourceIsWithinScope(source, scopeId) : !scopeId
1134
- }).map(([filePath]) => filePath)
1135
- if (options.progress) options.progress.files_total = paths.length
2107
+
2108
+ async separateFiles(filePaths, progress = null) {
1136
2109
  const summary = {
1137
- converted: 0, bytes_saved: 0, locked: 0, stale: 0, unmatched: 0,
1138
- incompatible: 0, unavailable: 0, failed: 0
1139
- }
1140
- const batch = options.batch_id || `batch-${crypto.randomUUID()}`
1141
- for (const filePath of paths) {
2110
+ separated: 0,
2111
+ bytes: 0,
2112
+ failed: 0,
2113
+ results: []
2114
+ }
2115
+ let separatedEntry = null
2116
+ let commonSourceId
2117
+ const affectedHashes = new Set()
2118
+ for (const filePath of new Set(filePaths.map((item) =>
2119
+ path.resolve(item)))) {
1142
2120
  try {
1143
- const result = await this.deduplicateExcludedFile(filePath, { batch_id: batch })
1144
- if (result.status === "converted" || result.status === "already") {
1145
- summary.converted += 1
1146
- summary.bytes_saved += result.bytes_saved || 0
1147
- } else if (result.status === "locked") {
1148
- summary.locked += 1
1149
- } else if (result.status === "stale") {
1150
- summary.stale += 1
1151
- } else if (result.status === "no-match" || result.status === "no-blob" ||
1152
- result.status === "size-mismatch") {
1153
- summary.unmatched += 1
1154
- } else if (result.status === "metadata-mismatch") {
1155
- summary.incompatible += 1
1156
- } else if (result.status === "unavailable" || result.status === "copy-mode") {
1157
- summary.unavailable += 1
2121
+ const entry = await this.registry.getFile(filePath)
2122
+ const result = entry && entry.status === "linked"
2123
+ ? await this.separate(filePath, {
2124
+ recordActivity: false,
2125
+ reclassify: false
2126
+ })
2127
+ : { status: "ineligible" }
2128
+ if (result.status === "detached") {
2129
+ summary.separated += 1
2130
+ summary.bytes += result.bytes || 0
2131
+ separatedEntry = entry
2132
+ affectedHashes.add(result.hash)
2133
+ const entrySourceId = entry.source_id || null
2134
+ commonSourceId = summary.separated === 1
2135
+ ? entrySourceId
2136
+ : commonSourceId === entrySourceId ? commonSourceId : null
1158
2137
  } else {
1159
2138
  summary.failed += 1
1160
2139
  }
2140
+ summary.results.push({ path: filePath, status: result.status })
1161
2141
  } catch (error) {
1162
2142
  summary.failed += 1
2143
+ summary.results.push({ path: filePath, status: "failed" })
2144
+ } finally {
2145
+ if (progress) progress.files_completed += 1
1163
2146
  }
1164
- if (options.progress) options.progress.files_completed += 1
1165
2147
  }
1166
- return summary
1167
- }
1168
- async deduplicateScope(scopeId, options = {}) {
1169
- await this.refreshSources()
1170
- const includeDescendants = options.includeDescendants === true
1171
- const source = scopeId
1172
- ? this._sources.find((item) => item.id === scopeId && (includeDescendants || item.kind !== "virtual"))
1173
- : null
1174
- if ((!includeDescendants || scopeId) && !source) {
1175
- return { error: "That location is no longer available. Scan again to refresh it." }
2148
+ if (affectedHashes.size) {
2149
+ await this.reclassifyHashes(affectedHashes)
1176
2150
  }
1177
- if (!includeDescendants) {
1178
- if (!source.shareable) return { error: "This location is on a disk that cannot share space with this vault." }
1179
- if (this.sourceAppIsRunning(source)) {
1180
- return { error: "This app has a running script. Stop it first, then deduplicate." }
1181
- }
1182
- }
1183
-
1184
- const registry = this.registry
1185
- const batch = options.batch_id || `batch-${crypto.randomUUID()}`
1186
- const summary = { converted: 0, bytes_saved: 0, locked: 0, stale: 0, incompatible: 0, unavailable: 0, failed: 0 }
1187
- const staleHashes = new Set()
1188
- const entries = [...registry.duplicates]
1189
- const matchesScope = (currentSource) => currentSource && currentSource.shareable &&
1190
- (includeDescendants ? this.sourceIsWithinScope(currentSource, scopeId) : currentSource.id === scopeId)
1191
- if (options.progress) {
1192
- options.progress.files_total = entries.reduce((total, [filePath, entry]) => {
1193
- const currentSource = this.sourceForPath(filePath, entry.source_id)
1194
- return total + (matchesScope(currentSource) ? 1 : 0)
1195
- }, 0)
1196
- }
1197
- const completeProgress = () => {
1198
- if (options.progress) options.progress.files_completed += 1
1199
- }
1200
- for (const [filePath, entry] of entries) {
1201
- const currentSource = this.sourceForPath(filePath, entry.source_id)
1202
- if (!matchesScope(currentSource)) continue
1203
- if (!await this.canonicalPathIsWithinSource(filePath, currentSource)) {
1204
- summary.stale += 1
1205
- completeProgress()
1206
- continue
1207
- }
1208
- const indexed = registry.scanIndex.get(filePath)
1209
- const expected = indexed && indexed.hash === entry.hash
1210
- ? indexed
1211
- : (entry.dev !== undefined && entry.ino !== undefined &&
1212
- entry.mtime !== undefined && entry.ctime !== undefined ? entry : null)
1213
- if (!expected) {
1214
- summary.stale += 1
1215
- completeProgress()
1216
- continue
1217
- }
1218
- if (staleHashes.has(entry.hash)) {
1219
- summary.stale += 1
1220
- completeProgress()
1221
- continue
1222
- }
1223
- try {
1224
- const result = await this.convert(filePath, entry.hash, {
1225
- app: currentSource.kind === "app" ? currentSource.app : (entry.app || null),
1226
- source_id: currentSource.id,
1227
- batch_id: batch,
1228
- source: currentSource,
1229
- expected
1230
- })
1231
- if (result.status === "converted" || result.status === "already") {
1232
- summary.converted += 1
1233
- summary.bytes_saved += result.bytes_saved || 0
1234
- } else if (result.status === "locked") {
1235
- summary.locked += 1
1236
- } else if (result.status === "stale") {
1237
- summary.stale += 1
1238
- } else if (result.status === "stale-blob") {
1239
- staleHashes.add(entry.hash)
1240
- summary.stale += 1
1241
- } else if (result.status === "metadata-mismatch") {
1242
- entry.unavailable_reason = "metadata"
1243
- summary.incompatible += 1
1244
- } else if (result.status === "unavailable" || result.status === "copy-mode") {
1245
- summary.unavailable += 1
1246
- } else {
1247
- summary.failed += 1
1248
- }
1249
- } catch (error) {
1250
- summary.failed += 1
1251
- }
1252
- completeProgress()
2151
+ const event = {
2152
+ kind: "detach",
2153
+ bytes: summary.bytes,
2154
+ files: summary.separated
2155
+ }
2156
+ if (summary.separated === 1 && separatedEntry) {
2157
+ event.hash = separatedEntry.hash
2158
+ event.path = separatedEntry.path
2159
+ event.app = separatedEntry.app
2160
+ event.source_id = separatedEntry.source_id
2161
+ } else if (commonSourceId) {
2162
+ event.source_id = commonSourceId
2163
+ }
2164
+ if (summary.separated && !await this.recordEvent(event)) {
2165
+ summary.activity_warning = true
1253
2166
  }
1254
- registry.schedulePersist()
1255
2167
  return summary
1256
2168
  }
1257
- // reclaim: delete an orphan's store name. Refuses when any app name remains.
1258
- async reclaim(hash, options = {}) {
1259
- if (!this.enabled) return { status: "disabled" }
1260
- const storePath = this.storePathFor(hash)
1261
- if (!this.registry.blobs.has(hash)) return { status: "not-found" }
1262
- const st = await this.storeStatIfPresent(storePath)
1263
- if (!st) {
1264
- const hasCopyNames = options.hasCopyNames === undefined
1265
- ? [...this.registry.links.values()].some((entry) => entry.hash === hash && entry.mode === "copy")
1266
- : options.hasCopyNames
1267
- if (hasCopyNames) return { status: "unavailable" }
1268
- if (!options.deferRegistry) this.registry.removeBlob(hash)
1269
- return options.deferRegistry ? { status: "gone", remove_hash: true } : { status: "gone" }
1270
- }
1271
- if (!st.isFile()) {
1272
- if (!options.deferRegistry) this.registry.removeBlob(hash)
1273
- return options.deferRegistry
1274
- ? { status: "invalid", remove_hash: true }
1275
- : { status: "invalid" }
1276
- }
1277
- if (st.nlink > 1) return { status: "in-use" }
1278
- await fs.promises.unlink(storePath)
1279
- if (!options.deferRegistry) this.registry.removeBlob(hash)
1280
- await this.recordEvent({ kind: "reclaim", hash, bytes_saved: st.size })
1281
- return options.deferRegistry
1282
- ? { status: "reclaimed", bytes_freed: st.size, remove_hash: true }
1283
- : { status: "reclaimed", bytes_freed: st.size }
1284
- }
1285
- // Startup registry verification (trigger 5): bounded by registry size,
1286
- // never a discovery walk. Prunes dead links, flags orphans, removes stray
1287
- // conversion tmp files, re-adopts blobs whose store name was deleted.
1288
- async verify(options = {}) {
1289
- if (!this.enabled || !this.registry) return
1290
- const preservePath = options.preservePath || (() => false)
1291
- const includePath = options.includePath || (() => true)
1292
- const handleAccessError = (error, filePath) => !!(
1293
- isAccessError(error) && options.onAccessError && options.onAccessError(error, filePath)
1294
- )
1295
- if (options.reconcile !== false) this.reconcileConfiguredSources()
1296
- for (const [linkPath, entry] of [...this.registry.links]) {
1297
- if (!includePath(linkPath)) continue
1298
- if (preservePath(linkPath)) continue
1299
- try {
1300
- const source = this.sourceForPath(linkPath, entry.source_id)
1301
- const managedPath = source && await this.canonicalPathIsWithinSource(linkPath, source, { strictErrors: true })
1302
- if (!managedPath) {
1303
- this.registry.untrack(linkPath)
1304
- continue
1305
- }
1306
- const st = await lstatIfPresent(linkPath)
1307
- if (!st || !st.isFile() || st.ino !== entry.ino || st.dev !== entry.dev) {
1308
- this.registry.untrack(linkPath)
1309
- }
1310
- } catch (error) {
1311
- if (!handleAccessError(error, linkPath)) throw error
1312
- }
1313
- }
1314
- for (const [dupPath, entry] of [...this.registry.duplicates]) {
1315
- if (!includePath(dupPath)) continue
1316
- if (preservePath(dupPath)) continue
1317
- let valid = false
1318
- try {
1319
- const source = this.sourceForPath(dupPath, entry.source_id)
1320
- const st = await lstatIfPresent(dupPath)
1321
- valid = !!(source && st && st.isFile() &&
1322
- await this.canonicalPathIsWithinSource(dupPath, source, { strictErrors: true }))
1323
- } catch (error) {
1324
- if (handleAccessError(error, dupPath)) continue
1325
- throw error
1326
- }
1327
- if (!valid) {
1328
- this.registry.untrack(dupPath)
1329
- continue
2169
+
2170
+ async separateMatchingFiles(selection, progress = null) {
2171
+ const summary = {
2172
+ separated: 0,
2173
+ bytes: 0,
2174
+ failed: 0,
2175
+ cancelled: false
2176
+ }
2177
+ let separatedEntry = null
2178
+ let commonSourceId
2179
+ let cursor = ""
2180
+ while (true) {
2181
+ if (this.fileActionCancelRequested) {
2182
+ summary.cancelled = true
2183
+ break
1330
2184
  }
1331
- const storeStat = await this.storeStatIfPresent(this.storePathFor(entry.hash))
1332
- if (storeStat && storeStat.isFile()) {
1333
- await unlinkIfSame(dupPath + TMP_SUFFIX, storeStat)
1334
- }
1335
- }
1336
- if (options.verifyBlobs === false) {
1337
- this.registry.schedulePersist()
1338
- return
1339
- }
1340
- const linksByHash = new Map()
1341
- for (const [linkPath, entry] of this.registry.links) {
1342
- if (!linksByHash.has(entry.hash)) linksByHash.set(entry.hash, [])
1343
- linksByHash.get(entry.hash).push([linkPath, entry])
1344
- }
1345
- const invalidBlobs = new Set()
1346
- for (const [hash, blob] of [...this.registry.blobs]) {
1347
- const storePath = this.storePathFor(hash)
1348
- const st = await this.storeStatIfPresent(storePath)
1349
- if (!st) {
1350
- const names = linksByHash.get(hash) || []
1351
- const copyNames = names.filter(([, entry]) => entry.mode === "copy")
1352
- // Deleting a name changes ctime, so trusted re-adoption compares the
1353
- // preserved identity, size, and mtime. Changed or legacy-unsnapshotted
1354
- // content waits for the next explicit scan instead of being assigned a
1355
- // stale hash filename during startup.
1356
- let readopted = false
1357
- let inaccessibleName = false
1358
- for (const [linkPath, entry] of names) {
1359
- if (entry.mode !== "link") continue
1360
- if (preservePath(linkPath)) {
1361
- inaccessibleName = true
1362
- continue
1363
- }
1364
- let source
1365
- let linkStat
1366
- try {
1367
- source = this.sourceForPath(linkPath, entry.source_id)
1368
- if (!source || !await this.canonicalPathIsWithinSource(linkPath, source, { strictErrors: true })) continue
1369
- linkStat = await lstatIfPresent(linkPath)
1370
- } catch (error) {
1371
- if (!handleAccessError(error, linkPath)) throw error
1372
- inaccessibleName = true
1373
- continue
1374
- }
1375
- const expected = this.registry.scanIndex.get(linkPath)
1376
- if (!linkStat || !expected || expected.hash !== hash ||
1377
- entry.dev !== linkStat.dev || entry.ino !== linkStat.ino ||
1378
- !sameContentState(expected, linkStat)) continue
1379
- const before = fileSnapshot(linkStat)
1380
- let created = false
1381
- try {
1382
- await this.storeStatIfPresent(storePath, { createParent: true })
1383
- await fs.promises.link(linkPath, storePath)
1384
- created = true
1385
- } catch (error) {
1386
- if (!error || error.code !== "EEXIST") throw error
1387
- const currentStore = await this.storeStatIfPresent(storePath)
1388
- const currentLink = await lstatIfPresent(linkPath)
1389
- if (!currentStore || !sameContentState(before, currentLink) ||
1390
- currentStore.dev !== linkStat.dev || currentStore.ino !== linkStat.ino) continue
1391
- }
1392
- const [currentLink, currentStore] = await Promise.all([
1393
- lstatIfPresent(linkPath),
1394
- this.storeStatIfPresent(storePath)
1395
- ])
1396
- if (!sameContentState(before, currentLink) || !currentStore ||
1397
- currentStore.dev !== linkStat.dev || currentStore.ino !== linkStat.ino) {
1398
- const ownsStoreName = created && currentStore && (
1399
- (currentStore.dev === linkStat.dev && currentStore.ino === linkStat.ino) ||
1400
- (currentLink && currentStore.dev === currentLink.dev && currentStore.ino === currentLink.ino)
1401
- )
1402
- if (ownsStoreName) await fs.promises.unlink(storePath)
1403
- continue
1404
- }
1405
- await this.refreshLinkSnapshots(hash, linkStat.dev, linkStat.ino)
1406
- readopted = true
2185
+ const rows = await this.registry.fileBatch(
2186
+ "linked",
2187
+ selection.sourceIds,
2188
+ cursor,
2189
+ 100,
2190
+ selection.query
2191
+ )
2192
+ if (!rows.length) break
2193
+ const affectedHashes = new Set()
2194
+ for (const row of rows) {
2195
+ if (this.fileActionCancelRequested) {
2196
+ summary.cancelled = true
1407
2197
  break
1408
2198
  }
1409
- // On a volume without file sharing support, copy-mode names are the
1410
- // durable content group. They remain useful for duplicate discovery
1411
- // even though there is no canonical file in the vault tree.
1412
- if (readopted || copyNames.length || inaccessibleName) {
1413
- blob.orphan = false
1414
- blob.verified_at = inaccessibleName && !readopted ? null : Date.now()
1415
- } else {
1416
- invalidBlobs.add(hash)
2199
+ cursor = row.path
2200
+ try {
2201
+ const entry = await this.registry.getFile(row.path)
2202
+ const result = entry && entry.status === "linked"
2203
+ ? await this.separate(row.path, {
2204
+ recordActivity: false,
2205
+ reclassify: false
2206
+ })
2207
+ : { status: "ineligible" }
2208
+ if (result.status === "detached") {
2209
+ summary.separated += 1
2210
+ summary.bytes += result.bytes || 0
2211
+ separatedEntry = entry
2212
+ affectedHashes.add(result.hash)
2213
+ const entrySourceId = entry.source_id || null
2214
+ commonSourceId = summary.separated === 1
2215
+ ? entrySourceId
2216
+ : commonSourceId === entrySourceId ? commonSourceId : null
2217
+ } else {
2218
+ summary.failed += 1
2219
+ }
2220
+ } catch (error) {
2221
+ summary.failed += 1
2222
+ } finally {
2223
+ if (progress) progress.files_completed += 1
1417
2224
  }
1418
- continue
1419
2225
  }
1420
- if (!st.isFile()) {
1421
- invalidBlobs.add(hash)
1422
- continue
2226
+ if (affectedHashes.size) {
2227
+ await this.reclassifyHashes(affectedHashes)
1423
2228
  }
1424
- blob.orphan = st.nlink === 1
1425
- blob.verified_at = Date.now()
1426
- }
1427
- this.registry.removeBlobs(invalidBlobs)
1428
- this.registry.schedulePersist()
1429
- }
1430
- // Rebuild derived state from disk: store filenames are hashes, stat gives
1431
- // nlink, and (dev, ino) matching re-associates app paths. The replacement
1432
- // maps are assembled off to the side and swapped in only after the walk so
1433
- // a long repair never persists a half-reset registry.
1434
- async rebuild(roots, options = {}) {
1435
- if (!this.enabled) return
1436
- const registry = this.registry
1437
- const preserveState = options.preserveState !== false
1438
- const preserved = preserveState ? {
1439
- blobs: new Map(registry.blobs),
1440
- links: new Map(registry.links),
1441
- excluded: new Map(registry.excluded),
1442
- totals: Object.assign({}, registry.totals),
1443
- lastScan: registry.lastScan ? Object.assign({}, registry.lastScan) : null,
1444
- sourceScans: new Map([...registry.sourceScans].map(([id, scan]) => [id, Object.assign({}, scan)])),
1445
- duplicates: new Map(registry.duplicates),
1446
- scanIndex: new Map(registry.scanIndex)
1447
- } : null
1448
- const rebuiltBlobs = new Map()
1449
- const rebuiltLinks = new Map()
1450
- const storeInoToHash = new Map()
1451
- let shards = []
2229
+ if (summary.cancelled) break
2230
+ }
2231
+ const event = {
2232
+ kind: "detach",
2233
+ bytes: summary.bytes,
2234
+ files: summary.separated
2235
+ }
2236
+ if (summary.separated === 1 && separatedEntry) {
2237
+ event.hash = separatedEntry.hash
2238
+ event.path = separatedEntry.path
2239
+ event.app = separatedEntry.app
2240
+ event.source_id = separatedEntry.source_id
2241
+ } else if (commonSourceId) {
2242
+ event.source_id = commonSourceId
2243
+ }
2244
+ if (summary.separated && !await this.recordEvent(event)) {
2245
+ summary.activity_warning = true
2246
+ }
2247
+ return summary
2248
+ }
2249
+
2250
+ async reclaim(hash, storeId = null) {
2251
+ if (typeof hash !== "string" || !SHA256_RE.test(hash)) {
2252
+ return { status: "not-found" }
2253
+ }
2254
+ const anchors = await this.registry.anchorsForHash(hash)
2255
+ const anchor = storeId
2256
+ ? anchors.find((item) => item.store_id === storeId)
2257
+ : anchors.length === 1 ? anchors[0] : null
2258
+ if (!anchor) return { status: "not-found" }
2259
+ const store = this._anchorStoresById.get(anchor.store_id)
2260
+ if (!store) return { status: "unavailable" }
2261
+ const storePath = this.storePathFor(hash, store.id)
2262
+ if (!samePath(storePath, anchor.path)) return { status: "stale" }
1452
2263
  try {
1453
- shards = await fs.promises.readdir(this.blobRoot, { withFileTypes: true })
2264
+ await this.validateAnchorStore(store, anchor.dev)
1454
2265
  } catch (error) {
1455
- if (!isMissingError(error)) throw error
2266
+ return { status: "stale" }
1456
2267
  }
1457
- for (const shardEntry of shards) {
1458
- if (!shardEntry.isDirectory() || shardEntry.isSymbolicLink() ||
1459
- !/^[0-9a-f]{2}$/.test(shardEntry.name)) continue
1460
- const shard = shardEntry.name
1461
- const shardPath = path.resolve(this.blobRoot, shard)
1462
- if (!await this.directoryIfSafe(shardPath)) continue
1463
- let names = []
1464
- try {
1465
- names = await fs.promises.readdir(shardPath)
1466
- } catch (error) {
1467
- if (isMissingError(error)) continue
1468
- throw error
1469
- }
1470
- for (const name of names) {
1471
- if (!SHA256_RE.test(name) || !name.startsWith(shard)) continue
2268
+ const stat = await this.storeStatIfPresent(storePath, {
2269
+ store_id: store.id
2270
+ })
2271
+ if (!stat) {
2272
+ await this.registry.removeAnchor(store.id, hash)
2273
+ return { status: "gone" }
2274
+ }
2275
+ const expected = {
2276
+ size: anchor.size,
2277
+ mtime: anchor.mtime,
2278
+ ctime: anchor.ctime,
2279
+ dev: anchor.dev,
2280
+ ino: anchor.ino
2281
+ }
2282
+ if (!sameSnapshot(expected, stat)) return { status: "stale" }
2283
+ if (stat.nlink !== 1) return { status: "in-use" }
2284
+ await fs.promises.unlink(storePath)
2285
+ await this.registry.removeAnchor(store.id, hash)
2286
+ const hasFiles = await this.registry.hasFilesForHash(hash)
2287
+ const remainingAnchors = await this.registry.anchorsForHash(hash)
2288
+ if (!hasFiles && !remainingAnchors.length) {
2289
+ await this.registry.removeContent(hash)
2290
+ }
2291
+ const result = { status: "reclaimed", bytes_freed: stat.size }
2292
+ if (!await this.recordEvent({
2293
+ kind: "reclaim",
2294
+ hash,
2295
+ bytes: stat.size
2296
+ })) result.activity_warning = true
2297
+ return result
2298
+ }
2299
+
2300
+ async reclaimAll() {
2301
+ const summary = { reclaimed: 0, bytes_freed: 0, failed: 0 }
2302
+ let cursor = { store_id: "", hash: "" }
2303
+ while (true) {
2304
+ const anchors = await this.registry.reclaimableBatch(cursor, 100)
2305
+ if (!anchors.length) break
2306
+ for (const row of anchors) {
2307
+ cursor = { store_id: row.store_id, hash: row.hash }
1472
2308
  try {
1473
- const st = await fs.promises.lstat(path.resolve(this.blobRoot, shard, name))
1474
- if (!st.isFile()) continue
1475
- const previous = registry.blobs.get(name)
1476
- rebuiltBlobs.set(name, {
1477
- size: st.size,
1478
- first_seen: previous && previous.first_seen ? previous.first_seen : Date.now(),
1479
- source_urls: previous && Array.isArray(previous.source_urls) ? [...previous.source_urls] : [],
1480
- verified_at: Date.now(),
1481
- orphan: st.nlink === 1
1482
- })
1483
- storeInoToHash.set(`${st.dev}:${st.ino}`, name)
2309
+ const result = await this.reclaim(row.hash, row.store_id)
2310
+ if (result.status === "reclaimed") {
2311
+ summary.reclaimed += 1
2312
+ summary.bytes_freed += result.bytes_freed || 0
2313
+ if (result.activity_warning) summary.activity_warning = true
2314
+ } else if (!["gone", "in-use"].includes(result.status)) {
2315
+ summary.failed += 1
2316
+ }
1484
2317
  } catch (error) {
1485
- if (!isMissingError(error)) throw error
2318
+ summary.failed += 1
1486
2319
  }
1487
2320
  }
1488
2321
  }
1489
- await this.refreshSources()
1490
- const walkRoots = roots
1491
- ? roots.map((root) => typeof root === "string" ? { root, source_id: null } : root)
1492
- : this.scanRoots()
1493
- for (const entry of walkRoots) {
1494
- await this.walkForInoMatch(
1495
- entry.root,
1496
- storeInoToHash,
1497
- entry.source_id === "pinokio" ? null : entry.source_id,
1498
- rebuiltLinks
1499
- )
1500
- }
1501
- const rebuiltDuplicates = new Map()
1502
- const rebuiltScanIndex = new Map()
1503
- const trustedHashes = new Set()
1504
- if (preserved) {
1505
- // Copy-mode content has no store inode to rediscover. Preserve only names
1506
- // whose exact scan snapshot still proves the recorded content group.
1507
- for (const [linkPath, link] of preserved.links) {
1508
- if (link.mode !== "copy" || preserved.excluded.has(linkPath)) continue
1509
- const blob = preserved.blobs.get(link.hash)
1510
- const expected = preserved.scanIndex.get(linkPath)
1511
- const source = this.sourceForPath(linkPath, link.source_id)
1512
- if (!blob || !expected || expected.hash !== link.hash || !source ||
1513
- !await this.canonicalPathIsWithinSource(linkPath, source, { strictErrors: true })) continue
1514
- try {
1515
- const st = await fs.promises.lstat(linkPath)
1516
- if (!st.isFile() || st.size < this.sizeThreshold ||
1517
- link.dev !== st.dev || link.ino !== st.ino ||
1518
- !sameSnapshot(expected, st)) continue
1519
- if (!rebuiltBlobs.has(link.hash)) {
1520
- rebuiltBlobs.set(link.hash, Object.assign({}, blob, {
1521
- size: st.size,
1522
- verified_at: Date.now(),
1523
- orphan: false
1524
- }))
1525
- }
1526
- rebuiltLinks.set(linkPath, Object.assign({}, link, {
1527
- app: source.kind === "app" ? source.app : (link.app || null),
1528
- source_id: source.id,
1529
- dev: st.dev,
1530
- ino: st.ino,
1531
- mode: "copy"
1532
- }))
1533
- } catch (error) {
1534
- if (!isMissingError(error)) throw error
1535
- }
2322
+ return summary
2323
+ }
2324
+
2325
+ async scanForScope(scopeId) {
2326
+ const direct = await this.registry.scanFor(scopeId)
2327
+ const key = scopeId || ""
2328
+ if (direct || !scopeId) {
2329
+ if (direct) this.lastScanCache.set(key, direct)
2330
+ else this.lastScanCache.delete(key)
2331
+ return direct
2332
+ }
2333
+ const global = await this.registry.scanFor()
2334
+ if (global) this.lastScanCache.set("", global)
2335
+ else this.lastScanCache.delete("")
2336
+ if (!global ||
2337
+ !global.source_files ||
2338
+ !Object.prototype.hasOwnProperty.call(
2339
+ global.source_files, scopeId)) {
2340
+ this.lastScanCache.delete(key)
2341
+ return null
2342
+ }
2343
+ const scoped = Object.assign({}, global, {
2344
+ files: global.source_files[scopeId] || 0,
2345
+ bytes_total: global.source_bytes[scopeId] || 0,
2346
+ hash_failures: global.source_hash_failures
2347
+ ? global.source_hash_failures[scopeId] || 0
2348
+ : 0
2349
+ })
2350
+ const sourceIds = new Set(this.scopeSourceIds(scopeId))
2351
+ scoped.exclusions = Array.isArray(global.exclusions)
2352
+ ? global.exclusions.filter((entry) =>
2353
+ entry && sourceIds.has(entry.source_id))
2354
+ : []
2355
+ scoped.partial = scoped.exclusions.length > 0
2356
+ scoped.outcome = scoped.partial
2357
+ ? "completed_with_exclusions"
2358
+ : "complete"
2359
+ this.lastScanCache.set(key, scoped)
2360
+ return scoped
2361
+ }
2362
+
2363
+ sourceCountMaps(summaryRows) {
2364
+ const counts = {
2365
+ all: {},
2366
+ duplicates: {},
2367
+ shareable: {}
2368
+ }
2369
+ const add = (target, sourceIdValue, amount) => {
2370
+ let current = this._sourcesById.get(sourceIdValue)
2371
+ const seen = new Set()
2372
+ while (current && !seen.has(current.id)) {
2373
+ target[current.id] = (target[current.id] || 0) + amount
2374
+ seen.add(current.id)
2375
+ current = current.parent_id
2376
+ ? this._sourcesById.get(current.parent_id)
2377
+ : null
1536
2378
  }
1537
- // A store filename is only a trustworthy content hash when at least one
1538
- // surviving name still matches the snapshot captured when that content
1539
- // was hashed. Repair itself never hashes, so unverified names are left
1540
- // without a scan-index entry and the next manual scan hashes them once.
1541
- for (const [linkPath, link] of rebuiltLinks) {
1542
- const expected = preserved.scanIndex.get(linkPath)
1543
- if (!expected || expected.hash !== link.hash) continue
1544
- try {
1545
- const st = await fs.promises.lstat(linkPath)
1546
- if (sameSnapshot(expected, st)) trustedHashes.add(link.hash)
1547
- } catch (error) {
1548
- if (!isMissingError(error)) throw error
1549
- }
2379
+ }
2380
+ for (const row of summaryRows) {
2381
+ const amount = Number(row.file_count) || 0
2382
+ if (!["reference", "duplicate", "linked", "unavailable"]
2383
+ .includes(row.status)) continue
2384
+ add(counts.all, row.source_id, amount)
2385
+ if (row.status === "duplicate" || row.status === "unavailable") {
2386
+ add(counts.duplicates, row.source_id, amount)
1550
2387
  }
1551
- for (const [linkPath, link] of rebuiltLinks) {
1552
- if (!trustedHashes.has(link.hash)) continue
1553
- try {
1554
- const st = await fs.promises.lstat(linkPath)
1555
- rebuiltScanIndex.set(linkPath, {
1556
- hash: link.hash, size: st.size, dev: st.dev, ino: st.ino,
1557
- mtime: st.mtimeMs, ctime: st.ctimeMs,
1558
- source_id: link.source_id || null
1559
- })
1560
- } catch (error) {
1561
- if (!isMissingError(error)) throw error
1562
- }
2388
+ if (row.status === "duplicate") {
2389
+ add(counts.shareable, row.source_id, amount)
1563
2390
  }
1564
- for (const [duplicatePath, duplicate] of preserved.duplicates) {
1565
- if (preserved.excluded.has(duplicatePath) || rebuiltLinks.has(duplicatePath) ||
1566
- !rebuiltBlobs.has(duplicate.hash) || !trustedHashes.has(duplicate.hash) ||
1567
- !this.sourceForPath(duplicatePath, duplicate.source_id)) continue
1568
- try {
1569
- const source = this.sourceForPath(duplicatePath, duplicate.source_id)
1570
- if (!await this.canonicalPathIsWithinSource(duplicatePath, source, { strictErrors: true })) continue
1571
- const st = await fs.promises.lstat(duplicatePath)
1572
- const indexed = preserved.scanIndex.get(duplicatePath)
1573
- const expected = indexed && indexed.hash === duplicate.hash ? indexed : duplicate
1574
- if (st.isFile() && sameSnapshot(expected, st)) {
1575
- rebuiltDuplicates.set(duplicatePath, Object.assign({}, duplicate, {
1576
- size: st.size, dev: st.dev, ino: st.ino, mtime: st.mtimeMs, ctime: st.ctimeMs
1577
- }))
1578
- rebuiltScanIndex.set(duplicatePath, {
1579
- hash: duplicate.hash, size: st.size, dev: st.dev, ino: st.ino,
1580
- mtime: st.mtimeMs, ctime: st.ctimeMs,
1581
- source_id: duplicate.source_id || expected.source_id || null
1582
- })
1583
- }
1584
- } catch (error) {
1585
- if (!isMissingError(error)) throw error
1586
- }
2391
+ }
2392
+ return counts
2393
+ }
2394
+
2395
+ publicFileItems(page) {
2396
+ const hashSiblings = new Map()
2397
+ for (const sibling of page.hashSiblings || []) {
2398
+ if (!hashSiblings.has(sibling.hash)) {
2399
+ hashSiblings.set(sibling.hash, [])
2400
+ }
2401
+ hashSiblings.get(sibling.hash).push(sibling)
2402
+ }
2403
+ const inodeSiblings = new Map()
2404
+ for (const sibling of page.inodeSiblings || []) {
2405
+ const key = `${sibling.dev}:${sibling.ino}`
2406
+ if (!inodeSiblings.has(key)) inodeSiblings.set(key, [])
2407
+ inodeSiblings.get(key).push(sibling)
2408
+ }
2409
+ return (page.rows || []).map((row) => {
2410
+ const sampledMatches = row.status === "linked"
2411
+ ? inodeSiblings.get(`${row.dev}:${row.ino}`) || []
2412
+ : hashSiblings.get(row.hash) || []
2413
+ const allMatches = [
2414
+ row,
2415
+ ...sampledMatches.filter((match) => match.path !== row.path)
2416
+ ]
2417
+ const locations = allMatches.map((match) =>
2418
+ Object.assign({
2419
+ path: match.path,
2420
+ app: match.app,
2421
+ dev: match.dev,
2422
+ ino: match.ino
2423
+ }, this.locationForPath(match.path, match.source_id)))
2424
+ const publicStatus = {
2425
+ reference: "tracked",
2426
+ duplicate: "duplicate",
2427
+ unavailable: "duplicate",
2428
+ linked: "shared"
2429
+ }[row.status]
2430
+ const result = Object.assign({
2431
+ path: row.path,
2432
+ hash: row.hash,
2433
+ size: row.size,
2434
+ app: row.app,
2435
+ status: publicStatus,
2436
+ shareable: row.status === "duplicate",
2437
+ unavailable_reason: row.status === "unavailable"
2438
+ ? row.unavailable_reason || "different_disk"
2439
+ : null,
2440
+ location_count: sampledMatches.length
2441
+ ? Number(sampledMatches[0].location_count) || 1
2442
+ : 1,
2443
+ locations
2444
+ }, this.locationForPath(row.path, row.source_id))
2445
+ if (row.status === "duplicate" || row.status === "unavailable") {
2446
+ const match = allMatches.find((candidate) =>
2447
+ candidate.path !== row.path)
2448
+ result.match = match
2449
+ ? Object.assign({
2450
+ path: match.path,
2451
+ app: match.app
2452
+ }, this.locationForPath(match.path, match.source_id))
2453
+ : null
1587
2454
  }
1588
- for (const excludedPath of preserved.excluded.keys()) {
1589
- rebuiltLinks.delete(excludedPath)
1590
- rebuiltScanIndex.delete(excludedPath)
1591
- }
1592
- }
1593
- const rebuiltByIno = new Map()
1594
- const rebuiltPathsByIno = new Map()
1595
- for (const [linkPath, entry] of rebuiltLinks) {
1596
- const key = registry.inoKey(entry.dev, entry.ino)
1597
- rebuiltByIno.set(key, entry.hash)
1598
- if (!rebuiltPathsByIno.has(key)) rebuiltPathsByIno.set(key, new Set())
1599
- rebuiltPathsByIno.get(key).add(linkPath)
1600
- }
1601
- registry.replaceState({
1602
- blobs: rebuiltBlobs,
1603
- links: rebuiltLinks,
1604
- scanIndex: rebuiltScanIndex,
1605
- duplicates: rebuiltDuplicates,
1606
- excluded: preserved ? preserved.excluded : new Map(),
1607
- lastScan: preserved ? preserved.lastScan : null,
1608
- sourceScans: preserved ? preserved.sourceScans : new Map(),
1609
- totals: preserved ? preserved.totals : { lifetime_bytes_saved: 0 },
1610
- byIno: rebuiltByIno,
1611
- pathsByIno: rebuiltPathsByIno
2455
+ return result
1612
2456
  })
1613
- if (options.flush !== false) await registry.flush()
1614
- }
1615
- async walkForInoMatch(root, storeInoToHash, preferredSourceId = null, outputLinks = null) {
1616
- const vaultRoot = path.resolve(this.root)
1617
- for await (const batch of walkBatches(root, {
1618
- concurrency: this.dirConcurrency,
1619
- skipDirectory: (full) => full === vaultRoot,
1620
- strictErrors: true
1621
- })) {
1622
- const files = batch.flatMap((group) => group.files.map((file) => file.path))
1623
- const stats = await statMany(files, this.statConcurrency, null, {
1624
- strictErrors: true,
1625
- followSymlinks: false
1626
- })
1627
- for (let index = 0; index < files.length; index++) {
1628
- const st = stats[index]
1629
- if (!st || !st.isFile() || st.size < this.sizeThreshold) continue
1630
- const hash = storeInoToHash.get(`${st.dev}:${st.ino}`)
1631
- if (hash) {
1632
- const source = this.sourceForPath(files[index], preferredSourceId)
1633
- const previous = this.registry.links.get(files[index])
1634
- const batchId = previous && previous.hash === hash && previous.mode === "link" &&
1635
- previous.dev === st.dev && previous.ino === st.ino
1636
- ? previous.batch_id || null
1637
- : null
1638
- const link = {
1639
- hash, app: source && source.kind === "app" ? source.app : null,
1640
- source_id: source ? source.id : null, dev: st.dev, ino: st.ino,
1641
- mode: "link", created: Date.now(),
1642
- batch_id: batchId
1643
- }
1644
- if (outputLinks) outputLinks.set(files[index], link)
1645
- else this.registry.addLink(files[index], link)
1646
- }
1647
- }
2457
+ }
2458
+
2459
+ publicDuplicateGroupItems(page) {
2460
+ return (page.rows || []).map((row) => Object.assign({
2461
+ kind: "duplicate_group",
2462
+ hash: row.hash,
2463
+ path: row.representative_path,
2464
+ size: Number(row.size) || 0,
2465
+ total_count: Number(row.total_count) || 0,
2466
+ eligible_count: Number(row.eligible_count) || 0,
2467
+ can_save: Number(row.can_save) || 0,
2468
+ app: row.representative_app || null
2469
+ }, this.locationForPath(
2470
+ row.representative_path,
2471
+ row.representative_source_id
2472
+ )))
2473
+ }
2474
+
2475
+ publicDuplicateChildItems(rows) {
2476
+ const publicStatus = {
2477
+ reference: "tracked",
2478
+ duplicate: "duplicate",
2479
+ unavailable: "duplicate",
2480
+ linked: "shared"
2481
+ }
2482
+ return (rows || []).map((row) => Object.assign({
2483
+ kind: "duplicate_path",
2484
+ path: row.path,
2485
+ hash: row.hash,
2486
+ size: Number(row.size) || 0,
2487
+ app: row.app || null,
2488
+ status: publicStatus[row.status],
2489
+ registry_status: row.status,
2490
+ shareable: row.status === "duplicate",
2491
+ selectable: !!row.selectable,
2492
+ unavailable_reason: row.status === "unavailable"
2493
+ ? row.unavailable_reason || "different_disk"
2494
+ : null
2495
+ }, this.locationForPath(row.path, row.source_id)))
2496
+ }
2497
+
2498
+ async duplicateGroupChildren(scopeId, hash, options = {}) {
2499
+ if (typeof hash !== "string" || !SHA256_RE.test(hash)) {
2500
+ throw new TypeError("Invalid vault content identifier.")
2501
+ }
2502
+ const locationId = typeof options.location_id === "string" &&
2503
+ options.location_id
2504
+ ? options.location_id
2505
+ : null
2506
+ const query = String(options.query || "").slice(0, 500).trim()
2507
+ const cursor = typeof options.cursor === "string"
2508
+ ? options.cursor.slice(0, 2048)
2509
+ : ""
2510
+ const pageSize = boundedInteger(
2511
+ options.page_size, 100, 1, STATUS_PAGE_SIZE)
2512
+ const result = await this.registry.duplicateGroupChildren({
2513
+ hash,
2514
+ sourceIds: this.scopeSourceIds(scopeId),
2515
+ activeSourceIds: this.scopeSourceIds(scopeId, locationId),
2516
+ query,
2517
+ cursor,
2518
+ pageSize,
2519
+ unrestricted: !scopeId,
2520
+ activeUnrestricted: !scopeId && !locationId
2521
+ })
2522
+ return {
2523
+ hash,
2524
+ items: this.publicDuplicateChildItems(result.rows),
2525
+ total: Number(result.total) || 0,
2526
+ next_cursor: result.nextCursor || null
1648
2527
  }
1649
2528
  }
1650
- cloudSyncProvider() {
1651
- const home = path.resolve(this.kernel.homedir || "").replace(/\\/g, "/").toLowerCase()
1652
- if (home.includes("/onedrive") || home.includes("/one drive")) return "OneDrive"
1653
- if (home.includes("/dropbox")) return "Dropbox"
1654
- if (home.includes("/library/mobile documents/") || home.includes("/icloud drive/")) return "iCloud Drive"
1655
- return null
2529
+
2530
+ async duplicateGroupSelection(scopeId, hash, options = {}) {
2531
+ if (typeof hash !== "string" || !SHA256_RE.test(hash)) {
2532
+ throw new TypeError("Invalid vault content identifier.")
2533
+ }
2534
+ const locationId = typeof options.location_id === "string" &&
2535
+ options.location_id
2536
+ ? options.location_id
2537
+ : null
2538
+ const result = await this.registry.duplicateGroupSelection({
2539
+ hash,
2540
+ sourceIds: this.scopeSourceIds(scopeId, locationId),
2541
+ query: String(options.query || "").slice(0, 500).trim(),
2542
+ unrestricted: !scopeId && !locationId
2543
+ })
2544
+ return {
2545
+ hash,
2546
+ paths: result.paths || [],
2547
+ exceeded: !!result.exceeded
2548
+ }
1656
2549
  }
1657
- // Global state for the vault page and the health endpoint. Answered from
1658
- // memory + one stat per blob and occupied shard; never a walk.
1659
- async status(scopeId = null) {
2550
+
2551
+ async status(scopeId = null, options = {}) {
1660
2552
  if (!this.enabled || !this.registry) return { enabled: false }
1661
- const registry = this.registry
1662
- const scope = scopeId ? this.scanSource(scopeId) : null
1663
- if (scopeId && !scope) throw new Error("That location is no longer available.")
1664
- const scopeHashes = scopeId ? new Set() : null
1665
- if (scopeHashes) {
1666
- for (const entry of registry.links.values()) {
1667
- if (entry.source_id === scopeId) scopeHashes.add(entry.hash)
1668
- }
1669
- for (const entry of registry.duplicates.values()) {
1670
- if (entry.source_id === scopeId) scopeHashes.add(entry.hash)
1671
- }
1672
- }
1673
- const blobs = []
1674
- const blobByHash = new Map()
1675
- const storeStats = new Map()
1676
- const namesByHash = new Map()
1677
- const undoBatchMap = new Map()
1678
- for (const [linkPath, entry] of registry.links) {
1679
- if (scopeHashes && !scopeHashes.has(entry.hash)) continue
1680
- if (!namesByHash.has(entry.hash)) namesByHash.set(entry.hash, [])
1681
- const location = this.locationForPath(linkPath, entry.source_id)
1682
- namesByHash.get(entry.hash).push(Object.assign({
1683
- path: linkPath, app: entry.app || null, mode: entry.mode
1684
- }, location))
1685
- if (entry.batch_id && (!scopeId || entry.source_id === scopeId)) {
1686
- if (!undoBatchMap.has(entry.batch_id)) {
1687
- undoBatchMap.set(entry.batch_id, { batch_id: entry.batch_id, files: 0, bytes: 0, ts: null })
1688
- }
1689
- const batch = undoBatchMap.get(entry.batch_id)
1690
- const blob = registry.blobs.get(entry.hash)
1691
- batch.files += 1
1692
- batch.bytes += blob && Number.isFinite(blob.size) ? blob.size : 0
1693
- }
1694
- }
1695
- const pendingBytesByHash = new Map()
1696
- for (const entry of registry.duplicates.values()) {
1697
- if (scopeId && entry.source_id !== scopeId) continue
1698
- pendingBytesByHash.set(entry.hash, (pendingBytesByHash.get(entry.hash) || 0) + (entry.size || 0))
1699
- }
1700
- const blobEntries = [...registry.blobs].filter(([hash]) => !scopeHashes || scopeHashes.has(hash))
1701
- if (!await this.directoryIfSafe(this.root) || !await this.directoryIfSafe(this.blobRoot)) {
1702
- throw unsafeStoragePath(this.blobRoot)
1703
- }
1704
- const storePaths = blobEntries.map(([hash]) => this.storePathFor(hash))
1705
- await Promise.all([...new Set(storePaths.map((storePath) => path.dirname(storePath)))]
1706
- .map((shard) => this.directoryIfSafe(shard)))
1707
- const blobStats = await statMany(
1708
- storePaths,
1709
- this.statConcurrency,
1710
- null,
1711
- { strictErrors: true, followSymlinks: false }
2553
+ const view = STATUS_VIEWS.has(options.view) ? options.view : "all"
2554
+ const groupDuplicates =
2555
+ view === "duplicates" && options.display_mode === "files"
2556
+ const statusFilter = STATUS_FILTERS.has(options.status_filter)
2557
+ ? options.status_filter
2558
+ : "all"
2559
+ const query = String(options.query || "").slice(0, 500).trim()
2560
+ const cursor = typeof options.cursor === "string"
2561
+ ? options.cursor.slice(0, 2048)
2562
+ : ""
2563
+ const pageSize = boundedInteger(
2564
+ options.page_size, STATUS_PAGE_SIZE, 1, STATUS_PAGE_SIZE)
2565
+ const requestedPage = boundedInteger(
2566
+ options.page, 0, 0, Number.MAX_SAFE_INTEGER)
2567
+ const locationId = typeof options.location_id === "string" &&
2568
+ options.location_id
2569
+ ? options.location_id
2570
+ : null
2571
+ const scopeSourceIds = this.scopeSourceIds(scopeId)
2572
+ const locationSourceIds = this.scopeSourceIds(
2573
+ scopeId,
2574
+ locationId
1712
2575
  )
1713
- let bytesOnDisk = 0
1714
- let wouldBe = 0
1715
- let trackedLogicalBytes = 0
1716
- for (let index = 0; index < blobEntries.length; index++) {
1717
- const [hash, blob] = blobEntries[index]
1718
- const names = namesByHash.get(hash) || []
1719
- const apps = new Set(names.map((name) => name.app).filter(Boolean))
1720
- const storeStat = blobStats[index] && blobStats[index].isFile() ? blobStats[index] : null
1721
- const nlink = storeStat ? storeStat.nlink : null
1722
- storeStats.set(hash, storeStat)
1723
- const size = blob.size || 0
1724
- const linkedCopy = storeStat || names.some((name) => name.mode === "link") ? 1 : 0
1725
- const physicalCopies = linkedCopy + names.filter((name) => name.mode === "copy").length
1726
- const pendingBytes = pendingBytesByHash.get(hash) || 0
1727
- trackedLogicalBytes += (size * names.length) + pendingBytes
1728
- bytesOnDisk += (size * physicalCopies) + pendingBytes
1729
- wouldBe += (size * Math.max(physicalCopies, names.length)) + pendingBytes
1730
- const orphan = !!(storeStat && storeStat.nlink === 1)
1731
- const publicBlob = {
1732
- hash, size: blob.size || 0, orphan, nlink,
1733
- names, apps: [...apps], source_urls: blob.source_urls || []
1734
- }
1735
- blobs.push(publicBlob)
1736
- blobByHash.set(hash, publicBlob)
1737
- }
1738
- if (!scopeId) {
1739
- // The scan total covers every regular file, including files below the
1740
- // candidate threshold and files the user kept separate. The registry
1741
- // figures above cover only content tracked by Save space. Add the
1742
- // remainder to both sides so the global comparison represents the
1743
- // complete scanned locations instead of silently omitting small files.
1744
- const lastScan = registry.scanFor()
1745
- const scannedBytes = lastScan && Number.isFinite(lastScan.bytes_total)
1746
- ? lastScan.bytes_total
1747
- : null
1748
- const untrackedScannedBytes = scannedBytes === null
1749
- ? 0
1750
- : Math.max(0, scannedBytes - trackedLogicalBytes)
1751
- bytesOnDisk += untrackedScannedBytes
1752
- wouldBe += untrackedScannedBytes
1753
- }
1754
- const duplicates = []
1755
- for (const [p, entry] of registry.duplicates) {
1756
- if (scopeId && entry.source_id !== scopeId) continue
1757
- const location = this.locationForPath(p, entry.source_id)
1758
- const source = this.sourceForPath(p, location.source_id)
1759
- const index = registry.scanIndex.get(p)
1760
- const storeStat = storeStats.get(entry.hash)
1761
- const shareable = !!(source && source.shareable && storeStat && !entry.unavailable_reason &&
1762
- (!index || index.dev === undefined || index.dev === storeStat.dev))
1763
- let unavailableReason = entry.unavailable_reason || null
1764
- if (!shareable && !unavailableReason) {
1765
- unavailableReason = source && storeStat && index && index.dev !== undefined && index.dev !== storeStat.dev
1766
- ? "different_disk"
1767
- : "unsupported_disk"
1768
- }
1769
- const blob = blobByHash.get(entry.hash)
1770
- const match = blob ? blob.names.find((name) => name.path !== p) || null : null
1771
- duplicates.push(Object.assign({
1772
- path: p, hash: entry.hash, size: entry.size || 0, app: entry.app || null,
1773
- shareable, unavailable_reason: unavailableReason, match
1774
- }, location))
1775
- }
1776
- const events = (await registry.readEvents()).reverse()
1777
- .filter((event) => !scopeId || event.source_id === scopeId)
1778
- .map((event) => {
1779
- const currentLocation = event.path ? this.locationForPath(event.path, event.source_id) : null
1780
- const fallbackLocation = currentLocation && currentLocation.source_id
1781
- ? currentLocation
1782
- : (event.path ? { relative_path: event.path } : {})
1783
- const publicEvent = Object.assign({}, fallbackLocation, event)
1784
- if (event.kind === "convert" && event.batch_id && event.path) {
1785
- const current = registry.links.get(event.path)
1786
- publicEvent.undoable = !!(current && (
1787
- current.batch_id === event.batch_id ||
1788
- (!current.batch_id && current.hash === event.hash)
1789
- ))
1790
- const batch = undoBatchMap.get(event.batch_id)
1791
- if (batch && Number.isFinite(event.ts)) {
1792
- batch.ts = Math.max(batch.ts || 0, event.ts)
1793
- }
1794
- }
1795
- return publicEvent
2576
+ const snapshot = await this.registry.statusSnapshot({
2577
+ scopeSourceIds,
2578
+ locationSourceIds,
2579
+ scoped: !!scopeId,
2580
+ scopeUnrestricted: !scopeId,
2581
+ locationUnrestricted: !scopeId && !locationId,
2582
+ view,
2583
+ statusFilter,
2584
+ query,
2585
+ pageSize,
2586
+ sizeSort: groupDuplicates
2587
+ ? options.size_sort || "desc"
2588
+ : options.size_sort,
2589
+ groupDuplicates,
2590
+ cursor
1796
2591
  })
1797
- const undoBatches = [...undoBatchMap.values()].sort((a, b) =>
1798
- (b.ts || 0) - (a.ts || 0) || a.batch_id.localeCompare(b.batch_id))
1799
- const scan = this.scanStatus()
1800
- const excluded = []
1801
- for (const [p, meta] of registry.excluded) {
1802
- if (scopeId && meta.source_id !== scopeId) continue
1803
- excluded.push(Object.assign({
1804
- path: p,
1805
- ts: meta.ts || null,
1806
- size: Number(meta.size) || 0
1807
- }, this.locationForPath(p, meta.source_id)))
1808
- }
1809
- const publicSources = this._sources.filter((source) => !scopeId || source.id === scopeId).map((source) => ({
1810
- id: source.id, kind: source.kind, label: source.label, root: source.root,
1811
- display_path: source.kind === "pinokio" ? source.root : (source.mount_path || source.root),
1812
- target_path: source.kind === "external" ? source.root : null,
1813
- parent_id: scopeId ? null : source.parent_id, app: source.app || null,
1814
- available: source.available !== false, shareable: source.kind === "virtual" ? null : !!source.shareable
1815
- }))
2592
+ let items
2593
+ if (view === "activity") {
2594
+ items = (snapshot.page.rows || []).map((event) => Object.assign(
2595
+ {},
2596
+ event.path
2597
+ ? this.locationForPath(event.path, event.source_id)
2598
+ : {},
2599
+ event
2600
+ ))
2601
+ } else if (view === "reclaimable") {
2602
+ items = snapshot.page.rows || []
2603
+ } else if (groupDuplicates) {
2604
+ items = this.publicDuplicateGroupItems(snapshot.page)
2605
+ } else {
2606
+ items = this.publicFileItems(snapshot.page)
2607
+ }
2608
+
2609
+ const lastScan = await this.scanForScope(scopeId)
2610
+ const before = lastScan && Number.isFinite(lastScan.bytes_total)
2611
+ ? lastScan.bytes_total
2612
+ : 0
2613
+ const saved = Math.max(0, Number(snapshot.saved) || 0)
2614
+ const currentCount = Number(snapshot.total) || 0
2615
+ const pageTotal = Number(snapshot.pageTotal) || 0
2616
+ const pages = Math.max(1, Math.ceil(pageTotal / pageSize))
2617
+ const publicSources = this._sources
2618
+ .filter((source) => !scopeId || source.id === scopeId)
2619
+ .map((source) => ({
2620
+ id: source.id,
2621
+ kind: source.kind,
2622
+ label: source.label,
2623
+ root: source.root,
2624
+ display_path: source.root,
2625
+ target_path: source.kind === "external" ? source.root : null,
2626
+ parent_id: scopeId ? null : source.parent_id,
2627
+ app: source.app || null,
2628
+ available: source.available !== false,
2629
+ shareable: source.kind === "virtual" ? null : !!source.shareable,
2630
+ removable: source.kind === "external" &&
2631
+ source.configured === true
2632
+ }))
2633
+ const sourceCounts = this.sourceCountMaps(snapshot.scopeRows)
1816
2634
  const result = {
1817
2635
  enabled: true,
1818
2636
  mode: this.mode,
1819
- scan,
1820
- last_scan: registry.scanFor(scopeId),
1821
- bytes_on_disk: bytesOnDisk,
1822
- bytes_without_sharing: wouldBe,
1823
- saved_by_sharing: wouldBe - bytesOnDisk,
1824
- lifetime_bytes_saved: registry.totals.lifetime_bytes_saved,
1825
- reclaimable: blobs.filter((b) => b.orphan).reduce((sum, b) => sum + b.size, 0),
1826
- pending_bytes: duplicates.filter((d) => d.shareable).reduce((sum, d) => sum + d.size, 0),
2637
+ scan: this.scanStatus(),
2638
+ last_scan: lastScan,
2639
+ bytes_without_sharing: before,
2640
+ bytes_on_disk: Math.max(0, before - saved),
2641
+ saved_by_sharing: saved,
2642
+ effective_bytes: Math.max(0, before - saved),
2643
+ reclaimable: Number(snapshot.reclaimable) || 0,
2644
+ pending_bytes: Number(snapshot.pending) || 0,
1827
2645
  file_action: this.fileActionStatus(scopeId),
1828
- activity_error: registry.eventError,
1829
- cloud_sync_warning: this.cloudSyncProvider(),
1830
- sources: publicSources, blobs, duplicates, excluded, events,
1831
- undo_batches: undoBatches
1832
- }
1833
- if (scopeId) {
1834
- const duplicateBytes = duplicates.reduce((sum, item) => sum + item.size, 0)
1835
- const excludedBytes = excluded.reduce((sum, item) => sum + item.size, 0)
1836
- let linkedBytes = 0
1837
- let effectiveLinkedBytes = 0
1838
- let sharedBytes = 0
1839
- for (const blob of blobs) {
1840
- const scopedNames = blob.names.filter((name) => name.source_id === scopeId)
1841
- const scopedLinks = scopedNames.filter((name) => name.mode === "link").length
1842
- const scopedCopies = scopedNames.length - scopedLinks
1843
- const registeredLinks = blob.names.filter((name) => name.mode === "link").length
1844
- // The store name is one hardlink but not a user-visible location.
1845
- // Registry count prevents stale filesystem metadata from over-attributing the inode.
1846
- const filesystemLinks = Number.isFinite(blob.nlink) ? Math.max(0, blob.nlink - 1) : 0
1847
- const sharingLocations = Math.max(1, registeredLinks, filesystemLinks)
1848
- linkedBytes += blob.size * scopedNames.length
1849
- effectiveLinkedBytes += (blob.size * scopedCopies) +
1850
- (blob.size * scopedLinks / sharingLocations)
1851
- if (registeredLinks >= 2) sharedBytes += blob.size * scopedLinks
1852
- }
1853
- const trackedBytes = linkedBytes + duplicateBytes + excludedBytes
1854
- const effectiveTrackedBytes = effectiveLinkedBytes + duplicateBytes + excludedBytes
1855
- const folderBytes = result.last_scan && Number.isFinite(result.last_scan.bytes_total)
1856
- ? result.last_scan.bytes_total
1857
- : null
1858
- result.scope_id = scopeId
1859
- result.tracked_bytes = trackedBytes
1860
- result.effective_bytes = folderBytes === null
1861
- ? null
1862
- : Math.max(0, folderBytes - trackedBytes) + effectiveTrackedBytes
1863
- result.shared_bytes = sharedBytes
1864
- result.reclaimable = 0
1865
- }
2646
+ sources: publicSources,
2647
+ items,
2648
+ inventory: {
2649
+ view,
2650
+ counts: snapshot.counts,
2651
+ source_counts: sourceCounts,
2652
+ shareable_by_source: sourceCounts.shareable,
2653
+ shareable_duplicates:
2654
+ Number(snapshot.shareableDuplicates) || 0,
2655
+ duplicate_locations:
2656
+ Number(snapshot.duplicateLocations) || 0,
2657
+ current: {
2658
+ count: currentCount,
2659
+ locations: Number(snapshot.currentLocations) || 0,
2660
+ shareable_bytes:
2661
+ Number(snapshot.currentShareableBytes) || 0,
2662
+ deduplicate_bytes:
2663
+ Number(snapshot.currentDeduplicateBytes) || 0,
2664
+ separate_count:
2665
+ Number(snapshot.currentSeparateCount) || 0,
2666
+ separate_bytes:
2667
+ Number(snapshot.currentSeparateBytes) || 0
2668
+ },
2669
+ page: requestedPage,
2670
+ page_size: pageSize,
2671
+ start: requestedPage * pageSize,
2672
+ end: Math.min(
2673
+ (requestedPage * pageSize) + items.length,
2674
+ pageTotal
2675
+ ),
2676
+ total: pageTotal,
2677
+ pages,
2678
+ cursor: cursor || null,
2679
+ next_cursor: snapshot.page.nextCursor || null,
2680
+ has_previous: requestedPage > 0,
2681
+ has_next: !!snapshot.page.nextCursor
2682
+ }
2683
+ }
2684
+ if (scopeId) result.scope_id = scopeId
1866
2685
  return result
1867
2686
  }
2687
+
1868
2688
  scanStatus() {
1869
- const sweeper = this.sweeper
1870
- if (!sweeper) return null
1871
- const pending = !!this.scanPromise && !sweeper.state.active
2689
+ if (!this.sweeper) return null
2690
+ const pending = !!this.scanPromise && !this.sweeper.state.active
1872
2691
  const state = pending
1873
- ? Object.assign(sweeper.idleState(), { phase: "queued", scope_id: this.scanScopeId })
1874
- : sweeper.state
2692
+ ? Object.assign(this.sweeper.idleState(), {
2693
+ phase: "queued",
2694
+ scope_id: this.scanScopeId
2695
+ })
2696
+ : this.sweeper.state
1875
2697
  return Object.assign({}, state, {
1876
- current_file: sweeper.currentHash ? path.basename(sweeper.currentHash.path) : null,
1877
- current_file_bytes: sweeper.currentHash ? sweeper.currentHash.bytes : null,
1878
- current_file_size: sweeper.currentHash ? sweeper.currentHash.size : null,
2698
+ current_file: this.sweeper.currentHash
2699
+ ? path.basename(this.sweeper.currentHash.path)
2700
+ : null,
2701
+ current_file_bytes: this.sweeper.currentHash
2702
+ ? this.sweeper.currentHash.bytes
2703
+ : null,
2704
+ current_file_size: this.sweeper.currentHash
2705
+ ? this.sweeper.currentHash.size
2706
+ : null,
1879
2707
  pending,
1880
2708
  error: this.scanError
1881
2709
  })
1882
2710
  }
2711
+
1883
2712
  fileActionStatus(scopeId = null) {
1884
2713
  const progress = this.fileActionProgress
1885
2714
  if (!progress) return null
1886
- if (scopeId) {
1887
- if (progress.scope_id && progress.scope_id !== scopeId) return null
1888
- if (progress.path) {
1889
- const source = this.sourceForPath(progress.path)
1890
- if (!source || source.id !== scopeId) return null
1891
- }
1892
- }
2715
+ if (scopeId && progress.scope_id &&
2716
+ progress.scope_id !== scopeId) return null
1893
2717
  return Object.assign({}, progress)
1894
2718
  }
1895
- progressStatus(scopeId = null) {
2719
+
2720
+ async progressStatus(scopeId = null) {
1896
2721
  return {
1897
2722
  enabled: !!this.enabled,
1898
2723
  scan: this.scanStatus(),
1899
2724
  file_action: this.fileActionStatus(scopeId),
1900
- last_scan: this.registry ? this.registry.scanFor(scopeId) : null
1901
- }
1902
- }
1903
- async copyOut(filePath, sourcePath, expectedTarget, expectedSource = expectedTarget, options = {}) {
1904
- const tmp = filePath + TMP_SUFFIX
1905
- let copiedStat = null
1906
- try {
1907
- await fs.promises.copyFile(sourcePath, tmp, fs.constants.COPYFILE_EXCL)
1908
- copiedStat = await fs.promises.lstat(tmp)
1909
- if (options.canReplace && !options.canReplace()) {
1910
- await unlinkIfSame(tmp, copiedStat)
1911
- return { status: "locked" }
1912
- }
1913
- const currentTarget = await lstatIfPresent(filePath)
1914
- const currentSource = sourcePath === filePath ? currentTarget : await lstatIfPresent(sourcePath)
1915
- const currentTmp = await lstatIfPresent(tmp)
1916
- if (!currentTarget || !currentSource || !sameSnapshot(expectedTarget, currentTarget) ||
1917
- !sameSnapshot(expectedSource, currentSource) || !sameIdentity(currentTmp, copiedStat)) {
1918
- await unlinkIfSame(tmp, copiedStat)
1919
- return { status: "stale" }
1920
- }
1921
- await fs.promises.rename(tmp, filePath)
1922
- let finalStat
1923
- try {
1924
- finalStat = await lstatIfPresent(filePath)
1925
- } catch (error) {
1926
- // The independent copy was committed by rename(). Preserve that
1927
- // completed action when only the post-commit metadata read failed.
1928
- finalStat = copiedStat
1929
- }
1930
- if (!finalStat || finalStat.dev !== copiedStat.dev || finalStat.ino !== copiedStat.ino) {
1931
- return { status: "stale" }
1932
- }
1933
- copiedStat = null
1934
- return { status: "copied", stat: finalStat }
1935
- } catch (error) {
1936
- await unlinkIfSame(tmp, copiedStat).catch(() => {})
1937
- if (error && error.code === "EEXIST") return { status: "conflict" }
1938
- if (isMissingError(error)) return { status: "stale" }
1939
- throw error
1940
- }
1941
- }
1942
- // detach: make one name an independent copy again ("go back"), and pin it
1943
- // so future scans neither re-list nor re-share it. Works on linked names
1944
- // (copies bytes out) and on pending duplicates (just ignores them).
1945
- async detach(filePath) {
1946
- if (!this.enabled) return { status: "disabled" }
1947
- await this.refreshSources()
1948
- const entry = this.registry.links.get(filePath)
1949
- if (entry) {
1950
- const source = this.sourceForPath(filePath, entry.source_id)
1951
- if (!source || !await this.canonicalPathIsWithinSource(filePath, source)) {
1952
- return { status: "stale" }
1953
- }
1954
- if (entry.mode === "copy") {
1955
- const st = await lstatIfPresent(filePath)
1956
- if (!st || !st.isFile() || st.dev !== entry.dev || st.ino !== entry.ino) return { status: "stale" }
1957
- this.registry.exclude(filePath, {
1958
- ts: Date.now(), source_id: entry.source_id || null, size: st.size
1959
- })
1960
- await this.recordEvent({
1961
- kind: "skip", hash: entry.hash, path: filePath,
1962
- app: entry.app || null, source_id: entry.source_id || null, size: st.size
1963
- })
1964
- return { status: "ignored" }
1965
- }
1966
- if (this.sourceAppIsRunning(source)) return { status: "locked" }
1967
- const before = await lstatIfPresent(filePath)
1968
- if (!before || !before.isFile() || before.dev !== entry.dev || before.ino !== entry.ino) return { status: "stale" }
1969
- const copied = await this.copyOut(filePath, filePath, fileSnapshot(before), fileSnapshot(before), {
1970
- canReplace: () => !this.sourceAppIsRunning(source)
1971
- })
1972
- if (copied.status !== "copied") return copied
1973
- const st = copied.stat
1974
- this.registry.exclude(filePath, { ts: Date.now(), source_id: entry.source_id || null, size: st.size })
1975
- await this.refreshLinkSnapshots(entry.hash, entry.dev, entry.ino)
1976
- await this.recordEvent({ kind: "detach", hash: entry.hash, path: filePath, app: entry.app || null, source_id: entry.source_id || null, size: st.size })
1977
- return { status: "detached" }
1978
- }
1979
- if (this.registry.duplicates.has(filePath)) {
1980
- const dup = this.registry.duplicates.get(filePath)
1981
- this.registry.exclude(filePath, { ts: Date.now(), source_id: dup.source_id || null, size: dup.size || 0 })
1982
- await this.recordEvent({ kind: "skip", hash: dup.hash, path: filePath, app: dup.app || null, source_id: dup.source_id || null, size: dup.size || 0 })
1983
- return { status: "ignored" }
1984
- }
1985
- return { status: "not-found" }
1986
- }
1987
- // Undo a conversion batch: replace each converted name with an independent
1988
- // copy of the bytes (spec, UX contract surface 3).
1989
- async undoBatch(batchId) {
1990
- if (!this.enabled || !batchId) return { undone: 0 }
1991
- await this.refreshSources()
1992
- const events = await this.registry.readEvents()
1993
- const conversionEvents = new Map(events
1994
- .filter((event) => event.kind === "convert" && event.batch_id === batchId && event.path)
1995
- .map((event) => [event.path, event]))
1996
- const targets = new Set(conversionEvents.keys())
1997
- for (const [filePath, entry] of this.registry.links) {
1998
- if (entry.batch_id === batchId) targets.add(filePath)
1999
- }
2000
- const summary = { undone: 0, bytes: 0, failed: 0 }
2001
- for (const filePath of targets) {
2002
- const event = conversionEvents.get(filePath) || {}
2003
- const entry = this.registry.links.get(filePath)
2004
- if (!entry || (entry.batch_id && entry.batch_id !== batchId) ||
2005
- (event.hash && entry.hash !== event.hash)) continue
2006
- const storePath = this.storePathFor(entry.hash)
2007
- try {
2008
- const source = this.sourceForPath(filePath, entry.source_id || event.source_id)
2009
- if (!source || !await this.canonicalPathIsWithinSource(filePath, source)) {
2010
- summary.failed += 1
2011
- continue
2012
- }
2013
- if (this.sourceAppIsRunning(source)) {
2014
- summary.failed += 1
2015
- continue
2016
- }
2017
- const st = await fs.promises.lstat(filePath)
2018
- const storeStat = await this.storeStatIfPresent(storePath)
2019
- if (!st.isFile() || !storeStat || !storeStat.isFile()) {
2020
- summary.failed += 1
2021
- continue
2022
- }
2023
- if (st.ino !== storeStat.ino || st.dev !== storeStat.dev ||
2024
- st.dev !== entry.dev || st.ino !== entry.ino) {
2025
- summary.failed += 1
2026
- continue
2027
- }
2028
- const copied = await this.copyOut(filePath, storePath, fileSnapshot(st), fileSnapshot(storeStat), {
2029
- canReplace: () => !this.sourceAppIsRunning(source)
2030
- })
2031
- if (copied.status !== "copied") {
2032
- summary.failed += 1
2033
- continue
2034
- }
2035
- const independentStat = copied.stat
2036
- const duplicateEntry = {
2037
- hash: entry.hash, size: storeStat.size, app: entry.app || null,
2038
- source_id: entry.source_id || event.source_id || null, discovered: Date.now(),
2039
- dev: independentStat.dev, ino: independentStat.ino,
2040
- mtime: independentStat.mtimeMs, ctime: independentStat.ctimeMs
2041
- }
2042
- const scanEntry = {
2043
- hash: entry.hash, size: independentStat.size,
2044
- dev: independentStat.dev, ino: independentStat.ino,
2045
- mtime: independentStat.mtimeMs, ctime: independentStat.ctimeMs,
2046
- source_id: entry.source_id || event.source_id || null
2047
- }
2048
- this.registry.setDuplicate(filePath, duplicateEntry, scanEntry)
2049
- await this.refreshLinkSnapshots(entry.hash, entry.dev, entry.ino)
2050
- const bytesSaved = event.bytes_saved || storeStat.size
2051
- this.registry.totals.lifetime_bytes_saved = Math.max(0,
2052
- this.registry.totals.lifetime_bytes_saved - bytesSaved)
2053
- await this.recordEvent({
2054
- kind: "undo", hash: entry.hash, path: filePath,
2055
- source_id: entry.source_id || event.source_id || null, batch_id: batchId,
2056
- bytes_saved: bytesSaved
2057
- })
2058
- summary.undone += 1
2059
- summary.bytes += storeStat.size
2060
- } catch (e) {
2061
- summary.failed += 1
2062
- }
2063
- }
2064
- this.registry.schedulePersist()
2065
- return summary
2066
- }
2067
- async reclaimAll() {
2068
- if (!this.enabled) return { reclaimed: 0, bytes_freed: 0 }
2069
- const summary = { reclaimed: 0, bytes_freed: 0, failed: 0 }
2070
- const removedHashes = new Set()
2071
- const copyHashes = new Set([...this.registry.links.values()]
2072
- .filter((entry) => entry.mode === "copy")
2073
- .map((entry) => entry.hash))
2074
- try {
2075
- for (const [hash] of [...this.registry.blobs]) {
2076
- const result = await this.reclaim(hash, {
2077
- deferRegistry: true,
2078
- hasCopyNames: copyHashes.has(hash)
2079
- })
2080
- if (result.remove_hash) removedHashes.add(hash)
2081
- if (result.status === "reclaimed") {
2082
- summary.reclaimed += 1
2083
- summary.bytes_freed += result.bytes_freed || 0
2084
- } else if (!["gone", "in-use", "unavailable"].includes(result.status)) summary.failed += 1
2085
- }
2086
- } finally {
2087
- this.registry.removeBlobs(removedHashes)
2725
+ last_scan: this.lastScanCache.get(scopeId || "") || null
2088
2726
  }
2089
- return summary
2090
2727
  }
2091
2728
  }
2092
2729
 
2093
- Vault.SIZE_THRESHOLD = SIZE_THRESHOLD
2094
- Vault.TMP_SUFFIX = TMP_SUFFIX
2095
-
2096
2730
  module.exports = Vault