pinokiod 8.0.50 → 8.0.53
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/kernel/index.js +4 -13
- package/kernel/vault/constants.js +6 -4
- package/kernel/vault/index.js +2370 -1736
- package/kernel/vault/registry.js +117 -500
- package/kernel/vault/registry_core.js +3387 -0
- package/kernel/vault/registry_worker.js +25 -0
- package/kernel/vault/snapshot.js +11 -4
- package/kernel/vault/sweeper.js +579 -572
- package/kernel/vault/walker.js +34 -15
- package/package.json +2 -1
- package/server/index.js +32 -5
- package/server/public/vault.css +251 -80
- package/server/public/vault.js +1350 -576
- package/server/views/partials/vault_workspace.ejs +19 -25
- package/test/vault-engine.test.js +749 -1581
- package/test/vault-sweep.test.js +1347 -1033
- package/test/vault-ui.test.js +883 -2708
package/kernel/vault/index.js
CHANGED
|
@@ -1,26 +1,37 @@
|
|
|
1
|
-
const fs = require(
|
|
2
|
-
const path = require(
|
|
3
|
-
const crypto = require(
|
|
4
|
-
const {
|
|
5
|
-
const
|
|
6
|
-
const
|
|
7
|
-
const
|
|
8
|
-
const { fileSnapshot, sameSnapshot, sameContentState } = require(
|
|
1
|
+
const fs = require("fs")
|
|
2
|
+
const path = require("path")
|
|
3
|
+
const crypto = require("crypto")
|
|
4
|
+
const { execFile } = require("child_process")
|
|
5
|
+
const { Worker } = require("worker_threads")
|
|
6
|
+
const Registry = require("./registry")
|
|
7
|
+
const Sweeper = require("./sweeper")
|
|
8
|
+
const { fileSnapshot, sameSnapshot, sameContentState } = require("./snapshot")
|
|
9
9
|
const {
|
|
10
|
-
SIZE_THRESHOLD,
|
|
11
|
-
|
|
12
|
-
|
|
10
|
+
SIZE_THRESHOLD,
|
|
11
|
+
CANDIDATE_SIZE_OPTIONS,
|
|
12
|
+
TMP_SUFFIX,
|
|
13
|
+
SHA256_RE,
|
|
14
|
+
DIR_CONCURRENCY,
|
|
15
|
+
STAT_CONCURRENCY,
|
|
16
|
+
HASH_INACTIVITY_MS
|
|
17
|
+
} = require("./constants")
|
|
13
18
|
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
// Nothing here runs automatically except startup verify; discovery happens
|
|
18
|
-
// only through the user's manual scan.
|
|
19
|
-
|
|
20
|
-
const NO_LINK_CODES = new Set(["EXDEV", "ENOTSUP", "ENOSYS"])
|
|
19
|
+
const NO_LINK_CODES = new Set([
|
|
20
|
+
"EXDEV", "ENOTSUP", "EOPNOTSUPP", "ENOSYS", "EPERM", "EACCES"
|
|
21
|
+
])
|
|
21
22
|
const LOCK_CODES = new Set(["EBUSY", "EPERM", "EACCES"])
|
|
22
|
-
const
|
|
23
|
-
const
|
|
23
|
+
const STATUS_PAGE_SIZE = 500
|
|
24
|
+
const MAX_BULK_FILE_ACTIONS = 500
|
|
25
|
+
const STATUS_VIEWS = new Set([
|
|
26
|
+
"all", "duplicates", "shared", "tracked", "reclaimable", "activity"
|
|
27
|
+
])
|
|
28
|
+
const STATUS_FILTERS = new Set([
|
|
29
|
+
"all", "duplicate", "shared", "tracked"
|
|
30
|
+
])
|
|
31
|
+
|
|
32
|
+
const isMissingError = (error) => !!(error &&
|
|
33
|
+
(error.code === "ENOENT" || error.code === "ENOTDIR"))
|
|
34
|
+
|
|
24
35
|
const lstatIfPresent = async (filePath) => {
|
|
25
36
|
try {
|
|
26
37
|
return await fs.promises.lstat(filePath)
|
|
@@ -29,9 +40,11 @@ const lstatIfPresent = async (filePath) => {
|
|
|
29
40
|
throw error
|
|
30
41
|
}
|
|
31
42
|
}
|
|
43
|
+
|
|
32
44
|
const sameIdentity = (left, right) => !!(
|
|
33
45
|
left && right && left.dev === right.dev && left.ino === right.ino
|
|
34
46
|
)
|
|
47
|
+
|
|
35
48
|
const unlinkIfSame = async (filePath, expected) => {
|
|
36
49
|
if (!expected) return false
|
|
37
50
|
const current = await lstatIfPresent(filePath)
|
|
@@ -44,41 +57,92 @@ const unlinkIfSame = async (filePath, expected) => {
|
|
|
44
57
|
throw error
|
|
45
58
|
}
|
|
46
59
|
}
|
|
60
|
+
|
|
47
61
|
const unsafeStoragePath = (filePath) => {
|
|
48
62
|
const error = new Error(`Storage path is not a real directory: ${filePath}`)
|
|
49
63
|
error.code = "EVAULTPATH"
|
|
50
64
|
return error
|
|
51
65
|
}
|
|
52
66
|
|
|
67
|
+
const unavailableStorage = (message) => {
|
|
68
|
+
const error = new Error(message)
|
|
69
|
+
error.code = "EVAULTUNAVAILABLE"
|
|
70
|
+
return error
|
|
71
|
+
}
|
|
72
|
+
|
|
53
73
|
const isPathWithin = (root, target) => {
|
|
54
|
-
const
|
|
55
|
-
return
|
|
74
|
+
const relative = path.relative(path.resolve(root), path.resolve(target))
|
|
75
|
+
return relative === "" || (
|
|
76
|
+
!relative.startsWith(`..${path.sep}`) &&
|
|
77
|
+
relative !== ".." &&
|
|
78
|
+
!path.isAbsolute(relative)
|
|
79
|
+
)
|
|
56
80
|
}
|
|
57
81
|
|
|
58
82
|
const samePath = (left, right) => {
|
|
59
|
-
const
|
|
60
|
-
const
|
|
61
|
-
return process.platform === "win32"
|
|
83
|
+
const first = path.resolve(left)
|
|
84
|
+
const second = path.resolve(right)
|
|
85
|
+
return process.platform === "win32"
|
|
86
|
+
? first.toLowerCase() === second.toLowerCase()
|
|
87
|
+
: first === second
|
|
62
88
|
}
|
|
63
89
|
|
|
64
90
|
const sameFileMetadata = (left, right) => {
|
|
65
91
|
if (!left || !right) return false
|
|
66
92
|
if ((left.mode & 0o7777) !== (right.mode & 0o7777)) return false
|
|
67
|
-
if (left.uid !== undefined && right.uid !== undefined &&
|
|
68
|
-
|
|
93
|
+
if (left.uid !== undefined && right.uid !== undefined &&
|
|
94
|
+
left.uid !== right.uid) return false
|
|
95
|
+
if (left.gid !== undefined && right.gid !== undefined &&
|
|
96
|
+
left.gid !== right.gid) return false
|
|
69
97
|
return true
|
|
70
98
|
}
|
|
71
99
|
|
|
72
100
|
const sourceId = (kind, name) => `${kind}:${encodeURIComponent(name)}`
|
|
101
|
+
const boundedInteger = (value, fallback, minimum, maximum) => {
|
|
102
|
+
const parsed = Number(value)
|
|
103
|
+
if (!Number.isSafeInteger(parsed)) return fallback
|
|
104
|
+
return Math.max(minimum, Math.min(maximum, parsed))
|
|
105
|
+
}
|
|
106
|
+
const rowSnapshot = (row) => ({
|
|
107
|
+
size: row.size,
|
|
108
|
+
mtime: row.mtime,
|
|
109
|
+
ctime: row.ctime,
|
|
110
|
+
dev: row.dev,
|
|
111
|
+
ino: row.ino
|
|
112
|
+
})
|
|
113
|
+
|
|
114
|
+
const revealInFileManager = (filePath, platform = process.platform) =>
|
|
115
|
+
new Promise((resolve, reject) => {
|
|
116
|
+
const command = platform === "darwin"
|
|
117
|
+
? "open"
|
|
118
|
+
: platform === "win32"
|
|
119
|
+
? "explorer.exe"
|
|
120
|
+
: "xdg-open"
|
|
121
|
+
const args = platform === "darwin"
|
|
122
|
+
? ["-R", filePath]
|
|
123
|
+
: platform === "win32"
|
|
124
|
+
? [`/select,${filePath}`]
|
|
125
|
+
: [path.dirname(filePath)]
|
|
126
|
+
execFile(command, args, {
|
|
127
|
+
timeout: 10000,
|
|
128
|
+
windowsHide: true
|
|
129
|
+
}, (error) => error ? reject(error) : resolve())
|
|
130
|
+
})
|
|
73
131
|
|
|
74
132
|
class Vault {
|
|
75
133
|
constructor(kernel) {
|
|
76
134
|
this.kernel = kernel
|
|
135
|
+
this.fileManagerLauncher = (filePath) =>
|
|
136
|
+
revealInFileManager(
|
|
137
|
+
filePath,
|
|
138
|
+
this.kernel.platform || process.platform
|
|
139
|
+
)
|
|
77
140
|
this.enabled = false
|
|
78
141
|
this.initialized = false
|
|
79
|
-
this.mode = null
|
|
80
|
-
this.volumeModes = new Map()
|
|
142
|
+
this.mode = null
|
|
143
|
+
this.volumeModes = new Map()
|
|
81
144
|
this.registry = null
|
|
145
|
+
this.sweeper = null
|
|
82
146
|
this.worker = null
|
|
83
147
|
this.workerJobs = new Map()
|
|
84
148
|
this.workerSeq = 0
|
|
@@ -89,597 +153,575 @@ class Vault {
|
|
|
89
153
|
this.dirConcurrency = DIR_CONCURRENCY
|
|
90
154
|
this.sizeThreshold = SIZE_THRESHOLD
|
|
91
155
|
this._sources = []
|
|
156
|
+
this._sourcesById = new Map()
|
|
92
157
|
this._sourceBases = new Map()
|
|
158
|
+
this._anchorStores = []
|
|
159
|
+
this._anchorStoresById = new Map()
|
|
160
|
+
this._anchorStoresByDevice = new Map()
|
|
161
|
+
this._fallbackConfig = null
|
|
93
162
|
this.operationTail = Promise.resolve()
|
|
94
163
|
this.initializationPromise = null
|
|
95
|
-
this.
|
|
164
|
+
this.scanPromise = null
|
|
96
165
|
this.scanError = null
|
|
97
166
|
this.scanScopeId = null
|
|
167
|
+
this.scanCancelRequested = false
|
|
168
|
+
this.lastScanCache = new Map()
|
|
98
169
|
this.fileActionProgress = null
|
|
170
|
+
this.fileActionCancelRequested = false
|
|
99
171
|
}
|
|
172
|
+
|
|
100
173
|
get root() {
|
|
101
174
|
return path.resolve(this.kernel.homedir, "vault")
|
|
102
175
|
}
|
|
176
|
+
|
|
103
177
|
get blobRoot() {
|
|
104
|
-
|
|
178
|
+
const store = this.defaultAnchorStore()
|
|
179
|
+
return store
|
|
180
|
+
? path.resolve(store.root, "sha256")
|
|
181
|
+
: path.resolve(this.globalConfigRoot(), "vault", "sha256")
|
|
105
182
|
}
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
}
|
|
109
|
-
storePathFor(hash) {
|
|
183
|
+
|
|
184
|
+
storePathFor(hash, storeId = null) {
|
|
110
185
|
if (typeof hash !== "string" || !SHA256_RE.test(hash)) {
|
|
111
186
|
throw new TypeError("Invalid vault content identifier.")
|
|
112
187
|
}
|
|
113
|
-
|
|
188
|
+
const store = storeId
|
|
189
|
+
? this._anchorStoresById.get(storeId)
|
|
190
|
+
: this.defaultAnchorStore()
|
|
191
|
+
if (!store) throw new Error("No anchor store is configured.")
|
|
192
|
+
return path.resolve(
|
|
193
|
+
store.root, "sha256", hash.slice(0, 2), hash)
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
globalConfigRoot() {
|
|
197
|
+
return this.kernel.store && typeof this.kernel.store.root === "string"
|
|
198
|
+
? path.resolve(this.kernel.store.root)
|
|
199
|
+
: path.resolve(this.kernel.homedir, ".pinokio")
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
readConfig() {
|
|
203
|
+
const stored = this.kernel.store &&
|
|
204
|
+
typeof this.kernel.store.get === "function"
|
|
205
|
+
? this.kernel.store.get("vault")
|
|
206
|
+
: this._fallbackConfig
|
|
207
|
+
const config = stored && typeof stored === "object" ? stored : {}
|
|
208
|
+
return {
|
|
209
|
+
locations: Array.isArray(config.locations)
|
|
210
|
+
? config.locations.filter((item) => typeof item === "string")
|
|
211
|
+
: [],
|
|
212
|
+
anchor_stores: Array.isArray(config.anchor_stores)
|
|
213
|
+
? config.anchor_stores.filter((item) =>
|
|
214
|
+
item &&
|
|
215
|
+
typeof item === "object" &&
|
|
216
|
+
typeof item.id === "string" &&
|
|
217
|
+
item.id &&
|
|
218
|
+
typeof item.root === "string" &&
|
|
219
|
+
typeof item.probe_path === "string")
|
|
220
|
+
: []
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
writeConfig(config) {
|
|
225
|
+
const value = {
|
|
226
|
+
locations: [...new Set((config.locations || [])
|
|
227
|
+
.filter((item) => typeof item === "string")
|
|
228
|
+
.map((item) => path.resolve(item)))],
|
|
229
|
+
anchor_stores: (config.anchor_stores || []).map((store) => ({
|
|
230
|
+
id: store.id,
|
|
231
|
+
root: path.resolve(store.root),
|
|
232
|
+
probe_path: path.resolve(store.probe_path)
|
|
233
|
+
}))
|
|
234
|
+
}
|
|
235
|
+
if (this.kernel.store && typeof this.kernel.store.set === "function") {
|
|
236
|
+
this.kernel.store.set("vault", value)
|
|
237
|
+
} else {
|
|
238
|
+
this._fallbackConfig = value
|
|
239
|
+
}
|
|
240
|
+
return value
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
configuredLocations() {
|
|
244
|
+
return this.readConfig().locations
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
defaultAnchorStore() {
|
|
248
|
+
const home = path.resolve(this.kernel.homedir)
|
|
249
|
+
return this._anchorStores.find((store) =>
|
|
250
|
+
store.available && isPathWithin(store.probe_path, home)) ||
|
|
251
|
+
this._anchorStores[0] ||
|
|
252
|
+
null
|
|
114
253
|
}
|
|
254
|
+
|
|
115
255
|
async directoryIfSafe(directory) {
|
|
116
|
-
const
|
|
117
|
-
if (
|
|
118
|
-
|
|
256
|
+
const stat = await lstatIfPresent(directory)
|
|
257
|
+
if (stat && (!stat.isDirectory() || stat.isSymbolicLink())) {
|
|
258
|
+
throw unsafeStoragePath(directory)
|
|
259
|
+
}
|
|
260
|
+
return stat
|
|
119
261
|
}
|
|
262
|
+
|
|
120
263
|
async ensureDirectory(directory) {
|
|
121
|
-
let
|
|
122
|
-
if (
|
|
264
|
+
let stat = await this.directoryIfSafe(directory)
|
|
265
|
+
if (stat) return stat
|
|
123
266
|
try {
|
|
124
|
-
await fs.promises.mkdir(directory)
|
|
267
|
+
await fs.promises.mkdir(directory, { mode: 0o700 })
|
|
125
268
|
} catch (error) {
|
|
126
269
|
if (!error || error.code !== "EEXIST") throw error
|
|
127
270
|
}
|
|
128
|
-
|
|
129
|
-
if (!
|
|
130
|
-
return
|
|
271
|
+
stat = await this.directoryIfSafe(directory)
|
|
272
|
+
if (!stat) throw unsafeStoragePath(directory)
|
|
273
|
+
return stat
|
|
131
274
|
}
|
|
275
|
+
|
|
132
276
|
async storeStatIfPresent(storePath, options = {}) {
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
277
|
+
const store = options.store_id
|
|
278
|
+
? this._anchorStoresById.get(options.store_id)
|
|
279
|
+
: this._anchorStores.find((candidate) =>
|
|
280
|
+
isPathWithin(path.resolve(candidate.root, "sha256"), storePath))
|
|
281
|
+
if (!store) {
|
|
282
|
+
throw unsafeStoragePath(storePath)
|
|
283
|
+
}
|
|
284
|
+
const blobRoot = path.resolve(store.root, "sha256")
|
|
285
|
+
if (!isPathWithin(blobRoot, storePath)) {
|
|
286
|
+
throw unsafeStoragePath(storePath)
|
|
287
|
+
}
|
|
288
|
+
if (options.createParent) {
|
|
289
|
+
await this.ensureAnchorStore(store, options.dev)
|
|
290
|
+
}
|
|
291
|
+
if (!await this.directoryIfSafe(store.root) ||
|
|
292
|
+
!await this.directoryIfSafe(blobRoot)) {
|
|
293
|
+
return null
|
|
136
294
|
}
|
|
137
295
|
const shard = path.dirname(storePath)
|
|
138
296
|
let shardStat = await this.directoryIfSafe(shard)
|
|
139
|
-
if (!shardStat && options.createParent)
|
|
297
|
+
if (!shardStat && options.createParent) {
|
|
298
|
+
shardStat = await this.ensureDirectory(shard)
|
|
299
|
+
}
|
|
140
300
|
if (!shardStat) return null
|
|
141
|
-
|
|
301
|
+
const stat = await lstatIfPresent(storePath)
|
|
302
|
+
if (stat && (!stat.isFile() || stat.isSymbolicLink())) {
|
|
303
|
+
throw unsafeStoragePath(storePath)
|
|
304
|
+
}
|
|
305
|
+
return stat
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
async ensureAnchorStore(store, expectedDev = null) {
|
|
309
|
+
if (!store) throw new Error("No anchor store is configured.")
|
|
310
|
+
const managementParent = path.dirname(store.root)
|
|
311
|
+
const parentStat = await lstatIfPresent(managementParent)
|
|
312
|
+
if (!parentStat) {
|
|
313
|
+
await fs.promises.mkdir(managementParent, {
|
|
314
|
+
recursive: true,
|
|
315
|
+
mode: 0o700
|
|
316
|
+
})
|
|
317
|
+
} else if (!parentStat.isDirectory() || parentStat.isSymbolicLink()) {
|
|
318
|
+
throw unsafeStoragePath(managementParent)
|
|
319
|
+
}
|
|
320
|
+
await this.ensureDirectory(store.root)
|
|
321
|
+
const markerPath = path.resolve(store.root, "store.json")
|
|
322
|
+
const markerStat = await lstatIfPresent(markerPath)
|
|
323
|
+
if (markerStat && (!markerStat.isFile() || markerStat.isSymbolicLink())) {
|
|
324
|
+
throw unsafeStoragePath(markerPath)
|
|
325
|
+
}
|
|
326
|
+
if (markerStat) {
|
|
327
|
+
let marker
|
|
328
|
+
try {
|
|
329
|
+
marker = JSON.parse(await fs.promises.readFile(markerPath, "utf8"))
|
|
330
|
+
} catch (error) {
|
|
331
|
+
throw unsafeStoragePath(markerPath)
|
|
332
|
+
}
|
|
333
|
+
if (!marker || marker.id !== store.id || marker.version !== 1) {
|
|
334
|
+
throw unsafeStoragePath(markerPath)
|
|
335
|
+
}
|
|
336
|
+
} else {
|
|
337
|
+
await fs.promises.writeFile(
|
|
338
|
+
markerPath,
|
|
339
|
+
`${JSON.stringify({ id: store.id, version: 1 }, null, 2)}\n`,
|
|
340
|
+
{ flag: "wx", mode: 0o600 }
|
|
341
|
+
)
|
|
342
|
+
}
|
|
343
|
+
const rootStat = await fs.promises.stat(store.root)
|
|
344
|
+
if (expectedDev !== null && rootStat.dev !== expectedDev) {
|
|
345
|
+
throw unavailableStorage(
|
|
346
|
+
"The configured anchor store is on a different filesystem.")
|
|
347
|
+
}
|
|
348
|
+
const mode = await this.probe(store.root)
|
|
349
|
+
store.available = true
|
|
350
|
+
store.dev = rootStat.dev
|
|
351
|
+
store.mode = mode
|
|
352
|
+
this._anchorStoresByDevice.set(rootStat.dev, store)
|
|
353
|
+
if (mode !== "link") {
|
|
354
|
+
throw unavailableStorage(
|
|
355
|
+
"This filesystem does not support hardlinks.")
|
|
356
|
+
}
|
|
357
|
+
await this.ensureDirectory(path.resolve(store.root, "sha256"))
|
|
358
|
+
return store
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
async validateAnchorStore(store, expectedDev = null) {
|
|
362
|
+
if (!store) throw new Error("No anchor store is configured.")
|
|
363
|
+
const markerPath = path.resolve(store.root, "store.json")
|
|
364
|
+
const [rootStat, markerStat] = await Promise.all([
|
|
365
|
+
this.directoryIfSafe(store.root),
|
|
366
|
+
lstatIfPresent(markerPath)
|
|
367
|
+
])
|
|
368
|
+
if (!rootStat ||
|
|
369
|
+
!markerStat ||
|
|
370
|
+
!markerStat.isFile() ||
|
|
371
|
+
markerStat.isSymbolicLink()) {
|
|
372
|
+
throw unsafeStoragePath(store.root)
|
|
373
|
+
}
|
|
374
|
+
let marker
|
|
375
|
+
try {
|
|
376
|
+
marker = JSON.parse(await fs.promises.readFile(markerPath, "utf8"))
|
|
377
|
+
} catch (error) {
|
|
378
|
+
throw unsafeStoragePath(markerPath)
|
|
379
|
+
}
|
|
380
|
+
if (!marker || marker.id !== store.id || marker.version !== 1) {
|
|
381
|
+
throw unsafeStoragePath(markerPath)
|
|
382
|
+
}
|
|
383
|
+
const current = await fs.promises.stat(store.root)
|
|
384
|
+
if (expectedDev !== null && current.dev !== expectedDev) {
|
|
385
|
+
throw unavailableStorage(
|
|
386
|
+
"The configured anchor store is on a different filesystem.")
|
|
387
|
+
}
|
|
388
|
+
store.available = true
|
|
389
|
+
store.dev = current.dev
|
|
390
|
+
return current
|
|
142
391
|
}
|
|
143
|
-
|
|
144
|
-
// while scans, conversions, repair, undo, detach, and reclaim can never
|
|
145
|
-
// publish interleaved filesystem/registry state.
|
|
392
|
+
|
|
146
393
|
runExclusive(operation) {
|
|
147
394
|
const pending = this.operationTail.then(operation, operation)
|
|
148
395
|
this.operationTail = pending.catch(() => {})
|
|
149
396
|
return pending
|
|
150
397
|
}
|
|
151
|
-
|
|
152
|
-
// flushed. Keep that distinction in the response: failed persistence is
|
|
153
|
-
// retried by Registry and must not make an already-completed action look as
|
|
154
|
-
// though it never happened.
|
|
398
|
+
|
|
155
399
|
runMutation(operation) {
|
|
156
|
-
return this.runExclusive(
|
|
157
|
-
const result = await operation()
|
|
158
|
-
try {
|
|
159
|
-
if (this.registry) await this.registry.flush()
|
|
160
|
-
} catch (error) {
|
|
161
|
-
return Object.assign({}, result, { persistence_warning: true })
|
|
162
|
-
}
|
|
163
|
-
return result
|
|
164
|
-
})
|
|
400
|
+
return this.runExclusive(operation)
|
|
165
401
|
}
|
|
402
|
+
|
|
166
403
|
async runFileAction(progress, operation) {
|
|
404
|
+
this.fileActionCancelRequested = false
|
|
167
405
|
this.fileActionProgress = progress
|
|
168
406
|
try {
|
|
169
407
|
return await operation(progress)
|
|
170
408
|
} finally {
|
|
171
|
-
if (this.fileActionProgress === progress)
|
|
409
|
+
if (this.fileActionProgress === progress) {
|
|
410
|
+
this.fileActionProgress = null
|
|
411
|
+
}
|
|
412
|
+
this.fileActionCancelRequested = false
|
|
172
413
|
}
|
|
173
414
|
}
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
if (this.
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
this.
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
throw error
|
|
184
|
-
})
|
|
185
|
-
.finally(() => {
|
|
186
|
-
this.scanPromise = null
|
|
187
|
-
this.scanScopeId = null
|
|
188
|
-
})
|
|
189
|
-
this.scanPromise.catch(() => {})
|
|
190
|
-
return { started: true }
|
|
415
|
+
|
|
416
|
+
cancelFileAction() {
|
|
417
|
+
if (!this.fileActionProgress ||
|
|
418
|
+
!this.fileActionProgress.cancelable) {
|
|
419
|
+
return { cancel_requested: false }
|
|
420
|
+
}
|
|
421
|
+
this.fileActionCancelRequested = true
|
|
422
|
+
this.fileActionProgress.cancel_requested = true
|
|
423
|
+
return { cancel_requested: true }
|
|
191
424
|
}
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
const
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
label: result.source.label,
|
|
203
|
-
target_path: result.source.root,
|
|
204
|
-
shareable: !!result.source.shareable
|
|
205
|
-
}
|
|
206
|
-
}
|
|
207
|
-
}
|
|
208
|
-
case "scan": {
|
|
209
|
-
if (payload.scope_id && !this.scanSource(payload.scope_id)) {
|
|
210
|
-
return { error: "That scan location is no longer available." }
|
|
211
|
-
}
|
|
212
|
-
let sizeThreshold = this.sizeThreshold
|
|
213
|
-
if (payload.candidate_size != null) {
|
|
214
|
-
if (!CANDIDATE_SIZE_OPTIONS.includes(payload.candidate_size)) {
|
|
215
|
-
return { error: "Choose a valid minimum file size." }
|
|
216
|
-
}
|
|
217
|
-
sizeThreshold = payload.candidate_size
|
|
425
|
+
|
|
426
|
+
async isEnabled() {
|
|
427
|
+
let value = process.env.PINOKIO_VAULT
|
|
428
|
+
if (value === undefined && this.kernel.homedir) {
|
|
429
|
+
try {
|
|
430
|
+
const raw = await fs.promises.readFile(
|
|
431
|
+
path.resolve(this.kernel.homedir, "ENVIRONMENT"), "utf8")
|
|
432
|
+
for (const line of raw.split("\n")) {
|
|
433
|
+
const match = line.match(/^\s*PINOKIO_VAULT\s*=\s*(.*)\s*$/)
|
|
434
|
+
if (match) value = match[1].trim()
|
|
218
435
|
}
|
|
219
|
-
|
|
436
|
+
} catch (error) {
|
|
437
|
+
if (!isMissingError(error)) throw error
|
|
220
438
|
}
|
|
221
|
-
case "deduplicate":
|
|
222
|
-
if (typeof payload.path === "string" && payload.path) {
|
|
223
|
-
return this.runMutation(() => this.runFileAction({
|
|
224
|
-
kind: "deduplicate-file",
|
|
225
|
-
path: path.resolve(payload.path)
|
|
226
|
-
}, () => this.deduplicateFile(payload.path, {
|
|
227
|
-
batch_id: `batch-${crypto.randomUUID()}`
|
|
228
|
-
})))
|
|
229
|
-
}
|
|
230
|
-
const selection = payload.selection || null
|
|
231
|
-
if (selection && selection !== "duplicates" && selection !== "kept-separate") {
|
|
232
|
-
return { error: "Choose files to deduplicate." }
|
|
233
|
-
}
|
|
234
|
-
if (payload.scope_id != null &&
|
|
235
|
-
(typeof payload.scope_id !== "string" || !payload.scope_id)) {
|
|
236
|
-
return { error: "Choose a valid location to deduplicate." }
|
|
237
|
-
}
|
|
238
|
-
const scopeId = typeof payload.scope_id === "string" && payload.scope_id
|
|
239
|
-
? payload.scope_id
|
|
240
|
-
: null
|
|
241
|
-
if (!selection && !scopeId) {
|
|
242
|
-
return { error: "Choose a location to deduplicate." }
|
|
243
|
-
}
|
|
244
|
-
return this.runMutation(() => this.runFileAction({
|
|
245
|
-
kind: "deduplicate",
|
|
246
|
-
scope_id: scopeId,
|
|
247
|
-
selection: selection || "duplicates",
|
|
248
|
-
files_total: 0,
|
|
249
|
-
files_completed: 0
|
|
250
|
-
}, async (progress) => {
|
|
251
|
-
const options = {
|
|
252
|
-
batch_id: `batch-${crypto.randomUUID()}`,
|
|
253
|
-
progress
|
|
254
|
-
}
|
|
255
|
-
return selection
|
|
256
|
-
? this.deduplicateSelection(selection, scopeId, options)
|
|
257
|
-
: this.deduplicateScope(scopeId, options)
|
|
258
|
-
}))
|
|
259
|
-
case "reclaim":
|
|
260
|
-
return this.runMutation(() => this.reclaim(payload.hash))
|
|
261
|
-
case "reclaim_all":
|
|
262
|
-
return this.runMutation(() => this.reclaimAll())
|
|
263
|
-
case "repair":
|
|
264
|
-
case "rebuild":
|
|
265
|
-
if (this.scanPromise || (this.sweeper && this.sweeper.state.active)) {
|
|
266
|
-
return { error: "Wait for the current scan to finish before repairing the index." }
|
|
267
|
-
}
|
|
268
|
-
return this.runMutation(async () => {
|
|
269
|
-
await this.rebuild(undefined, { flush: false })
|
|
270
|
-
return { done: true }
|
|
271
|
-
})
|
|
272
|
-
case "undo":
|
|
273
|
-
return this.runMutation(() => this.undoBatch(payload.batch_id))
|
|
274
|
-
case "detach":
|
|
275
|
-
if (typeof payload.path !== "string" || !payload.path) return { status: "not-found" }
|
|
276
|
-
return this.runMutation(() => {
|
|
277
|
-
const targetPath = path.resolve(payload.path)
|
|
278
|
-
const kind = this.registry.duplicates.has(targetPath)
|
|
279
|
-
? "keep-separate"
|
|
280
|
-
: "make-separate"
|
|
281
|
-
return this.runFileAction({ kind, path: targetPath }, () => this.detach(targetPath))
|
|
282
|
-
})
|
|
283
|
-
default:
|
|
284
|
-
return { error: "unknown action" }
|
|
285
439
|
}
|
|
440
|
+
return String(value).toLowerCase() !== "false"
|
|
286
441
|
}
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
442
|
+
|
|
443
|
+
async init(options = {}) {
|
|
444
|
+
this.enabled = await this.isEnabled()
|
|
445
|
+
if (!this.enabled) return { enabled: false }
|
|
446
|
+
if (options.deferStorage) return { enabled: true }
|
|
447
|
+
return this.initializeStorage()
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
async initializeStorage() {
|
|
451
|
+
if (this.initialized) return { enabled: true, mode: this.mode }
|
|
452
|
+
await this.ensureDirectory(this.root)
|
|
453
|
+
this.registry = new Registry(this.root)
|
|
454
|
+
await this.registry.load()
|
|
455
|
+
await this.importLegacyLocationConfig()
|
|
456
|
+
await this.refreshAnchorStores()
|
|
457
|
+
await this.migrateLegacyAnchorStore()
|
|
458
|
+
this.mode = this.defaultAnchorStore() &&
|
|
459
|
+
this.defaultAnchorStore().mode === "copy"
|
|
460
|
+
? "copy"
|
|
461
|
+
: "link"
|
|
462
|
+
await this.refreshSources()
|
|
463
|
+
this.sweeper = new Sweeper(this)
|
|
464
|
+
this.initialized = true
|
|
465
|
+
return { enabled: true, mode: this.mode }
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
async ensureInitialized() {
|
|
469
|
+
if (!this.enabled) return { enabled: false }
|
|
470
|
+
if (this.initialized) return { enabled: true, mode: this.mode }
|
|
471
|
+
if (!this.initializationPromise) {
|
|
472
|
+
this.initializationPromise = this.initializeStorage().finally(() => {
|
|
473
|
+
this.initializationPromise = null
|
|
474
|
+
})
|
|
475
|
+
}
|
|
476
|
+
return this.initializationPromise
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
async openWorkspace() {
|
|
480
|
+
if (!this.initialized) return this.ensureInitialized()
|
|
481
|
+
await this.refreshAnchorStores()
|
|
482
|
+
await this.refreshSources()
|
|
483
|
+
return { enabled: true, mode: this.mode }
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
async importLegacyLocationConfig() {
|
|
487
|
+
const config = this.readConfig()
|
|
488
|
+
const legacy = await this.registry.externalSources()
|
|
489
|
+
const locations = [...new Set([
|
|
490
|
+
...config.locations.map((item) => path.resolve(item)),
|
|
491
|
+
...legacy.map((item) => path.resolve(item))
|
|
492
|
+
])]
|
|
493
|
+
if (locations.length !== config.locations.length ||
|
|
494
|
+
locations.some((item, index) =>
|
|
495
|
+
item !== path.resolve(config.locations[index] || ""))) {
|
|
496
|
+
this.writeConfig(Object.assign({}, config, { locations }))
|
|
497
|
+
}
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
async migrateLegacyAnchorStore() {
|
|
501
|
+
const legacyRoot = path.resolve(this.root, "sha256")
|
|
502
|
+
const legacyStat = await lstatIfPresent(legacyRoot)
|
|
503
|
+
if (!legacyStat) return false
|
|
504
|
+
if (!legacyStat.isDirectory() || legacyStat.isSymbolicLink()) {
|
|
505
|
+
throw unsafeStoragePath(legacyRoot)
|
|
506
|
+
}
|
|
507
|
+
if (!(await fs.promises.readdir(legacyRoot)).length) {
|
|
508
|
+
await fs.promises.rmdir(legacyRoot)
|
|
509
|
+
return true
|
|
510
|
+
}
|
|
511
|
+
const store = this.anchorStoreForDevice(legacyStat.dev)
|
|
512
|
+
if (!store) {
|
|
513
|
+
throw unavailableStorage(
|
|
514
|
+
"The legacy anchor store has no configured store on its filesystem.")
|
|
515
|
+
}
|
|
516
|
+
const targetRoot = path.resolve(store.root, "sha256")
|
|
517
|
+
if (samePath(legacyRoot, targetRoot)) return false
|
|
518
|
+
|
|
519
|
+
await this.ensureAnchorStore(store, legacyStat.dev)
|
|
520
|
+
const targetEntries = await fs.promises.readdir(targetRoot)
|
|
521
|
+
if (targetEntries.length) {
|
|
522
|
+
throw unavailableStorage(
|
|
523
|
+
"Both the legacy and configured anchor stores contain files.")
|
|
524
|
+
}
|
|
525
|
+
await fs.promises.rmdir(targetRoot)
|
|
296
526
|
try {
|
|
297
|
-
|
|
527
|
+
await fs.promises.rename(legacyRoot, targetRoot)
|
|
298
528
|
} catch (error) {
|
|
299
|
-
|
|
529
|
+
await this.ensureDirectory(targetRoot).catch(() => {})
|
|
530
|
+
throw error
|
|
300
531
|
}
|
|
301
|
-
|
|
302
|
-
|
|
532
|
+
return true
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
async refreshAnchorStores() {
|
|
536
|
+
let config = this.readConfig()
|
|
537
|
+
const candidates = [
|
|
538
|
+
{
|
|
539
|
+
probe_path: path.resolve(this.kernel.homedir),
|
|
540
|
+
preferred_root: path.resolve(this.globalConfigRoot(), "vault"),
|
|
541
|
+
home: true
|
|
542
|
+
},
|
|
543
|
+
...config.locations.map((location) => ({
|
|
544
|
+
probe_path: path.resolve(location),
|
|
545
|
+
home: false
|
|
546
|
+
}))
|
|
547
|
+
]
|
|
548
|
+
const configured = []
|
|
549
|
+
for (const entry of config.anchor_stores) {
|
|
550
|
+
if (typeof entry.id !== "string" ||
|
|
551
|
+
!entry.id ||
|
|
552
|
+
typeof entry.root !== "string" ||
|
|
553
|
+
typeof entry.probe_path !== "string") continue
|
|
554
|
+
configured.push({
|
|
555
|
+
id: entry.id,
|
|
556
|
+
root: path.resolve(entry.root),
|
|
557
|
+
probe_path: path.resolve(entry.probe_path)
|
|
558
|
+
})
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
const observed = []
|
|
562
|
+
for (const store of configured) {
|
|
303
563
|
try {
|
|
304
|
-
const
|
|
305
|
-
|
|
306
|
-
source.available = st.isDirectory()
|
|
307
|
-
source.shareable = source.available && storeDev !== null && st.dev === storeDev && this.mode === "link"
|
|
564
|
+
const stat = await fs.promises.stat(store.probe_path)
|
|
565
|
+
if (stat.isDirectory()) observed.push({ store, dev: stat.dev })
|
|
308
566
|
} catch (error) {
|
|
309
567
|
if (!isMissingError(error)) throw error
|
|
310
|
-
source.available = false
|
|
311
|
-
source.shareable = false
|
|
312
568
|
}
|
|
313
|
-
|
|
569
|
+
}
|
|
570
|
+
let changed = configured.length !== config.anchor_stores.length
|
|
571
|
+
for (const candidate of candidates) {
|
|
572
|
+
let stat
|
|
573
|
+
try {
|
|
574
|
+
stat = await fs.promises.stat(candidate.probe_path)
|
|
575
|
+
} catch (error) {
|
|
576
|
+
if (isMissingError(error)) continue
|
|
577
|
+
throw error
|
|
578
|
+
}
|
|
579
|
+
if (!stat.isDirectory() ||
|
|
580
|
+
observed.some((item) => item.dev === stat.dev)) continue
|
|
581
|
+
const filesystemAnchor = candidate.home
|
|
582
|
+
? null
|
|
583
|
+
: await this.filesystemAnchor(candidate.probe_path, stat.dev)
|
|
584
|
+
const store = {
|
|
585
|
+
id: crypto.randomUUID(),
|
|
586
|
+
root: candidate.home
|
|
587
|
+
? candidate.preferred_root
|
|
588
|
+
: path.resolve(filesystemAnchor, ".pinokio", "vault"),
|
|
589
|
+
probe_path: candidate.probe_path
|
|
590
|
+
}
|
|
591
|
+
configured.push(store)
|
|
592
|
+
observed.push({ store, dev: stat.dev })
|
|
593
|
+
changed = true
|
|
594
|
+
}
|
|
595
|
+
if (changed) {
|
|
596
|
+
config = this.writeConfig(Object.assign({}, config, {
|
|
597
|
+
anchor_stores: configured
|
|
598
|
+
}))
|
|
314
599
|
}
|
|
315
600
|
|
|
316
|
-
const
|
|
317
|
-
|
|
318
|
-
|
|
601
|
+
const stores = []
|
|
602
|
+
for (const entry of config.anchor_stores) {
|
|
603
|
+
const store = {
|
|
604
|
+
id: entry.id,
|
|
605
|
+
root: path.resolve(entry.root),
|
|
606
|
+
probe_path: path.resolve(entry.probe_path),
|
|
607
|
+
available: false,
|
|
608
|
+
dev: null,
|
|
609
|
+
mode: null
|
|
610
|
+
}
|
|
319
611
|
try {
|
|
320
|
-
|
|
612
|
+
const probeStat = await fs.promises.stat(store.probe_path)
|
|
613
|
+
if (!probeStat.isDirectory()) {
|
|
614
|
+
stores.push(store)
|
|
615
|
+
continue
|
|
616
|
+
}
|
|
617
|
+
const rootStat = await lstatIfPresent(store.root)
|
|
618
|
+
if (rootStat &&
|
|
619
|
+
(!rootStat.isDirectory() || rootStat.isSymbolicLink())) {
|
|
620
|
+
throw unsafeStoragePath(store.root)
|
|
621
|
+
}
|
|
622
|
+
store.available = true
|
|
623
|
+
store.dev = probeStat.dev
|
|
624
|
+
store.mode = this.volumeModes.get(store.dev) || null
|
|
321
625
|
} catch (error) {
|
|
322
626
|
if (!isMissingError(error)) throw error
|
|
323
627
|
}
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
root: canonical, mount_path: mountPath, parent_id: "external"
|
|
337
|
-
}))
|
|
338
|
-
} catch (error) {
|
|
339
|
-
if (!isMissingError(error)) throw error
|
|
340
|
-
}
|
|
628
|
+
stores.push(store)
|
|
629
|
+
}
|
|
630
|
+
this._anchorStores = stores
|
|
631
|
+
this._anchorStoresById = new Map(
|
|
632
|
+
stores.map((store) => [store.id, store]))
|
|
633
|
+
this._anchorStoresByDevice = new Map()
|
|
634
|
+
for (const store of stores) {
|
|
635
|
+
if (!store.available || !Number.isFinite(store.dev)) continue
|
|
636
|
+
if (this._anchorStoresByDevice.has(store.dev)) {
|
|
637
|
+
store.available = false
|
|
638
|
+
store.error = "duplicate_device"
|
|
639
|
+
continue
|
|
341
640
|
}
|
|
641
|
+
this._anchorStoresByDevice.set(store.dev, store)
|
|
342
642
|
}
|
|
643
|
+
return stores
|
|
644
|
+
}
|
|
343
645
|
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
646
|
+
async filesystemAnchor(directory, expectedDev = null) {
|
|
647
|
+
let current = path.resolve(directory)
|
|
648
|
+
const first = await fs.promises.stat(current)
|
|
649
|
+
const dev = expectedDev === null ? first.dev : expectedDev
|
|
650
|
+
if (!first.isDirectory() || first.dev !== dev) {
|
|
651
|
+
throw new Error("The configured location is not on the expected filesystem.")
|
|
349
652
|
}
|
|
350
|
-
|
|
351
|
-
const
|
|
352
|
-
if (
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
653
|
+
while (true) {
|
|
654
|
+
const parent = path.dirname(current)
|
|
655
|
+
if (parent === current) return current
|
|
656
|
+
let parentStat
|
|
657
|
+
try {
|
|
658
|
+
parentStat = await fs.promises.stat(parent)
|
|
659
|
+
} catch (error) {
|
|
660
|
+
if (isMissingError(error)) return current
|
|
661
|
+
throw error
|
|
357
662
|
}
|
|
663
|
+
if (parentStat.dev !== dev) return current
|
|
664
|
+
current = parent
|
|
358
665
|
}
|
|
359
|
-
|
|
360
|
-
if (await this.directoryIfSafe(this.sourceRoot)) {
|
|
361
|
-
await addExternalEntries(this.sourceRoot, "vault:")
|
|
362
|
-
}
|
|
666
|
+
}
|
|
363
667
|
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
homeEntries = await fs.promises.readdir(home, { withFileTypes: true })
|
|
367
|
-
} catch (error) {
|
|
368
|
-
if (!isMissingError(error)) throw error
|
|
369
|
-
}
|
|
370
|
-
for (const entry of homeEntries) {
|
|
371
|
-
if (!entry.isDirectory() || entry.isSymbolicLink() || entry.name === "api" || entry.name === "vault") continue
|
|
372
|
-
sources.push(await decorate({
|
|
373
|
-
id: sourceId("folder", entry.name), kind: "folder", label: entry.name,
|
|
374
|
-
root: path.resolve(home, entry.name), parent_id: "pinokio"
|
|
375
|
-
}))
|
|
376
|
-
}
|
|
377
|
-
const homeSource = sources.find((source) => source.id === "pinokio")
|
|
378
|
-
if (homeSource) await decorate(homeSource)
|
|
379
|
-
this._sources = sources
|
|
380
|
-
this._sourceBases = new Map()
|
|
381
|
-
for (const source of sources) {
|
|
382
|
-
if (!source.root || source.kind === "virtual") continue
|
|
383
|
-
for (const base of [source.root, source.mount_path]) {
|
|
384
|
-
if (!base) continue
|
|
385
|
-
const resolved = path.resolve(base)
|
|
386
|
-
const key = process.platform === "win32" ? resolved.toLowerCase() : resolved
|
|
387
|
-
if (!this._sourceBases.has(key)) this._sourceBases.set(key, [])
|
|
388
|
-
this._sourceBases.get(key).push(source)
|
|
389
|
-
}
|
|
390
|
-
}
|
|
391
|
-
return sources
|
|
668
|
+
anchorStoreForDevice(dev) {
|
|
669
|
+
return this._anchorStoresByDevice.get(dev) || null
|
|
392
670
|
}
|
|
393
|
-
|
|
394
|
-
|
|
671
|
+
|
|
672
|
+
anchorStores() {
|
|
673
|
+
return this._anchorStores
|
|
395
674
|
}
|
|
396
|
-
async addExternalSource(folderPath) {
|
|
397
|
-
if (!this.enabled) throw new Error("Save space is disabled.")
|
|
398
|
-
if (typeof folderPath !== "string" || !path.isAbsolute(folderPath.trim())) {
|
|
399
|
-
throw new Error("Choose a valid folder.")
|
|
400
|
-
}
|
|
401
|
-
const requested = folderPath.trim()
|
|
402
|
-
let canonical
|
|
403
|
-
let stats
|
|
404
|
-
try {
|
|
405
|
-
canonical = path.resolve(await fs.promises.realpath(requested))
|
|
406
|
-
stats = await fs.promises.stat(canonical)
|
|
407
|
-
} catch (error) {
|
|
408
|
-
throw new Error("That folder is no longer available.")
|
|
409
|
-
}
|
|
410
|
-
if (!stats.isDirectory()) throw new Error("Choose a folder, not a file.")
|
|
411
675
|
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
}
|
|
676
|
+
anchorStoreForPath(filePath) {
|
|
677
|
+
return this._anchorStores.find((store) =>
|
|
678
|
+
isPathWithin(path.resolve(store.root, "sha256"), filePath)) || null
|
|
679
|
+
}
|
|
417
680
|
|
|
418
|
-
|
|
419
|
-
const
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
const importRoot = this.sourceRoot
|
|
425
|
-
await this.ensureDirectory(importRoot)
|
|
426
|
-
const baseLabel = path.basename(canonical) || "external-folder"
|
|
427
|
-
let mountPath = null
|
|
428
|
-
for (let index = 1; index < 10000; index++) {
|
|
429
|
-
const label = index === 1 ? baseLabel : `${baseLabel}-${index}`
|
|
430
|
-
const candidate = path.resolve(importRoot, label)
|
|
681
|
+
storageRoots() {
|
|
682
|
+
const roots = [
|
|
683
|
+
this.root,
|
|
684
|
+
...this._anchorStores.map((store) => store.root)
|
|
685
|
+
].map((item) => path.resolve(item))
|
|
686
|
+
for (const root of [...roots]) {
|
|
431
687
|
try {
|
|
432
|
-
|
|
433
|
-
mountPath = candidate
|
|
434
|
-
break
|
|
688
|
+
roots.push(path.resolve(fs.realpathSync(root)))
|
|
435
689
|
} catch (error) {
|
|
436
|
-
if (error
|
|
437
|
-
if (error && (error.code === "EACCES" || error.code === "EPERM")) {
|
|
438
|
-
throw new Error("Pinokio does not have permission to add that folder.")
|
|
439
|
-
}
|
|
440
|
-
throw new Error("Pinokio could not add that folder.")
|
|
690
|
+
if (!isMissingError(error)) throw error
|
|
441
691
|
}
|
|
442
692
|
}
|
|
443
|
-
|
|
693
|
+
return [...new Set(roots)]
|
|
694
|
+
}
|
|
444
695
|
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
if (!source) throw new Error("The folder could not be added. Restart Pinokio and try again.")
|
|
449
|
-
return { created: true, source }
|
|
450
|
-
} catch (error) {
|
|
451
|
-
// Adding a source is transactional. Roll back only the exact directory
|
|
452
|
-
// link created above; preserve any path an external writer replaced.
|
|
453
|
-
try {
|
|
454
|
-
const [mounted, target] = await Promise.all([
|
|
455
|
-
fs.promises.lstat(mountPath),
|
|
456
|
-
fs.promises.realpath(mountPath)
|
|
457
|
-
])
|
|
458
|
-
if (mounted.isSymbolicLink() && samePath(target, canonical)) {
|
|
459
|
-
await fs.promises.unlink(mountPath)
|
|
460
|
-
}
|
|
461
|
-
} catch {
|
|
462
|
-
// The original error remains authoritative; an unverified path is
|
|
463
|
-
// deliberately preserved rather than deleted.
|
|
464
|
-
}
|
|
465
|
-
throw error
|
|
466
|
-
}
|
|
467
|
-
}
|
|
468
|
-
sourceForPath(filePath, preferredId) {
|
|
469
|
-
let cursor = path.resolve(filePath)
|
|
470
|
-
while (true) {
|
|
471
|
-
const key = process.platform === "win32" ? cursor.toLowerCase() : cursor
|
|
472
|
-
const matches = this._sourceBases.get(key)
|
|
473
|
-
if (matches && matches.length) {
|
|
474
|
-
return matches.find((source) => source.id === preferredId) || matches[0]
|
|
475
|
-
}
|
|
476
|
-
const parent = path.dirname(cursor)
|
|
477
|
-
if (parent === cursor) return null
|
|
478
|
-
cursor = parent
|
|
479
|
-
}
|
|
480
|
-
}
|
|
481
|
-
async canonicalPathIsWithinSource(filePath, source, options = {}) {
|
|
482
|
-
if (!source || !source.root) return false
|
|
483
|
-
try {
|
|
484
|
-
const [canonicalRoot, canonicalFile] = await Promise.all([
|
|
485
|
-
fs.promises.realpath(source.root),
|
|
486
|
-
fs.promises.realpath(filePath)
|
|
487
|
-
])
|
|
488
|
-
return isPathWithin(canonicalRoot, canonicalFile)
|
|
489
|
-
} catch (error) {
|
|
490
|
-
if (options.strictErrors && !isMissingError(error)) throw error
|
|
491
|
-
return false
|
|
492
|
-
}
|
|
493
|
-
}
|
|
494
|
-
locationForPath(filePath, preferredId) {
|
|
495
|
-
const source = this.sourceForPath(filePath, preferredId)
|
|
496
|
-
if (!source) return { source_id: null, relative_path: path.basename(filePath) }
|
|
497
|
-
const absolute = path.resolve(filePath)
|
|
498
|
-
let base = source.root
|
|
499
|
-
if (source.mount_path && isPathWithin(source.mount_path, absolute)) base = source.mount_path
|
|
500
|
-
const relative = path.relative(base, absolute).split(path.sep).join("/") || path.basename(absolute)
|
|
501
|
-
return {
|
|
502
|
-
source_id: source.id,
|
|
503
|
-
source_kind: source.kind,
|
|
504
|
-
source_label: source.label,
|
|
505
|
-
relative_path: relative
|
|
506
|
-
}
|
|
507
|
-
}
|
|
508
|
-
async recordEvent(event) {
|
|
509
|
-
let entry = event
|
|
510
|
-
if (event.path) {
|
|
511
|
-
const location = this.locationForPath(event.path, event.source_id)
|
|
512
|
-
const context = location.source_id ? location : { relative_path: event.path }
|
|
513
|
-
entry = Object.assign({}, event, context)
|
|
514
|
-
}
|
|
515
|
-
try {
|
|
516
|
-
await this.registry.appendEvent(entry)
|
|
517
|
-
return true
|
|
518
|
-
} catch (error) {
|
|
519
|
-
return false
|
|
520
|
-
}
|
|
521
|
-
}
|
|
522
|
-
scanSource(scopeId) {
|
|
523
|
-
if (!scopeId) return null
|
|
524
|
-
return this._sources.find((source) =>
|
|
525
|
-
source.id === scopeId && source.kind !== "virtual" && source.available && source.root) || null
|
|
526
|
-
}
|
|
527
|
-
scanRoots(scopeId = null) {
|
|
528
|
-
if (scopeId) {
|
|
529
|
-
const source = this.scanSource(scopeId)
|
|
530
|
-
return source ? [{ root: path.resolve(source.root), source_id: source.id }] : []
|
|
531
|
-
}
|
|
532
|
-
const home = path.resolve(this.kernel.homedir)
|
|
533
|
-
const candidates = [{ root: home, source_id: "pinokio" }]
|
|
534
|
-
for (const source of this._sources) {
|
|
535
|
-
if (source.kind !== "external" || !source.available || !source.root) continue
|
|
536
|
-
const canonical = path.resolve(source.root)
|
|
537
|
-
if (isPathWithin(home, canonical) || isPathWithin(canonical, home)) continue
|
|
538
|
-
candidates.push({ root: canonical, source_id: source.id })
|
|
539
|
-
}
|
|
540
|
-
candidates.sort((left, right) => left.root.length - right.root.length)
|
|
541
|
-
const roots = []
|
|
542
|
-
for (const candidate of candidates) {
|
|
543
|
-
if (roots.some((existing) => isPathWithin(existing.root, candidate.root))) continue
|
|
544
|
-
roots.push(candidate)
|
|
545
|
-
}
|
|
546
|
-
return roots
|
|
547
|
-
}
|
|
548
|
-
reconcileConfiguredSources() {
|
|
549
|
-
if (!this.registry) return
|
|
550
|
-
const configured = (filePath, sourceId) => !!this.sourceForPath(filePath, sourceId)
|
|
551
|
-
for (const [filePath, entry] of [...this.registry.links]) {
|
|
552
|
-
if (!configured(filePath, entry.source_id)) this.registry.untrack(filePath)
|
|
553
|
-
}
|
|
554
|
-
for (const [filePath, entry] of [...this.registry.duplicates]) {
|
|
555
|
-
if (!configured(filePath, entry.source_id)) this.registry.untrack(filePath)
|
|
556
|
-
}
|
|
557
|
-
for (const [filePath, entry] of [...this.registry.scanIndex]) {
|
|
558
|
-
if (!configured(filePath, entry.source_id)) this.registry.untrack(filePath)
|
|
559
|
-
}
|
|
560
|
-
}
|
|
561
|
-
// Kill switch: system ENVIRONMENT variable PINOKIO_VAULT, default unset =
|
|
562
|
-
// enabled. Read directly (single key) to avoid loading the environment
|
|
563
|
-
// module chain before the kernel is fully up.
|
|
564
|
-
async isEnabled() {
|
|
565
|
-
let value = process.env.PINOKIO_VAULT
|
|
566
|
-
if (value === undefined && this.kernel.homedir) {
|
|
567
|
-
try {
|
|
568
|
-
const raw = await fs.promises.readFile(path.resolve(this.kernel.homedir, "ENVIRONMENT"), "utf8")
|
|
569
|
-
for (const line of raw.split("\n")) {
|
|
570
|
-
const m = line.match(/^\s*PINOKIO_VAULT\s*=\s*(.*)\s*$/)
|
|
571
|
-
if (m) value = m[1].trim()
|
|
572
|
-
}
|
|
573
|
-
} catch (error) {
|
|
574
|
-
if (!isMissingError(error)) throw error
|
|
575
|
-
}
|
|
576
|
-
}
|
|
577
|
-
return String(value).toLowerCase() !== "false"
|
|
578
|
-
}
|
|
579
|
-
// Disabled ⇒ do nothing observable: no directory, no registry, no probes.
|
|
580
|
-
async init(options = {}) {
|
|
581
|
-
this.enabled = await this.isEnabled()
|
|
582
|
-
if (!this.enabled) return { enabled: false }
|
|
583
|
-
if (options.existingOnly) {
|
|
584
|
-
try {
|
|
585
|
-
await fs.promises.stat(this.root)
|
|
586
|
-
} catch (error) {
|
|
587
|
-
if (error && error.code === "ENOENT") return { enabled: true, fresh: true }
|
|
588
|
-
throw error
|
|
589
|
-
}
|
|
590
|
-
}
|
|
591
|
-
return this.initializeStorage()
|
|
592
|
-
}
|
|
593
|
-
async initializeStorage() {
|
|
594
|
-
if (this.initialized) return { enabled: true, mode: this.mode }
|
|
595
|
-
await this.ensureDirectory(this.root)
|
|
596
|
-
await this.ensureDirectory(this.blobRoot)
|
|
597
|
-
this.registry = new Registry(this.root)
|
|
598
|
-
this.mode = await this.probe(this.root)
|
|
599
|
-
await this.refreshSources()
|
|
600
|
-
const loaded = await this.registry.load()
|
|
601
|
-
const missingWithBlobs = !loaded.existed && await this.hasStoredBlobs()
|
|
602
|
-
if (loaded.corrupt || missingWithBlobs) {
|
|
603
|
-
await this.rebuild(undefined, { preserveState: false })
|
|
604
|
-
}
|
|
605
|
-
this.sweeper = new Sweeper(this)
|
|
606
|
-
this.initialized = true
|
|
607
|
-
return {
|
|
608
|
-
enabled: true,
|
|
609
|
-
mode: this.mode,
|
|
610
|
-
corrupt_recovered: !!loaded.corrupt,
|
|
611
|
-
missing_recovered: missingWithBlobs
|
|
612
|
-
}
|
|
696
|
+
isStorageRoot(directory) {
|
|
697
|
+
const target = path.resolve(directory)
|
|
698
|
+
return this.storageRoots().some((root) => samePath(root, target))
|
|
613
699
|
}
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
if (!this.initializationPromise) {
|
|
618
|
-
this.initializationPromise = (async () => {
|
|
619
|
-
const result = this.initialized
|
|
620
|
-
? { enabled: true, mode: this.mode }
|
|
621
|
-
: await this.initializeStorage()
|
|
622
|
-
await this.runExclusive(async () => {
|
|
623
|
-
await this.verify()
|
|
624
|
-
await this.registry.flush()
|
|
625
|
-
})
|
|
626
|
-
this.verificationPending = false
|
|
627
|
-
return result
|
|
628
|
-
})().catch((error) => {
|
|
629
|
-
if (this.initialized) this.verificationPending = true
|
|
630
|
-
throw error
|
|
631
|
-
}).finally(() => {
|
|
632
|
-
this.initializationPromise = null
|
|
633
|
-
})
|
|
634
|
-
}
|
|
635
|
-
return this.initializationPromise
|
|
636
|
-
}
|
|
637
|
-
async hasStoredBlobs() {
|
|
638
|
-
let shards = []
|
|
639
|
-
try {
|
|
640
|
-
shards = await fs.promises.readdir(this.blobRoot, { withFileTypes: true })
|
|
641
|
-
} catch (error) {
|
|
642
|
-
if (isMissingError(error)) return false
|
|
643
|
-
throw error
|
|
644
|
-
}
|
|
645
|
-
for (const shard of shards) {
|
|
646
|
-
if (!shard.isDirectory() || !/^[0-9a-f]{2}$/.test(shard.name)) continue
|
|
647
|
-
const shardPath = path.resolve(this.blobRoot, shard.name)
|
|
648
|
-
const shardStat = await this.directoryIfSafe(shardPath)
|
|
649
|
-
if (!shardStat) continue
|
|
650
|
-
let names = []
|
|
651
|
-
try {
|
|
652
|
-
names = await fs.promises.readdir(shardPath)
|
|
653
|
-
} catch (error) {
|
|
654
|
-
if (isMissingError(error)) continue
|
|
655
|
-
throw error
|
|
656
|
-
}
|
|
657
|
-
if (names.some((name) => SHA256_RE.test(name) && name.startsWith(shard.name))) return true
|
|
658
|
-
}
|
|
659
|
-
return false
|
|
660
|
-
}
|
|
661
|
-
// Capability probe: temp file + node:fs.link + nlink check. Once per volume.
|
|
662
|
-
async probe(dir) {
|
|
663
|
-
const dev = (await fs.promises.stat(dir)).dev
|
|
700
|
+
|
|
701
|
+
async probe(directory) {
|
|
702
|
+
const dev = (await fs.promises.stat(directory)).dev
|
|
664
703
|
if (this.volumeModes.has(dev)) return this.volumeModes.get(dev)
|
|
665
|
-
const
|
|
666
|
-
|
|
704
|
+
const first = path.resolve(
|
|
705
|
+
directory, `.pinokio-probe-${crypto.randomBytes(6).toString("hex")}`)
|
|
706
|
+
const second = `${first}-link`
|
|
667
707
|
let mode = "copy"
|
|
668
|
-
let
|
|
669
|
-
let
|
|
708
|
+
let firstStat = null
|
|
709
|
+
let secondStat = null
|
|
670
710
|
let failure = null
|
|
671
711
|
try {
|
|
672
|
-
await fs.promises.writeFile(
|
|
673
|
-
|
|
674
|
-
await fs.promises.link(
|
|
675
|
-
|
|
676
|
-
if (sameIdentity(
|
|
712
|
+
await fs.promises.writeFile(first, "probe", { flag: "wx" })
|
|
713
|
+
firstStat = await fs.promises.lstat(first)
|
|
714
|
+
await fs.promises.link(first, second)
|
|
715
|
+
secondStat = await fs.promises.lstat(second)
|
|
716
|
+
if (sameIdentity(firstStat, secondStat) && secondStat.nlink === 2) {
|
|
717
|
+
mode = "link"
|
|
718
|
+
}
|
|
677
719
|
} catch (error) {
|
|
678
720
|
if (!NO_LINK_CODES.has(error && error.code)) failure = error
|
|
679
721
|
} finally {
|
|
680
722
|
try {
|
|
681
|
-
await unlinkIfSame(
|
|
682
|
-
await unlinkIfSame(
|
|
723
|
+
await unlinkIfSame(second, secondStat)
|
|
724
|
+
await unlinkIfSame(first, firstStat)
|
|
683
725
|
} catch (error) {
|
|
684
726
|
if (!failure) failure = error
|
|
685
727
|
}
|
|
@@ -688,6 +730,7 @@ class Vault {
|
|
|
688
730
|
this.volumeModes.set(dev, mode)
|
|
689
731
|
return mode
|
|
690
732
|
}
|
|
733
|
+
|
|
691
734
|
failHashWorker(worker, error, terminate = false) {
|
|
692
735
|
if (this.worker === worker) {
|
|
693
736
|
if (this.workerIdleTimer) clearTimeout(this.workerIdleTimer)
|
|
@@ -702,6 +745,7 @@ class Vault {
|
|
|
702
745
|
}
|
|
703
746
|
if (terminate) worker.terminate().catch(() => {})
|
|
704
747
|
}
|
|
748
|
+
|
|
705
749
|
async hashFile(filePath, options = {}) {
|
|
706
750
|
if (this.workerIdleTimer) {
|
|
707
751
|
clearTimeout(this.workerIdleTimer)
|
|
@@ -711,7 +755,9 @@ class Vault {
|
|
|
711
755
|
const worker = new Worker(path.resolve(__dirname, "hash_worker.js"))
|
|
712
756
|
this.worker = worker
|
|
713
757
|
worker.unref()
|
|
714
|
-
worker.on("message", ({
|
|
758
|
+
worker.on("message", ({
|
|
759
|
+
id, hash, size, bytes_read: bytesRead, error, code
|
|
760
|
+
}) => {
|
|
715
761
|
const job = this.workerJobs.get(id)
|
|
716
762
|
if (!job || job.worker !== worker) return
|
|
717
763
|
if (Number.isFinite(bytesRead)) {
|
|
@@ -729,8 +775,6 @@ class Vault {
|
|
|
729
775
|
job.reportProgress(size)
|
|
730
776
|
job.resolve({ hash, size })
|
|
731
777
|
}
|
|
732
|
-
// Keep the worker warm across the scan queue. Starting one worker per
|
|
733
|
-
// file costs ~30ms and adds up quickly on a first scan.
|
|
734
778
|
if (this.workerJobs.size === 0 && this.worker === worker) {
|
|
735
779
|
this.workerIdleTimer = setTimeout(() => {
|
|
736
780
|
this.workerIdleTimer = null
|
|
@@ -744,37 +788,49 @@ class Vault {
|
|
|
744
788
|
})
|
|
745
789
|
worker.on("error", (error) => {
|
|
746
790
|
if (this.worker !== worker) return
|
|
747
|
-
|
|
791
|
+
const failure = new Error(error && error.message
|
|
792
|
+
? error.message
|
|
793
|
+
: "Hash worker failed.")
|
|
794
|
+
failure.code = "EVAULTHASHWORKER"
|
|
795
|
+
this.failHashWorker(worker, failure)
|
|
748
796
|
})
|
|
749
797
|
worker.on("exit", (code) => {
|
|
750
|
-
if (this.worker
|
|
751
|
-
|
|
798
|
+
if (this.worker === worker) {
|
|
799
|
+
const failure = new Error(`hash worker exited with code ${code}`)
|
|
800
|
+
failure.code = "EVAULTHASHWORKER"
|
|
801
|
+
this.failHashWorker(worker, failure)
|
|
802
|
+
}
|
|
752
803
|
})
|
|
753
804
|
}
|
|
754
805
|
const worker = this.worker
|
|
755
806
|
const id = ++this.workerSeq
|
|
756
807
|
return new Promise((resolve, reject) => {
|
|
757
|
-
const onProgress = typeof options.onProgress === "function"
|
|
808
|
+
const onProgress = typeof options.onProgress === "function"
|
|
809
|
+
? options.onProgress
|
|
810
|
+
: null
|
|
758
811
|
const job = {
|
|
759
812
|
worker,
|
|
760
813
|
resolve,
|
|
761
814
|
reject,
|
|
762
815
|
reportProgress: (bytes) => {
|
|
763
816
|
if (!onProgress) return
|
|
764
|
-
|
|
765
|
-
|
|
817
|
+
try {
|
|
818
|
+
onProgress(bytes)
|
|
819
|
+
} catch (error) {}
|
|
766
820
|
},
|
|
767
821
|
inactivityTimer: null,
|
|
768
822
|
resetInactivity: null
|
|
769
823
|
}
|
|
770
824
|
job.resetInactivity = () => {
|
|
771
825
|
clearTimeout(job.inactivityTimer)
|
|
772
|
-
const
|
|
826
|
+
const timeout = Math.max(
|
|
827
|
+
1, Number(this.hashInactivityMs) || HASH_INACTIVITY_MS)
|
|
773
828
|
job.inactivityTimer = setTimeout(() => {
|
|
774
|
-
const failure = new Error(
|
|
829
|
+
const failure = new Error(
|
|
830
|
+
`Timed out while reading ${path.basename(filePath)}`)
|
|
775
831
|
failure.code = "ETIMEDOUT"
|
|
776
832
|
this.failHashWorker(worker, failure, true)
|
|
777
|
-
},
|
|
833
|
+
}, timeout)
|
|
778
834
|
if (job.inactivityTimer.unref) job.inactivityTimer.unref()
|
|
779
835
|
}
|
|
780
836
|
this.workerJobs.set(id, job)
|
|
@@ -782,1315 +838,1893 @@ class Vault {
|
|
|
782
838
|
try {
|
|
783
839
|
worker.postMessage({ id, filePath })
|
|
784
840
|
} catch (error) {
|
|
785
|
-
|
|
841
|
+
const failure = new Error(error && error.message
|
|
842
|
+
? error.message
|
|
843
|
+
: "Hash worker could not accept work.")
|
|
844
|
+
failure.code = "EVAULTHASHWORKER"
|
|
845
|
+
this.failHashWorker(worker, failure, true)
|
|
786
846
|
}
|
|
787
847
|
})
|
|
788
848
|
}
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
const
|
|
792
|
-
const
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
849
|
+
|
|
850
|
+
async refreshSources() {
|
|
851
|
+
const home = path.resolve(this.kernel.homedir)
|
|
852
|
+
const apiRoot = path.resolve(home, "api")
|
|
853
|
+
const sources = [
|
|
854
|
+
{
|
|
855
|
+
id: "pinokio", kind: "pinokio", label: "Pinokio",
|
|
856
|
+
root: home, parent_id: null
|
|
857
|
+
},
|
|
858
|
+
{
|
|
859
|
+
id: "apps", kind: "virtual", label: "Apps",
|
|
860
|
+
root: apiRoot, parent_id: "pinokio"
|
|
861
|
+
},
|
|
862
|
+
{
|
|
863
|
+
id: "external", kind: "virtual", label: "External folders",
|
|
864
|
+
root: null, parent_id: null
|
|
865
|
+
}
|
|
866
|
+
]
|
|
867
|
+
const decorate = async (source) => {
|
|
868
|
+
if (!source.root) return source
|
|
796
869
|
try {
|
|
797
|
-
const
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
if (entry.hash !== hash || entry.mode !== "link" ||
|
|
816
|
-
entry.dev !== storeStat.dev || entry.ino !== storeStat.ino) continue
|
|
817
|
-
const expected = this.registry.scanIndex.get(linkPath)
|
|
818
|
-
if (expected && expected.hash === hash && sameSnapshot(expected, storeStat)) {
|
|
819
|
-
return { valid: true, snapshot: fileSnapshot(storeStat) }
|
|
870
|
+
const [stat, realRoot] = await Promise.all([
|
|
871
|
+
fs.promises.lstat(source.root),
|
|
872
|
+
fs.promises.realpath(source.root)
|
|
873
|
+
])
|
|
874
|
+
source.canonical_root = path.resolve(realRoot)
|
|
875
|
+
source.dev = stat.dev
|
|
876
|
+
source.available = stat.isDirectory() &&
|
|
877
|
+
!stat.isSymbolicLink() &&
|
|
878
|
+
(source.kind !== "external" || samePath(realRoot, source.root))
|
|
879
|
+
const store = this.anchorStoreForDevice(stat.dev)
|
|
880
|
+
source.shareable = source.available &&
|
|
881
|
+
!!store &&
|
|
882
|
+
store.mode !== "copy"
|
|
883
|
+
source.store_id = store ? store.id : null
|
|
884
|
+
} catch (error) {
|
|
885
|
+
if (!isMissingError(error)) throw error
|
|
886
|
+
source.available = false
|
|
887
|
+
source.shareable = false
|
|
820
888
|
}
|
|
889
|
+
return source
|
|
821
890
|
}
|
|
822
891
|
|
|
823
|
-
|
|
824
|
-
let hashed
|
|
892
|
+
let apps = []
|
|
825
893
|
try {
|
|
826
|
-
|
|
894
|
+
apps = await fs.promises.readdir(apiRoot, { withFileTypes: true })
|
|
827
895
|
} catch (error) {
|
|
828
|
-
|
|
896
|
+
if (!isMissingError(error)) throw error
|
|
897
|
+
}
|
|
898
|
+
for (const entry of apps) {
|
|
899
|
+
if (!entry.isDirectory() || entry.isSymbolicLink()) continue
|
|
900
|
+
sources.push(await decorate({
|
|
901
|
+
id: sourceId("app", entry.name),
|
|
902
|
+
kind: "app",
|
|
903
|
+
label: entry.name,
|
|
904
|
+
app: entry.name,
|
|
905
|
+
root: path.resolve(apiRoot, entry.name),
|
|
906
|
+
parent_id: "apps"
|
|
907
|
+
}))
|
|
908
|
+
}
|
|
909
|
+
|
|
910
|
+
for (const configuredPath of this.configuredLocations()) {
|
|
911
|
+
sources.push(await decorate({
|
|
912
|
+
id: sourceId("external", configuredPath),
|
|
913
|
+
kind: "external",
|
|
914
|
+
label: path.basename(configuredPath) || configuredPath,
|
|
915
|
+
root: configuredPath,
|
|
916
|
+
parent_id: "external",
|
|
917
|
+
configured: true
|
|
918
|
+
}))
|
|
829
919
|
}
|
|
830
|
-
|
|
920
|
+
|
|
921
|
+
let homeEntries = []
|
|
831
922
|
try {
|
|
832
|
-
|
|
923
|
+
homeEntries = await fs.promises.readdir(home, { withFileTypes: true })
|
|
833
924
|
} catch (error) {
|
|
834
|
-
|
|
925
|
+
if (!isMissingError(error)) throw error
|
|
835
926
|
}
|
|
836
|
-
|
|
837
|
-
|
|
927
|
+
for (const entry of homeEntries) {
|
|
928
|
+
if (!entry.isDirectory() || entry.isSymbolicLink() ||
|
|
929
|
+
entry.name === "api" || entry.name === "vault") continue
|
|
930
|
+
sources.push(await decorate({
|
|
931
|
+
id: sourceId("folder", entry.name),
|
|
932
|
+
kind: "folder",
|
|
933
|
+
label: entry.name,
|
|
934
|
+
root: path.resolve(home, entry.name),
|
|
935
|
+
parent_id: "pinokio"
|
|
936
|
+
}))
|
|
838
937
|
}
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
})
|
|
861
|
-
if (storeStat) {
|
|
862
|
-
if (storeStat.dev === st.dev && storeStat.ino === st.ino) {
|
|
863
|
-
const current = await lstatIfPresent(filePath)
|
|
864
|
-
if (!sameSnapshot(before, current)) return { status: "stale" }
|
|
865
|
-
this.registry.addBlob(hash, { size: st.size, source_urls: meta.source_urls || [] })
|
|
866
|
-
this.registry.addLink(filePath, linkEntry)
|
|
867
|
-
await this.refreshLinkSnapshots(hash, st.dev, st.ino)
|
|
868
|
-
return { status: "already" }
|
|
869
|
-
}
|
|
870
|
-
const current = await lstatIfPresent(filePath)
|
|
871
|
-
if (!sameSnapshot(before, current)) return { status: "stale" }
|
|
872
|
-
return { status: "duplicate", storePath }
|
|
873
|
-
}
|
|
874
|
-
const registerCopy = async () => {
|
|
875
|
-
const current = await lstatIfPresent(filePath)
|
|
876
|
-
if (!sameSnapshot(before, current)) return { status: "stale" }
|
|
877
|
-
linkEntry.mode = "copy"
|
|
878
|
-
this.registry.addBlob(hash, { size: st.size, source_urls: meta.source_urls || [] })
|
|
879
|
-
this.registry.addLink(filePath, linkEntry)
|
|
880
|
-
await this.refreshLinkSnapshots(hash, st.dev, st.ino)
|
|
881
|
-
await this.recordEvent({
|
|
882
|
-
kind: "adopt", hash, path: filePath, app: meta.app || null,
|
|
883
|
-
source_id: meta.source_id || null, bytes_saved: 0, mode: "copy"
|
|
884
|
-
})
|
|
885
|
-
return { status: "copy-mode" }
|
|
938
|
+
|
|
939
|
+
const pinokio = sources.find((source) => source.id === "pinokio")
|
|
940
|
+
if (pinokio) await decorate(pinokio)
|
|
941
|
+
this._sources = sources
|
|
942
|
+
this._sourcesById = new Map(
|
|
943
|
+
sources.map((source) => [source.id, source])
|
|
944
|
+
)
|
|
945
|
+
this._sourceBases = new Map()
|
|
946
|
+
for (const source of sources) {
|
|
947
|
+
if (!source.root || source.kind === "virtual") continue
|
|
948
|
+
const bases = new Set([
|
|
949
|
+
path.resolve(source.root),
|
|
950
|
+
source.canonical_root && path.resolve(source.canonical_root)
|
|
951
|
+
].filter(Boolean))
|
|
952
|
+
for (const resolved of bases) {
|
|
953
|
+
const key = process.platform === "win32"
|
|
954
|
+
? resolved.toLowerCase()
|
|
955
|
+
: resolved
|
|
956
|
+
if (!this._sourceBases.has(key)) this._sourceBases.set(key, [])
|
|
957
|
+
this._sourceBases.get(key).push(source)
|
|
958
|
+
}
|
|
886
959
|
}
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
960
|
+
return sources
|
|
961
|
+
}
|
|
962
|
+
|
|
963
|
+
sources() {
|
|
964
|
+
return this._sources
|
|
965
|
+
}
|
|
966
|
+
|
|
967
|
+
sourceForPath(filePath, preferredId = null) {
|
|
968
|
+
let cursor = path.resolve(filePath)
|
|
969
|
+
while (true) {
|
|
970
|
+
const key = process.platform === "win32"
|
|
971
|
+
? cursor.toLowerCase()
|
|
972
|
+
: cursor
|
|
973
|
+
const matches = this._sourceBases.get(key)
|
|
974
|
+
if (matches && matches.length) {
|
|
975
|
+
return matches.find((source) => source.id === preferredId) || matches[0]
|
|
893
976
|
}
|
|
894
|
-
|
|
977
|
+
const parent = path.dirname(cursor)
|
|
978
|
+
if (parent === cursor) return null
|
|
979
|
+
cursor = parent
|
|
895
980
|
}
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
981
|
+
}
|
|
982
|
+
|
|
983
|
+
sourceIsWithinScope(source, scopeId) {
|
|
984
|
+
if (!scopeId) return true
|
|
985
|
+
const seen = new Set()
|
|
986
|
+
let current = source
|
|
987
|
+
while (current && !seen.has(current.id)) {
|
|
988
|
+
if (current.id === scopeId) return true
|
|
989
|
+
seen.add(current.id)
|
|
990
|
+
current = this._sourcesById.get(current.parent_id)
|
|
991
|
+
}
|
|
992
|
+
return false
|
|
993
|
+
}
|
|
994
|
+
|
|
995
|
+
scanSource(scopeId) {
|
|
996
|
+
if (!scopeId) return null
|
|
997
|
+
const source = this._sourcesById.get(scopeId)
|
|
998
|
+
return source &&
|
|
999
|
+
source.kind !== "virtual" &&
|
|
1000
|
+
source.available &&
|
|
1001
|
+
source.root
|
|
1002
|
+
? source
|
|
1003
|
+
: null
|
|
1004
|
+
}
|
|
1005
|
+
|
|
1006
|
+
scanRoots(scopeId = null) {
|
|
1007
|
+
if (scopeId) {
|
|
1008
|
+
const source = this.scanSource(scopeId)
|
|
1009
|
+
return source
|
|
1010
|
+
? [{ root: path.resolve(source.root), source_id: source.id }]
|
|
1011
|
+
: []
|
|
1012
|
+
}
|
|
1013
|
+
const pinokio = this._sourcesById.get("pinokio")
|
|
1014
|
+
const candidates = [
|
|
1015
|
+
{
|
|
1016
|
+
root: path.resolve(pinokio && pinokio.root || this.kernel.homedir),
|
|
1017
|
+
canonical_root: path.resolve(
|
|
1018
|
+
pinokio && pinokio.canonical_root ||
|
|
1019
|
+
pinokio && pinokio.root ||
|
|
1020
|
+
this.kernel.homedir
|
|
1021
|
+
),
|
|
1022
|
+
source_id: "pinokio"
|
|
1023
|
+
},
|
|
1024
|
+
...this._sources
|
|
1025
|
+
.filter((source) =>
|
|
1026
|
+
source.kind === "external" && source.available && source.root)
|
|
1027
|
+
.map((source) => ({
|
|
1028
|
+
root: path.resolve(source.root),
|
|
1029
|
+
canonical_root: path.resolve(source.canonical_root || source.root),
|
|
1030
|
+
source_id: source.id
|
|
1031
|
+
}))
|
|
1032
|
+
].sort((left, right) =>
|
|
1033
|
+
left.canonical_root.length - right.canonical_root.length ||
|
|
1034
|
+
left.canonical_root.localeCompare(right.canonical_root))
|
|
1035
|
+
const roots = []
|
|
1036
|
+
for (const candidate of candidates) {
|
|
1037
|
+
const canonical = candidate.canonical_root
|
|
1038
|
+
if (roots.some((root) =>
|
|
1039
|
+
isPathWithin(root.canonical_root, canonical))) continue
|
|
1040
|
+
for (let index = roots.length - 1; index >= 0; index--) {
|
|
1041
|
+
if (isPathWithin(canonical, roots[index].canonical_root)) {
|
|
1042
|
+
roots.splice(index, 1)
|
|
914
1043
|
}
|
|
915
1044
|
}
|
|
916
|
-
|
|
1045
|
+
roots.push(candidate)
|
|
917
1046
|
}
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
1047
|
+
return roots.map(({ root, source_id: sourceIdValue }) => ({
|
|
1048
|
+
root,
|
|
1049
|
+
source_id: sourceIdValue
|
|
1050
|
+
}))
|
|
1051
|
+
}
|
|
1052
|
+
|
|
1053
|
+
scopeSourceIds(scopeId = null, locationId = null) {
|
|
1054
|
+
return this._sources
|
|
1055
|
+
.filter((source) => source.kind !== "virtual")
|
|
1056
|
+
.filter((source) => !scopeId || this.sourceIsWithinScope(source, scopeId))
|
|
1057
|
+
.filter((source) =>
|
|
1058
|
+
!locationId || this.sourceIsWithinScope(source, locationId))
|
|
1059
|
+
.map((source) => source.id)
|
|
1060
|
+
}
|
|
1061
|
+
|
|
1062
|
+
async canonicalPathIsWithinSource(filePath, source) {
|
|
1063
|
+
if (!source || !source.root) return false
|
|
930
1064
|
try {
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
// Windows metadata operations can change ctime without changing the
|
|
940
|
-
// file's identity or bytes. Recover only that isolated mismatch by
|
|
941
|
-
// hashing the target again, then retain the exact pre-rename check below.
|
|
942
|
-
if (!sameContentState(meta.expected, targetStat)) return { status: "stale" }
|
|
943
|
-
const before = fileSnapshot(targetStat)
|
|
944
|
-
let verifiedTarget
|
|
945
|
-
try {
|
|
946
|
-
verifiedTarget = await this.hashFile(targetPath)
|
|
947
|
-
} catch (error) {
|
|
948
|
-
return { status: "stale" }
|
|
949
|
-
}
|
|
950
|
-
const after = await lstatIfPresent(targetPath)
|
|
951
|
-
if (!sameSnapshot(before, after) ||
|
|
952
|
-
verifiedTarget.hash !== hash || verifiedTarget.size !== after.size) {
|
|
953
|
-
return { status: "stale" }
|
|
1065
|
+
const rootStat = await fs.promises.lstat(source.root)
|
|
1066
|
+
if (!rootStat.isDirectory() || rootStat.isSymbolicLink()) return false
|
|
1067
|
+
const [root, target] = await Promise.all([
|
|
1068
|
+
fs.promises.realpath(source.root),
|
|
1069
|
+
fs.promises.realpath(filePath)
|
|
1070
|
+
])
|
|
1071
|
+
if (source.kind === "external" && !samePath(root, source.root)) {
|
|
1072
|
+
return false
|
|
954
1073
|
}
|
|
955
|
-
|
|
1074
|
+
return isPathWithin(root, target)
|
|
1075
|
+
} catch (error) {
|
|
1076
|
+
if (!isMissingError(error)) throw error
|
|
1077
|
+
return false
|
|
956
1078
|
}
|
|
957
|
-
|
|
958
|
-
|
|
1079
|
+
}
|
|
1080
|
+
|
|
1081
|
+
async revealFile(scopeId, filePath) {
|
|
1082
|
+
if (typeof filePath !== "string" ||
|
|
1083
|
+
!path.isAbsolute(filePath)) {
|
|
1084
|
+
return { error: "Choose a valid file." }
|
|
959
1085
|
}
|
|
960
|
-
if (
|
|
961
|
-
|
|
962
|
-
await this.refreshLinkSnapshots(hash, storeStat.dev, storeStat.ino)
|
|
963
|
-
return { status: "already" }
|
|
1086
|
+
if (scopeId && !this._sourcesById.has(scopeId)) {
|
|
1087
|
+
return { error: "That location is no longer available." }
|
|
964
1088
|
}
|
|
965
|
-
|
|
966
|
-
|
|
1089
|
+
const entry = await this.registry.getFile(path.resolve(filePath))
|
|
1090
|
+
if (!entry || entry.unavailable_reason === "stale") {
|
|
1091
|
+
return { error: "This file is no longer tracked. Scan again to refresh this view." }
|
|
967
1092
|
}
|
|
968
|
-
|
|
969
|
-
|
|
1093
|
+
const source = this.sourceForPath(entry.path, entry.source_id)
|
|
1094
|
+
if (!source || !this.sourceIsWithinScope(source, scopeId)) {
|
|
1095
|
+
return { error: "This file is outside the current location." }
|
|
1096
|
+
}
|
|
1097
|
+
if (!await this.canonicalPathIsWithinSource(entry.path, source)) {
|
|
1098
|
+
return { error: "This file is no longer available at the scanned location." }
|
|
1099
|
+
}
|
|
1100
|
+
const current = await lstatIfPresent(entry.path)
|
|
1101
|
+
if (!current || !current.isFile()) {
|
|
1102
|
+
return { error: "This file is no longer available at the scanned location." }
|
|
970
1103
|
}
|
|
971
|
-
const verifiedStore = await this.verifyStoreContent(hash, storePath, storeStat)
|
|
972
|
-
if (!verifiedStore.valid) return { status: "stale-blob" }
|
|
973
|
-
if (meta.source && this.sourceAppIsRunning(meta.source)) return { status: "locked" }
|
|
974
|
-
const tmp = targetPath + TMP_SUFFIX
|
|
975
|
-
let ownsTmp = false
|
|
976
1104
|
try {
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
1105
|
+
await this.fileManagerLauncher(entry.path)
|
|
1106
|
+
} catch (error) {
|
|
1107
|
+
return { error: "The file manager could not open this file." }
|
|
1108
|
+
}
|
|
1109
|
+
return { revealed: true }
|
|
1110
|
+
}
|
|
1111
|
+
|
|
1112
|
+
locationForPath(filePath, preferredId = null) {
|
|
1113
|
+
const source = this.sourceForPath(filePath, preferredId)
|
|
1114
|
+
if (!source) {
|
|
1115
|
+
return {
|
|
1116
|
+
source_id: null,
|
|
1117
|
+
relative_path: path.basename(filePath)
|
|
1118
|
+
}
|
|
1119
|
+
}
|
|
1120
|
+
const resolvedPath = path.resolve(filePath)
|
|
1121
|
+
const sourceBase = source.canonical_root &&
|
|
1122
|
+
isPathWithin(source.canonical_root, resolvedPath)
|
|
1123
|
+
? source.canonical_root
|
|
1124
|
+
: source.root
|
|
1125
|
+
const relative = path.relative(sourceBase, resolvedPath)
|
|
1126
|
+
.split(path.sep).join("/") || path.basename(filePath)
|
|
1127
|
+
return {
|
|
1128
|
+
source_id: source.id,
|
|
1129
|
+
source_kind: source.kind,
|
|
1130
|
+
source_label: source.label,
|
|
1131
|
+
relative_path: relative
|
|
1132
|
+
}
|
|
1133
|
+
}
|
|
1134
|
+
|
|
1135
|
+
async addExternalSource(folderPath) {
|
|
1136
|
+
if (typeof folderPath !== "string" ||
|
|
1137
|
+
!path.isAbsolute(folderPath.trim())) {
|
|
1138
|
+
throw new Error("Choose a valid folder.")
|
|
1139
|
+
}
|
|
1140
|
+
const canonical = path.resolve(
|
|
1141
|
+
await fs.promises.realpath(folderPath.trim()))
|
|
1142
|
+
const stat = await fs.promises.stat(canonical)
|
|
1143
|
+
if (!stat.isDirectory()) throw new Error("Choose a folder, not a file.")
|
|
1144
|
+
|
|
1145
|
+
const home = path.resolve(await fs.promises.realpath(this.kernel.homedir))
|
|
1146
|
+
if (isPathWithin(home, canonical)) {
|
|
1147
|
+
throw new Error(
|
|
1148
|
+
"That folder is already inside Pinokio and is included in scans.")
|
|
1149
|
+
}
|
|
1150
|
+
await this.refreshSources()
|
|
1151
|
+
const existing = this._sources.find((source) =>
|
|
1152
|
+
source.kind === "external" &&
|
|
1153
|
+
source.root &&
|
|
1154
|
+
samePath(source.root, canonical))
|
|
1155
|
+
if (existing) return { created: false, source: existing }
|
|
1156
|
+
|
|
1157
|
+
const config = this.readConfig()
|
|
1158
|
+
this.writeConfig(Object.assign({}, config, {
|
|
1159
|
+
locations: [...config.locations, canonical]
|
|
1160
|
+
}))
|
|
1161
|
+
await this.refreshAnchorStores()
|
|
1162
|
+
await this.refreshSources()
|
|
1163
|
+
const source = this._sources.find((candidate) =>
|
|
1164
|
+
candidate.kind === "external" &&
|
|
1165
|
+
samePath(candidate.root, canonical))
|
|
1166
|
+
return { created: true, source }
|
|
1167
|
+
}
|
|
1168
|
+
|
|
1169
|
+
async removeExternalSource(sourceIdToRemove) {
|
|
1170
|
+
await this.refreshSources()
|
|
1171
|
+
const source = this._sources.find((candidate) =>
|
|
1172
|
+
candidate.id === sourceIdToRemove &&
|
|
1173
|
+
candidate.kind === "external" &&
|
|
1174
|
+
candidate.configured)
|
|
1175
|
+
if (!source) {
|
|
1176
|
+
return { error: "That external folder is no longer configured." }
|
|
1177
|
+
}
|
|
1178
|
+
const config = this.readConfig()
|
|
1179
|
+
this.writeConfig(Object.assign({}, config, {
|
|
1180
|
+
locations: config.locations.filter((item) =>
|
|
1181
|
+
!samePath(item, source.root))
|
|
1182
|
+
}))
|
|
1183
|
+
await this.registry.removeExternalSourceState(source.root, source.id)
|
|
1184
|
+
await this.refreshAnchorStores()
|
|
1185
|
+
await this.refreshSources()
|
|
1186
|
+
return {
|
|
1187
|
+
removed: true,
|
|
1188
|
+
source_id: source.id,
|
|
1189
|
+
label: source.label
|
|
1190
|
+
}
|
|
1191
|
+
}
|
|
1192
|
+
|
|
1193
|
+
async recordEvent(event) {
|
|
1194
|
+
try {
|
|
1195
|
+
await this.registry.addEvent(event)
|
|
1196
|
+
return true
|
|
1197
|
+
} catch (error) {
|
|
1198
|
+
return false
|
|
1199
|
+
}
|
|
1200
|
+
}
|
|
1201
|
+
|
|
1202
|
+
startScan(scopeId = null, sizeThreshold = this.sizeThreshold) {
|
|
1203
|
+
if (!this.enabled || !this.sweeper) {
|
|
1204
|
+
return { started: false, disabled: true }
|
|
1205
|
+
}
|
|
1206
|
+
if (this.scanPromise) return { started: false, already_running: true }
|
|
1207
|
+
this.sizeThreshold = sizeThreshold
|
|
1208
|
+
this.scanError = null
|
|
1209
|
+
this.scanScopeId = scopeId
|
|
1210
|
+
this.scanCancelRequested = false
|
|
1211
|
+
this.scanPromise = this.runExclusive(() => {
|
|
1212
|
+
if (this.scanCancelRequested) {
|
|
1213
|
+
this.sweeper.state = Object.assign(this.sweeper.idleState(), {
|
|
1214
|
+
phase: "cancelled",
|
|
1215
|
+
scope_id: scopeId
|
|
1216
|
+
})
|
|
1217
|
+
return { cancelled: true }
|
|
1218
|
+
}
|
|
1219
|
+
return this.sweeper.scan(scopeId)
|
|
1220
|
+
}).catch((error) => {
|
|
1221
|
+
this.scanError = error && error.message
|
|
1222
|
+
? error.message
|
|
1223
|
+
: String(error)
|
|
1224
|
+
}).finally(() => {
|
|
1225
|
+
this.scanPromise = null
|
|
1226
|
+
this.scanScopeId = null
|
|
1227
|
+
this.scanCancelRequested = false
|
|
1228
|
+
})
|
|
1229
|
+
return { started: true }
|
|
1230
|
+
}
|
|
1231
|
+
|
|
1232
|
+
cancelScan() {
|
|
1233
|
+
if (!this.scanPromise || !this.sweeper) {
|
|
1234
|
+
return { cancel_requested: false }
|
|
1235
|
+
}
|
|
1236
|
+
this.scanCancelRequested = true
|
|
1237
|
+
if (this.sweeper.state.active && this.sweeper.currentHash &&
|
|
1238
|
+
this.worker) {
|
|
1239
|
+
const error = new Error("Scan cancelled.")
|
|
1240
|
+
error.code = "EVAULTCANCELLED"
|
|
1241
|
+
this.failHashWorker(this.worker, error, true)
|
|
1242
|
+
}
|
|
1243
|
+
return {
|
|
1244
|
+
cancel_requested: this.sweeper.state.active
|
|
1245
|
+
? this.sweeper.cancel()
|
|
1246
|
+
: true
|
|
1247
|
+
}
|
|
1248
|
+
}
|
|
1249
|
+
|
|
1250
|
+
async perform(action, payload = {}) {
|
|
1251
|
+
if (!this.enabled) return { error: "Save space is disabled." }
|
|
1252
|
+
await this.ensureInitialized()
|
|
1253
|
+
switch (action) {
|
|
1254
|
+
case "add_source": {
|
|
1255
|
+
const result = await this.runExclusive(() =>
|
|
1256
|
+
this.addExternalSource(payload.path))
|
|
1257
|
+
return {
|
|
1258
|
+
created: result.created,
|
|
1259
|
+
source: {
|
|
1260
|
+
id: result.source.id,
|
|
1261
|
+
label: result.source.label,
|
|
1262
|
+
target_path: result.source.root,
|
|
1263
|
+
shareable: !!result.source.shareable
|
|
1264
|
+
}
|
|
989
1265
|
}
|
|
990
1266
|
}
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
1267
|
+
case "remove_source":
|
|
1268
|
+
if (typeof payload.source_id !== "string" || !payload.source_id) {
|
|
1269
|
+
return { error: "Choose an external folder to remove." }
|
|
1270
|
+
}
|
|
1271
|
+
return this.runMutation(() =>
|
|
1272
|
+
this.removeExternalSource(payload.source_id))
|
|
1273
|
+
case "scan": {
|
|
1274
|
+
if (payload.scope_id && !this.scanSource(payload.scope_id)) {
|
|
1275
|
+
return { error: "That scan location is no longer available." }
|
|
1276
|
+
}
|
|
1277
|
+
let threshold = this.sizeThreshold
|
|
1278
|
+
if (payload.candidate_size != null) {
|
|
1279
|
+
if (!CANDIDATE_SIZE_OPTIONS.includes(payload.candidate_size)) {
|
|
1280
|
+
return { error: "Choose a valid minimum file size." }
|
|
1281
|
+
}
|
|
1282
|
+
threshold = payload.candidate_size
|
|
1283
|
+
}
|
|
1284
|
+
return this.startScan(payload.scope_id || null, threshold)
|
|
1285
|
+
}
|
|
1286
|
+
case "cancel_scan":
|
|
1287
|
+
return this.cancelScan()
|
|
1288
|
+
case "cancel_file_action":
|
|
1289
|
+
return this.cancelFileAction()
|
|
1290
|
+
case "reveal":
|
|
1291
|
+
return this.revealFile(
|
|
1292
|
+
typeof payload.scope_id === "string" && payload.scope_id
|
|
1293
|
+
? payload.scope_id
|
|
1294
|
+
: null,
|
|
1295
|
+
payload.path
|
|
1296
|
+
)
|
|
1297
|
+
case "deduplicate": {
|
|
1298
|
+
const scopeId = typeof payload.scope_id === "string" &&
|
|
1299
|
+
payload.scope_id
|
|
1300
|
+
? payload.scope_id
|
|
1301
|
+
: null
|
|
1302
|
+
if (scopeId && !this._sources.some((source) =>
|
|
1303
|
+
source.id === scopeId)) {
|
|
1304
|
+
return { error: "That location is no longer available." }
|
|
1305
|
+
}
|
|
1306
|
+
if (typeof payload.path === "string" && payload.path) {
|
|
1307
|
+
return this.runMutation(() => this.runFileAction({
|
|
1308
|
+
kind: "deduplicate-file",
|
|
1309
|
+
path: path.resolve(payload.path)
|
|
1310
|
+
}, () => this.deduplicateFile(payload.path)))
|
|
1311
|
+
}
|
|
1312
|
+
const filesTotal = await this.countActionFiles("duplicate", scopeId)
|
|
1313
|
+
return this.runMutation(() => this.runFileAction({
|
|
1314
|
+
kind: "deduplicate",
|
|
1315
|
+
scope_id: scopeId,
|
|
1316
|
+
files_total: filesTotal,
|
|
1317
|
+
files_completed: 0
|
|
1318
|
+
}, (progress) => this.deduplicateScope(scopeId, { progress })))
|
|
1319
|
+
}
|
|
1320
|
+
case "deduplicate_files": {
|
|
1321
|
+
if (!Array.isArray(payload.paths) ||
|
|
1322
|
+
payload.paths.length === 0 ||
|
|
1323
|
+
payload.paths.length > MAX_BULK_FILE_ACTIONS ||
|
|
1324
|
+
payload.paths.some((item) =>
|
|
1325
|
+
typeof item !== "string" || !item)) {
|
|
1326
|
+
return { error: "Choose valid duplicate files to deduplicate." }
|
|
1327
|
+
}
|
|
1328
|
+
const paths = [...new Set(payload.paths.map((item) =>
|
|
1329
|
+
path.resolve(item)))]
|
|
1330
|
+
return this.runMutation(() => this.runFileAction({
|
|
1331
|
+
kind: "deduplicate-files",
|
|
1332
|
+
files_total: paths.length,
|
|
1333
|
+
files_completed: 0
|
|
1334
|
+
}, (progress) => this.deduplicateFiles(paths, progress)))
|
|
997
1335
|
}
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1336
|
+
case "detach":
|
|
1337
|
+
if (typeof payload.path !== "string" || !payload.path) {
|
|
1338
|
+
return { status: "not-found" }
|
|
1339
|
+
}
|
|
1340
|
+
return this.runMutation(() => this.runFileAction({
|
|
1341
|
+
kind: "make-separate",
|
|
1342
|
+
path: path.resolve(payload.path)
|
|
1343
|
+
}, () => this.separate(payload.path)))
|
|
1344
|
+
case "separate_files": {
|
|
1345
|
+
if (!Array.isArray(payload.paths) ||
|
|
1346
|
+
payload.paths.length === 0 ||
|
|
1347
|
+
payload.paths.length > MAX_BULK_FILE_ACTIONS ||
|
|
1348
|
+
payload.paths.some((item) =>
|
|
1349
|
+
typeof item !== "string" || !item)) {
|
|
1350
|
+
return { error: "Choose valid deduplicated files to separate." }
|
|
1351
|
+
}
|
|
1352
|
+
return this.runMutation(() => this.runFileAction({
|
|
1353
|
+
kind: "separate-files",
|
|
1354
|
+
files_total: new Set(payload.paths.map((item) =>
|
|
1355
|
+
path.resolve(item))).size,
|
|
1356
|
+
files_completed: 0
|
|
1357
|
+
}, (progress) => this.separateFiles(payload.paths, progress)))
|
|
1358
|
+
}
|
|
1359
|
+
case "separate_all": {
|
|
1360
|
+
const selection = this.separateSelection(payload)
|
|
1361
|
+
if (selection.error) return { error: selection.error }
|
|
1362
|
+
const total = await this.registry.matchingFileSummary(
|
|
1363
|
+
"linked", selection.sourceIds, selection.query)
|
|
1364
|
+
if (!total.count) {
|
|
1365
|
+
return { error: "No matching deduplicated files remain." }
|
|
1366
|
+
}
|
|
1367
|
+
return this.runMutation(() => this.runFileAction({
|
|
1368
|
+
kind: "separate-files",
|
|
1369
|
+
files_total: total.count,
|
|
1370
|
+
files_completed: 0,
|
|
1371
|
+
all_matching: true,
|
|
1372
|
+
cancelable: true,
|
|
1373
|
+
cancel_requested: false
|
|
1374
|
+
}, (progress) =>
|
|
1375
|
+
this.separateMatchingFiles(selection, progress)))
|
|
1002
1376
|
}
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1377
|
+
case "reclaim":
|
|
1378
|
+
return this.runMutation(() =>
|
|
1379
|
+
this.reclaim(payload.hash, payload.store_id))
|
|
1380
|
+
case "reclaim_all":
|
|
1381
|
+
return this.runMutation(() => this.reclaimAll())
|
|
1382
|
+
default:
|
|
1383
|
+
return { error: "unknown action" }
|
|
1384
|
+
}
|
|
1385
|
+
}
|
|
1386
|
+
|
|
1387
|
+
sourceAppIsRunning(source) {
|
|
1388
|
+
if (!source || source.kind !== "app") return false
|
|
1389
|
+
const appRoot = path.resolve(this.kernel.homedir, "api", source.app)
|
|
1390
|
+
const api = this.kernel.api || {}
|
|
1391
|
+
const running = api.running || {}
|
|
1392
|
+
const runningPaths = api.running_paths || {}
|
|
1393
|
+
return Object.keys(running).some((id) => {
|
|
1394
|
+
if (!running[id]) return false
|
|
1395
|
+
const runningPath = runningPaths[id] ||
|
|
1396
|
+
(path.isAbsolute(id) ? id.split("?")[0] : null)
|
|
1397
|
+
return !!(runningPath && isPathWithin(appRoot, runningPath))
|
|
1398
|
+
})
|
|
1399
|
+
}
|
|
1400
|
+
|
|
1401
|
+
async refreshInodeSnapshots(hash, dev, ino, storeId = null) {
|
|
1402
|
+
const store = storeId
|
|
1403
|
+
? this._anchorStoresById.get(storeId)
|
|
1404
|
+
: this.anchorStoreForDevice(dev)
|
|
1405
|
+
const storePath = store ? this.storePathFor(hash, store.id) : null
|
|
1406
|
+
const storeStat = storePath
|
|
1407
|
+
? await this.storeStatIfPresent(storePath, { store_id: store.id })
|
|
1408
|
+
: null
|
|
1409
|
+
let inodeStat = storeStat &&
|
|
1410
|
+
storeStat.dev === dev &&
|
|
1411
|
+
storeStat.ino === ino
|
|
1412
|
+
? storeStat
|
|
1413
|
+
: null
|
|
1414
|
+
if (!inodeStat) {
|
|
1415
|
+
const row = await this.registry.firstFileForInode(hash, dev, ino)
|
|
1416
|
+
if (row) {
|
|
1417
|
+
try {
|
|
1418
|
+
const stat = await fs.promises.lstat(row.path)
|
|
1419
|
+
if (stat.isFile() && stat.dev === dev && stat.ino === ino) {
|
|
1420
|
+
inodeStat = stat
|
|
1421
|
+
}
|
|
1422
|
+
} catch (error) {}
|
|
1007
1423
|
}
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
}
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1424
|
+
}
|
|
1425
|
+
if (inodeStat) {
|
|
1426
|
+
await this.registry.updateInodeSnapshots(
|
|
1427
|
+
dev, ino, fileSnapshot(inodeStat))
|
|
1428
|
+
}
|
|
1429
|
+
const content = await this.registry.getContent(hash)
|
|
1430
|
+
if (content) {
|
|
1431
|
+
await this.registry.upsertContent(Object.assign({}, content, {
|
|
1432
|
+
hash,
|
|
1433
|
+
size: storeStat ? storeStat.size : content.size
|
|
1434
|
+
}))
|
|
1435
|
+
}
|
|
1436
|
+
if (store && storeStat) {
|
|
1437
|
+
await this.registry.upsertAnchor({
|
|
1438
|
+
store_id: store.id,
|
|
1439
|
+
hash,
|
|
1440
|
+
path: storePath,
|
|
1441
|
+
verified_at: Date.now(),
|
|
1442
|
+
dev: storeStat.dev,
|
|
1443
|
+
ino: storeStat.ino,
|
|
1444
|
+
size: storeStat.size,
|
|
1445
|
+
mtime: storeStat.mtimeMs,
|
|
1446
|
+
ctime: storeStat.ctimeMs,
|
|
1447
|
+
nlink: storeStat.nlink,
|
|
1448
|
+
mode: storeStat.mode,
|
|
1449
|
+
uid: storeStat.uid,
|
|
1450
|
+
gid: storeStat.gid
|
|
1451
|
+
})
|
|
1452
|
+
} else if (store) {
|
|
1453
|
+
await this.registry.removeAnchor(store.id, hash)
|
|
1454
|
+
}
|
|
1455
|
+
}
|
|
1456
|
+
|
|
1457
|
+
async reclassifyHashes(hashes) {
|
|
1458
|
+
const stores = this.anchorStores()
|
|
1459
|
+
.filter((store) =>
|
|
1460
|
+
store.available && Number.isFinite(store.dev))
|
|
1461
|
+
.map((store) => ({
|
|
1462
|
+
store_id: store.id,
|
|
1463
|
+
dev: store.dev,
|
|
1464
|
+
can_link: store.mode !== "copy"
|
|
1465
|
+
}))
|
|
1466
|
+
for (const hash of new Set(hashes)) {
|
|
1467
|
+
const anchorSnapshots = []
|
|
1468
|
+
for (const store of this.anchorStores()) {
|
|
1469
|
+
const storePath = this.storePathFor(hash, store.id)
|
|
1470
|
+
const stat = await this.storeStatIfPresent(storePath, {
|
|
1471
|
+
store_id: store.id
|
|
1472
|
+
})
|
|
1473
|
+
if (stat) anchorSnapshots.push(fileSnapshot(stat))
|
|
1022
1474
|
}
|
|
1023
|
-
|
|
1024
|
-
|
|
1475
|
+
await this.registry.reclassifyHash(
|
|
1476
|
+
hash,
|
|
1477
|
+
stores,
|
|
1478
|
+
anchorSnapshots
|
|
1479
|
+
)
|
|
1480
|
+
}
|
|
1481
|
+
}
|
|
1482
|
+
|
|
1483
|
+
async verifyStoreContent(hash, storePath, storeStat, storeId = null) {
|
|
1484
|
+
if (!storeStat || !storeStat.isFile()) return { valid: false }
|
|
1485
|
+
const store = storeId
|
|
1486
|
+
? this._anchorStoresById.get(storeId)
|
|
1487
|
+
: this.anchorStoreForPath(storePath)
|
|
1488
|
+
if (!store) return { valid: false }
|
|
1489
|
+
const content = await this.registry.getContent(hash)
|
|
1490
|
+
const anchor = await this.registry.getAnchor(store.id, hash)
|
|
1491
|
+
if (anchor &&
|
|
1492
|
+
anchor.verified_at &&
|
|
1493
|
+
anchor.dev === storeStat.dev &&
|
|
1494
|
+
anchor.ino === storeStat.ino &&
|
|
1495
|
+
anchor.size === storeStat.size &&
|
|
1496
|
+
anchor.mtime === storeStat.mtimeMs &&
|
|
1497
|
+
anchor.ctime === storeStat.ctimeMs) {
|
|
1498
|
+
return { valid: true, snapshot: fileSnapshot(storeStat) }
|
|
1499
|
+
}
|
|
1500
|
+
const before = fileSnapshot(storeStat)
|
|
1501
|
+
let hashed
|
|
1502
|
+
try {
|
|
1503
|
+
hashed = await this.hashFile(storePath)
|
|
1504
|
+
} catch (error) {
|
|
1505
|
+
return { valid: false }
|
|
1506
|
+
}
|
|
1507
|
+
const after = await this.storeStatIfPresent(storePath)
|
|
1508
|
+
if (!after ||
|
|
1509
|
+
!sameSnapshot(before, after) ||
|
|
1510
|
+
hashed.hash !== hash ||
|
|
1511
|
+
hashed.size !== after.size) {
|
|
1512
|
+
return { valid: false }
|
|
1513
|
+
}
|
|
1514
|
+
await this.registry.upsertContent({
|
|
1515
|
+
hash,
|
|
1516
|
+
size: after.size,
|
|
1517
|
+
first_seen: content && content.first_seen,
|
|
1518
|
+
verified_at: Date.now()
|
|
1519
|
+
})
|
|
1520
|
+
await this.registry.upsertAnchor({
|
|
1521
|
+
store_id: store.id,
|
|
1522
|
+
hash,
|
|
1523
|
+
path: storePath,
|
|
1524
|
+
verified_at: Date.now(),
|
|
1525
|
+
dev: after.dev,
|
|
1526
|
+
ino: after.ino,
|
|
1527
|
+
size: after.size,
|
|
1528
|
+
mtime: after.mtimeMs,
|
|
1529
|
+
ctime: after.ctimeMs,
|
|
1530
|
+
nlink: after.nlink,
|
|
1531
|
+
mode: after.mode,
|
|
1532
|
+
uid: after.uid,
|
|
1533
|
+
gid: after.gid
|
|
1534
|
+
})
|
|
1535
|
+
return { valid: true, snapshot: fileSnapshot(after) }
|
|
1536
|
+
}
|
|
1537
|
+
|
|
1538
|
+
async adopt(filePath, hash, expected, selectedStore = null) {
|
|
1539
|
+
const current = await lstatIfPresent(filePath)
|
|
1540
|
+
if (!current ||
|
|
1541
|
+
!current.isFile() ||
|
|
1542
|
+
!sameSnapshot(expected, current)) {
|
|
1543
|
+
return { status: "stale" }
|
|
1544
|
+
}
|
|
1545
|
+
const store = selectedStore || this.anchorStoreForDevice(current.dev)
|
|
1546
|
+
if (!store) return { status: "unavailable" }
|
|
1547
|
+
const storePath = this.storePathFor(hash, store.id)
|
|
1548
|
+
let storeStat
|
|
1549
|
+
try {
|
|
1550
|
+
storeStat = await this.storeStatIfPresent(storePath, {
|
|
1551
|
+
createParent: true,
|
|
1552
|
+
dev: current.dev,
|
|
1553
|
+
store_id: store.id
|
|
1554
|
+
})
|
|
1555
|
+
} catch (error) {
|
|
1556
|
+
if (error && (
|
|
1557
|
+
error.code === "EVAULTUNAVAILABLE" ||
|
|
1558
|
+
NO_LINK_CODES.has(error.code) ||
|
|
1559
|
+
LOCK_CODES.has(error.code)
|
|
1560
|
+
)) {
|
|
1561
|
+
return { status: "unavailable" }
|
|
1025
1562
|
}
|
|
1026
|
-
throw
|
|
1563
|
+
throw error
|
|
1564
|
+
}
|
|
1565
|
+
if (storeStat) {
|
|
1566
|
+
if (sameIdentity(storeStat, current)) return { status: "ready" }
|
|
1567
|
+
return { status: "exists" }
|
|
1027
1568
|
}
|
|
1028
|
-
let st
|
|
1029
1569
|
try {
|
|
1030
|
-
|
|
1570
|
+
await fs.promises.link(filePath, storePath)
|
|
1031
1571
|
} catch (error) {
|
|
1032
|
-
if (
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
st = storeStat
|
|
1572
|
+
if (NO_LINK_CODES.has(error && error.code)) {
|
|
1573
|
+
return { status: "unavailable" }
|
|
1574
|
+
}
|
|
1575
|
+
throw error
|
|
1037
1576
|
}
|
|
1038
|
-
|
|
1577
|
+
storeStat = await this.storeStatIfPresent(storePath, {
|
|
1578
|
+
store_id: store.id
|
|
1579
|
+
})
|
|
1580
|
+
const after = await lstatIfPresent(filePath)
|
|
1581
|
+
if (!storeStat ||
|
|
1582
|
+
!after ||
|
|
1583
|
+
!sameIdentity(storeStat, after) ||
|
|
1584
|
+
!sameContentState(expected, after)) {
|
|
1585
|
+
if (storeStat) await unlinkIfSame(storePath, storeStat)
|
|
1039
1586
|
return { status: "stale" }
|
|
1040
1587
|
}
|
|
1041
|
-
this.registry.
|
|
1042
|
-
|
|
1043
|
-
|
|
1588
|
+
const existing = await this.registry.getFile(filePath)
|
|
1589
|
+
await this.registry.upsertContent({
|
|
1590
|
+
hash,
|
|
1591
|
+
size: storeStat.size,
|
|
1592
|
+
verified_at: Date.now(),
|
|
1593
|
+
first_seen: Date.now()
|
|
1044
1594
|
})
|
|
1045
|
-
await this.
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1595
|
+
await this.registry.upsertAnchor({
|
|
1596
|
+
store_id: store.id,
|
|
1597
|
+
hash,
|
|
1598
|
+
path: storePath,
|
|
1599
|
+
verified_at: Date.now(),
|
|
1600
|
+
dev: storeStat.dev,
|
|
1601
|
+
ino: storeStat.ino,
|
|
1602
|
+
size: storeStat.size,
|
|
1603
|
+
mtime: storeStat.mtimeMs,
|
|
1604
|
+
ctime: storeStat.ctimeMs,
|
|
1605
|
+
nlink: storeStat.nlink,
|
|
1606
|
+
mode: storeStat.mode,
|
|
1607
|
+
uid: storeStat.uid,
|
|
1608
|
+
gid: storeStat.gid
|
|
1050
1609
|
})
|
|
1051
|
-
|
|
1610
|
+
await this.registry.upsertFile(Object.assign({}, existing, {
|
|
1611
|
+
path: filePath,
|
|
1612
|
+
hash,
|
|
1613
|
+
size: after.size,
|
|
1614
|
+
mtime: after.mtimeMs,
|
|
1615
|
+
ctime: after.ctimeMs,
|
|
1616
|
+
dev: after.dev,
|
|
1617
|
+
ino: after.ino,
|
|
1618
|
+
source_id: existing && existing.source_id,
|
|
1619
|
+
app: existing && existing.app,
|
|
1620
|
+
status: "linked"
|
|
1621
|
+
}))
|
|
1622
|
+
await this.refreshInodeSnapshots(hash, after.dev, after.ino, store.id)
|
|
1623
|
+
return { status: "ready", store_id: store.id }
|
|
1052
1624
|
}
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
const
|
|
1056
|
-
const
|
|
1057
|
-
|
|
1058
|
-
const
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1625
|
+
|
|
1626
|
+
async ensureAnchorForHash(hash, targetStat) {
|
|
1627
|
+
const dev = targetStat.dev
|
|
1628
|
+
const store = this.anchorStoreForDevice(dev)
|
|
1629
|
+
if (!store) return { status: "unavailable" }
|
|
1630
|
+
const storePath = this.storePathFor(hash, store.id)
|
|
1631
|
+
let existingStore
|
|
1632
|
+
try {
|
|
1633
|
+
existingStore = await this.storeStatIfPresent(storePath, {
|
|
1634
|
+
store_id: store.id
|
|
1635
|
+
})
|
|
1636
|
+
if (existingStore) await this.validateAnchorStore(store, dev)
|
|
1637
|
+
} catch (error) {
|
|
1638
|
+
if (error && (
|
|
1639
|
+
error.code === "EVAULTPATH" ||
|
|
1640
|
+
error.code === "EVAULTUNAVAILABLE" ||
|
|
1641
|
+
NO_LINK_CODES.has(error.code) ||
|
|
1642
|
+
LOCK_CODES.has(error.code)
|
|
1643
|
+
)) {
|
|
1644
|
+
return { status: "unavailable" }
|
|
1645
|
+
}
|
|
1646
|
+
throw error
|
|
1647
|
+
}
|
|
1648
|
+
if (existingStore) {
|
|
1649
|
+
if (existingStore.dev !== dev) return { status: "unavailable" }
|
|
1650
|
+
const verified = await this.verifyStoreContent(
|
|
1651
|
+
hash, storePath, existingStore, store.id)
|
|
1652
|
+
return verified.valid
|
|
1653
|
+
? { status: "ready", stat: existingStore, store_id: store.id }
|
|
1654
|
+
: { status: "stale" }
|
|
1655
|
+
}
|
|
1656
|
+
|
|
1657
|
+
const candidate = await this.registry.anchorCandidate(
|
|
1658
|
+
hash,
|
|
1659
|
+
dev,
|
|
1660
|
+
targetStat.mode & 0o7777,
|
|
1661
|
+
targetStat.uid,
|
|
1662
|
+
targetStat.gid
|
|
1663
|
+
)
|
|
1664
|
+
if (!candidate) return { status: "no-source" }
|
|
1665
|
+
const source = this.sourceForPath(candidate.path, candidate.source_id)
|
|
1666
|
+
if (!source ||
|
|
1667
|
+
!await this.canonicalPathIsWithinSource(candidate.path, source)) {
|
|
1668
|
+
return { status: "stale" }
|
|
1669
|
+
}
|
|
1670
|
+
if (this.sourceAppIsRunning(source)) return { status: "locked" }
|
|
1671
|
+
const before = await lstatIfPresent(candidate.path)
|
|
1672
|
+
if (!before ||
|
|
1673
|
+
!before.isFile() ||
|
|
1674
|
+
!sameSnapshot(rowSnapshot(candidate), before)) {
|
|
1675
|
+
return { status: "stale" }
|
|
1676
|
+
}
|
|
1677
|
+
const hashed = await this.hashFile(candidate.path)
|
|
1678
|
+
const after = await lstatIfPresent(candidate.path)
|
|
1679
|
+
if (!after ||
|
|
1680
|
+
!sameSnapshot(fileSnapshot(before), after) ||
|
|
1681
|
+
hashed.hash !== hash) {
|
|
1682
|
+
return { status: "stale" }
|
|
1683
|
+
}
|
|
1684
|
+
const adopted = await this.adopt(
|
|
1685
|
+
candidate.path, hash, fileSnapshot(after), store)
|
|
1686
|
+
if (adopted.status !== "ready") return adopted
|
|
1687
|
+
return {
|
|
1688
|
+
status: "ready",
|
|
1689
|
+
store_id: store.id,
|
|
1690
|
+
stat: await this.storeStatIfPresent(storePath, {
|
|
1691
|
+
store_id: store.id
|
|
1692
|
+
})
|
|
1693
|
+
}
|
|
1064
1694
|
}
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1695
|
+
|
|
1696
|
+
async convert(targetPath) {
|
|
1697
|
+
const target = await this.registry.getFile(targetPath)
|
|
1698
|
+
if (!target ||
|
|
1699
|
+
target.unavailable_reason === "stale" ||
|
|
1700
|
+
target.status !== "duplicate" ||
|
|
1701
|
+
!target.hash) {
|
|
1702
|
+
return { status: "not-found" }
|
|
1703
|
+
}
|
|
1704
|
+
const source = this.sourceForPath(target.path, target.source_id)
|
|
1705
|
+
if (!source ||
|
|
1706
|
+
!await this.canonicalPathIsWithinSource(target.path, source)) {
|
|
1707
|
+
return { status: "stale" }
|
|
1068
1708
|
}
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
if (!excluded) return { status: "not-found" }
|
|
1076
|
-
const source = this.sourceForPath(targetPath, excluded.source_id)
|
|
1077
|
-
if (!source || !await this.canonicalPathIsWithinSource(targetPath, source)) {
|
|
1709
|
+
if (this.sourceAppIsRunning(source)) return { status: "locked" }
|
|
1710
|
+
|
|
1711
|
+
let targetStat = await lstatIfPresent(target.path)
|
|
1712
|
+
if (!targetStat ||
|
|
1713
|
+
!targetStat.isFile() ||
|
|
1714
|
+
!sameSnapshot(rowSnapshot(target), targetStat)) {
|
|
1078
1715
|
return { status: "stale" }
|
|
1079
1716
|
}
|
|
1080
|
-
|
|
1081
|
-
if (
|
|
1717
|
+
const prepared = await this.ensureAnchorForHash(target.hash, targetStat)
|
|
1718
|
+
if (prepared.status !== "ready") {
|
|
1719
|
+
if (prepared.status === "unavailable") {
|
|
1720
|
+
await this.reclassifyHashes([target.hash])
|
|
1721
|
+
}
|
|
1722
|
+
return prepared
|
|
1723
|
+
}
|
|
1724
|
+
const storePath = this.storePathFor(target.hash, prepared.store_id)
|
|
1725
|
+
let storeStat = await this.storeStatIfPresent(
|
|
1726
|
+
storePath, { store_id: prepared.store_id })
|
|
1727
|
+
if (!storeStat) return { status: "stale" }
|
|
1728
|
+
if (storeStat.dev !== targetStat.dev) {
|
|
1729
|
+
return { status: "unavailable" }
|
|
1730
|
+
}
|
|
1731
|
+
if (sameIdentity(storeStat, targetStat)) {
|
|
1732
|
+
await this.registry.upsertFile(Object.assign({}, target, {
|
|
1733
|
+
status: "linked"
|
|
1734
|
+
}))
|
|
1735
|
+
return { status: "already", bytes_saved: 0 }
|
|
1736
|
+
}
|
|
1737
|
+
if (storeStat.size !== targetStat.size) {
|
|
1738
|
+
return { status: "size-mismatch" }
|
|
1739
|
+
}
|
|
1740
|
+
if (!sameFileMetadata(storeStat, targetStat)) {
|
|
1741
|
+
return { status: "metadata-mismatch" }
|
|
1742
|
+
}
|
|
1743
|
+
const verified = await this.verifyStoreContent(
|
|
1744
|
+
target.hash, storePath, storeStat, prepared.store_id)
|
|
1745
|
+
if (!verified.valid) return { status: "stale" }
|
|
1746
|
+
|
|
1747
|
+
const temporary = `${target.path}${TMP_SUFFIX}`
|
|
1748
|
+
let temporaryStat = null
|
|
1749
|
+
let committedStat = null
|
|
1750
|
+
try {
|
|
1751
|
+
await fs.promises.link(storePath, temporary)
|
|
1752
|
+
temporaryStat = await fs.promises.lstat(temporary)
|
|
1753
|
+
if (this.sourceAppIsRunning(source)) {
|
|
1754
|
+
await unlinkIfSame(temporary, temporaryStat)
|
|
1755
|
+
return { status: "locked" }
|
|
1756
|
+
}
|
|
1757
|
+
const [currentTarget, currentStore, currentTemporary] =
|
|
1758
|
+
await Promise.all([
|
|
1759
|
+
lstatIfPresent(target.path),
|
|
1760
|
+
this.storeStatIfPresent(storePath, {
|
|
1761
|
+
store_id: prepared.store_id
|
|
1762
|
+
}),
|
|
1763
|
+
lstatIfPresent(temporary)
|
|
1764
|
+
])
|
|
1765
|
+
if (!currentTarget ||
|
|
1766
|
+
!sameSnapshot(fileSnapshot(targetStat), currentTarget) ||
|
|
1767
|
+
!currentStore ||
|
|
1768
|
+
!currentTemporary ||
|
|
1769
|
+
!sameIdentity(currentStore, currentTemporary) ||
|
|
1770
|
+
!sameContentState(verified.snapshot, currentStore)) {
|
|
1771
|
+
await unlinkIfSame(temporary, temporaryStat)
|
|
1772
|
+
return { status: "stale" }
|
|
1773
|
+
}
|
|
1774
|
+
committedStat = temporaryStat
|
|
1775
|
+
await fs.promises.rename(temporary, target.path)
|
|
1776
|
+
temporaryStat = null
|
|
1777
|
+
} catch (error) {
|
|
1778
|
+
if (temporaryStat) {
|
|
1779
|
+
await unlinkIfSame(temporary, temporaryStat).catch(() => {})
|
|
1780
|
+
}
|
|
1781
|
+
if (LOCK_CODES.has(error && error.code)) return { status: "locked" }
|
|
1782
|
+
if (NO_LINK_CODES.has(error && error.code)) {
|
|
1783
|
+
return { status: "unavailable" }
|
|
1784
|
+
}
|
|
1785
|
+
if (error && error.code === "EEXIST") return { status: "conflict" }
|
|
1786
|
+
throw error
|
|
1787
|
+
}
|
|
1788
|
+
|
|
1789
|
+
let finalStat
|
|
1790
|
+
try {
|
|
1791
|
+
finalStat = await fs.promises.lstat(target.path)
|
|
1792
|
+
} catch (error) {
|
|
1793
|
+
if (!committedStat) throw error
|
|
1794
|
+
finalStat = committedStat
|
|
1795
|
+
}
|
|
1796
|
+
storeStat = await this.storeStatIfPresent(storePath, {
|
|
1797
|
+
store_id: prepared.store_id
|
|
1798
|
+
})
|
|
1799
|
+
if (!storeStat || !sameIdentity(finalStat, storeStat)) {
|
|
1800
|
+
return { status: "stale" }
|
|
1801
|
+
}
|
|
1802
|
+
await this.registry.upsertFile(Object.assign({}, target, {
|
|
1803
|
+
size: finalStat.size,
|
|
1804
|
+
mtime: finalStat.mtimeMs,
|
|
1805
|
+
ctime: finalStat.ctimeMs,
|
|
1806
|
+
dev: finalStat.dev,
|
|
1807
|
+
ino: finalStat.ino,
|
|
1808
|
+
status: "linked"
|
|
1809
|
+
}))
|
|
1810
|
+
await this.refreshInodeSnapshots(
|
|
1811
|
+
target.hash, finalStat.dev, finalStat.ino, prepared.store_id)
|
|
1812
|
+
return {
|
|
1813
|
+
status: "converted",
|
|
1814
|
+
bytes_saved: finalStat.size,
|
|
1815
|
+
hash: target.hash,
|
|
1816
|
+
path: target.path,
|
|
1817
|
+
app: target.app,
|
|
1818
|
+
source_id: target.source_id
|
|
1819
|
+
}
|
|
1820
|
+
}
|
|
1821
|
+
|
|
1822
|
+
async countActionFiles(status, scopeId = null) {
|
|
1823
|
+
const sourceIds = this.scopeSourceIds(scopeId)
|
|
1824
|
+
if (scopeId && !sourceIds.length) return 0
|
|
1825
|
+
return this.registry.countActionFiles(status, sourceIds)
|
|
1826
|
+
}
|
|
1827
|
+
|
|
1828
|
+
separateSelection(payload = {}) {
|
|
1829
|
+
const scopeId = typeof payload.scope_id === "string" &&
|
|
1830
|
+
payload.scope_id
|
|
1831
|
+
? payload.scope_id
|
|
1832
|
+
: null
|
|
1833
|
+
const locationId = typeof payload.location_id === "string" &&
|
|
1834
|
+
payload.location_id
|
|
1835
|
+
? payload.location_id
|
|
1836
|
+
: null
|
|
1837
|
+
if (scopeId && !this._sourcesById.has(scopeId)) {
|
|
1838
|
+
return { error: "That location is no longer available." }
|
|
1839
|
+
}
|
|
1840
|
+
if (locationId && !this._sourcesById.has(locationId)) {
|
|
1841
|
+
return { error: "That location is no longer available." }
|
|
1842
|
+
}
|
|
1843
|
+
const view = STATUS_VIEWS.has(payload.view) ? payload.view : "all"
|
|
1844
|
+
const statusFilter = STATUS_FILTERS.has(payload.status_filter)
|
|
1845
|
+
? payload.status_filter
|
|
1846
|
+
: "all"
|
|
1847
|
+
if (view !== "shared" &&
|
|
1848
|
+
!(view === "all" &&
|
|
1849
|
+
(statusFilter === "all" || statusFilter === "shared"))) {
|
|
1850
|
+
return {
|
|
1851
|
+
error: "The current view has no deduplicated files to separate."
|
|
1852
|
+
}
|
|
1853
|
+
}
|
|
1854
|
+
const sourceIds = this.scopeSourceIds(scopeId, locationId)
|
|
1855
|
+
if (!sourceIds.length) {
|
|
1856
|
+
return { error: "That location is no longer available." }
|
|
1857
|
+
}
|
|
1858
|
+
return {
|
|
1859
|
+
sourceIds,
|
|
1860
|
+
query: String(payload.query || "").slice(0, 500).trim()
|
|
1861
|
+
}
|
|
1862
|
+
}
|
|
1863
|
+
|
|
1864
|
+
async deduplicateFile(filePath) {
|
|
1865
|
+
const result = await this.convert(path.resolve(filePath))
|
|
1866
|
+
if (result.status === "converted") {
|
|
1867
|
+
if (!await this.recordEvent({
|
|
1868
|
+
kind: "convert",
|
|
1869
|
+
hash: result.hash,
|
|
1870
|
+
app: result.app,
|
|
1871
|
+
source_id: result.source_id,
|
|
1872
|
+
bytes: result.bytes_saved,
|
|
1873
|
+
files: 1
|
|
1874
|
+
})) {
|
|
1875
|
+
result.activity_warning = true
|
|
1876
|
+
}
|
|
1877
|
+
}
|
|
1878
|
+
return result
|
|
1879
|
+
}
|
|
1880
|
+
|
|
1881
|
+
deduplicationSummary() {
|
|
1882
|
+
return {
|
|
1883
|
+
converted: 0,
|
|
1884
|
+
bytes_saved: 0,
|
|
1885
|
+
locked: 0,
|
|
1886
|
+
stale: 0,
|
|
1887
|
+
incompatible: 0,
|
|
1888
|
+
unavailable: 0,
|
|
1889
|
+
failed: 0
|
|
1890
|
+
}
|
|
1891
|
+
}
|
|
1892
|
+
|
|
1893
|
+
addDeduplicationResult(summary, result) {
|
|
1894
|
+
if (result.status === "converted") {
|
|
1895
|
+
summary.converted += 1
|
|
1896
|
+
summary.bytes_saved += result.bytes_saved || 0
|
|
1897
|
+
} else if (result.status === "already") {
|
|
1898
|
+
// Another selected path already caused this inode to be shared.
|
|
1899
|
+
} else if (result.status === "locked") {
|
|
1900
|
+
summary.locked += 1
|
|
1901
|
+
} else if (result.status === "metadata-mismatch") {
|
|
1902
|
+
summary.incompatible += 1
|
|
1903
|
+
} else if (result.status === "unavailable") {
|
|
1904
|
+
summary.unavailable += 1
|
|
1905
|
+
} else {
|
|
1906
|
+
summary.stale += 1
|
|
1907
|
+
}
|
|
1908
|
+
}
|
|
1909
|
+
|
|
1910
|
+
async recordDeduplicationSummary(
|
|
1911
|
+
summary,
|
|
1912
|
+
representative = null,
|
|
1913
|
+
sourceId = null
|
|
1914
|
+
) {
|
|
1915
|
+
if (!summary.converted) return
|
|
1916
|
+
const event = {
|
|
1917
|
+
kind: "convert",
|
|
1918
|
+
bytes: summary.bytes_saved,
|
|
1919
|
+
files: summary.converted
|
|
1920
|
+
}
|
|
1921
|
+
if (summary.converted === 1 && representative) {
|
|
1922
|
+
event.hash = representative.hash
|
|
1923
|
+
event.path = representative.path
|
|
1924
|
+
event.app = representative.app
|
|
1925
|
+
event.source_id = representative.source_id
|
|
1926
|
+
} else if (sourceId) {
|
|
1927
|
+
event.source_id = sourceId
|
|
1928
|
+
}
|
|
1929
|
+
if (!await this.recordEvent(event)) summary.activity_warning = true
|
|
1930
|
+
}
|
|
1931
|
+
|
|
1932
|
+
async deduplicateFiles(filePaths, progress = null) {
|
|
1933
|
+
const summary = Object.assign(this.deduplicationSummary(), {
|
|
1934
|
+
results: []
|
|
1935
|
+
})
|
|
1936
|
+
let representative = null
|
|
1937
|
+
let commonSourceId
|
|
1938
|
+
for (const filePath of new Set(filePaths.map((item) =>
|
|
1939
|
+
path.resolve(item)))) {
|
|
1940
|
+
try {
|
|
1941
|
+
const result = await this.convert(filePath)
|
|
1942
|
+
this.addDeduplicationResult(summary, result)
|
|
1943
|
+
if (result.status === "converted") {
|
|
1944
|
+
if (!representative) representative = result
|
|
1945
|
+
const sourceId = result.source_id || null
|
|
1946
|
+
commonSourceId = commonSourceId === undefined
|
|
1947
|
+
? sourceId
|
|
1948
|
+
: commonSourceId === sourceId ? commonSourceId : null
|
|
1949
|
+
}
|
|
1950
|
+
summary.results.push({ path: filePath, status: result.status })
|
|
1951
|
+
} catch (error) {
|
|
1952
|
+
summary.failed += 1
|
|
1953
|
+
summary.results.push({ path: filePath, status: "failed" })
|
|
1954
|
+
} finally {
|
|
1955
|
+
if (progress) progress.files_completed += 1
|
|
1956
|
+
}
|
|
1957
|
+
}
|
|
1958
|
+
await this.recordDeduplicationSummary(
|
|
1959
|
+
summary,
|
|
1960
|
+
representative,
|
|
1961
|
+
commonSourceId
|
|
1962
|
+
)
|
|
1963
|
+
return summary
|
|
1964
|
+
}
|
|
1965
|
+
|
|
1966
|
+
async deduplicateScope(scopeId = null, options = {}) {
|
|
1967
|
+
const sourceIds = this.scopeSourceIds(scopeId)
|
|
1968
|
+
const summary = this.deduplicationSummary()
|
|
1969
|
+
if (scopeId && !sourceIds.length) return summary
|
|
1970
|
+
let cursor = ""
|
|
1971
|
+
while (true) {
|
|
1972
|
+
const rows = await this.registry.fileBatch(
|
|
1973
|
+
"duplicate", sourceIds, cursor, 100)
|
|
1974
|
+
if (!rows.length) break
|
|
1975
|
+
for (const row of rows) {
|
|
1976
|
+
cursor = row.path
|
|
1977
|
+
try {
|
|
1978
|
+
const result = await this.convert(row.path)
|
|
1979
|
+
this.addDeduplicationResult(summary, result)
|
|
1980
|
+
} catch (error) {
|
|
1981
|
+
summary.failed += 1
|
|
1982
|
+
}
|
|
1983
|
+
if (options.progress) options.progress.files_completed += 1
|
|
1984
|
+
}
|
|
1985
|
+
}
|
|
1986
|
+
if (summary.converted) {
|
|
1987
|
+
const scopedSource = scopeId
|
|
1988
|
+
? this._sources.find((source) => source.id === scopeId)
|
|
1989
|
+
: null
|
|
1990
|
+
await this.recordDeduplicationSummary(
|
|
1991
|
+
summary,
|
|
1992
|
+
null,
|
|
1993
|
+
scopedSource && scopedSource.kind !== "virtual"
|
|
1994
|
+
? scopeId
|
|
1995
|
+
: null
|
|
1996
|
+
)
|
|
1997
|
+
}
|
|
1998
|
+
return summary
|
|
1999
|
+
}
|
|
2000
|
+
|
|
2001
|
+
async copyOut(filePath, expected, source) {
|
|
2002
|
+
const temporary = `${filePath}${TMP_SUFFIX}`
|
|
2003
|
+
let temporaryStat = null
|
|
2004
|
+
try {
|
|
2005
|
+
await fs.promises.copyFile(
|
|
2006
|
+
filePath, temporary, fs.constants.COPYFILE_EXCL)
|
|
2007
|
+
if (Number.isFinite(expected.atimeMs) &&
|
|
2008
|
+
Number.isFinite(expected.mtimeMs)) {
|
|
2009
|
+
await fs.promises.utimes(
|
|
2010
|
+
temporary,
|
|
2011
|
+
new Date(expected.atimeMs),
|
|
2012
|
+
new Date(expected.mtimeMs)
|
|
2013
|
+
)
|
|
2014
|
+
}
|
|
2015
|
+
temporaryStat = await fs.promises.lstat(temporary)
|
|
2016
|
+
if (this.sourceAppIsRunning(source)) {
|
|
2017
|
+
await unlinkIfSame(temporary, temporaryStat)
|
|
2018
|
+
return { status: "locked" }
|
|
2019
|
+
}
|
|
2020
|
+
const [current, currentTemporary] = await Promise.all([
|
|
2021
|
+
lstatIfPresent(filePath),
|
|
2022
|
+
lstatIfPresent(temporary)
|
|
2023
|
+
])
|
|
2024
|
+
if (!current ||
|
|
2025
|
+
!sameSnapshot(expected, current) ||
|
|
2026
|
+
!sameIdentity(temporaryStat, currentTemporary)) {
|
|
2027
|
+
await unlinkIfSame(temporary, temporaryStat)
|
|
2028
|
+
return { status: "stale" }
|
|
2029
|
+
}
|
|
2030
|
+
await fs.promises.rename(temporary, filePath)
|
|
2031
|
+
let finalStat
|
|
2032
|
+
try {
|
|
2033
|
+
finalStat = await fs.promises.lstat(filePath)
|
|
2034
|
+
} catch (error) {
|
|
2035
|
+
// rename() already committed the separate copy. Preserve that
|
|
2036
|
+
// successful action if only the post-commit metadata read failed.
|
|
2037
|
+
finalStat = temporaryStat
|
|
2038
|
+
}
|
|
2039
|
+
if (!sameIdentity(finalStat, temporaryStat)) {
|
|
2040
|
+
return { status: "stale" }
|
|
2041
|
+
}
|
|
2042
|
+
temporaryStat = null
|
|
2043
|
+
return {
|
|
2044
|
+
status: "copied",
|
|
2045
|
+
stat: finalStat
|
|
2046
|
+
}
|
|
2047
|
+
} catch (error) {
|
|
2048
|
+
if (temporaryStat) {
|
|
2049
|
+
await unlinkIfSame(temporary, temporaryStat).catch(() => {})
|
|
2050
|
+
}
|
|
2051
|
+
if (error && error.code === "EEXIST") return { status: "conflict" }
|
|
2052
|
+
if (isMissingError(error)) return { status: "stale" }
|
|
2053
|
+
if (LOCK_CODES.has(error && error.code)) return { status: "locked" }
|
|
2054
|
+
throw error
|
|
2055
|
+
}
|
|
2056
|
+
}
|
|
1082
2057
|
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
2058
|
+
async separate(filePath, options = {}) {
|
|
2059
|
+
const entry = await this.registry.getFile(filePath)
|
|
2060
|
+
if (!entry || entry.unavailable_reason === "stale") {
|
|
2061
|
+
return { status: "not-found" }
|
|
2062
|
+
}
|
|
2063
|
+
const source = this.sourceForPath(entry.path, entry.source_id)
|
|
2064
|
+
if (!source ||
|
|
2065
|
+
!await this.canonicalPathIsWithinSource(entry.path, source)) {
|
|
1089
2066
|
return { status: "stale" }
|
|
1090
2067
|
}
|
|
1091
|
-
const
|
|
1092
|
-
if (!
|
|
2068
|
+
const current = await lstatIfPresent(entry.path)
|
|
2069
|
+
if (!current ||
|
|
2070
|
+
!current.isFile() ||
|
|
2071
|
+
!sameSnapshot(rowSnapshot(entry), current)) {
|
|
1093
2072
|
return { status: "stale" }
|
|
1094
2073
|
}
|
|
1095
|
-
if (
|
|
1096
|
-
if (
|
|
1097
|
-
const
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
2074
|
+
if (entry.status !== "linked") return { status: "not-found" }
|
|
2075
|
+
if (this.sourceAppIsRunning(source)) return { status: "locked" }
|
|
2076
|
+
const copied = await this.copyOut(
|
|
2077
|
+
entry.path, current, source)
|
|
2078
|
+
if (copied.status !== "copied") return copied
|
|
2079
|
+
await this.registry.upsertFile(Object.assign(
|
|
2080
|
+
{},
|
|
2081
|
+
entry,
|
|
2082
|
+
fileSnapshot(copied.stat),
|
|
2083
|
+
{
|
|
2084
|
+
status: "linked",
|
|
2085
|
+
unavailable_reason: null
|
|
2086
|
+
}
|
|
2087
|
+
))
|
|
2088
|
+
await this.refreshInodeSnapshots(entry.hash, entry.dev, entry.ino)
|
|
2089
|
+
if (options.reclassify !== false) {
|
|
2090
|
+
await this.reclassifyHashes([entry.hash])
|
|
1109
2091
|
}
|
|
2092
|
+
const result = {
|
|
2093
|
+
status: "detached",
|
|
2094
|
+
bytes: entry.size,
|
|
2095
|
+
hash: entry.hash
|
|
2096
|
+
}
|
|
2097
|
+
if (options.recordActivity !== false && !await this.recordEvent({
|
|
2098
|
+
kind: "detach",
|
|
2099
|
+
hash: entry.hash,
|
|
2100
|
+
path: entry.path,
|
|
2101
|
+
app: entry.app,
|
|
2102
|
+
source_id: entry.source_id,
|
|
2103
|
+
bytes: entry.size
|
|
2104
|
+
})) result.activity_warning = true
|
|
1110
2105
|
return result
|
|
1111
2106
|
}
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
const seen = new Set()
|
|
1115
|
-
let current = source
|
|
1116
|
-
while (current && !seen.has(current.id)) {
|
|
1117
|
-
if (current.id === scopeId) return true
|
|
1118
|
-
seen.add(current.id)
|
|
1119
|
-
current = this._sources.find((item) => item.id === current.parent_id)
|
|
1120
|
-
}
|
|
1121
|
-
return false
|
|
1122
|
-
}
|
|
1123
|
-
async deduplicateSelection(selection, scopeId, options = {}) {
|
|
1124
|
-
if (selection === "duplicates") {
|
|
1125
|
-
return this.deduplicateScope(scopeId, Object.assign({}, options, { includeDescendants: true }))
|
|
1126
|
-
}
|
|
1127
|
-
await this.refreshSources()
|
|
1128
|
-
if (scopeId && !this._sources.some((source) => source.id === scopeId)) {
|
|
1129
|
-
return { error: "That location is no longer available. Scan again to refresh it." }
|
|
1130
|
-
}
|
|
1131
|
-
const paths = [...this.registry.excluded].filter(([filePath, entry]) => {
|
|
1132
|
-
const source = this.sourceForPath(filePath, entry.source_id)
|
|
1133
|
-
return source ? this.sourceIsWithinScope(source, scopeId) : !scopeId
|
|
1134
|
-
}).map(([filePath]) => filePath)
|
|
1135
|
-
if (options.progress) options.progress.files_total = paths.length
|
|
2107
|
+
|
|
2108
|
+
async separateFiles(filePaths, progress = null) {
|
|
1136
2109
|
const summary = {
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
2110
|
+
separated: 0,
|
|
2111
|
+
bytes: 0,
|
|
2112
|
+
failed: 0,
|
|
2113
|
+
results: []
|
|
2114
|
+
}
|
|
2115
|
+
let separatedEntry = null
|
|
2116
|
+
let commonSourceId
|
|
2117
|
+
const affectedHashes = new Set()
|
|
2118
|
+
for (const filePath of new Set(filePaths.map((item) =>
|
|
2119
|
+
path.resolve(item)))) {
|
|
1142
2120
|
try {
|
|
1143
|
-
const
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
2121
|
+
const entry = await this.registry.getFile(filePath)
|
|
2122
|
+
const result = entry && entry.status === "linked"
|
|
2123
|
+
? await this.separate(filePath, {
|
|
2124
|
+
recordActivity: false,
|
|
2125
|
+
reclassify: false
|
|
2126
|
+
})
|
|
2127
|
+
: { status: "ineligible" }
|
|
2128
|
+
if (result.status === "detached") {
|
|
2129
|
+
summary.separated += 1
|
|
2130
|
+
summary.bytes += result.bytes || 0
|
|
2131
|
+
separatedEntry = entry
|
|
2132
|
+
affectedHashes.add(result.hash)
|
|
2133
|
+
const entrySourceId = entry.source_id || null
|
|
2134
|
+
commonSourceId = summary.separated === 1
|
|
2135
|
+
? entrySourceId
|
|
2136
|
+
: commonSourceId === entrySourceId ? commonSourceId : null
|
|
1158
2137
|
} else {
|
|
1159
2138
|
summary.failed += 1
|
|
1160
2139
|
}
|
|
2140
|
+
summary.results.push({ path: filePath, status: result.status })
|
|
1161
2141
|
} catch (error) {
|
|
1162
2142
|
summary.failed += 1
|
|
2143
|
+
summary.results.push({ path: filePath, status: "failed" })
|
|
2144
|
+
} finally {
|
|
2145
|
+
if (progress) progress.files_completed += 1
|
|
1163
2146
|
}
|
|
1164
|
-
if (options.progress) options.progress.files_completed += 1
|
|
1165
2147
|
}
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
async deduplicateScope(scopeId, options = {}) {
|
|
1169
|
-
await this.refreshSources()
|
|
1170
|
-
const includeDescendants = options.includeDescendants === true
|
|
1171
|
-
const source = scopeId
|
|
1172
|
-
? this._sources.find((item) => item.id === scopeId && (includeDescendants || item.kind !== "virtual"))
|
|
1173
|
-
: null
|
|
1174
|
-
if ((!includeDescendants || scopeId) && !source) {
|
|
1175
|
-
return { error: "That location is no longer available. Scan again to refresh it." }
|
|
2148
|
+
if (affectedHashes.size) {
|
|
2149
|
+
await this.reclassifyHashes(affectedHashes)
|
|
1176
2150
|
}
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
options.progress.files_total = entries.reduce((total, [filePath, entry]) => {
|
|
1193
|
-
const currentSource = this.sourceForPath(filePath, entry.source_id)
|
|
1194
|
-
return total + (matchesScope(currentSource) ? 1 : 0)
|
|
1195
|
-
}, 0)
|
|
1196
|
-
}
|
|
1197
|
-
const completeProgress = () => {
|
|
1198
|
-
if (options.progress) options.progress.files_completed += 1
|
|
1199
|
-
}
|
|
1200
|
-
for (const [filePath, entry] of entries) {
|
|
1201
|
-
const currentSource = this.sourceForPath(filePath, entry.source_id)
|
|
1202
|
-
if (!matchesScope(currentSource)) continue
|
|
1203
|
-
if (!await this.canonicalPathIsWithinSource(filePath, currentSource)) {
|
|
1204
|
-
summary.stale += 1
|
|
1205
|
-
completeProgress()
|
|
1206
|
-
continue
|
|
1207
|
-
}
|
|
1208
|
-
const indexed = registry.scanIndex.get(filePath)
|
|
1209
|
-
const expected = indexed && indexed.hash === entry.hash
|
|
1210
|
-
? indexed
|
|
1211
|
-
: (entry.dev !== undefined && entry.ino !== undefined &&
|
|
1212
|
-
entry.mtime !== undefined && entry.ctime !== undefined ? entry : null)
|
|
1213
|
-
if (!expected) {
|
|
1214
|
-
summary.stale += 1
|
|
1215
|
-
completeProgress()
|
|
1216
|
-
continue
|
|
1217
|
-
}
|
|
1218
|
-
if (staleHashes.has(entry.hash)) {
|
|
1219
|
-
summary.stale += 1
|
|
1220
|
-
completeProgress()
|
|
1221
|
-
continue
|
|
1222
|
-
}
|
|
1223
|
-
try {
|
|
1224
|
-
const result = await this.convert(filePath, entry.hash, {
|
|
1225
|
-
app: currentSource.kind === "app" ? currentSource.app : (entry.app || null),
|
|
1226
|
-
source_id: currentSource.id,
|
|
1227
|
-
batch_id: batch,
|
|
1228
|
-
source: currentSource,
|
|
1229
|
-
expected
|
|
1230
|
-
})
|
|
1231
|
-
if (result.status === "converted" || result.status === "already") {
|
|
1232
|
-
summary.converted += 1
|
|
1233
|
-
summary.bytes_saved += result.bytes_saved || 0
|
|
1234
|
-
} else if (result.status === "locked") {
|
|
1235
|
-
summary.locked += 1
|
|
1236
|
-
} else if (result.status === "stale") {
|
|
1237
|
-
summary.stale += 1
|
|
1238
|
-
} else if (result.status === "stale-blob") {
|
|
1239
|
-
staleHashes.add(entry.hash)
|
|
1240
|
-
summary.stale += 1
|
|
1241
|
-
} else if (result.status === "metadata-mismatch") {
|
|
1242
|
-
entry.unavailable_reason = "metadata"
|
|
1243
|
-
summary.incompatible += 1
|
|
1244
|
-
} else if (result.status === "unavailable" || result.status === "copy-mode") {
|
|
1245
|
-
summary.unavailable += 1
|
|
1246
|
-
} else {
|
|
1247
|
-
summary.failed += 1
|
|
1248
|
-
}
|
|
1249
|
-
} catch (error) {
|
|
1250
|
-
summary.failed += 1
|
|
1251
|
-
}
|
|
1252
|
-
completeProgress()
|
|
2151
|
+
const event = {
|
|
2152
|
+
kind: "detach",
|
|
2153
|
+
bytes: summary.bytes,
|
|
2154
|
+
files: summary.separated
|
|
2155
|
+
}
|
|
2156
|
+
if (summary.separated === 1 && separatedEntry) {
|
|
2157
|
+
event.hash = separatedEntry.hash
|
|
2158
|
+
event.path = separatedEntry.path
|
|
2159
|
+
event.app = separatedEntry.app
|
|
2160
|
+
event.source_id = separatedEntry.source_id
|
|
2161
|
+
} else if (commonSourceId) {
|
|
2162
|
+
event.source_id = commonSourceId
|
|
2163
|
+
}
|
|
2164
|
+
if (summary.separated && !await this.recordEvent(event)) {
|
|
2165
|
+
summary.activity_warning = true
|
|
1253
2166
|
}
|
|
1254
|
-
registry.schedulePersist()
|
|
1255
2167
|
return summary
|
|
1256
2168
|
}
|
|
1257
|
-
|
|
1258
|
-
async
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
if (!options.deferRegistry) this.registry.removeBlob(hash)
|
|
1273
|
-
return options.deferRegistry
|
|
1274
|
-
? { status: "invalid", remove_hash: true }
|
|
1275
|
-
: { status: "invalid" }
|
|
1276
|
-
}
|
|
1277
|
-
if (st.nlink > 1) return { status: "in-use" }
|
|
1278
|
-
await fs.promises.unlink(storePath)
|
|
1279
|
-
if (!options.deferRegistry) this.registry.removeBlob(hash)
|
|
1280
|
-
await this.recordEvent({ kind: "reclaim", hash, bytes_saved: st.size })
|
|
1281
|
-
return options.deferRegistry
|
|
1282
|
-
? { status: "reclaimed", bytes_freed: st.size, remove_hash: true }
|
|
1283
|
-
: { status: "reclaimed", bytes_freed: st.size }
|
|
1284
|
-
}
|
|
1285
|
-
// Startup registry verification (trigger 5): bounded by registry size,
|
|
1286
|
-
// never a discovery walk. Prunes dead links, flags orphans, removes stray
|
|
1287
|
-
// conversion tmp files, re-adopts blobs whose store name was deleted.
|
|
1288
|
-
async verify(options = {}) {
|
|
1289
|
-
if (!this.enabled || !this.registry) return
|
|
1290
|
-
const preservePath = options.preservePath || (() => false)
|
|
1291
|
-
const includePath = options.includePath || (() => true)
|
|
1292
|
-
const handleAccessError = (error, filePath) => !!(
|
|
1293
|
-
isAccessError(error) && options.onAccessError && options.onAccessError(error, filePath)
|
|
1294
|
-
)
|
|
1295
|
-
if (options.reconcile !== false) this.reconcileConfiguredSources()
|
|
1296
|
-
for (const [linkPath, entry] of [...this.registry.links]) {
|
|
1297
|
-
if (!includePath(linkPath)) continue
|
|
1298
|
-
if (preservePath(linkPath)) continue
|
|
1299
|
-
try {
|
|
1300
|
-
const source = this.sourceForPath(linkPath, entry.source_id)
|
|
1301
|
-
const managedPath = source && await this.canonicalPathIsWithinSource(linkPath, source, { strictErrors: true })
|
|
1302
|
-
if (!managedPath) {
|
|
1303
|
-
this.registry.untrack(linkPath)
|
|
1304
|
-
continue
|
|
1305
|
-
}
|
|
1306
|
-
const st = await lstatIfPresent(linkPath)
|
|
1307
|
-
if (!st || !st.isFile() || st.ino !== entry.ino || st.dev !== entry.dev) {
|
|
1308
|
-
this.registry.untrack(linkPath)
|
|
1309
|
-
}
|
|
1310
|
-
} catch (error) {
|
|
1311
|
-
if (!handleAccessError(error, linkPath)) throw error
|
|
1312
|
-
}
|
|
1313
|
-
}
|
|
1314
|
-
for (const [dupPath, entry] of [...this.registry.duplicates]) {
|
|
1315
|
-
if (!includePath(dupPath)) continue
|
|
1316
|
-
if (preservePath(dupPath)) continue
|
|
1317
|
-
let valid = false
|
|
1318
|
-
try {
|
|
1319
|
-
const source = this.sourceForPath(dupPath, entry.source_id)
|
|
1320
|
-
const st = await lstatIfPresent(dupPath)
|
|
1321
|
-
valid = !!(source && st && st.isFile() &&
|
|
1322
|
-
await this.canonicalPathIsWithinSource(dupPath, source, { strictErrors: true }))
|
|
1323
|
-
} catch (error) {
|
|
1324
|
-
if (handleAccessError(error, dupPath)) continue
|
|
1325
|
-
throw error
|
|
1326
|
-
}
|
|
1327
|
-
if (!valid) {
|
|
1328
|
-
this.registry.untrack(dupPath)
|
|
1329
|
-
continue
|
|
2169
|
+
|
|
2170
|
+
async separateMatchingFiles(selection, progress = null) {
|
|
2171
|
+
const summary = {
|
|
2172
|
+
separated: 0,
|
|
2173
|
+
bytes: 0,
|
|
2174
|
+
failed: 0,
|
|
2175
|
+
cancelled: false
|
|
2176
|
+
}
|
|
2177
|
+
let separatedEntry = null
|
|
2178
|
+
let commonSourceId
|
|
2179
|
+
let cursor = ""
|
|
2180
|
+
while (true) {
|
|
2181
|
+
if (this.fileActionCancelRequested) {
|
|
2182
|
+
summary.cancelled = true
|
|
2183
|
+
break
|
|
1330
2184
|
}
|
|
1331
|
-
const
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
linksByHash.get(entry.hash).push([linkPath, entry])
|
|
1344
|
-
}
|
|
1345
|
-
const invalidBlobs = new Set()
|
|
1346
|
-
for (const [hash, blob] of [...this.registry.blobs]) {
|
|
1347
|
-
const storePath = this.storePathFor(hash)
|
|
1348
|
-
const st = await this.storeStatIfPresent(storePath)
|
|
1349
|
-
if (!st) {
|
|
1350
|
-
const names = linksByHash.get(hash) || []
|
|
1351
|
-
const copyNames = names.filter(([, entry]) => entry.mode === "copy")
|
|
1352
|
-
// Deleting a name changes ctime, so trusted re-adoption compares the
|
|
1353
|
-
// preserved identity, size, and mtime. Changed or legacy-unsnapshotted
|
|
1354
|
-
// content waits for the next explicit scan instead of being assigned a
|
|
1355
|
-
// stale hash filename during startup.
|
|
1356
|
-
let readopted = false
|
|
1357
|
-
let inaccessibleName = false
|
|
1358
|
-
for (const [linkPath, entry] of names) {
|
|
1359
|
-
if (entry.mode !== "link") continue
|
|
1360
|
-
if (preservePath(linkPath)) {
|
|
1361
|
-
inaccessibleName = true
|
|
1362
|
-
continue
|
|
1363
|
-
}
|
|
1364
|
-
let source
|
|
1365
|
-
let linkStat
|
|
1366
|
-
try {
|
|
1367
|
-
source = this.sourceForPath(linkPath, entry.source_id)
|
|
1368
|
-
if (!source || !await this.canonicalPathIsWithinSource(linkPath, source, { strictErrors: true })) continue
|
|
1369
|
-
linkStat = await lstatIfPresent(linkPath)
|
|
1370
|
-
} catch (error) {
|
|
1371
|
-
if (!handleAccessError(error, linkPath)) throw error
|
|
1372
|
-
inaccessibleName = true
|
|
1373
|
-
continue
|
|
1374
|
-
}
|
|
1375
|
-
const expected = this.registry.scanIndex.get(linkPath)
|
|
1376
|
-
if (!linkStat || !expected || expected.hash !== hash ||
|
|
1377
|
-
entry.dev !== linkStat.dev || entry.ino !== linkStat.ino ||
|
|
1378
|
-
!sameContentState(expected, linkStat)) continue
|
|
1379
|
-
const before = fileSnapshot(linkStat)
|
|
1380
|
-
let created = false
|
|
1381
|
-
try {
|
|
1382
|
-
await this.storeStatIfPresent(storePath, { createParent: true })
|
|
1383
|
-
await fs.promises.link(linkPath, storePath)
|
|
1384
|
-
created = true
|
|
1385
|
-
} catch (error) {
|
|
1386
|
-
if (!error || error.code !== "EEXIST") throw error
|
|
1387
|
-
const currentStore = await this.storeStatIfPresent(storePath)
|
|
1388
|
-
const currentLink = await lstatIfPresent(linkPath)
|
|
1389
|
-
if (!currentStore || !sameContentState(before, currentLink) ||
|
|
1390
|
-
currentStore.dev !== linkStat.dev || currentStore.ino !== linkStat.ino) continue
|
|
1391
|
-
}
|
|
1392
|
-
const [currentLink, currentStore] = await Promise.all([
|
|
1393
|
-
lstatIfPresent(linkPath),
|
|
1394
|
-
this.storeStatIfPresent(storePath)
|
|
1395
|
-
])
|
|
1396
|
-
if (!sameContentState(before, currentLink) || !currentStore ||
|
|
1397
|
-
currentStore.dev !== linkStat.dev || currentStore.ino !== linkStat.ino) {
|
|
1398
|
-
const ownsStoreName = created && currentStore && (
|
|
1399
|
-
(currentStore.dev === linkStat.dev && currentStore.ino === linkStat.ino) ||
|
|
1400
|
-
(currentLink && currentStore.dev === currentLink.dev && currentStore.ino === currentLink.ino)
|
|
1401
|
-
)
|
|
1402
|
-
if (ownsStoreName) await fs.promises.unlink(storePath)
|
|
1403
|
-
continue
|
|
1404
|
-
}
|
|
1405
|
-
await this.refreshLinkSnapshots(hash, linkStat.dev, linkStat.ino)
|
|
1406
|
-
readopted = true
|
|
2185
|
+
const rows = await this.registry.fileBatch(
|
|
2186
|
+
"linked",
|
|
2187
|
+
selection.sourceIds,
|
|
2188
|
+
cursor,
|
|
2189
|
+
100,
|
|
2190
|
+
selection.query
|
|
2191
|
+
)
|
|
2192
|
+
if (!rows.length) break
|
|
2193
|
+
const affectedHashes = new Set()
|
|
2194
|
+
for (const row of rows) {
|
|
2195
|
+
if (this.fileActionCancelRequested) {
|
|
2196
|
+
summary.cancelled = true
|
|
1407
2197
|
break
|
|
1408
2198
|
}
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
2199
|
+
cursor = row.path
|
|
2200
|
+
try {
|
|
2201
|
+
const entry = await this.registry.getFile(row.path)
|
|
2202
|
+
const result = entry && entry.status === "linked"
|
|
2203
|
+
? await this.separate(row.path, {
|
|
2204
|
+
recordActivity: false,
|
|
2205
|
+
reclassify: false
|
|
2206
|
+
})
|
|
2207
|
+
: { status: "ineligible" }
|
|
2208
|
+
if (result.status === "detached") {
|
|
2209
|
+
summary.separated += 1
|
|
2210
|
+
summary.bytes += result.bytes || 0
|
|
2211
|
+
separatedEntry = entry
|
|
2212
|
+
affectedHashes.add(result.hash)
|
|
2213
|
+
const entrySourceId = entry.source_id || null
|
|
2214
|
+
commonSourceId = summary.separated === 1
|
|
2215
|
+
? entrySourceId
|
|
2216
|
+
: commonSourceId === entrySourceId ? commonSourceId : null
|
|
2217
|
+
} else {
|
|
2218
|
+
summary.failed += 1
|
|
2219
|
+
}
|
|
2220
|
+
} catch (error) {
|
|
2221
|
+
summary.failed += 1
|
|
2222
|
+
} finally {
|
|
2223
|
+
if (progress) progress.files_completed += 1
|
|
1417
2224
|
}
|
|
1418
|
-
continue
|
|
1419
2225
|
}
|
|
1420
|
-
if (
|
|
1421
|
-
|
|
1422
|
-
continue
|
|
2226
|
+
if (affectedHashes.size) {
|
|
2227
|
+
await this.reclassifyHashes(affectedHashes)
|
|
1423
2228
|
}
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
const
|
|
1450
|
-
const
|
|
1451
|
-
|
|
2229
|
+
if (summary.cancelled) break
|
|
2230
|
+
}
|
|
2231
|
+
const event = {
|
|
2232
|
+
kind: "detach",
|
|
2233
|
+
bytes: summary.bytes,
|
|
2234
|
+
files: summary.separated
|
|
2235
|
+
}
|
|
2236
|
+
if (summary.separated === 1 && separatedEntry) {
|
|
2237
|
+
event.hash = separatedEntry.hash
|
|
2238
|
+
event.path = separatedEntry.path
|
|
2239
|
+
event.app = separatedEntry.app
|
|
2240
|
+
event.source_id = separatedEntry.source_id
|
|
2241
|
+
} else if (commonSourceId) {
|
|
2242
|
+
event.source_id = commonSourceId
|
|
2243
|
+
}
|
|
2244
|
+
if (summary.separated && !await this.recordEvent(event)) {
|
|
2245
|
+
summary.activity_warning = true
|
|
2246
|
+
}
|
|
2247
|
+
return summary
|
|
2248
|
+
}
|
|
2249
|
+
|
|
2250
|
+
async reclaim(hash, storeId = null) {
|
|
2251
|
+
if (typeof hash !== "string" || !SHA256_RE.test(hash)) {
|
|
2252
|
+
return { status: "not-found" }
|
|
2253
|
+
}
|
|
2254
|
+
const anchors = await this.registry.anchorsForHash(hash)
|
|
2255
|
+
const anchor = storeId
|
|
2256
|
+
? anchors.find((item) => item.store_id === storeId)
|
|
2257
|
+
: anchors.length === 1 ? anchors[0] : null
|
|
2258
|
+
if (!anchor) return { status: "not-found" }
|
|
2259
|
+
const store = this._anchorStoresById.get(anchor.store_id)
|
|
2260
|
+
if (!store) return { status: "unavailable" }
|
|
2261
|
+
const storePath = this.storePathFor(hash, store.id)
|
|
2262
|
+
if (!samePath(storePath, anchor.path)) return { status: "stale" }
|
|
1452
2263
|
try {
|
|
1453
|
-
|
|
2264
|
+
await this.validateAnchorStore(store, anchor.dev)
|
|
1454
2265
|
} catch (error) {
|
|
1455
|
-
|
|
2266
|
+
return { status: "stale" }
|
|
1456
2267
|
}
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
2268
|
+
const stat = await this.storeStatIfPresent(storePath, {
|
|
2269
|
+
store_id: store.id
|
|
2270
|
+
})
|
|
2271
|
+
if (!stat) {
|
|
2272
|
+
await this.registry.removeAnchor(store.id, hash)
|
|
2273
|
+
return { status: "gone" }
|
|
2274
|
+
}
|
|
2275
|
+
const expected = {
|
|
2276
|
+
size: anchor.size,
|
|
2277
|
+
mtime: anchor.mtime,
|
|
2278
|
+
ctime: anchor.ctime,
|
|
2279
|
+
dev: anchor.dev,
|
|
2280
|
+
ino: anchor.ino
|
|
2281
|
+
}
|
|
2282
|
+
if (!sameSnapshot(expected, stat)) return { status: "stale" }
|
|
2283
|
+
if (stat.nlink !== 1) return { status: "in-use" }
|
|
2284
|
+
await fs.promises.unlink(storePath)
|
|
2285
|
+
await this.registry.removeAnchor(store.id, hash)
|
|
2286
|
+
const hasFiles = await this.registry.hasFilesForHash(hash)
|
|
2287
|
+
const remainingAnchors = await this.registry.anchorsForHash(hash)
|
|
2288
|
+
if (!hasFiles && !remainingAnchors.length) {
|
|
2289
|
+
await this.registry.removeContent(hash)
|
|
2290
|
+
}
|
|
2291
|
+
const result = { status: "reclaimed", bytes_freed: stat.size }
|
|
2292
|
+
if (!await this.recordEvent({
|
|
2293
|
+
kind: "reclaim",
|
|
2294
|
+
hash,
|
|
2295
|
+
bytes: stat.size
|
|
2296
|
+
})) result.activity_warning = true
|
|
2297
|
+
return result
|
|
2298
|
+
}
|
|
2299
|
+
|
|
2300
|
+
async reclaimAll() {
|
|
2301
|
+
const summary = { reclaimed: 0, bytes_freed: 0, failed: 0 }
|
|
2302
|
+
let cursor = { store_id: "", hash: "" }
|
|
2303
|
+
while (true) {
|
|
2304
|
+
const anchors = await this.registry.reclaimableBatch(cursor, 100)
|
|
2305
|
+
if (!anchors.length) break
|
|
2306
|
+
for (const row of anchors) {
|
|
2307
|
+
cursor = { store_id: row.store_id, hash: row.hash }
|
|
1472
2308
|
try {
|
|
1473
|
-
const
|
|
1474
|
-
if (
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
orphan: st.nlink === 1
|
|
1482
|
-
})
|
|
1483
|
-
storeInoToHash.set(`${st.dev}:${st.ino}`, name)
|
|
2309
|
+
const result = await this.reclaim(row.hash, row.store_id)
|
|
2310
|
+
if (result.status === "reclaimed") {
|
|
2311
|
+
summary.reclaimed += 1
|
|
2312
|
+
summary.bytes_freed += result.bytes_freed || 0
|
|
2313
|
+
if (result.activity_warning) summary.activity_warning = true
|
|
2314
|
+
} else if (!["gone", "in-use"].includes(result.status)) {
|
|
2315
|
+
summary.failed += 1
|
|
2316
|
+
}
|
|
1484
2317
|
} catch (error) {
|
|
1485
|
-
|
|
2318
|
+
summary.failed += 1
|
|
1486
2319
|
}
|
|
1487
2320
|
}
|
|
1488
2321
|
}
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
2322
|
+
return summary
|
|
2323
|
+
}
|
|
2324
|
+
|
|
2325
|
+
async scanForScope(scopeId) {
|
|
2326
|
+
const direct = await this.registry.scanFor(scopeId)
|
|
2327
|
+
const key = scopeId || ""
|
|
2328
|
+
if (direct || !scopeId) {
|
|
2329
|
+
if (direct) this.lastScanCache.set(key, direct)
|
|
2330
|
+
else this.lastScanCache.delete(key)
|
|
2331
|
+
return direct
|
|
2332
|
+
}
|
|
2333
|
+
const global = await this.registry.scanFor()
|
|
2334
|
+
if (global) this.lastScanCache.set("", global)
|
|
2335
|
+
else this.lastScanCache.delete("")
|
|
2336
|
+
if (!global ||
|
|
2337
|
+
!global.source_files ||
|
|
2338
|
+
!Object.prototype.hasOwnProperty.call(
|
|
2339
|
+
global.source_files, scopeId)) {
|
|
2340
|
+
this.lastScanCache.delete(key)
|
|
2341
|
+
return null
|
|
2342
|
+
}
|
|
2343
|
+
const scoped = Object.assign({}, global, {
|
|
2344
|
+
files: global.source_files[scopeId] || 0,
|
|
2345
|
+
bytes_total: global.source_bytes[scopeId] || 0,
|
|
2346
|
+
hash_failures: global.source_hash_failures
|
|
2347
|
+
? global.source_hash_failures[scopeId] || 0
|
|
2348
|
+
: 0
|
|
2349
|
+
})
|
|
2350
|
+
const sourceIds = new Set(this.scopeSourceIds(scopeId))
|
|
2351
|
+
scoped.exclusions = Array.isArray(global.exclusions)
|
|
2352
|
+
? global.exclusions.filter((entry) =>
|
|
2353
|
+
entry && sourceIds.has(entry.source_id))
|
|
2354
|
+
: []
|
|
2355
|
+
scoped.partial = scoped.exclusions.length > 0
|
|
2356
|
+
scoped.outcome = scoped.partial
|
|
2357
|
+
? "completed_with_exclusions"
|
|
2358
|
+
: "complete"
|
|
2359
|
+
this.lastScanCache.set(key, scoped)
|
|
2360
|
+
return scoped
|
|
2361
|
+
}
|
|
2362
|
+
|
|
2363
|
+
sourceCountMaps(summaryRows) {
|
|
2364
|
+
const counts = {
|
|
2365
|
+
all: {},
|
|
2366
|
+
duplicates: {},
|
|
2367
|
+
shareable: {}
|
|
2368
|
+
}
|
|
2369
|
+
const add = (target, sourceIdValue, amount) => {
|
|
2370
|
+
let current = this._sourcesById.get(sourceIdValue)
|
|
2371
|
+
const seen = new Set()
|
|
2372
|
+
while (current && !seen.has(current.id)) {
|
|
2373
|
+
target[current.id] = (target[current.id] || 0) + amount
|
|
2374
|
+
seen.add(current.id)
|
|
2375
|
+
current = current.parent_id
|
|
2376
|
+
? this._sourcesById.get(current.parent_id)
|
|
2377
|
+
: null
|
|
1536
2378
|
}
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
const st = await fs.promises.lstat(linkPath)
|
|
1546
|
-
if (sameSnapshot(expected, st)) trustedHashes.add(link.hash)
|
|
1547
|
-
} catch (error) {
|
|
1548
|
-
if (!isMissingError(error)) throw error
|
|
1549
|
-
}
|
|
2379
|
+
}
|
|
2380
|
+
for (const row of summaryRows) {
|
|
2381
|
+
const amount = Number(row.file_count) || 0
|
|
2382
|
+
if (!["reference", "duplicate", "linked", "unavailable"]
|
|
2383
|
+
.includes(row.status)) continue
|
|
2384
|
+
add(counts.all, row.source_id, amount)
|
|
2385
|
+
if (row.status === "duplicate" || row.status === "unavailable") {
|
|
2386
|
+
add(counts.duplicates, row.source_id, amount)
|
|
1550
2387
|
}
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
try {
|
|
1554
|
-
const st = await fs.promises.lstat(linkPath)
|
|
1555
|
-
rebuiltScanIndex.set(linkPath, {
|
|
1556
|
-
hash: link.hash, size: st.size, dev: st.dev, ino: st.ino,
|
|
1557
|
-
mtime: st.mtimeMs, ctime: st.ctimeMs,
|
|
1558
|
-
source_id: link.source_id || null
|
|
1559
|
-
})
|
|
1560
|
-
} catch (error) {
|
|
1561
|
-
if (!isMissingError(error)) throw error
|
|
1562
|
-
}
|
|
2388
|
+
if (row.status === "duplicate") {
|
|
2389
|
+
add(counts.shareable, row.source_id, amount)
|
|
1563
2390
|
}
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
2391
|
+
}
|
|
2392
|
+
return counts
|
|
2393
|
+
}
|
|
2394
|
+
|
|
2395
|
+
publicFileItems(page) {
|
|
2396
|
+
const hashSiblings = new Map()
|
|
2397
|
+
for (const sibling of page.hashSiblings || []) {
|
|
2398
|
+
if (!hashSiblings.has(sibling.hash)) {
|
|
2399
|
+
hashSiblings.set(sibling.hash, [])
|
|
2400
|
+
}
|
|
2401
|
+
hashSiblings.get(sibling.hash).push(sibling)
|
|
2402
|
+
}
|
|
2403
|
+
const inodeSiblings = new Map()
|
|
2404
|
+
for (const sibling of page.inodeSiblings || []) {
|
|
2405
|
+
const key = `${sibling.dev}:${sibling.ino}`
|
|
2406
|
+
if (!inodeSiblings.has(key)) inodeSiblings.set(key, [])
|
|
2407
|
+
inodeSiblings.get(key).push(sibling)
|
|
2408
|
+
}
|
|
2409
|
+
return (page.rows || []).map((row) => {
|
|
2410
|
+
const sampledMatches = row.status === "linked"
|
|
2411
|
+
? inodeSiblings.get(`${row.dev}:${row.ino}`) || []
|
|
2412
|
+
: hashSiblings.get(row.hash) || []
|
|
2413
|
+
const allMatches = [
|
|
2414
|
+
row,
|
|
2415
|
+
...sampledMatches.filter((match) => match.path !== row.path)
|
|
2416
|
+
]
|
|
2417
|
+
const locations = allMatches.map((match) =>
|
|
2418
|
+
Object.assign({
|
|
2419
|
+
path: match.path,
|
|
2420
|
+
app: match.app,
|
|
2421
|
+
dev: match.dev,
|
|
2422
|
+
ino: match.ino
|
|
2423
|
+
}, this.locationForPath(match.path, match.source_id)))
|
|
2424
|
+
const publicStatus = {
|
|
2425
|
+
reference: "tracked",
|
|
2426
|
+
duplicate: "duplicate",
|
|
2427
|
+
unavailable: "duplicate",
|
|
2428
|
+
linked: "shared"
|
|
2429
|
+
}[row.status]
|
|
2430
|
+
const result = Object.assign({
|
|
2431
|
+
path: row.path,
|
|
2432
|
+
hash: row.hash,
|
|
2433
|
+
size: row.size,
|
|
2434
|
+
app: row.app,
|
|
2435
|
+
status: publicStatus,
|
|
2436
|
+
shareable: row.status === "duplicate",
|
|
2437
|
+
unavailable_reason: row.status === "unavailable"
|
|
2438
|
+
? row.unavailable_reason || "different_disk"
|
|
2439
|
+
: null,
|
|
2440
|
+
location_count: sampledMatches.length
|
|
2441
|
+
? Number(sampledMatches[0].location_count) || 1
|
|
2442
|
+
: 1,
|
|
2443
|
+
locations
|
|
2444
|
+
}, this.locationForPath(row.path, row.source_id))
|
|
2445
|
+
if (row.status === "duplicate" || row.status === "unavailable") {
|
|
2446
|
+
const match = allMatches.find((candidate) =>
|
|
2447
|
+
candidate.path !== row.path)
|
|
2448
|
+
result.match = match
|
|
2449
|
+
? Object.assign({
|
|
2450
|
+
path: match.path,
|
|
2451
|
+
app: match.app
|
|
2452
|
+
}, this.locationForPath(match.path, match.source_id))
|
|
2453
|
+
: null
|
|
1587
2454
|
}
|
|
1588
|
-
|
|
1589
|
-
rebuiltLinks.delete(excludedPath)
|
|
1590
|
-
rebuiltScanIndex.delete(excludedPath)
|
|
1591
|
-
}
|
|
1592
|
-
}
|
|
1593
|
-
const rebuiltByIno = new Map()
|
|
1594
|
-
const rebuiltPathsByIno = new Map()
|
|
1595
|
-
for (const [linkPath, entry] of rebuiltLinks) {
|
|
1596
|
-
const key = registry.inoKey(entry.dev, entry.ino)
|
|
1597
|
-
rebuiltByIno.set(key, entry.hash)
|
|
1598
|
-
if (!rebuiltPathsByIno.has(key)) rebuiltPathsByIno.set(key, new Set())
|
|
1599
|
-
rebuiltPathsByIno.get(key).add(linkPath)
|
|
1600
|
-
}
|
|
1601
|
-
registry.replaceState({
|
|
1602
|
-
blobs: rebuiltBlobs,
|
|
1603
|
-
links: rebuiltLinks,
|
|
1604
|
-
scanIndex: rebuiltScanIndex,
|
|
1605
|
-
duplicates: rebuiltDuplicates,
|
|
1606
|
-
excluded: preserved ? preserved.excluded : new Map(),
|
|
1607
|
-
lastScan: preserved ? preserved.lastScan : null,
|
|
1608
|
-
sourceScans: preserved ? preserved.sourceScans : new Map(),
|
|
1609
|
-
totals: preserved ? preserved.totals : { lifetime_bytes_saved: 0 },
|
|
1610
|
-
byIno: rebuiltByIno,
|
|
1611
|
-
pathsByIno: rebuiltPathsByIno
|
|
2455
|
+
return result
|
|
1612
2456
|
})
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
2457
|
+
}
|
|
2458
|
+
|
|
2459
|
+
publicDuplicateGroupItems(page) {
|
|
2460
|
+
return (page.rows || []).map((row) => Object.assign({
|
|
2461
|
+
kind: "duplicate_group",
|
|
2462
|
+
hash: row.hash,
|
|
2463
|
+
path: row.representative_path,
|
|
2464
|
+
size: Number(row.size) || 0,
|
|
2465
|
+
total_count: Number(row.total_count) || 0,
|
|
2466
|
+
eligible_count: Number(row.eligible_count) || 0,
|
|
2467
|
+
can_save: Number(row.can_save) || 0,
|
|
2468
|
+
app: row.representative_app || null
|
|
2469
|
+
}, this.locationForPath(
|
|
2470
|
+
row.representative_path,
|
|
2471
|
+
row.representative_source_id
|
|
2472
|
+
)))
|
|
2473
|
+
}
|
|
2474
|
+
|
|
2475
|
+
publicDuplicateChildItems(rows) {
|
|
2476
|
+
const publicStatus = {
|
|
2477
|
+
reference: "tracked",
|
|
2478
|
+
duplicate: "duplicate",
|
|
2479
|
+
unavailable: "duplicate",
|
|
2480
|
+
linked: "shared"
|
|
2481
|
+
}
|
|
2482
|
+
return (rows || []).map((row) => Object.assign({
|
|
2483
|
+
kind: "duplicate_path",
|
|
2484
|
+
path: row.path,
|
|
2485
|
+
hash: row.hash,
|
|
2486
|
+
size: Number(row.size) || 0,
|
|
2487
|
+
app: row.app || null,
|
|
2488
|
+
status: publicStatus[row.status],
|
|
2489
|
+
registry_status: row.status,
|
|
2490
|
+
shareable: row.status === "duplicate",
|
|
2491
|
+
selectable: !!row.selectable,
|
|
2492
|
+
unavailable_reason: row.status === "unavailable"
|
|
2493
|
+
? row.unavailable_reason || "different_disk"
|
|
2494
|
+
: null
|
|
2495
|
+
}, this.locationForPath(row.path, row.source_id)))
|
|
2496
|
+
}
|
|
2497
|
+
|
|
2498
|
+
async duplicateGroupChildren(scopeId, hash, options = {}) {
|
|
2499
|
+
if (typeof hash !== "string" || !SHA256_RE.test(hash)) {
|
|
2500
|
+
throw new TypeError("Invalid vault content identifier.")
|
|
2501
|
+
}
|
|
2502
|
+
const locationId = typeof options.location_id === "string" &&
|
|
2503
|
+
options.location_id
|
|
2504
|
+
? options.location_id
|
|
2505
|
+
: null
|
|
2506
|
+
const query = String(options.query || "").slice(0, 500).trim()
|
|
2507
|
+
const cursor = typeof options.cursor === "string"
|
|
2508
|
+
? options.cursor.slice(0, 2048)
|
|
2509
|
+
: ""
|
|
2510
|
+
const pageSize = boundedInteger(
|
|
2511
|
+
options.page_size, 100, 1, STATUS_PAGE_SIZE)
|
|
2512
|
+
const result = await this.registry.duplicateGroupChildren({
|
|
2513
|
+
hash,
|
|
2514
|
+
sourceIds: this.scopeSourceIds(scopeId),
|
|
2515
|
+
activeSourceIds: this.scopeSourceIds(scopeId, locationId),
|
|
2516
|
+
query,
|
|
2517
|
+
cursor,
|
|
2518
|
+
pageSize,
|
|
2519
|
+
unrestricted: !scopeId,
|
|
2520
|
+
activeUnrestricted: !scopeId && !locationId
|
|
2521
|
+
})
|
|
2522
|
+
return {
|
|
2523
|
+
hash,
|
|
2524
|
+
items: this.publicDuplicateChildItems(result.rows),
|
|
2525
|
+
total: Number(result.total) || 0,
|
|
2526
|
+
next_cursor: result.nextCursor || null
|
|
1648
2527
|
}
|
|
1649
2528
|
}
|
|
1650
|
-
|
|
1651
|
-
|
|
1652
|
-
if (
|
|
1653
|
-
|
|
1654
|
-
|
|
1655
|
-
|
|
2529
|
+
|
|
2530
|
+
async duplicateGroupSelection(scopeId, hash, options = {}) {
|
|
2531
|
+
if (typeof hash !== "string" || !SHA256_RE.test(hash)) {
|
|
2532
|
+
throw new TypeError("Invalid vault content identifier.")
|
|
2533
|
+
}
|
|
2534
|
+
const locationId = typeof options.location_id === "string" &&
|
|
2535
|
+
options.location_id
|
|
2536
|
+
? options.location_id
|
|
2537
|
+
: null
|
|
2538
|
+
const result = await this.registry.duplicateGroupSelection({
|
|
2539
|
+
hash,
|
|
2540
|
+
sourceIds: this.scopeSourceIds(scopeId, locationId),
|
|
2541
|
+
query: String(options.query || "").slice(0, 500).trim(),
|
|
2542
|
+
unrestricted: !scopeId && !locationId
|
|
2543
|
+
})
|
|
2544
|
+
return {
|
|
2545
|
+
hash,
|
|
2546
|
+
paths: result.paths || [],
|
|
2547
|
+
exceeded: !!result.exceeded
|
|
2548
|
+
}
|
|
1656
2549
|
}
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
async status(scopeId = null) {
|
|
2550
|
+
|
|
2551
|
+
async status(scopeId = null, options = {}) {
|
|
1660
2552
|
if (!this.enabled || !this.registry) return { enabled: false }
|
|
1661
|
-
const
|
|
1662
|
-
const
|
|
1663
|
-
|
|
1664
|
-
const
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
const
|
|
1674
|
-
|
|
1675
|
-
const
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
path: linkPath, app: entry.app || null, mode: entry.mode
|
|
1684
|
-
}, location))
|
|
1685
|
-
if (entry.batch_id && (!scopeId || entry.source_id === scopeId)) {
|
|
1686
|
-
if (!undoBatchMap.has(entry.batch_id)) {
|
|
1687
|
-
undoBatchMap.set(entry.batch_id, { batch_id: entry.batch_id, files: 0, bytes: 0, ts: null })
|
|
1688
|
-
}
|
|
1689
|
-
const batch = undoBatchMap.get(entry.batch_id)
|
|
1690
|
-
const blob = registry.blobs.get(entry.hash)
|
|
1691
|
-
batch.files += 1
|
|
1692
|
-
batch.bytes += blob && Number.isFinite(blob.size) ? blob.size : 0
|
|
1693
|
-
}
|
|
1694
|
-
}
|
|
1695
|
-
const pendingBytesByHash = new Map()
|
|
1696
|
-
for (const entry of registry.duplicates.values()) {
|
|
1697
|
-
if (scopeId && entry.source_id !== scopeId) continue
|
|
1698
|
-
pendingBytesByHash.set(entry.hash, (pendingBytesByHash.get(entry.hash) || 0) + (entry.size || 0))
|
|
1699
|
-
}
|
|
1700
|
-
const blobEntries = [...registry.blobs].filter(([hash]) => !scopeHashes || scopeHashes.has(hash))
|
|
1701
|
-
if (!await this.directoryIfSafe(this.root) || !await this.directoryIfSafe(this.blobRoot)) {
|
|
1702
|
-
throw unsafeStoragePath(this.blobRoot)
|
|
1703
|
-
}
|
|
1704
|
-
const storePaths = blobEntries.map(([hash]) => this.storePathFor(hash))
|
|
1705
|
-
await Promise.all([...new Set(storePaths.map((storePath) => path.dirname(storePath)))]
|
|
1706
|
-
.map((shard) => this.directoryIfSafe(shard)))
|
|
1707
|
-
const blobStats = await statMany(
|
|
1708
|
-
storePaths,
|
|
1709
|
-
this.statConcurrency,
|
|
1710
|
-
null,
|
|
1711
|
-
{ strictErrors: true, followSymlinks: false }
|
|
2553
|
+
const view = STATUS_VIEWS.has(options.view) ? options.view : "all"
|
|
2554
|
+
const groupDuplicates =
|
|
2555
|
+
view === "duplicates" && options.display_mode === "files"
|
|
2556
|
+
const statusFilter = STATUS_FILTERS.has(options.status_filter)
|
|
2557
|
+
? options.status_filter
|
|
2558
|
+
: "all"
|
|
2559
|
+
const query = String(options.query || "").slice(0, 500).trim()
|
|
2560
|
+
const cursor = typeof options.cursor === "string"
|
|
2561
|
+
? options.cursor.slice(0, 2048)
|
|
2562
|
+
: ""
|
|
2563
|
+
const pageSize = boundedInteger(
|
|
2564
|
+
options.page_size, STATUS_PAGE_SIZE, 1, STATUS_PAGE_SIZE)
|
|
2565
|
+
const requestedPage = boundedInteger(
|
|
2566
|
+
options.page, 0, 0, Number.MAX_SAFE_INTEGER)
|
|
2567
|
+
const locationId = typeof options.location_id === "string" &&
|
|
2568
|
+
options.location_id
|
|
2569
|
+
? options.location_id
|
|
2570
|
+
: null
|
|
2571
|
+
const scopeSourceIds = this.scopeSourceIds(scopeId)
|
|
2572
|
+
const locationSourceIds = this.scopeSourceIds(
|
|
2573
|
+
scopeId,
|
|
2574
|
+
locationId
|
|
1712
2575
|
)
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
|
|
1717
|
-
|
|
1718
|
-
|
|
1719
|
-
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
|
|
1725
|
-
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
bytesOnDisk += (size * physicalCopies) + pendingBytes
|
|
1729
|
-
wouldBe += (size * Math.max(physicalCopies, names.length)) + pendingBytes
|
|
1730
|
-
const orphan = !!(storeStat && storeStat.nlink === 1)
|
|
1731
|
-
const publicBlob = {
|
|
1732
|
-
hash, size: blob.size || 0, orphan, nlink,
|
|
1733
|
-
names, apps: [...apps], source_urls: blob.source_urls || []
|
|
1734
|
-
}
|
|
1735
|
-
blobs.push(publicBlob)
|
|
1736
|
-
blobByHash.set(hash, publicBlob)
|
|
1737
|
-
}
|
|
1738
|
-
if (!scopeId) {
|
|
1739
|
-
// The scan total covers every regular file, including files below the
|
|
1740
|
-
// candidate threshold and files the user kept separate. The registry
|
|
1741
|
-
// figures above cover only content tracked by Save space. Add the
|
|
1742
|
-
// remainder to both sides so the global comparison represents the
|
|
1743
|
-
// complete scanned locations instead of silently omitting small files.
|
|
1744
|
-
const lastScan = registry.scanFor()
|
|
1745
|
-
const scannedBytes = lastScan && Number.isFinite(lastScan.bytes_total)
|
|
1746
|
-
? lastScan.bytes_total
|
|
1747
|
-
: null
|
|
1748
|
-
const untrackedScannedBytes = scannedBytes === null
|
|
1749
|
-
? 0
|
|
1750
|
-
: Math.max(0, scannedBytes - trackedLogicalBytes)
|
|
1751
|
-
bytesOnDisk += untrackedScannedBytes
|
|
1752
|
-
wouldBe += untrackedScannedBytes
|
|
1753
|
-
}
|
|
1754
|
-
const duplicates = []
|
|
1755
|
-
for (const [p, entry] of registry.duplicates) {
|
|
1756
|
-
if (scopeId && entry.source_id !== scopeId) continue
|
|
1757
|
-
const location = this.locationForPath(p, entry.source_id)
|
|
1758
|
-
const source = this.sourceForPath(p, location.source_id)
|
|
1759
|
-
const index = registry.scanIndex.get(p)
|
|
1760
|
-
const storeStat = storeStats.get(entry.hash)
|
|
1761
|
-
const shareable = !!(source && source.shareable && storeStat && !entry.unavailable_reason &&
|
|
1762
|
-
(!index || index.dev === undefined || index.dev === storeStat.dev))
|
|
1763
|
-
let unavailableReason = entry.unavailable_reason || null
|
|
1764
|
-
if (!shareable && !unavailableReason) {
|
|
1765
|
-
unavailableReason = source && storeStat && index && index.dev !== undefined && index.dev !== storeStat.dev
|
|
1766
|
-
? "different_disk"
|
|
1767
|
-
: "unsupported_disk"
|
|
1768
|
-
}
|
|
1769
|
-
const blob = blobByHash.get(entry.hash)
|
|
1770
|
-
const match = blob ? blob.names.find((name) => name.path !== p) || null : null
|
|
1771
|
-
duplicates.push(Object.assign({
|
|
1772
|
-
path: p, hash: entry.hash, size: entry.size || 0, app: entry.app || null,
|
|
1773
|
-
shareable, unavailable_reason: unavailableReason, match
|
|
1774
|
-
}, location))
|
|
1775
|
-
}
|
|
1776
|
-
const events = (await registry.readEvents()).reverse()
|
|
1777
|
-
.filter((event) => !scopeId || event.source_id === scopeId)
|
|
1778
|
-
.map((event) => {
|
|
1779
|
-
const currentLocation = event.path ? this.locationForPath(event.path, event.source_id) : null
|
|
1780
|
-
const fallbackLocation = currentLocation && currentLocation.source_id
|
|
1781
|
-
? currentLocation
|
|
1782
|
-
: (event.path ? { relative_path: event.path } : {})
|
|
1783
|
-
const publicEvent = Object.assign({}, fallbackLocation, event)
|
|
1784
|
-
if (event.kind === "convert" && event.batch_id && event.path) {
|
|
1785
|
-
const current = registry.links.get(event.path)
|
|
1786
|
-
publicEvent.undoable = !!(current && (
|
|
1787
|
-
current.batch_id === event.batch_id ||
|
|
1788
|
-
(!current.batch_id && current.hash === event.hash)
|
|
1789
|
-
))
|
|
1790
|
-
const batch = undoBatchMap.get(event.batch_id)
|
|
1791
|
-
if (batch && Number.isFinite(event.ts)) {
|
|
1792
|
-
batch.ts = Math.max(batch.ts || 0, event.ts)
|
|
1793
|
-
}
|
|
1794
|
-
}
|
|
1795
|
-
return publicEvent
|
|
2576
|
+
const snapshot = await this.registry.statusSnapshot({
|
|
2577
|
+
scopeSourceIds,
|
|
2578
|
+
locationSourceIds,
|
|
2579
|
+
scoped: !!scopeId,
|
|
2580
|
+
scopeUnrestricted: !scopeId,
|
|
2581
|
+
locationUnrestricted: !scopeId && !locationId,
|
|
2582
|
+
view,
|
|
2583
|
+
statusFilter,
|
|
2584
|
+
query,
|
|
2585
|
+
pageSize,
|
|
2586
|
+
sizeSort: groupDuplicates
|
|
2587
|
+
? options.size_sort || "desc"
|
|
2588
|
+
: options.size_sort,
|
|
2589
|
+
groupDuplicates,
|
|
2590
|
+
cursor
|
|
1796
2591
|
})
|
|
1797
|
-
|
|
1798
|
-
|
|
1799
|
-
|
|
1800
|
-
|
|
1801
|
-
|
|
1802
|
-
|
|
1803
|
-
|
|
1804
|
-
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
|
|
1808
|
-
}
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
|
|
1814
|
-
|
|
1815
|
-
|
|
2592
|
+
let items
|
|
2593
|
+
if (view === "activity") {
|
|
2594
|
+
items = (snapshot.page.rows || []).map((event) => Object.assign(
|
|
2595
|
+
{},
|
|
2596
|
+
event.path
|
|
2597
|
+
? this.locationForPath(event.path, event.source_id)
|
|
2598
|
+
: {},
|
|
2599
|
+
event
|
|
2600
|
+
))
|
|
2601
|
+
} else if (view === "reclaimable") {
|
|
2602
|
+
items = snapshot.page.rows || []
|
|
2603
|
+
} else if (groupDuplicates) {
|
|
2604
|
+
items = this.publicDuplicateGroupItems(snapshot.page)
|
|
2605
|
+
} else {
|
|
2606
|
+
items = this.publicFileItems(snapshot.page)
|
|
2607
|
+
}
|
|
2608
|
+
|
|
2609
|
+
const lastScan = await this.scanForScope(scopeId)
|
|
2610
|
+
const before = lastScan && Number.isFinite(lastScan.bytes_total)
|
|
2611
|
+
? lastScan.bytes_total
|
|
2612
|
+
: 0
|
|
2613
|
+
const saved = Math.max(0, Number(snapshot.saved) || 0)
|
|
2614
|
+
const currentCount = Number(snapshot.total) || 0
|
|
2615
|
+
const pageTotal = Number(snapshot.pageTotal) || 0
|
|
2616
|
+
const pages = Math.max(1, Math.ceil(pageTotal / pageSize))
|
|
2617
|
+
const publicSources = this._sources
|
|
2618
|
+
.filter((source) => !scopeId || source.id === scopeId)
|
|
2619
|
+
.map((source) => ({
|
|
2620
|
+
id: source.id,
|
|
2621
|
+
kind: source.kind,
|
|
2622
|
+
label: source.label,
|
|
2623
|
+
root: source.root,
|
|
2624
|
+
display_path: source.root,
|
|
2625
|
+
target_path: source.kind === "external" ? source.root : null,
|
|
2626
|
+
parent_id: scopeId ? null : source.parent_id,
|
|
2627
|
+
app: source.app || null,
|
|
2628
|
+
available: source.available !== false,
|
|
2629
|
+
shareable: source.kind === "virtual" ? null : !!source.shareable,
|
|
2630
|
+
removable: source.kind === "external" &&
|
|
2631
|
+
source.configured === true
|
|
2632
|
+
}))
|
|
2633
|
+
const sourceCounts = this.sourceCountMaps(snapshot.scopeRows)
|
|
1816
2634
|
const result = {
|
|
1817
2635
|
enabled: true,
|
|
1818
2636
|
mode: this.mode,
|
|
1819
|
-
scan,
|
|
1820
|
-
last_scan:
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
saved_by_sharing:
|
|
1824
|
-
|
|
1825
|
-
reclaimable:
|
|
1826
|
-
pending_bytes:
|
|
2637
|
+
scan: this.scanStatus(),
|
|
2638
|
+
last_scan: lastScan,
|
|
2639
|
+
bytes_without_sharing: before,
|
|
2640
|
+
bytes_on_disk: Math.max(0, before - saved),
|
|
2641
|
+
saved_by_sharing: saved,
|
|
2642
|
+
effective_bytes: Math.max(0, before - saved),
|
|
2643
|
+
reclaimable: Number(snapshot.reclaimable) || 0,
|
|
2644
|
+
pending_bytes: Number(snapshot.pending) || 0,
|
|
1827
2645
|
file_action: this.fileActionStatus(scopeId),
|
|
1828
|
-
|
|
1829
|
-
|
|
1830
|
-
|
|
1831
|
-
|
|
1832
|
-
|
|
1833
|
-
|
|
1834
|
-
|
|
1835
|
-
|
|
1836
|
-
|
|
1837
|
-
|
|
1838
|
-
|
|
1839
|
-
|
|
1840
|
-
|
|
1841
|
-
|
|
1842
|
-
|
|
1843
|
-
|
|
1844
|
-
|
|
1845
|
-
|
|
1846
|
-
|
|
1847
|
-
|
|
1848
|
-
|
|
1849
|
-
|
|
1850
|
-
|
|
1851
|
-
|
|
1852
|
-
|
|
1853
|
-
|
|
1854
|
-
|
|
1855
|
-
|
|
1856
|
-
|
|
1857
|
-
|
|
1858
|
-
|
|
1859
|
-
|
|
1860
|
-
|
|
1861
|
-
|
|
1862
|
-
:
|
|
1863
|
-
|
|
1864
|
-
|
|
1865
|
-
}
|
|
2646
|
+
sources: publicSources,
|
|
2647
|
+
items,
|
|
2648
|
+
inventory: {
|
|
2649
|
+
view,
|
|
2650
|
+
counts: snapshot.counts,
|
|
2651
|
+
source_counts: sourceCounts,
|
|
2652
|
+
shareable_by_source: sourceCounts.shareable,
|
|
2653
|
+
shareable_duplicates:
|
|
2654
|
+
Number(snapshot.shareableDuplicates) || 0,
|
|
2655
|
+
duplicate_locations:
|
|
2656
|
+
Number(snapshot.duplicateLocations) || 0,
|
|
2657
|
+
current: {
|
|
2658
|
+
count: currentCount,
|
|
2659
|
+
locations: Number(snapshot.currentLocations) || 0,
|
|
2660
|
+
shareable_bytes:
|
|
2661
|
+
Number(snapshot.currentShareableBytes) || 0,
|
|
2662
|
+
deduplicate_bytes:
|
|
2663
|
+
Number(snapshot.currentDeduplicateBytes) || 0,
|
|
2664
|
+
separate_count:
|
|
2665
|
+
Number(snapshot.currentSeparateCount) || 0,
|
|
2666
|
+
separate_bytes:
|
|
2667
|
+
Number(snapshot.currentSeparateBytes) || 0
|
|
2668
|
+
},
|
|
2669
|
+
page: requestedPage,
|
|
2670
|
+
page_size: pageSize,
|
|
2671
|
+
start: requestedPage * pageSize,
|
|
2672
|
+
end: Math.min(
|
|
2673
|
+
(requestedPage * pageSize) + items.length,
|
|
2674
|
+
pageTotal
|
|
2675
|
+
),
|
|
2676
|
+
total: pageTotal,
|
|
2677
|
+
pages,
|
|
2678
|
+
cursor: cursor || null,
|
|
2679
|
+
next_cursor: snapshot.page.nextCursor || null,
|
|
2680
|
+
has_previous: requestedPage > 0,
|
|
2681
|
+
has_next: !!snapshot.page.nextCursor
|
|
2682
|
+
}
|
|
2683
|
+
}
|
|
2684
|
+
if (scopeId) result.scope_id = scopeId
|
|
1866
2685
|
return result
|
|
1867
2686
|
}
|
|
2687
|
+
|
|
1868
2688
|
scanStatus() {
|
|
1869
|
-
|
|
1870
|
-
|
|
1871
|
-
const pending = !!this.scanPromise && !sweeper.state.active
|
|
2689
|
+
if (!this.sweeper) return null
|
|
2690
|
+
const pending = !!this.scanPromise && !this.sweeper.state.active
|
|
1872
2691
|
const state = pending
|
|
1873
|
-
? Object.assign(sweeper.idleState(), {
|
|
1874
|
-
|
|
2692
|
+
? Object.assign(this.sweeper.idleState(), {
|
|
2693
|
+
phase: "queued",
|
|
2694
|
+
scope_id: this.scanScopeId
|
|
2695
|
+
})
|
|
2696
|
+
: this.sweeper.state
|
|
1875
2697
|
return Object.assign({}, state, {
|
|
1876
|
-
current_file:
|
|
1877
|
-
|
|
1878
|
-
|
|
2698
|
+
current_file: this.sweeper.currentHash
|
|
2699
|
+
? path.basename(this.sweeper.currentHash.path)
|
|
2700
|
+
: null,
|
|
2701
|
+
current_file_bytes: this.sweeper.currentHash
|
|
2702
|
+
? this.sweeper.currentHash.bytes
|
|
2703
|
+
: null,
|
|
2704
|
+
current_file_size: this.sweeper.currentHash
|
|
2705
|
+
? this.sweeper.currentHash.size
|
|
2706
|
+
: null,
|
|
1879
2707
|
pending,
|
|
1880
2708
|
error: this.scanError
|
|
1881
2709
|
})
|
|
1882
2710
|
}
|
|
2711
|
+
|
|
1883
2712
|
fileActionStatus(scopeId = null) {
|
|
1884
2713
|
const progress = this.fileActionProgress
|
|
1885
2714
|
if (!progress) return null
|
|
1886
|
-
if (scopeId
|
|
1887
|
-
|
|
1888
|
-
if (progress.path) {
|
|
1889
|
-
const source = this.sourceForPath(progress.path)
|
|
1890
|
-
if (!source || source.id !== scopeId) return null
|
|
1891
|
-
}
|
|
1892
|
-
}
|
|
2715
|
+
if (scopeId && progress.scope_id &&
|
|
2716
|
+
progress.scope_id !== scopeId) return null
|
|
1893
2717
|
return Object.assign({}, progress)
|
|
1894
2718
|
}
|
|
1895
|
-
|
|
2719
|
+
|
|
2720
|
+
async progressStatus(scopeId = null) {
|
|
1896
2721
|
return {
|
|
1897
2722
|
enabled: !!this.enabled,
|
|
1898
2723
|
scan: this.scanStatus(),
|
|
1899
2724
|
file_action: this.fileActionStatus(scopeId),
|
|
1900
|
-
last_scan: this.
|
|
1901
|
-
}
|
|
1902
|
-
}
|
|
1903
|
-
async copyOut(filePath, sourcePath, expectedTarget, expectedSource = expectedTarget, options = {}) {
|
|
1904
|
-
const tmp = filePath + TMP_SUFFIX
|
|
1905
|
-
let copiedStat = null
|
|
1906
|
-
try {
|
|
1907
|
-
await fs.promises.copyFile(sourcePath, tmp, fs.constants.COPYFILE_EXCL)
|
|
1908
|
-
copiedStat = await fs.promises.lstat(tmp)
|
|
1909
|
-
if (options.canReplace && !options.canReplace()) {
|
|
1910
|
-
await unlinkIfSame(tmp, copiedStat)
|
|
1911
|
-
return { status: "locked" }
|
|
1912
|
-
}
|
|
1913
|
-
const currentTarget = await lstatIfPresent(filePath)
|
|
1914
|
-
const currentSource = sourcePath === filePath ? currentTarget : await lstatIfPresent(sourcePath)
|
|
1915
|
-
const currentTmp = await lstatIfPresent(tmp)
|
|
1916
|
-
if (!currentTarget || !currentSource || !sameSnapshot(expectedTarget, currentTarget) ||
|
|
1917
|
-
!sameSnapshot(expectedSource, currentSource) || !sameIdentity(currentTmp, copiedStat)) {
|
|
1918
|
-
await unlinkIfSame(tmp, copiedStat)
|
|
1919
|
-
return { status: "stale" }
|
|
1920
|
-
}
|
|
1921
|
-
await fs.promises.rename(tmp, filePath)
|
|
1922
|
-
let finalStat
|
|
1923
|
-
try {
|
|
1924
|
-
finalStat = await lstatIfPresent(filePath)
|
|
1925
|
-
} catch (error) {
|
|
1926
|
-
// The independent copy was committed by rename(). Preserve that
|
|
1927
|
-
// completed action when only the post-commit metadata read failed.
|
|
1928
|
-
finalStat = copiedStat
|
|
1929
|
-
}
|
|
1930
|
-
if (!finalStat || finalStat.dev !== copiedStat.dev || finalStat.ino !== copiedStat.ino) {
|
|
1931
|
-
return { status: "stale" }
|
|
1932
|
-
}
|
|
1933
|
-
copiedStat = null
|
|
1934
|
-
return { status: "copied", stat: finalStat }
|
|
1935
|
-
} catch (error) {
|
|
1936
|
-
await unlinkIfSame(tmp, copiedStat).catch(() => {})
|
|
1937
|
-
if (error && error.code === "EEXIST") return { status: "conflict" }
|
|
1938
|
-
if (isMissingError(error)) return { status: "stale" }
|
|
1939
|
-
throw error
|
|
1940
|
-
}
|
|
1941
|
-
}
|
|
1942
|
-
// detach: make one name an independent copy again ("go back"), and pin it
|
|
1943
|
-
// so future scans neither re-list nor re-share it. Works on linked names
|
|
1944
|
-
// (copies bytes out) and on pending duplicates (just ignores them).
|
|
1945
|
-
async detach(filePath) {
|
|
1946
|
-
if (!this.enabled) return { status: "disabled" }
|
|
1947
|
-
await this.refreshSources()
|
|
1948
|
-
const entry = this.registry.links.get(filePath)
|
|
1949
|
-
if (entry) {
|
|
1950
|
-
const source = this.sourceForPath(filePath, entry.source_id)
|
|
1951
|
-
if (!source || !await this.canonicalPathIsWithinSource(filePath, source)) {
|
|
1952
|
-
return { status: "stale" }
|
|
1953
|
-
}
|
|
1954
|
-
if (entry.mode === "copy") {
|
|
1955
|
-
const st = await lstatIfPresent(filePath)
|
|
1956
|
-
if (!st || !st.isFile() || st.dev !== entry.dev || st.ino !== entry.ino) return { status: "stale" }
|
|
1957
|
-
this.registry.exclude(filePath, {
|
|
1958
|
-
ts: Date.now(), source_id: entry.source_id || null, size: st.size
|
|
1959
|
-
})
|
|
1960
|
-
await this.recordEvent({
|
|
1961
|
-
kind: "skip", hash: entry.hash, path: filePath,
|
|
1962
|
-
app: entry.app || null, source_id: entry.source_id || null, size: st.size
|
|
1963
|
-
})
|
|
1964
|
-
return { status: "ignored" }
|
|
1965
|
-
}
|
|
1966
|
-
if (this.sourceAppIsRunning(source)) return { status: "locked" }
|
|
1967
|
-
const before = await lstatIfPresent(filePath)
|
|
1968
|
-
if (!before || !before.isFile() || before.dev !== entry.dev || before.ino !== entry.ino) return { status: "stale" }
|
|
1969
|
-
const copied = await this.copyOut(filePath, filePath, fileSnapshot(before), fileSnapshot(before), {
|
|
1970
|
-
canReplace: () => !this.sourceAppIsRunning(source)
|
|
1971
|
-
})
|
|
1972
|
-
if (copied.status !== "copied") return copied
|
|
1973
|
-
const st = copied.stat
|
|
1974
|
-
this.registry.exclude(filePath, { ts: Date.now(), source_id: entry.source_id || null, size: st.size })
|
|
1975
|
-
await this.refreshLinkSnapshots(entry.hash, entry.dev, entry.ino)
|
|
1976
|
-
await this.recordEvent({ kind: "detach", hash: entry.hash, path: filePath, app: entry.app || null, source_id: entry.source_id || null, size: st.size })
|
|
1977
|
-
return { status: "detached" }
|
|
1978
|
-
}
|
|
1979
|
-
if (this.registry.duplicates.has(filePath)) {
|
|
1980
|
-
const dup = this.registry.duplicates.get(filePath)
|
|
1981
|
-
this.registry.exclude(filePath, { ts: Date.now(), source_id: dup.source_id || null, size: dup.size || 0 })
|
|
1982
|
-
await this.recordEvent({ kind: "skip", hash: dup.hash, path: filePath, app: dup.app || null, source_id: dup.source_id || null, size: dup.size || 0 })
|
|
1983
|
-
return { status: "ignored" }
|
|
1984
|
-
}
|
|
1985
|
-
return { status: "not-found" }
|
|
1986
|
-
}
|
|
1987
|
-
// Undo a conversion batch: replace each converted name with an independent
|
|
1988
|
-
// copy of the bytes (spec, UX contract surface 3).
|
|
1989
|
-
async undoBatch(batchId) {
|
|
1990
|
-
if (!this.enabled || !batchId) return { undone: 0 }
|
|
1991
|
-
await this.refreshSources()
|
|
1992
|
-
const events = await this.registry.readEvents()
|
|
1993
|
-
const conversionEvents = new Map(events
|
|
1994
|
-
.filter((event) => event.kind === "convert" && event.batch_id === batchId && event.path)
|
|
1995
|
-
.map((event) => [event.path, event]))
|
|
1996
|
-
const targets = new Set(conversionEvents.keys())
|
|
1997
|
-
for (const [filePath, entry] of this.registry.links) {
|
|
1998
|
-
if (entry.batch_id === batchId) targets.add(filePath)
|
|
1999
|
-
}
|
|
2000
|
-
const summary = { undone: 0, bytes: 0, failed: 0 }
|
|
2001
|
-
for (const filePath of targets) {
|
|
2002
|
-
const event = conversionEvents.get(filePath) || {}
|
|
2003
|
-
const entry = this.registry.links.get(filePath)
|
|
2004
|
-
if (!entry || (entry.batch_id && entry.batch_id !== batchId) ||
|
|
2005
|
-
(event.hash && entry.hash !== event.hash)) continue
|
|
2006
|
-
const storePath = this.storePathFor(entry.hash)
|
|
2007
|
-
try {
|
|
2008
|
-
const source = this.sourceForPath(filePath, entry.source_id || event.source_id)
|
|
2009
|
-
if (!source || !await this.canonicalPathIsWithinSource(filePath, source)) {
|
|
2010
|
-
summary.failed += 1
|
|
2011
|
-
continue
|
|
2012
|
-
}
|
|
2013
|
-
if (this.sourceAppIsRunning(source)) {
|
|
2014
|
-
summary.failed += 1
|
|
2015
|
-
continue
|
|
2016
|
-
}
|
|
2017
|
-
const st = await fs.promises.lstat(filePath)
|
|
2018
|
-
const storeStat = await this.storeStatIfPresent(storePath)
|
|
2019
|
-
if (!st.isFile() || !storeStat || !storeStat.isFile()) {
|
|
2020
|
-
summary.failed += 1
|
|
2021
|
-
continue
|
|
2022
|
-
}
|
|
2023
|
-
if (st.ino !== storeStat.ino || st.dev !== storeStat.dev ||
|
|
2024
|
-
st.dev !== entry.dev || st.ino !== entry.ino) {
|
|
2025
|
-
summary.failed += 1
|
|
2026
|
-
continue
|
|
2027
|
-
}
|
|
2028
|
-
const copied = await this.copyOut(filePath, storePath, fileSnapshot(st), fileSnapshot(storeStat), {
|
|
2029
|
-
canReplace: () => !this.sourceAppIsRunning(source)
|
|
2030
|
-
})
|
|
2031
|
-
if (copied.status !== "copied") {
|
|
2032
|
-
summary.failed += 1
|
|
2033
|
-
continue
|
|
2034
|
-
}
|
|
2035
|
-
const independentStat = copied.stat
|
|
2036
|
-
const duplicateEntry = {
|
|
2037
|
-
hash: entry.hash, size: storeStat.size, app: entry.app || null,
|
|
2038
|
-
source_id: entry.source_id || event.source_id || null, discovered: Date.now(),
|
|
2039
|
-
dev: independentStat.dev, ino: independentStat.ino,
|
|
2040
|
-
mtime: independentStat.mtimeMs, ctime: independentStat.ctimeMs
|
|
2041
|
-
}
|
|
2042
|
-
const scanEntry = {
|
|
2043
|
-
hash: entry.hash, size: independentStat.size,
|
|
2044
|
-
dev: independentStat.dev, ino: independentStat.ino,
|
|
2045
|
-
mtime: independentStat.mtimeMs, ctime: independentStat.ctimeMs,
|
|
2046
|
-
source_id: entry.source_id || event.source_id || null
|
|
2047
|
-
}
|
|
2048
|
-
this.registry.setDuplicate(filePath, duplicateEntry, scanEntry)
|
|
2049
|
-
await this.refreshLinkSnapshots(entry.hash, entry.dev, entry.ino)
|
|
2050
|
-
const bytesSaved = event.bytes_saved || storeStat.size
|
|
2051
|
-
this.registry.totals.lifetime_bytes_saved = Math.max(0,
|
|
2052
|
-
this.registry.totals.lifetime_bytes_saved - bytesSaved)
|
|
2053
|
-
await this.recordEvent({
|
|
2054
|
-
kind: "undo", hash: entry.hash, path: filePath,
|
|
2055
|
-
source_id: entry.source_id || event.source_id || null, batch_id: batchId,
|
|
2056
|
-
bytes_saved: bytesSaved
|
|
2057
|
-
})
|
|
2058
|
-
summary.undone += 1
|
|
2059
|
-
summary.bytes += storeStat.size
|
|
2060
|
-
} catch (e) {
|
|
2061
|
-
summary.failed += 1
|
|
2062
|
-
}
|
|
2063
|
-
}
|
|
2064
|
-
this.registry.schedulePersist()
|
|
2065
|
-
return summary
|
|
2066
|
-
}
|
|
2067
|
-
async reclaimAll() {
|
|
2068
|
-
if (!this.enabled) return { reclaimed: 0, bytes_freed: 0 }
|
|
2069
|
-
const summary = { reclaimed: 0, bytes_freed: 0, failed: 0 }
|
|
2070
|
-
const removedHashes = new Set()
|
|
2071
|
-
const copyHashes = new Set([...this.registry.links.values()]
|
|
2072
|
-
.filter((entry) => entry.mode === "copy")
|
|
2073
|
-
.map((entry) => entry.hash))
|
|
2074
|
-
try {
|
|
2075
|
-
for (const [hash] of [...this.registry.blobs]) {
|
|
2076
|
-
const result = await this.reclaim(hash, {
|
|
2077
|
-
deferRegistry: true,
|
|
2078
|
-
hasCopyNames: copyHashes.has(hash)
|
|
2079
|
-
})
|
|
2080
|
-
if (result.remove_hash) removedHashes.add(hash)
|
|
2081
|
-
if (result.status === "reclaimed") {
|
|
2082
|
-
summary.reclaimed += 1
|
|
2083
|
-
summary.bytes_freed += result.bytes_freed || 0
|
|
2084
|
-
} else if (!["gone", "in-use", "unavailable"].includes(result.status)) summary.failed += 1
|
|
2085
|
-
}
|
|
2086
|
-
} finally {
|
|
2087
|
-
this.registry.removeBlobs(removedHashes)
|
|
2725
|
+
last_scan: this.lastScanCache.get(scopeId || "") || null
|
|
2088
2726
|
}
|
|
2089
|
-
return summary
|
|
2090
2727
|
}
|
|
2091
2728
|
}
|
|
2092
2729
|
|
|
2093
|
-
Vault.SIZE_THRESHOLD = SIZE_THRESHOLD
|
|
2094
|
-
Vault.TMP_SUFFIX = TMP_SUFFIX
|
|
2095
|
-
|
|
2096
2730
|
module.exports = Vault
|