pinokiod 8.0.41 → 8.0.44
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/kernel/vault/constants.js +1 -5
- package/kernel/vault/hash_worker.js +1 -7
- package/kernel/vault/index.js +515 -873
- package/kernel/vault/registry.js +35 -157
- package/kernel/vault/sweeper.js +106 -181
- package/kernel/vault/walker.js +1 -8
- package/package.json +1 -1
- package/server/index.js +3 -20
- package/server/public/storage-size.js +3 -1
- package/server/public/style.css +0 -13
- package/server/public/vault.css +60 -173
- package/server/public/vault.js +397 -626
- package/server/views/app.ejs +6 -85
- package/server/views/partials/main_sidebar.ejs +1 -4
- package/server/views/partials/vault_workspace.ejs +2 -5
- package/server/views/vault.ejs +1 -1
- package/server/views/vault_app.ejs +1 -1
- package/test/vault-engine.test.js +123 -212
- package/test/vault-sweep.test.js +163 -292
- package/test/vault-ui.test.js +718 -979
- package/server/public/vault-nav.js +0 -96
package/kernel/vault/sweeper.js
CHANGED
|
@@ -50,7 +50,6 @@ class Sweeper {
|
|
|
50
50
|
this.hashJobsByIno = new Map()
|
|
51
51
|
this.completedHashesByIno = new Map()
|
|
52
52
|
this.inaccessiblePaths = new Set()
|
|
53
|
-
this.inaccessibleCodes = new Map()
|
|
54
53
|
}
|
|
55
54
|
idleState() {
|
|
56
55
|
return {
|
|
@@ -77,27 +76,12 @@ class Sweeper {
|
|
|
77
76
|
this.hashJobsByIno.clear()
|
|
78
77
|
this.completedHashesByIno.clear()
|
|
79
78
|
this.inaccessiblePaths.clear()
|
|
80
|
-
this.inaccessibleCodes.clear()
|
|
81
|
-
this.fatalError = null
|
|
82
|
-
const registryWasDirty = !!(
|
|
83
|
-
this.vault.registry.persistDirty || this.vault.registry.persistTimer
|
|
84
|
-
)
|
|
85
|
-
const registrySnapshot = this.vault.registry.snapshotState()
|
|
86
79
|
this.vault.registry.beginBatch()
|
|
87
|
-
this.vault.beginScanDraft()
|
|
88
80
|
let completed = false
|
|
89
81
|
let incomplete = false
|
|
90
|
-
let rollbackMetadataChanged = false
|
|
91
82
|
try {
|
|
92
83
|
await this.vault.refreshSources()
|
|
93
84
|
if (!scopeId) this.vault.reconcileConfiguredSources()
|
|
94
|
-
if (!scopeId) {
|
|
95
|
-
for (const source of this.vault.sources()) {
|
|
96
|
-
if (source.kind === "external" && source.available === false && source.root) {
|
|
97
|
-
this.recordUnavailable(source.root, source.error_code)
|
|
98
|
-
}
|
|
99
|
-
}
|
|
100
|
-
}
|
|
101
85
|
const scanRoots = this.vault.scanRoots(scopeId)
|
|
102
86
|
if (!scanRoots.length) throw new Error("That scan location is no longer available.")
|
|
103
87
|
const previous = this.vault.registry.scanFor(scopeId)
|
|
@@ -136,9 +120,6 @@ class Sweeper {
|
|
|
136
120
|
await this.settle()
|
|
137
121
|
this.state.hash_wait_duration_ms = Date.now() - hashWaitStarted
|
|
138
122
|
const scanMeta = {
|
|
139
|
-
ts: Date.now(),
|
|
140
|
-
complete: true,
|
|
141
|
-
candidate_min_bytes: this.vault.sizeThreshold,
|
|
142
123
|
dirs: this.state.dirs,
|
|
143
124
|
files: this.state.files,
|
|
144
125
|
bytes_total: this.state.bytes_total,
|
|
@@ -156,8 +137,7 @@ class Sweeper {
|
|
|
156
137
|
count_duration_ms: this.state.count_duration_ms,
|
|
157
138
|
walk_duration_ms: this.state.walk_duration_ms,
|
|
158
139
|
hash_wait_duration_ms: this.state.hash_wait_duration_ms,
|
|
159
|
-
hash_duration_ms: this.state.hash_duration_ms
|
|
160
|
-
unreadable: []
|
|
140
|
+
hash_duration_ms: this.state.hash_duration_ms
|
|
161
141
|
}
|
|
162
142
|
// Refresh dead-name and orphan state from filesystem identity after
|
|
163
143
|
// every completed discovery pass, as required by the Vault contract.
|
|
@@ -173,60 +153,35 @@ class Sweeper {
|
|
|
173
153
|
})
|
|
174
154
|
this.state.duration_ms = Date.now() - this.state.started
|
|
175
155
|
incomplete = this.state.inaccessible > 0
|
|
176
|
-
scanMeta.ts = Date.now()
|
|
177
|
-
scanMeta.complete = !incomplete
|
|
178
|
-
scanMeta.duration_ms = this.state.duration_ms
|
|
179
|
-
scanMeta.unreadable = [...this.inaccessiblePaths].map((filePath) => ({
|
|
180
|
-
path: filePath,
|
|
181
|
-
code: this.inaccessibleCodes.get(filePath) || "EACCES"
|
|
182
|
-
}))
|
|
183
|
-
this.refreshExcludedVerification(scanRoots)
|
|
184
|
-
if (incomplete) this.markUnavailableRows()
|
|
185
|
-
this.vault.registry.setScanAttempt(scanMeta, scopeId)
|
|
186
156
|
if (!incomplete) {
|
|
157
|
+
scanMeta.ts = Date.now()
|
|
158
|
+
scanMeta.duration_ms = this.state.duration_ms
|
|
187
159
|
this.vault.registry.setLastScan(scanMeta, scopeId)
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
source_files: { [sourceId]: scanMeta.source_files[sourceId] || 0 },
|
|
210
|
-
source_hash_failures: {
|
|
211
|
-
[sourceId]: scanMeta.source_hash_failures[sourceId] || 0
|
|
212
|
-
},
|
|
213
|
-
unreadable: sourceIncomplete
|
|
214
|
-
? scanMeta.unreadable.filter((entry) =>
|
|
215
|
-
isPathWithin(source.root, entry.path) || isPathWithin(entry.path, source.root))
|
|
216
|
-
: []
|
|
160
|
+
if (!scopeId) {
|
|
161
|
+
for (const source of this.vault.sources()) {
|
|
162
|
+
if (source.kind !== "app" || !source.available) continue
|
|
163
|
+
const sourceId = source.id
|
|
164
|
+
const bytes = scanMeta.source_bytes[sourceId] || 0
|
|
165
|
+
this.vault.registry.setLastScan({
|
|
166
|
+
ts: scanMeta.ts,
|
|
167
|
+
duration_ms: scanMeta.duration_ms,
|
|
168
|
+
count_duration_ms: scanMeta.count_duration_ms,
|
|
169
|
+
walk_duration_ms: scanMeta.walk_duration_ms,
|
|
170
|
+
hash_wait_duration_ms: scanMeta.hash_wait_duration_ms,
|
|
171
|
+
files: scanMeta.source_files[sourceId] || 0,
|
|
172
|
+
bytes_total: bytes,
|
|
173
|
+
home_bytes_total: 0,
|
|
174
|
+
hash_failures: scanMeta.source_hash_failures[sourceId] || 0,
|
|
175
|
+
source_bytes: { [sourceId]: bytes },
|
|
176
|
+
source_files: { [sourceId]: scanMeta.source_files[sourceId] || 0 },
|
|
177
|
+
source_hash_failures: {
|
|
178
|
+
[sourceId]: scanMeta.source_hash_failures[sourceId] || 0
|
|
179
|
+
}
|
|
180
|
+
}, sourceId)
|
|
217
181
|
}
|
|
218
|
-
this.vault.registry.setScanAttempt(scopedMeta, sourceId)
|
|
219
|
-
if (!sourceIncomplete) this.vault.registry.setLastScan(scopedMeta, sourceId)
|
|
220
182
|
}
|
|
221
183
|
}
|
|
222
|
-
await this.vault.commitScanDraft()
|
|
223
184
|
completed = true
|
|
224
|
-
} catch (error) {
|
|
225
|
-
// Drain every queued hash before restoring the snapshot; otherwise a
|
|
226
|
-
// late worker could mutate the restored registry after rollback.
|
|
227
|
-
await this.settle().catch(() => {})
|
|
228
|
-
rollbackMetadataChanged = await this.vault.abortScanDraft(registrySnapshot)
|
|
229
|
-
throw error
|
|
230
185
|
} finally {
|
|
231
186
|
// A failed walk must not leave background hash work or per-scan inode
|
|
232
187
|
// state alive when the next manual scan starts.
|
|
@@ -236,15 +191,7 @@ class Sweeper {
|
|
|
236
191
|
this.completedHashesByIno.clear()
|
|
237
192
|
this.state.active = false
|
|
238
193
|
this.state.phase = completed ? (incomplete ? "incomplete" : "complete") : "failed"
|
|
239
|
-
|
|
240
|
-
await this.vault.registry.endBatch(completed ? { flush: true } : { discard: true })
|
|
241
|
-
} catch (error) {
|
|
242
|
-
if (!completed) throw error
|
|
243
|
-
this.vault.scanPersistenceWarning = true
|
|
244
|
-
}
|
|
245
|
-
if (!completed && (registryWasDirty || rollbackMetadataChanged)) {
|
|
246
|
-
this.vault.registry.schedulePersist()
|
|
247
|
-
}
|
|
194
|
+
await this.vault.registry.endBatch({ flush: true })
|
|
248
195
|
}
|
|
249
196
|
return {
|
|
250
197
|
dirs: this.state.dirs, files: this.state.files,
|
|
@@ -255,19 +202,12 @@ class Sweeper {
|
|
|
255
202
|
}
|
|
256
203
|
recordInaccessible(error, filePath) {
|
|
257
204
|
if (!isAccessError(error)) return false
|
|
258
|
-
return this.recordUnavailable(filePath, error.code)
|
|
259
|
-
}
|
|
260
|
-
recordUnavailable(filePath, code = null) {
|
|
261
205
|
const target = path.resolve(filePath)
|
|
262
206
|
if ([...this.inaccessiblePaths].some((existing) => isPathWithin(existing, target))) return true
|
|
263
207
|
for (const existing of [...this.inaccessiblePaths]) {
|
|
264
|
-
if (isPathWithin(target, existing))
|
|
265
|
-
this.inaccessiblePaths.delete(existing)
|
|
266
|
-
this.inaccessibleCodes.delete(existing)
|
|
267
|
-
}
|
|
208
|
+
if (isPathWithin(target, existing)) this.inaccessiblePaths.delete(existing)
|
|
268
209
|
}
|
|
269
210
|
this.inaccessiblePaths.add(target)
|
|
270
|
-
this.inaccessibleCodes.set(target, code || "EACCES")
|
|
271
211
|
this.state.inaccessible = this.inaccessiblePaths.size
|
|
272
212
|
this.state.inaccessible_paths = [...this.inaccessiblePaths].slice(0, 20)
|
|
273
213
|
return true
|
|
@@ -275,33 +215,12 @@ class Sweeper {
|
|
|
275
215
|
isInaccessible(filePath) {
|
|
276
216
|
return [...this.inaccessiblePaths].some((target) => isPathWithin(target, filePath))
|
|
277
217
|
}
|
|
278
|
-
markUnavailableRows() {
|
|
279
|
-
const mark = (entries) => {
|
|
280
|
-
for (const [filePath, entry] of entries) {
|
|
281
|
-
if (!this.isInaccessible(filePath)) continue
|
|
282
|
-
entry.unverified = true
|
|
283
|
-
}
|
|
284
|
-
}
|
|
285
|
-
mark(this.vault.registry.links)
|
|
286
|
-
mark(this.vault.registry.duplicates)
|
|
287
|
-
mark(this.vault.registry.excluded)
|
|
288
|
-
this.vault.registry.schedulePersist()
|
|
289
|
-
}
|
|
290
|
-
refreshExcludedVerification(scanRoots) {
|
|
291
|
-
for (const [filePath, entry] of this.vault.registry.excluded) {
|
|
292
|
-
if (!scanRoots.some((source) => isPathWithin(source.root, filePath))) continue
|
|
293
|
-
entry.unverified = this.isInaccessible(filePath)
|
|
294
|
-
}
|
|
295
|
-
this.vault.registry.schedulePersist()
|
|
296
|
-
}
|
|
297
218
|
async countFiles(root) {
|
|
298
219
|
const vaultRoot = this.vault.root
|
|
299
220
|
for await (const directoryResults of walkBatches(root, {
|
|
300
221
|
concurrency: this.dirConcurrency,
|
|
301
222
|
skipDirectory: (full) => full === vaultRoot,
|
|
302
223
|
strictErrors: true,
|
|
303
|
-
onRootMissing: (error, filePath) =>
|
|
304
|
-
this.recordUnavailable(filePath, error && error.code),
|
|
305
224
|
onError: (error, filePath) => this.recordInaccessible(error, filePath)
|
|
306
225
|
})) {
|
|
307
226
|
this.state.counted_dirs += directoryResults.filter((group) => group.firstChunk).length
|
|
@@ -311,11 +230,6 @@ class Sweeper {
|
|
|
311
230
|
}
|
|
312
231
|
async settle() {
|
|
313
232
|
await this.hashQueue.drained()
|
|
314
|
-
if (this.fatalError) {
|
|
315
|
-
const error = this.fatalError
|
|
316
|
-
this.fatalError = null
|
|
317
|
-
throw error
|
|
318
|
-
}
|
|
319
233
|
}
|
|
320
234
|
async walk(root, preferredSourceId = null) {
|
|
321
235
|
const vaultRoot = this.vault.root
|
|
@@ -323,8 +237,6 @@ class Sweeper {
|
|
|
323
237
|
concurrency: this.dirConcurrency,
|
|
324
238
|
skipDirectory: (full) => full === vaultRoot,
|
|
325
239
|
strictErrors: true,
|
|
326
|
-
onRootMissing: (error, filePath) =>
|
|
327
|
-
this.recordUnavailable(filePath, error && error.code),
|
|
328
240
|
onError: (error, filePath) => this.recordInaccessible(error, filePath)
|
|
329
241
|
})) {
|
|
330
242
|
this.state.dirs += directoryResults.filter((group) => group.firstChunk).length
|
|
@@ -374,42 +286,33 @@ class Sweeper {
|
|
|
374
286
|
this.state.total_files = this.state.files
|
|
375
287
|
}
|
|
376
288
|
}
|
|
377
|
-
async considerStat(filePath, st, preferredSourceId = null
|
|
289
|
+
async considerStat(filePath, st, preferredSourceId = null) {
|
|
378
290
|
if (filePath.endsWith(TMP_SUFFIX)) return
|
|
379
291
|
const registry = this.vault.registry
|
|
380
292
|
const source = this.vault.sourceForPath(filePath, preferredSourceId)
|
|
381
293
|
const sourceId = source ? source.id : null
|
|
382
294
|
const appName = source && source.kind === "app" ? source.app : null
|
|
383
295
|
const sourceKey = sourceId || preferredSourceId || "pinokio"
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
if (!preferredSourceId) this.state.home_bytes_total += st.size
|
|
389
|
-
}
|
|
296
|
+
this.countFile(sourceKey)
|
|
297
|
+
this.state.bytes_total += st.size
|
|
298
|
+
this.state.source_bytes[sourceKey] = (this.state.source_bytes[sourceKey] || 0) + st.size
|
|
299
|
+
if (!preferredSourceId) this.state.home_bytes_total += st.size
|
|
390
300
|
if (registry.excluded.has(filePath)) {
|
|
391
301
|
registry.untrack(filePath)
|
|
392
302
|
return
|
|
393
303
|
}
|
|
394
304
|
if (st.size < this.vault.sizeThreshold) {
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
registry.untrack(filePath)
|
|
398
|
-
return
|
|
399
|
-
}
|
|
400
|
-
} else {
|
|
401
|
-
this.state.candidates += 1
|
|
305
|
+
await this.untrackBelowThreshold(filePath, st, { app: appName, source_id: sourceId })
|
|
306
|
+
return
|
|
402
307
|
}
|
|
308
|
+
this.state.candidates += 1
|
|
403
309
|
// Known and unchanged: nothing to do.
|
|
404
310
|
const inoHash = registry.byIno.get(registry.inoKey(st.dev, st.ino))
|
|
405
311
|
if (inoHash && st.nlink > 1) {
|
|
406
312
|
const cached = registry.scanIndex.get(filePath)
|
|
407
313
|
const unchanged = cached && cached.hash === inoHash && sameSnapshot(cached, st)
|
|
408
314
|
if (unchanged) {
|
|
409
|
-
registry.addLink(filePath, {
|
|
410
|
-
hash: inoHash, app: appName, source_id: sourceId,
|
|
411
|
-
dev: st.dev, ino: st.ino, unverified: false
|
|
412
|
-
})
|
|
315
|
+
registry.addLink(filePath, { hash: inoHash, app: appName, source_id: sourceId, dev: st.dev, ino: st.ino, mode: "link" })
|
|
413
316
|
return
|
|
414
317
|
}
|
|
415
318
|
}
|
|
@@ -458,28 +361,52 @@ class Sweeper {
|
|
|
458
361
|
invalidateLinkedInode(hash, dev, ino) {
|
|
459
362
|
const registry = this.vault.registry
|
|
460
363
|
for (const [filePath, entry] of [...registry.links]) {
|
|
461
|
-
if (entry.hash === hash && entry.dev === dev && entry.ino === ino) {
|
|
364
|
+
if (entry.hash === hash && entry.mode === "link" && entry.dev === dev && entry.ino === ino) {
|
|
462
365
|
registry.untrack(filePath)
|
|
463
366
|
}
|
|
464
367
|
}
|
|
465
|
-
registry.
|
|
368
|
+
const hasCopies = [...registry.links.values()].some((entry) =>
|
|
369
|
+
entry.hash === hash && entry.mode === "copy")
|
|
370
|
+
if (hasCopies) {
|
|
371
|
+
const blob = registry.blobs.get(hash)
|
|
372
|
+
if (blob) {
|
|
373
|
+
blob.orphan = false
|
|
374
|
+
blob.verified_at = null
|
|
375
|
+
registry.schedulePersist()
|
|
376
|
+
}
|
|
377
|
+
} else {
|
|
378
|
+
registry.removeBlob(hash)
|
|
379
|
+
}
|
|
466
380
|
}
|
|
467
|
-
|
|
381
|
+
async untrackBelowThreshold(filePath, st, meta = {}) {
|
|
468
382
|
const registry = this.vault.registry
|
|
469
383
|
const linked = registry.links.get(filePath)
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
384
|
+
if (linked && linked.mode === "link") {
|
|
385
|
+
let storeStat = null
|
|
386
|
+
try {
|
|
387
|
+
storeStat = await this.vault.storeStatIfPresent(this.vault.storePathFor(linked.hash))
|
|
388
|
+
} catch (error) {
|
|
389
|
+
if (!isMissingError(error)) throw error
|
|
390
|
+
}
|
|
391
|
+
if (storeStat && storeStat.dev === st.dev && storeStat.ino === st.ino) {
|
|
392
|
+
const cached = registry.scanIndex.get(filePath)
|
|
393
|
+
try {
|
|
394
|
+
await fs.promises.unlink(this.vault.storePathFor(linked.hash))
|
|
395
|
+
} catch (error) {
|
|
396
|
+
if (!error || error.code !== "ENOENT") return
|
|
397
|
+
}
|
|
398
|
+
this.invalidateLinkedInode(linked.hash, st.dev, st.ino)
|
|
399
|
+
if (cached && !sameSnapshot(cached, st)) {
|
|
400
|
+
await this.vault.recordEvent({
|
|
401
|
+
kind: "diverged", hash: linked.hash, path: filePath,
|
|
402
|
+
app: meta.app || linked.app || null,
|
|
403
|
+
source_id: meta.source_id || linked.source_id || null,
|
|
404
|
+
size: st.size
|
|
405
|
+
})
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
registry.untrack(filePath)
|
|
483
410
|
}
|
|
484
411
|
async _hashJob(job) {
|
|
485
412
|
const primary = job.names[0]
|
|
@@ -503,8 +430,7 @@ class Sweeper {
|
|
|
503
430
|
try {
|
|
504
431
|
st = name === primary ? primaryStat : await fs.promises.lstat(name.filePath)
|
|
505
432
|
} catch (e) {
|
|
506
|
-
|
|
507
|
-
throw e
|
|
433
|
+
continue
|
|
508
434
|
}
|
|
509
435
|
// Same (device, inode) is physical identity, not a content guess.
|
|
510
436
|
// Size and mtime also prevent applying a completed digest after an
|
|
@@ -528,7 +454,6 @@ class Sweeper {
|
|
|
528
454
|
const sourceId = primary.source_id || "pinokio"
|
|
529
455
|
this.state.source_hash_failures[sourceId] =
|
|
530
456
|
(this.state.source_hash_failures[sourceId] || 0) + 1
|
|
531
|
-
if (!this.fatalError) this.fatalError = e
|
|
532
457
|
}
|
|
533
458
|
} finally {
|
|
534
459
|
this.state.hash_duration_ms += Date.now() - started
|
|
@@ -565,29 +490,34 @@ class Sweeper {
|
|
|
565
490
|
const sourceId = source ? source.id : null
|
|
566
491
|
const appName = source && source.kind === "app" ? source.app : null
|
|
567
492
|
// Divergence: this inode was registered under a different hash — the
|
|
568
|
-
// shared content was written in place.
|
|
569
|
-
// when the scan commits so a later fatal error leaves the filesystem and
|
|
570
|
-
// registry at their pre-scan state.
|
|
493
|
+
// shared content was written in place. Evict the stale blob.
|
|
571
494
|
const priorHash = registry.byIno.get(registry.inoKey(st.dev, st.ino))
|
|
572
495
|
if (priorHash && priorHash !== hash) {
|
|
573
496
|
const stalePath = this.vault.storePathFor(priorHash)
|
|
574
|
-
let
|
|
497
|
+
let removedStaleStore = false
|
|
575
498
|
try {
|
|
576
499
|
const staleStat = await this.vault.storeStatIfPresent(stalePath)
|
|
577
500
|
if (staleStat && staleStat.isFile() && staleStat.dev === st.dev && staleStat.ino === st.ino) {
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
filePath,
|
|
581
|
-
...(registry.pathsByIno.get(inodeKey) || [])
|
|
582
|
-
])
|
|
583
|
-
retiredStaleStore = true
|
|
501
|
+
await fs.promises.unlink(stalePath)
|
|
502
|
+
removedStaleStore = true
|
|
584
503
|
}
|
|
585
504
|
} catch (error) {
|
|
586
505
|
if (!isMissingError(error)) throw error
|
|
587
506
|
}
|
|
588
|
-
if (
|
|
507
|
+
if (removedStaleStore) this.invalidateLinkedInode(priorHash, st.dev, st.ino)
|
|
589
508
|
else registry.untrack(filePath)
|
|
590
509
|
this.vault.recordEvent({ kind: "diverged", hash: priorHash, path: filePath, app: appName, source_id: sourceId, size: st.size })
|
|
510
|
+
if (removedStaleStore) {
|
|
511
|
+
// Removing the stale store name changes ctime on this inode. Refresh
|
|
512
|
+
// the trusted snapshot before adoption, but reject any content change.
|
|
513
|
+
const current = await fs.promises.lstat(filePath)
|
|
514
|
+
if (!sameContentState(fileSnapshot(st), current)) {
|
|
515
|
+
registry.untrack(filePath)
|
|
516
|
+
this.state.unstable_hashes += 1
|
|
517
|
+
return
|
|
518
|
+
}
|
|
519
|
+
st = current
|
|
520
|
+
}
|
|
591
521
|
}
|
|
592
522
|
const storePath = this.vault.storePathFor(hash)
|
|
593
523
|
let storeStat = null
|
|
@@ -597,6 +527,14 @@ class Sweeper {
|
|
|
597
527
|
if (!isMissingError(error)) throw error
|
|
598
528
|
}
|
|
599
529
|
const registered = registry.links.get(filePath)
|
|
530
|
+
if (registered && registered.mode === "copy" &&
|
|
531
|
+
registered.hash === hash && registered.dev === st.dev && registered.ino === st.ino) {
|
|
532
|
+
registry.addLink(filePath, {
|
|
533
|
+
hash, app: appName, source_id: sourceId, dev: st.dev, ino: st.ino, mode: "copy"
|
|
534
|
+
})
|
|
535
|
+
this.updateScanIndex(filePath, st, hash, sourceId)
|
|
536
|
+
return
|
|
537
|
+
}
|
|
600
538
|
if (!storeStat && ((source && source.shareable) || !registry.blobs.has(hash) || (registered &&
|
|
601
539
|
registered.hash === hash && registered.dev === st.dev && registered.ino === st.ino))) {
|
|
602
540
|
const adopted = await this.vault.adopt(filePath, hash, {
|
|
@@ -609,7 +547,7 @@ class Sweeper {
|
|
|
609
547
|
this.state.unstable_hashes += 1
|
|
610
548
|
return
|
|
611
549
|
}
|
|
612
|
-
if (adopted.status
|
|
550
|
+
if (adopted.status !== "duplicate") {
|
|
613
551
|
return
|
|
614
552
|
}
|
|
615
553
|
try {
|
|
@@ -619,10 +557,7 @@ class Sweeper {
|
|
|
619
557
|
}
|
|
620
558
|
}
|
|
621
559
|
if (storeStat && storeStat.dev === st.dev && storeStat.ino === st.ino) {
|
|
622
|
-
registry.addLink(filePath, {
|
|
623
|
-
hash, app: appName, source_id: sourceId,
|
|
624
|
-
dev: st.dev, ino: st.ino, unverified: false
|
|
625
|
-
})
|
|
560
|
+
registry.addLink(filePath, { hash, app: appName, source_id: sourceId, dev: st.dev, ino: st.ino, mode: "link" })
|
|
626
561
|
this.updateScanIndex(filePath, st, hash, sourceId)
|
|
627
562
|
return
|
|
628
563
|
}
|
|
@@ -632,8 +567,7 @@ class Sweeper {
|
|
|
632
567
|
}
|
|
633
568
|
const previous = registry.setDuplicate(filePath, {
|
|
634
569
|
hash, size: st.size, app: appName, source_id: sourceId, discovered: Date.now(),
|
|
635
|
-
dev: st.dev, ino: st.ino, mtime: st.mtimeMs, ctime: st.ctimeMs
|
|
636
|
-
unverified: false
|
|
570
|
+
dev: st.dev, ino: st.ino, mtime: st.mtimeMs, ctime: st.ctimeMs
|
|
637
571
|
}, scanEntry)
|
|
638
572
|
if (!previous || previous.hash !== hash) {
|
|
639
573
|
this.vault.recordEvent({ kind: "found", hash, path: filePath, app: appName, source_id: sourceId, size: st.size })
|
|
@@ -667,21 +601,12 @@ class Sweeper {
|
|
|
667
601
|
this.state.source_bytes[sourceKey] = (this.state.source_bytes[sourceKey] || 0) + st.size
|
|
668
602
|
if (!preferredSourceId) this.state.home_bytes_total += st.size
|
|
669
603
|
if (!SHA256_RE.test(name)) continue
|
|
670
|
-
if (this.vault.registry.excluded.has(full)) {
|
|
671
|
-
this.vault.registry.untrack(full)
|
|
672
|
-
continue
|
|
673
|
-
}
|
|
674
604
|
if (st.size < this.vault.sizeThreshold) {
|
|
675
|
-
|
|
676
|
-
if (this.vault.registry.links.has(full)) {
|
|
677
|
-
await this.considerStat(full, st, preferredSourceId, { count: false })
|
|
678
|
-
} else {
|
|
679
|
-
this.vault.registry.untrack(full)
|
|
680
|
-
}
|
|
605
|
+
await this.untrackBelowThreshold(full, st, { app: appName, source_id: sourceId })
|
|
681
606
|
continue
|
|
682
|
-
} else {
|
|
683
|
-
this.state.candidates += 1
|
|
684
607
|
}
|
|
608
|
+
if (this.vault.registry.excluded.has(full)) continue
|
|
609
|
+
this.state.candidates += 1
|
|
685
610
|
await this.classify(full, st, name, preferredSourceId)
|
|
686
611
|
}
|
|
687
612
|
}
|
package/kernel/vault/walker.js
CHANGED
|
@@ -10,13 +10,12 @@ const isHandledError = (options, error, target) => !!(
|
|
|
10
10
|
// One traversal primitive for Scan and Repair. It owns only directory
|
|
11
11
|
// discovery and symlink policy; callers decide how files are classified.
|
|
12
12
|
async function * walkBatches(root, options = {}) {
|
|
13
|
-
const resolvedRoot = path.resolve(root)
|
|
14
13
|
const concurrency = Math.max(1, Number(options.concurrency) || 1)
|
|
15
14
|
const entryBatchSize = Math.max(1, Number(options.entryBatchSize) || ENTRY_BATCH_SIZE)
|
|
16
15
|
const statConcurrency = Math.max(1, Number(options.statConcurrency) || 32)
|
|
17
16
|
const skipDirectory = options.skipDirectory || (() => false)
|
|
18
17
|
const strictErrors = !!options.strictErrors
|
|
19
|
-
const pending = [{ dir:
|
|
18
|
+
const pending = [{ dir: path.resolve(root), followRootSymlink: true }]
|
|
20
19
|
const active = []
|
|
21
20
|
try {
|
|
22
21
|
while (pending.length || active.length) {
|
|
@@ -34,9 +33,6 @@ async function * walkBatches(root, options = {}) {
|
|
|
34
33
|
}
|
|
35
34
|
active.push({ dir, handle: await fs.promises.opendir(dir), firstChunk: true })
|
|
36
35
|
} catch (error) {
|
|
37
|
-
if (isMissingError(error) && dir === resolvedRoot && options.onRootMissing) {
|
|
38
|
-
options.onRootMissing(error, dir)
|
|
39
|
-
}
|
|
40
36
|
if (strictErrors && !isMissingError(error) && !isHandledError(options, error, dir)) throw error
|
|
41
37
|
yield [{ dir, entries: null, files: [], discoveredDirs: 0, discoveredFiles: 0, firstChunk: true }]
|
|
42
38
|
}
|
|
@@ -55,9 +51,6 @@ async function * walkBatches(root, options = {}) {
|
|
|
55
51
|
entries.push(entry)
|
|
56
52
|
}
|
|
57
53
|
} catch (error) {
|
|
58
|
-
if (isMissingError(error) && task.dir === resolvedRoot && options.onRootMissing) {
|
|
59
|
-
options.onRootMissing(error, task.dir)
|
|
60
|
-
}
|
|
61
54
|
if (strictErrors && !isMissingError(error) && !isHandledError(options, error, task.dir)) throw error
|
|
62
55
|
done = true
|
|
63
56
|
}
|
package/package.json
CHANGED
package/server/index.js
CHANGED
|
@@ -15741,25 +15741,6 @@ class Server {
|
|
|
15741
15741
|
|
|
15742
15742
|
// Vault dashboard data: registry-backed with bounded per-blob stats,
|
|
15743
15743
|
// never a discovery walk. Scans run ONLY via the explicit action below.
|
|
15744
|
-
this.app.get("/info/dedup/nav", ex(async (req, res) => {
|
|
15745
|
-
if (!privacyFilterCache.isSameOriginRequest(req)) {
|
|
15746
|
-
res.sendStatus(403)
|
|
15747
|
-
return
|
|
15748
|
-
}
|
|
15749
|
-
const vault = this.kernel.vault
|
|
15750
|
-
if (vault && vault.ready) await vault.ready
|
|
15751
|
-
if (!vault || !vault.enabled) {
|
|
15752
|
-
res.sendStatus(404)
|
|
15753
|
-
return
|
|
15754
|
-
}
|
|
15755
|
-
res.set("Cache-Control", "no-store")
|
|
15756
|
-
const appName = req.query && typeof req.query.app === "string"
|
|
15757
|
-
? req.query.app
|
|
15758
|
-
: null
|
|
15759
|
-
res.json(appName
|
|
15760
|
-
? vault.navigationStatusForApp(appName)
|
|
15761
|
-
: vault.navigationStatus())
|
|
15762
|
-
}))
|
|
15763
15744
|
this.app.get("/info/dedup", ex(async (req, res) => {
|
|
15764
15745
|
if (!privacyFilterCache.isSameOriginRequest(req)) {
|
|
15765
15746
|
res.sendStatus(403)
|
|
@@ -15771,6 +15752,7 @@ class Server {
|
|
|
15771
15752
|
res.sendStatus(404)
|
|
15772
15753
|
return
|
|
15773
15754
|
}
|
|
15755
|
+
await vault.ensureInitialized()
|
|
15774
15756
|
res.set("Cache-Control", "no-store")
|
|
15775
15757
|
const scopeId = req.query && typeof req.query.scope_id === "string"
|
|
15776
15758
|
? req.query.scope_id
|
|
@@ -15796,7 +15778,7 @@ class Server {
|
|
|
15796
15778
|
return
|
|
15797
15779
|
}
|
|
15798
15780
|
await vault.ensureInitialized()
|
|
15799
|
-
res.render("vault", { theme: this.theme, agent: req.agent })
|
|
15781
|
+
res.render("vault", { theme: this.theme, platform: this.kernel.platform, agent: req.agent })
|
|
15800
15782
|
}))
|
|
15801
15783
|
this.app.get("/vault/app/:name", ex(async (req, res) => {
|
|
15802
15784
|
if (!privacyFilterCache.isSameOriginRequest(req)) {
|
|
@@ -15819,6 +15801,7 @@ class Server {
|
|
|
15819
15801
|
}
|
|
15820
15802
|
res.render("vault_app", {
|
|
15821
15803
|
theme: this.theme,
|
|
15804
|
+
platform: this.kernel.platform,
|
|
15822
15805
|
agent: req.agent,
|
|
15823
15806
|
scope_id: source.id
|
|
15824
15807
|
})
|
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
(function (root) {
|
|
2
|
-
const base = 1000
|
|
3
2
|
const units = ["B", "KB", "MB", "GB", "TB"]
|
|
4
3
|
|
|
5
4
|
root.PinokioFormatStorageSize = (value) => {
|
|
6
5
|
const bytes = Number(value)
|
|
7
6
|
if (!Number.isFinite(bytes) || bytes <= 0) return "0 B"
|
|
7
|
+
const platform = root.document && root.document.body &&
|
|
8
|
+
root.document.body.dataset.platform
|
|
9
|
+
const base = platform === "win32" ? 1024 : 1000
|
|
8
10
|
const index = Math.min(units.length - 1, Math.floor(Math.log(bytes) / Math.log(base)))
|
|
9
11
|
const scaled = bytes / Math.pow(base, index)
|
|
10
12
|
return `${Number(scaled.toFixed(index ? 2 : 0))} ${units[index]}`
|
package/server/public/style.css
CHANGED
|
@@ -1567,14 +1567,6 @@ body.main-sidebar-page aside.main-sidebar .tab.selected .main-sidebar-action-ico
|
|
|
1567
1567
|
background: rgba(45, 180, 92, 0.14);
|
|
1568
1568
|
color: #247a43;
|
|
1569
1569
|
}
|
|
1570
|
-
.main-sidebar [data-main-sidebar-vault-tab][data-vault-state="new"]:not(.selected) {
|
|
1571
|
-
background: var(--pinokio-sidebar-tab-hover);
|
|
1572
|
-
}
|
|
1573
|
-
.main-sidebar .main-sidebar-status-badge[data-state="new"] {
|
|
1574
|
-
border: 1px solid rgba(71, 85, 105, 0.22);
|
|
1575
|
-
background: transparent;
|
|
1576
|
-
color: rgba(31, 41, 55, 0.72);
|
|
1577
|
-
}
|
|
1578
1570
|
body.dark .main-sidebar .main-sidebar-status-badge {
|
|
1579
1571
|
background: rgba(255, 255, 255, 0.08);
|
|
1580
1572
|
color: rgba(229, 231, 235, 0.68);
|
|
@@ -1583,11 +1575,6 @@ body.dark .main-sidebar .main-sidebar-status-badge[data-state="on"] {
|
|
|
1583
1575
|
background: rgba(74, 222, 128, 0.14);
|
|
1584
1576
|
color: #86efac;
|
|
1585
1577
|
}
|
|
1586
|
-
body.dark .main-sidebar .main-sidebar-status-badge[data-state="new"] {
|
|
1587
|
-
border-color: rgba(226, 232, 240, 0.22);
|
|
1588
|
-
background: transparent;
|
|
1589
|
-
color: rgba(248, 250, 252, 0.76);
|
|
1590
|
-
}
|
|
1591
1578
|
.main-sidebar .tab.submenu {
|
|
1592
1579
|
padding-left: 14px !important;
|
|
1593
1580
|
}
|