pinokiod 8.0.38 → 8.0.39
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/kernel/vault/hash_worker.js +6 -1
- package/kernel/vault/sweeper.js +145 -29
- package/package.json +1 -1
- package/server/index.js +7 -0
- package/server/views/vault.ejs +75 -2
- package/test/vault-sweep.test.js +51 -1
- package/test/vault-ui.test.js +23 -0
|
@@ -2,10 +2,15 @@ const { parentPort } = require('worker_threads')
|
|
|
2
2
|
const crypto = require('crypto')
|
|
3
3
|
const fs = require('fs')
|
|
4
4
|
|
|
5
|
+
// Large model files otherwise generate a very high number of 64 KiB stream
|
|
6
|
+
// events. This changes only the read granularity; every byte still feeds the
|
|
7
|
+
// same sha256 digest, one file at a time.
|
|
8
|
+
const HASH_READ_SIZE = 1024 * 1024
|
|
9
|
+
|
|
5
10
|
parentPort.on('message', ({ id, filePath }) => {
|
|
6
11
|
const hash = crypto.createHash('sha256')
|
|
7
12
|
let size = 0
|
|
8
|
-
const stream = fs.createReadStream(filePath)
|
|
13
|
+
const stream = fs.createReadStream(filePath, { highWaterMark: HASH_READ_SIZE })
|
|
9
14
|
stream.on('data', (chunk) => {
|
|
10
15
|
size += chunk.length
|
|
11
16
|
hash.update(chunk)
|
package/kernel/vault/sweeper.js
CHANGED
|
@@ -12,8 +12,26 @@ const fastq = require('fastq')
|
|
|
12
12
|
|
|
13
13
|
const SHA256_RE = /^[0-9a-f]{64}$/
|
|
14
14
|
const TMP_SUFFIX = ".pinokio-dedup-tmp"
|
|
15
|
+
const DIR_CONCURRENCY = 8
|
|
15
16
|
const STAT_CONCURRENCY = 32
|
|
16
17
|
|
|
18
|
+
const fileSnapshot = (st) => ({
|
|
19
|
+
dev: st.dev,
|
|
20
|
+
ino: st.ino,
|
|
21
|
+
size: st.size,
|
|
22
|
+
mtime: st.mtimeMs,
|
|
23
|
+
ctime: st.ctimeMs
|
|
24
|
+
})
|
|
25
|
+
|
|
26
|
+
const sameSnapshot = (snapshot, st) => !!(
|
|
27
|
+
snapshot && st &&
|
|
28
|
+
snapshot.dev === st.dev &&
|
|
29
|
+
snapshot.ino === st.ino &&
|
|
30
|
+
snapshot.size === st.size &&
|
|
31
|
+
snapshot.mtime === st.mtimeMs &&
|
|
32
|
+
snapshot.ctime === st.ctimeMs
|
|
33
|
+
)
|
|
34
|
+
|
|
17
35
|
class Sweeper {
|
|
18
36
|
constructor(vault) {
|
|
19
37
|
this.vault = vault
|
|
@@ -23,14 +41,20 @@ class Sweeper {
|
|
|
23
41
|
this.hashQueue = fastq.promise(this, this._hashJob, 1)
|
|
24
42
|
this.stats = { hashes: 0, ingested: 0 }
|
|
25
43
|
this.statConcurrency = vault.statConcurrency || STAT_CONCURRENCY
|
|
44
|
+
this.dirConcurrency = vault.dirConcurrency || DIR_CONCURRENCY
|
|
26
45
|
this.metadataBaseCache = new Map()
|
|
46
|
+
this.hashJobsByIno = new Map()
|
|
27
47
|
}
|
|
28
48
|
idleState() {
|
|
29
49
|
return {
|
|
30
50
|
active: false, dirs: 0, files: 0, bytes_total: 0,
|
|
31
51
|
home_bytes_total: 0, source_bytes: {},
|
|
32
|
-
candidates: 0, hashed: 0, hash_bytes: 0, queued: 0,
|
|
33
|
-
|
|
52
|
+
candidates: 0, hashed: 0, hash_total: 0, hash_bytes: 0, queued: 0,
|
|
53
|
+
inode_reuses: 0, unstable_hashes: 0,
|
|
54
|
+
source_ids: [], estimated_files: null, estimated_walk_weight: null,
|
|
55
|
+
started: null, duration_ms: null,
|
|
56
|
+
walk_duration_ms: null, hash_wait_duration_ms: null,
|
|
57
|
+
hash_duration_ms: 0
|
|
34
58
|
}
|
|
35
59
|
}
|
|
36
60
|
// The one entry point: the Pinokio home plus explicit linked imports.
|
|
@@ -38,12 +62,35 @@ class Sweeper {
|
|
|
38
62
|
if (this.state.active) return { already_running: true }
|
|
39
63
|
this.state = Object.assign(this.idleState(), { active: true, started: Date.now() })
|
|
40
64
|
this.metadataBaseCache.clear()
|
|
65
|
+
this.hashJobsByIno.clear()
|
|
41
66
|
try {
|
|
42
67
|
await this.vault.refreshSources()
|
|
43
|
-
|
|
68
|
+
const scanRoots = this.vault.scanRoots()
|
|
69
|
+
this.state.source_ids = scanRoots.map((source) => source.source_id).sort()
|
|
70
|
+
const previous = this.vault.registry.lastScan
|
|
71
|
+
const previousSourceIds = previous
|
|
72
|
+
? (Array.isArray(previous.source_ids) ? previous.source_ids : Object.keys(previous.source_bytes || {})).slice().sort()
|
|
73
|
+
: []
|
|
74
|
+
const sameSources = previousSourceIds.length === this.state.source_ids.length &&
|
|
75
|
+
previousSourceIds.every((sourceId, index) => sourceId === this.state.source_ids[index])
|
|
76
|
+
if (previous && previous.files > 0 && sameSources) {
|
|
77
|
+
this.state.estimated_files = previous.files
|
|
78
|
+
const measuredWeight = previous.duration_ms > 0 && previous.walk_duration_ms >= 0
|
|
79
|
+
? previous.walk_duration_ms / previous.duration_ms
|
|
80
|
+
: 0.8
|
|
81
|
+
this.state.estimated_walk_weight = Math.max(0.15, Math.min(0.98, measuredWeight))
|
|
82
|
+
}
|
|
83
|
+
const walkStarted = Date.now()
|
|
84
|
+
for (const source of scanRoots) {
|
|
85
|
+
if (!Object.prototype.hasOwnProperty.call(this.state.source_bytes, source.source_id)) {
|
|
86
|
+
this.state.source_bytes[source.source_id] = 0
|
|
87
|
+
}
|
|
44
88
|
await this.walk(source.root, source.source_id === "pinokio" ? null : source.source_id)
|
|
45
89
|
}
|
|
90
|
+
this.state.walk_duration_ms = Date.now() - walkStarted
|
|
91
|
+
const hashWaitStarted = Date.now()
|
|
46
92
|
await this.settle()
|
|
93
|
+
this.state.hash_wait_duration_ms = Date.now() - hashWaitStarted
|
|
47
94
|
this.state.duration_ms = Date.now() - this.state.started
|
|
48
95
|
this.vault.registry.setLastScan({
|
|
49
96
|
ts: Date.now(),
|
|
@@ -52,12 +99,23 @@ class Sweeper {
|
|
|
52
99
|
bytes_total: this.state.bytes_total,
|
|
53
100
|
home_bytes_total: this.state.home_bytes_total,
|
|
54
101
|
source_bytes: Object.assign({}, this.state.source_bytes),
|
|
102
|
+
source_ids: this.state.source_ids.slice(),
|
|
55
103
|
candidates: this.state.candidates,
|
|
56
104
|
hashed: this.state.hashed,
|
|
105
|
+
hash_total: this.state.hash_total,
|
|
57
106
|
hash_bytes: this.state.hash_bytes,
|
|
107
|
+
inode_reuses: this.state.inode_reuses,
|
|
108
|
+
unstable_hashes: this.state.unstable_hashes,
|
|
109
|
+
walk_duration_ms: this.state.walk_duration_ms,
|
|
110
|
+
hash_wait_duration_ms: this.state.hash_wait_duration_ms,
|
|
111
|
+
hash_duration_ms: this.state.hash_duration_ms,
|
|
58
112
|
duration_ms: this.state.duration_ms
|
|
59
113
|
})
|
|
60
114
|
} finally {
|
|
115
|
+
// A failed walk must not leave background hash work or per-scan inode
|
|
116
|
+
// state alive when the next manual scan starts.
|
|
117
|
+
await this.settle().catch(() => {})
|
|
118
|
+
this.hashJobsByIno.clear()
|
|
61
119
|
this.state.active = false
|
|
62
120
|
}
|
|
63
121
|
return {
|
|
@@ -72,30 +130,45 @@ class Sweeper {
|
|
|
72
130
|
const vaultRoot = this.vault.root
|
|
73
131
|
const stack = [root]
|
|
74
132
|
while (stack.length) {
|
|
75
|
-
const
|
|
76
|
-
this.
|
|
77
|
-
|
|
78
|
-
try {
|
|
79
|
-
entries = await fs.promises.readdir(dir, { withFileTypes: true })
|
|
80
|
-
} catch (e) {
|
|
81
|
-
continue
|
|
133
|
+
const dirs = []
|
|
134
|
+
while (dirs.length < this.dirConcurrency && stack.length) {
|
|
135
|
+
dirs.push(stack.pop())
|
|
82
136
|
}
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
137
|
+
const directoryResults = await Promise.all(dirs.map(async (dir) => {
|
|
138
|
+
this.state.dirs += 1
|
|
139
|
+
try {
|
|
140
|
+
const entries = await fs.promises.readdir(dir, { withFileTypes: true })
|
|
141
|
+
return { dir, entries }
|
|
142
|
+
} catch (e) {
|
|
143
|
+
return { dir, entries: null }
|
|
144
|
+
}
|
|
145
|
+
}))
|
|
146
|
+
const files = []
|
|
147
|
+
const hfDirectories = []
|
|
148
|
+
for (const { dir, entries } of directoryResults) {
|
|
149
|
+
if (!entries) continue
|
|
150
|
+
// HF hub blobs dir: filenames ARE sha256 for LFS files — no hashing.
|
|
151
|
+
if (path.basename(dir) === "blobs" && /^(models|datasets|spaces)--/.test(path.basename(path.dirname(dir)))) {
|
|
152
|
+
hfDirectories.push({ dir, entries })
|
|
153
|
+
continue
|
|
154
|
+
}
|
|
155
|
+
for (const entry of entries) {
|
|
156
|
+
if (entry.isSymbolicLink()) continue
|
|
157
|
+
const full = path.resolve(dir, entry.name)
|
|
158
|
+
if (entry.isDirectory()) {
|
|
159
|
+
if (full === vaultRoot) continue
|
|
160
|
+
stack.push(full)
|
|
161
|
+
} else if (entry.isFile()) {
|
|
162
|
+
files.push(full)
|
|
163
|
+
}
|
|
94
164
|
}
|
|
95
165
|
}
|
|
96
|
-
|
|
97
|
-
|
|
166
|
+
// One metadata pool is shared by all files in this directory batch, so
|
|
167
|
+
// trees made of many small directories still use the bounded workers.
|
|
98
168
|
await this.considerFiles(files, preferredSourceId)
|
|
169
|
+
for (const { dir, entries } of hfDirectories) {
|
|
170
|
+
await this.ingestHfBlobs(dir, entries, preferredSourceId)
|
|
171
|
+
}
|
|
99
172
|
}
|
|
100
173
|
}
|
|
101
174
|
async considerFiles(filePaths, preferredSourceId = null) {
|
|
@@ -165,20 +238,63 @@ class Sweeper {
|
|
|
165
238
|
await this.classify(filePath, st, harvested, sourceId)
|
|
166
239
|
return
|
|
167
240
|
}
|
|
241
|
+
// Only coalesce an inode which the filesystem itself reports as having
|
|
242
|
+
// multiple names. This avoids trusting placeholder inode values on
|
|
243
|
+
// filesystems which do not expose usable identity metadata.
|
|
244
|
+
const reusableInode = st.nlink > 1 && st.ino !== undefined && st.ino !== 0
|
|
245
|
+
const inodeKey = reusableInode ? registry.inoKey(st.dev, st.ino) : null
|
|
246
|
+
const existingJob = inodeKey ? this.hashJobsByIno.get(inodeKey) : null
|
|
247
|
+
const name = { filePath, source_id: sourceId, expected: fileSnapshot(st) }
|
|
248
|
+
if (existingJob) {
|
|
249
|
+
existingJob.names.push(name)
|
|
250
|
+
this.state.inode_reuses += 1
|
|
251
|
+
return
|
|
252
|
+
}
|
|
253
|
+
const job = { inodeKey, names: [name] }
|
|
254
|
+
if (inodeKey) this.hashJobsByIno.set(inodeKey, job)
|
|
255
|
+
this.state.hash_total += 1
|
|
168
256
|
this.state.queued += 1
|
|
169
|
-
this.hashQueue.push(
|
|
257
|
+
this.hashQueue.push(job).catch(() => {})
|
|
170
258
|
}
|
|
171
|
-
async _hashJob(
|
|
172
|
-
|
|
259
|
+
async _hashJob(job) {
|
|
260
|
+
const primary = job.names[0]
|
|
261
|
+
const started = Date.now()
|
|
262
|
+
this.currentHash = { path: primary.filePath }
|
|
173
263
|
try {
|
|
174
|
-
const { hash, size } = await this.vault.hashFile(filePath)
|
|
264
|
+
const { hash, size } = await this.vault.hashFile(primary.filePath)
|
|
175
265
|
this.stats.hashes += 1
|
|
176
266
|
this.state.hashed += 1
|
|
177
267
|
this.state.hash_bytes += size || 0
|
|
178
|
-
const
|
|
179
|
-
|
|
268
|
+
const primaryStat = await fs.promises.stat(primary.filePath)
|
|
269
|
+
// Never publish a digest if the path changed while its bytes were read.
|
|
270
|
+
// The next manual scan can retry a continuously-mutating file safely.
|
|
271
|
+
if (size !== primaryStat.size || !sameSnapshot(primary.expected, primaryStat)) {
|
|
272
|
+
this.state.unstable_hashes += 1
|
|
273
|
+
return
|
|
274
|
+
}
|
|
275
|
+
let index = 0
|
|
276
|
+
while (index < job.names.length) {
|
|
277
|
+
const name = job.names[index++]
|
|
278
|
+
let st
|
|
279
|
+
try {
|
|
280
|
+
st = name === primary ? primaryStat : await fs.promises.stat(name.filePath)
|
|
281
|
+
} catch (e) {
|
|
282
|
+
continue
|
|
283
|
+
}
|
|
284
|
+
// Same (device, inode) is physical identity, not a content guess.
|
|
285
|
+
// Size and mtime also prevent applying a completed digest after an
|
|
286
|
+
// in-place mutation. ctime is omitted here because adoption itself
|
|
287
|
+
// adds a name and therefore legitimately changes inode metadata.
|
|
288
|
+
if (st.dev !== primaryStat.dev || st.ino !== primaryStat.ino ||
|
|
289
|
+
st.size !== primaryStat.size || st.mtimeMs !== primaryStat.mtimeMs) continue
|
|
290
|
+
await this.classify(name.filePath, st, hash, name.source_id)
|
|
291
|
+
}
|
|
180
292
|
} catch (e) {
|
|
181
293
|
} finally {
|
|
294
|
+
this.state.hash_duration_ms += Date.now() - started
|
|
295
|
+
if (job.inodeKey && this.hashJobsByIno.get(job.inodeKey) === job) {
|
|
296
|
+
this.hashJobsByIno.delete(job.inodeKey)
|
|
297
|
+
}
|
|
182
298
|
this.currentHash = null
|
|
183
299
|
this.state.queued = Math.max(0, this.state.queued - 1)
|
|
184
300
|
}
|
package/package.json
CHANGED
package/server/index.js
CHANGED
|
@@ -15747,6 +15747,13 @@ class Server {
|
|
|
15747
15747
|
res.json(await vault.status())
|
|
15748
15748
|
}))
|
|
15749
15749
|
this.app.get("/vault", ex(async (req, res) => {
|
|
15750
|
+
const { install_required, requirements_pending } = await this.kernel.bin.check({
|
|
15751
|
+
bin: this.kernel.bin.preset("dev"),
|
|
15752
|
+
})
|
|
15753
|
+
if (requirements_pending || install_required) {
|
|
15754
|
+
res.redirect(`/setup/dev?callback=${encodeURIComponent(req.originalUrl)}`)
|
|
15755
|
+
return
|
|
15756
|
+
}
|
|
15750
15757
|
res.render("vault", { theme: this.theme, agent: req.agent })
|
|
15751
15758
|
}))
|
|
15752
15759
|
this.app.post("/vault/action", ex(async (req, res) => {
|
package/server/views/vault.ejs
CHANGED
|
@@ -191,6 +191,10 @@ button.vault-metric:focus-visible,
|
|
|
191
191
|
border-bottom: 1px solid var(--task-border);
|
|
192
192
|
font-size: 12.5px;
|
|
193
193
|
}
|
|
194
|
+
.vault-scan-state {
|
|
195
|
+
position: relative;
|
|
196
|
+
overflow: hidden;
|
|
197
|
+
}
|
|
194
198
|
.vault-scan-state.show,
|
|
195
199
|
.vault-result.show,
|
|
196
200
|
.vault-feedback.show { display: flex; }
|
|
@@ -199,6 +203,48 @@ button.vault-metric:focus-visible,
|
|
|
199
203
|
.vault-result strong { font-weight: 650; }
|
|
200
204
|
.vault-result-detail,
|
|
201
205
|
.vault-scan-detail { color: var(--task-muted); }
|
|
206
|
+
.vault-scan-detail {
|
|
207
|
+
min-width: 0;
|
|
208
|
+
overflow: hidden;
|
|
209
|
+
text-overflow: ellipsis;
|
|
210
|
+
white-space: nowrap;
|
|
211
|
+
}
|
|
212
|
+
.vault-scan-percent {
|
|
213
|
+
flex: 0 0 auto;
|
|
214
|
+
margin-left: auto;
|
|
215
|
+
color: var(--task-muted);
|
|
216
|
+
font-size: 11.5px;
|
|
217
|
+
font-variant-numeric: tabular-nums;
|
|
218
|
+
font-weight: 650;
|
|
219
|
+
}
|
|
220
|
+
.vault-progress-track {
|
|
221
|
+
position: absolute;
|
|
222
|
+
right: 0;
|
|
223
|
+
bottom: 0;
|
|
224
|
+
left: 0;
|
|
225
|
+
height: 2px;
|
|
226
|
+
overflow: hidden;
|
|
227
|
+
background: color-mix(in srgb, var(--task-accent) 12%, transparent);
|
|
228
|
+
}
|
|
229
|
+
.vault-progress-bar {
|
|
230
|
+
display: block;
|
|
231
|
+
width: 100%;
|
|
232
|
+
height: 100%;
|
|
233
|
+
background: var(--task-accent);
|
|
234
|
+
transform-origin: left center;
|
|
235
|
+
}
|
|
236
|
+
.vault-progress-bar.determinate {
|
|
237
|
+
transform: scaleX(var(--vault-progress, 0));
|
|
238
|
+
transition: transform 320ms cubic-bezier(.22, 1, .36, 1);
|
|
239
|
+
}
|
|
240
|
+
.vault-progress-bar.indeterminate {
|
|
241
|
+
transform: scaleX(1);
|
|
242
|
+
animation: vault-progress-pulse 1.4s ease-in-out infinite;
|
|
243
|
+
}
|
|
244
|
+
@keyframes vault-progress-pulse {
|
|
245
|
+
0%, 100% { opacity: .22; }
|
|
246
|
+
50% { opacity: .78; }
|
|
247
|
+
}
|
|
202
248
|
.vault-result .vault-button { min-height: 28px; margin-left: auto; padding: 4px 10px; }
|
|
203
249
|
.vault-feedback { min-height: 34px; background: var(--task-soft); }
|
|
204
250
|
.vault-feedback.error { color: var(--vault-warning); }
|
|
@@ -563,6 +609,12 @@ button.vault-metric:focus-visible,
|
|
|
563
609
|
}
|
|
564
610
|
@media (prefers-reduced-motion: reduce) {
|
|
565
611
|
.vault-page * { scroll-behavior: auto !important; }
|
|
612
|
+
.vault-progress-bar { transition: none; }
|
|
613
|
+
.vault-progress-bar.indeterminate {
|
|
614
|
+
animation: none;
|
|
615
|
+
opacity: .5;
|
|
616
|
+
transform: scaleX(1);
|
|
617
|
+
}
|
|
566
618
|
}
|
|
567
619
|
</style>
|
|
568
620
|
</head>
|
|
@@ -657,6 +709,9 @@ const COPY = {
|
|
|
657
709
|
repairing: "Repairing…",
|
|
658
710
|
repair_done: "Vault index repaired",
|
|
659
711
|
scan_progress: "Scanning your configured locations",
|
|
712
|
+
scan_analyzing: "Analyzing large files",
|
|
713
|
+
scan_finishing: "Finishing scan",
|
|
714
|
+
scan_estimate_help: "Estimated from your last completed scan",
|
|
660
715
|
scan_folders: "folders checked",
|
|
661
716
|
scan_files: "files checked",
|
|
662
717
|
analyzing: "analyzing",
|
|
@@ -1225,11 +1280,29 @@ const renderOverview = () => {
|
|
|
1225
1280
|
const scanState = el("vault-scan-state")
|
|
1226
1281
|
if (scanning) {
|
|
1227
1282
|
const scan = data.scan
|
|
1283
|
+
const walking = scan.walk_duration_ms == null
|
|
1284
|
+
const hashTotal = scan.hash_total || 0
|
|
1285
|
+
const hashDone = Math.min(scan.hashed || 0, hashTotal)
|
|
1286
|
+
const hashRatio = hashTotal ? Math.min(1, hashDone / hashTotal) : 1
|
|
1287
|
+
const phase = walking ? COPY.scan_progress : hashTotal ? COPY.scan_analyzing : COPY.scan_finishing
|
|
1288
|
+
const hasEstimate = Number.isFinite(scan.estimated_files) && scan.estimated_files > 0
|
|
1289
|
+
const walkWeight = Number.isFinite(scan.estimated_walk_weight) ? scan.estimated_walk_weight : 0.8
|
|
1290
|
+
const walkRatio = hasEstimate ? Math.min(1, (scan.files || 0) / scan.estimated_files) : null
|
|
1291
|
+
const progressRatio = walking
|
|
1292
|
+
? (hasEstimate ? walkRatio * walkWeight : null)
|
|
1293
|
+
: (hasEstimate ? walkWeight + ((1 - walkWeight) * hashRatio) : hashRatio)
|
|
1294
|
+
const progressValue = progressRatio == null ? null : Math.max(0, Math.min(100, Math.round(progressRatio * 100)))
|
|
1228
1295
|
const details = [`${scan.dirs || 0} ${COPY.scan_folders}`, `${scan.files || 0} ${COPY.scan_files}`, fmt(scan.bytes_total || 0)]
|
|
1229
1296
|
if (scan.current_file) details.push(`${COPY.analyzing} ${scan.current_file}`)
|
|
1230
|
-
if (scan.queued > 1) details.push(`${scan.queued} ${COPY.waiting}`)
|
|
1297
|
+
if (walking && scan.queued > 1) details.push(`${scan.queued} ${COPY.waiting}`)
|
|
1298
|
+
const estimated = hasEstimate && progressValue < 100
|
|
1299
|
+
const percent = progressValue == null ? "" : `<span class="vault-scan-percent"${estimated ? ` title="${attr(COPY.scan_estimate_help)}"` : ""}>${estimated ? "~" : ""}${progressValue}%</span>`
|
|
1300
|
+
const progressAttrs = progressValue == null
|
|
1301
|
+
? `role="progressbar" aria-label="${attr(phase)}"`
|
|
1302
|
+
: `role="progressbar" aria-label="${attr(phase)}" aria-valuemin="0" aria-valuemax="100" aria-valuenow="${progressValue}"${estimated ? ` aria-valuetext="${attr(`About ${progressValue} percent. ${COPY.scan_estimate_help}.`)}"` : ""}`
|
|
1303
|
+
const progressStyle = progressValue == null ? "" : ` style="--vault-progress:${progressRatio}"`
|
|
1231
1304
|
scanState.classList.add("show")
|
|
1232
|
-
scanState.innerHTML = `<i class="fa-solid fa-circle-notch fa-spin"></i><strong>${esc(
|
|
1305
|
+
scanState.innerHTML = `<i class="fa-solid fa-circle-notch fa-spin"></i><strong>${esc(phase)}</strong><span class="vault-scan-detail">${esc(details.join(" · "))}</span>${percent}<span class="vault-progress-track" ${progressAttrs}><span class="vault-progress-bar ${progressValue == null ? "indeterminate" : "determinate"}"${progressStyle}></span></span>`
|
|
1233
1306
|
} else {
|
|
1234
1307
|
scanState.classList.remove("show")
|
|
1235
1308
|
scanState.innerHTML = ""
|
package/test/vault-sweep.test.js
CHANGED
|
@@ -48,6 +48,10 @@ describe('vault manual scan (phase 3)', () => {
|
|
|
48
48
|
assert.ok(result.bytes_total >= content.length + 4, 'folder total includes small files')
|
|
49
49
|
assert.ok(vault.registry.lastScan && vault.registry.lastScan.bytes_total === result.bytes_total)
|
|
50
50
|
assert.ok(vault.registry.lastScan.duration_ms >= 0, 'scan duration is measured')
|
|
51
|
+
assert.ok(vault.registry.lastScan.walk_duration_ms >= 0, 'walk duration is measured')
|
|
52
|
+
assert.ok(vault.registry.lastScan.hash_duration_ms >= 0, 'hash duration is measured')
|
|
53
|
+
assert.strictEqual(vault.registry.lastScan.hash_total, 1, 'hash work has a determinate total after walking')
|
|
54
|
+
assert.deepStrictEqual(vault.registry.lastScan.source_ids, ['pinokio'], 'scan source set is recorded for later estimates')
|
|
51
55
|
})
|
|
52
56
|
|
|
53
57
|
test('scans NEVER convert: every byte-identical copy is pending, even in HF caches', async () => {
|
|
@@ -163,15 +167,17 @@ describe('vault manual scan (phase 3)', () => {
|
|
|
163
167
|
await writeFile(path.resolve(home, 'api', 'appA', 'm.bin'), crypto.randomBytes(4096))
|
|
164
168
|
await vault.sweeper.scan()
|
|
165
169
|
const after = vault.sweeper.stats.hashes
|
|
170
|
+
const priorFiles = vault.registry.lastScan.files
|
|
166
171
|
await vault.sweeper.scan()
|
|
167
172
|
assert.strictEqual(vault.sweeper.stats.hashes, after)
|
|
173
|
+
assert.strictEqual(vault.sweeper.state.estimated_files, priorFiles, 'same-source rescan uses the prior exact file count')
|
|
168
174
|
})
|
|
169
175
|
|
|
170
176
|
test('file metadata inspection uses bounded concurrency without changing results', async () => {
|
|
171
177
|
const { home, vault } = await makeEnv()
|
|
172
178
|
const dir = path.resolve(home, 'api', 'bulk')
|
|
173
179
|
await Promise.all(Array.from({ length: 48 }, (_, index) =>
|
|
174
|
-
writeFile(path.resolve(dir, `
|
|
180
|
+
writeFile(path.resolve(dir, `dir-${index}`, 'small.txt'), 'x')))
|
|
175
181
|
const realStat = fs.promises.stat
|
|
176
182
|
let active = 0
|
|
177
183
|
let maximum = 0
|
|
@@ -194,9 +200,53 @@ describe('vault manual scan (phase 3)', () => {
|
|
|
194
200
|
fs.promises.stat = realStat
|
|
195
201
|
}
|
|
196
202
|
assert.ok(maximum > 1, `expected concurrent metadata reads, saw ${maximum}`)
|
|
203
|
+
assert.ok(maximum <= vault.statConcurrency, `metadata concurrency exceeded its bound: ${maximum}`)
|
|
197
204
|
assert.strictEqual(vault.registry.lastScan.files >= 48, true)
|
|
198
205
|
})
|
|
199
206
|
|
|
207
|
+
test('multiple names for one inode share one exact hash job', async () => {
|
|
208
|
+
const { home, vault } = await makeEnv()
|
|
209
|
+
const content = crypto.randomBytes(4096)
|
|
210
|
+
const a = await writeFile(path.resolve(home, 'api', 'appA', 'model.bin'), content)
|
|
211
|
+
const b = path.resolve(home, 'api', 'appB', 'model.bin')
|
|
212
|
+
await fs.promises.mkdir(path.dirname(b), { recursive: true })
|
|
213
|
+
await fs.promises.link(a, b)
|
|
214
|
+
|
|
215
|
+
await vault.sweeper.scan()
|
|
216
|
+
|
|
217
|
+
assert.strictEqual(vault.registry.lastScan.hashed, 1, 'physical content was hashed once')
|
|
218
|
+
assert.strictEqual(vault.registry.lastScan.inode_reuses, 1, 'second name reused the inode hash job')
|
|
219
|
+
assert.strictEqual(vault.registry.duplicates.size, 0, 'existing hard links are already shared')
|
|
220
|
+
assert.strictEqual(vault.registry.links.get(a).hash, sha256(content))
|
|
221
|
+
assert.strictEqual(vault.registry.links.get(b).hash, sha256(content))
|
|
222
|
+
})
|
|
223
|
+
|
|
224
|
+
test('a file changed during hashing never publishes the stale digest', async () => {
|
|
225
|
+
const { home, vault } = await makeEnv()
|
|
226
|
+
const original = crypto.randomBytes(4096)
|
|
227
|
+
const oldHash = sha256(original)
|
|
228
|
+
const file = await writeFile(path.resolve(home, 'api', 'appA', 'changing.bin'), original)
|
|
229
|
+
const realHashFile = vault.hashFile.bind(vault)
|
|
230
|
+
vault.hashFile = async (target) => {
|
|
231
|
+
const result = await realHashFile(target)
|
|
232
|
+
await fs.promises.writeFile(target, crypto.randomBytes(original.length))
|
|
233
|
+
const future = new Date(Date.now() + 2000)
|
|
234
|
+
await fs.promises.utimes(target, future, future)
|
|
235
|
+
return result
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
try {
|
|
239
|
+
await vault.sweeper.scan()
|
|
240
|
+
} finally {
|
|
241
|
+
vault.hashFile = realHashFile
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
assert.strictEqual(vault.registry.lastScan.unstable_hashes, 1)
|
|
245
|
+
assert.strictEqual(vault.registry.blobs.has(oldHash), false, 'stale digest was discarded')
|
|
246
|
+
assert.strictEqual(vault.registry.scanIndex.has(file), false, 'unstable path remains unclassified')
|
|
247
|
+
assert.strictEqual((await fs.promises.stat(file)).nlink, 1, 'unstable file was not adopted')
|
|
248
|
+
})
|
|
249
|
+
|
|
200
250
|
test('conversion tmp files are never candidates', async () => {
|
|
201
251
|
const { home, vault } = await makeEnv()
|
|
202
252
|
const stray = await writeFile(path.resolve(home, 'api', 'appA', 'm.bin.pinokio-dedup-tmp'), crypto.randomBytes(4096))
|
package/test/vault-ui.test.js
CHANGED
|
@@ -204,6 +204,16 @@ describe('vault dashboard backend (phase 4)', () => {
|
|
|
204
204
|
assert.doesNotMatch(vaultPage, /Matching locations/)
|
|
205
205
|
assert.match(vaultPage, /scan_waiting:\s*"Waiting for scan results"/)
|
|
206
206
|
assert.match(vaultPage, /view === "all" && !scanning \? `<button class="vault-button" type="button" id="btn-empty-scan"/)
|
|
207
|
+
assert.match(vaultPage, /class="vault-progress-track"/)
|
|
208
|
+
assert.match(vaultPage, /role="progressbar"/)
|
|
209
|
+
assert.match(vaultPage, /scan\.walk_duration_ms == null/)
|
|
210
|
+
assert.match(vaultPage, /const hashTotal = scan\.hash_total/)
|
|
211
|
+
assert.match(vaultPage, /const hasEstimate = Number\.isFinite\(scan\.estimated_files\)/)
|
|
212
|
+
assert.match(vaultPage, /scan_estimate_help:\s*"Estimated from your last completed scan"/)
|
|
213
|
+
assert.match(vaultPage, /\$\{estimated \? "~" : ""\}\$\{progressValue\}%/)
|
|
214
|
+
assert.match(vaultPage, /\.vault-progress-bar\.determinate \{[\s\S]*?transform:\s*scaleX/)
|
|
215
|
+
assert.match(vaultPage, /\.vault-progress-bar\.indeterminate \{[\s\S]*?transform:\s*scaleX\(1\)[\s\S]*?animation:\s*vault-progress-pulse/)
|
|
216
|
+
assert.doesNotMatch(vaultPage, /vault-progress-sweep/)
|
|
207
217
|
assert.match(vaultPage, /body\.vault-page \.task-container \{[\s\S]*?overflow:\s*hidden/)
|
|
208
218
|
assert.match(vaultPage, /\.vault-explorer \{[\s\S]*?flex:\s*1 1 auto[\s\S]*?overflow:\s*hidden/)
|
|
209
219
|
assert.match(vaultPage, /\.vault-rail \{[\s\S]*?overflow-y:\s*auto[\s\S]*?overscroll-behavior:\s*contain/)
|
|
@@ -217,6 +227,19 @@ describe('vault dashboard backend (phase 4)', () => {
|
|
|
217
227
|
assert.match(vaultPage, /vault-source-line depth-\$\{Math\.min\(depth, 2\)\} \$\{pathText \? "has-path" : ""\}/)
|
|
218
228
|
})
|
|
219
229
|
|
|
230
|
+
test('vault route provisions the dev requirements needed by its folder picker', async () => {
|
|
231
|
+
const serverSource = await fs.promises.readFile(path.resolve(__dirname, '..', 'server', 'index.js'), 'utf8')
|
|
232
|
+
const routeStart = serverSource.indexOf('this.app.get("/vault"')
|
|
233
|
+
const routeEnd = serverSource.indexOf('this.app.post("/vault/action"', routeStart)
|
|
234
|
+
assert.ok(routeStart >= 0 && routeEnd > routeStart, 'vault route must be present')
|
|
235
|
+
const route = serverSource.slice(routeStart, routeEnd)
|
|
236
|
+
assert.match(route, /this\.kernel\.bin\.check\(/)
|
|
237
|
+
assert.match(route, /this\.kernel\.bin\.preset\("dev"\)/)
|
|
238
|
+
assert.match(route, /requirements_pending \|\| install_required/)
|
|
239
|
+
assert.match(route, /\/setup\/dev\?callback=\$\{encodeURIComponent\(req\.originalUrl\)\}/)
|
|
240
|
+
assert.ok(route.indexOf('res.redirect') < route.indexOf('res.render("vault"'), 'requirements redirect must happen before rendering Vault')
|
|
241
|
+
})
|
|
242
|
+
|
|
220
243
|
test('deduplicate batches share a batch id and are undoable as one', async () => {
|
|
221
244
|
const { home, vault } = await makeEnv()
|
|
222
245
|
const c1 = crypto.randomBytes(4096)
|