pinokiod 8.0.80 → 8.0.100
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/kernel/shell.js +7 -0
- package/kernel/vault/constants.js +4 -2
- package/kernel/vault/index.js +4 -0
- package/kernel/vault/registry_core.js +1 -1
- package/package.json +1 -1
- package/server/public/vault.js +7 -5
- package/test/shell-run-template.test.js +41 -0
- package/test/vault-automatic-scans.test.js +1 -1
- package/test/vault-engine.test.js +184 -69
- package/test/vault-sweep.test.js +38 -21
- package/test/vault-ui.test.js +10 -9
package/kernel/shell.js
CHANGED
|
@@ -1263,6 +1263,13 @@ class Shell {
|
|
|
1263
1263
|
if (cuda_script) {
|
|
1264
1264
|
conda_activation.push(`CALL "${cuda_script}" > "${cuda_log}" 2>&1`)
|
|
1265
1265
|
}
|
|
1266
|
+
// The conda-forge VS activation script prepends Library\lib before
|
|
1267
|
+
// calling vcvars, but vcvars rebuilds LIB and drops that entry.
|
|
1268
|
+
// Restore both the active Conda environment and managed CUDA import
|
|
1269
|
+
// libraries after all compiler activation scripts have completed.
|
|
1270
|
+
// CALL's second expansion is required because these commands are
|
|
1271
|
+
// joined on one cmd.exe line and must read LIB after vcvars updates it.
|
|
1272
|
+
conda_activation.push(`CALL set "LIB=%%CONDA_PREFIX%%\\Library\\lib;%%CUDA_HOME%%\\lib;%%LIB%%"`)
|
|
1266
1273
|
} catch (e) {
|
|
1267
1274
|
console.log('vc vars setup', e)
|
|
1268
1275
|
}
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
const CANDIDATE_SIZE_BASE = process.platform === "win32" ? 1024 : 1000
|
|
2
2
|
const SIZE_THRESHOLD = 100 * CANDIDATE_SIZE_BASE ** 2
|
|
3
|
-
const CANDIDATE_SIZE_OPTIONS = [
|
|
4
|
-
.
|
|
3
|
+
const CANDIDATE_SIZE_OPTIONS = [10, 50, 100, 500]
|
|
4
|
+
.map((value) => value * CANDIDATE_SIZE_BASE ** 2)
|
|
5
5
|
.concat(CANDIDATE_SIZE_BASE ** 3)
|
|
6
|
+
const MINIMUM_CANDIDATE_SIZE = CANDIDATE_SIZE_OPTIONS[0]
|
|
6
7
|
const isCandidateFileSize = (size, threshold) => size > 0 && size >= threshold
|
|
7
8
|
|
|
8
9
|
module.exports = {
|
|
10
|
+
MINIMUM_CANDIDATE_SIZE,
|
|
9
11
|
SIZE_THRESHOLD,
|
|
10
12
|
CANDIDATE_SIZE_OPTIONS,
|
|
11
13
|
isCandidateFileSize,
|
package/kernel/vault/index.js
CHANGED
|
@@ -11,6 +11,7 @@ const AutomaticScans = require("./automatic_scans")
|
|
|
11
11
|
const { fileSnapshot, sameSnapshot, sameContentState } = require("./snapshot")
|
|
12
12
|
const { cancelledError } = require("./operation_errors")
|
|
13
13
|
const {
|
|
14
|
+
MINIMUM_CANDIDATE_SIZE,
|
|
14
15
|
SIZE_THRESHOLD,
|
|
15
16
|
CANDIDATE_SIZE_OPTIONS,
|
|
16
17
|
TMP_SUFFIX,
|
|
@@ -2417,6 +2418,9 @@ class Vault {
|
|
|
2417
2418
|
unavailable_reason: unavailableReason
|
|
2418
2419
|
}
|
|
2419
2420
|
}
|
|
2421
|
+
if (targetStat.size < MINIMUM_CANDIDATE_SIZE) {
|
|
2422
|
+
return markUnavailable("below_minimum_size")
|
|
2423
|
+
}
|
|
2420
2424
|
const prepared = await this.ensureAnchorForHash(target.hash, targetStat)
|
|
2421
2425
|
if (prepared.status !== "ready") {
|
|
2422
2426
|
if (prepared.status === "unavailable") {
|
|
@@ -3227,7 +3227,7 @@ class RegistryCore {
|
|
|
3227
3227
|
beginScan(scopeId = null) {
|
|
3228
3228
|
// The scan schema is reset for every run, so a compact integer is enough.
|
|
3229
3229
|
// Repeating a UUID in tens of millions of temporary rows and indexes turns
|
|
3230
|
-
//
|
|
3230
|
+
// a low-threshold scan into many gigabytes of avoidable I/O.
|
|
3231
3231
|
const id = 1
|
|
3232
3232
|
this.scanSizes.clear()
|
|
3233
3233
|
this.scanInodes.clear()
|
package/package.json
CHANGED
package/server/public/vault.js
CHANGED
|
@@ -90,7 +90,7 @@ const COPY = {
|
|
|
90
90
|
manual_description: "Scan only when requested",
|
|
91
91
|
storage_saved_headline: "{size} saved by deduplicating",
|
|
92
92
|
storage_unique_headline: "{size} unique to this app",
|
|
93
|
-
storage_still_used: "
|
|
93
|
+
storage_still_used: "In use",
|
|
94
94
|
storage_saved: "Saved",
|
|
95
95
|
storage_unique: "Unique",
|
|
96
96
|
storage_shared: "Shared",
|
|
@@ -192,6 +192,7 @@ const COPY = {
|
|
|
192
192
|
hardlinks_unavailable: "This disk does not support shared files",
|
|
193
193
|
anchor_conflict: "The stored matching copy could not be verified",
|
|
194
194
|
permission_denied: "Disk Saver cannot modify this file or its folder",
|
|
195
|
+
below_minimum_size: "This file is below the 10 MB safety minimum",
|
|
195
196
|
cannot_share_safely: "This file cannot safely share storage",
|
|
196
197
|
changed_since_scan: "Some files changed since the scan. Scan again before deduplicating them.",
|
|
197
198
|
deduplicate_locked: "Stop the app before deduplicating this file.",
|
|
@@ -394,8 +395,8 @@ const candidateSizeBase = document.body.dataset.platform === "win32" ? 1024 : 10
|
|
|
394
395
|
const AUTOMATIC_SUPPORTED = document.body.dataset.platform === "darwin" ||
|
|
395
396
|
document.body.dataset.platform === "win32"
|
|
396
397
|
const defaultCandidateSize = 100 * candidateSizeBase ** 2
|
|
397
|
-
const candidateSizeOptions = [
|
|
398
|
-
.
|
|
398
|
+
const candidateSizeOptions = [10, 50, 100, 500]
|
|
399
|
+
.map((value) => value * candidateSizeBase ** 2)
|
|
399
400
|
.concat(candidateSizeBase ** 3)
|
|
400
401
|
const PAGE_SIZE = 500
|
|
401
402
|
|
|
@@ -494,7 +495,7 @@ const candidateSize = () => {
|
|
|
494
495
|
return candidateSizeOptions.includes(value) ? value : defaultCandidateSize
|
|
495
496
|
}
|
|
496
497
|
const candidateSizeLabel = (size = candidateSize()) => {
|
|
497
|
-
return
|
|
498
|
+
return `${fmt(size)}+`
|
|
498
499
|
}
|
|
499
500
|
const renderCandidateSizeControl = () => {
|
|
500
501
|
const size = candidateSize()
|
|
@@ -1657,7 +1658,8 @@ const unavailableLabel = (item) => ({
|
|
|
1657
1658
|
hardlinks: COPY.hardlinks_unavailable,
|
|
1658
1659
|
different_disk: COPY.different_disk,
|
|
1659
1660
|
anchor_conflict: COPY.anchor_conflict,
|
|
1660
|
-
permission_denied: COPY.permission_denied
|
|
1661
|
+
permission_denied: COPY.permission_denied,
|
|
1662
|
+
below_minimum_size: COPY.below_minimum_size
|
|
1661
1663
|
}[item.unavailable_reason] || COPY.cannot_share_safely)
|
|
1662
1664
|
const statusMarkup = (item) => {
|
|
1663
1665
|
if (item.status === "unavailable") {
|
|
@@ -174,6 +174,47 @@ test('Shell.activate keeps managed Python downloads automatic on every platform'
|
|
|
174
174
|
}
|
|
175
175
|
})
|
|
176
176
|
|
|
177
|
+
test('Shell.activate restores Conda and CUDA library paths after Windows compiler activation', async (t) => {
|
|
178
|
+
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'pinokio-shell-windows-lib-'))
|
|
179
|
+
t.after(() => fs.rm(root, { recursive: true, force: true }))
|
|
180
|
+
const activateRoot = path.join(root, 'bin', 'miniforge', 'etc', 'conda', 'activate.d', 'pinokio')
|
|
181
|
+
const compilerScript = path.join(activateRoot, 'vs2019_compiler_vars.bat')
|
|
182
|
+
const cudaScript = path.join(activateRoot, '~cuda-nvcc_activate.bat')
|
|
183
|
+
await fs.mkdir(activateRoot, { recursive: true })
|
|
184
|
+
await fs.writeFile(compilerScript, '@echo off\n')
|
|
185
|
+
await fs.writeFile(cudaScript, '@echo off\n')
|
|
186
|
+
|
|
187
|
+
const kernel = createKernel(root)
|
|
188
|
+
kernel.bin = {
|
|
189
|
+
activationCommands: () => [],
|
|
190
|
+
path: (...parts) => path.join(root, 'bin', ...parts),
|
|
191
|
+
vs_path_env: {},
|
|
192
|
+
}
|
|
193
|
+
const shell = createShell(kernel)
|
|
194
|
+
shell.platform = 'win32'
|
|
195
|
+
shell.shell = 'cmd.exe'
|
|
196
|
+
shell.env = {}
|
|
197
|
+
|
|
198
|
+
const params = await shell.activate({
|
|
199
|
+
build: true,
|
|
200
|
+
path: root,
|
|
201
|
+
message: ['uv pip install cupy'],
|
|
202
|
+
})
|
|
203
|
+
|
|
204
|
+
const activation = params.message[0]
|
|
205
|
+
const compilerIndex = activation.indexOf(`CALL "${compilerScript}"`)
|
|
206
|
+
const cudaIndex = activation.indexOf(`CALL "${cudaScript}"`)
|
|
207
|
+
const libraryCommand = 'CALL set "LIB=%%CONDA_PREFIX%%\\Library\\lib;%%CUDA_HOME%%\\lib;%%LIB%%"'
|
|
208
|
+
const libraryIndex = activation.indexOf(libraryCommand)
|
|
209
|
+
|
|
210
|
+
assert.notEqual(compilerIndex, -1)
|
|
211
|
+
assert.notEqual(cudaIndex, -1)
|
|
212
|
+
assert.notEqual(libraryIndex, -1)
|
|
213
|
+
assert.ok(compilerIndex < cudaIndex)
|
|
214
|
+
assert.ok(cudaIndex < libraryIndex)
|
|
215
|
+
assert.equal(params.message[1], 'uv pip install cupy')
|
|
216
|
+
})
|
|
217
|
+
|
|
177
218
|
test('Shell.init_env defaults the uv HTTP timeout without overriding apps', async () => {
|
|
178
219
|
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'pinokio-shell-uv-timeout-'))
|
|
179
220
|
await fs.mkdir(path.join(root, 'api'), { recursive: true })
|
|
@@ -1247,7 +1247,7 @@ describe("automatic app checks", () => {
|
|
|
1247
1247
|
await fs.promises.writeFile(second, "same-content")
|
|
1248
1248
|
const vault = await makeVault(home)
|
|
1249
1249
|
vault.lastScanCache.set("", { candidate_min_bytes: 1 })
|
|
1250
|
-
|
|
1250
|
+
vault.automaticScans.candidateThreshold = async () => 1
|
|
1251
1251
|
const hashedPaths = []
|
|
1252
1252
|
const originalHashFile = vault.hashFile.bind(vault)
|
|
1253
1253
|
vault.hashFile = async (filePath, options) => {
|
|
@@ -9,9 +9,12 @@ const Vault = require("../kernel/vault")
|
|
|
9
9
|
const Registry = require("../kernel/vault/registry")
|
|
10
10
|
const {
|
|
11
11
|
CANDIDATE_SIZE_OPTIONS,
|
|
12
|
+
MINIMUM_CANDIDATE_SIZE,
|
|
12
13
|
SIZE_THRESHOLD
|
|
13
14
|
} = require("../kernel/vault/constants")
|
|
14
15
|
|
|
16
|
+
const candidateBase = process.platform === "win32" ? 1024 : 1000
|
|
17
|
+
|
|
15
18
|
const homes = []
|
|
16
19
|
|
|
17
20
|
const makeHome = async () => {
|
|
@@ -35,7 +38,7 @@ const makeVault = async ({ candidateSize = CANDIDATE_SIZE_OPTIONS[0] } = {}) =>
|
|
|
35
38
|
const vault = new Vault(kernel)
|
|
36
39
|
kernel.vault = vault
|
|
37
40
|
await vault.init()
|
|
38
|
-
vault.sizeThreshold =
|
|
41
|
+
vault.sizeThreshold = 1
|
|
39
42
|
if (candidateSize !== null) {
|
|
40
43
|
await vault.perform("set_candidate_size", {
|
|
41
44
|
candidate_size: candidateSize
|
|
@@ -50,13 +53,29 @@ const write = async (filePath, contents) => {
|
|
|
50
53
|
return filePath
|
|
51
54
|
}
|
|
52
55
|
|
|
56
|
+
const writeCandidate = async (
|
|
57
|
+
filePath,
|
|
58
|
+
contents,
|
|
59
|
+
size = MINIMUM_CANDIDATE_SIZE
|
|
60
|
+
) => {
|
|
61
|
+
await write(filePath, contents)
|
|
62
|
+
await fs.promises.truncate(filePath, size)
|
|
63
|
+
return filePath
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
const assertCandidateContents = async (filePath, contents, size) => {
|
|
67
|
+
const expected = Buffer.alloc(size)
|
|
68
|
+
contents.copy(expected)
|
|
69
|
+
assert.deepEqual(await fs.promises.readFile(filePath), expected)
|
|
70
|
+
}
|
|
71
|
+
|
|
53
72
|
const duplicatePair = async (home, name = "model.bin") => {
|
|
54
73
|
const contents = crypto.randomBytes(4096)
|
|
55
|
-
const first = await
|
|
74
|
+
const first = await writeCandidate(
|
|
56
75
|
path.join(home, "api", "first", name), contents)
|
|
57
|
-
const second = await
|
|
76
|
+
const second = await writeCandidate(
|
|
58
77
|
path.join(home, "api", "second", name), contents)
|
|
59
|
-
return { contents, first, second }
|
|
78
|
+
return { contents, first, second, size: MINIMUM_CANDIDATE_SIZE }
|
|
60
79
|
}
|
|
61
80
|
|
|
62
81
|
const close = async (vault) => {
|
|
@@ -268,6 +287,33 @@ describe("Save Space engine", () => {
|
|
|
268
287
|
await close(vault)
|
|
269
288
|
})
|
|
270
289
|
|
|
290
|
+
test("minimum size settings reject values below 10 MB", async () => {
|
|
291
|
+
const { vault } = await makeVault({ candidateSize: null })
|
|
292
|
+
|
|
293
|
+
for (const candidateSize of [0, candidateBase ** 2]) {
|
|
294
|
+
assert.match((await vault.perform("set_candidate_size", {
|
|
295
|
+
candidate_size: candidateSize
|
|
296
|
+
})).error, /valid minimum file size/i)
|
|
297
|
+
}
|
|
298
|
+
assert.equal(CANDIDATE_SIZE_OPTIONS[0],
|
|
299
|
+
MINIMUM_CANDIDATE_SIZE)
|
|
300
|
+
assert.deepEqual(CANDIDATE_SIZE_OPTIONS, [
|
|
301
|
+
10, 50, 100, 500, 1000
|
|
302
|
+
].map((value) => value * candidateBase ** 2))
|
|
303
|
+
assert.equal(await vault.registry.scanSetting(), null)
|
|
304
|
+
|
|
305
|
+
await close(vault)
|
|
306
|
+
})
|
|
307
|
+
|
|
308
|
+
test("legacy minimum size settings fall back to 100 MB", async () => {
|
|
309
|
+
const { vault } = await makeVault({ candidateSize: null })
|
|
310
|
+
await vault.registry.setScanSetting(null, candidateBase ** 2)
|
|
311
|
+
|
|
312
|
+
assert.equal((await vault.status()).candidate_min_bytes, SIZE_THRESHOLD)
|
|
313
|
+
|
|
314
|
+
await close(vault)
|
|
315
|
+
})
|
|
316
|
+
|
|
271
317
|
test("minimum size settings reject malformed scopes", async () => {
|
|
272
318
|
const { vault } = await makeVault({ candidateSize: null })
|
|
273
319
|
|
|
@@ -677,11 +723,11 @@ describe("Save Space engine", () => {
|
|
|
677
723
|
const { home, vault } = await makeVault()
|
|
678
724
|
const outside = await makeOutside()
|
|
679
725
|
const contents = crypto.randomBytes(4096)
|
|
680
|
-
const pinokioFile = await
|
|
726
|
+
const pinokioFile = await writeCandidate(
|
|
681
727
|
path.join(home, "api", "app", "model.bin"), contents)
|
|
682
|
-
const direct = await
|
|
728
|
+
const direct = await writeCandidate(
|
|
683
729
|
path.join(outside, "other-app", "model.bin"), contents)
|
|
684
|
-
const nested = await
|
|
730
|
+
const nested = await writeCandidate(
|
|
685
731
|
path.join(outside, "other-app", "models", "model.bin"),
|
|
686
732
|
contents
|
|
687
733
|
)
|
|
@@ -721,11 +767,11 @@ describe("Save Space engine", () => {
|
|
|
721
767
|
assert.equal(discovery.phase, "complete", discovery.error)
|
|
722
768
|
assert.equal(discovery.result_count, 1)
|
|
723
769
|
assert.equal(discovery.result_files, 2)
|
|
724
|
-
assert.equal(discovery.result_bytes,
|
|
770
|
+
assert.equal(discovery.result_bytes, MINIMUM_CANDIDATE_SIZE * 2)
|
|
725
771
|
assert.equal(discovery.candidates_known, true)
|
|
726
772
|
assert.equal(discovery.processed, discovery.candidates)
|
|
727
773
|
assert.equal(discovery.verified_files, 2)
|
|
728
|
-
assert.equal(discovery.verified_bytes,
|
|
774
|
+
assert.equal(discovery.verified_bytes, MINIMUM_CANDIDATE_SIZE * 2)
|
|
729
775
|
const results = await vault.folderDiscoveryResults()
|
|
730
776
|
assert.equal(results.total, 1)
|
|
731
777
|
assert.equal(results.root.folder,
|
|
@@ -733,7 +779,7 @@ describe("Save Space engine", () => {
|
|
|
733
779
|
assert.equal(results.items[0].folder,
|
|
734
780
|
path.join(await fs.promises.realpath(outside), "other-app"))
|
|
735
781
|
assert.equal(results.items[0].file_count, 2)
|
|
736
|
-
assert.equal(results.items[0].bytes,
|
|
782
|
+
assert.equal(results.items[0].bytes, MINIMUM_CANDIDATE_SIZE * 2)
|
|
737
783
|
assert.equal(results.items[0].eligible_file_count, 2)
|
|
738
784
|
const recommendations = await vault.registry
|
|
739
785
|
.folderDiscoveryRecommendations(vault.folderFinder.runId)
|
|
@@ -772,20 +818,20 @@ describe("Save Space engine", () => {
|
|
|
772
818
|
const first = crypto.randomBytes(4096)
|
|
773
819
|
const second = crypto.randomBytes(6144)
|
|
774
820
|
const unrelated = crypto.randomBytes(7168)
|
|
775
|
-
await
|
|
776
|
-
await
|
|
777
|
-
const unrelatedPath = await
|
|
821
|
+
await writeCandidate(path.join(home, "api", "first", "model.bin"), first)
|
|
822
|
+
await writeCandidate(path.join(home, "api", "second", "model.bin"), second)
|
|
823
|
+
const unrelatedPath = await writeCandidate(
|
|
778
824
|
path.join(home, "api", "unrelated-a", "model.bin"), unrelated)
|
|
779
|
-
await
|
|
825
|
+
await writeCandidate(
|
|
780
826
|
path.join(home, "api", "unrelated-b", "model.bin"), unrelated)
|
|
781
827
|
const comfy = path.join(outside, ".comfycraft")
|
|
782
828
|
const release = path.join(outside, "pinokio_2026_0627")
|
|
783
|
-
await
|
|
784
|
-
await
|
|
829
|
+
await writeCandidate(path.join(comfy, "kits", "ace", "model.bin"), first)
|
|
830
|
+
await writeCandidate(path.join(comfy, "kits", "hello", "model.bin"), second)
|
|
785
831
|
await write(path.join(comfy, "logs", "history.bin"),
|
|
786
832
|
crypto.randomBytes(2048))
|
|
787
|
-
await
|
|
788
|
-
await
|
|
833
|
+
await writeCandidate(path.join(release, "api", "first", "model.bin"), first)
|
|
834
|
+
await writeCandidate(path.join(release, "api", "second", "model.bin"), second)
|
|
789
835
|
|
|
790
836
|
await vault.sweeper.scan()
|
|
791
837
|
const unrelatedHash = (await vault.registry.getFile(unrelatedPath)).hash
|
|
@@ -823,7 +869,7 @@ describe("Save Space engine", () => {
|
|
|
823
869
|
const kitChildren = await vault.folderDiscoveryChildren(
|
|
824
870
|
comfyChildren.items[0].folder)
|
|
825
871
|
assert.equal(kitChildren.items.length, 2)
|
|
826
|
-
assert.equal(comfyResult.eligible_file_count,
|
|
872
|
+
assert.equal(comfyResult.eligible_file_count, 2)
|
|
827
873
|
assert.equal(comfyResult.file_count, 2)
|
|
828
874
|
|
|
829
875
|
const selected = [
|
|
@@ -889,9 +935,9 @@ describe("Save Space engine", () => {
|
|
|
889
935
|
const contents = crypto.randomBytes(8192)
|
|
890
936
|
await write(path.join(home, "api", "baseline", "unique.bin"),
|
|
891
937
|
crypto.randomBytes(contents.length))
|
|
892
|
-
const first = await
|
|
938
|
+
const first = await writeCandidate(
|
|
893
939
|
path.join(outside, "first-app", "model.bin"), contents)
|
|
894
|
-
const second = await
|
|
940
|
+
const second = await writeCandidate(
|
|
895
941
|
path.join(outside, "second-app", "model.bin"), contents)
|
|
896
942
|
|
|
897
943
|
await vault.sweeper.scan()
|
|
@@ -904,7 +950,7 @@ describe("Save Space engine", () => {
|
|
|
904
950
|
const discovery = vault.folderDiscoveryStatus()
|
|
905
951
|
const results = await vault.folderDiscoveryResults()
|
|
906
952
|
assert.equal(discovery.result_files, 2)
|
|
907
|
-
assert.equal(discovery.result_bytes,
|
|
953
|
+
assert.equal(discovery.result_bytes, MINIMUM_CANDIDATE_SIZE * 2)
|
|
908
954
|
assert.equal(results.selection.selected_count, 0)
|
|
909
955
|
await vault.perform("update_folder_discovery_selection", {
|
|
910
956
|
root: discovery.root,
|
|
@@ -915,7 +961,7 @@ describe("Save Space engine", () => {
|
|
|
915
961
|
const staged = (await vault.folderDiscoveryResults()).selection
|
|
916
962
|
assert.equal(staged.selected_count, 1)
|
|
917
963
|
assert.equal(staged.selected_files, 2)
|
|
918
|
-
assert.equal(staged.potential_savings,
|
|
964
|
+
assert.equal(staged.potential_savings, MINIMUM_CANDIDATE_SIZE)
|
|
919
965
|
|
|
920
966
|
vault.hashFile = async () => {
|
|
921
967
|
throw new Error("Adding staged results must not hash files again.")
|
|
@@ -936,8 +982,9 @@ describe("Save Space engine", () => {
|
|
|
936
982
|
assert.equal(secondRow.status, "duplicate")
|
|
937
983
|
const status = await vault.status()
|
|
938
984
|
assert.equal(status.bytes_without_sharing, contents.length)
|
|
939
|
-
assert.equal(status.logical_bytes,
|
|
940
|
-
|
|
985
|
+
assert.equal(status.logical_bytes,
|
|
986
|
+
MINIMUM_CANDIDATE_SIZE * 2 + contents.length)
|
|
987
|
+
assert.equal(status.pending_bytes, MINIMUM_CANDIDATE_SIZE)
|
|
941
988
|
await close(vault)
|
|
942
989
|
})
|
|
943
990
|
|
|
@@ -957,7 +1004,8 @@ describe("Save Space engine", () => {
|
|
|
957
1004
|
await vault.sweeper.scan()
|
|
958
1005
|
assert.equal((await vault.registry.files()).length, 0)
|
|
959
1006
|
|
|
960
|
-
await
|
|
1007
|
+
await writeCandidate(
|
|
1008
|
+
path.join(outside, "models", "model.bin"), pair.contents, pair.size)
|
|
961
1009
|
assert.equal((await vault.perform("find_folders", {
|
|
962
1010
|
path: outside,
|
|
963
1011
|
candidate_size: CANDIDATE_SIZE_OPTIONS[0]
|
|
@@ -967,7 +1015,7 @@ describe("Save Space engine", () => {
|
|
|
967
1015
|
const discovery = vault.folderDiscoveryStatus()
|
|
968
1016
|
assert.equal(discovery.phase, "complete", discovery.error)
|
|
969
1017
|
assert.equal(discovery.result_files, 1)
|
|
970
|
-
assert.equal(discovery.result_bytes, pair.
|
|
1018
|
+
assert.equal(discovery.result_bytes, pair.size)
|
|
971
1019
|
const results = await vault.folderDiscoveryResults()
|
|
972
1020
|
assert.equal(results.total, 1)
|
|
973
1021
|
assert.equal(results.selection.selected_count, 0)
|
|
@@ -979,7 +1027,7 @@ describe("Save Space engine", () => {
|
|
|
979
1027
|
})
|
|
980
1028
|
const selected = (await vault.folderDiscoveryResults()).selection
|
|
981
1029
|
assert.equal(selected.selected_files, 1)
|
|
982
|
-
assert.equal(selected.potential_savings, pair.
|
|
1030
|
+
assert.equal(selected.potential_savings, pair.size)
|
|
983
1031
|
await close(vault)
|
|
984
1032
|
})
|
|
985
1033
|
|
|
@@ -987,8 +1035,8 @@ describe("Save Space engine", () => {
|
|
|
987
1035
|
const { home, vault } = await makeVault()
|
|
988
1036
|
const outside = await makeOutside()
|
|
989
1037
|
const contents = crypto.randomBytes(4096)
|
|
990
|
-
await
|
|
991
|
-
const outsideFile = await
|
|
1038
|
+
await writeCandidate(path.join(home, "api", "app", "model.bin"), contents)
|
|
1039
|
+
const outsideFile = await writeCandidate(
|
|
992
1040
|
path.join(outside, "models", "model.bin"), contents)
|
|
993
1041
|
const canonicalOutsideFile = await fs.promises.realpath(outsideFile)
|
|
994
1042
|
await vault.sweeper.scan()
|
|
@@ -1037,18 +1085,45 @@ describe("Save Space engine", () => {
|
|
|
1037
1085
|
const { home, vault } = await makeVault()
|
|
1038
1086
|
const outside = await makeOutside()
|
|
1039
1087
|
const contents = crypto.randomBytes(64)
|
|
1040
|
-
await
|
|
1088
|
+
await writeCandidate(
|
|
1089
|
+
path.join(home, "api", "context", "model.bin"), contents)
|
|
1041
1090
|
for (let index = 0; index < 501; index++) {
|
|
1042
|
-
await
|
|
1091
|
+
await writeCandidate(path.join(
|
|
1043
1092
|
outside,
|
|
1044
1093
|
`app-${String(index).padStart(3, "0")}`,
|
|
1045
1094
|
"models",
|
|
1046
1095
|
"model.bin"
|
|
1047
1096
|
), contents)
|
|
1048
1097
|
}
|
|
1049
|
-
await write(path.join(outside, "unrelated.bin"),
|
|
1050
|
-
crypto.randomBytes(contents.length * 1000))
|
|
1051
1098
|
await vault.sweeper.scan()
|
|
1099
|
+
const resultHash = crypto.createHash("sha256")
|
|
1100
|
+
.update(contents)
|
|
1101
|
+
.update(Buffer.alloc(MINIMUM_CANDIDATE_SIZE - contents.length))
|
|
1102
|
+
.digest("hex")
|
|
1103
|
+
vault.hashFile = async (filePath) => ({
|
|
1104
|
+
hash: resultHash,
|
|
1105
|
+
size: (await fs.promises.stat(filePath)).size
|
|
1106
|
+
})
|
|
1107
|
+
const prepareResults = vault.registry.prepareFolderDiscoveryResults
|
|
1108
|
+
.bind(vault.registry)
|
|
1109
|
+
vault.registry.prepareFolderDiscoveryResults = async (runId, root) => {
|
|
1110
|
+
const stat = await fs.promises.stat(root)
|
|
1111
|
+
// Force recommendations below the root without creating a huge file.
|
|
1112
|
+
const staged = await vault.registry.stageFolderDiscoveryFiles(runId, [{
|
|
1113
|
+
path: path.join(root, "unrelated.bin"),
|
|
1114
|
+
size: MINIMUM_CANDIDATE_SIZE * 1000,
|
|
1115
|
+
mtime: stat.mtimeMs,
|
|
1116
|
+
ctime: stat.ctimeMs,
|
|
1117
|
+
dev: stat.dev,
|
|
1118
|
+
ino: 0,
|
|
1119
|
+
nlink: 1,
|
|
1120
|
+
mode: stat.mode,
|
|
1121
|
+
uid: stat.uid,
|
|
1122
|
+
gid: stat.gid
|
|
1123
|
+
}])
|
|
1124
|
+
assert.equal(staged.changes, 1)
|
|
1125
|
+
return prepareResults(runId, root)
|
|
1126
|
+
}
|
|
1052
1127
|
assert.equal((await vault.perform("find_folders", {
|
|
1053
1128
|
path: outside,
|
|
1054
1129
|
candidate_size: CANDIDATE_SIZE_OPTIONS[0]
|
|
@@ -1084,13 +1159,24 @@ describe("Save Space engine", () => {
|
|
|
1084
1159
|
const outside = await makeOutside()
|
|
1085
1160
|
const first = crypto.randomBytes(1000)
|
|
1086
1161
|
const second = crypto.randomBytes(1000)
|
|
1087
|
-
await
|
|
1088
|
-
|
|
1089
|
-
await
|
|
1090
|
-
|
|
1162
|
+
await writeCandidate(
|
|
1163
|
+
path.join(home, "api", "first", "model.bin"), first)
|
|
1164
|
+
await writeCandidate(
|
|
1165
|
+
path.join(home, "api", "second", "model.bin"), second)
|
|
1166
|
+
await writeCandidate(
|
|
1167
|
+
path.join(outside, "group", "a", "model.bin"), first)
|
|
1168
|
+
await writeCandidate(
|
|
1169
|
+
path.join(outside, "group", "b", "model.bin"), second)
|
|
1091
1170
|
for (let index = 0; index < extraSizes.length; index++) {
|
|
1092
|
-
|
|
1093
|
-
|
|
1171
|
+
const size = Math.max(
|
|
1172
|
+
MINIMUM_CANDIDATE_SIZE,
|
|
1173
|
+
MINIMUM_CANDIDATE_SIZE * extraSizes[index] / 1000
|
|
1174
|
+
)
|
|
1175
|
+
await writeCandidate(
|
|
1176
|
+
path.join(outside, "group", `extra-${index}.bin`),
|
|
1177
|
+
crypto.randomBytes(extraSizes[index]),
|
|
1178
|
+
size
|
|
1179
|
+
)
|
|
1094
1180
|
}
|
|
1095
1181
|
await vault.sweeper.scan()
|
|
1096
1182
|
await vault.perform("find_folders", {
|
|
@@ -1249,13 +1335,15 @@ describe("Save Space engine", () => {
|
|
|
1249
1335
|
const contents = crypto.randomBytes(4096)
|
|
1250
1336
|
const stale = []
|
|
1251
1337
|
for (let index = 0; index < 32; index++) {
|
|
1252
|
-
stale.push(await
|
|
1338
|
+
stale.push(await writeCandidate(
|
|
1253
1339
|
path.join(home, "api", "app", `a${String(index).padStart(2, "0")}.bin`),
|
|
1254
1340
|
contents
|
|
1255
1341
|
))
|
|
1256
1342
|
}
|
|
1257
|
-
await
|
|
1258
|
-
|
|
1343
|
+
await writeCandidate(
|
|
1344
|
+
path.join(home, "api", "app", "z-valid.bin"), contents)
|
|
1345
|
+
await writeCandidate(
|
|
1346
|
+
path.join(outside, "models", "model.bin"), contents)
|
|
1259
1347
|
await vault.sweeper.scan()
|
|
1260
1348
|
|
|
1261
1349
|
for (let index = 0; index < stale.length; index++) {
|
|
@@ -1338,8 +1426,10 @@ describe("Save Space engine", () => {
|
|
|
1338
1426
|
const { home, vault } = await makeVault()
|
|
1339
1427
|
const outside = await makeOutside()
|
|
1340
1428
|
const contents = crypto.randomBytes(4096)
|
|
1341
|
-
await
|
|
1342
|
-
|
|
1429
|
+
await writeCandidate(
|
|
1430
|
+
path.join(home, "api", "app", "model.bin"), contents)
|
|
1431
|
+
await writeCandidate(
|
|
1432
|
+
path.join(outside, "models", "model.bin"), contents)
|
|
1343
1433
|
await vault.sweeper.scan()
|
|
1344
1434
|
const publishedFiles = await vault.registry.countFiles()
|
|
1345
1435
|
vault.hashFile = async () => {
|
|
@@ -1387,7 +1477,7 @@ describe("Save Space engine", () => {
|
|
|
1387
1477
|
store: kernel.store
|
|
1388
1478
|
})
|
|
1389
1479
|
await replacement.init()
|
|
1390
|
-
replacement.sizeThreshold = pair.
|
|
1480
|
+
replacement.sizeThreshold = pair.size * 2
|
|
1391
1481
|
|
|
1392
1482
|
assert.deepEqual(replacement.configuredLocations(), [])
|
|
1393
1483
|
assert.equal(await replacement.globalScanReady(), false)
|
|
@@ -1401,6 +1491,33 @@ describe("Save Space engine", () => {
|
|
|
1401
1491
|
await close(replacement)
|
|
1402
1492
|
})
|
|
1403
1493
|
|
|
1494
|
+
test("deduplication refuses legacy scan results below 10 MB", async () => {
|
|
1495
|
+
const { home, vault } = await makeVault()
|
|
1496
|
+
const contents = crypto.randomBytes(4096)
|
|
1497
|
+
const pair = {
|
|
1498
|
+
first: await write(path.join(home, "api", "first", "small.bin"),
|
|
1499
|
+
contents),
|
|
1500
|
+
second: await write(path.join(home, "api", "second", "small.bin"),
|
|
1501
|
+
contents)
|
|
1502
|
+
}
|
|
1503
|
+
await vault.sweeper.scan()
|
|
1504
|
+
const duplicate = [...await vault.registry.files({
|
|
1505
|
+
statuses: ["duplicate"]
|
|
1506
|
+
})][0]
|
|
1507
|
+
const result = await vault.perform("deduplicate", {
|
|
1508
|
+
path: duplicate.path
|
|
1509
|
+
})
|
|
1510
|
+
|
|
1511
|
+
assert.equal(result.status, "unavailable")
|
|
1512
|
+
assert.equal(result.unavailable_reason, "below_minimum_size")
|
|
1513
|
+
const [first, second] = await Promise.all([
|
|
1514
|
+
fs.promises.stat(pair.first),
|
|
1515
|
+
fs.promises.stat(pair.second)
|
|
1516
|
+
])
|
|
1517
|
+
assert.notEqual(first.ino, second.ino)
|
|
1518
|
+
await close(vault)
|
|
1519
|
+
})
|
|
1520
|
+
|
|
1404
1521
|
test("explicit deduplication creates the first anchor and changes one path atomically", async () => {
|
|
1405
1522
|
const { home, vault } = await makeVault()
|
|
1406
1523
|
const pair = await duplicatePair(home)
|
|
@@ -1418,7 +1535,7 @@ describe("Save Space engine", () => {
|
|
|
1418
1535
|
path: duplicate.path
|
|
1419
1536
|
})
|
|
1420
1537
|
assert.equal(result.status, "converted")
|
|
1421
|
-
assert.equal(result.bytes_saved, pair.
|
|
1538
|
+
assert.equal(result.bytes_saved, pair.size)
|
|
1422
1539
|
|
|
1423
1540
|
const firstStat = await fs.promises.stat(pair.first)
|
|
1424
1541
|
const secondStat = await fs.promises.stat(pair.second)
|
|
@@ -1691,7 +1808,7 @@ describe("Save Space engine", () => {
|
|
|
1691
1808
|
unavailable_reason: "hardlinks"
|
|
1692
1809
|
})
|
|
1693
1810
|
assert.equal((await fs.promises.stat(duplicate.path)).ino, before.ino)
|
|
1694
|
-
|
|
1811
|
+
await assertCandidateContents(duplicate.path, pair.contents, pair.size)
|
|
1695
1812
|
const rows = await vault.registry.files({ hash: duplicate.hash })
|
|
1696
1813
|
assert.equal(rows.every((row) =>
|
|
1697
1814
|
row.status === "unavailable" &&
|
|
@@ -1739,7 +1856,7 @@ describe("Save Space engine", () => {
|
|
|
1739
1856
|
assert.equal(row.status, "unavailable")
|
|
1740
1857
|
assert.equal(row.unavailable_reason, "permission_denied")
|
|
1741
1858
|
assert.equal((await fs.promises.stat(duplicate.path)).ino, before.ino)
|
|
1742
|
-
|
|
1859
|
+
await assertCandidateContents(duplicate.path, pair.contents, pair.size)
|
|
1743
1860
|
const status = await vault.status(null, { view: "duplicates" })
|
|
1744
1861
|
const unavailable = await vault.status(null, { view: "unavailable" })
|
|
1745
1862
|
assert.equal(status.inventory.counts.duplicates, 0)
|
|
@@ -1903,7 +2020,7 @@ describe("Save Space engine", () => {
|
|
|
1903
2020
|
assert.equal(row.status, "unavailable")
|
|
1904
2021
|
assert.equal(row.unavailable_reason, "metadata")
|
|
1905
2022
|
assert.equal((await fs.promises.stat(duplicate.path)).ino, before.ino)
|
|
1906
|
-
|
|
2023
|
+
await assertCandidateContents(duplicate.path, pair.contents, pair.size)
|
|
1907
2024
|
await close(vault)
|
|
1908
2025
|
})
|
|
1909
2026
|
|
|
@@ -1950,7 +2067,7 @@ describe("Save Space engine", () => {
|
|
|
1950
2067
|
assert.equal(row.status, "unavailable")
|
|
1951
2068
|
assert.equal(row.unavailable_reason, "different_disk")
|
|
1952
2069
|
assert.equal((await fs.promises.stat(duplicate.path)).ino, before.ino)
|
|
1953
|
-
|
|
2070
|
+
await assertCandidateContents(duplicate.path, pair.contents, pair.size)
|
|
1954
2071
|
await close(vault)
|
|
1955
2072
|
})
|
|
1956
2073
|
|
|
@@ -1990,7 +2107,7 @@ describe("Save Space engine", () => {
|
|
|
1990
2107
|
assert.equal(fs.existsSync(
|
|
1991
2108
|
`${duplicate.path}.pinokio-dedup-tmp`), false)
|
|
1992
2109
|
assert.equal((await fs.promises.stat(duplicate.path)).ino, before.ino)
|
|
1993
|
-
|
|
2110
|
+
await assertCandidateContents(duplicate.path, pair.contents, pair.size)
|
|
1994
2111
|
await close(vault)
|
|
1995
2112
|
})
|
|
1996
2113
|
|
|
@@ -2009,7 +2126,8 @@ describe("Save Space engine", () => {
|
|
|
2009
2126
|
)
|
|
2010
2127
|
const anchor = vault.storePathFor(duplicate.hash)
|
|
2011
2128
|
await fs.promises.unlink(anchor)
|
|
2012
|
-
await
|
|
2129
|
+
await writeCandidate(anchor, crypto.randomBytes(pair.contents.length),
|
|
2130
|
+
pair.size)
|
|
2013
2131
|
const before = await fs.promises.stat(duplicate.path)
|
|
2014
2132
|
|
|
2015
2133
|
const result = await vault.perform("deduplicate", {
|
|
@@ -2018,10 +2136,7 @@ describe("Save Space engine", () => {
|
|
|
2018
2136
|
|
|
2019
2137
|
assert.equal(result.status, "stale")
|
|
2020
2138
|
assert.equal((await fs.promises.stat(duplicate.path)).ino, before.ino)
|
|
2021
|
-
|
|
2022
|
-
await fs.promises.readFile(duplicate.path),
|
|
2023
|
-
pair.contents
|
|
2024
|
-
)
|
|
2139
|
+
await assertCandidateContents(duplicate.path, pair.contents, pair.size)
|
|
2025
2140
|
await close(vault)
|
|
2026
2141
|
})
|
|
2027
2142
|
|
|
@@ -2098,7 +2213,7 @@ describe("Save Space engine", () => {
|
|
|
2098
2213
|
assert.equal(matching.inventory.current.separate_count, 2)
|
|
2099
2214
|
assert.equal(
|
|
2100
2215
|
matching.inventory.current.separate_bytes,
|
|
2101
|
-
firstPair.
|
|
2216
|
+
firstPair.size * 2
|
|
2102
2217
|
)
|
|
2103
2218
|
|
|
2104
2219
|
const result = await vault.perform("separate_all", {
|
|
@@ -2161,11 +2276,11 @@ describe("Save Space engine", () => {
|
|
|
2161
2276
|
await vault.sweeper.scan()
|
|
2162
2277
|
let status = await vault.status()
|
|
2163
2278
|
|
|
2164
|
-
assert.equal(status.bytes_without_sharing, pair.
|
|
2165
|
-
assert.equal(status.logical_bytes, pair.
|
|
2279
|
+
assert.equal(status.bytes_without_sharing, pair.size * 2)
|
|
2280
|
+
assert.equal(status.logical_bytes, pair.size * 2)
|
|
2166
2281
|
assert.equal(status.saved_by_sharing, 0)
|
|
2167
|
-
assert.equal(status.bytes_on_disk, pair.
|
|
2168
|
-
assert.equal(status.pending_bytes, pair.
|
|
2282
|
+
assert.equal(status.bytes_on_disk, pair.size * 2)
|
|
2283
|
+
assert.equal(status.pending_bytes, pair.size)
|
|
2169
2284
|
assert.equal(Object.hasOwn(status, "shared_logical_bytes"), false)
|
|
2170
2285
|
let appStatus = await vault.status("app:first")
|
|
2171
2286
|
assert.equal(appStatus.shared_logical_bytes, 0)
|
|
@@ -2175,21 +2290,21 @@ describe("Save Space engine", () => {
|
|
|
2175
2290
|
})][0]
|
|
2176
2291
|
await vault.perform("deduplicate", { path: duplicate.path })
|
|
2177
2292
|
status = await vault.status()
|
|
2178
|
-
assert.equal(status.saved_by_sharing, pair.
|
|
2179
|
-
assert.equal(status.logical_bytes, pair.
|
|
2180
|
-
assert.equal(status.bytes_on_disk, pair.
|
|
2293
|
+
assert.equal(status.saved_by_sharing, pair.size)
|
|
2294
|
+
assert.equal(status.logical_bytes, pair.size * 2)
|
|
2295
|
+
assert.equal(status.bytes_on_disk, pair.size)
|
|
2181
2296
|
assert.equal(status.pending_bytes, 0)
|
|
2182
2297
|
appStatus = await vault.status("app:first", {
|
|
2183
2298
|
view: "duplicates",
|
|
2184
2299
|
query: "does-not-match"
|
|
2185
2300
|
})
|
|
2186
|
-
assert.equal(appStatus.shared_logical_bytes, pair.
|
|
2301
|
+
assert.equal(appStatus.shared_logical_bytes, pair.size)
|
|
2187
2302
|
|
|
2188
2303
|
await vault.perform("separate_files", { paths: [duplicate.path] })
|
|
2189
2304
|
status = await vault.status()
|
|
2190
2305
|
assert.equal(status.saved_by_sharing, 0)
|
|
2191
|
-
assert.equal(status.bytes_on_disk, pair.
|
|
2192
|
-
assert.equal(status.pending_bytes, pair.
|
|
2306
|
+
assert.equal(status.bytes_on_disk, pair.size * 2)
|
|
2307
|
+
assert.equal(status.pending_bytes, pair.size)
|
|
2193
2308
|
await close(vault)
|
|
2194
2309
|
})
|
|
2195
2310
|
|
|
@@ -2254,7 +2369,7 @@ describe("Save Space engine", () => {
|
|
|
2254
2369
|
})).status, "detached")
|
|
2255
2370
|
assert.notEqual((await fs.promises.stat(duplicate.path)).ino, linked.ino)
|
|
2256
2371
|
assert.equal((await vault.registry.getFile(duplicate.path)).status, "duplicate")
|
|
2257
|
-
|
|
2372
|
+
await assertCandidateContents(duplicate.path, pair.contents, pair.size)
|
|
2258
2373
|
assert.equal((await vault.perform("review_again", {
|
|
2259
2374
|
path: duplicate.path
|
|
2260
2375
|
})).error, "unknown action")
|
package/test/vault-sweep.test.js
CHANGED
|
@@ -6,6 +6,7 @@ const os = require("node:os")
|
|
|
6
6
|
const path = require("node:path")
|
|
7
7
|
const Vault = require("../kernel/vault")
|
|
8
8
|
const RegistryCore = require("../kernel/vault/registry_core")
|
|
9
|
+
const { MINIMUM_CANDIDATE_SIZE } = require("../kernel/vault/constants")
|
|
9
10
|
const { statMany } = require("../kernel/vault/walker")
|
|
10
11
|
|
|
11
12
|
const homes = []
|
|
@@ -20,6 +21,12 @@ const write = async (filePath, contents) => {
|
|
|
20
21
|
return filePath
|
|
21
22
|
}
|
|
22
23
|
|
|
24
|
+
const writeCandidate = async (filePath, contents) => {
|
|
25
|
+
await write(filePath, contents)
|
|
26
|
+
await fs.promises.truncate(filePath, MINIMUM_CANDIDATE_SIZE)
|
|
27
|
+
return filePath
|
|
28
|
+
}
|
|
29
|
+
|
|
23
30
|
const makeVault = async (threshold = 1) => {
|
|
24
31
|
const home = await fs.promises.mkdtemp(
|
|
25
32
|
path.join(os.tmpdir(), "pinokio-vault-scan-"))
|
|
@@ -455,7 +462,7 @@ describe("Save Space scans", () => {
|
|
|
455
462
|
assert.equal((await vault.registry.getFile(linked)).status, "linked")
|
|
456
463
|
})
|
|
457
464
|
|
|
458
|
-
test("
|
|
465
|
+
test("a low-threshold scan excludes empty files and retains more than one page", async () => {
|
|
459
466
|
const { home, vault } = await makeVault(0)
|
|
460
467
|
const root = path.join(home, "api", "many")
|
|
461
468
|
const paths = []
|
|
@@ -490,7 +497,7 @@ describe("Save Space scans", () => {
|
|
|
490
497
|
.concat(secondPage.items.map((item) => item.path))).size, paths.length)
|
|
491
498
|
})
|
|
492
499
|
|
|
493
|
-
test("
|
|
500
|
+
test("a low-threshold scan sends only matching size groups to hash work", async () => {
|
|
494
501
|
const { home, vault } = await makeVault(0)
|
|
495
502
|
const root = path.join(home, "api", "windows")
|
|
496
503
|
for (let index = 1; index <= 300; index++) {
|
|
@@ -676,9 +683,10 @@ describe("Save Space scans", () => {
|
|
|
676
683
|
test("an anchor sharing a scanned inode is not counted as a second read", async () => {
|
|
677
684
|
const { home, vault } = await makeVault()
|
|
678
685
|
const contents = crypto.randomBytes(4096)
|
|
679
|
-
const first = await
|
|
686
|
+
const first = await writeCandidate(
|
|
680
687
|
path.join(home, "api", "one", "model.bin"), contents)
|
|
681
|
-
await
|
|
688
|
+
await writeCandidate(
|
|
689
|
+
path.join(home, "api", "two", "model.bin"), contents)
|
|
682
690
|
await vault.sweeper.scan()
|
|
683
691
|
const duplicate = [...await vault.registry.files({
|
|
684
692
|
statuses: ["duplicate"]
|
|
@@ -703,10 +711,11 @@ describe("Save Space scans", () => {
|
|
|
703
711
|
assert.equal(reads, 1)
|
|
704
712
|
assert.equal(vault.sweeper.state.hash_work_files, 1)
|
|
705
713
|
assert.equal(vault.sweeper.state.hash_files_completed, 1)
|
|
706
|
-
assert.equal(vault.sweeper.state.hash_work_bytes,
|
|
714
|
+
assert.equal(vault.sweeper.state.hash_work_bytes,
|
|
715
|
+
MINIMUM_CANDIDATE_SIZE)
|
|
707
716
|
assert.equal(
|
|
708
717
|
vault.sweeper.state.hash_bytes_completed,
|
|
709
|
-
|
|
718
|
+
MINIMUM_CANDIDATE_SIZE
|
|
710
719
|
)
|
|
711
720
|
assert.equal((await vault.registry.getFile(first)).hash, duplicate.hash)
|
|
712
721
|
})
|
|
@@ -1375,8 +1384,10 @@ describe("Save Space scans", () => {
|
|
|
1375
1384
|
test("an anchor fallback does not change the discovered hash workload", async () => {
|
|
1376
1385
|
const { home, vault } = await makeVault()
|
|
1377
1386
|
const contents = crypto.randomBytes(4096)
|
|
1378
|
-
await
|
|
1379
|
-
|
|
1387
|
+
await writeCandidate(
|
|
1388
|
+
path.join(home, "api", "one", "model.bin"), contents)
|
|
1389
|
+
await writeCandidate(
|
|
1390
|
+
path.join(home, "api", "two", "model.bin"), contents)
|
|
1380
1391
|
await vault.sweeper.scan()
|
|
1381
1392
|
const duplicate = [...await vault.registry.files({
|
|
1382
1393
|
statuses: ["duplicate"]
|
|
@@ -1429,7 +1440,7 @@ describe("Save Space scans", () => {
|
|
|
1429
1440
|
const progress = vault.scanStatus()
|
|
1430
1441
|
assert.equal(progress.hash_work_files, 1)
|
|
1431
1442
|
assert.equal(progress.hash_files_completed, 0)
|
|
1432
|
-
assert.equal(progress.hash_work_bytes,
|
|
1443
|
+
assert.equal(progress.hash_work_bytes, MINIMUM_CANDIDATE_SIZE)
|
|
1433
1444
|
assert.equal(progress.hash_bytes_completed, 0)
|
|
1434
1445
|
} finally {
|
|
1435
1446
|
releaseAnchor()
|
|
@@ -1445,10 +1456,11 @@ describe("Save Space scans", () => {
|
|
|
1445
1456
|
assert.equal(result.outcome, "completed_with_exclusions")
|
|
1446
1457
|
assert.equal(vault.sweeper.state.hash_work_files, 1)
|
|
1447
1458
|
assert.equal(vault.sweeper.state.hash_files_completed, 1)
|
|
1448
|
-
assert.equal(vault.sweeper.state.hash_work_bytes,
|
|
1459
|
+
assert.equal(vault.sweeper.state.hash_work_bytes,
|
|
1460
|
+
MINIMUM_CANDIDATE_SIZE)
|
|
1449
1461
|
assert.equal(
|
|
1450
1462
|
vault.sweeper.state.hash_bytes_completed,
|
|
1451
|
-
|
|
1463
|
+
MINIMUM_CANDIDATE_SIZE
|
|
1452
1464
|
)
|
|
1453
1465
|
})
|
|
1454
1466
|
|
|
@@ -1602,12 +1614,15 @@ describe("Save Space scans", () => {
|
|
|
1602
1614
|
test("a missing database is rebuilt by a normal scan, including managed links below the threshold", async () => {
|
|
1603
1615
|
const { home, vault, store } = await makeVault()
|
|
1604
1616
|
const contents = crypto.randomBytes(4096)
|
|
1605
|
-
await
|
|
1606
|
-
|
|
1617
|
+
await writeCandidate(
|
|
1618
|
+
path.join(home, "api", "one", "model.bin"), contents)
|
|
1619
|
+
await writeCandidate(
|
|
1620
|
+
path.join(home, "api", "two", "model.bin"), contents)
|
|
1607
1621
|
await vault.sweeper.scan()
|
|
1608
1622
|
const duplicate = [...await vault.registry.files({
|
|
1609
1623
|
statuses: ["duplicate"]
|
|
1610
1624
|
})][0]
|
|
1625
|
+
const hash = duplicate.hash
|
|
1611
1626
|
await vault.perform("deduplicate", { path: duplicate.path })
|
|
1612
1627
|
const managedPath = duplicate.path
|
|
1613
1628
|
|
|
@@ -1619,7 +1634,7 @@ describe("Save Space scans", () => {
|
|
|
1619
1634
|
store
|
|
1620
1635
|
})
|
|
1621
1636
|
await replacement.init()
|
|
1622
|
-
replacement.sizeThreshold =
|
|
1637
|
+
replacement.sizeThreshold = MINIMUM_CANDIDATE_SIZE * 2
|
|
1623
1638
|
vaults.push(replacement)
|
|
1624
1639
|
assert.equal(await replacement.registry.countFiles(), 0)
|
|
1625
1640
|
|
|
@@ -1628,21 +1643,23 @@ describe("Save Space scans", () => {
|
|
|
1628
1643
|
const rebuilt = await replacement.registry.getFile(managedPath)
|
|
1629
1644
|
assert.ok(rebuilt)
|
|
1630
1645
|
assert.equal(rebuilt.status, "linked")
|
|
1631
|
-
assert.equal(rebuilt.hash,
|
|
1646
|
+
assert.equal(rebuilt.hash, hash)
|
|
1632
1647
|
})
|
|
1633
1648
|
|
|
1634
1649
|
test("managed links remain tracked below the threshold if their anchor was removed externally", async () => {
|
|
1635
1650
|
const { home, vault } = await makeVault()
|
|
1636
1651
|
const contents = crypto.randomBytes(4096)
|
|
1637
|
-
await
|
|
1638
|
-
|
|
1652
|
+
await writeCandidate(
|
|
1653
|
+
path.join(home, "api", "one", "model.bin"), contents)
|
|
1654
|
+
await writeCandidate(
|
|
1655
|
+
path.join(home, "api", "two", "model.bin"), contents)
|
|
1639
1656
|
await vault.sweeper.scan()
|
|
1640
1657
|
const duplicate = [...await vault.registry.files({
|
|
1641
1658
|
statuses: ["duplicate"]
|
|
1642
1659
|
})][0]
|
|
1643
1660
|
await vault.perform("deduplicate", { path: duplicate.path })
|
|
1644
1661
|
await fs.promises.unlink(vault.storePathFor(duplicate.hash))
|
|
1645
|
-
vault.sizeThreshold =
|
|
1662
|
+
vault.sizeThreshold = MINIMUM_CANDIDATE_SIZE * 2
|
|
1646
1663
|
|
|
1647
1664
|
await vault.sweeper.scan()
|
|
1648
1665
|
|
|
@@ -1744,11 +1761,11 @@ describe("Save Space scans", () => {
|
|
|
1744
1761
|
}
|
|
1745
1762
|
const { home, vault } = await makeVault()
|
|
1746
1763
|
const contents = crypto.randomBytes(4096)
|
|
1747
|
-
const incompatible = await
|
|
1764
|
+
const incompatible = await writeCandidate(
|
|
1748
1765
|
path.join(home, "api", "a", "model.bin"), contents)
|
|
1749
|
-
const compatibleOne = await
|
|
1766
|
+
const compatibleOne = await writeCandidate(
|
|
1750
1767
|
path.join(home, "api", "b", "model.bin"), contents)
|
|
1751
|
-
const compatibleTwo = await
|
|
1768
|
+
const compatibleTwo = await writeCandidate(
|
|
1752
1769
|
path.join(home, "api", "c", "model.bin"), contents)
|
|
1753
1770
|
await fs.promises.chmod(incompatible, 0o600)
|
|
1754
1771
|
await fs.promises.chmod(compatibleOne, 0o644)
|
package/test/vault-ui.test.js
CHANGED
|
@@ -520,8 +520,9 @@ describe("Save Space interface", () => {
|
|
|
520
520
|
assert.doesNotMatch(combined,
|
|
521
521
|
/Keep separate|Kept separate|Review again|review_again/)
|
|
522
522
|
assert.doesNotMatch(combined, /\bundo\b/i)
|
|
523
|
-
assert.match(combined,
|
|
524
|
-
|
|
523
|
+
assert.match(combined,
|
|
524
|
+
/const candidateSizeOptions = \[10, 50, 100, 500\]/)
|
|
525
|
+
assert.doesNotMatch(combined, /const candidateSizeOptions = \[0\]/)
|
|
525
526
|
assert.match(combined, /The rest of the scan completed\./)
|
|
526
527
|
assert.doesNotMatch(combined, /Previous completed results were kept\./)
|
|
527
528
|
assert.match(combined, /Scan completed with exclusions/)
|
|
@@ -617,11 +618,11 @@ describe("Save Space interface", () => {
|
|
|
617
618
|
const globalChart = globalDocument.querySelector(".vault-storage-chart")
|
|
618
619
|
|
|
619
620
|
assert.match(globalHeadline.textContent, /saved by deduplicating/)
|
|
620
|
-
assert.match(globalChart.getAttribute("aria-label"), /
|
|
621
|
+
assert.match(globalChart.getAttribute("aria-label"), /In use:/)
|
|
621
622
|
assert.match(globalChart.getAttribute("aria-label"), /Saved:/)
|
|
622
623
|
assert.match(globalChart.getAttribute("aria-label"), /Can save:/)
|
|
623
624
|
assert.match(globalDocument.querySelector(
|
|
624
|
-
".vault-storage-key.occupied").textContent, /
|
|
625
|
+
".vault-storage-key.occupied").textContent, /In use/)
|
|
625
626
|
assert.match(globalDocument.querySelector(
|
|
626
627
|
".vault-storage-key.optimized").textContent, /Saved/)
|
|
627
628
|
assert.match(globalDocument.querySelector(
|
|
@@ -747,9 +748,9 @@ describe("Save Space interface", () => {
|
|
|
747
748
|
assert.match(document.getElementById("btn-scan").textContent,
|
|
748
749
|
/Scan again/)
|
|
749
750
|
assert.equal(requests.some((request) => request.action === "scan"), false)
|
|
750
|
-
document.querySelector('[data-candidate-size="0"]')
|
|
751
|
-
assert.
|
|
752
|
-
|
|
751
|
+
assert.equal(document.querySelector('[data-candidate-size="0"]'), null)
|
|
752
|
+
assert.equal(document.querySelector(
|
|
753
|
+
`[data-candidate-size="${candidateBase ** 2}"]`), null)
|
|
753
754
|
assert.equal(requests.some((request) => request.action === "scan"), false)
|
|
754
755
|
document.querySelector(`[data-candidate-size="${100 * candidateBase ** 2}"]`).click()
|
|
755
756
|
assert.ok(document.querySelector('.vault-all-locations[data-source=""]'))
|
|
@@ -892,8 +893,8 @@ describe("Save Space interface", () => {
|
|
|
892
893
|
|
|
893
894
|
test("an app displays and immediately saves its own minimum size", async () => {
|
|
894
895
|
const candidateBase = process.platform === "win32" ? 1024 : 1000
|
|
895
|
-
const appMinimum = candidateBase ** 2
|
|
896
|
-
const globalMinimum =
|
|
896
|
+
const appMinimum = 10 * candidateBase ** 2
|
|
897
|
+
const globalMinimum = 100 * candidateBase ** 2
|
|
897
898
|
const selected = 50 * candidateBase ** 2
|
|
898
899
|
const { dom, requests } = await makePage(fixture([], {
|
|
899
900
|
candidate_min_bytes: appMinimum,
|