pinokiod 8.0.37 → 8.0.39

package/test/vault-engine.test.js

@@ -188,6 +188,11 @@ describe('vault engine (phase 1)', () => {
     assert.strictEqual(fresh.registry.blobs.get(hash).orphan, false)
     assert.strictEqual(fresh.registry.links.get(a).hash, hash)
     assert.strictEqual(fresh.registry.links.get(a).app, 'appA')
+    assert.strictEqual(
+      fresh.registry.links.has(fresh.storePathFor(hash)),
+      false,
+      'the internal vault name is never exposed as a tracked location'
+    )
   })
 
   test('item 9: torn events line tolerated; corrupt registry.json triggers rebuild, not failure', async () => {
@@ -224,7 +229,31 @@ describe('vault engine (phase 1)', () => {
     assert.strictEqual(fs.existsSync(path.resolve(vault.root, 'registry.json.tmp')), false)
   })
 
-  test('rebuild walk prunes noise dirs and skips symlinks', async () => {
+  test('manual index repair preserves user decisions, history, and pending review state', async () => {
+    const h = await home()
+    const vault = await makeVault(h)
+    const content = crypto.randomBytes(4096)
+    const hash = sha256(content)
+    const original = await writeFile(path.resolve(h, 'api', 'appA', 'm.bin'), content)
+    const pending = await writeFile(path.resolve(h, 'api', 'appB', 'm.bin'), content)
+    const independent = await writeFile(path.resolve(h, 'cache', 'keep.bin'), crypto.randomBytes(2048))
+    await vault.adopt(original, hash, { app: 'appA', source_urls: ['https://example.test/model'] })
+    vault.registry.duplicates.set(pending, { hash, size: content.length, app: 'appB', discovered: Date.now() })
+    vault.registry.excluded.set(independent, { ts: Date.now(), size: 2048 })
+    vault.registry.totals.lifetime_bytes_saved = 12345
+    vault.registry.lastScan = { ts: 111, dirs: 2, files: 3, bytes_total: 999 }
+
+    await vault.rebuild([])
+
+    assert.ok(vault.registry.excluded.has(independent), 'keep-independent decision survives')
+    assert.strictEqual(vault.registry.excluded.get(independent).size, 2048)
+    assert.strictEqual(vault.registry.totals.lifetime_bytes_saved, 12345)
+    assert.deepStrictEqual(vault.registry.lastScan, { ts: 111, dirs: 2, files: 3, bytes_total: 999 })
+    assert.ok(vault.registry.duplicates.has(pending), 'pending review item survives when still valid')
+    assert.deepStrictEqual(vault.registry.blobs.get(hash).source_urls, ['https://example.test/model'])
+  })
+
+  test('rebuild walks every directory without name heuristics and still skips symlinks', async () => {
     const h = await home()
     const vault = await makeVault(h)
     const big = Buffer.alloc(Vault.SIZE_THRESHOLD, 3)
@@ -234,19 +263,77 @@ describe('vault engine (phase 1)', () => {
     await fs.promises.unlink(path.resolve(vault.root, 'registry.json')).catch(() => {})
     const fresh = await makeVault(h)
     await fresh.rebuild()
-    // Blob is found (store side), but the env/ path must NOT be re-associated.
     assert.ok(fresh.registry.blobs.has(hash))
-    assert.strictEqual(fresh.registry.links.has(inEnv), false)
+    assert.strictEqual(fresh.registry.links.has(inEnv), true, 'directory names never hide tracked files')
+  })
+
+  test('hash worker computes sha256 and is reused across a scan batch', async () => {
+    const h = await home()
+    const vault = await makeVault(h)
+    const first = crypto.randomBytes(1024 * 1024)
+    const second = crypto.randomBytes(1024 * 1024)
+    const firstFile = await writeFile(path.resolve(h, 'api', 'appA', 'x.bin'), first)
+    const secondFile = await writeFile(path.resolve(h, 'api', 'appA', 'y.bin'), second)
+    const firstResult = await vault.hashFile(firstFile)
+    const secondResult = await vault.hashFile(secondFile)
+    assert.strictEqual(firstResult.hash, sha256(first))
+    assert.strictEqual(firstResult.size, first.length)
+    assert.strictEqual(secondResult.hash, sha256(second))
+    assert.strictEqual(vault.workerStarts, 1, 'worker startup is amortized across files')
+  })
+
+  test('unsupported conversion leaves the independent copy pending and unregistered', async () => {
+    const h = await home()
+    const vault = await makeVault(h)
+    const content = crypto.randomBytes(4096)
+    const hash = sha256(content)
+    const original = await writeFile(path.resolve(h, 'api', 'appA', 'm.bin'), content)
+    const pending = await writeFile(path.resolve(h, 'api', 'appB', 'm.bin'), content)
+    await vault.adopt(original, hash, { app: 'appA' })
+    const realLink = fs.promises.link
+    fs.promises.link = async (source, target) => {
+      if (target === pending + Vault.TMP_SUFFIX) {
+        const error = new Error('different volume')
+        error.code = 'EXDEV'
+        throw error
+      }
+      return realLink(source, target)
+    }
+    try {
+      const result = await vault.convert(pending, hash, { app: 'appB' })
+      assert.strictEqual(result.status, 'unavailable')
+    } finally {
+      fs.promises.link = realLink
+    }
+    assert.strictEqual(vault.registry.links.has(pending), false)
+    assert.deepStrictEqual(await fs.promises.readFile(pending), content)
   })
 
-  test('hash worker computes correct sha256', async () => {
+  test('permission errors during conversion are retryable locks, not unsupported disks', async () => {
     const h = await home()
     const vault = await makeVault(h)
-    const content = crypto.randomBytes(1024 * 1024)
-    const file = await writeFile(path.resolve(h, 'api', 'appA', 'x.bin'), content)
-    const result = await vault.hashFile(file)
-    assert.strictEqual(result.hash, sha256(content))
-    assert.strictEqual(result.size, content.length)
+    const content = crypto.randomBytes(4096)
+    const hash = sha256(content)
+    const original = await writeFile(path.resolve(h, 'api', 'appA', 'm.bin'), content)
+    const pending = await writeFile(path.resolve(h, 'api', 'appB', 'm.bin'), content)
+    await vault.adopt(original, hash, { app: 'appA' })
+    const realLink = fs.promises.link
+    fs.promises.link = async (source, target) => {
+      if (target === pending + Vault.TMP_SUFFIX) {
+        const error = new Error('file is in use')
+        error.code = 'EPERM'
+        throw error
+      }
+      return realLink(source, target)
+    }
+    try {
+      const result = await vault.convert(pending, hash, { app: 'appB' })
+      assert.strictEqual(result.status, 'locked')
+    } finally {
+      fs.promises.link = realLink
+    }
+    assert.strictEqual(vault.registry.links.has(pending), false)
+    assert.deepStrictEqual(await fs.promises.readFile(pending), content)
   })
 
   test('item 11: exFAT volume enters copy mode (macOS loopback; skipped elsewhere)', { skip: process.platform !== 'darwin' }, async (t) => {

package/test/vault-sweep.test.js

@@ -47,6 +47,11 @@ describe('vault manual scan (phase 3)', () => {
     assert.ok(vault.registry.blobs.has(sha256(content)))
     assert.ok(result.bytes_total >= content.length + 4, 'folder total includes small files')
     assert.ok(vault.registry.lastScan && vault.registry.lastScan.bytes_total === result.bytes_total)
+    assert.ok(vault.registry.lastScan.duration_ms >= 0, 'scan duration is measured')
+    assert.ok(vault.registry.lastScan.walk_duration_ms >= 0, 'walk duration is measured')
+    assert.ok(vault.registry.lastScan.hash_duration_ms >= 0, 'hash duration is measured')
+    assert.strictEqual(vault.registry.lastScan.hash_total, 1, 'hash work has a determinate total after walking')
+    assert.deepStrictEqual(vault.registry.lastScan.source_ids, ['pinokio'], 'scan source set is recorded for later estimates')
   })
 
   test('scans NEVER convert: every byte-identical copy is pending, even in HF caches', async () => {
@@ -123,6 +128,28 @@ describe('vault manual scan (phase 3)', () => {
     assert.strictEqual(status.last_scan.source_bytes[source.id], content.length, 'external totals remain attributable')
   })
 
+  test('adding an external source creates one persistent import without starting a scan', async () => {
+    const { home, vault } = await makeEnv()
+    const external = await fs.promises.mkdtemp(path.resolve(os.tmpdir(), 'pinokio-added-external-'))
+    homes.push(external)
+    const content = crypto.randomBytes(4096)
+    const imported = await writeFile(path.resolve(external, 'models', 'added.bin'), content)
+
+    const added = await vault.addExternalSource(external)
+    assert.strictEqual(added.created, true)
+    assert.strictEqual(added.source.kind, 'external')
+    assert.strictEqual(added.source.parent_id, 'external')
+    assert.strictEqual(vault.registry.scanIndex.size, 0, 'adding a source never scans it')
+    assert.strictEqual((await fs.promises.lstat(added.source.mount_path)).isSymbolicLink(), true)
+
+    const repeated = await vault.addExternalSource(external)
+    assert.strictEqual(repeated.created, false, 'the same physical folder is not imported twice')
+    assert.strictEqual(vault.sources().filter((source) => source.kind === 'external').length, 1)
+
+    await vault.sweeper.scan()
+    assert.ok(vault.registry.scanIndex.has(path.resolve(await fs.promises.realpath(imported))))
+  })
+
   test('excluded paths are respected: no adoption, no pending, no re-listing', async () => {
     const { home, vault } = await makeEnv()
     const content = crypto.randomBytes(4096)
@@ -140,8 +167,84 @@ describe('vault manual scan (phase 3)', () => {
     await writeFile(path.resolve(home, 'api', 'appA', 'm.bin'), crypto.randomBytes(4096))
     await vault.sweeper.scan()
     const after = vault.sweeper.stats.hashes
+    const priorFiles = vault.registry.lastScan.files
     await vault.sweeper.scan()
     assert.strictEqual(vault.sweeper.stats.hashes, after)
+    assert.strictEqual(vault.sweeper.state.estimated_files, priorFiles, 'same-source rescan uses the prior exact file count')
+  })
+
+  test('file metadata inspection uses bounded concurrency without changing results', async () => {
+    const { home, vault } = await makeEnv()
+    const dir = path.resolve(home, 'api', 'bulk')
+    await Promise.all(Array.from({ length: 48 }, (_, index) =>
+      writeFile(path.resolve(dir, `dir-${index}`, 'small.txt'), 'x')))
+    const realStat = fs.promises.stat
+    let active = 0
+    let maximum = 0
+    fs.promises.stat = async (target) => {
+      if (String(target).startsWith(dir + path.sep)) {
+        active += 1
+        maximum = Math.max(maximum, active)
+        await new Promise((resolve) => setTimeout(resolve, 3))
+        try {
+          return await realStat(target)
+        } finally {
+          active -= 1
+        }
+      }
+      return realStat(target)
+    }
+    try {
+      await vault.sweeper.scan()
+    } finally {
+      fs.promises.stat = realStat
+    }
+    assert.ok(maximum > 1, `expected concurrent metadata reads, saw ${maximum}`)
+    assert.ok(maximum <= vault.statConcurrency, `metadata concurrency exceeded its bound: ${maximum}`)
+    assert.strictEqual(vault.registry.lastScan.files >= 48, true)
+  })
+
+  test('multiple names for one inode share one exact hash job', async () => {
+    const { home, vault } = await makeEnv()
+    const content = crypto.randomBytes(4096)
+    const a = await writeFile(path.resolve(home, 'api', 'appA', 'model.bin'), content)
+    const b = path.resolve(home, 'api', 'appB', 'model.bin')
+    await fs.promises.mkdir(path.dirname(b), { recursive: true })
+    await fs.promises.link(a, b)
+
+    await vault.sweeper.scan()
+
+    assert.strictEqual(vault.registry.lastScan.hashed, 1, 'physical content was hashed once')
+    assert.strictEqual(vault.registry.lastScan.inode_reuses, 1, 'second name reused the inode hash job')
+    assert.strictEqual(vault.registry.duplicates.size, 0, 'existing hard links are already shared')
+    assert.strictEqual(vault.registry.links.get(a).hash, sha256(content))
+    assert.strictEqual(vault.registry.links.get(b).hash, sha256(content))
+  })
+
+  test('a file changed during hashing never publishes the stale digest', async () => {
+    const { home, vault } = await makeEnv()
+    const original = crypto.randomBytes(4096)
+    const oldHash = sha256(original)
+    const file = await writeFile(path.resolve(home, 'api', 'appA', 'changing.bin'), original)
+    const realHashFile = vault.hashFile.bind(vault)
+    vault.hashFile = async (target) => {
+      const result = await realHashFile(target)
+      await fs.promises.writeFile(target, crypto.randomBytes(original.length))
+      const future = new Date(Date.now() + 2000)
+      await fs.promises.utimes(target, future, future)
+      return result
+    }
+
+    try {
+      await vault.sweeper.scan()
+    } finally {
+      vault.hashFile = realHashFile
+    }
+
+    assert.strictEqual(vault.registry.lastScan.unstable_hashes, 1)
+    assert.strictEqual(vault.registry.blobs.has(oldHash), false, 'stale digest was discarded')
+    assert.strictEqual(vault.registry.scanIndex.has(file), false, 'unstable path remains unclassified')
+    assert.strictEqual((await fs.promises.stat(file)).nlink, 1, 'unstable file was not adopted')
   })
 
   test('conversion tmp files are never candidates', async () => {

package/test/vault-ui.test.js

@@ -67,6 +67,7 @@ describe('vault dashboard backend (phase 4)', () => {
     assert.deepStrictEqual(await fs.promises.readFile(b), content)
     assert.strictEqual((await fs.promises.stat(a)).nlink, 2)
     assert.ok(vault.registry.excluded.has(b))
+    assert.strictEqual(vault.registry.excluded.get(b).size, content.length)
     assert.strictEqual(fs.existsSync(b + '.pinokio-dedup-tmp'), false)
 
     // A scan neither re-adopts nor re-lists the pinned path.
@@ -95,6 +96,7 @@ describe('vault dashboard backend (phase 4)', () => {
     assert.strictEqual(result.status, 'ignored')
     assert.strictEqual(vault.registry.duplicates.has(b), false)
     assert.ok(vault.registry.excluded.has(b))
+    assert.strictEqual(vault.registry.excluded.get(b).size, content.length)
   })
 
   test('status(): disk figures, scan metadata, excluded list', async () => {
@@ -107,6 +109,7 @@ describe('vault dashboard backend (phase 4)', () => {
     assert.strictEqual(status.bytes_on_disk, content.length)
     assert.ok(status.last_scan && status.last_scan.files > 0)
     assert.strictEqual(status.excluded.length, 1)
+    assert.strictEqual(status.excluded[0].size, content.length)
     assert.strictEqual(status.blobs.length, 1)
     assert.deepStrictEqual(status.blobs[0].apps, ['appA'])
     assert.ok(status.scan && status.scan.active === false)
@@ -149,6 +152,94 @@ describe('vault dashboard backend (phase 4)', () => {
     assert.doesNotMatch(style[0], /text-overflow:\s*ellipsis/)
   })
 
+  test('copy-mode tracking never claims physical sharing savings', async () => {
+    const { home, vault } = await makeEnv()
+    const content = crypto.randomBytes(4096)
+    const hash = sha256(content)
+    const first = await writeFile(path.resolve(home, 'api', 'appA', 'm.bin'), content)
+    const second = await writeFile(path.resolve(home, 'api', 'appB', 'm.bin'), content)
+    const firstStat = await fs.promises.stat(first)
+    const secondStat = await fs.promises.stat(second)
+    vault.registry.addBlob(hash, { size: content.length })
+    vault.registry.addLink(first, { hash, app: 'appA', dev: firstStat.dev, ino: firstStat.ino, mode: 'copy' })
+    vault.registry.addLink(second, { hash, app: 'appB', dev: secondStat.dev, ino: secondStat.ino, mode: 'copy' })
+
+    const status = await vault.status()
+    assert.strictEqual(status.bytes_on_disk, content.length * 2)
+    assert.strictEqual(status.saved_by_sharing, 0)
+  })
+
+  test('repair is advanced, metrics distinguish detected and explicit savings, and refresh retries', async () => {
+    const vaultPage = await fs.promises.readFile(path.resolve(__dirname, '..', 'server', 'views', 'vault.ejs'), 'utf8')
+    assert.match(vaultPage, /already_shared:\s*"Already shared"/)
+    assert.match(vaultPage, /saved_by_vault:\s*"Saved by Vault"/)
+    assert.match(vaultPage, /repair_index:\s*"Repair Vault index"/)
+    assert.match(vaultPage, /<details class='vault-advanced'/)
+    assert.doesNotMatch(vaultPage, /id='btn-rebuild'/)
+    const refreshBlock = vaultPage.match(/const refresh = async \(\) => \{[\s\S]*?\n\}/)
+    assert.ok(refreshBlock)
+    assert.match(refreshBlock[0], /finally/)
+    assert.doesNotMatch(vaultPage, /priorScanning/)
+  })
+
+  test('vault uses compact desktop density without truncating duplicate matches', async () => {
+    const vaultPage = await fs.promises.readFile(path.resolve(__dirname, '..', 'server', 'views', 'vault.ejs'), 'utf8')
+    assert.match(vaultPage, /--vault-control-height:\s*28px/)
+    assert.match(vaultPage, /--vault-row-height:\s*44px/)
+    assert.match(vaultPage, /--vault-tree-row:\s*26px/)
+    assert.match(vaultPage, /--vault-tree-path-row:\s*34px/)
+    assert.match(vaultPage, /\.vault-shell \*,?[\s\S]*?box-sizing:\s*border-box/)
+    assert.match(vaultPage, /grid-template-columns:\s*248px minmax\(0, 1fr\)/)
+    assert.match(vaultPage, /\.vault-toolbar \{[\s\S]*?min-height:\s*44px/)
+    assert.match(vaultPage, /\.vault-match-path \{[\s\S]*?overflow-wrap:\s*anywhere/)
+    assert.doesNotMatch(vaultPage, /<header class='task-shell-header'>/)
+    assert.doesNotMatch(vaultPage, /id='vault-(?:title|subtitle)'/)
+    assert.match(vaultPage, /\.vault-overview \{[\s\S]*?padding:\s*7px var\(--vault-inline\)/)
+    assert.match(vaultPage, /\.vault-metric \{[\s\S]*?border:\s*0;[\s\S]*?border-left:\s*1px solid var\(--task-border\)/)
+    assert.match(vaultPage, /button\.vault-metric \{[\s\S]*?font:\s*inherit/)
+    assert.match(vaultPage, /id='btn-add-source'/)
+    assert.match(vaultPage, /<script src="\/Socket\.js"><\/script>/)
+    assert.match(vaultPage, /action:\s*"add_source"/)
+    assert.match(vaultPage, /identical_contents_at:\s*"Identical contents at"/)
+    assert.doesNotMatch(vaultPage, /Matching locations/)
+    assert.match(vaultPage, /scan_waiting:\s*"Waiting for scan results"/)
+    assert.match(vaultPage, /view === "all" && !scanning \? `<button class="vault-button" type="button" id="btn-empty-scan"/)
+    assert.match(vaultPage, /class="vault-progress-track"/)
+    assert.match(vaultPage, /role="progressbar"/)
+    assert.match(vaultPage, /scan\.walk_duration_ms == null/)
+    assert.match(vaultPage, /const hashTotal = scan\.hash_total/)
+    assert.match(vaultPage, /const hasEstimate = Number\.isFinite\(scan\.estimated_files\)/)
+    assert.match(vaultPage, /scan_estimate_help:\s*"Estimated from your last completed scan"/)
+    assert.match(vaultPage, /\$\{estimated \? "~" : ""\}\$\{progressValue\}%/)
+    assert.match(vaultPage, /\.vault-progress-bar\.determinate \{[\s\S]*?transform:\s*scaleX/)
+    assert.match(vaultPage, /\.vault-progress-bar\.indeterminate \{[\s\S]*?transform:\s*scaleX\(1\)[\s\S]*?animation:\s*vault-progress-pulse/)
+    assert.doesNotMatch(vaultPage, /vault-progress-sweep/)
+    assert.match(vaultPage, /body\.vault-page \.task-container \{[\s\S]*?overflow:\s*hidden/)
+    assert.match(vaultPage, /\.vault-explorer \{[\s\S]*?flex:\s*1 1 auto[\s\S]*?overflow:\s*hidden/)
+    assert.match(vaultPage, /\.vault-rail \{[\s\S]*?overflow-y:\s*auto[\s\S]*?overscroll-behavior:\s*contain/)
+    assert.match(vaultPage, /\.vault-pane \{[\s\S]*?grid-template-rows:\s*auto minmax\(0, 1fr\) auto/)
+    assert.doesNotMatch(vaultPage, /vault-pane-head/)
+    assert.match(vaultPage, /const toolbarSummary = \(visibleItems\) =>/)
+    assert.match(vaultPage, /id="vault-toolbar-summary"/)
+    assert.match(vaultPage, /\.vault-table-wrap \{[\s\S]*?overflow:\s*auto[\s\S]*?overscroll-behavior:\s*contain/)
+    assert.match(vaultPage, /#vault-locations \{[\s\S]*?flex-direction:\s*column[\s\S]*?justify-content:\s*flex-start[\s\S]*?gap:\s*0/)
+    assert.match(vaultPage, /\.vault-source-line\.depth-2 \{ padding-left:\s*24px; \}/)
+    assert.match(vaultPage, /vault-source-line depth-\$\{Math\.min\(depth, 2\)\} \$\{pathText \? "has-path" : ""\}/)
+  })
+
+  test('vault route provisions the dev requirements needed by its folder picker', async () => {
+    const serverSource = await fs.promises.readFile(path.resolve(__dirname, '..', 'server', 'index.js'), 'utf8')
+    const routeStart = serverSource.indexOf('this.app.get("/vault"')
+    const routeEnd = serverSource.indexOf('this.app.post("/vault/action"', routeStart)
+    assert.ok(routeStart >= 0 && routeEnd > routeStart, 'vault route must be present')
+    const route = serverSource.slice(routeStart, routeEnd)
+    assert.match(route, /this\.kernel\.bin\.check\(/)
+    assert.match(route, /this\.kernel\.bin\.preset\("dev"\)/)
+    assert.match(route, /requirements_pending \|\| install_required/)
+    assert.match(route, /\/setup\/dev\?callback=\$\{encodeURIComponent\(req\.originalUrl\)\}/)
+    assert.ok(route.indexOf('res.redirect') < route.indexOf('res.render("vault"'), 'requirements redirect must happen before rendering Vault')
+  })
+
   test('deduplicate batches share a batch id and are undoable as one', async () => {
     const { home, vault } = await makeEnv()
     const c1 = crypto.randomBytes(4096)
@@ -194,6 +285,23 @@ describe('vault dashboard backend (phase 4)', () => {
     assert.ok(vault.registry.duplicates.has(cacheCopy), 'other non-app scope remains pending')
   })
 
+  test('unavailable conversions remain pending for review', async () => {
+    const { home, vault } = await makeEnv()
+    const content = crypto.randomBytes(4096)
+    const hash = sha256(content)
+    const original = await writeFile(path.resolve(home, 'api', 'appA', 'm.bin'), content)
+    const pending = await writeFile(path.resolve(home, 'api', 'appB', 'm.bin'), content)
+    await vault.adopt(original, hash, { app: 'appA' })
+    vault.registry.duplicates.set(pending, { hash, size: content.length, app: 'appB' })
+    const realConvert = vault.convert.bind(vault)
+    vault.convert = async () => ({ status: 'unavailable' })
+    const result = await vault.sweeper.convertPending('appB')
+    vault.convert = realConvert
+    assert.strictEqual(result.unavailable, 1)
+    assert.ok(vault.registry.duplicates.has(pending))
+    assert.strictEqual(vault.registry.links.has(pending), false)
+  })
+
   test('status separates app, Pinokio folder, and external source metadata', async (t) => {
     const { home, vault } = await makeEnv()
     const external = await fs.promises.mkdtemp(path.resolve(os.tmpdir(), 'pinokio-ui-external-'))