pinokiod 8.0.39 → 8.0.40

package/test/vault-engine.test.js

@@ -1,360 +0,0 @@
-const { test, describe, before, after } = require('node:test')
-const assert = require('node:assert')
-const fs = require('fs')
-const os = require('os')
-const path = require('path')
-const crypto = require('crypto')
-const { execSync } = require('child_process')
-const Vault = require('../kernel/vault')
-
-const sha256 = (buf) => crypto.createHash('sha256').update(buf).digest('hex')
-
-const makeHome = async () => {
-  const home = await fs.promises.mkdtemp(path.resolve(os.tmpdir(), 'pinokio-vault-test-'))
-  await fs.promises.mkdir(path.resolve(home, 'api'), { recursive: true })
-  return home
-}
-
-const fakeKernel = (home) => ({
-  homedir: home,
-  platform: process.platform,
-  path: (...args) => path.resolve(home, ...args)
-})
-
-const makeVault = async (home) => {
-  const vault = new Vault(fakeKernel(home))
-  await vault.init()
-  return vault
-}
-
-const writeFile = async (p, content) => {
-  await fs.promises.mkdir(path.dirname(p), { recursive: true })
-  await fs.promises.writeFile(p, content)
-  return p
-}
-
-describe('vault engine (phase 1)', () => {
-  let homes = []
-  const home = async () => {
-    const h = await makeHome()
-    homes.push(h)
-    return h
-  }
-  after(async () => {
-    for (const h of homes) {
-      await fs.promises.rm(h, { recursive: true, force: true }).catch(() => {})
-    }
-  })
-
-  test('item 1: PINOKIO_VAULT=false creates nothing and disables all ops', async () => {
-    const h = await home()
-    await writeFile(path.resolve(h, 'ENVIRONMENT'), 'PINOKIO_VAULT=false\n')
-    const vault = new Vault(fakeKernel(h))
-    const result = await vault.init()
-    assert.strictEqual(result.enabled, false)
-    assert.strictEqual(fs.existsSync(path.resolve(h, 'vault')), false)
-    const file = await writeFile(path.resolve(h, 'api', 'app', 'model.bin'), 'data')
-    assert.strictEqual((await vault.adopt(file, sha256(Buffer.from('data')))).status, 'disabled')
-    assert.strictEqual(fs.existsSync(path.resolve(h, 'vault')), false)
-  })
-
-  test('kill switch: process.env override wins', async () => {
-    const h = await home()
-    process.env.PINOKIO_VAULT = 'false'
-    try {
-      const vault = new Vault(fakeKernel(h))
-      const result = await vault.init()
-      assert.strictEqual(result.enabled, false)
-    } finally {
-      delete process.env.PINOKIO_VAULT
-    }
-  })
-
-  test('adopt: metadata-only, store name shares the inode', async () => {
-    const h = await home()
-    const vault = await makeVault(h)
-    const content = crypto.randomBytes(4096)
-    const hash = sha256(content)
-    const file = await writeFile(path.resolve(h, 'api', 'appA', 'models', 'm.safetensors'), content)
-    const result = await vault.adopt(file, hash, { app: 'appA' })
-    assert.strictEqual(result.status, 'adopted')
-    const st = await fs.promises.stat(file)
-    const storeStat = await fs.promises.stat(vault.storePathFor(hash))
-    assert.strictEqual(st.ino, storeStat.ino)
-    assert.strictEqual(st.nlink, 2)
-    assert.ok(vault.registry.blobs.has(hash))
-    assert.strictEqual(vault.registry.links.get(file).hash, hash)
-  })
-
-  test('item 5: convert links duplicate to blob, content intact, other copies untouched', async () => {
-    const h = await home()
-    const vault = await makeVault(h)
-    const content = crypto.randomBytes(8192)
-    const hash = sha256(content)
-    const a = await writeFile(path.resolve(h, 'api', 'appA', 'm.bin'), content)
-    const b = await writeFile(path.resolve(h, 'api', 'appB', 'm.bin'), content)
-    await vault.adopt(a, hash, { app: 'appA' })
-    const result = await vault.convert(b, hash, { app: 'appB', batch_id: 'batch1' })
-    assert.strictEqual(result.status, 'converted')
-    const stA = await fs.promises.stat(a)
-    const stB = await fs.promises.stat(b)
-    assert.strictEqual(stA.ino, stB.ino)
-    assert.strictEqual(stB.nlink, 3)
-    assert.deepStrictEqual(await fs.promises.readFile(b), content)
-    assert.strictEqual(vault.registry.totals.lifetime_bytes_saved, content.length)
-    const events = await vault.registry.readEvents()
-    assert.ok(events.some((e) => e.kind === 'convert' && e.batch_id === 'batch1'))
-  })
-
-  test('item 5: simulated crash between link and rename leaves target intact; verify cleans stray tmp', async () => {
-    const h = await home()
-    const vault = await makeVault(h)
-    const content = crypto.randomBytes(4096)
-    const hash = sha256(content)
-    const a = await writeFile(path.resolve(h, 'api', 'appA', 'm.bin'), content)
-    await vault.adopt(a, hash, { app: 'appA' })
-    // Simulate a crash mid-convert: tmp link exists next to a registered path.
-    await fs.promises.link(vault.storePathFor(hash), a + Vault.TMP_SUFFIX)
-    assert.deepStrictEqual(await fs.promises.readFile(a), content)
-    await vault.verify()
-    assert.strictEqual(fs.existsSync(a + Vault.TMP_SUFFIX), false)
-    assert.deepStrictEqual(await fs.promises.readFile(a), content)
-  })
-
-  test('verify: orphan detection via nlink, dead link pruning, store re-adoption', async () => {
-    const h = await home()
-    const vault = await makeVault(h)
-    const content = crypto.randomBytes(4096)
-    const hash = sha256(content)
-    const a = await writeFile(path.resolve(h, 'api', 'appA', 'm.bin'), content)
-    await vault.adopt(a, hash, { app: 'appA' })
-
-    // Delete the app copy: blob becomes an orphan, link is pruned.
-    await fs.promises.unlink(a)
-    await vault.verify()
-    assert.strictEqual(vault.registry.links.has(a), false)
-    assert.strictEqual(vault.registry.blobs.get(hash).orphan, true)
-
-    // Recreate the content as a NEW inode: adopt refuses (duplicate), convert
-    // links it. Then delete the STORE name: verify re-adopts from the link.
-    const b = await writeFile(path.resolve(h, 'api', 'appB', 'm.bin'), content)
-    assert.strictEqual((await vault.adopt(b, hash, { app: 'appB' })).status, 'duplicate')
-    assert.strictEqual((await vault.convert(b, hash, { app: 'appB' })).status, 'converted')
-    await fs.promises.unlink(vault.storePathFor(hash))
-    await vault.verify()
-    assert.strictEqual(fs.existsSync(vault.storePathFor(hash)), true)
-    const st = await fs.promises.stat(b)
-    assert.strictEqual(st.nlink, 2)
-  })
-
-  test('reclaim: refuses while in use, frees orphans', async () => {
-    const h = await home()
-    const vault = await makeVault(h)
-    const content = crypto.randomBytes(4096)
-    const hash = sha256(content)
-    const a = await writeFile(path.resolve(h, 'api', 'appA', 'm.bin'), content)
-    await vault.adopt(a, hash)
-    assert.strictEqual((await vault.reclaim(hash)).status, 'in-use')
-    await fs.promises.unlink(a)
-    const result = await vault.reclaim(hash)
-    assert.strictEqual(result.status, 'reclaimed')
-    assert.strictEqual(result.bytes_freed, content.length)
-    assert.strictEqual(fs.existsSync(vault.storePathFor(hash)), false)
-    assert.strictEqual(vault.registry.blobs.has(hash), false)
-  })
-
-  test('item 9: registry deleted => rebuild reproduces blobs, links, orphans from disk', async () => {
-    const h = await home()
-    const vault = await makeVault(h)
-    // Rebuild's ino-match walk only considers files >= SIZE_THRESHOLD, so
-    // lower the threshold surrogate by using large-enough sparse-ish files.
-    const big = Buffer.alloc(Vault.SIZE_THRESHOLD, 7)
-    const hash = sha256(big)
-    const a = await writeFile(path.resolve(h, 'api', 'appA', 'big.bin'), big)
-    await vault.adopt(a, hash, { app: 'appA' })
-    const orphanContent = Buffer.alloc(Vault.SIZE_THRESHOLD, 9)
-    const orphanHash = sha256(orphanContent)
-    const b = await writeFile(path.resolve(h, 'api', 'appB', 'big2.bin'), orphanContent)
-    await vault.adopt(b, orphanHash, { app: 'appB' })
-    await fs.promises.unlink(b)
-
-    await fs.promises.unlink(path.resolve(vault.root, 'registry.json')).catch(() => {})
-    await fs.promises.unlink(path.resolve(vault.root, 'events.ndjson')).catch(() => {})
-    const fresh = await makeVault(h)
-    await fresh.rebuild()
-    assert.ok(fresh.registry.blobs.has(hash))
-    assert.ok(fresh.registry.blobs.has(orphanHash))
-    assert.strictEqual(fresh.registry.blobs.get(orphanHash).orphan, true)
-    assert.strictEqual(fresh.registry.blobs.get(hash).orphan, false)
-    assert.strictEqual(fresh.registry.links.get(a).hash, hash)
-    assert.strictEqual(fresh.registry.links.get(a).app, 'appA')
-    assert.strictEqual(
-      fresh.registry.links.has(fresh.storePathFor(hash)),
-      false,
-      'the internal vault name is never exposed as a tracked location'
-    )
-  })
-
-  test('item 9: torn events line tolerated; corrupt registry.json triggers rebuild, not failure', async () => {
-    const h = await home()
-    const vault = await makeVault(h)
-    const content = crypto.randomBytes(4096)
-    const hash = sha256(content)
-    const a = await writeFile(path.resolve(h, 'api', 'appA', 'm.bin'), content)
-    await vault.adopt(a, hash)
-    await vault.registry.flush()
-    // Torn final line
-    await fs.promises.appendFile(path.resolve(vault.root, 'events.ndjson'), '{"kind":"conv')
-    const events = await vault.registry.readEvents()
-    assert.ok(events.every((e) => typeof e === 'object'))
-    // Corrupt snapshot
-    await fs.promises.writeFile(path.resolve(vault.root, 'registry.json'), '{corrupted!!')
-    const fresh = new Vault(fakeKernel(h))
-    const result = await fresh.init()
-    assert.strictEqual(result.enabled, true)
-    assert.strictEqual(result.corrupt_recovered, true)
-    assert.ok(fresh.registry.blobs.has(hash))
-  })
-
-  test('registry snapshot flushes serialize without racing the shared temp file', async () => {
-    const h = await home()
-    const vault = await makeVault(h)
-    const content = crypto.randomBytes(4096)
-    const file = await writeFile(path.resolve(h, 'api', 'appA', 'm.bin'), content)
-    await vault.adopt(file, sha256(content), { app: 'appA' })
-
-    await Promise.all([vault.registry.flush(), vault.registry.flush(), vault.registry.flush()])
-    const snapshot = JSON.parse(await fs.promises.readFile(path.resolve(vault.root, 'registry.json'), 'utf8'))
-    assert.ok(snapshot.blobs[sha256(content)])
-    assert.strictEqual(fs.existsSync(path.resolve(vault.root, 'registry.json.tmp')), false)
-  })
-
-  test('manual index repair preserves user decisions, history, and pending review state', async () => {
-    const h = await home()
-    const vault = await makeVault(h)
-    const content = crypto.randomBytes(4096)
-    const hash = sha256(content)
-    const original = await writeFile(path.resolve(h, 'api', 'appA', 'm.bin'), content)
-    const pending = await writeFile(path.resolve(h, 'api', 'appB', 'm.bin'), content)
-    const independent = await writeFile(path.resolve(h, 'cache', 'keep.bin'), crypto.randomBytes(2048))
-    await vault.adopt(original, hash, { app: 'appA', source_urls: ['https://example.test/model'] })
-    vault.registry.duplicates.set(pending, { hash, size: content.length, app: 'appB', discovered: Date.now() })
-    vault.registry.excluded.set(independent, { ts: Date.now(), size: 2048 })
-    vault.registry.totals.lifetime_bytes_saved = 12345
-    vault.registry.lastScan = { ts: 111, dirs: 2, files: 3, bytes_total: 999 }
-
-    await vault.rebuild([])
-
-    assert.ok(vault.registry.excluded.has(independent), 'keep-independent decision survives')
-    assert.strictEqual(vault.registry.excluded.get(independent).size, 2048)
-    assert.strictEqual(vault.registry.totals.lifetime_bytes_saved, 12345)
-    assert.deepStrictEqual(vault.registry.lastScan, { ts: 111, dirs: 2, files: 3, bytes_total: 999 })
-    assert.ok(vault.registry.duplicates.has(pending), 'pending review item survives when still valid')
-    assert.deepStrictEqual(vault.registry.blobs.get(hash).source_urls, ['https://example.test/model'])
-  })
-
-  test('rebuild walks every directory without name heuristics and still skips symlinks', async () => {
-    const h = await home()
-    const vault = await makeVault(h)
-    const big = Buffer.alloc(Vault.SIZE_THRESHOLD, 3)
-    const hash = sha256(big)
-    const inEnv = await writeFile(path.resolve(h, 'api', 'appA', 'env', 'big.bin'), big)
-    await vault.adopt(inEnv, hash, { app: 'appA' })
-    await fs.promises.unlink(path.resolve(vault.root, 'registry.json')).catch(() => {})
-    const fresh = await makeVault(h)
-    await fresh.rebuild()
-    assert.ok(fresh.registry.blobs.has(hash))
-    assert.strictEqual(fresh.registry.links.has(inEnv), true, 'directory names never hide tracked files')
-  })
-
-  test('hash worker computes sha256 and is reused across a scan batch', async () => {
-    const h = await home()
-    const vault = await makeVault(h)
-    const first = crypto.randomBytes(1024 * 1024)
-    const second = crypto.randomBytes(1024 * 1024)
-    const firstFile = await writeFile(path.resolve(h, 'api', 'appA', 'x.bin'), first)
-    const secondFile = await writeFile(path.resolve(h, 'api', 'appA', 'y.bin'), second)
-    const firstResult = await vault.hashFile(firstFile)
-    const secondResult = await vault.hashFile(secondFile)
-    assert.strictEqual(firstResult.hash, sha256(first))
-    assert.strictEqual(firstResult.size, first.length)
-    assert.strictEqual(secondResult.hash, sha256(second))
-    assert.strictEqual(vault.workerStarts, 1, 'worker startup is amortized across files')
-  })
-
-  test('unsupported conversion leaves the independent copy pending and unregistered', async () => {
-    const h = await home()
-    const vault = await makeVault(h)
-    const content = crypto.randomBytes(4096)
-    const hash = sha256(content)
-    const original = await writeFile(path.resolve(h, 'api', 'appA', 'm.bin'), content)
-    const pending = await writeFile(path.resolve(h, 'api', 'appB', 'm.bin'), content)
-    await vault.adopt(original, hash, { app: 'appA' })
-    const realLink = fs.promises.link
-    fs.promises.link = async (source, target) => {
-      if (target === pending + Vault.TMP_SUFFIX) {
-        const error = new Error('different volume')
-        error.code = 'EXDEV'
-        throw error
-      }
-      return realLink(source, target)
-    }
-    try {
-      const result = await vault.convert(pending, hash, { app: 'appB' })
-      assert.strictEqual(result.status, 'unavailable')
-    } finally {
-      fs.promises.link = realLink
-    }
-    assert.strictEqual(vault.registry.links.has(pending), false)
-    assert.deepStrictEqual(await fs.promises.readFile(pending), content)
-  })
-
-  test('permission errors during conversion are retryable locks, not unsupported disks', async () => {
-    const h = await home()
-    const vault = await makeVault(h)
-    const content = crypto.randomBytes(4096)
-    const hash = sha256(content)
-    const original = await writeFile(path.resolve(h, 'api', 'appA', 'm.bin'), content)
-    const pending = await writeFile(path.resolve(h, 'api', 'appB', 'm.bin'), content)
-    await vault.adopt(original, hash, { app: 'appA' })
-    const realLink = fs.promises.link
-    fs.promises.link = async (source, target) => {
-      if (target === pending + Vault.TMP_SUFFIX) {
-        const error = new Error('file is in use')
-        error.code = 'EPERM'
-        throw error
-      }
-      return realLink(source, target)
-    }
-    try {
-      const result = await vault.convert(pending, hash, { app: 'appB' })
-      assert.strictEqual(result.status, 'locked')
-    } finally {
-      fs.promises.link = realLink
-    }
-    assert.strictEqual(vault.registry.links.has(pending), false)
-    assert.deepStrictEqual(await fs.promises.readFile(pending), content)
-  })
-
-  test('item 11: exFAT volume enters copy mode (macOS loopback; skipped elsewhere)', { skip: process.platform !== 'darwin' }, async (t) => {
-    const h = await home()
-    const vault = await makeVault(h)
-    const dmg = path.resolve(h, 'exfat-test.dmg')
-    let mountPoint = null
-    try {
-      execSync(`hdiutil create -size 16m -fs exFAT -volname VAULTPROBE "${dmg}"`, { stdio: 'pipe' })
-      const out = execSync(`hdiutil attach "${dmg}" -nobrowse`, { encoding: 'utf8' })
-      const m = out.match(/(\/Volumes\/\S+)\s*$/m)
-      mountPoint = m && m[1]
-    } catch (e) {
-      t.skip('hdiutil exFAT loopback unavailable: ' + e.message)
-      return
-    }
-    try {
-      const mode = await vault.probe(mountPoint)
-      assert.strictEqual(mode, 'copy')
-    } finally {
-      if (mountPoint) execSync(`hdiutil detach "${mountPoint}" -force`, { stdio: 'pipe' })
-    }
-  })
-})