pinokiod 8.0.41 → 8.0.44

package/test/vault-engine.test.js

@@ -93,73 +93,10 @@ describe('vault engine (phase 1)', () => {
 
     assert.deepStrictEqual(startup, { enabled: true, fresh: true })
     assert.strictEqual(fs.existsSync(path.resolve(h, 'vault')), false)
-    assert.deepStrictEqual(vault.navigationStatus(), {
-      any_scan: false,
-      scanned: false,
-      scanning: false,
-      pending_bytes: 0
-    })
-    assert.strictEqual(fs.existsSync(path.resolve(h, 'vault')), false, 'navigation status is read-only')
     await vault.ensureInitialized()
     assert.strictEqual(fs.existsSync(path.resolve(h, 'vault', 'registry.json')), true)
   })
 
-  test('navigation status distinguishes global and app scans without filesystem work', async () => {
-    const h = await home()
-    const pending = await writeFile(path.resolve(h, 'api', 'appA', 'pending.bin'), 'pending')
-    const vault = await makeVault(h)
-    const hash = sha256(Buffer.from('pending'))
-    const sourceId = 'app:appA'
-    vault.registry.addBlob(hash, { size: 7000 })
-    vault.registry.setDuplicate(pending, {
-      hash,
-      size: 7000,
-      app: 'appA',
-      source_id: sourceId
-    })
-    vault.registry.setLastScan({ ts: 1, files: 1 }, sourceId)
-
-    assert.deepStrictEqual(vault.navigationStatus(), {
-      any_scan: true,
-      scanned: false,
-      scanning: false,
-      pending_bytes: 7000,
-      incomplete: false
-    })
-    assert.deepStrictEqual(vault.navigationStatusForApp('appA'), {
-      any_scan: true,
-      scanned: true,
-      scanning: false,
-      pending_bytes: 7000,
-      incomplete: false
-    })
-    assert.deepStrictEqual(vault.navigationStatusForApp('appB'), {
-      any_scan: true,
-      scanned: false,
-      scanning: false,
-      pending_bytes: 0,
-      incomplete: false
-    })
-
-    vault.registry.exclude(pending, {
-      ts: Date.now(),
-      size: 7000,
-      source_id: sourceId,
-      unverified: true
-    })
-    assert.strictEqual(vault.navigationStatus().incomplete, true)
-    assert.strictEqual(vault.navigationStatusForApp('appA').incomplete, true)
-
-    vault.sweeper.state = Object.assign(vault.sweeper.idleState(), {
-      active: true,
-      phase: 'discovering',
-      scope_id: sourceId
-    })
-    assert.strictEqual(vault.navigationStatus().scanning, true)
-    assert.strictEqual(vault.navigationStatusForApp('appA').scanning, true)
-    assert.strictEqual(vault.navigationStatusForApp('appB').scanning, false)
-  })
-
   test('adopt: metadata-only, store name shares the inode', async () => {
     const h = await home()
     const vault = await makeVault(h)
@@ -222,6 +159,7 @@ describe('vault engine (phase 1)', () => {
     assert.strictEqual(stA.ino, stB.ino)
     assert.strictEqual(stB.nlink, 3)
     assert.deepStrictEqual(await fs.promises.readFile(b), content)
+    assert.strictEqual(vault.registry.totals.lifetime_bytes_saved, content.length)
     const events = await vault.registry.readEvents()
     assert.ok(events.some((e) => e.kind === 'convert' && e.batch_id === 'batch1'))
   })
@@ -266,6 +204,7 @@ describe('vault engine (phase 1)', () => {
     const duplicateStat = await fs.promises.stat(duplicate)
     assert.strictEqual(duplicateStat.ino, storeStat.ino)
     assert.strictEqual(vault.registry.links.get(duplicate).batch_id, 'committed')
+    assert.strictEqual(vault.registry.totals.lifetime_bytes_saved, content.length)
   })
 
   test('a failed activity append does not misreport a completed conversion', async () => {
@@ -326,6 +265,7 @@ describe('vault engine (phase 1)', () => {
     assert.strictEqual(result.status, 'stale')
     assert.deepStrictEqual(await fs.promises.readFile(duplicate), replacement)
     assert.strictEqual(vault.registry.links.has(duplicate), false)
+    assert.strictEqual(vault.registry.totals.lifetime_bytes_saved, 0)
   })
 
   test('convert refuses a final store metadata change', {
@@ -378,11 +318,6 @@ describe('vault engine (phase 1)', () => {
       dev: duplicateStat.dev, ino: duplicateStat.ino,
       mtime: duplicateStat.mtimeMs, ctime: duplicateStat.ctimeMs
     })
-    vault.registry.setScanEntry(duplicate, {
-      hash, size: content.length, source_id: source.id,
-      dev: duplicateStat.dev, ino: duplicateStat.ino,
-      mtime: duplicateStat.mtimeMs, ctime: duplicateStat.ctimeMs
-    })
     // This is the actual crash state: the target is still pending and the
     // temporary name already shares the stored file's identity.
     await fs.promises.link(vault.storePathFor(hash), duplicate + Vault.TMP_SUFFIX)
@@ -413,6 +348,29 @@ describe('vault engine (phase 1)', () => {
     assert.strictEqual(await fs.promises.readFile(registeredTmp, 'utf8'), 'other user data')
   })
 
+  test('verification preserves a copy-mode hardlink using the reserved suffix', async () => {
+    const h = await home()
+    const vault = await makeVault(h)
+    const content = crypto.randomBytes(4096)
+    const hash = sha256(content)
+    const file = await writeFile(path.resolve(h, 'api', 'appA', 'm.bin'), content)
+    await vault.refreshSources()
+    const source = vault.sources().find((item) => item.kind === 'app' && item.app === 'appA')
+    const st = await fs.promises.lstat(file)
+    vault.registry.addBlob(hash, { size: st.size })
+    vault.registry.addLink(file, {
+      hash, app: 'appA', source_id: source.id,
+      dev: st.dev, ino: st.ino, mode: 'copy'
+    })
+    const suffix = file + Vault.TMP_SUFFIX
+    await fs.promises.link(file, suffix)
+
+    await vault.verify()
+
+    assert.strictEqual(fs.existsSync(suffix), true)
+    assert.strictEqual(vault.registry.links.has(file), true)
+  })
+
   test('verify: orphan detection via nlink, dead link pruning, store re-adoption', async () => {
     const h = await home()
     const vault = await makeVault(h)
@@ -457,25 +415,28 @@ describe('vault engine (phase 1)', () => {
       return realRealpath(target, options)
     }
     try {
-      await vault.verify()
+      await assert.rejects(vault.verify(), (error) => error && error.code === 'EIO')
     } finally {
       fs.promises.realpath = realRealpath
     }
 
     assert.strictEqual(vault.registry.links.get(file).batch_id, 'keep-batch')
-    assert.strictEqual(vault.registry.links.get(file).unverified, true)
     assert.ok(vault.registry.blobs.has(hash))
     assert.deepStrictEqual(await fs.promises.readFile(file), content)
   })
 
-  test('verify re-adopts a trusted linked name without claiming an independent copy', async () => {
+  test('verify re-adopts a trusted linked name when a copy-mode name also exists', async () => {
     const h = await home()
     const vault = await makeVault(h)
     const content = crypto.randomBytes(4096)
     const hash = sha256(content)
     const linked = await writeFile(path.resolve(h, 'api', 'appA', 'm.bin'), content)
-    await writeFile(path.resolve(h, 'api', 'appB', 'm.bin'), content)
+    const copied = await writeFile(path.resolve(h, 'api', 'appB', 'm.bin'), content)
     await vault.adopt(linked, hash, { app: 'appA' })
+    const copiedStat = await fs.promises.stat(copied)
+    vault.registry.addLink(copied, {
+      hash, app: 'appB', dev: copiedStat.dev, ino: copiedStat.ino, mode: 'copy'
+    })
     await fs.promises.unlink(vault.storePathFor(hash))
 
     await vault.verify()
@@ -658,8 +619,7 @@ describe('vault engine (phase 1)', () => {
     await fs.promises.utimes(pendingPath, future, future)
 
     const result = await vault.perform('deduplicate', {
-      scope_id: vault.registry.duplicates.get(pendingPath).source_id,
-      paths: [pendingPath]
+      scope_id: vault.registry.duplicates.get(pendingPath).source_id
     })
     assert.strictEqual(result.stale, 1)
     assert.strictEqual(result.converted, 0)
@@ -679,10 +639,7 @@ describe('vault engine (phase 1)', () => {
     await fs.promises.writeFile(original, changedContent)
 
     const pendingEntry = vault.registry.duplicates.get(pending)
-    const result = await vault.perform('deduplicate', {
-      scope_id: pendingEntry.source_id,
-      paths: [pending]
-    })
+    const result = await vault.perform('deduplicate', { scope_id: pendingEntry.source_id })
 
     assert.strictEqual(result.converted, 0)
     assert.strictEqual(result.stale, 1)
@@ -704,10 +661,7 @@ describe('vault engine (phase 1)', () => {
     await vault.sweeper.scan()
     const before = await fs.promises.stat(pending)
     const pendingEntry = vault.registry.duplicates.get(pending)
-    const result = await vault.perform('deduplicate', {
-      scope_id: pendingEntry.source_id,
-      paths: [pending]
-    })
+    const result = await vault.perform('deduplicate', { scope_id: pendingEntry.source_id })
     const after = await fs.promises.stat(pending)
 
     if (process.platform === 'win32') assert.ok(result.converted >= 0)
@@ -737,7 +691,7 @@ describe('vault engine (phase 1)', () => {
       dev: pendingStat.dev, ino: pendingStat.ino, mtime: pendingStat.mtimeMs
     })
 
-    const result = await vault.perform('deduplicate', { scope_id: appB.id, paths: [pending] })
+    const result = await vault.perform('deduplicate', { scope_id: appB.id })
 
     assert.strictEqual(result.stale, 1)
     assert.strictEqual(result.converted, 0)
@@ -745,9 +699,11 @@ describe('vault engine (phase 1)', () => {
     assert.ok(vault.registry.duplicates.has(pending))
   })
 
-  test('item 9: registry deleted => startup recovers only managed blobs', async () => {
+  test('item 9: registry deleted => rebuild reproduces blobs, links, orphans from disk', async () => {
     const h = await home()
     const vault = await makeVault(h)
+    // Rebuild's ino-match walk only considers files >= SIZE_THRESHOLD, so
+    // lower the threshold surrogate by using large-enough sparse-ish files.
     const big = Buffer.alloc(Vault.SIZE_THRESHOLD, 7)
     const hash = sha256(big)
     const a = await writeFile(path.resolve(h, 'api', 'appA', 'big.bin'), big)
@@ -766,11 +722,14 @@ describe('vault engine (phase 1)', () => {
     assert.ok(fresh.registry.blobs.has(hash))
     assert.ok(fresh.registry.blobs.has(orphanHash))
     assert.strictEqual(fresh.registry.blobs.get(orphanHash).orphan, true)
-    assert.strictEqual(fresh.registry.blobs.get(hash).orphan, false,
-      'the visible hardlink still keeps the managed inode live')
-    assert.strictEqual(fresh.registry.links.size, 0,
-      'startup recovery never walks visible source trees')
-    assert.strictEqual(fresh.registry.links.has(a), false)
+    assert.strictEqual(fresh.registry.blobs.get(hash).orphan, false)
+    assert.strictEqual(fresh.registry.links.get(a).hash, hash)
+    assert.strictEqual(fresh.registry.links.get(a).app, 'appA')
+    assert.strictEqual(
+      fresh.registry.links.has(fresh.storePathFor(hash)),
+      false,
+      'the internal vault name is never exposed as a tracked location'
+    )
   })
 
   test('item 9: torn events line tolerated; corrupt registry.json triggers rebuild, not failure', async () => {
@@ -1134,149 +1093,99 @@ describe('vault engine (phase 1)', () => {
     assert.ok(batches.every((batch) => /^batch-[0-9a-f-]{36}$/.test(batch)))
   })
 
-  test('deduplicate batches expose queued and exact completed file progress', async () => {
+  test('deduplication reports transient file progress and clears it when complete', async () => {
     const h = await home()
     const vault = await makeVault(h)
-    vault.sizeThreshold = 1
-    const files = []
-    for (let index = 0; index < 3; index += 1) {
-      const content = crypto.randomBytes(4096)
-      files.push({
-        content,
-        hash: sha256(content),
-        original: await writeFile(path.resolve(h, 'api', 'appA', `${index}.bin`), content),
-        pending: await writeFile(path.resolve(h, 'api', 'appB', `${index}.bin`), content)
-      })
-    }
-    await vault.refreshSources()
-    const appA = vault.sources().find((source) => source.kind === 'app' && source.app === 'appA')
-    const appB = vault.sources().find((source) => source.kind === 'app' && source.app === 'appB')
-    for (const file of files) {
-      await vault.adopt(file.original, file.hash, { app: 'appA', source_id: appA.id })
-      const st = await fs.promises.stat(file.pending)
-      const entry = {
-        hash: file.hash,
-        size: st.size,
-        app: 'appB',
-        source_id: appB.id,
-        dev: st.dev,
-        ino: st.ino,
-        mtime: st.mtimeMs,
-        ctime: st.ctimeMs
-      }
-      vault.registry.setDuplicate(file.pending, entry, entry)
+    const source = {
+      id: 'app:appA',
+      kind: 'app',
+      app: 'appA',
+      root: path.resolve(h, 'api', 'appA'),
+      shareable: true
     }
+    vault.refreshSources = async () => {}
+    vault._sources = [source]
+    vault.sourceForPath = () => source
+    vault.sourceAppIsRunning = () => false
+    vault.canonicalPathIsWithinSource = async () => true
+    vault.registry.duplicates = new Map([
+      ['/models/one.bin', { hash: 'a'.repeat(64), dev: 1, ino: 1, mtime: 1, ctime: 1 }],
+      ['/models/two.bin', { hash: 'b'.repeat(64), dev: 1, ino: 2, mtime: 1, ctime: 1 }]
+    ])
 
-    let releaseQueue
-    const queueGate = new Promise((resolve) => { releaseQueue = resolve })
-    const blocker = vault.runExclusive(() => queueGate)
-    const samples = []
-    const realConvert = vault.convert.bind(vault)
-    vault.convert = async (...args) => {
-      const action = vault.progressStatus(appB.id).file_actions
-        .find((item) => item.kind === 'deduplicate')
-      samples.push({
-        phase: action.phase,
-        completed: action.files_completed,
-        total: action.files_total
-      })
-      return realConvert(...args)
+    let calls = 0
+    let releaseSecond
+    let reportSecond
+    const secondStarted = new Promise((resolve) => { reportSecond = resolve })
+    const secondGate = new Promise((resolve) => { releaseSecond = resolve })
+    vault.convert = async () => {
+      calls += 1
+      if (calls === 2) {
+        reportSecond()
+        await secondGate
+      }
+      return { status: 'converted', bytes_saved: 1 }
     }
 
-    const pending = vault.perform('deduplicate', {
-      scope_id: appB.id,
-      paths: files.map((file) => file.pending)
-    })
-    await new Promise((resolve) => setImmediate(resolve))
-    const queued = vault.progressStatus(appB.id).file_actions
-      .find((item) => item.kind === 'deduplicate')
-    assert.strictEqual(queued.phase, 'queued')
-    assert.strictEqual(queued.files_completed, 0)
-    assert.strictEqual(queued.files_total, 3)
-
-    releaseQueue()
-    await blocker
+    const pending = vault.perform('deduplicate', { scope_id: source.id })
+    await secondStarted
+    const active = vault.progressStatus().file_action
+    const dashboard = await vault.status()
+    releaseSecond()
     const result = await pending
 
-    assert.strictEqual(result.converted, 3)
-    assert.deepStrictEqual(samples, [
-      { phase: 'deduplicating', completed: 0, total: 3 },
-      { phase: 'deduplicating', completed: 1, total: 3 },
-      { phase: 'deduplicating', completed: 2, total: 3 }
-    ])
-    assert.deepStrictEqual(vault.progressStatus(appB.id).file_actions, [])
+    assert.deepStrictEqual(active, {
+      kind: 'deduplicate',
+      scope_id: source.id,
+      selection: 'duplicates',
+      files_total: 2,
+      files_completed: 1
+    })
+    assert.deepStrictEqual(dashboard.file_action, active)
+    assert.strictEqual(result.converted, 2)
+    assert.strictEqual(vault.progressStatus().file_action, null)
   })
 
-  test('Deduplicate all reports aggregate progress and keeps one batch per location', async () => {
+  test('verification keeps copy-mode content groups without a managed file', async () => {
     const h = await home()
     const vault = await makeVault(h)
-    const batches = []
-    const samples = []
-    vault.deduplicateScope = async (scopeId, options) => {
-      batches.push({ scopeId, batchId: options.batch_id, paths: options.paths })
-      for (const filePath of options.paths) {
-        options.onFileComplete()
-        const action = vault.progressStatus().file_actions
-          .find((item) => item.kind === 'deduplicate_all')
-        samples.push({
-          path: filePath,
-          completed: action.files_completed,
-          total: action.files_total,
-          sourceId: action.source_id
-        })
-      }
-      return { converted: options.paths.length, bytes_saved: options.paths.length * 100 }
-    }
+    const content = crypto.randomBytes(4096)
+    const hash = sha256(content)
+    const file = await writeFile(path.resolve(h, 'api', 'appA', 'model.bin'), content)
+    const st = await fs.promises.stat(file)
+    vault.registry.addBlob(hash, { size: content.length })
+    vault.registry.addLink(file, { hash, app: 'appA', dev: st.dev, ino: st.ino, mode: 'copy' })
 
-    const result = await vault.perform('deduplicate_all', {
-      groups: [
-        { scope_id: 'app:a', paths: ['/a/one', '/a/two'] },
-        { scope_id: 'app:b', paths: ['/b/one'] }
-      ]
-    })
+    await vault.verify()
 
-    assert.strictEqual(result.converted, 3)
-    assert.strictEqual(result.bytes_saved, 300)
-    assert.deepStrictEqual(samples, [
-      { path: '/a/one', completed: 1, total: 3, sourceId: null },
-      { path: '/a/two', completed: 2, total: 3, sourceId: null },
-      { path: '/b/one', completed: 3, total: 3, sourceId: null }
-    ])
-    assert.deepStrictEqual(batches.map((batch) => batch.scopeId), ['app:a', 'app:b'])
-    assert.strictEqual(new Set(batches.map((batch) => batch.batchId)).size, 2)
-    assert.ok(batches.every((batch) => /^batch-[0-9a-f-]{36}$/.test(batch.batchId)))
-    assert.deepStrictEqual(vault.progressStatus().file_actions, [])
+    assert.ok(vault.registry.blobs.has(hash))
+    assert.ok(vault.registry.links.has(file))
+    assert.strictEqual(vault.registry.blobs.get(hash).orphan, false)
+    assert.strictEqual((await vault.reclaim(hash)).status, 'unavailable')
   })
 
-  test('legacy copy entries migrate to unavailable pending suggestions', async () => {
+  test('manual repair preserves verified copy-mode content groups', async () => {
     const h = await home()
-    const root = path.resolve(h, 'vault')
-    await fs.promises.mkdir(root, { recursive: true })
+    const vault = await makeVault(h)
     const content = crypto.randomBytes(4096)
     const hash = sha256(content)
     const file = await writeFile(path.resolve(h, 'api', 'appA', 'model.bin'), content)
-    const st = await fs.promises.stat(file)
-    const registry = new Registry(root)
-    await fs.promises.writeFile(registry.snapshotPath, JSON.stringify({
-      blobs: { [hash]: { size: content.length } },
-      links: {
-        [file]: {
-          hash, app: 'appA', source_id: 'app:appA',
-          dev: st.dev, ino: st.ino, mode: 'copy'
-        }
-      },
-      scan_index: {
-        [file]: {
-          hash, size: st.size, source_id: 'app:appA',
-          dev: st.dev, ino: st.ino, mtime: st.mtimeMs, ctime: st.ctimeMs
-        }
-      }
-    }))
+    await vault.refreshSources()
+    const source = vault.sources().find((item) => item.kind === 'app' && item.app === 'appA')
+    vault.sizeThreshold = 1
+    vault.mode = 'copy'
+    assert.strictEqual((await vault.adopt(file, hash, {
+      app: 'appA', source_id: source.id
+    })).status, 'copy-mode')
+    vault.registry.lastScan = { ts: 123, files: 1 }
 
-    await registry.load()
+    await vault.rebuild()
 
-    assert.strictEqual(registry.links.has(file), false)
-    assert.strictEqual(registry.duplicates.get(file).unavailable_reason, 'unsupported_disk')
+    assert.ok(vault.registry.blobs.has(hash))
+    assert.strictEqual(vault.registry.links.get(file).mode, 'copy')
+    assert.strictEqual(vault.registry.scanIndex.get(file).hash, hash)
+    assert.deepStrictEqual(vault.registry.lastScan, { ts: 123, files: 1 })
+    assert.deepStrictEqual(await fs.promises.readFile(file), content)
   })
 
   test('manual index repair preserves user decisions, history, and pending review state', async () => {
@@ -1298,12 +1207,14 @@ describe('vault engine (phase 1)', () => {
     vault.registry.duplicates.set(pending, pendingEntry)
     vault.registry.scanIndex.set(pending, pendingEntry)
     vault.registry.excluded.set(independent, { ts: Date.now(), size: 2048 })
+    vault.registry.totals.lifetime_bytes_saved = 12345
     vault.registry.lastScan = { ts: 111, dirs: 2, files: 3, bytes_total: 999 }
 
     await vault.rebuild()
 
     assert.ok(vault.registry.excluded.has(independent), 'keep-independent decision survives')
     assert.strictEqual(vault.registry.excluded.get(independent).size, 2048)
+    assert.strictEqual(vault.registry.totals.lifetime_bytes_saved, 12345)
     assert.deepStrictEqual(vault.registry.lastScan, { ts: 111, dirs: 2, files: 3, bytes_total: 999 })
     assert.ok(vault.registry.duplicates.has(pending), 'pending review item survives when still valid')
     assert.deepStrictEqual(vault.registry.blobs.get(hash).source_urls, ['https://example.test/model'])
@@ -1586,7 +1497,7 @@ describe('vault engine (phase 1)', () => {
     assert.deepStrictEqual(await fs.promises.readFile(pending), content)
   })
 
-  test('item 11: exFAT volume is unsupported (macOS loopback; skipped elsewhere)', { skip: process.platform !== 'darwin' }, async (t) => {
+  test('item 11: exFAT volume enters copy mode (macOS loopback; skipped elsewhere)', { skip: process.platform !== 'darwin' }, async (t) => {
     const h = await home()
     const vault = await makeVault(h)
     const dmg = path.resolve(h, 'exfat-test.dmg')
@@ -1602,7 +1513,7 @@ describe('vault engine (phase 1)', () => {
     }
     try {
       const mode = await vault.probe(mountPoint)
-      assert.strictEqual(mode, 'unsupported')
+      assert.strictEqual(mode, 'copy')
     } finally {
       if (mountPoint) execSync(`hdiutil detach "${mountPoint}" -force`, { stdio: 'pipe' })
     }