pinokiod 8.0.37 → 8.0.38

package/kernel/vault/index.js

@@ -14,17 +14,40 @@ const Sweeper = require('./sweeper')
 
 const SIZE_THRESHOLD = 100 * 1024 * 1024
 const TMP_SUFFIX = ".pinokio-dedup-tmp"
-const PRUNE_DIRS = new Set(["node_modules", ".git", "env", "venv", "__pycache__", "logs"])
 const NO_LINK_CODES = new Set(["EXDEV", "ENOTSUP", "EPERM", "ENOSYS"])
 const LOCK_CODES = new Set(["EBUSY", "EPERM", "EACCES"])
+const STAT_CONCURRENCY = 32
 
 const isPathWithin = (root, target) => {
   const rel = path.relative(path.resolve(root), path.resolve(target))
   return rel === "" || (!rel.startsWith(`..${path.sep}`) && rel !== ".." && !path.isAbsolute(rel))
 }
 
+const samePath = (left, right) => {
+  const a = path.resolve(left)
+  const b = path.resolve(right)
+  return process.platform === "win32" ? a.toLowerCase() === b.toLowerCase() : a === b
+}
+
 const sourceId = (kind, name) => `${kind}:${encodeURIComponent(name)}`
 
+const statMany = async (paths, concurrency = STAT_CONCURRENCY) => {
+  const results = new Array(paths.length)
+  let cursor = 0
+  const workers = Array.from({ length: Math.min(concurrency, paths.length) }, async () => {
+    while (cursor < paths.length) {
+      const index = cursor++
+      try {
+        results[index] = await fs.promises.stat(paths[index])
+      } catch (error) {
+        results[index] = null
+      }
+    }
+  })
+  await Promise.all(workers)
+  return results
+}
+
 class Vault {
   constructor(kernel) {
     this.kernel = kernel
@@ -36,6 +59,10 @@ class Vault {
     this.worker = null
     this.workerJobs = new Map()
     this.workerSeq = 0
+    this.workerStarts = 0
+    this.workerIdleTimer = null
+    this.workerIdleMs = 750
+    this.statConcurrency = STAT_CONCURRENCY
     this.sizeThreshold = SIZE_THRESHOLD
     this._sources = []
   }
@@ -115,6 +142,60 @@ class Vault {
   sources() {
     return this._sources
   }
+  async addExternalSource(folderPath) {
+    if (typeof folderPath !== "string" || !path.isAbsolute(folderPath.trim())) {
+      throw new Error("Choose a valid folder.")
+    }
+    const requested = folderPath.trim()
+    let canonical
+    let stats
+    try {
+      canonical = path.resolve(await fs.promises.realpath(requested))
+      stats = await fs.promises.stat(canonical)
+    } catch (error) {
+      throw new Error("That folder is no longer available.")
+    }
+    if (!stats.isDirectory()) throw new Error("Choose a folder, not a file.")
+
+    const home = path.resolve(this.kernel.homedir)
+    let canonicalHome = home
+    try { canonicalHome = path.resolve(await fs.promises.realpath(home)) } catch (error) {}
+    if (isPathWithin(canonicalHome, canonical) || isPathWithin(canonical, canonicalHome)) {
+      throw new Error("That folder is already inside Pinokio and is included in scans.")
+    }
+
+    await this.refreshSources()
+    const existing = this._sources.find((source) => source.kind === "external" && source.root && samePath(source.root, canonical))
+    if (existing) {
+      return { created: false, source: existing }
+    }
+
+    const apiRoot = path.resolve(home, "api")
+    await fs.promises.mkdir(apiRoot, { recursive: true })
+    const baseLabel = path.basename(canonical) || "external-folder"
+    let mountPath = null
+    for (let index = 1; index < 10000; index++) {
+      const label = index === 1 ? baseLabel : `${baseLabel}-${index}`
+      const candidate = path.resolve(apiRoot, label)
+      try {
+        await fs.promises.symlink(canonical, candidate, this.kernel.platform === "win32" ? "junction" : "dir")
+        mountPath = candidate
+        break
+      } catch (error) {
+        if (error && error.code === "EEXIST") continue
+        if (error && (error.code === "EACCES" || error.code === "EPERM")) {
+          throw new Error("Pinokio does not have permission to add that folder.")
+        }
+        throw new Error("Pinokio could not add that folder.")
+      }
+    }
+    if (!mountPath) throw new Error("Pinokio could not create a unique name for that folder.")
+
+    await this.refreshSources()
+    const source = this._sources.find((item) => item.kind === "external" && item.mount_path && samePath(item.mount_path, mountPath))
+    if (!source) throw new Error("The folder was added but could not be loaded. Restart Pinokio and try again.")
+    return { created: true, source }
+  }
   sourceForPath(filePath, preferredId) {
     const sources = this._sources || []
     if (preferredId) {
@@ -181,7 +262,7 @@ class Vault {
     await this.refreshSources()
     const loaded = await this.registry.load()
     if (loaded.corrupt) {
-      await this.rebuild()
+      await this.rebuild(undefined, { preserveState: false })
     }
     this.sweeper = new Sweeper(this)
     this.initialized = true
@@ -213,9 +294,14 @@ class Vault {
     return mode
   }
   async hashFile(filePath) {
+    if (this.workerIdleTimer) {
+      clearTimeout(this.workerIdleTimer)
+      this.workerIdleTimer = null
+    }
     if (!this.worker) {
       const worker = new Worker(path.resolve(__dirname, "hash_worker.js"))
       this.worker = worker
+      this.workerStarts += 1
       worker.unref()
       worker.on("message", ({ id, hash, size, error }) => {
         const job = this.workerJobs.get(id)
@@ -223,22 +309,31 @@ class Vault {
         this.workerJobs.delete(id)
         if (error) job.reject(new Error(error))
         else job.resolve({ hash, size })
-        // Terminate when idle: lingering workers hold pipe handles that can
-        // keep processes alive (observed under node --test); respawning for
-        // the next batch costs ~30ms and hashing is idle-priority anyway.
+        // Keep the worker warm across the scan queue. Starting one worker per
+        // file costs ~30ms and adds up quickly on a first scan.
         if (this.workerJobs.size === 0 && this.worker === worker) {
-          this.worker = null
-          worker.terminate().catch(() => {})
+          this.workerIdleTimer = setTimeout(() => {
+            this.workerIdleTimer = null
+            if (this.workerJobs.size === 0 && this.worker === worker) {
+              this.worker = null
+              worker.terminate().catch(() => {})
+            }
+          }, this.workerIdleMs)
+          if (this.workerIdleTimer.unref) this.workerIdleTimer.unref()
         }
       })
       worker.on("error", (error) => {
         if (this.worker !== worker) return
+        if (this.workerIdleTimer) clearTimeout(this.workerIdleTimer)
+        this.workerIdleTimer = null
         for (const job of this.workerJobs.values()) job.reject(error)
         this.workerJobs.clear()
         this.worker = null
       })
       worker.on("exit", (code) => {
         if (this.worker !== worker) return
+        if (this.workerIdleTimer) clearTimeout(this.workerIdleTimer)
+        this.workerIdleTimer = null
         for (const job of this.workerJobs.values()) {
           job.reject(new Error(`hash worker exited with code ${code}`))
         }
@@ -314,7 +409,6 @@ class Vault {
     if (storeStat.size !== targetStat.size) {
       return { status: "size-mismatch" }
     }
-    const volumeMode = this.volumeModes.get(targetStat.dev)
     const tmp = targetPath + TMP_SUFFIX
     try {
       try {
@@ -330,13 +424,12 @@ class Vault {
       await fs.promises.rename(tmp, targetPath)
     } catch (e) {
       await fs.promises.unlink(tmp).catch(() => {})
-      if (NO_LINK_CODES.has(e.code) && volumeMode !== "link") {
-        this.registry.addLink(targetPath, { hash, app: meta.app || null, source_id: meta.source_id || null, dev: targetStat.dev, ino: targetStat.ino, mode: "copy" })
-        return { status: "copy-mode" }
-      }
       if (LOCK_CODES.has(e.code)) {
         return { status: "locked" }
       }
+      if (NO_LINK_CODES.has(e.code)) {
+        return { status: "unavailable", code: e.code }
+      }
       throw e
     }
     const st = await fs.promises.stat(targetPath)
@@ -413,11 +506,22 @@ class Vault {
     }
     this.registry.schedulePersist()
   }
-  // Rebuild from disk alone: store filenames are hashes, stat gives nlink,
-  // (dev, ino) matching re-associates app paths. No hashing required.
-  async rebuild(roots) {
+  // Rebuild derived state from disk: store filenames are hashes, stat gives
+  // nlink, and (dev, ino) matching re-associates app paths. The replacement
+  // maps are assembled off to the side and swapped in only after the walk so
+  // a long repair never persists a half-reset registry.
+  async rebuild(roots, options = {}) {
     if (!this.enabled) return
-    this.registry.reset()
+    const registry = this.registry
+    const preserveState = options.preserveState !== false
+    const preserved = preserveState ? {
+      excluded: new Map(registry.excluded),
+      totals: Object.assign({}, registry.totals),
+      lastScan: registry.lastScan ? Object.assign({}, registry.lastScan) : null,
+      duplicates: new Map(registry.duplicates)
+    } : null
+    const rebuiltBlobs = new Map()
+    const rebuiltLinks = new Map()
     const storeInoToHash = new Map()
     let shards = []
     try {
@@ -432,8 +536,14 @@ class Vault {
         if (!/^[0-9a-f]{64}$/.test(name)) continue
         try {
           const st = await fs.promises.stat(path.resolve(this.blobRoot, shard, name))
-          this.registry.addBlob(name, { size: st.size })
-          this.registry.blobs.get(name).orphan = st.nlink === 1
+          const previous = registry.blobs.get(name)
+          rebuiltBlobs.set(name, {
+            size: st.size,
+            first_seen: previous && previous.first_seen ? previous.first_seen : Date.now(),
+            source_urls: previous && Array.isArray(previous.source_urls) ? [...previous.source_urls] : [],
+            verified_at: Date.now(),
+            orphan: st.nlink === 1
+          })
           storeInoToHash.set(`${st.dev}:${st.ino}`, name)
         } catch (e) {}
       }
@@ -443,34 +553,73 @@ class Vault {
       ? roots.map((root) => typeof root === "string" ? { root, source_id: null } : root)
       : this.scanRoots()
     for (const entry of walkRoots) {
-      await this.walkForInoMatch(entry.root, storeInoToHash, entry.source_id === "pinokio" ? null : entry.source_id)
+      await this.walkForInoMatch(
+        entry.root,
+        storeInoToHash,
+        entry.source_id === "pinokio" ? null : entry.source_id,
+        rebuiltLinks
+      )
+    }
+    const rebuiltDuplicates = new Map()
+    if (preserved) {
+      for (const [duplicatePath, duplicate] of preserved.duplicates) {
+        if (preserved.excluded.has(duplicatePath) || rebuiltLinks.has(duplicatePath) || !rebuiltBlobs.has(duplicate.hash)) continue
+        try {
+          const st = await fs.promises.stat(duplicatePath)
+          if (st.isFile() && (!duplicate.size || duplicate.size === st.size)) {
+            rebuiltDuplicates.set(duplicatePath, Object.assign({}, duplicate, { size: st.size }))
+          }
+        } catch (error) {}
+      }
     }
-    await this.registry.flush()
+    const rebuiltByIno = new Map()
+    for (const entry of rebuiltLinks.values()) {
+      rebuiltByIno.set(registry.inoKey(entry.dev, entry.ino), entry.hash)
+    }
+    registry.blobs = rebuiltBlobs
+    registry.links = rebuiltLinks
+    registry.scanIndex = new Map()
+    registry.duplicates = rebuiltDuplicates
+    registry.excluded = preserved ? preserved.excluded : new Map()
+    registry.lastScan = preserved ? preserved.lastScan : null
+    registry.totals = preserved ? preserved.totals : { lifetime_bytes_saved: 0 }
+    registry.byIno = rebuiltByIno
+    await registry.flush()
   }
-  async walkForInoMatch(dir, storeInoToHash, preferredSourceId = null) {
-    let entries = []
-    try {
-      entries = await fs.promises.readdir(dir, { withFileTypes: true })
-    } catch (e) { return }
-    for (const entry of entries) {
-      const full = path.resolve(dir, entry.name)
-      if (entry.isSymbolicLink()) continue
-      if (entry.isDirectory()) {
-        if (PRUNE_DIRS.has(entry.name)) continue
-        await this.walkForInoMatch(full, storeInoToHash, preferredSourceId)
-      } else if (entry.isFile()) {
-        try {
-          const st = await fs.promises.stat(full)
-          if (st.size < SIZE_THRESHOLD) continue
-          const hash = storeInoToHash.get(`${st.dev}:${st.ino}`)
-          if (hash) {
-            const source = this.sourceForPath(full, preferredSourceId)
-            this.registry.addLink(full, {
-              hash, app: source && source.kind === "app" ? source.app : null,
-              source_id: source ? source.id : null, dev: st.dev, ino: st.ino, mode: "link"
-            })
+  async walkForInoMatch(root, storeInoToHash, preferredSourceId = null, outputLinks = null) {
+    const stack = [root]
+    const vaultRoot = path.resolve(this.root)
+    while (stack.length) {
+      const dir = stack.pop()
+      let entries = []
+      try {
+        entries = await fs.promises.readdir(dir, { withFileTypes: true })
+      } catch (error) { continue }
+      const files = []
+      for (const entry of entries) {
+        const full = path.resolve(dir, entry.name)
+        if (entry.isSymbolicLink()) continue
+        if (entry.isDirectory()) {
+          if (full === vaultRoot) continue
+          stack.push(full)
+        }
+        else if (entry.isFile()) files.push(full)
+      }
+      const stats = await statMany(files, this.statConcurrency)
+      for (let index = 0; index < files.length; index++) {
+        const st = stats[index]
+        if (!st || st.size < this.sizeThreshold) continue
+        const hash = storeInoToHash.get(`${st.dev}:${st.ino}`)
+        if (hash) {
+          const source = this.sourceForPath(files[index], preferredSourceId)
+          const link = {
+            hash, app: source && source.kind === "app" ? source.app : null,
+            source_id: source ? source.id : null, dev: st.dev, ino: st.ino,
+            mode: "link", created: Date.now()
           }
-        } catch (e) {}
+          if (outputLinks) outputLinks.set(files[index], link)
+          else this.registry.addLink(files[index], link)
+        }
       }
     }
   }
@@ -484,19 +633,21 @@ class Vault {
     if (!this.enabled || !this.registry) return { enabled: false }
     const registry = this.registry
     const blobs = []
+    const blobByHash = new Map()
     const storeStats = new Map()
+    const namesByHash = new Map()
+    for (const [linkPath, entry] of registry.links) {
+      if (!namesByHash.has(entry.hash)) namesByHash.set(entry.hash, [])
+      const location = this.locationForPath(linkPath, entry.source_id)
+      namesByHash.get(entry.hash).push(Object.assign({
+        path: linkPath, app: entry.app || null, mode: entry.mode
+      }, location))
+    }
     let bytesOnDisk = 0
     let wouldBe = 0
     for (const [hash, blob] of registry.blobs) {
-      const names = []
-      const apps = new Set()
-      for (const [linkPath, entry] of registry.links) {
-        if (entry.hash === hash) {
-          const location = this.locationForPath(linkPath, entry.source_id)
-          names.push(Object.assign({ path: linkPath, app: entry.app || null, mode: entry.mode }, location))
-          if (entry.app) apps.add(entry.app)
-        }
-      }
+      const names = namesByHash.get(hash) || []
+      const apps = new Set(names.map((name) => name.app).filter(Boolean))
       let nlink = null
       let storeStat = null
       try {
@@ -504,12 +655,16 @@ class Vault {
         nlink = storeStat.nlink
       } catch (e) {}
       storeStats.set(hash, storeStat)
-      bytesOnDisk += blob.size || 0
-      wouldBe += (blob.size || 0) * Math.max(1, names.length)
-      blobs.push({
+      const size = blob.size || 0
+      const physicalCopies = Math.max(1, (storeStat ? 1 : 0) + names.filter((name) => name.mode === "copy").length)
+      bytesOnDisk += size * physicalCopies
+      wouldBe += size * Math.max(physicalCopies, names.length)
+      const publicBlob = {
         hash, size: blob.size || 0, orphan: !!blob.orphan, nlink,
         names, apps: [...apps], source_urls: blob.source_urls || []
-      })
+      }
+      blobs.push(publicBlob)
+      blobByHash.set(hash, publicBlob)
     }
     const duplicates = []
     for (const [p, entry] of registry.duplicates) {
@@ -518,20 +673,21 @@ class Vault {
       const index = registry.scanIndex.get(p)
       const storeStat = storeStats.get(entry.hash)
       const shareable = !!(source && source.shareable && storeStat && (!index || index.dev === undefined || index.dev === storeStat.dev))
-      const blob = blobs.find((item) => item.hash === entry.hash)
+      const blob = blobByHash.get(entry.hash)
       duplicates.push(Object.assign({
         path: p, hash: entry.hash, size: entry.size || 0, app: entry.app || null,
         shareable, matches: blob ? blob.names : []
       }, location))
     }
     const events = (await registry.readEvents()).slice(-100).reverse()
-    const sweeper = this.sweeper
-    const scan = sweeper ? Object.assign({}, sweeper.state, {
-      current_file: sweeper.currentHash ? path.basename(sweeper.currentHash.path) : null
-    }) : null
+    const scan = this.scanStatus()
     const excluded = []
     for (const [p, meta] of registry.excluded) {
-      excluded.push(Object.assign({ path: p, ts: meta.ts || null }, this.locationForPath(p, meta.source_id)))
+      let size = Number(meta.size) || 0
+      if (!size) {
+        try { size = (await fs.promises.stat(p)).size } catch (error) {}
+      }
+      excluded.push(Object.assign({ path: p, ts: meta.ts || null, size }, this.locationForPath(p, meta.source_id)))
     }
     const publicSources = this._sources.map((source) => ({
       id: source.id, kind: source.kind, label: source.label, root: source.root,
@@ -555,6 +711,19 @@ class Vault {
       sources: publicSources, blobs, duplicates, excluded, events
     }
   }
+  scanStatus() {
+    const sweeper = this.sweeper
+    return sweeper ? Object.assign({}, sweeper.state, {
+      current_file: sweeper.currentHash ? path.basename(sweeper.currentHash.path) : null
+    }) : null
+  }
+  progressStatus() {
+    return {
+      enabled: !!this.enabled,
+      scan: this.scanStatus(),
+      last_scan: this.registry ? this.registry.lastScan : null
+    }
+  }
   // detach: make one name an independent copy again ("go back"), and pin it
   // so future scans neither re-list nor re-share it. Works on linked names
   // (copies bytes out) and on pending duplicates (just ignores them).
@@ -562,22 +731,22 @@ class Vault {
     if (!this.enabled) return { status: "disabled" }
     const entry = this.registry.links.get(filePath)
     if (entry) {
-      const storePath = this.storePathFor(entry.hash)
       const tmp = filePath + TMP_SUFFIX
       await fs.promises.copyFile(filePath, tmp)
       await fs.promises.rename(tmp, filePath)
+      const st = await fs.promises.stat(filePath)
       this.registry.removeLink(filePath)
-      this.registry.excluded.set(filePath, { ts: Date.now(), source_id: entry.source_id || null })
+      this.registry.excluded.set(filePath, { ts: Date.now(), source_id: entry.source_id || null, size: st.size })
       this.registry.schedulePersist()
-      await this.registry.appendEvent({ kind: "detach", hash: entry.hash, path: filePath, app: entry.app || null, source_id: entry.source_id || null })
+      await this.registry.appendEvent({ kind: "detach", hash: entry.hash, path: filePath, app: entry.app || null, source_id: entry.source_id || null, size: st.size })
       return { status: "detached" }
     }
     if (this.registry.duplicates.has(filePath)) {
       const dup = this.registry.duplicates.get(filePath)
       this.registry.duplicates.delete(filePath)
-      this.registry.excluded.set(filePath, { ts: Date.now(), source_id: dup.source_id || null })
+      this.registry.excluded.set(filePath, { ts: Date.now(), source_id: dup.source_id || null, size: dup.size || 0 })
       this.registry.schedulePersist()
-      await this.registry.appendEvent({ kind: "detach", hash: dup.hash, path: filePath, app: dup.app || null, source_id: dup.source_id || null })
+      await this.registry.appendEvent({ kind: "detach", hash: dup.hash, path: filePath, app: dup.app || null, source_id: dup.source_id || null, size: dup.size || 0 })
       return { status: "ignored" }
     }
     return { status: "not-found" }
@@ -620,7 +789,8 @@ class Vault {
           this.registry.totals.lifetime_bytes_saved - (event.bytes_saved || 0))
         await this.registry.appendEvent({
           kind: "undo", hash: event.hash, path: event.path,
-          source_id: entry.source_id || event.source_id || null, batch_id: batchId
+          source_id: entry.source_id || event.source_id || null, batch_id: batchId,
+          bytes_saved: event.bytes_saved || storeStat.size
         })
         summary.undone += 1
         summary.bytes += storeStat.size
@@ -646,6 +816,5 @@ class Vault {
 
 Vault.SIZE_THRESHOLD = SIZE_THRESHOLD
 Vault.TMP_SUFFIX = TMP_SUFFIX
-Vault.PRUNE_DIRS = PRUNE_DIRS
 
 module.exports = Vault

package/kernel/vault/registry.js

@@ -19,8 +19,8 @@ class Registry {
     this.links = new Map()      // path -> { hash, app, source_id, dev, ino, created, mode }
     this.scanIndex = new Map()  // path -> { size, mtime, dev, ino, hash, source_id }
     this.duplicates = new Map() // path -> { hash, size, app, source_id, discovered } pending user conversion
-    this.excluded = new Map()   // path -> { ts, source_id } user chose "keep independent"
-    this.lastScan = null        // { ts, dirs, files, bytes_total, home_bytes_total, source_bytes }
+    this.excluded = new Map()   // path -> { ts, source_id, size } user chose "keep independent"
+    this.lastScan = null        // scan totals, duration, and hash work
     this.totals = { lifetime_bytes_saved: 0 }
     this.byIno = new Map()      // "dev:ino" -> hash
   }

package/kernel/vault/sweeper.js

@@ -12,6 +12,7 @@ const fastq = require('fastq')
 
 const SHA256_RE = /^[0-9a-f]{64}$/
 const TMP_SUFFIX = ".pinokio-dedup-tmp"
+const STAT_CONCURRENCY = 32
 
 class Sweeper {
   constructor(vault) {
@@ -21,31 +22,40 @@ class Sweeper {
     this.currentHash = null
     this.hashQueue = fastq.promise(this, this._hashJob, 1)
     this.stats = { hashes: 0, ingested: 0 }
+    this.statConcurrency = vault.statConcurrency || STAT_CONCURRENCY
+    this.metadataBaseCache = new Map()
   }
   idleState() {
     return {
       active: false, dirs: 0, files: 0, bytes_total: 0,
       home_bytes_total: 0, source_bytes: {},
-      candidates: 0, hashed: 0, queued: 0, started: null
+      candidates: 0, hashed: 0, hash_bytes: 0, queued: 0,
+      started: null, duration_ms: null
     }
   }
   // The one entry point: the Pinokio home plus explicit linked imports.
   async scan() {
     if (this.state.active) return { already_running: true }
     this.state = Object.assign(this.idleState(), { active: true, started: Date.now() })
+    this.metadataBaseCache.clear()
     try {
       await this.vault.refreshSources()
       for (const source of this.vault.scanRoots()) {
         await this.walk(source.root, source.source_id === "pinokio" ? null : source.source_id)
       }
       await this.settle()
+      this.state.duration_ms = Date.now() - this.state.started
       this.vault.registry.setLastScan({
         ts: Date.now(),
         dirs: this.state.dirs,
         files: this.state.files,
         bytes_total: this.state.bytes_total,
         home_bytes_total: this.state.home_bytes_total,
-        source_bytes: Object.assign({}, this.state.source_bytes)
+        source_bytes: Object.assign({}, this.state.source_bytes),
+        candidates: this.state.candidates,
+        hashed: this.state.hashed,
+        hash_bytes: this.state.hash_bytes,
+        duration_ms: this.state.duration_ms
       })
     } finally {
       this.state.active = false
@@ -81,21 +91,46 @@ class Sweeper {
         if (entry.isDirectory()) {
           if (full === vaultRoot) continue
           stack.push(full)
-        } else if (entry.isFile()) {
-          await this.considerFile(full, preferredSourceId)
         }
       }
+      const files = entries.filter((entry) => entry.isFile() && !entry.isSymbolicLink())
+        .map((entry) => path.resolve(dir, entry.name))
+      await this.considerFiles(files, preferredSourceId)
+    }
+  }
+  async considerFiles(filePaths, preferredSourceId = null) {
+    const stats = new Array(filePaths.length)
+    let cursor = 0
+    const workers = Array.from({ length: Math.min(this.statConcurrency, filePaths.length) }, async () => {
+      while (cursor < filePaths.length) {
+        const index = cursor++
+        try {
+          stats[index] = await fs.promises.stat(filePaths[index])
+        } catch (error) {
+          stats[index] = null
+        }
+      }
+    })
+    await Promise.all(workers)
+    // Preserve deterministic classification order while parallelizing only
+    // the expensive filesystem metadata reads.
+    for (let index = 0; index < filePaths.length; index++) {
+      if (stats[index]) await this.considerStat(filePaths[index], stats[index], preferredSourceId)
     }
   }
   async considerFile(filePath, preferredSourceId = null) {
     if (filePath.endsWith(TMP_SUFFIX)) return
-    const registry = this.vault.registry
     let st
     try {
       st = await fs.promises.stat(filePath)
     } catch (e) {
       return
     }
+    await this.considerStat(filePath, st, preferredSourceId)
+  }
+  async considerStat(filePath, st, preferredSourceId = null) {
+    if (filePath.endsWith(TMP_SUFFIX)) return
+    const registry = this.vault.registry
     this.state.files += 1
     this.state.bytes_total += st.size
     const sourceKey = preferredSourceId || "pinokio"
@@ -136,9 +171,10 @@ class Sweeper {
   async _hashJob({ filePath, source_id }) {
     this.currentHash = { path: filePath }
     try {
-      const { hash } = await this.vault.hashFile(filePath)
+      const { hash, size } = await this.vault.hashFile(filePath)
       this.stats.hashes += 1
       this.state.hashed += 1
+      this.state.hash_bytes += size || 0
       const st = await fs.promises.stat(filePath)
       await this.classify(filePath, st, hash, source_id)
     } catch (e) {
@@ -224,35 +260,43 @@ class Sweeper {
   // via the recorded timestamp against mtime (huggingface_hub's own rule).
   async harvestLocalDirEtag(filePath, st) {
     try {
-      let dir = path.dirname(filePath)
+      const start = path.dirname(filePath)
+      let dir = start
       const stop = path.resolve(this.kernel.homedir)
-      const rel = []
+      const visited = []
+      let base = null
       for (let depth = 0; depth < 12; depth++) {
+        if (this.metadataBaseCache.has(dir)) {
+          base = this.metadataBaseCache.get(dir)
+          break
+        }
+        visited.push(dir)
         const metaRoot = path.resolve(dir, ".cache", "huggingface")
         let exists = false
         try {
           exists = (await fs.promises.stat(metaRoot)).isDirectory()
         } catch (e) {}
         if (exists) {
-          const metaPath = path.resolve(metaRoot, ...rel, path.basename(filePath) + ".metadata")
-          let raw
-          try {
-            raw = await fs.promises.readFile(metaPath, "utf8")
-          } catch (e) {
-            return null
-          }
-          const lines = raw.split("\n")
-          const etag = String(lines[1] || "").replace(/"/g, "").trim().toLowerCase()
-          const ts = parseFloat(lines[2] || "0") * 1000
-          if (SHA256_RE.test(etag) && st.mtimeMs <= ts + 1000) {
-            return etag
-          }
-          return null
+          base = dir
+          break
         }
-        if (dir === stop || path.dirname(dir) === dir) return null
-        rel.unshift(path.basename(dir))
+        if (dir === stop || path.dirname(dir) === dir) break
         dir = path.dirname(dir)
       }
+      for (const visitedDir of visited) this.metadataBaseCache.set(visitedDir, base)
+      if (!base) return null
+      const relative = path.relative(base, filePath)
+      const metaPath = path.resolve(base, ".cache", "huggingface", relative + ".metadata")
+      let raw
+      try {
+        raw = await fs.promises.readFile(metaPath, "utf8")
+      } catch (e) {
+        return null
+      }
+      const lines = raw.split("\n")
+      const etag = String(lines[1] || "").replace(/"/g, "").trim().toLowerCase()
+      const ts = parseFloat(lines[2] || "0") * 1000
+      if (SHA256_RE.test(etag) && st.mtimeMs <= ts + 1000) return etag
     } catch (e) {}
     return null
   }
@@ -293,8 +337,9 @@ class Sweeper {
           summary.bytes_saved += result.bytes_saved || 0
         } else if (result.status === "locked") {
           summary.locked += 1
+        } else if (result.status === "unavailable" || result.status === "copy-mode") {
+          summary.unavailable += 1
         } else {
-          registry.duplicates.delete(filePath)
           summary.failed += 1
         }
       } catch (e) {