pinokiod 8.0.39 → 8.0.41

package/kernel/vault/sweeper.js

@@ -1,6 +1,31 @@
 const fs = require('fs')
 const path = require('path')
 const fastq = require('fastq')
+const { walkBatches, statMany } = require('./walker')
+const { fileSnapshot, sameSnapshot, sameContentState } = require('./snapshot')
+const { SHA256_RE, TMP_SUFFIX, DIR_CONCURRENCY, STAT_CONCURRENCY, HASH_QUEUE_LIMIT } = require('./constants')
+
+const isMissingError = (error) => !!(error && (error.code === "ENOENT" || error.code === "ENOTDIR"))
+const isAccessError = (error) => !!(error && (error.code === "EACCES" || error.code === "EPERM"))
+const isPathWithin = (root, target) => {
+  const rel = path.relative(path.resolve(root), path.resolve(target))
+  return rel === "" || (!rel.startsWith(`..${path.sep}`) && rel !== ".." && !path.isAbsolute(rel))
+}
+const readFileNoFollow = async (filePath) => {
+  const before = await fs.promises.lstat(filePath)
+  if (!before.isFile()) return null
+  const handle = await fs.promises.open(
+    filePath,
+    fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW || 0)
+  )
+  try {
+    const opened = await handle.stat()
+    if (!opened.isFile() || opened.dev !== before.dev || opened.ino !== before.ino) return null
+    return await handle.readFile("utf8")
+  } finally {
+    await handle.close()
+  }
+}
 
 // Manual scan engine (spec/requirements/shared-model-store.md).
 // Scans run ONLY when the user asks — there are no automatic triggers.
@@ -10,28 +35,6 @@ const fastq = require('fastq')
 // which mutates nothing) and lists byte-identical copies as pending, for the
 // user to deduplicate explicitly.
 
-const SHA256_RE = /^[0-9a-f]{64}$/
-const TMP_SUFFIX = ".pinokio-dedup-tmp"
-const DIR_CONCURRENCY = 8
-const STAT_CONCURRENCY = 32
-
-const fileSnapshot = (st) => ({
-  dev: st.dev,
-  ino: st.ino,
-  size: st.size,
-  mtime: st.mtimeMs,
-  ctime: st.ctimeMs
-})
-
-const sameSnapshot = (snapshot, st) => !!(
-  snapshot && st &&
-  snapshot.dev === st.dev &&
-  snapshot.ino === st.ino &&
-  snapshot.size === st.size &&
-  snapshot.mtime === st.mtimeMs &&
-  snapshot.ctime === st.ctimeMs
-)
-
 class Sweeper {
   constructor(vault) {
     this.vault = vault
@@ -39,47 +42,87 @@ class Sweeper {
     this.state = this.idleState()
     this.currentHash = null
     this.hashQueue = fastq.promise(this, this._hashJob, 1)
-    this.stats = { hashes: 0, ingested: 0 }
     this.statConcurrency = vault.statConcurrency || STAT_CONCURRENCY
     this.dirConcurrency = vault.dirConcurrency || DIR_CONCURRENCY
+    this.hashQueueLimit = vault.hashQueueLimit || HASH_QUEUE_LIMIT
+    this.hashCapacityWaiters = []
     this.metadataBaseCache = new Map()
     this.hashJobsByIno = new Map()
+    this.completedHashesByIno = new Map()
+    this.inaccessiblePaths = new Set()
+    this.inaccessibleCodes = new Map()
   }
   idleState() {
     return {
-      active: false, dirs: 0, files: 0, bytes_total: 0,
-      home_bytes_total: 0, source_bytes: {},
+      active: false, phase: "idle", dirs: 0, files: 0, bytes_total: 0,
+      counted_dirs: 0, counted_files: 0, total_files: null, count_estimate_files: null,
+      home_bytes_total: 0, source_bytes: {}, source_files: {}, source_hash_failures: {}, scope_id: null,
       candidates: 0, hashed: 0, hash_total: 0, hash_bytes: 0, queued: 0,
-      inode_reuses: 0, unstable_hashes: 0,
-      source_ids: [], estimated_files: null, estimated_walk_weight: null,
+      inode_reuses: 0, unstable_hashes: 0, hash_failures: 0,
+      inaccessible: 0, inaccessible_paths: [],
+      estimated_count_weight: 0.4, estimated_walk_weight: 0.5,
       started: null, duration_ms: null,
-      walk_duration_ms: null, hash_wait_duration_ms: null,
+      count_duration_ms: null, walk_duration_ms: null, hash_wait_duration_ms: null,
       hash_duration_ms: 0
     }
   }
-  // The one entry point: the Pinokio home plus explicit linked imports.
-  async scan() {
+  // A global scan covers Pinokio plus explicit imports. A scoped scan covers
+  // exactly one physical source (used by an app's Save space page).
+  async scan(scopeId = null) {
     if (this.state.active) return { already_running: true }
-    this.state = Object.assign(this.idleState(), { active: true, started: Date.now() })
+    this.state = Object.assign(this.idleState(), {
+      active: true, phase: "counting", started: Date.now(), scope_id: scopeId
+    })
     this.metadataBaseCache.clear()
     this.hashJobsByIno.clear()
+    this.completedHashesByIno.clear()
+    this.inaccessiblePaths.clear()
+    this.inaccessibleCodes.clear()
+    this.fatalError = null
+    const registryWasDirty = !!(
+      this.vault.registry.persistDirty || this.vault.registry.persistTimer
+    )
+    const registrySnapshot = this.vault.registry.snapshotState()
+    this.vault.registry.beginBatch()
+    this.vault.beginScanDraft()
+    let completed = false
+    let incomplete = false
+    let rollbackMetadataChanged = false
     try {
       await this.vault.refreshSources()
-      const scanRoots = this.vault.scanRoots()
-      this.state.source_ids = scanRoots.map((source) => source.source_id).sort()
-      const previous = this.vault.registry.lastScan
-      const previousSourceIds = previous
-        ? (Array.isArray(previous.source_ids) ? previous.source_ids : Object.keys(previous.source_bytes || {})).slice().sort()
-        : []
-      const sameSources = previousSourceIds.length === this.state.source_ids.length &&
-        previousSourceIds.every((sourceId, index) => sourceId === this.state.source_ids[index])
-      if (previous && previous.files > 0 && sameSources) {
-        this.state.estimated_files = previous.files
-        const measuredWeight = previous.duration_ms > 0 && previous.walk_duration_ms >= 0
-          ? previous.walk_duration_ms / previous.duration_ms
-          : 0.8
-        this.state.estimated_walk_weight = Math.max(0.15, Math.min(0.98, measuredWeight))
+      if (!scopeId) this.vault.reconcileConfiguredSources()
+      if (!scopeId) {
+        for (const source of this.vault.sources()) {
+          if (source.kind === "external" && source.available === false && source.root) {
+            this.recordUnavailable(source.root, source.error_code)
+          }
+        }
+      }
+      const scanRoots = this.vault.scanRoots(scopeId)
+      if (!scanRoots.length) throw new Error("That scan location is no longer available.")
+      const previous = this.vault.registry.scanFor(scopeId)
+      if (previous) {
+        if (Number.isFinite(previous.files) && previous.files > 0) {
+          this.state.count_estimate_files = previous.files
+        }
+        const duration = Math.max(0, Number(previous.duration_ms) || 0)
+        const walkDuration = Math.max(0, Number(previous.walk_duration_ms) || 0)
+        const waitDuration = Math.max(0, Number(previous.hash_wait_duration_ms) || 0)
+        const countDuration = Number.isFinite(previous.count_duration_ms)
+          ? Math.max(0, previous.count_duration_ms)
+          : Math.max(0, duration - walkDuration - waitDuration)
+        const finishDuration = Math.max(1, duration - countDuration - walkDuration)
+        const measuredDuration = countDuration + walkDuration + finishDuration
+        if (measuredDuration > 1) {
+          this.state.estimated_count_weight = countDuration / measuredDuration
+          this.state.estimated_walk_weight = walkDuration / measuredDuration
+        }
       }
+      const countStarted = Date.now()
+      for (const source of scanRoots) await this.countFiles(source.root)
+      this.state.count_duration_ms = Date.now() - countStarted
+      this.state.total_files = this.state.counted_files
+      this.state.phase = "discovering"
       const walkStarted = Date.now()
       for (const source of scanRoots) {
         if (!Object.prototype.hasOwnProperty.call(this.state.source_bytes, source.source_id)) {
@@ -88,80 +131,219 @@ class Sweeper {
         await this.walk(source.root, source.source_id === "pinokio" ? null : source.source_id)
       }
       this.state.walk_duration_ms = Date.now() - walkStarted
+      this.state.phase = "analyzing"
       const hashWaitStarted = Date.now()
       await this.settle()
       this.state.hash_wait_duration_ms = Date.now() - hashWaitStarted
-      this.state.duration_ms = Date.now() - this.state.started
-      this.vault.registry.setLastScan({
+      const scanMeta = {
         ts: Date.now(),
+        complete: true,
+        candidate_min_bytes: this.vault.sizeThreshold,
         dirs: this.state.dirs,
         files: this.state.files,
         bytes_total: this.state.bytes_total,
         home_bytes_total: this.state.home_bytes_total,
         source_bytes: Object.assign({}, this.state.source_bytes),
-        source_ids: this.state.source_ids.slice(),
+        source_files: Object.assign({}, this.state.source_files),
+        source_hash_failures: Object.assign({}, this.state.source_hash_failures),
         candidates: this.state.candidates,
         hashed: this.state.hashed,
         hash_total: this.state.hash_total,
         hash_bytes: this.state.hash_bytes,
         inode_reuses: this.state.inode_reuses,
         unstable_hashes: this.state.unstable_hashes,
+        hash_failures: this.state.hash_failures,
+        count_duration_ms: this.state.count_duration_ms,
         walk_duration_ms: this.state.walk_duration_ms,
         hash_wait_duration_ms: this.state.hash_wait_duration_ms,
         hash_duration_ms: this.state.hash_duration_ms,
-        duration_ms: this.state.duration_ms
+        unreadable: []
+      }
+      // Refresh dead-name and orphan state from filesystem identity after
+      // every completed discovery pass, as required by the Vault contract.
+      this.state.phase = "verifying"
+      await this.vault.verify({
+        includePath: scopeId
+          ? (filePath) => isPathWithin(scanRoots[0].root, filePath)
+          : undefined,
+        reconcile: !scopeId,
+        verifyBlobs: !scopeId,
+        preservePath: (filePath) => this.isInaccessible(filePath),
+        onAccessError: (error, filePath) => this.recordInaccessible(error, filePath)
       })
+      this.state.duration_ms = Date.now() - this.state.started
+      incomplete = this.state.inaccessible > 0
+      scanMeta.ts = Date.now()
+      scanMeta.complete = !incomplete
+      scanMeta.duration_ms = this.state.duration_ms
+      scanMeta.unreadable = [...this.inaccessiblePaths].map((filePath) => ({
+        path: filePath,
+        code: this.inaccessibleCodes.get(filePath) || "EACCES"
+      }))
+      this.refreshExcludedVerification(scanRoots)
+      if (incomplete) this.markUnavailableRows()
+      this.vault.registry.setScanAttempt(scanMeta, scopeId)
+      if (!incomplete) {
+        this.vault.registry.setLastScan(scanMeta, scopeId)
+      }
+      if (!scopeId) {
+        for (const source of this.vault.sources()) {
+          if (source.kind !== "app" || !source.available) continue
+          const sourceId = source.id
+          const sourceIncomplete = [...this.inaccessiblePaths].some((unreadable) =>
+            isPathWithin(source.root, unreadable) || isPathWithin(unreadable, source.root))
+          const bytes = scanMeta.source_bytes[sourceId] || 0
+          const scopedMeta = {
+            ts: scanMeta.ts,
+            complete: !sourceIncomplete,
+            duration_ms: scanMeta.duration_ms,
+            count_duration_ms: scanMeta.count_duration_ms,
+            walk_duration_ms: scanMeta.walk_duration_ms,
+            hash_wait_duration_ms: scanMeta.hash_wait_duration_ms,
+            candidate_min_bytes: scanMeta.candidate_min_bytes,
+            files: scanMeta.source_files[sourceId] || 0,
+            bytes_total: bytes,
+            home_bytes_total: 0,
+            hash_failures: scanMeta.source_hash_failures[sourceId] || 0,
+            source_bytes: { [sourceId]: bytes },
+            source_files: { [sourceId]: scanMeta.source_files[sourceId] || 0 },
+            source_hash_failures: {
+              [sourceId]: scanMeta.source_hash_failures[sourceId] || 0
+            },
+            unreadable: sourceIncomplete
+              ? scanMeta.unreadable.filter((entry) =>
+                  isPathWithin(source.root, entry.path) || isPathWithin(entry.path, source.root))
+              : []
+          }
+          this.vault.registry.setScanAttempt(scopedMeta, sourceId)
+          if (!sourceIncomplete) this.vault.registry.setLastScan(scopedMeta, sourceId)
+        }
+      }
+      await this.vault.commitScanDraft()
+      completed = true
+    } catch (error) {
+      // Drain every queued hash before restoring the snapshot; otherwise a
+      // late worker could mutate the restored registry after rollback.
+      await this.settle().catch(() => {})
+      rollbackMetadataChanged = await this.vault.abortScanDraft(registrySnapshot)
+      throw error
     } finally {
       // A failed walk must not leave background hash work or per-scan inode
       // state alive when the next manual scan starts.
       await this.settle().catch(() => {})
+      await this.vault.registry.eventPromise
       this.hashJobsByIno.clear()
+      this.completedHashesByIno.clear()
       this.state.active = false
+      this.state.phase = completed ? (incomplete ? "incomplete" : "complete") : "failed"
+      try {
+        await this.vault.registry.endBatch(completed ? { flush: true } : { discard: true })
+      } catch (error) {
+        if (!completed) throw error
+        this.vault.scanPersistenceWarning = true
+      }
+      if (!completed && (registryWasDirty || rollbackMetadataChanged)) {
+        this.vault.registry.schedulePersist()
+      }
     }
     return {
       dirs: this.state.dirs, files: this.state.files,
-      bytes_total: this.state.bytes_total, candidates: this.state.candidates
+      bytes_total: this.state.bytes_total, candidates: this.state.candidates,
+      incomplete, inaccessible: this.state.inaccessible,
+      inaccessible_paths: this.state.inaccessible_paths.slice()
+    }
+  }
+  recordInaccessible(error, filePath) {
+    if (!isAccessError(error)) return false
+    return this.recordUnavailable(filePath, error.code)
+  }
+  recordUnavailable(filePath, code = null) {
+    const target = path.resolve(filePath)
+    if ([...this.inaccessiblePaths].some((existing) => isPathWithin(existing, target))) return true
+    for (const existing of [...this.inaccessiblePaths]) {
+      if (isPathWithin(target, existing)) {
+        this.inaccessiblePaths.delete(existing)
+        this.inaccessibleCodes.delete(existing)
+      }
+    }
+    this.inaccessiblePaths.add(target)
+    this.inaccessibleCodes.set(target, code || "EACCES")
+    this.state.inaccessible = this.inaccessiblePaths.size
+    this.state.inaccessible_paths = [...this.inaccessiblePaths].slice(0, 20)
+    return true
+  }
+  isInaccessible(filePath) {
+    return [...this.inaccessiblePaths].some((target) => isPathWithin(target, filePath))
+  }
+  markUnavailableRows() {
+    const mark = (entries) => {
+      for (const [filePath, entry] of entries) {
+        if (!this.isInaccessible(filePath)) continue
+        entry.unverified = true
+      }
+    }
+    mark(this.vault.registry.links)
+    mark(this.vault.registry.duplicates)
+    mark(this.vault.registry.excluded)
+    this.vault.registry.schedulePersist()
+  }
+  refreshExcludedVerification(scanRoots) {
+    for (const [filePath, entry] of this.vault.registry.excluded) {
+      if (!scanRoots.some((source) => isPathWithin(source.root, filePath))) continue
+      entry.unverified = this.isInaccessible(filePath)
+    }
+    this.vault.registry.schedulePersist()
+  }
+  async countFiles(root) {
+    const vaultRoot = this.vault.root
+    for await (const directoryResults of walkBatches(root, {
+      concurrency: this.dirConcurrency,
+      skipDirectory: (full) => full === vaultRoot,
+      strictErrors: true,
+      onRootMissing: (error, filePath) =>
+        this.recordUnavailable(filePath, error && error.code),
+      onError: (error, filePath) => this.recordInaccessible(error, filePath)
+    })) {
+      this.state.counted_dirs += directoryResults.filter((group) => group.firstChunk).length
+      this.state.counted_files += directoryResults.reduce((sum, group) => sum +
+        group.files.filter((file) => !file.path.endsWith(TMP_SUFFIX)).length, 0)
     }
   }
   async settle() {
     await this.hashQueue.drained()
+    if (this.fatalError) {
+      const error = this.fatalError
+      this.fatalError = null
+      throw error
+    }
   }
   async walk(root, preferredSourceId = null) {
     const vaultRoot = this.vault.root
-    const stack = [root]
-    while (stack.length) {
-      const dirs = []
-      while (dirs.length < this.dirConcurrency && stack.length) {
-        dirs.push(stack.pop())
-      }
-      const directoryResults = await Promise.all(dirs.map(async (dir) => {
-        this.state.dirs += 1
-        try {
-          const entries = await fs.promises.readdir(dir, { withFileTypes: true })
-          return { dir, entries }
-        } catch (e) {
-          return { dir, entries: null }
-        }
-      }))
+    for await (const directoryResults of walkBatches(root, {
+      concurrency: this.dirConcurrency,
+      skipDirectory: (full) => full === vaultRoot,
+      strictErrors: true,
+      onRootMissing: (error, filePath) =>
+        this.recordUnavailable(filePath, error && error.code),
+      onError: (error, filePath) => this.recordInaccessible(error, filePath)
+    })) {
+      this.state.dirs += directoryResults.filter((group) => group.firstChunk).length
       const files = []
       const hfDirectories = []
-      for (const { dir, entries } of directoryResults) {
+      for (const { dir, entries, files: directoryFiles } of directoryResults) {
         if (!entries) continue
-        // HF hub blobs dir: filenames ARE sha256 for LFS files — no hashing.
-        if (path.basename(dir) === "blobs" && /^(models|datasets|spaces)--/.test(path.basename(path.dirname(dir)))) {
-          hfDirectories.push({ dir, entries })
-          continue
-        }
-        for (const entry of entries) {
-          if (entry.isSymbolicLink()) continue
-          const full = path.resolve(dir, entry.name)
-          if (entry.isDirectory()) {
-            if (full === vaultRoot) continue
-            stack.push(full)
-          } else if (entry.isFile()) {
-            files.push(full)
+        const isHfBlobDirectory = path.basename(dir) === "blobs" &&
+          /^(models|datasets|spaces)--/.test(path.basename(path.dirname(dir)))
+        const hashEntries = []
+        for (const file of directoryFiles) {
+          // A 64-hex HF blob name is a hash-free input. Everything else still
+          // follows the generic candidate path.
+          if (isHfBlobDirectory && file.entry.isFile() && SHA256_RE.test(file.entry.name)) {
+            hashEntries.push(file.entry)
           }
+          else files.push(file.path)
         }
+        if (hashEntries.length) hfDirectories.push({ dir, entries: hashEntries })
       }
       // One metadata pool is shared by all files in this directory batch, so
       // trees made of many small directories still use the bounded workers.
@@ -172,68 +354,75 @@ class Sweeper {
     }
   }
   async considerFiles(filePaths, preferredSourceId = null) {
-    const stats = new Array(filePaths.length)
-    let cursor = 0
-    const workers = Array.from({ length: Math.min(this.statConcurrency, filePaths.length) }, async () => {
-      while (cursor < filePaths.length) {
-        const index = cursor++
-        try {
-          stats[index] = await fs.promises.stat(filePaths[index])
-        } catch (error) {
-          stats[index] = null
-        }
-      }
+    const stats = await statMany(filePaths, this.statConcurrency, null, {
+      followSymlinks: false,
+      strictErrors: true,
+      onError: (error, filePath) => this.recordInaccessible(error, filePath)
     })
-    await Promise.all(workers)
     // Preserve deterministic classification order while parallelizing only
     // the expensive filesystem metadata reads.
     for (let index = 0; index < filePaths.length; index++) {
-      if (stats[index]) await this.considerStat(filePaths[index], stats[index], preferredSourceId)
+      if (stats[index] && stats[index].isFile()) {
+        await this.considerStat(filePaths[index], stats[index], preferredSourceId)
+      }
     }
   }
-  async considerFile(filePath, preferredSourceId = null) {
-    if (filePath.endsWith(TMP_SUFFIX)) return
-    let st
-    try {
-      st = await fs.promises.stat(filePath)
-    } catch (e) {
-      return
+  countFile(sourceId) {
+    this.state.files += 1
+    this.state.source_files[sourceId] = (this.state.source_files[sourceId] || 0) + 1
+    if (this.state.total_files !== null && this.state.files > this.state.total_files) {
+      this.state.total_files = this.state.files
     }
-    await this.considerStat(filePath, st, preferredSourceId)
   }
-  async considerStat(filePath, st, preferredSourceId = null) {
+  async considerStat(filePath, st, preferredSourceId = null, options = {}) {
     if (filePath.endsWith(TMP_SUFFIX)) return
     const registry = this.vault.registry
-    this.state.files += 1
-    this.state.bytes_total += st.size
-    const sourceKey = preferredSourceId || "pinokio"
-    this.state.source_bytes[sourceKey] = (this.state.source_bytes[sourceKey] || 0) + st.size
-    if (!preferredSourceId) this.state.home_bytes_total += st.size
-    if (st.size < this.vault.sizeThreshold) return
-    if (registry.excluded.has(filePath)) return
-    this.state.candidates += 1
     const source = this.vault.sourceForPath(filePath, preferredSourceId)
     const sourceId = source ? source.id : null
     const appName = source && source.kind === "app" ? source.app : null
+    const sourceKey = sourceId || preferredSourceId || "pinokio"
+    if (options.count !== false) {
+      this.countFile(sourceKey)
+      this.state.bytes_total += st.size
+      this.state.source_bytes[sourceKey] = (this.state.source_bytes[sourceKey] || 0) + st.size
+      if (!preferredSourceId) this.state.home_bytes_total += st.size
+    }
+    if (registry.excluded.has(filePath)) {
+      registry.untrack(filePath)
+      return
+    }
+    if (st.size < this.vault.sizeThreshold) {
+      if (this.preserveTrackedBelowThreshold(filePath, st, appName, sourceId)) return
+      if (!registry.links.has(filePath)) {
+        registry.untrack(filePath)
+        return
+      }
+    } else {
+      this.state.candidates += 1
+    }
     // Known and unchanged: nothing to do.
     const inoHash = registry.byIno.get(registry.inoKey(st.dev, st.ino))
     if (inoHash && st.nlink > 1) {
       const cached = registry.scanIndex.get(filePath)
-      const unchanged = cached && cached.size === st.size && cached.mtime === st.mtimeMs && cached.ino === st.ino
-      if (unchanged || !cached) {
-        if (!registry.links.has(filePath)) {
-          registry.addLink(filePath, { hash: inoHash, app: appName, source_id: sourceId, dev: st.dev, ino: st.ino, mode: "link" })
-        }
-        if (!cached) this.updateScanIndex(filePath, st, inoHash, sourceId)
+      const unchanged = cached && cached.hash === inoHash && sameSnapshot(cached, st)
+      if (unchanged) {
+        registry.addLink(filePath, {
+          hash: inoHash, app: appName, source_id: sourceId,
+          dev: st.dev, ino: st.ino, unverified: false
+        })
         return
       }
     }
     const cached = registry.scanIndex.get(filePath)
-    if (cached && cached.size === st.size && cached.mtime === st.mtimeMs && cached.ino === st.ino && cached.hash) {
+    if (cached && cached.hash && sameSnapshot(cached, st)) {
       await this.classify(filePath, st, cached.hash, sourceId)
       return
     }
-    const harvested = await this.harvestLocalDirEtag(filePath, st)
+    const harvested = await this.harvestLocalDirEtag(
+      filePath,
+      st,
+      source && source.root ? source.root : this.kernel.homedir
+    )
     if (harvested) {
       await this.classify(filePath, st, harvested, sourceId)
       return
@@ -250,11 +439,47 @@ class Sweeper {
       this.state.inode_reuses += 1
       return
     }
+    const completed = inodeKey ? this.completedHashesByIno.get(inodeKey) : null
+    if (completed && sameContentState(completed.expected, st)) {
+      this.state.inode_reuses += 1
+      await this.classify(filePath, st, completed.hash, sourceId)
+      return
+    }
+    if (completed) this.completedHashesByIno.delete(inodeKey)
     const job = { inodeKey, names: [name] }
     if (inodeKey) this.hashJobsByIno.set(inodeKey, job)
     this.state.hash_total += 1
     this.state.queued += 1
     this.hashQueue.push(job).catch(() => {})
+    if (this.state.queued >= this.hashQueueLimit) {
+      await new Promise((resolve) => this.hashCapacityWaiters.push(resolve))
+    }
+  }
+  invalidateLinkedInode(hash, dev, ino) {
+    const registry = this.vault.registry
+    for (const [filePath, entry] of [...registry.links]) {
+      if (entry.hash === hash && entry.dev === dev && entry.ino === ino) {
+        registry.untrack(filePath)
+      }
+    }
+    registry.removeBlob(hash)
+  }
+  preserveTrackedBelowThreshold(filePath, st, appName, sourceId) {
+    const registry = this.vault.registry
+    const linked = registry.links.get(filePath)
+    const cached = registry.scanIndex.get(filePath)
+    if (!linked || !cached || cached.hash !== linked.hash ||
+        linked.dev !== st.dev || linked.ino !== st.ino ||
+        !sameSnapshot(cached, st)) return false
+    registry.addLink(filePath, {
+      hash: linked.hash,
+      app: appName,
+      source_id: sourceId,
+      dev: st.dev,
+      ino: st.ino,
+      unverified: false
+    })
+    return true
   }
   async _hashJob(job) {
     const primary = job.names[0]
@@ -262,10 +487,9 @@ class Sweeper {
     this.currentHash = { path: primary.filePath }
     try {
       const { hash, size } = await this.vault.hashFile(primary.filePath)
-      this.stats.hashes += 1
       this.state.hashed += 1
       this.state.hash_bytes += size || 0
-      const primaryStat = await fs.promises.stat(primary.filePath)
+      const primaryStat = await fs.promises.lstat(primary.filePath)
       // Never publish a digest if the path changed while its bytes were read.
       // The next manual scan can retry a continuously-mutating file safely.
       if (size !== primaryStat.size || !sameSnapshot(primary.expected, primaryStat)) {
@@ -277,19 +501,35 @@ class Sweeper {
         const name = job.names[index++]
         let st
         try {
-          st = name === primary ? primaryStat : await fs.promises.stat(name.filePath)
+          st = name === primary ? primaryStat : await fs.promises.lstat(name.filePath)
         } catch (e) {
-          continue
+          if (isMissingError(e) || this.recordInaccessible(e, name.filePath)) continue
+          throw e
         }
         // Same (device, inode) is physical identity, not a content guess.
         // Size and mtime also prevent applying a completed digest after an
         // in-place mutation. ctime is omitted here because adoption itself
         // adds a name and therefore legitimately changes inode metadata.
-        if (st.dev !== primaryStat.dev || st.ino !== primaryStat.ino ||
-            st.size !== primaryStat.size || st.mtimeMs !== primaryStat.mtimeMs) continue
+        if (!sameContentState(fileSnapshot(primaryStat), st)) continue
         await this.classify(name.filePath, st, hash, name.source_id)
       }
+      if (job.inodeKey) {
+        const current = await fs.promises.lstat(primary.filePath)
+        if (sameContentState(fileSnapshot(primaryStat), current)) {
+          this.completedHashesByIno.set(job.inodeKey, {
+            hash,
+            expected: fileSnapshot(current)
+          })
+        }
+      }
     } catch (e) {
+      if (!this.recordInaccessible(e, primary.filePath)) {
+        this.state.hash_failures += 1
+        const sourceId = primary.source_id || "pinokio"
+        this.state.source_hash_failures[sourceId] =
+          (this.state.source_hash_failures[sourceId] || 0) + 1
+        if (!this.fatalError) this.fatalError = e
+      }
     } finally {
       this.state.hash_duration_ms += Date.now() - started
       if (job.inodeKey && this.hashJobsByIno.get(job.inodeKey) === job) {
@@ -297,58 +537,113 @@ class Sweeper {
       }
       this.currentHash = null
       this.state.queued = Math.max(0, this.state.queued - 1)
+      if (this.state.queued < this.hashQueueLimit) {
+        for (const resolve of this.hashCapacityWaiters.splice(0)) resolve()
+      }
     }
   }
   // hash known → adopt if new content; otherwise a byte-identical copy:
   // ALWAYS pending, never converted by a scan.
   async classify(filePath, st, hash, preferredSourceId = null) {
     const registry = this.vault.registry
+    try {
+      const current = await fs.promises.lstat(filePath)
+      // A sibling name of the same inode may already have been adopted in
+      // this hash job, which changes ctime but not identity or content.
+      if (!sameContentState(fileSnapshot(st), current)) {
+        registry.untrack(filePath)
+        this.state.unstable_hashes += 1
+        return
+      }
+      st = current
+    } catch (error) {
+      if (!isMissingError(error)) throw error
+      registry.untrack(filePath)
+      return
+    }
     const source = this.vault.sourceForPath(filePath, preferredSourceId)
     const sourceId = source ? source.id : null
     const appName = source && source.kind === "app" ? source.app : null
     // Divergence: this inode was registered under a different hash — the
-    // shared content was written in place. Evict the stale blob.
+    // shared content was written in place. Retire the stale store name only
+    // when the scan commits so a later fatal error leaves the filesystem and
+    // registry at their pre-scan state.
     const priorHash = registry.byIno.get(registry.inoKey(st.dev, st.ino))
     if (priorHash && priorHash !== hash) {
       const stalePath = this.vault.storePathFor(priorHash)
+      let retiredStaleStore = false
       try {
-        const staleStat = await fs.promises.stat(stalePath)
-        if (staleStat.dev === st.dev && staleStat.ino === st.ino) {
-          await fs.promises.unlink(stalePath)
+        const staleStat = await this.vault.storeStatIfPresent(stalePath)
+        if (staleStat && staleStat.isFile() && staleStat.dev === st.dev && staleStat.ino === st.ino) {
+          const inodeKey = registry.inoKey(st.dev, st.ino)
+          this.vault.rememberRetiredScanStore(stalePath, staleStat, [
+            filePath,
+            ...(registry.pathsByIno.get(inodeKey) || [])
+          ])
+          retiredStaleStore = true
         }
-      } catch (e) {}
-      registry.removeBlob(priorHash)
-      registry.appendEvent({ kind: "diverged", hash: priorHash, path: filePath, app: appName, source_id: sourceId, size: st.size })
+      } catch (error) {
+        if (!isMissingError(error)) throw error
+      }
+      if (retiredStaleStore) this.invalidateLinkedInode(priorHash, st.dev, st.ino)
+      else registry.untrack(filePath)
+      this.vault.recordEvent({ kind: "diverged", hash: priorHash, path: filePath, app: appName, source_id: sourceId, size: st.size })
     }
     const storePath = this.vault.storePathFor(hash)
     let storeStat = null
     try {
-      storeStat = await fs.promises.stat(storePath)
-    } catch (e) {}
-    if (!storeStat && !registry.blobs.has(hash)) {
-      await this.vault.adopt(filePath, hash, { app: appName, source_id: sourceId })
-      this.updateScanIndex(filePath, st, hash, sourceId)
-      return
+      storeStat = await this.vault.storeStatIfPresent(storePath)
+    } catch (error) {
+      if (!isMissingError(error)) throw error
     }
-    if (storeStat && storeStat.dev === st.dev && storeStat.ino === st.ino) {
-      if (!registry.links.has(filePath)) {
-        registry.addLink(filePath, { hash, app: appName, source_id: sourceId, dev: st.dev, ino: st.ino, mode: "link" })
+    const registered = registry.links.get(filePath)
+    if (!storeStat && ((source && source.shareable) || !registry.blobs.has(hash) || (registered &&
+        registered.hash === hash && registered.dev === st.dev && registered.ino === st.ino))) {
+      const adopted = await this.vault.adopt(filePath, hash, {
+        app: appName,
+        source_id: sourceId,
+        expected: fileSnapshot(st)
+      })
+      if (adopted.status === "stale") {
+        registry.untrack(filePath)
+        this.state.unstable_hashes += 1
+        return
+      }
+      if (adopted.status === "adopted" || adopted.status === "already") {
+        return
+      }
+      try {
+        storeStat = await this.vault.storeStatIfPresent(storePath)
+      } catch (error) {
+        if (!isMissingError(error)) throw error
       }
+    }
+    if (storeStat && storeStat.dev === st.dev && storeStat.ino === st.ino) {
+      registry.addLink(filePath, {
+        hash, app: appName, source_id: sourceId,
+        dev: st.dev, ino: st.ino, unverified: false
+      })
       this.updateScanIndex(filePath, st, hash, sourceId)
       return
     }
-    if (!registry.duplicates.has(filePath)) {
-      registry.duplicates.set(filePath, { hash, size: st.size, app: appName, source_id: sourceId, discovered: Date.now() })
-      registry.schedulePersist()
-      registry.appendEvent({ kind: "found", hash, path: filePath, app: appName, source_id: sourceId, size: st.size })
+    const scanEntry = {
+      size: st.size, mtime: st.mtimeMs, ctime: st.ctimeMs,
+      dev: st.dev, ino: st.ino, hash: hash || null, source_id: sourceId
+    }
+    const previous = registry.setDuplicate(filePath, {
+      hash, size: st.size, app: appName, source_id: sourceId, discovered: Date.now(),
+      dev: st.dev, ino: st.ino, mtime: st.mtimeMs, ctime: st.ctimeMs,
+      unverified: false
+    }, scanEntry)
+    if (!previous || previous.hash !== hash) {
+      this.vault.recordEvent({ kind: "found", hash, path: filePath, app: appName, source_id: sourceId, size: st.size })
     }
-    this.updateScanIndex(filePath, st, hash, sourceId)
   }
   updateScanIndex(filePath, st, hash, sourceId = null) {
-    this.vault.registry.scanIndex.set(filePath, {
-      size: st.size, mtime: st.mtimeMs, dev: st.dev, ino: st.ino, hash: hash || null, source_id: sourceId
+    this.vault.registry.setScanEntry(filePath, {
+      size: st.size, mtime: st.mtimeMs, ctime: st.ctimeMs,
+      dev: st.dev, ino: st.ino, hash: hash || null, source_id: sourceId
     })
-    this.vault.registry.schedulePersist()
   }
   async ingestHfBlobs(dir, entries, preferredSourceId = null) {
     for (const entry of entries) {
@@ -357,28 +652,48 @@ class Sweeper {
       const full = path.resolve(dir, name)
       let st
       try {
-        st = await fs.promises.stat(full)
-      } catch (e) { continue }
-      this.state.files += 1
+        st = await fs.promises.lstat(full)
+      } catch (e) {
+        if (!isMissingError(e) && !this.recordInaccessible(e, full)) throw e
+        continue
+      }
+      if (!st.isFile()) continue
+      const source = this.vault.sourceForPath(full, preferredSourceId)
+      const sourceId = source ? source.id : null
+      const appName = source && source.kind === "app" ? source.app : null
+      const sourceKey = sourceId || preferredSourceId || "pinokio"
+      this.countFile(sourceKey)
       this.state.bytes_total += st.size
-      const sourceKey = preferredSourceId || "pinokio"
       this.state.source_bytes[sourceKey] = (this.state.source_bytes[sourceKey] || 0) + st.size
       if (!preferredSourceId) this.state.home_bytes_total += st.size
       if (!SHA256_RE.test(name)) continue
-      if (st.size < this.vault.sizeThreshold) continue
-      if (this.vault.registry.excluded.has(full)) continue
-      this.state.candidates += 1
-      this.stats.ingested += 1
+      if (this.vault.registry.excluded.has(full)) {
+        this.vault.registry.untrack(full)
+        continue
+      }
+      if (st.size < this.vault.sizeThreshold) {
+        if (this.preserveTrackedBelowThreshold(full, st, appName, sourceId)) continue
+        if (this.vault.registry.links.has(full)) {
+          await this.considerStat(full, st, preferredSourceId, { count: false })
+        } else {
+          this.vault.registry.untrack(full)
+        }
+        continue
+      } else {
+        this.state.candidates += 1
+      }
       await this.classify(full, st, name, preferredSourceId)
     }
   }
   // hf CLI --local-dir metadata: etag is the sha256 for LFS files; freshness
   // via the recorded timestamp against mtime (huggingface_hub's own rule).
-  async harvestLocalDirEtag(filePath, st) {
+  async harvestLocalDirEtag(filePath, st, sourceRoot) {
     try {
       const start = path.dirname(filePath)
       let dir = start
-      const stop = path.resolve(this.kernel.homedir)
+      // Metadata outside the configured source is not authoritative for a
+      // scanned file, even when the source happens to be nested below it.
+      const stop = path.resolve(sourceRoot || this.kernel.homedir)
       const visited = []
       let base = null
       for (let depth = 0; depth < 12; depth++) {
@@ -390,7 +705,7 @@ class Sweeper {
         const metaRoot = path.resolve(dir, ".cache", "huggingface")
         let exists = false
         try {
-          exists = (await fs.promises.stat(metaRoot)).isDirectory()
+          exists = (await fs.promises.lstat(metaRoot)).isDirectory()
         } catch (e) {}
         if (exists) {
           base = dir
@@ -402,69 +717,24 @@ class Sweeper {
       for (const visitedDir of visited) this.metadataBaseCache.set(visitedDir, base)
       if (!base) return null
       const relative = path.relative(base, filePath)
-      const metaPath = path.resolve(base, ".cache", "huggingface", relative + ".metadata")
+      const metaRoot = path.resolve(base, ".cache", "huggingface")
+      const metaRootStat = await fs.promises.lstat(metaRoot)
+      if (!metaRootStat.isDirectory()) return null
+      const metaPath = path.resolve(metaRoot, relative + ".metadata")
       let raw
       try {
-        raw = await fs.promises.readFile(metaPath, "utf8")
+        raw = await readFileNoFollow(metaPath)
       } catch (e) {
         return null
       }
+      if (raw === null) return null
       const lines = raw.split("\n")
       const etag = String(lines[1] || "").replace(/"/g, "").trim().toLowerCase()
       const ts = parseFloat(lines[2] || "0") * 1000
-      if (SHA256_RE.test(etag) && st.mtimeMs <= ts + 1000) return etag
+      if (SHA256_RE.test(etag) && ts >= st.mtimeMs) return etag
     } catch (e) {}
     return null
   }
-  appNameFor(filePath) {
-    return this.vault.appNameFor(filePath)
-  }
-  // User-triggered batch conversion (the Deduplicate buttons). Locked files
-  // stay pending; the user can retry after stopping the app.
-  async convertPending(appName, opts = {}) {
-    const registry = this.vault.registry
-    const batch = opts.batch_id || `batch-${Date.now()}`
-    const scopeId = opts.scope_id || null
-    const summary = { converted: 0, bytes_saved: 0, locked: 0, unavailable: 0, failed: 0 }
-    for (const [filePath, entry] of [...registry.duplicates]) {
-      const location = this.vault.locationForPath(filePath, entry.source_id)
-      if (scopeId && location.source_id !== scopeId) continue
-      if (!scopeId && appName !== undefined && (entry.app || null) !== appName) continue
-      try {
-        if (scopeId) {
-          const source = this.vault.sourceForPath(filePath, location.source_id)
-          let targetStat = null
-          let storeStat = null
-          try {
-            targetStat = await fs.promises.stat(filePath)
-            storeStat = await fs.promises.stat(this.vault.storePathFor(entry.hash))
-          } catch (e) {}
-          if (!source || !source.shareable || !targetStat || !storeStat || targetStat.dev !== storeStat.dev) {
-            summary.unavailable += 1
-            continue
-          }
-        }
-        const result = await this.vault.convert(filePath, entry.hash, {
-          app: entry.app, source_id: location.source_id, batch_id: batch
-        })
-        if (result.status === "converted" || result.status === "already") {
-          registry.duplicates.delete(filePath)
-          summary.converted += 1
-          summary.bytes_saved += result.bytes_saved || 0
-        } else if (result.status === "locked") {
-          summary.locked += 1
-        } else if (result.status === "unavailable" || result.status === "copy-mode") {
-          summary.unavailable += 1
-        } else {
-          summary.failed += 1
-        }
-      } catch (e) {
-        summary.failed += 1
-      }
-    }
-    registry.schedulePersist()
-    return summary
-  }
 }
 
 module.exports = Sweeper