pinokiod 8.0.39 → 8.0.40

package/kernel/vault/registry.js

@@ -1,169 +0,0 @@
-const fs = require('fs')
-const path = require('path')
-
-// Persisted record of blobs and their known names.
-// The registry is a cache of the filesystem, never the source of truth
-// (spec/requirements/shared-model-store.md "Registry").
-class Registry {
-  constructor(root) {
-    this.root = root
-    this.snapshotPath = path.resolve(root, "registry.json")
-    this.eventsPath = path.resolve(root, "events.ndjson")
-    this.reset()
-    this.persistTimer = null
-    this.persistDelay = 500
-    this.flushPromise = Promise.resolve()
-  }
-  reset() {
-    this.blobs = new Map()      // hash -> { size, first_seen, source_urls, verified_at, orphan }
-    this.links = new Map()      // path -> { hash, app, source_id, dev, ino, created, mode }
-    this.scanIndex = new Map()  // path -> { size, mtime, dev, ino, hash, source_id }
-    this.duplicates = new Map() // path -> { hash, size, app, source_id, discovered } pending user conversion
-    this.excluded = new Map()   // path -> { ts, source_id, size } user chose "keep independent"
-    this.lastScan = null        // scan totals, duration, and hash work
-    this.totals = { lifetime_bytes_saved: 0 }
-    this.byIno = new Map()      // "dev:ino" -> hash
-  }
-  setLastScan(meta) {
-    this.lastScan = meta
-    this.schedulePersist()
-  }
-  inoKey(dev, ino) {
-    return `${dev}:${ino}`
-  }
-  addBlob(hash, meta) {
-    if (!this.blobs.has(hash)) {
-      this.blobs.set(hash, Object.assign({
-        size: 0,
-        first_seen: Date.now(),
-        source_urls: [],
-        verified_at: null
-      }, meta))
-    } else if (meta && meta.source_urls && meta.source_urls.length) {
-      const blob = this.blobs.get(hash)
-      for (const url of meta.source_urls) {
-        if (!blob.source_urls.includes(url)) blob.source_urls.push(url)
-      }
-    }
-    this.schedulePersist()
-  }
-  addLink(filePath, entry) {
-    this.links.set(filePath, Object.assign({ created: Date.now(), mode: "link" }, entry))
-    if (entry.dev !== undefined && entry.ino !== undefined) {
-      this.byIno.set(this.inoKey(entry.dev, entry.ino), entry.hash)
-    }
-    this.schedulePersist()
-  }
-  removeLink(filePath) {
-    const entry = this.links.get(filePath)
-    if (entry) {
-      this.links.delete(filePath)
-      const key = this.inoKey(entry.dev, entry.ino)
-      let stillUsed = false
-      for (const other of this.links.values()) {
-        if (this.inoKey(other.dev, other.ino) === key) { stillUsed = true; break }
-      }
-      if (!stillUsed) this.byIno.delete(key)
-      this.schedulePersist()
-    }
-  }
-  removeBlob(hash) {
-    this.blobs.delete(hash)
-    for (const [p, entry] of [...this.links]) {
-      if (entry.hash === hash) this.removeLink(p)
-    }
-    this.schedulePersist()
-  }
-  addSaved(bytes) {
-    this.totals.lifetime_bytes_saved += bytes
-    this.schedulePersist()
-  }
-  // Returns { corrupt: true } when the snapshot exists but cannot be parsed,
-  // so the caller can rebuild from disk instead of blocking startup.
-  async load() {
-    this.reset()
-    let raw
-    try {
-      raw = await fs.promises.readFile(this.snapshotPath, "utf8")
-    } catch (e) {
-      return { corrupt: false, existed: false }
-    }
-    let json
-    try {
-      json = JSON.parse(raw)
-    } catch (e) {
-      return { corrupt: true, existed: true }
-    }
-    for (const [k, v] of Object.entries(json.blobs || {})) this.blobs.set(k, v)
-    for (const [k, v] of Object.entries(json.links || {})) {
-      this.links.set(k, v)
-      if (v.dev !== undefined && v.ino !== undefined) {
-        this.byIno.set(this.inoKey(v.dev, v.ino), v.hash)
-      }
-    }
-    for (const [k, v] of Object.entries(json.scan_index || {})) this.scanIndex.set(k, v)
-    for (const [k, v] of Object.entries(json.duplicates || {})) this.duplicates.set(k, v)
-    for (const [k, v] of Object.entries(json.excluded || {})) this.excluded.set(k, v)
-    if (json.last_scan) this.lastScan = json.last_scan
-    if (json.totals && typeof json.totals.lifetime_bytes_saved === "number") {
-      this.totals = json.totals
-    }
-    return { corrupt: false, existed: true }
-  }
-  schedulePersist() {
-    if (this.persistTimer) return
-    this.persistTimer = setTimeout(() => {
-      this.persistTimer = null
-      this.flush().catch(() => {})
-    }, this.persistDelay)
-    if (this.persistTimer.unref) this.persistTimer.unref()
-  }
-  async flush() {
-    if (this.persistTimer) {
-      clearTimeout(this.persistTimer)
-      this.persistTimer = null
-    }
-    const json = {
-      version: 1,
-      blobs: Object.fromEntries(this.blobs),
-      links: Object.fromEntries(this.links),
-      scan_index: Object.fromEntries(this.scanIndex),
-      duplicates: Object.fromEntries(this.duplicates),
-      excluded: Object.fromEntries(this.excluded),
-      last_scan: this.lastScan,
-      totals: this.totals
-    }
-    const write = async () => {
-      const tmp = this.snapshotPath + ".tmp"
-      await fs.promises.mkdir(this.root, { recursive: true })
-      await fs.promises.writeFile(tmp, JSON.stringify(json))
-      await fs.promises.rename(tmp, this.snapshotPath)
-    }
-    const pending = this.flushPromise.then(write, write)
-    this.flushPromise = pending.catch(() => {})
-    return pending
-  }
-  async appendEvent(event) {
-    const line = JSON.stringify(Object.assign({ ts: Date.now() }, event)) + "\n"
-    await fs.promises.appendFile(this.eventsPath, line).catch(() => {})
-  }
-  // Torn final line (crash mid-append) is discarded, per persistence rules.
-  async readEvents() {
-    let raw
-    try {
-      raw = await fs.promises.readFile(this.eventsPath, "utf8")
-    } catch (e) {
-      return []
-    }
-    const events = []
-    for (const line of raw.split("\n")) {
-      if (!line.trim()) continue
-      try {
-        events.push(JSON.parse(line))
-      } catch (e) {}
-    }
-    return events
-  }
-}
-
-module.exports = Registry

package/kernel/vault/sweeper.js

@@ -1,470 +0,0 @@
-const fs = require('fs')
-const path = require('path')
-const fastq = require('fastq')
-
-// Manual scan engine (spec/requirements/shared-model-store.md).
-// Scans run ONLY when the user asks — there are no automatic triggers.
-// The walk is generic: every regular file counts toward folder totals, and
-// every file >= the size threshold is a candidate. No name heuristics.
-// A scan never converts anything: it adopts new content (adds a vault name,
-// which mutates nothing) and lists byte-identical copies as pending, for the
-// user to deduplicate explicitly.
-
-const SHA256_RE = /^[0-9a-f]{64}$/
-const TMP_SUFFIX = ".pinokio-dedup-tmp"
-const DIR_CONCURRENCY = 8
-const STAT_CONCURRENCY = 32
-
-const fileSnapshot = (st) => ({
-  dev: st.dev,
-  ino: st.ino,
-  size: st.size,
-  mtime: st.mtimeMs,
-  ctime: st.ctimeMs
-})
-
-const sameSnapshot = (snapshot, st) => !!(
-  snapshot && st &&
-  snapshot.dev === st.dev &&
-  snapshot.ino === st.ino &&
-  snapshot.size === st.size &&
-  snapshot.mtime === st.mtimeMs &&
-  snapshot.ctime === st.ctimeMs
-)
-
-class Sweeper {
-  constructor(vault) {
-    this.vault = vault
-    this.kernel = vault.kernel
-    this.state = this.idleState()
-    this.currentHash = null
-    this.hashQueue = fastq.promise(this, this._hashJob, 1)
-    this.stats = { hashes: 0, ingested: 0 }
-    this.statConcurrency = vault.statConcurrency || STAT_CONCURRENCY
-    this.dirConcurrency = vault.dirConcurrency || DIR_CONCURRENCY
-    this.metadataBaseCache = new Map()
-    this.hashJobsByIno = new Map()
-  }
-  idleState() {
-    return {
-      active: false, dirs: 0, files: 0, bytes_total: 0,
-      home_bytes_total: 0, source_bytes: {},
-      candidates: 0, hashed: 0, hash_total: 0, hash_bytes: 0, queued: 0,
-      inode_reuses: 0, unstable_hashes: 0,
-      source_ids: [], estimated_files: null, estimated_walk_weight: null,
-      started: null, duration_ms: null,
-      walk_duration_ms: null, hash_wait_duration_ms: null,
-      hash_duration_ms: 0
-    }
-  }
-  // The one entry point: the Pinokio home plus explicit linked imports.
-  async scan() {
-    if (this.state.active) return { already_running: true }
-    this.state = Object.assign(this.idleState(), { active: true, started: Date.now() })
-    this.metadataBaseCache.clear()
-    this.hashJobsByIno.clear()
-    try {
-      await this.vault.refreshSources()
-      const scanRoots = this.vault.scanRoots()
-      this.state.source_ids = scanRoots.map((source) => source.source_id).sort()
-      const previous = this.vault.registry.lastScan
-      const previousSourceIds = previous
-        ? (Array.isArray(previous.source_ids) ? previous.source_ids : Object.keys(previous.source_bytes || {})).slice().sort()
-        : []
-      const sameSources = previousSourceIds.length === this.state.source_ids.length &&
-        previousSourceIds.every((sourceId, index) => sourceId === this.state.source_ids[index])
-      if (previous && previous.files > 0 && sameSources) {
-        this.state.estimated_files = previous.files
-        const measuredWeight = previous.duration_ms > 0 && previous.walk_duration_ms >= 0
-          ? previous.walk_duration_ms / previous.duration_ms
-          : 0.8
-        this.state.estimated_walk_weight = Math.max(0.15, Math.min(0.98, measuredWeight))
-      }
-      const walkStarted = Date.now()
-      for (const source of scanRoots) {
-        if (!Object.prototype.hasOwnProperty.call(this.state.source_bytes, source.source_id)) {
-          this.state.source_bytes[source.source_id] = 0
-        }
-        await this.walk(source.root, source.source_id === "pinokio" ? null : source.source_id)
-      }
-      this.state.walk_duration_ms = Date.now() - walkStarted
-      const hashWaitStarted = Date.now()
-      await this.settle()
-      this.state.hash_wait_duration_ms = Date.now() - hashWaitStarted
-      this.state.duration_ms = Date.now() - this.state.started
-      this.vault.registry.setLastScan({
-        ts: Date.now(),
-        dirs: this.state.dirs,
-        files: this.state.files,
-        bytes_total: this.state.bytes_total,
-        home_bytes_total: this.state.home_bytes_total,
-        source_bytes: Object.assign({}, this.state.source_bytes),
-        source_ids: this.state.source_ids.slice(),
-        candidates: this.state.candidates,
-        hashed: this.state.hashed,
-        hash_total: this.state.hash_total,
-        hash_bytes: this.state.hash_bytes,
-        inode_reuses: this.state.inode_reuses,
-        unstable_hashes: this.state.unstable_hashes,
-        walk_duration_ms: this.state.walk_duration_ms,
-        hash_wait_duration_ms: this.state.hash_wait_duration_ms,
-        hash_duration_ms: this.state.hash_duration_ms,
-        duration_ms: this.state.duration_ms
-      })
-    } finally {
-      // A failed walk must not leave background hash work or per-scan inode
-      // state alive when the next manual scan starts.
-      await this.settle().catch(() => {})
-      this.hashJobsByIno.clear()
-      this.state.active = false
-    }
-    return {
-      dirs: this.state.dirs, files: this.state.files,
-      bytes_total: this.state.bytes_total, candidates: this.state.candidates
-    }
-  }
-  async settle() {
-    await this.hashQueue.drained()
-  }
-  async walk(root, preferredSourceId = null) {
-    const vaultRoot = this.vault.root
-    const stack = [root]
-    while (stack.length) {
-      const dirs = []
-      while (dirs.length < this.dirConcurrency && stack.length) {
-        dirs.push(stack.pop())
-      }
-      const directoryResults = await Promise.all(dirs.map(async (dir) => {
-        this.state.dirs += 1
-        try {
-          const entries = await fs.promises.readdir(dir, { withFileTypes: true })
-          return { dir, entries }
-        } catch (e) {
-          return { dir, entries: null }
-        }
-      }))
-      const files = []
-      const hfDirectories = []
-      for (const { dir, entries } of directoryResults) {
-        if (!entries) continue
-        // HF hub blobs dir: filenames ARE sha256 for LFS files — no hashing.
-        if (path.basename(dir) === "blobs" && /^(models|datasets|spaces)--/.test(path.basename(path.dirname(dir)))) {
-          hfDirectories.push({ dir, entries })
-          continue
-        }
-        for (const entry of entries) {
-          if (entry.isSymbolicLink()) continue
-          const full = path.resolve(dir, entry.name)
-          if (entry.isDirectory()) {
-            if (full === vaultRoot) continue
-            stack.push(full)
-          } else if (entry.isFile()) {
-            files.push(full)
-          }
-        }
-      }
-      // One metadata pool is shared by all files in this directory batch, so
-      // trees made of many small directories still use the bounded workers.
-      await this.considerFiles(files, preferredSourceId)
-      for (const { dir, entries } of hfDirectories) {
-        await this.ingestHfBlobs(dir, entries, preferredSourceId)
-      }
-    }
-  }
-  async considerFiles(filePaths, preferredSourceId = null) {
-    const stats = new Array(filePaths.length)
-    let cursor = 0
-    const workers = Array.from({ length: Math.min(this.statConcurrency, filePaths.length) }, async () => {
-      while (cursor < filePaths.length) {
-        const index = cursor++
-        try {
-          stats[index] = await fs.promises.stat(filePaths[index])
-        } catch (error) {
-          stats[index] = null
-        }
-      }
-    })
-    await Promise.all(workers)
-    // Preserve deterministic classification order while parallelizing only
-    // the expensive filesystem metadata reads.
-    for (let index = 0; index < filePaths.length; index++) {
-      if (stats[index]) await this.considerStat(filePaths[index], stats[index], preferredSourceId)
-    }
-  }
-  async considerFile(filePath, preferredSourceId = null) {
-    if (filePath.endsWith(TMP_SUFFIX)) return
-    let st
-    try {
-      st = await fs.promises.stat(filePath)
-    } catch (e) {
-      return
-    }
-    await this.considerStat(filePath, st, preferredSourceId)
-  }
-  async considerStat(filePath, st, preferredSourceId = null) {
-    if (filePath.endsWith(TMP_SUFFIX)) return
-    const registry = this.vault.registry
-    this.state.files += 1
-    this.state.bytes_total += st.size
-    const sourceKey = preferredSourceId || "pinokio"
-    this.state.source_bytes[sourceKey] = (this.state.source_bytes[sourceKey] || 0) + st.size
-    if (!preferredSourceId) this.state.home_bytes_total += st.size
-    if (st.size < this.vault.sizeThreshold) return
-    if (registry.excluded.has(filePath)) return
-    this.state.candidates += 1
-    const source = this.vault.sourceForPath(filePath, preferredSourceId)
-    const sourceId = source ? source.id : null
-    const appName = source && source.kind === "app" ? source.app : null
-    // Known and unchanged: nothing to do.
-    const inoHash = registry.byIno.get(registry.inoKey(st.dev, st.ino))
-    if (inoHash && st.nlink > 1) {
-      const cached = registry.scanIndex.get(filePath)
-      const unchanged = cached && cached.size === st.size && cached.mtime === st.mtimeMs && cached.ino === st.ino
-      if (unchanged || !cached) {
-        if (!registry.links.has(filePath)) {
-          registry.addLink(filePath, { hash: inoHash, app: appName, source_id: sourceId, dev: st.dev, ino: st.ino, mode: "link" })
-        }
-        if (!cached) this.updateScanIndex(filePath, st, inoHash, sourceId)
-        return
-      }
-    }
-    const cached = registry.scanIndex.get(filePath)
-    if (cached && cached.size === st.size && cached.mtime === st.mtimeMs && cached.ino === st.ino && cached.hash) {
-      await this.classify(filePath, st, cached.hash, sourceId)
-      return
-    }
-    const harvested = await this.harvestLocalDirEtag(filePath, st)
-    if (harvested) {
-      await this.classify(filePath, st, harvested, sourceId)
-      return
-    }
-    // Only coalesce an inode which the filesystem itself reports as having
-    // multiple names. This avoids trusting placeholder inode values on
-    // filesystems which do not expose usable identity metadata.
-    const reusableInode = st.nlink > 1 && st.ino !== undefined && st.ino !== 0
-    const inodeKey = reusableInode ? registry.inoKey(st.dev, st.ino) : null
-    const existingJob = inodeKey ? this.hashJobsByIno.get(inodeKey) : null
-    const name = { filePath, source_id: sourceId, expected: fileSnapshot(st) }
-    if (existingJob) {
-      existingJob.names.push(name)
-      this.state.inode_reuses += 1
-      return
-    }
-    const job = { inodeKey, names: [name] }
-    if (inodeKey) this.hashJobsByIno.set(inodeKey, job)
-    this.state.hash_total += 1
-    this.state.queued += 1
-    this.hashQueue.push(job).catch(() => {})
-  }
-  async _hashJob(job) {
-    const primary = job.names[0]
-    const started = Date.now()
-    this.currentHash = { path: primary.filePath }
-    try {
-      const { hash, size } = await this.vault.hashFile(primary.filePath)
-      this.stats.hashes += 1
-      this.state.hashed += 1
-      this.state.hash_bytes += size || 0
-      const primaryStat = await fs.promises.stat(primary.filePath)
-      // Never publish a digest if the path changed while its bytes were read.
-      // The next manual scan can retry a continuously-mutating file safely.
-      if (size !== primaryStat.size || !sameSnapshot(primary.expected, primaryStat)) {
-        this.state.unstable_hashes += 1
-        return
-      }
-      let index = 0
-      while (index < job.names.length) {
-        const name = job.names[index++]
-        let st
-        try {
-          st = name === primary ? primaryStat : await fs.promises.stat(name.filePath)
-        } catch (e) {
-          continue
-        }
-        // Same (device, inode) is physical identity, not a content guess.
-        // Size and mtime also prevent applying a completed digest after an
-        // in-place mutation. ctime is omitted here because adoption itself
-        // adds a name and therefore legitimately changes inode metadata.
-        if (st.dev !== primaryStat.dev || st.ino !== primaryStat.ino ||
-            st.size !== primaryStat.size || st.mtimeMs !== primaryStat.mtimeMs) continue
-        await this.classify(name.filePath, st, hash, name.source_id)
-      }
-    } catch (e) {
-    } finally {
-      this.state.hash_duration_ms += Date.now() - started
-      if (job.inodeKey && this.hashJobsByIno.get(job.inodeKey) === job) {
-        this.hashJobsByIno.delete(job.inodeKey)
-      }
-      this.currentHash = null
-      this.state.queued = Math.max(0, this.state.queued - 1)
-    }
-  }
-  // hash known → adopt if new content; otherwise a byte-identical copy:
-  // ALWAYS pending, never converted by a scan.
-  async classify(filePath, st, hash, preferredSourceId = null) {
-    const registry = this.vault.registry
-    const source = this.vault.sourceForPath(filePath, preferredSourceId)
-    const sourceId = source ? source.id : null
-    const appName = source && source.kind === "app" ? source.app : null
-    // Divergence: this inode was registered under a different hash — the
-    // shared content was written in place. Evict the stale blob.
-    const priorHash = registry.byIno.get(registry.inoKey(st.dev, st.ino))
-    if (priorHash && priorHash !== hash) {
-      const stalePath = this.vault.storePathFor(priorHash)
-      try {
-        const staleStat = await fs.promises.stat(stalePath)
-        if (staleStat.dev === st.dev && staleStat.ino === st.ino) {
-          await fs.promises.unlink(stalePath)
-        }
-      } catch (e) {}
-      registry.removeBlob(priorHash)
-      registry.appendEvent({ kind: "diverged", hash: priorHash, path: filePath, app: appName, source_id: sourceId, size: st.size })
-    }
-    const storePath = this.vault.storePathFor(hash)
-    let storeStat = null
-    try {
-      storeStat = await fs.promises.stat(storePath)
-    } catch (e) {}
-    if (!storeStat && !registry.blobs.has(hash)) {
-      await this.vault.adopt(filePath, hash, { app: appName, source_id: sourceId })
-      this.updateScanIndex(filePath, st, hash, sourceId)
-      return
-    }
-    if (storeStat && storeStat.dev === st.dev && storeStat.ino === st.ino) {
-      if (!registry.links.has(filePath)) {
-        registry.addLink(filePath, { hash, app: appName, source_id: sourceId, dev: st.dev, ino: st.ino, mode: "link" })
-      }
-      this.updateScanIndex(filePath, st, hash, sourceId)
-      return
-    }
-    if (!registry.duplicates.has(filePath)) {
-      registry.duplicates.set(filePath, { hash, size: st.size, app: appName, source_id: sourceId, discovered: Date.now() })
-      registry.schedulePersist()
-      registry.appendEvent({ kind: "found", hash, path: filePath, app: appName, source_id: sourceId, size: st.size })
-    }
-    this.updateScanIndex(filePath, st, hash, sourceId)
-  }
-  updateScanIndex(filePath, st, hash, sourceId = null) {
-    this.vault.registry.scanIndex.set(filePath, {
-      size: st.size, mtime: st.mtimeMs, dev: st.dev, ino: st.ino, hash: hash || null, source_id: sourceId
-    })
-    this.vault.registry.schedulePersist()
-  }
-  async ingestHfBlobs(dir, entries, preferredSourceId = null) {
-    for (const entry of entries) {
-      if (!entry.isFile() || entry.isSymbolicLink()) continue
-      const name = entry.name
-      const full = path.resolve(dir, name)
-      let st
-      try {
-        st = await fs.promises.stat(full)
-      } catch (e) { continue }
-      this.state.files += 1
-      this.state.bytes_total += st.size
-      const sourceKey = preferredSourceId || "pinokio"
-      this.state.source_bytes[sourceKey] = (this.state.source_bytes[sourceKey] || 0) + st.size
-      if (!preferredSourceId) this.state.home_bytes_total += st.size
-      if (!SHA256_RE.test(name)) continue
-      if (st.size < this.vault.sizeThreshold) continue
-      if (this.vault.registry.excluded.has(full)) continue
-      this.state.candidates += 1
-      this.stats.ingested += 1
-      await this.classify(full, st, name, preferredSourceId)
-    }
-  }
-  // hf CLI --local-dir metadata: etag is the sha256 for LFS files; freshness
-  // via the recorded timestamp against mtime (huggingface_hub's own rule).
-  async harvestLocalDirEtag(filePath, st) {
-    try {
-      const start = path.dirname(filePath)
-      let dir = start
-      const stop = path.resolve(this.kernel.homedir)
-      const visited = []
-      let base = null
-      for (let depth = 0; depth < 12; depth++) {
-        if (this.metadataBaseCache.has(dir)) {
-          base = this.metadataBaseCache.get(dir)
-          break
-        }
-        visited.push(dir)
-        const metaRoot = path.resolve(dir, ".cache", "huggingface")
-        let exists = false
-        try {
-          exists = (await fs.promises.stat(metaRoot)).isDirectory()
-        } catch (e) {}
-        if (exists) {
-          base = dir
-          break
-        }
-        if (dir === stop || path.dirname(dir) === dir) break
-        dir = path.dirname(dir)
-      }
-      for (const visitedDir of visited) this.metadataBaseCache.set(visitedDir, base)
-      if (!base) return null
-      const relative = path.relative(base, filePath)
-      const metaPath = path.resolve(base, ".cache", "huggingface", relative + ".metadata")
-      let raw
-      try {
-        raw = await fs.promises.readFile(metaPath, "utf8")
-      } catch (e) {
-        return null
-      }
-      const lines = raw.split("\n")
-      const etag = String(lines[1] || "").replace(/"/g, "").trim().toLowerCase()
-      const ts = parseFloat(lines[2] || "0") * 1000
-      if (SHA256_RE.test(etag) && st.mtimeMs <= ts + 1000) return etag
-    } catch (e) {}
-    return null
-  }
-  appNameFor(filePath) {
-    return this.vault.appNameFor(filePath)
-  }
-  // User-triggered batch conversion (the Deduplicate buttons). Locked files
-  // stay pending; the user can retry after stopping the app.
-  async convertPending(appName, opts = {}) {
-    const registry = this.vault.registry
-    const batch = opts.batch_id || `batch-${Date.now()}`
-    const scopeId = opts.scope_id || null
-    const summary = { converted: 0, bytes_saved: 0, locked: 0, unavailable: 0, failed: 0 }
-    for (const [filePath, entry] of [...registry.duplicates]) {
-      const location = this.vault.locationForPath(filePath, entry.source_id)
-      if (scopeId && location.source_id !== scopeId) continue
-      if (!scopeId && appName !== undefined && (entry.app || null) !== appName) continue
-      try {
-        if (scopeId) {
-          const source = this.vault.sourceForPath(filePath, location.source_id)
-          let targetStat = null
-          let storeStat = null
-          try {
-            targetStat = await fs.promises.stat(filePath)
-            storeStat = await fs.promises.stat(this.vault.storePathFor(entry.hash))
-          } catch (e) {}
-          if (!source || !source.shareable || !targetStat || !storeStat || targetStat.dev !== storeStat.dev) {
-            summary.unavailable += 1
-            continue
-          }
-        }
-        const result = await this.vault.convert(filePath, entry.hash, {
-          app: entry.app, source_id: location.source_id, batch_id: batch
-        })
-        if (result.status === "converted" || result.status === "already") {
-          registry.duplicates.delete(filePath)
-          summary.converted += 1
-          summary.bytes_saved += result.bytes_saved || 0
-        } else if (result.status === "locked") {
-          summary.locked += 1
-        } else if (result.status === "unavailable" || result.status === "copy-mode") {
-          summary.unavailable += 1
-        } else {
-          summary.failed += 1
-        }
-      } catch (e) {
-        summary.failed += 1
-      }
-    }
-    registry.schedulePersist()
-    return summary
-  }
-}
-
-module.exports = Sweeper