pinokiod 8.0.37 → 8.0.39

package/kernel/vault/sweeper.js

@@ -12,6 +12,25 @@ const fastq = require('fastq')
 
 const SHA256_RE = /^[0-9a-f]{64}$/
 const TMP_SUFFIX = ".pinokio-dedup-tmp"
+const DIR_CONCURRENCY = 8
+const STAT_CONCURRENCY = 32
+
+const fileSnapshot = (st) => ({
+  dev: st.dev,
+  ino: st.ino,
+  size: st.size,
+  mtime: st.mtimeMs,
+  ctime: st.ctimeMs
+})
+
+const sameSnapshot = (snapshot, st) => !!(
+  snapshot && st &&
+  snapshot.dev === st.dev &&
+  snapshot.ino === st.ino &&
+  snapshot.size === st.size &&
+  snapshot.mtime === st.mtimeMs &&
+  snapshot.ctime === st.ctimeMs
+)
 
 class Sweeper {
   constructor(vault) {
@@ -21,33 +40,82 @@ class Sweeper {
     this.currentHash = null
     this.hashQueue = fastq.promise(this, this._hashJob, 1)
     this.stats = { hashes: 0, ingested: 0 }
+    this.statConcurrency = vault.statConcurrency || STAT_CONCURRENCY
+    this.dirConcurrency = vault.dirConcurrency || DIR_CONCURRENCY
+    this.metadataBaseCache = new Map()
+    this.hashJobsByIno = new Map()
   }
   idleState() {
     return {
       active: false, dirs: 0, files: 0, bytes_total: 0,
       home_bytes_total: 0, source_bytes: {},
-      candidates: 0, hashed: 0, queued: 0, started: null
+      candidates: 0, hashed: 0, hash_total: 0, hash_bytes: 0, queued: 0,
+      inode_reuses: 0, unstable_hashes: 0,
+      source_ids: [], estimated_files: null, estimated_walk_weight: null,
+      started: null, duration_ms: null,
+      walk_duration_ms: null, hash_wait_duration_ms: null,
+      hash_duration_ms: 0
     }
   }
   // The one entry point: the Pinokio home plus explicit linked imports.
   async scan() {
     if (this.state.active) return { already_running: true }
     this.state = Object.assign(this.idleState(), { active: true, started: Date.now() })
+    this.metadataBaseCache.clear()
+    this.hashJobsByIno.clear()
     try {
       await this.vault.refreshSources()
-      for (const source of this.vault.scanRoots()) {
+      const scanRoots = this.vault.scanRoots()
+      this.state.source_ids = scanRoots.map((source) => source.source_id).sort()
+      const previous = this.vault.registry.lastScan
+      const previousSourceIds = previous
+        ? (Array.isArray(previous.source_ids) ? previous.source_ids : Object.keys(previous.source_bytes || {})).slice().sort()
+        : []
+      const sameSources = previousSourceIds.length === this.state.source_ids.length &&
+        previousSourceIds.every((sourceId, index) => sourceId === this.state.source_ids[index])
+      if (previous && previous.files > 0 && sameSources) {
+        this.state.estimated_files = previous.files
+        const measuredWeight = previous.duration_ms > 0 && previous.walk_duration_ms >= 0
+          ? previous.walk_duration_ms / previous.duration_ms
+          : 0.8
+        this.state.estimated_walk_weight = Math.max(0.15, Math.min(0.98, measuredWeight))
+      }
+      const walkStarted = Date.now()
+      for (const source of scanRoots) {
+        if (!Object.prototype.hasOwnProperty.call(this.state.source_bytes, source.source_id)) {
+          this.state.source_bytes[source.source_id] = 0
+        }
         await this.walk(source.root, source.source_id === "pinokio" ? null : source.source_id)
       }
+      this.state.walk_duration_ms = Date.now() - walkStarted
+      const hashWaitStarted = Date.now()
       await this.settle()
+      this.state.hash_wait_duration_ms = Date.now() - hashWaitStarted
+      this.state.duration_ms = Date.now() - this.state.started
       this.vault.registry.setLastScan({
         ts: Date.now(),
         dirs: this.state.dirs,
         files: this.state.files,
         bytes_total: this.state.bytes_total,
         home_bytes_total: this.state.home_bytes_total,
-        source_bytes: Object.assign({}, this.state.source_bytes)
+        source_bytes: Object.assign({}, this.state.source_bytes),
+        source_ids: this.state.source_ids.slice(),
+        candidates: this.state.candidates,
+        hashed: this.state.hashed,
+        hash_total: this.state.hash_total,
+        hash_bytes: this.state.hash_bytes,
+        inode_reuses: this.state.inode_reuses,
+        unstable_hashes: this.state.unstable_hashes,
+        walk_duration_ms: this.state.walk_duration_ms,
+        hash_wait_duration_ms: this.state.hash_wait_duration_ms,
+        hash_duration_ms: this.state.hash_duration_ms,
+        duration_ms: this.state.duration_ms
       })
     } finally {
+      // A failed walk must not leave background hash work or per-scan inode
+      // state alive when the next manual scan starts.
+      await this.settle().catch(() => {})
+      this.hashJobsByIno.clear()
       this.state.active = false
     }
     return {
@@ -62,40 +130,80 @@ class Sweeper {
     const vaultRoot = this.vault.root
     const stack = [root]
     while (stack.length) {
-      const dir = stack.pop()
-      this.state.dirs += 1
-      let entries
-      try {
-        entries = await fs.promises.readdir(dir, { withFileTypes: true })
-      } catch (e) {
-        continue
+      const dirs = []
+      while (dirs.length < this.dirConcurrency && stack.length) {
+        dirs.push(stack.pop())
+      }
+      const directoryResults = await Promise.all(dirs.map(async (dir) => {
+        this.state.dirs += 1
+        try {
+          const entries = await fs.promises.readdir(dir, { withFileTypes: true })
+          return { dir, entries }
+        } catch (e) {
+          return { dir, entries: null }
+        }
+      }))
+      const files = []
+      const hfDirectories = []
+      for (const { dir, entries } of directoryResults) {
+        if (!entries) continue
+        // HF hub blobs dir: filenames ARE sha256 for LFS files — no hashing.
+        if (path.basename(dir) === "blobs" && /^(models|datasets|spaces)--/.test(path.basename(path.dirname(dir)))) {
+          hfDirectories.push({ dir, entries })
+          continue
+        }
+        for (const entry of entries) {
+          if (entry.isSymbolicLink()) continue
+          const full = path.resolve(dir, entry.name)
+          if (entry.isDirectory()) {
+            if (full === vaultRoot) continue
+            stack.push(full)
+          } else if (entry.isFile()) {
+            files.push(full)
+          }
+        }
       }
-      // HF hub blobs dir: filenames ARE sha256 for LFS files — no hashing.
-      if (path.basename(dir) === "blobs" && /^(models|datasets|spaces)--/.test(path.basename(path.dirname(dir)))) {
+      // One metadata pool is shared by all files in this directory batch, so
+      // trees made of many small directories still use the bounded workers.
+      await this.considerFiles(files, preferredSourceId)
+      for (const { dir, entries } of hfDirectories) {
         await this.ingestHfBlobs(dir, entries, preferredSourceId)
-        continue
       }
-      for (const entry of entries) {
-        if (entry.isSymbolicLink()) continue
-        const full = path.resolve(dir, entry.name)
-        if (entry.isDirectory()) {
-          if (full === vaultRoot) continue
-          stack.push(full)
-        } else if (entry.isFile()) {
-          await this.considerFile(full, preferredSourceId)
+    }
+  }
+  async considerFiles(filePaths, preferredSourceId = null) {
+    const stats = new Array(filePaths.length)
+    let cursor = 0
+    const workers = Array.from({ length: Math.min(this.statConcurrency, filePaths.length) }, async () => {
+      while (cursor < filePaths.length) {
+        const index = cursor++
+        try {
+          stats[index] = await fs.promises.stat(filePaths[index])
+        } catch (error) {
+          stats[index] = null
         }
       }
+    })
+    await Promise.all(workers)
+    // Preserve deterministic classification order while parallelizing only
+    // the expensive filesystem metadata reads.
+    for (let index = 0; index < filePaths.length; index++) {
+      if (stats[index]) await this.considerStat(filePaths[index], stats[index], preferredSourceId)
     }
   }
   async considerFile(filePath, preferredSourceId = null) {
     if (filePath.endsWith(TMP_SUFFIX)) return
-    const registry = this.vault.registry
     let st
     try {
       st = await fs.promises.stat(filePath)
     } catch (e) {
       return
     }
+    await this.considerStat(filePath, st, preferredSourceId)
+  }
+  async considerStat(filePath, st, preferredSourceId = null) {
+    if (filePath.endsWith(TMP_SUFFIX)) return
+    const registry = this.vault.registry
     this.state.files += 1
     this.state.bytes_total += st.size
     const sourceKey = preferredSourceId || "pinokio"
@@ -130,19 +238,63 @@ class Sweeper {
       await this.classify(filePath, st, harvested, sourceId)
       return
     }
+    // Only coalesce an inode which the filesystem itself reports as having
+    // multiple names. This avoids trusting placeholder inode values on
+    // filesystems which do not expose usable identity metadata.
+    const reusableInode = st.nlink > 1 && st.ino !== undefined && st.ino !== 0
+    const inodeKey = reusableInode ? registry.inoKey(st.dev, st.ino) : null
+    const existingJob = inodeKey ? this.hashJobsByIno.get(inodeKey) : null
+    const name = { filePath, source_id: sourceId, expected: fileSnapshot(st) }
+    if (existingJob) {
+      existingJob.names.push(name)
+      this.state.inode_reuses += 1
+      return
+    }
+    const job = { inodeKey, names: [name] }
+    if (inodeKey) this.hashJobsByIno.set(inodeKey, job)
+    this.state.hash_total += 1
     this.state.queued += 1
-    this.hashQueue.push({ filePath, source_id: sourceId }).catch(() => {})
+    this.hashQueue.push(job).catch(() => {})
   }
-  async _hashJob({ filePath, source_id }) {
-    this.currentHash = { path: filePath }
+  async _hashJob(job) {
+    const primary = job.names[0]
+    const started = Date.now()
+    this.currentHash = { path: primary.filePath }
     try {
-      const { hash } = await this.vault.hashFile(filePath)
+      const { hash, size } = await this.vault.hashFile(primary.filePath)
       this.stats.hashes += 1
       this.state.hashed += 1
-      const st = await fs.promises.stat(filePath)
-      await this.classify(filePath, st, hash, source_id)
+      this.state.hash_bytes += size || 0
+      const primaryStat = await fs.promises.stat(primary.filePath)
+      // Never publish a digest if the path changed while its bytes were read.
+      // The next manual scan can retry a continuously-mutating file safely.
+      if (size !== primaryStat.size || !sameSnapshot(primary.expected, primaryStat)) {
+        this.state.unstable_hashes += 1
+        return
+      }
+      let index = 0
+      while (index < job.names.length) {
+        const name = job.names[index++]
+        let st
+        try {
+          st = name === primary ? primaryStat : await fs.promises.stat(name.filePath)
+        } catch (e) {
+          continue
+        }
+        // Same (device, inode) is physical identity, not a content guess.
+        // Size and mtime also prevent applying a completed digest after an
+        // in-place mutation. ctime is omitted here because adoption itself
+        // adds a name and therefore legitimately changes inode metadata.
+        if (st.dev !== primaryStat.dev || st.ino !== primaryStat.ino ||
+            st.size !== primaryStat.size || st.mtimeMs !== primaryStat.mtimeMs) continue
+        await this.classify(name.filePath, st, hash, name.source_id)
+      }
     } catch (e) {
     } finally {
+      this.state.hash_duration_ms += Date.now() - started
+      if (job.inodeKey && this.hashJobsByIno.get(job.inodeKey) === job) {
+        this.hashJobsByIno.delete(job.inodeKey)
+      }
       this.currentHash = null
       this.state.queued = Math.max(0, this.state.queued - 1)
     }
@@ -224,35 +376,43 @@ class Sweeper {
   // via the recorded timestamp against mtime (huggingface_hub's own rule).
   async harvestLocalDirEtag(filePath, st) {
     try {
-      let dir = path.dirname(filePath)
+      const start = path.dirname(filePath)
+      let dir = start
       const stop = path.resolve(this.kernel.homedir)
-      const rel = []
+      const visited = []
+      let base = null
       for (let depth = 0; depth < 12; depth++) {
+        if (this.metadataBaseCache.has(dir)) {
+          base = this.metadataBaseCache.get(dir)
+          break
+        }
+        visited.push(dir)
         const metaRoot = path.resolve(dir, ".cache", "huggingface")
         let exists = false
         try {
           exists = (await fs.promises.stat(metaRoot)).isDirectory()
         } catch (e) {}
         if (exists) {
-          const metaPath = path.resolve(metaRoot, ...rel, path.basename(filePath) + ".metadata")
-          let raw
-          try {
-            raw = await fs.promises.readFile(metaPath, "utf8")
-          } catch (e) {
-            return null
-          }
-          const lines = raw.split("\n")
-          const etag = String(lines[1] || "").replace(/"/g, "").trim().toLowerCase()
-          const ts = parseFloat(lines[2] || "0") * 1000
-          if (SHA256_RE.test(etag) && st.mtimeMs <= ts + 1000) {
-            return etag
-          }
-          return null
+          base = dir
+          break
         }
-        if (dir === stop || path.dirname(dir) === dir) return null
-        rel.unshift(path.basename(dir))
+        if (dir === stop || path.dirname(dir) === dir) break
         dir = path.dirname(dir)
       }
+      for (const visitedDir of visited) this.metadataBaseCache.set(visitedDir, base)
+      if (!base) return null
+      const relative = path.relative(base, filePath)
+      const metaPath = path.resolve(base, ".cache", "huggingface", relative + ".metadata")
+      let raw
+      try {
+        raw = await fs.promises.readFile(metaPath, "utf8")
+      } catch (e) {
+        return null
+      }
+      const lines = raw.split("\n")
+      const etag = String(lines[1] || "").replace(/"/g, "").trim().toLowerCase()
+      const ts = parseFloat(lines[2] || "0") * 1000
+      if (SHA256_RE.test(etag) && st.mtimeMs <= ts + 1000) return etag
     } catch (e) {}
     return null
   }
@@ -293,8 +453,9 @@ class Sweeper {
           summary.bytes_saved += result.bytes_saved || 0
         } else if (result.status === "locked") {
           summary.locked += 1
+        } else if (result.status === "unavailable" || result.status === "copy-mode") {
+          summary.unavailable += 1
         } else {
-          registry.duplicates.delete(filePath)
           summary.failed += 1
         }
       } catch (e) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pinokiod",
-  "version": "8.0.37",
+  "version": "8.0.39",
   "description": "",
   "main": "index.js",
   "scripts": {

package/server/index.js

@@ -15740,9 +15740,20 @@ class Server {
         res.json({ enabled: false })
         return
       }
+      if (req.query && req.query.progress === "1") {
+        res.json(vault.progressStatus())
+        return
+      }
       res.json(await vault.status())
     }))
     this.app.get("/vault", ex(async (req, res) => {
+      const { install_required, requirements_pending } = await this.kernel.bin.check({
+        bin: this.kernel.bin.preset("dev"),
+      })
+      if (requirements_pending || install_required) {
+        res.redirect(`/setup/dev?callback=${encodeURIComponent(req.originalUrl)}`)
+        return
+      }
       res.render("vault", { theme: this.theme, agent: req.agent })
     }))
     this.app.post("/vault/action", ex(async (req, res) => {
@@ -15754,6 +15765,19 @@ class Server {
       const body = req.body || {}
       try {
         switch (body.action) {
+          case "add_source": {
+            const result = await vault.addExternalSource(body.path)
+            res.json({
+              created: result.created,
+              source: {
+                id: result.source.id,
+                label: result.source.label,
+                target_path: result.source.root,
+                shareable: !!result.source.shareable
+              }
+            })
+            return
+          }
           case "deduplicate": {
             const scopeId = typeof body.scope_id === "string" ? body.scope_id : null
             if (scopeId) {
@@ -15803,7 +15827,12 @@ class Server {
             vault.sweeper.scan().catch(() => {})
             res.json({ started: true })
             return
+          case "repair":
           case "rebuild":
+            if (vault.sweeper && vault.sweeper.state.active) {
+              res.json({ error: "Wait for the current scan to finish before repairing the index." })
+              return
+            }
             await vault.rebuild()
             res.json({ done: true })
             return