pinokiod 8.0.36 → 8.0.37

package/kernel/vault/registry.js

@@ -0,0 +1,169 @@
+const fs = require('fs')
+const path = require('path')
+
+// Persisted record of blobs and their known names.
+// The registry is a cache of the filesystem, never the source of truth
+// (spec/requirements/shared-model-store.md "Registry").
+class Registry {
+  constructor(root) {
+    this.root = root
+    this.snapshotPath = path.resolve(root, "registry.json")
+    this.eventsPath = path.resolve(root, "events.ndjson")
+    this.reset()
+    this.persistTimer = null
+    this.persistDelay = 500
+    this.flushPromise = Promise.resolve()
+  }
+  reset() {
+    this.blobs = new Map()      // hash -> { size, first_seen, source_urls, verified_at, orphan }
+    this.links = new Map()      // path -> { hash, app, source_id, dev, ino, created, mode }
+    this.scanIndex = new Map()  // path -> { size, mtime, dev, ino, hash, source_id }
+    this.duplicates = new Map() // path -> { hash, size, app, source_id, discovered } pending user conversion
+    this.excluded = new Map()   // path -> { ts, source_id } user chose "keep independent"
+    this.lastScan = null        // { ts, dirs, files, bytes_total, home_bytes_total, source_bytes }
+    this.totals = { lifetime_bytes_saved: 0 }
+    this.byIno = new Map()      // "dev:ino" -> hash
+  }
+  setLastScan(meta) {
+    this.lastScan = meta
+    this.schedulePersist()
+  }
+  inoKey(dev, ino) {
+    return `${dev}:${ino}`
+  }
+  addBlob(hash, meta) {
+    if (!this.blobs.has(hash)) {
+      this.blobs.set(hash, Object.assign({
+        size: 0,
+        first_seen: Date.now(),
+        source_urls: [],
+        verified_at: null
+      }, meta))
+    } else if (meta && meta.source_urls && meta.source_urls.length) {
+      const blob = this.blobs.get(hash)
+      for (const url of meta.source_urls) {
+        if (!blob.source_urls.includes(url)) blob.source_urls.push(url)
+      }
+    }
+    this.schedulePersist()
+  }
+  addLink(filePath, entry) {
+    this.links.set(filePath, Object.assign({ created: Date.now(), mode: "link" }, entry))
+    if (entry.dev !== undefined && entry.ino !== undefined) {
+      this.byIno.set(this.inoKey(entry.dev, entry.ino), entry.hash)
+    }
+    this.schedulePersist()
+  }
+  removeLink(filePath) {
+    const entry = this.links.get(filePath)
+    if (entry) {
+      this.links.delete(filePath)
+      const key = this.inoKey(entry.dev, entry.ino)
+      let stillUsed = false
+      for (const other of this.links.values()) {
+        if (this.inoKey(other.dev, other.ino) === key) { stillUsed = true; break }
+      }
+      if (!stillUsed) this.byIno.delete(key)
+      this.schedulePersist()
+    }
+  }
+  removeBlob(hash) {
+    this.blobs.delete(hash)
+    for (const [p, entry] of [...this.links]) {
+      if (entry.hash === hash) this.removeLink(p)
+    }
+    this.schedulePersist()
+  }
+  addSaved(bytes) {
+    this.totals.lifetime_bytes_saved += bytes
+    this.schedulePersist()
+  }
+  // Returns { corrupt: true } when the snapshot exists but cannot be parsed,
+  // so the caller can rebuild from disk instead of blocking startup.
+  async load() {
+    this.reset()
+    let raw
+    try {
+      raw = await fs.promises.readFile(this.snapshotPath, "utf8")
+    } catch (e) {
+      return { corrupt: false, existed: false }
+    }
+    let json
+    try {
+      json = JSON.parse(raw)
+    } catch (e) {
+      return { corrupt: true, existed: true }
+    }
+    for (const [k, v] of Object.entries(json.blobs || {})) this.blobs.set(k, v)
+    for (const [k, v] of Object.entries(json.links || {})) {
+      this.links.set(k, v)
+      if (v.dev !== undefined && v.ino !== undefined) {
+        this.byIno.set(this.inoKey(v.dev, v.ino), v.hash)
+      }
+    }
+    for (const [k, v] of Object.entries(json.scan_index || {})) this.scanIndex.set(k, v)
+    for (const [k, v] of Object.entries(json.duplicates || {})) this.duplicates.set(k, v)
+    for (const [k, v] of Object.entries(json.excluded || {})) this.excluded.set(k, v)
+    if (json.last_scan) this.lastScan = json.last_scan
+    if (json.totals && typeof json.totals.lifetime_bytes_saved === "number") {
+      this.totals = json.totals
+    }
+    return { corrupt: false, existed: true }
+  }
+  schedulePersist() {
+    if (this.persistTimer) return
+    this.persistTimer = setTimeout(() => {
+      this.persistTimer = null
+      this.flush().catch(() => {})
+    }, this.persistDelay)
+    if (this.persistTimer.unref) this.persistTimer.unref()
+  }
+  async flush() {
+    if (this.persistTimer) {
+      clearTimeout(this.persistTimer)
+      this.persistTimer = null
+    }
+    const json = {
+      version: 1,
+      blobs: Object.fromEntries(this.blobs),
+      links: Object.fromEntries(this.links),
+      scan_index: Object.fromEntries(this.scanIndex),
+      duplicates: Object.fromEntries(this.duplicates),
+      excluded: Object.fromEntries(this.excluded),
+      last_scan: this.lastScan,
+      totals: this.totals
+    }
+    const write = async () => {
+      const tmp = this.snapshotPath + ".tmp"
+      await fs.promises.mkdir(this.root, { recursive: true })
+      await fs.promises.writeFile(tmp, JSON.stringify(json))
+      await fs.promises.rename(tmp, this.snapshotPath)
+    }
+    const pending = this.flushPromise.then(write, write)
+    this.flushPromise = pending.catch(() => {})
+    return pending
+  }
+  async appendEvent(event) {
+    const line = JSON.stringify(Object.assign({ ts: Date.now() }, event)) + "\n"
+    await fs.promises.appendFile(this.eventsPath, line).catch(() => {})
+  }
+  // Torn final line (crash mid-append) is discarded, per persistence rules.
+  async readEvents() {
+    let raw
+    try {
+      raw = await fs.promises.readFile(this.eventsPath, "utf8")
+    } catch (e) {
+      return []
+    }
+    const events = []
+    for (const line of raw.split("\n")) {
+      if (!line.trim()) continue
+      try {
+        events.push(JSON.parse(line))
+      } catch (e) {}
+    }
+    return events
+  }
+}
+
+module.exports = Registry

package/kernel/vault/sweeper.js

@@ -0,0 +1,309 @@
+const fs = require('fs')
+const path = require('path')
+const fastq = require('fastq')
+
+// Manual scan engine (spec/requirements/shared-model-store.md).
+// Scans run ONLY when the user asks — there are no automatic triggers.
+// The walk is generic: every regular file counts toward folder totals, and
+// every file >= the size threshold is a candidate. No name heuristics.
+// A scan never converts anything: it adopts new content (adds a vault name,
+// which mutates nothing) and lists byte-identical copies as pending, for the
+// user to deduplicate explicitly.
+
+const SHA256_RE = /^[0-9a-f]{64}$/
+const TMP_SUFFIX = ".pinokio-dedup-tmp"
+
+class Sweeper {
+  constructor(vault) {
+    this.vault = vault
+    this.kernel = vault.kernel
+    this.state = this.idleState()
+    this.currentHash = null
+    this.hashQueue = fastq.promise(this, this._hashJob, 1)
+    this.stats = { hashes: 0, ingested: 0 }
+  }
+  idleState() {
+    return {
+      active: false, dirs: 0, files: 0, bytes_total: 0,
+      home_bytes_total: 0, source_bytes: {},
+      candidates: 0, hashed: 0, queued: 0, started: null
+    }
+  }
+  // The one entry point: the Pinokio home plus explicit linked imports.
+  async scan() {
+    if (this.state.active) return { already_running: true }
+    this.state = Object.assign(this.idleState(), { active: true, started: Date.now() })
+    try {
+      await this.vault.refreshSources()
+      for (const source of this.vault.scanRoots()) {
+        await this.walk(source.root, source.source_id === "pinokio" ? null : source.source_id)
+      }
+      await this.settle()
+      this.vault.registry.setLastScan({
+        ts: Date.now(),
+        dirs: this.state.dirs,
+        files: this.state.files,
+        bytes_total: this.state.bytes_total,
+        home_bytes_total: this.state.home_bytes_total,
+        source_bytes: Object.assign({}, this.state.source_bytes)
+      })
+    } finally {
+      this.state.active = false
+    }
+    return {
+      dirs: this.state.dirs, files: this.state.files,
+      bytes_total: this.state.bytes_total, candidates: this.state.candidates
+    }
+  }
+  async settle() {
+    await this.hashQueue.drained()
+  }
+  async walk(root, preferredSourceId = null) {
+    const vaultRoot = this.vault.root
+    const stack = [root]
+    while (stack.length) {
+      const dir = stack.pop()
+      this.state.dirs += 1
+      let entries
+      try {
+        entries = await fs.promises.readdir(dir, { withFileTypes: true })
+      } catch (e) {
+        continue
+      }
+      // HF hub blobs dir: filenames ARE sha256 for LFS files — no hashing.
+      if (path.basename(dir) === "blobs" && /^(models|datasets|spaces)--/.test(path.basename(path.dirname(dir)))) {
+        await this.ingestHfBlobs(dir, entries, preferredSourceId)
+        continue
+      }
+      for (const entry of entries) {
+        if (entry.isSymbolicLink()) continue
+        const full = path.resolve(dir, entry.name)
+        if (entry.isDirectory()) {
+          if (full === vaultRoot) continue
+          stack.push(full)
+        } else if (entry.isFile()) {
+          await this.considerFile(full, preferredSourceId)
+        }
+      }
+    }
+  }
+  async considerFile(filePath, preferredSourceId = null) {
+    if (filePath.endsWith(TMP_SUFFIX)) return
+    const registry = this.vault.registry
+    let st
+    try {
+      st = await fs.promises.stat(filePath)
+    } catch (e) {
+      return
+    }
+    this.state.files += 1
+    this.state.bytes_total += st.size
+    const sourceKey = preferredSourceId || "pinokio"
+    this.state.source_bytes[sourceKey] = (this.state.source_bytes[sourceKey] || 0) + st.size
+    if (!preferredSourceId) this.state.home_bytes_total += st.size
+    if (st.size < this.vault.sizeThreshold) return
+    if (registry.excluded.has(filePath)) return
+    this.state.candidates += 1
+    const source = this.vault.sourceForPath(filePath, preferredSourceId)
+    const sourceId = source ? source.id : null
+    const appName = source && source.kind === "app" ? source.app : null
+    // Known and unchanged: nothing to do.
+    const inoHash = registry.byIno.get(registry.inoKey(st.dev, st.ino))
+    if (inoHash && st.nlink > 1) {
+      const cached = registry.scanIndex.get(filePath)
+      const unchanged = cached && cached.size === st.size && cached.mtime === st.mtimeMs && cached.ino === st.ino
+      if (unchanged || !cached) {
+        if (!registry.links.has(filePath)) {
+          registry.addLink(filePath, { hash: inoHash, app: appName, source_id: sourceId, dev: st.dev, ino: st.ino, mode: "link" })
+        }
+        if (!cached) this.updateScanIndex(filePath, st, inoHash, sourceId)
+        return
+      }
+    }
+    const cached = registry.scanIndex.get(filePath)
+    if (cached && cached.size === st.size && cached.mtime === st.mtimeMs && cached.ino === st.ino && cached.hash) {
+      await this.classify(filePath, st, cached.hash, sourceId)
+      return
+    }
+    const harvested = await this.harvestLocalDirEtag(filePath, st)
+    if (harvested) {
+      await this.classify(filePath, st, harvested, sourceId)
+      return
+    }
+    this.state.queued += 1
+    this.hashQueue.push({ filePath, source_id: sourceId }).catch(() => {})
+  }
+  async _hashJob({ filePath, source_id }) {
+    this.currentHash = { path: filePath }
+    try {
+      const { hash } = await this.vault.hashFile(filePath)
+      this.stats.hashes += 1
+      this.state.hashed += 1
+      const st = await fs.promises.stat(filePath)
+      await this.classify(filePath, st, hash, source_id)
+    } catch (e) {
+    } finally {
+      this.currentHash = null
+      this.state.queued = Math.max(0, this.state.queued - 1)
+    }
+  }
+  // hash known → adopt if new content; otherwise a byte-identical copy:
+  // ALWAYS pending, never converted by a scan.
+  async classify(filePath, st, hash, preferredSourceId = null) {
+    const registry = this.vault.registry
+    const source = this.vault.sourceForPath(filePath, preferredSourceId)
+    const sourceId = source ? source.id : null
+    const appName = source && source.kind === "app" ? source.app : null
+    // Divergence: this inode was registered under a different hash — the
+    // shared content was written in place. Evict the stale blob.
+    const priorHash = registry.byIno.get(registry.inoKey(st.dev, st.ino))
+    if (priorHash && priorHash !== hash) {
+      const stalePath = this.vault.storePathFor(priorHash)
+      try {
+        const staleStat = await fs.promises.stat(stalePath)
+        if (staleStat.dev === st.dev && staleStat.ino === st.ino) {
+          await fs.promises.unlink(stalePath)
+        }
+      } catch (e) {}
+      registry.removeBlob(priorHash)
+      registry.appendEvent({ kind: "diverged", hash: priorHash, path: filePath, app: appName, source_id: sourceId, size: st.size })
+    }
+    const storePath = this.vault.storePathFor(hash)
+    let storeStat = null
+    try {
+      storeStat = await fs.promises.stat(storePath)
+    } catch (e) {}
+    if (!storeStat && !registry.blobs.has(hash)) {
+      await this.vault.adopt(filePath, hash, { app: appName, source_id: sourceId })
+      this.updateScanIndex(filePath, st, hash, sourceId)
+      return
+    }
+    if (storeStat && storeStat.dev === st.dev && storeStat.ino === st.ino) {
+      if (!registry.links.has(filePath)) {
+        registry.addLink(filePath, { hash, app: appName, source_id: sourceId, dev: st.dev, ino: st.ino, mode: "link" })
+      }
+      this.updateScanIndex(filePath, st, hash, sourceId)
+      return
+    }
+    if (!registry.duplicates.has(filePath)) {
+      registry.duplicates.set(filePath, { hash, size: st.size, app: appName, source_id: sourceId, discovered: Date.now() })
+      registry.schedulePersist()
+      registry.appendEvent({ kind: "found", hash, path: filePath, app: appName, source_id: sourceId, size: st.size })
+    }
+    this.updateScanIndex(filePath, st, hash, sourceId)
+  }
+  updateScanIndex(filePath, st, hash, sourceId = null) {
+    this.vault.registry.scanIndex.set(filePath, {
+      size: st.size, mtime: st.mtimeMs, dev: st.dev, ino: st.ino, hash: hash || null, source_id: sourceId
+    })
+    this.vault.registry.schedulePersist()
+  }
+  async ingestHfBlobs(dir, entries, preferredSourceId = null) {
+    for (const entry of entries) {
+      if (!entry.isFile() || entry.isSymbolicLink()) continue
+      const name = entry.name
+      const full = path.resolve(dir, name)
+      let st
+      try {
+        st = await fs.promises.stat(full)
+      } catch (e) { continue }
+      this.state.files += 1
+      this.state.bytes_total += st.size
+      const sourceKey = preferredSourceId || "pinokio"
+      this.state.source_bytes[sourceKey] = (this.state.source_bytes[sourceKey] || 0) + st.size
+      if (!preferredSourceId) this.state.home_bytes_total += st.size
+      if (!SHA256_RE.test(name)) continue
+      if (st.size < this.vault.sizeThreshold) continue
+      if (this.vault.registry.excluded.has(full)) continue
+      this.state.candidates += 1
+      this.stats.ingested += 1
+      await this.classify(full, st, name, preferredSourceId)
+    }
+  }
+  // hf CLI --local-dir metadata: etag is the sha256 for LFS files; freshness
+  // via the recorded timestamp against mtime (huggingface_hub's own rule).
+  async harvestLocalDirEtag(filePath, st) {
+    try {
+      let dir = path.dirname(filePath)
+      const stop = path.resolve(this.kernel.homedir)
+      const rel = []
+      for (let depth = 0; depth < 12; depth++) {
+        const metaRoot = path.resolve(dir, ".cache", "huggingface")
+        let exists = false
+        try {
+          exists = (await fs.promises.stat(metaRoot)).isDirectory()
+        } catch (e) {}
+        if (exists) {
+          const metaPath = path.resolve(metaRoot, ...rel, path.basename(filePath) + ".metadata")
+          let raw
+          try {
+            raw = await fs.promises.readFile(metaPath, "utf8")
+          } catch (e) {
+            return null
+          }
+          const lines = raw.split("\n")
+          const etag = String(lines[1] || "").replace(/"/g, "").trim().toLowerCase()
+          const ts = parseFloat(lines[2] || "0") * 1000
+          if (SHA256_RE.test(etag) && st.mtimeMs <= ts + 1000) {
+            return etag
+          }
+          return null
+        }
+        if (dir === stop || path.dirname(dir) === dir) return null
+        rel.unshift(path.basename(dir))
+        dir = path.dirname(dir)
+      }
+    } catch (e) {}
+    return null
+  }
+  appNameFor(filePath) {
+    return this.vault.appNameFor(filePath)
+  }
+  // User-triggered batch conversion (the Deduplicate buttons). Locked files
+  // stay pending; the user can retry after stopping the app.
+  async convertPending(appName, opts = {}) {
+    const registry = this.vault.registry
+    const batch = opts.batch_id || `batch-${Date.now()}`
+    const scopeId = opts.scope_id || null
+    const summary = { converted: 0, bytes_saved: 0, locked: 0, unavailable: 0, failed: 0 }
+    for (const [filePath, entry] of [...registry.duplicates]) {
+      const location = this.vault.locationForPath(filePath, entry.source_id)
+      if (scopeId && location.source_id !== scopeId) continue
+      if (!scopeId && appName !== undefined && (entry.app || null) !== appName) continue
+      try {
+        if (scopeId) {
+          const source = this.vault.sourceForPath(filePath, location.source_id)
+          let targetStat = null
+          let storeStat = null
+          try {
+            targetStat = await fs.promises.stat(filePath)
+            storeStat = await fs.promises.stat(this.vault.storePathFor(entry.hash))
+          } catch (e) {}
+          if (!source || !source.shareable || !targetStat || !storeStat || targetStat.dev !== storeStat.dev) {
+            summary.unavailable += 1
+            continue
+          }
+        }
+        const result = await this.vault.convert(filePath, entry.hash, {
+          app: entry.app, source_id: location.source_id, batch_id: batch
+        })
+        if (result.status === "converted" || result.status === "already") {
+          registry.duplicates.delete(filePath)
+          summary.converted += 1
+          summary.bytes_saved += result.bytes_saved || 0
+        } else if (result.status === "locked") {
+          summary.locked += 1
+        } else {
+          registry.duplicates.delete(filePath)
+          summary.failed += 1
+        }
+      } catch (e) {
+        summary.failed += 1
+      }
+    }
+    registry.schedulePersist()
+    return summary
+  }
+}
+
+module.exports = Sweeper

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pinokiod",
-  "version": "8.0.36",
+  "version": "8.0.37",
   "description": "",
   "main": "index.js",
   "scripts": {

package/server/index.js

@@ -15732,6 +15732,97 @@ class Server {
 //    }))
 
 
+    // Vault dashboard data: everything from memory, never a walk. The scan
+    // itself runs ONLY via the explicit scan action below.
+    this.app.get("/info/dedup", ex(async (req, res) => {
+      const vault = this.kernel.vault
+      if (!vault || !vault.enabled) {
+        res.json({ enabled: false })
+        return
+      }
+      res.json(await vault.status())
+    }))
+    this.app.get("/vault", ex(async (req, res) => {
+      res.render("vault", { theme: this.theme, agent: req.agent })
+    }))
+    this.app.post("/vault/action", ex(async (req, res) => {
+      const vault = this.kernel.vault
+      if (!vault || !vault.enabled) {
+        res.json({ error: "vault is not enabled" })
+        return
+      }
+      const body = req.body || {}
+      try {
+        switch (body.action) {
+          case "deduplicate": {
+            const scopeId = typeof body.scope_id === "string" ? body.scope_id : null
+            if (scopeId) {
+              const source = vault.sources().find((item) => item.id === scopeId && item.kind !== "virtual")
+              if (!source) {
+                res.json({ error: "That location is no longer available. Scan again to refresh it." })
+                return
+              }
+              if (!source.shareable) {
+                res.json({ error: "This location is on a disk that cannot share space with this vault." })
+                return
+              }
+              if (source.kind === "app") {
+                const appRoot = this.kernel.path("api", source.app) + path.sep
+                const running = this.kernel.api && this.kernel.api.running ? this.kernel.api.running : {}
+                const busy = Object.keys(running).some((k) => running[k] && k.startsWith(appRoot))
+                if (busy) {
+                  res.json({ error: "This app has a running script. Stop it first, then deduplicate." })
+                  return
+                }
+              }
+              res.json(await vault.sweeper.convertPending(undefined, {
+                scope_id: scopeId, batch_id: `batch-${Date.now()}`
+              }))
+              return
+            }
+            const app = Object.prototype.hasOwnProperty.call(body, "app") ? (body.app || null) : undefined
+            if (app) {
+              const appRoot = this.kernel.path("api", app) + path.sep
+              const running = this.kernel.api && this.kernel.api.running ? this.kernel.api.running : {}
+              const busy = Object.keys(running).some((k) => running[k] && k.startsWith(appRoot))
+              if (busy) {
+                res.json({ error: "This app has a running script. Stop it first, then deduplicate." })
+                return
+              }
+            }
+            res.json(await vault.sweeper.convertPending(app, { batch_id: `batch-${Date.now()}` }))
+            return
+          }
+          case "reclaim":
+            res.json(await vault.reclaim(body.hash))
+            return
+          case "reclaim_all":
+            res.json(await vault.reclaimAll())
+            return
+          case "scan":
+            vault.sweeper.scan().catch(() => {})
+            res.json({ started: true })
+            return
+          case "rebuild":
+            await vault.rebuild()
+            res.json({ done: true })
+            return
+          case "undo":
+            res.json(await vault.undoBatch(body.batch_id))
+            return
+          case "detach":
+            res.json(await vault.detach(body.path))
+            return
+          case "reshare":
+            res.json(await vault.reshare(body.path))
+            return
+          default:
+            res.json({ error: "unknown action" })
+        }
+      } catch (e) {
+        res.json({ error: e && e.message ? e.message : String(e) })
+      }
+    }))
     this.app.get("/info/scripts", ex(async (req, res) => {
     /*
       returns something like this by using the this.kernel.memory.local variable, extracting the api name and adding all running scripts in each associated array, setting the uri as the script path, and the local variables as the local attribute

package/server/views/partials/main_sidebar.ejs

@@ -26,6 +26,7 @@
     <div class='main-sidebar-section' aria-label="Manage">
       <div class="main-sidebar-section-title">Manage</div>
       <a class="tab <%= sidebarSelected === 'checkpoints' ? 'selected' : '' %>" href="/checkpoints"><i class="fa-solid fa-clock-rotate-left"></i><div class='caption'>Checkpoints</div></a>
+      <a class="tab <%= sidebarSelected === 'vault' ? 'selected' : '' %>" href="/vault"><i class="fa-solid fa-hard-drive"></i><div class='caption'>Vault</div></a>
       <a class="tab <%= sidebarSelected === 'tools' ? 'selected' : '' %>" href="/tools"><i class="fa-solid fa-toolbox"></i><div class='caption'>Tools</div></a>
       <a class="tab <%= sidebarSelected === 'plugins' ? 'selected' : '' %>" href="/plugins"><i class="fa-solid fa-plug-circle-bolt"></i><div class='caption'>Plugins</div></a>
       <a class="tab <%= sidebarSelected === 'tasks' ? 'selected' : '' %>" href="/tasks"><i class="fa-solid fa-list-check"></i><div class='caption'>Tasks</div></a>