pinokiod 8.0.39 → 8.0.41

package/kernel/vault/walker.js

@@ -0,0 +1,142 @@
+const fs = require('fs')
+const path = require('path')
+const { ENTRY_BATCH_SIZE } = require('./constants')
+
+const isMissingError = (error) => !!(error && (error.code === "ENOENT" || error.code === "ENOTDIR"))
+const isHandledError = (options, error, target) => !!(
+  !isMissingError(error) && options.onError && options.onError(error, target)
+)
+
+// One traversal primitive for Scan and Repair. It owns only directory
+// discovery and symlink policy; callers decide how files are classified.
+async function * walkBatches(root, options = {}) {
+  const resolvedRoot = path.resolve(root)
+  const concurrency = Math.max(1, Number(options.concurrency) || 1)
+  const entryBatchSize = Math.max(1, Number(options.entryBatchSize) || ENTRY_BATCH_SIZE)
+  const statConcurrency = Math.max(1, Number(options.statConcurrency) || 32)
+  const skipDirectory = options.skipDirectory || (() => false)
+  const strictErrors = !!options.strictErrors
+  const pending = [{ dir: resolvedRoot, followRootSymlink: true }]
+  const active = []
+  try {
+    while (pending.length || active.length) {
+      while (active.length < concurrency && pending.length) {
+        const next = pending.pop()
+        const dir = next.dir
+        try {
+          let st = await fs.promises.lstat(dir)
+          if (next.followRootSymlink && st.isSymbolicLink()) {
+            st = await fs.promises.stat(dir)
+          }
+          if (!st.isDirectory()) {
+            yield [{ dir, entries: null, files: [], discoveredDirs: 0, discoveredFiles: 0, firstChunk: true }]
+            continue
+          }
+          active.push({ dir, handle: await fs.promises.opendir(dir), firstChunk: true })
+        } catch (error) {
+          if (isMissingError(error) && dir === resolvedRoot && options.onRootMissing) {
+            options.onRootMissing(error, dir)
+          }
+          if (strictErrors && !isMissingError(error) && !isHandledError(options, error, dir)) throw error
+          yield [{ dir, entries: null, files: [], discoveredDirs: 0, discoveredFiles: 0, firstChunk: true }]
+        }
+      }
+      if (!active.length) continue
+      const batch = await Promise.all(active.map(async (task) => {
+        const entries = []
+        let done = false
+        try {
+          while (entries.length < entryBatchSize) {
+            const entry = await task.handle.read()
+            if (!entry) {
+              done = true
+              break
+            }
+            entries.push(entry)
+          }
+        } catch (error) {
+          if (isMissingError(error) && task.dir === resolvedRoot && options.onRootMissing) {
+            options.onRootMissing(error, task.dir)
+          }
+          if (strictErrors && !isMissingError(error) && !isHandledError(options, error, task.dir)) throw error
+          done = true
+        }
+        return {
+          task, dir: task.dir, entries, done, firstChunk: task.firstChunk,
+          files: [], discoveredDirs: 0, discoveredFiles: 0
+        }
+      }))
+      const unknown = []
+      for (const group of batch) {
+        group.task.firstChunk = false
+        for (const entry of group.entries) {
+          if (entry.isSymbolicLink()) continue
+          const full = path.resolve(group.dir, entry.name)
+          if (entry.isDirectory()) {
+            if (skipDirectory(full)) continue
+            pending.push({ dir: full, followRootSymlink: false })
+            group.discoveredDirs += 1
+          } else if (entry.isFile()) {
+            group.files.push({ path: full, entry })
+            group.discoveredFiles += 1
+          } else {
+            unknown.push({ group, entry, full })
+          }
+        }
+      }
+      if (unknown.length) {
+        const stats = await statMany(
+          unknown.map((item) => item.full),
+          statConcurrency,
+          null,
+          { followSymlinks: false, strictErrors, onError: options.onError }
+        )
+        for (let index = 0; index < unknown.length; index++) {
+          const item = unknown[index]
+          const st = stats[index]
+          if (!st || st.isSymbolicLink()) continue
+          if (st.isDirectory()) {
+            if (skipDirectory(item.full)) continue
+            pending.push({ dir: item.full, followRootSymlink: false })
+            item.group.discoveredDirs += 1
+          } else if (st.isFile()) {
+            item.group.files.push({ path: item.full, entry: item.entry })
+            item.group.discoveredFiles += 1
+          }
+        }
+      }
+      yield batch.map(({ task, done, ...group }) => group)
+      for (let index = active.length - 1; index >= 0; index--) {
+        if (!batch[index].done) continue
+        await active[index].handle.close().catch(() => {})
+        active.splice(index, 1)
+      }
+    }
+  } finally {
+    await Promise.all(active.map((task) => task.handle.close().catch(() => {})))
+  }
+}
+
+const statMany = async (paths, concurrency = 32, onSettled = null, options = {}) => {
+  const results = new Array(paths.length)
+  const readMetadata = options.followSymlinks === false ? fs.promises.lstat : fs.promises.stat
+  let cursor = 0
+  const workers = Array.from({ length: Math.min(concurrency, paths.length) }, async () => {
+    while (cursor < paths.length) {
+      const index = cursor++
+      try {
+        results[index] = await readMetadata(paths[index])
+      } catch (error) {
+        if (options.strictErrors && !isMissingError(error) &&
+            !isHandledError(options, error, paths[index])) throw error
+        results[index] = null
+      } finally {
+        if (onSettled) onSettled(index, results[index])
+      }
+    }
+  })
+  await Promise.all(workers)
+  return results
+}
+
+module.exports = { walkBatches, statMany }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pinokiod",
-  "version": "8.0.39",
+  "version": "8.0.41",
   "description": "",
   "main": "index.js",
   "scripts": {

package/server/index.js

@@ -1990,6 +1990,9 @@ class Server {
     result.hasSnapshots = !!(options && options.hasSnapshots)
     result.pendingSnapshotId = options && options.pendingSnapshotId ? String(options.pendingSnapshotId) : null
     result.registryEnabled = registryEnabled
+    const vault = this.kernel.vault
+    if (vault && vault.ready) await vault.ready
+    result.vault_enabled = !!(vault && vault.enabled)
     if (!registryEnabled) {
       result.pendingSnapshotId = null
     }
@@ -6664,6 +6667,10 @@ class Server {
 
     this.started = false
     this.app = express();
+    this.app.use((req, res, next) => {
+      res.locals.vaultEnabled = !!(this.kernel.vault && this.kernel.vault.enabled)
+      next()
+    })
     this.app.use(cors({
       origin: '*'
     }));
@@ -15732,21 +15739,55 @@ class Server {
 //    }))
 
 
-    // Vault dashboard data: everything from memory, never a walk. The scan
-    // itself runs ONLY via the explicit scan action below.
+    // Vault dashboard data: registry-backed with bounded per-blob stats,
+    // never a discovery walk. Scans run ONLY via the explicit action below.
+    this.app.get("/info/dedup/nav", ex(async (req, res) => {
+      if (!privacyFilterCache.isSameOriginRequest(req)) {
+        res.sendStatus(403)
+        return
+      }
+      const vault = this.kernel.vault
+      if (vault && vault.ready) await vault.ready
+      if (!vault || !vault.enabled) {
+        res.sendStatus(404)
+        return
+      }
+      res.set("Cache-Control", "no-store")
+      const appName = req.query && typeof req.query.app === "string"
+        ? req.query.app
+        : null
+      res.json(appName
+        ? vault.navigationStatusForApp(appName)
+        : vault.navigationStatus())
+    }))
     this.app.get("/info/dedup", ex(async (req, res) => {
+      if (!privacyFilterCache.isSameOriginRequest(req)) {
+        res.sendStatus(403)
+        return
+      }
       const vault = this.kernel.vault
+      if (vault && vault.ready) await vault.ready
       if (!vault || !vault.enabled) {
-        res.json({ enabled: false })
+        res.sendStatus(404)
         return
       }
+      res.set("Cache-Control", "no-store")
+      const scopeId = req.query && typeof req.query.scope_id === "string"
+        ? req.query.scope_id
+        : null
       if (req.query && req.query.progress === "1") {
-        res.json(vault.progressStatus())
+        res.json(vault.progressStatus(scopeId))
         return
       }
-      res.json(await vault.status())
+      res.json(await vault.status(scopeId))
     }))
     this.app.get("/vault", ex(async (req, res) => {
+      const vault = this.kernel.vault
+      if (vault && vault.ready) await vault.ready
+      if (!vault || !vault.enabled) {
+        res.sendStatus(404)
+        return
+      }
       const { install_required, requirements_pending } = await this.kernel.bin.check({
         bin: this.kernel.bin.preset("dev"),
       })
@@ -15754,100 +15795,48 @@ class Server {
         res.redirect(`/setup/dev?callback=${encodeURIComponent(req.originalUrl)}`)
         return
       }
+      await vault.ensureInitialized()
       res.render("vault", { theme: this.theme, agent: req.agent })
     }))
+    this.app.get("/vault/app/:name", ex(async (req, res) => {
+      if (!privacyFilterCache.isSameOriginRequest(req)) {
+        res.sendStatus(403)
+        return
+      }
+      const vault = this.kernel.vault
+      if (vault && vault.ready) await vault.ready
+      if (!vault || !vault.enabled) {
+        res.sendStatus(404)
+        return
+      }
+      await vault.ensureInitialized()
+      await vault.refreshSources()
+      const source = vault.sources().find((item) =>
+        item.kind === "app" && item.app === req.params.name && item.available)
+      if (!source) {
+        res.sendStatus(404)
+        return
+      }
+      res.render("vault_app", {
+        theme: this.theme,
+        agent: req.agent,
+        scope_id: source.id
+      })
+    }))
     this.app.post("/vault/action", ex(async (req, res) => {
+      if (!privacyFilterCache.isSameOriginRequest(req)) {
+        res.sendStatus(403)
+        return
+      }
       const vault = this.kernel.vault
+      if (vault && vault.ready) await vault.ready
       if (!vault || !vault.enabled) {
-        res.json({ error: "vault is not enabled" })
+        res.sendStatus(404)
         return
       }
       const body = req.body || {}
       try {
-        switch (body.action) {
-          case "add_source": {
-            const result = await vault.addExternalSource(body.path)
-            res.json({
-              created: result.created,
-              source: {
-                id: result.source.id,
-                label: result.source.label,
-                target_path: result.source.root,
-                shareable: !!result.source.shareable
-              }
-            })
-            return
-          }
-          case "deduplicate": {
-            const scopeId = typeof body.scope_id === "string" ? body.scope_id : null
-            if (scopeId) {
-              const source = vault.sources().find((item) => item.id === scopeId && item.kind !== "virtual")
-              if (!source) {
-                res.json({ error: "That location is no longer available. Scan again to refresh it." })
-                return
-              }
-              if (!source.shareable) {
-                res.json({ error: "This location is on a disk that cannot share space with this vault." })
-                return
-              }
-              if (source.kind === "app") {
-                const appRoot = this.kernel.path("api", source.app) + path.sep
-                const running = this.kernel.api && this.kernel.api.running ? this.kernel.api.running : {}
-                const busy = Object.keys(running).some((k) => running[k] && k.startsWith(appRoot))
-                if (busy) {
-                  res.json({ error: "This app has a running script. Stop it first, then deduplicate." })
-                  return
-                }
-              }
-              res.json(await vault.sweeper.convertPending(undefined, {
-                scope_id: scopeId, batch_id: `batch-${Date.now()}`
-              }))
-              return
-            }
-            const app = Object.prototype.hasOwnProperty.call(body, "app") ? (body.app || null) : undefined
-            if (app) {
-              const appRoot = this.kernel.path("api", app) + path.sep
-              const running = this.kernel.api && this.kernel.api.running ? this.kernel.api.running : {}
-              const busy = Object.keys(running).some((k) => running[k] && k.startsWith(appRoot))
-              if (busy) {
-                res.json({ error: "This app has a running script. Stop it first, then deduplicate." })
-                return
-              }
-            }
-            res.json(await vault.sweeper.convertPending(app, { batch_id: `batch-${Date.now()}` }))
-            return
-          }
-          case "reclaim":
-            res.json(await vault.reclaim(body.hash))
-            return
-          case "reclaim_all":
-            res.json(await vault.reclaimAll())
-            return
-          case "scan":
-            vault.sweeper.scan().catch(() => {})
-            res.json({ started: true })
-            return
-          case "repair":
-          case "rebuild":
-            if (vault.sweeper && vault.sweeper.state.active) {
-              res.json({ error: "Wait for the current scan to finish before repairing the index." })
-              return
-            }
-            await vault.rebuild()
-            res.json({ done: true })
-            return
-          case "undo":
-            res.json(await vault.undoBatch(body.batch_id))
-            return
-          case "detach":
-            res.json(await vault.detach(body.path))
-            return
-          case "reshare":
-            res.json(await vault.reshare(body.path))
-            return
-          default:
-            res.json({ error: "unknown action" })
-        }
+        res.json(await vault.perform(body.action, body))
       } catch (e) {
         res.json({ error: e && e.message ? e.message : String(e) })
       }

package/server/lib/privacy_filter_cache.js

@@ -66,7 +66,7 @@ function assertKnownAsset(relativePath) {
   return normalized
 }
 
-function isInstallRequestAllowed(req) {
+function isSameOriginRequest(req) {
   const fetchSite = String(req && typeof req.get === 'function' ? req.get('Sec-Fetch-Site') || '' : '').trim().toLowerCase()
   if (fetchSite && fetchSite !== 'same-origin' && fetchSite !== 'none') {
     return false
@@ -90,6 +90,8 @@ function isInstallRequestAllowed(req) {
   }
 }
 
+const isInstallRequestAllowed = isSameOriginRequest
+
 function assetPath(kernel, relativePath) {
   const normalized = assertKnownAsset(relativePath)
   return path.resolve(modelRoot(kernel), ...normalized.split('/'))
@@ -280,6 +282,7 @@ module.exports = {
   normalizeDtype,
   cacheRoot,
   modelRoot,
+  isSameOriginRequest,
   isInstallRequestAllowed,
   status,
   ensure

package/server/public/storage-size.js

@@ -0,0 +1,12 @@
+(function (root) {
+  const base = 1000
+  const units = ["B", "KB", "MB", "GB", "TB"]
+
+  root.PinokioFormatStorageSize = (value) => {
+    const bytes = Number(value)
+    if (!Number.isFinite(bytes) || bytes <= 0) return "0 B"
+    const index = Math.min(units.length - 1, Math.floor(Math.log(bytes) / Math.log(base)))
+    const scaled = bytes / Math.pow(base, index)
+    return `${Number(scaled.toFixed(index ? 2 : 0))} ${units[index]}`
+  }
+})(window)

package/server/public/style.css

@@ -1567,6 +1567,14 @@ body.main-sidebar-page aside.main-sidebar .tab.selected .main-sidebar-action-ico
   background: rgba(45, 180, 92, 0.14);
   color: #247a43;
 }
+.main-sidebar [data-main-sidebar-vault-tab][data-vault-state="new"]:not(.selected) {
+  background: var(--pinokio-sidebar-tab-hover);
+}
+.main-sidebar .main-sidebar-status-badge[data-state="new"] {
+  border: 1px solid rgba(71, 85, 105, 0.22);
+  background: transparent;
+  color: rgba(31, 41, 55, 0.72);
+}
 body.dark .main-sidebar .main-sidebar-status-badge {
   background: rgba(255, 255, 255, 0.08);
   color: rgba(229, 231, 235, 0.68);
@@ -1575,6 +1583,11 @@ body.dark .main-sidebar .main-sidebar-status-badge[data-state="on"] {
   background: rgba(74, 222, 128, 0.14);
   color: #86efac;
 }
+body.dark .main-sidebar .main-sidebar-status-badge[data-state="new"] {
+  border-color: rgba(226, 232, 240, 0.22);
+  background: transparent;
+  color: rgba(248, 250, 252, 0.76);
+}
 .main-sidebar .tab.submenu {
   padding-left: 14px !important;
 }

package/server/public/tab-link-popover.js

@@ -25,6 +25,9 @@
     if (!link.hasAttribute("href")) {
       return false
     }
+    if (link.dataset.tabLinkPopover === "false") {
+      return false
+    }
     const href = link.getAttribute("href")
     return typeof href === "string" && href.trim().length > 0
   }

package/server/public/vault-nav.js

@@ -0,0 +1,96 @@
+(() => {
+  if (window.__pinokioVaultNavigationStatusInit) return
+  window.__pinokioVaultNavigationStatusInit = true
+
+  const formatBytes = (value) => {
+    const bytes = Number(value)
+    if (!Number.isFinite(bytes) || bytes <= 0) return ""
+    const units = ["B", "KB", "MB", "GB", "TB"]
+    const index = Math.min(units.length - 1, Math.floor(Math.log(bytes) / Math.log(1000)))
+    const scaled = bytes / Math.pow(1000, index)
+    const digits = scaled >= 10 || index === 0 ? 0 : 1
+    return `${Number(scaled.toFixed(digits))} ${units[index]}`
+  }
+
+  const init = () => {
+    const tab = document.querySelector("[data-main-sidebar-vault-tab], [data-app-vault-tab]")
+    if (!tab) return
+    const appName = tab.dataset.appVaultName || ""
+    const appMode = !!appName
+    const status = tab.querySelector(appMode
+      ? "[data-app-vault-status]"
+      : "[data-main-sidebar-vault-status]")
+    const prompt = tab.querySelector("[data-app-vault-prompt]")
+    if (!status) return
+    let requestId = 0
+
+    const render = (data) => {
+      let state = ""
+      let label = ""
+      let title = "Save space"
+      if (data && data.scanning) {
+        state = "scanning"
+        label = "Scanning"
+        title = appMode ? "Save space, scanning this app" : "Save space, scanning"
+      } else if (!data || !data.any_scan) {
+        state = "new"
+        label = "NEW"
+        title = "Save space — find duplicate files and use less disk"
+      } else if (!data.scanned) {
+        state = "not-scanned"
+        label = "Not scanned"
+        title = appMode ? "Save space, this app has not been scanned" : "Save space, not scanned"
+      } else if (Number(data.pending_bytes) > 0) {
+        state = "pending"
+        label = `${data.incomplete ? "≥" : ""}${formatBytes(data.pending_bytes)}`
+        title = `Save space, ${data.incomplete ? "at least " : ""}${formatBytes(data.pending_bytes)} can be saved${appMode ? " in this app" : ""}`
+      }
+
+      tab.dataset.vaultState = state || "ready"
+      tab.setAttribute("aria-label", title)
+      tab.setAttribute("title", title)
+      if (prompt) prompt.hidden = state !== "new"
+      status.hidden = !label
+      status.textContent = label
+      if (state) status.dataset.state = state
+      else delete status.dataset.state
+    }
+
+    const load = () => {
+      const currentRequest = ++requestId
+      const query = appMode ? `?app=${encodeURIComponent(appName)}` : ""
+      fetch(`/info/dedup/nav${query}`, { cache: "no-store" })
+        .then((response) => {
+          if (!response.ok) throw new Error(`Save space status failed: ${response.status}`)
+          return response.json()
+        })
+        .then((data) => {
+          if (currentRequest === requestId) render(data)
+        })
+        .catch(() => {
+          if (currentRequest !== requestId) return
+          status.hidden = true
+          if (prompt) prompt.hidden = true
+          delete status.dataset.state
+          delete tab.dataset.vaultState
+          tab.removeAttribute("aria-label")
+          tab.removeAttribute("title")
+        })
+    }
+
+    document.addEventListener("pinokio:vault-status-changed", load)
+    window.addEventListener("message", (event) => {
+      if (event.origin === window.location.origin &&
+          event.data && event.data.type === "pinokio:vault-status-changed") {
+        load()
+      }
+    })
+    load()
+  }
+
+  if (document.readyState === "loading") {
+    document.addEventListener("DOMContentLoaded", init, { once: true })
+  } else {
+    init()
+  }
+})()