pinokiod 8.0.69 → 8.0.71

package/kernel/index.js

@@ -1062,15 +1062,15 @@ class Kernel {
   kill() {
     process.kill(process.pid, "SIGTERM")
   }
-  async disposeVaultAutomaticScans() {
+  async disposeVault() {
     const vault = this.vault
-    if (!vault || !vault.automaticScans) return false
+    if (!vault) return false
     if (vault.ready) await Promise.resolve(vault.ready).catch(() => {})
     try {
-      await vault.automaticScans.dispose()
+      await vault.dispose()
       return true
     } catch (error) {
-      console.warn("Vault automatic cleanup error:",
+      console.warn("Vault cleanup error:",
         error && error.message ? error.message : error)
       return false
     }
@@ -1092,7 +1092,7 @@ class Kernel {
     })
 
     let home = this.store.get("home") || process.env.PINOKIO_HOME
-    await this.disposeVaultAutomaticScans()
+    await this.disposeVault()
     this.vault = null
     this.homedir = home
 

package/kernel/shell.js

@@ -290,6 +290,7 @@ class Shell {
       }
     }
 
+    setDefaultEnvValue(this.env, "UV_HTTP_TIMEOUT", "60")
     setDefaultEnvValue(this.env, "HF_HUB_DISABLE_UPDATE_CHECK", "1")
     setDefaultEnvValue(this.env, "HF_TOKEN_PATH", this.kernel.envs?.HF_TOKEN_PATH || path.resolve(this.kernel.homedir, "cache", "HF_AUTH", "token"))
 

package/kernel/vault/automatic_scans.js

@@ -2,9 +2,12 @@ const fs = require("fs")
 const path = require("path")
 const {
   SIZE_THRESHOLD,
-  isCandidateFileSize
+  isCandidateFileSize,
+  SHA256_RE,
+  ENTRY_BATCH_SIZE
 } = require("./constants")
 const { statMany } = require("./walker")
+const { sameSnapshot } = require("./snapshot")
 const {
   cancelledError,
   isPathError
@@ -24,6 +27,11 @@ const inside = (root, candidate) => {
   )
 }
 
+const pathKey = (value) => {
+  const resolved = path.resolve(value)
+  return process.platform === "win32" ? resolved.toLowerCase() : resolved
+}
+
 class AutomaticScans {
   constructor(vault) {
     this.vault = vault
@@ -745,6 +753,7 @@ class AutomaticScans {
     }
     this.active.cancelled = true
     this.active.reason = reason
+    if (this.active.controller) this.active.controller.abort()
     this.log("check-cancel-requested", { app, reason })
     return true
   }
@@ -755,6 +764,90 @@ class AutomaticScans {
     }
   }
 
+  automaticHashKey(entry) {
+    return [
+      this.automaticIdentityKey(entry),
+      entry.size,
+      entry.mtime,
+      entry.ctime
+    ].join("\0")
+  }
+
+  automaticIdentityKey(entry) {
+    return entry.ino !== 0
+      ? `inode:${entry.dev}:${entry.ino}`
+      : `path:${pathKey(entry.path)}`
+  }
+
+  logVerificationSkip(app, entry, reason, counts, error = null) {
+    if (error) counts.path_errors += 1
+    else counts.unstable_hashes += 1
+    this.log("verification-skipped", {
+      app,
+      path: entry.path,
+      kind: entry.kind,
+      reason,
+      code: error && error.code,
+      message: error && error.message
+    })
+  }
+
+  async verifiedAutomaticHash(active, entry, memory, verifiedHashes, counts) {
+    this.checkpoint(active)
+    let current
+    try {
+      current = await fs.promises.lstat(entry.path)
+    } catch (error) {
+      if (!isPathError(error)) throw error
+      this.logVerificationSkip(active.app, entry, "unreadable", counts, error)
+      return null
+    }
+    this.checkpoint(active)
+    if (!current.isFile() || current.isSymbolicLink() ||
+        !sameSnapshot(entry, current)) {
+      this.logVerificationSkip(
+        active.app, entry, "snapshot-changed", counts)
+      return null
+    }
+    const key = this.automaticHashKey(entry)
+    const reusable = SHA256_RE.test(entry.hash || "")
+      ? entry.hash
+      : memory.get(key)
+    if (reusable) {
+      counts.hash_reuses += 1
+      memory.set(key, reusable)
+      verifiedHashes.set(pathKey(entry.path), Object.assign({}, entry, {
+        hash: reusable
+      }))
+      return reusable
+    }
+    let verified
+    try {
+      verified = await this.vault.scanner.hashStable(entry, {
+        signal: active.controller && active.controller.signal
+      })
+    } catch (error) {
+      if (error && error.code === "EVAULTCANCELLED") throw error
+      if (!isPathError(error)) throw error
+      counts.hash_failures += 1
+      this.logVerificationSkip(active.app, entry, "hash-failed", counts, error)
+      return null
+    }
+    this.checkpoint(active)
+    if (!verified.stable || !SHA256_RE.test(verified.result.hash || "")) {
+      this.logVerificationSkip(
+        active.app, entry, "changed-during-hash", counts)
+      return null
+    }
+    counts.hashed += 1
+    counts.hash_bytes += Math.max(0, Number(verified.result.size) || 0)
+    memory.set(key, verified.result.hash)
+    verifiedHashes.set(pathKey(entry.path), Object.assign({}, entry, {
+      hash: verified.result.hash
+    }))
+    return verified.result.hash
+  }
+
   async runPrecheck(active) {
     const app = active.app
     const root = path.resolve(this.vault.kernel.homedir, "api", app)
@@ -768,18 +861,24 @@ class AutomaticScans {
       files: 0,
       bytes: 0,
       candidates: 0,
-      path_errors: 0
+      path_errors: 0,
+      hashed: 0,
+      hash_bytes: 0,
+      hash_reuses: 0,
+      hash_failures: 0,
+      unstable_hashes: 0
     }
     const threshold = await this.candidateThreshold()
     const paths = [...new Set(active.paths || [])].filter((filePath) =>
       typeof filePath === "string" && inside(root, filePath))
+    const verifiedHashes = new Map()
     await this.vault.registry.beginAutomaticPrecheck(app)
     this.log("check-started", {
       app,
       root,
       threshold_bytes: threshold,
       changed_paths: paths.length,
-      policy: "metadata-only"
+      policy: "metadata-prefilter-sha256"
     })
     try {
       this.checkpoint(active)
@@ -817,8 +916,14 @@ class AutomaticScans {
         candidates.push({
           path: paths[index],
           size: stat.size,
+          mtime: stat.mtimeMs,
+          ctime: stat.ctimeMs,
           dev: stat.dev,
-          ino: stat.ino
+          ino: stat.ino,
+          nlink: stat.nlink,
+          mode: stat.mode,
+          uid: stat.uid,
+          gid: stat.gid
         })
       }
       counts.candidates = candidates.length
@@ -831,15 +936,123 @@ class AutomaticScans {
         error.code = "ENOENT"
         throw error
       }
-      const result = await this.vault.registry.automaticPrecheckResult(app)
+      const verifiedMatches = []
+      let group = null
+      const preparePeers = () => {
+        if (!group || group.unmatched) return
+        group.unmatched = new Map()
+        group.unmatchedIdentities = new Map()
+        for (const [hash, value] of group.candidatesByHash) {
+          if (value.identities.size > 1) {
+            for (const candidate of value.candidates) {
+              group.matched.add(pathKey(candidate.path))
+            }
+          } else {
+            const identity = value.identities.values().next().value
+            group.unmatched.set(hash, {
+              identity,
+              candidates: value.candidates
+            })
+            group.unmatchedIdentities.set(
+              identity,
+              (group.unmatchedIdentities.get(identity) || 0) + 1)
+          }
+        }
+      }
+      const finishGroup = () => {
+        if (!group) return
+        preparePeers()
+        for (const [hash, value] of group.candidatesByHash) {
+          for (const candidate of value.candidates) {
+            if (!group.matched.has(pathKey(candidate.path))) continue
+            verifiedMatches.push({ path: candidate.path, hash })
+          }
+        }
+        group = null
+      }
+      let cursor = null
+      do {
+        const page = await this.vault.registry.automaticPrecheckEntries(
+          app, cursor, ENTRY_BATCH_SIZE)
+        this.checkpoint(active)
+        for (const entry of page.entries) {
+          const key = `${entry.dev}\0${entry.size}`
+          if (!group || group.key !== key) {
+            finishGroup()
+            group = {
+              key,
+              candidatesByHash: new Map(),
+              matched: new Set(),
+              unmatched: null,
+              unmatchedIdentities: null,
+              memory: new Map()
+            }
+          }
+          if (entry.kind === "changed") {
+            const candidate = entry
+            this.checkpoint(active)
+            const hash = await this.verifiedAutomaticHash(
+              active, candidate, group.memory, verifiedHashes, counts)
+            if (!hash) continue
+            let value = group.candidatesByHash.get(hash)
+            if (!value) {
+              value = { candidates: [], identities: new Set() }
+              group.candidatesByHash.set(hash, value)
+            }
+            value.candidates.push(candidate)
+            value.identities.add(this.automaticIdentityKey(candidate))
+            continue
+          }
+          preparePeers()
+          if (!group.unmatched.size) continue
+          this.checkpoint(active)
+          const identity = this.automaticIdentityKey(entry)
+          if (group.unmatchedIdentities.size === 1 &&
+              group.unmatchedIdentities.has(identity)) continue
+          const hash = await this.verifiedAutomaticHash(
+            active, entry, group.memory, verifiedHashes, counts)
+          const wanted = hash && group.unmatched.get(hash)
+          if (!wanted || wanted.identity === identity) {
+            continue
+          }
+          for (const candidate of wanted.candidates) {
+            group.matched.add(pathKey(candidate.path))
+          }
+          group.unmatched.delete(hash)
+          const remaining = group.unmatchedIdentities.get(wanted.identity) - 1
+          if (remaining) {
+            group.unmatchedIdentities.set(wanted.identity, remaining)
+          } else {
+            group.unmatchedIdentities.delete(wanted.identity)
+          }
+        }
+        if (verifiedHashes.size >= ENTRY_BATCH_SIZE) {
+          await this.vault.registry.rememberHashCache(
+            [...verifiedHashes.values()])
+          verifiedHashes.clear()
+          this.checkpoint(active)
+        }
+        cursor = page.next_cursor
+      } while (cursor)
+      finishGroup()
+      this.checkpoint(active)
+      const result = await this.vault.registry.automaticPrecheckResult(
+        app, verifiedMatches)
       this.checkpoint(active)
       return Object.assign({}, counts, {
         signature: result.signature,
-        possible_files: Math.max(0, Number(result.files) || 0),
+        verified_files: Math.max(0, Number(result.files) || 0),
         duration_ms: Date.now() - startedAt
       })
     } finally {
-      await this.vault.registry.abortAutomaticPrecheck(app)
+      try {
+        if (verifiedHashes.size) {
+          await this.vault.registry.rememberHashCache(
+            [...verifiedHashes.values()])
+        }
+      } finally {
+        await this.vault.registry.abortAutomaticPrecheck(app)
+      }
     }
   }
 
@@ -894,6 +1107,7 @@ class AutomaticScans {
       paths: [...new Set(entry.paths || [])],
       cancelled: false,
       reason: null,
+      controller: new AbortController(),
       promise: null
     }
     this.active = active
@@ -933,7 +1147,12 @@ class AutomaticScans {
           files: result && result.files,
           bytes: result && result.bytes,
           candidates: result && result.candidates,
-          possible_files: result && result.possible_files,
+          verified_files: result && result.verified_files,
+          hashed: result && result.hashed,
+          hash_bytes: result && result.hash_bytes,
+          hash_reuses: result && result.hash_reuses,
+          hash_failures: result && result.hash_failures,
+          unstable_hashes: result && result.unstable_hashes,
           path_errors: result && result.path_errors,
           error: error && error.message
         })
@@ -1037,8 +1256,8 @@ class AutomaticScans {
   }
 
   async publishResultNow(app, result = {}) {
-    const possibleFiles = Math.max(0, Number(result.possible_files) || 0)
-    if (possibleFiles > 0 && result.signature) {
+    const verifiedFiles = Math.max(0, Number(result.verified_files) || 0)
+    if (verifiedFiles > 0 && result.signature) {
       const acknowledged = (this.settings.get(app) || {})
         .acknowledged_signature
       const options = { signature: result.signature }
@@ -1059,9 +1278,9 @@ class AutomaticScans {
         previous: null,
         hidden: acknowledged === result.signature
       })
-      this.log("possible-matches", {
+      this.log("verified-matches", {
         app,
-        possible_files: possibleFiles,
+        verified_files: verifiedFiles,
         acknowledged: acknowledged === result.signature
       })
       return "result"
@@ -1076,7 +1295,7 @@ class AutomaticScans {
       await this.vault.registry.setAutomaticAppScanState(app, null)
     }
     this.entries.delete(app)
-    this.log("no-possible-matches", { app })
+    this.log("no-verified-matches", { app })
     return "empty"
   }
 
@@ -1246,17 +1465,17 @@ class AutomaticScans {
 
   async acknowledge(app, signature) {
     if (this.disposed) {
-      return { error: "That possible-match result is no longer available." }
+      return { error: "That automatic duplicate result is no longer available." }
     }
     return this.withAppTransition(app, async () => {
       await this.hydrate()
       if (this.disposed) {
-        return { error: "That possible-match result is no longer available." }
+        return { error: "That automatic duplicate result is no longer available." }
       }
       const current = this.entries.get(app)
       const entry = this.publicResult(current)
       if (!entry) {
-        return { error: "That possible-match result is no longer available." }
+        return { error: "That automatic duplicate result is no longer available." }
       }
       if (typeof signature !== "string" || signature !== entry.signature) {
         return { stale: true, app }

package/kernel/vault/hash_worker.js

@@ -7,14 +7,27 @@ const fs = require('fs')
 // same sha256 digest, one file at a time.
 const HASH_READ_SIZE = 1024 * 1024
 const HASH_PROGRESS_INTERVAL_MS = 1000
+const jobs = new Map()
 
-parentPort.on('message', ({ id, filePath }) => {
+parentPort.on('message', ({ id, filePath, cancel }) => {
+  if (cancel) {
+    const job = jobs.get(id)
+    if (!job) return
+    const error = new Error('Hashing cancelled.')
+    error.code = 'EVAULTCANCELLED'
+    job.stream.destroy(error)
+    return
+  }
   const hash = crypto.createHash('sha256')
   let size = 0
   let lastProgressAt = 0
   let complete = false
   const flags = fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW || 0)
-  const stream = fs.createReadStream(filePath, { flags, highWaterMark: HASH_READ_SIZE })
+  const stream = fs.createReadStream(filePath, {
+    flags,
+    highWaterMark: HASH_READ_SIZE
+  })
+  jobs.set(id, { stream })
   stream.on('data', (chunk) => {
     size += chunk.length
     hash.update(chunk)
@@ -27,11 +40,13 @@ parentPort.on('message', ({ id, filePath }) => {
   stream.on('error', (error) => {
     if (complete) return
     complete = true
+    jobs.delete(id)
     parentPort.postMessage({ id, error: error.message, code: error.code })
   })
   stream.on('end', () => {
     if (complete) return
     complete = true
+    jobs.delete(id)
     parentPort.postMessage({ id, hash: hash.digest('hex'), size })
   })
 })

package/kernel/vault/index.js

@@ -9,6 +9,7 @@ const Sweeper = require("./sweeper")
 const FolderFinder = require("./folder_finder")
 const AutomaticScans = require("./automatic_scans")
 const { fileSnapshot, sameSnapshot, sameContentState } = require("./snapshot")
+const { cancelledError } = require("./operation_errors")
 const {
   SIZE_THRESHOLD,
   CANDIDATE_SIZE_OPTIONS,
@@ -230,6 +231,7 @@ class Vault {
     this._anchorStoresByDevice = new Map()
     this.operationTail = Promise.resolve()
     this.registryInitializationPromise = null
+    this.registryRestartPromise = null
     this.initializationPromise = null
     this.scanPromise = null
     this.scanCompletionPromise = null
@@ -246,6 +248,42 @@ class Vault {
     this.automaticScans = new AutomaticScans(this)
   }
 
+  async dispose() {
+    const failures = []
+    const settle = async (promise) => {
+      if (!promise) return
+      try {
+        await promise
+      } catch (error) {
+        failures.push(error)
+      }
+    }
+
+    await settle(this.automaticScans.dispose())
+    if (this.scanPromise) this.cancelScan()
+    if (this.folderDiscoveryPromise) this.cancelFolderDiscovery()
+    if (this.fileActionProgress) this.cancelFileAction()
+    await settle(this.operationTail)
+    await settle(this.scanCompletionPromise)
+    await settle(this.folderDiscoveryCommitPromise)
+    await settle(this.registryRestartPromise)
+
+    if (this.worker) {
+      const worker = this.worker
+      this.failHashWorker(worker, cancelledError("Vault disposed."))
+      await settle(worker.terminate())
+    }
+    if (this.registry) {
+      const registry = this.registry
+      this.registry = null
+      await settle(registry.close())
+    }
+    this.initialized = false
+    this.sweeper = null
+    this.folderFinder = null
+    if (failures.length) throw failures[0]
+  }
+
   get root() {
     return path.resolve(this.kernel.homedir, "vault")
   }
@@ -614,6 +652,9 @@ class Vault {
   }
 
   ensureRegistryInitialized() {
+    if (this.registryRestartPromise) {
+      return this.registryRestartPromise.then(() => ({ enabled: true }))
+    }
     if (this.registry) return Promise.resolve({ enabled: true })
     if (!this.registryInitializationPromise) {
       this.registryInitializationPromise = this.initializeRegistryStorage()
@@ -640,6 +681,7 @@ class Vault {
 
   async ensureInitialized() {
     if (!this.enabled) return { enabled: false }
+    await this.ensureRegistryInitialized()
     if (this.initialized) return { enabled: true, mode: this.mode }
     if (!this.initializationPromise) {
       this.initializationPromise = this.initializeStorage().finally(() => {
@@ -651,6 +693,7 @@ class Vault {
 
   async openWorkspace() {
     if (!this.initialized) return this.ensureInitialized()
+    await this.ensureRegistryInitialized()
     await this.refreshAnchorStores()
     await this.refreshSources()
     return { enabled: true, mode: this.mode }
@@ -936,7 +979,7 @@ class Vault {
     }
     for (const [id, job] of [...this.workerJobs]) {
       if (job.worker !== worker) continue
-      clearTimeout(job.inactivityTimer)
+      job.cleanup()
       this.workerJobs.delete(id)
       job.reject(error)
     }
@@ -944,6 +987,10 @@ class Vault {
   }
 
   async hashFile(filePath, options = {}) {
+    const signal = options.signal
+    if (signal && signal.aborted) {
+      throw cancelledError("Hashing cancelled.")
+    }
     if (this.workerIdleTimer) {
       clearTimeout(this.workerIdleTimer)
       this.workerIdleTimer = null
@@ -962,7 +1009,7 @@ class Vault {
           job.reportProgress(bytesRead)
           return
         }
-        clearTimeout(job.inactivityTimer)
+        job.cleanup()
         this.workerJobs.delete(id)
         if (error) {
           const failure = new Error(error)
@@ -1016,7 +1063,22 @@ class Vault {
           } catch (error) {}
         },
         inactivityTimer: null,
-        resetInactivity: null
+        resetInactivity: null,
+        cancel: null,
+        cleanup: null
+      }
+      job.cancel = () => {
+        if (!this.workerJobs.has(id)) return
+        try {
+          worker.postMessage({ id, cancel: true })
+        } catch (error) {
+          const failure = cancelledError("Hashing cancelled.")
+          this.failHashWorker(worker, failure, true)
+        }
+      }
+      job.cleanup = () => {
+        clearTimeout(job.inactivityTimer)
+        if (signal) signal.removeEventListener("abort", job.cancel)
       }
       job.resetInactivity = () => {
         clearTimeout(job.inactivityTimer)
@@ -1031,9 +1093,11 @@ class Vault {
         if (job.inactivityTimer.unref) job.inactivityTimer.unref()
       }
       this.workerJobs.set(id, job)
+      if (signal) signal.addEventListener("abort", job.cancel, { once: true })
       job.resetInactivity()
       try {
         worker.postMessage({ id, filePath })
+        if (signal && signal.aborted) job.cancel()
       } catch (error) {
         const failure = new Error(error && error.message
           ? error.message
@@ -1696,6 +1760,9 @@ class Vault {
     this.scanPromise = tracked
     const completion = tracked.then(async () => {
       try {
+        if (this.registryRestartPromise) {
+          await this.registryRestartPromise
+        }
         await this.automaticScans.scanFinished({
           scopeId,
           result: scanResult,
@@ -1719,17 +1786,47 @@ class Vault {
     if (!this.scanPromise || !this.sweeper) {
       return { cancel_requested: false }
     }
+    // Publication is one atomic commit. Once it starts, let it finish so a
+    // committed result can never be reported as cancelled.
+    if (this.sweeper.state.active &&
+        this.sweeper.state.phase === "publishing") {
+      return { cancel_requested: false }
+    }
     this.scanCancelRequested = true
+    const error = new Error("Scan cancelled.")
+    error.code = "EVAULTCANCELLED"
+    const cancelRequested = this.sweeper.state.active
+      ? this.sweeper.cancel()
+      : true
     if (this.sweeper.state.active && this.sweeper.currentHash &&
         this.worker) {
-      const error = new Error("Scan cancelled.")
-      error.code = "EVAULTCANCELLED"
       this.failHashWorker(this.worker, error, true)
     }
+    if (this.sweeper.state.active && this.registry &&
+        !this.registryRestartPromise) {
+      const registry = this.registry
+      const restart = registry.restart(error).catch((restartError) => {
+        if (this.registry === registry) {
+          this.registry = null
+          this.initialized = false
+          this.sweeper = null
+        }
+        this.scanError = restartError && restartError.message
+          ? restartError.message
+          : String(restartError)
+        throw restartError
+      })
+      const tracked = restart.finally(() => {
+        if (this.registryRestartPromise === tracked) {
+          this.registryRestartPromise = null
+        }
+      })
+      this.registryRestartPromise = tracked
+      // The action returns immediately; subsequent work awaits this promise.
+      tracked.catch(() => {})
+    }
     return {
-      cancel_requested: this.sweeper.state.active
-        ? this.sweeper.cancel()
-        : true
+      cancel_requested: cancelRequested
     }
   }