pinokiod 8.0.69 → 8.0.72

package/kernel/index.js

@@ -1062,15 +1062,15 @@ class Kernel {
   kill() {
     process.kill(process.pid, "SIGTERM")
   }
-  async disposeVaultAutomaticScans() {
+  async disposeVault() {
     const vault = this.vault
-    if (!vault || !vault.automaticScans) return false
+    if (!vault) return false
     if (vault.ready) await Promise.resolve(vault.ready).catch(() => {})
     try {
-      await vault.automaticScans.dispose()
+      await vault.dispose()
       return true
     } catch (error) {
-      console.warn("Vault automatic cleanup error:",
+      console.warn("Vault cleanup error:",
         error && error.message ? error.message : error)
       return false
     }
@@ -1092,7 +1092,7 @@ class Kernel {
     })
 
     let home = this.store.get("home") || process.env.PINOKIO_HOME
-    await this.disposeVaultAutomaticScans()
+    await this.disposeVault()
     this.vault = null
     this.homedir = home
 

package/kernel/shell.js

@@ -290,6 +290,7 @@ class Shell {
       }
     }
 
+    setDefaultEnvValue(this.env, "UV_HTTP_TIMEOUT", "60")
     setDefaultEnvValue(this.env, "HF_HUB_DISABLE_UPDATE_CHECK", "1")
     setDefaultEnvValue(this.env, "HF_TOKEN_PATH", this.kernel.envs?.HF_TOKEN_PATH || path.resolve(this.kernel.homedir, "cache", "HF_AUTH", "token"))
 

package/kernel/vault/automatic_scans.js

@@ -2,9 +2,12 @@ const fs = require("fs")
 const path = require("path")
 const {
   SIZE_THRESHOLD,
-  isCandidateFileSize
+  isCandidateFileSize,
+  SHA256_RE,
+  ENTRY_BATCH_SIZE
 } = require("./constants")
 const { statMany } = require("./walker")
+const { fileSnapshot, sameSnapshot } = require("./snapshot")
 const {
   cancelledError,
   isPathError
@@ -24,6 +27,11 @@ const inside = (root, candidate) => {
   )
 }
 
+const pathKey = (value) => {
+  const resolved = path.resolve(value)
+  return process.platform === "win32" ? resolved.toLowerCase() : resolved
+}
+
 class AutomaticScans {
   constructor(vault) {
     this.vault = vault
@@ -44,6 +52,7 @@ class AutomaticScans {
     this.manualDepth = 0
     this.observedApps = new Set()
     this.changedPaths = new Map()
+    this.changedPathIndex = new Map()
     this.pendingStops = new Map()
     this.lifecycleWork = new Set()
     this.stopSettleMs = STOP_SETTLE_MS
@@ -128,6 +137,7 @@ class AutomaticScans {
     }
     this.observedApps.clear()
     this.changedPaths.clear()
+    this.changedPathIndex.clear()
     if (watcherError) throw watcherError
   }
 
@@ -144,7 +154,8 @@ class AutomaticScans {
       if (!event || !["create", "update", "delete"].includes(event.type)) {
         continue
       }
-      this.recordChangedPath(event.path)
+      if (event.type === "delete") this.removeChangedPath(event.path)
+      else this.recordChangedPath(event.path)
     }
   }
 
@@ -163,19 +174,95 @@ class AutomaticScans {
       paths = new Set()
       this.changedPaths.set(app, paths)
     }
+    let index = this.changedPathIndex.get(app)
+    if (!index) {
+      index = { exact: new Map(), descendants: new Map() }
+      this.changedPathIndex.set(app, index)
+    }
+    const key = pathKey(resolved)
+    if (index.exact.has(key)) return true
     paths.add(resolved)
+    index.exact.set(key, resolved)
+    this.addChangedPathToIndex(index.descendants, appRoot, resolved)
+    return true
+  }
+
+  changedPathParentKeys(appRoot, filePath) {
+    const rootKey = pathKey(appRoot)
+    const keys = []
+    let current = path.dirname(path.resolve(filePath))
+    while (true) {
+      const key = pathKey(current)
+      keys.push(key)
+      if (key === rootKey) return keys
+      current = path.dirname(current)
+    }
+  }
+
+  addChangedPathToIndex(index, appRoot, filePath) {
+    for (const key of this.changedPathParentKeys(appRoot, filePath)) {
+      let descendants = index.get(key)
+      if (!descendants) {
+        descendants = new Set()
+        index.set(key, descendants)
+      }
+      descendants.add(filePath)
+    }
+  }
+
+  removeChangedPathFromIndex(index, appRoot, filePath) {
+    index.exact.delete(pathKey(filePath))
+    for (const key of this.changedPathParentKeys(appRoot, filePath)) {
+      const descendants = index.descendants.get(key)
+      if (!descendants) continue
+      descendants.delete(filePath)
+      if (!descendants.size) index.descendants.delete(key)
+    }
+  }
+
+  removeChangedPath(filePath) {
+    if (this.disposed || !this.supported ||
+        typeof filePath !== "string" || !filePath) {
+      return false
+    }
+    const app = this.appForLaunchPath(filePath)
+    if (!app || !this.observedApps.has(app)) return false
+    const paths = this.changedPaths.get(app)
+    if (!paths) return false
+    const resolved = path.resolve(filePath)
+    const appRoot = path.resolve(this.vault.kernel.homedir, "api", app)
+    const index = this.changedPathIndex.get(app)
+    if (!index) return false
+    const key = pathKey(resolved)
+    const exact = index.exact.get(key)
+    const descendants = index.descendants.get(key)
+    if (!exact && !descendants) return false
+    const matches = new Set(descendants || [])
+    if (exact) matches.add(exact)
+    for (const candidate of matches) {
+      paths.delete(candidate)
+      this.removeChangedPathFromIndex(index, appRoot, candidate)
+    }
+    if (!paths.size) this.discardChangedPaths(app)
     return true
   }
 
   discardChangedPaths(app) {
     this.changedPaths.delete(app)
+    this.changedPathIndex.delete(app)
   }
 
   consumeChangedPaths(app, paths) {
     const collected = this.changedPaths.get(app)
     if (!collected) return
-    for (const filePath of paths || []) collected.delete(filePath)
-    if (!collected.size) this.changedPaths.delete(app)
+    const appRoot = path.resolve(this.vault.kernel.homedir, "api", app)
+    const index = this.changedPathIndex.get(app)
+    for (const filePath of paths || []) {
+      const resolved = path.resolve(filePath)
+      if (!collected.delete(resolved) || !index) continue
+      this.removeChangedPathFromIndex(index, appRoot, resolved)
+    }
+    if (!collected.size) this.discardChangedPaths(app)
   }
 
   async candidateThreshold() {
@@ -745,6 +832,7 @@ class AutomaticScans {
     }
     this.active.cancelled = true
     this.active.reason = reason
+    if (this.active.controller) this.active.controller.abort()
     this.log("check-cancel-requested", { app, reason })
     return true
   }
@@ -755,6 +843,111 @@ class AutomaticScans {
     }
   }
 
+  automaticHashKey(entry) {
+    return [
+      this.automaticIdentityKey(entry),
+      entry.size,
+      entry.mtime,
+      entry.ctime
+    ].join("\0")
+  }
+
+  automaticIdentityKey(entry) {
+    return entry.ino !== 0
+      ? `inode:${entry.dev}:${entry.ino}`
+      : `path:${pathKey(entry.path)}`
+  }
+
+  logVerificationSkip(app, entry, reason, counts, error = null) {
+    if (error) counts.path_errors += 1
+    else counts.unstable_hashes += 1
+    this.log("verification-skipped", {
+      app,
+      path: entry.path,
+      kind: entry.kind,
+      reason,
+      code: error && error.code,
+      message: error && error.message
+    })
+  }
+
+  async verifiedAutomaticHash(active, entry, memory, verifiedHashes, counts) {
+    this.checkpoint(active)
+    let current
+    try {
+      current = await fs.promises.lstat(entry.path)
+    } catch (error) {
+      if (!isPathError(error)) throw error
+      if (entry.kind === "changed" && isMissing(error)) {
+        counts.path_errors += 1
+        return null
+      }
+      this.logVerificationSkip(active.app, entry, "unreadable", counts, error)
+      return null
+    }
+    this.checkpoint(active)
+    if (!current.isFile() || current.isSymbolicLink() ||
+        current.dev !== entry.dev || current.size !== entry.size) {
+      this.logVerificationSkip(
+        active.app, entry, "snapshot-changed", counts)
+      return null
+    }
+    const currentEntry = Object.assign({}, entry, fileSnapshot(current))
+    const key = this.automaticHashKey(currentEntry)
+    const cachedSnapshot = {
+      size: entry.cached_size,
+      mtime: entry.cached_mtime,
+      ctime: entry.cached_ctime,
+      dev: entry.cached_dev,
+      ino: entry.cached_ino
+    }
+    const reusable = memory.get(key) ||
+      (sameSnapshot(entry, current) && SHA256_RE.test(entry.hash || "")
+        ? entry.hash
+        : null) ||
+      (sameSnapshot(cachedSnapshot, current) &&
+        SHA256_RE.test(entry.cached_hash || "")
+        ? entry.cached_hash
+        : null)
+    if (reusable) {
+      counts.hash_reuses += 1
+      memory.set(key, reusable)
+      verifiedHashes.set(pathKey(entry.path), Object.assign({}, currentEntry, {
+        hash: reusable
+      }))
+      return reusable
+    }
+    let verified
+    try {
+      verified = await this.vault.scanner.hashStable(currentEntry, {
+        signal: active.controller && active.controller.signal
+      })
+    } catch (error) {
+      if (error && error.code === "EVAULTCANCELLED") throw error
+      if (!isPathError(error)) throw error
+      counts.hash_failures += 1
+      if (entry.kind === "changed" && isMissing(error)) {
+        counts.path_errors += 1
+        return null
+      }
+      this.logVerificationSkip(active.app, entry, "hash-failed", counts, error)
+      return null
+    }
+    this.checkpoint(active)
+    if (!verified.stable || !SHA256_RE.test(verified.result.hash || "")) {
+      this.logVerificationSkip(
+        active.app, entry, "changed-during-hash", counts)
+      return null
+    }
+    counts.hashed += 1
+    counts.hash_bytes += Math.max(0, Number(verified.result.size) || 0)
+    memory.set(key, verified.result.hash)
+    verifiedHashes.set(pathKey(entry.path), Object.assign({}, currentEntry, {
+      hash: verified.result.hash
+    }))
+    return verified.result.hash
+  }
+
   async runPrecheck(active) {
     const app = active.app
     const root = path.resolve(this.vault.kernel.homedir, "api", app)
@@ -768,18 +961,24 @@ class AutomaticScans {
       files: 0,
       bytes: 0,
       candidates: 0,
-      path_errors: 0
+      path_errors: 0,
+      hashed: 0,
+      hash_bytes: 0,
+      hash_reuses: 0,
+      hash_failures: 0,
+      unstable_hashes: 0
     }
     const threshold = await this.candidateThreshold()
     const paths = [...new Set(active.paths || [])].filter((filePath) =>
       typeof filePath === "string" && inside(root, filePath))
+    const verifiedHashes = new Map()
     await this.vault.registry.beginAutomaticPrecheck(app)
     this.log("check-started", {
       app,
       root,
       threshold_bytes: threshold,
       changed_paths: paths.length,
-      policy: "metadata-only"
+      policy: "metadata-prefilter-sha256"
     })
     try {
       this.checkpoint(active)
@@ -789,6 +988,7 @@ class AutomaticScans {
         onError: (error, filePath) => {
           if (!isPathError(error)) return false
           counts.path_errors += 1
+          if (isMissing(error)) return true
           this.log("path-skipped", {
             app,
             path: filePath,
@@ -817,8 +1017,14 @@ class AutomaticScans {
         candidates.push({
           path: paths[index],
           size: stat.size,
+          mtime: stat.mtimeMs,
+          ctime: stat.ctimeMs,
           dev: stat.dev,
-          ino: stat.ino
+          ino: stat.ino,
+          nlink: stat.nlink,
+          mode: stat.mode,
+          uid: stat.uid,
+          gid: stat.gid
         })
       }
       counts.candidates = candidates.length
@@ -831,15 +1037,123 @@ class AutomaticScans {
         error.code = "ENOENT"
         throw error
       }
-      const result = await this.vault.registry.automaticPrecheckResult(app)
+      const verifiedMatches = []
+      let group = null
+      const preparePeers = () => {
+        if (!group || group.unmatched) return
+        group.unmatched = new Map()
+        group.unmatchedIdentities = new Map()
+        for (const [hash, value] of group.candidatesByHash) {
+          if (value.identities.size > 1) {
+            for (const candidate of value.candidates) {
+              group.matched.add(pathKey(candidate.path))
+            }
+          } else {
+            const identity = value.identities.values().next().value
+            group.unmatched.set(hash, {
+              identity,
+              candidates: value.candidates
+            })
+            group.unmatchedIdentities.set(
+              identity,
+              (group.unmatchedIdentities.get(identity) || 0) + 1)
+          }
+        }
+      }
+      const finishGroup = () => {
+        if (!group) return
+        preparePeers()
+        for (const [hash, value] of group.candidatesByHash) {
+          for (const candidate of value.candidates) {
+            if (!group.matched.has(pathKey(candidate.path))) continue
+            verifiedMatches.push({ path: candidate.path, hash })
+          }
+        }
+        group = null
+      }
+      let cursor = null
+      do {
+        const page = await this.vault.registry.automaticPrecheckEntries(
+          app, cursor, ENTRY_BATCH_SIZE)
+        this.checkpoint(active)
+        for (const entry of page.entries) {
+          const key = `${entry.dev}\0${entry.size}`
+          if (!group || group.key !== key) {
+            finishGroup()
+            group = {
+              key,
+              candidatesByHash: new Map(),
+              matched: new Set(),
+              unmatched: null,
+              unmatchedIdentities: null,
+              memory: new Map()
+            }
+          }
+          if (entry.kind === "changed") {
+            const candidate = entry
+            this.checkpoint(active)
+            const hash = await this.verifiedAutomaticHash(
+              active, candidate, group.memory, verifiedHashes, counts)
+            if (!hash) continue
+            let value = group.candidatesByHash.get(hash)
+            if (!value) {
+              value = { candidates: [], identities: new Set() }
+              group.candidatesByHash.set(hash, value)
+            }
+            value.candidates.push(candidate)
+            value.identities.add(this.automaticIdentityKey(candidate))
+            continue
+          }
+          preparePeers()
+          if (!group.unmatched.size) continue
+          this.checkpoint(active)
+          const identity = this.automaticIdentityKey(entry)
+          if (group.unmatchedIdentities.size === 1 &&
+              group.unmatchedIdentities.has(identity)) continue
+          const hash = await this.verifiedAutomaticHash(
+            active, entry, group.memory, verifiedHashes, counts)
+          const wanted = hash && group.unmatched.get(hash)
+          if (!wanted || wanted.identity === identity) {
+            continue
+          }
+          for (const candidate of wanted.candidates) {
+            group.matched.add(pathKey(candidate.path))
+          }
+          group.unmatched.delete(hash)
+          const remaining = group.unmatchedIdentities.get(wanted.identity) - 1
+          if (remaining) {
+            group.unmatchedIdentities.set(wanted.identity, remaining)
+          } else {
+            group.unmatchedIdentities.delete(wanted.identity)
+          }
+        }
+        if (verifiedHashes.size >= ENTRY_BATCH_SIZE) {
+          await this.vault.registry.rememberHashCache(
+            [...verifiedHashes.values()])
+          verifiedHashes.clear()
+          this.checkpoint(active)
+        }
+        cursor = page.next_cursor
+      } while (cursor)
+      finishGroup()
+      this.checkpoint(active)
+      const result = await this.vault.registry.automaticPrecheckResult(
+        app, verifiedMatches)
       this.checkpoint(active)
       return Object.assign({}, counts, {
         signature: result.signature,
-        possible_files: Math.max(0, Number(result.files) || 0),
+        verified_files: Math.max(0, Number(result.files) || 0),
         duration_ms: Date.now() - startedAt
       })
     } finally {
-      await this.vault.registry.abortAutomaticPrecheck(app)
+      try {
+        if (verifiedHashes.size) {
+          await this.vault.registry.rememberHashCache(
+            [...verifiedHashes.values()])
+        }
+      } finally {
+        await this.vault.registry.abortAutomaticPrecheck(app)
+      }
     }
   }
 
@@ -894,6 +1208,7 @@ class AutomaticScans {
       paths: [...new Set(entry.paths || [])],
       cancelled: false,
       reason: null,
+      controller: new AbortController(),
       promise: null
     }
     this.active = active
@@ -933,7 +1248,12 @@ class AutomaticScans {
           files: result && result.files,
           bytes: result && result.bytes,
           candidates: result && result.candidates,
-          possible_files: result && result.possible_files,
+          verified_files: result && result.verified_files,
+          hashed: result && result.hashed,
+          hash_bytes: result && result.hash_bytes,
+          hash_reuses: result && result.hash_reuses,
+          hash_failures: result && result.hash_failures,
+          unstable_hashes: result && result.unstable_hashes,
           path_errors: result && result.path_errors,
           error: error && error.message
         })
@@ -1037,8 +1357,8 @@ class AutomaticScans {
   }
 
   async publishResultNow(app, result = {}) {
-    const possibleFiles = Math.max(0, Number(result.possible_files) || 0)
-    if (possibleFiles > 0 && result.signature) {
+    const verifiedFiles = Math.max(0, Number(result.verified_files) || 0)
+    if (verifiedFiles > 0 && result.signature) {
       const acknowledged = (this.settings.get(app) || {})
         .acknowledged_signature
       const options = { signature: result.signature }
@@ -1059,9 +1379,9 @@ class AutomaticScans {
         previous: null,
         hidden: acknowledged === result.signature
       })
-      this.log("possible-matches", {
+      this.log("verified-matches", {
         app,
-        possible_files: possibleFiles,
+        verified_files: verifiedFiles,
         acknowledged: acknowledged === result.signature
       })
       return "result"
@@ -1076,7 +1396,7 @@ class AutomaticScans {
       await this.vault.registry.setAutomaticAppScanState(app, null)
     }
     this.entries.delete(app)
-    this.log("no-possible-matches", { app })
+    this.log("no-verified-matches", { app })
     return "empty"
   }
 
@@ -1246,17 +1566,17 @@ class AutomaticScans {
 
   async acknowledge(app, signature) {
     if (this.disposed) {
-      return { error: "That possible-match result is no longer available." }
+      return { error: "That automatic duplicate result is no longer available." }
     }
     return this.withAppTransition(app, async () => {
       await this.hydrate()
       if (this.disposed) {
-        return { error: "That possible-match result is no longer available." }
+        return { error: "That automatic duplicate result is no longer available." }
       }
       const current = this.entries.get(app)
       const entry = this.publicResult(current)
       if (!entry) {
-        return { error: "That possible-match result is no longer available." }
+        return { error: "That automatic duplicate result is no longer available." }
       }
       if (typeof signature !== "string" || signature !== entry.signature) {
         return { stale: true, app }

package/kernel/vault/hash_worker.js

@@ -7,14 +7,27 @@ const fs = require('fs')
 // same sha256 digest, one file at a time.
 const HASH_READ_SIZE = 1024 * 1024
 const HASH_PROGRESS_INTERVAL_MS = 1000
+const jobs = new Map()
 
-parentPort.on('message', ({ id, filePath }) => {
+parentPort.on('message', ({ id, filePath, cancel }) => {
+  if (cancel) {
+    const job = jobs.get(id)
+    if (!job) return
+    const error = new Error('Hashing cancelled.')
+    error.code = 'EVAULTCANCELLED'
+    job.stream.destroy(error)
+    return
+  }
   const hash = crypto.createHash('sha256')
   let size = 0
   let lastProgressAt = 0
   let complete = false
   const flags = fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW || 0)
-  const stream = fs.createReadStream(filePath, { flags, highWaterMark: HASH_READ_SIZE })
+  const stream = fs.createReadStream(filePath, {
+    flags,
+    highWaterMark: HASH_READ_SIZE
+  })
+  jobs.set(id, { stream })
   stream.on('data', (chunk) => {
     size += chunk.length
     hash.update(chunk)
@@ -27,11 +40,13 @@ parentPort.on('message', ({ id, filePath }) => {
   stream.on('error', (error) => {
     if (complete) return
     complete = true
+    jobs.delete(id)
     parentPort.postMessage({ id, error: error.message, code: error.code })
   })
   stream.on('end', () => {
     if (complete) return
     complete = true
+    jobs.delete(id)
     parentPort.postMessage({ id, hash: hash.digest('hex'), size })
   })
 })