pinokiod 8.0.39 → 8.0.41

package/kernel/api/hf/index.js

@@ -214,7 +214,7 @@ class HF {
 
     if (!force) {
       const existing = await connect.keys("huggingface", authContext)
-      if (existing && existing.access_token) {
+      if (existing && existing.access_token && await connect.connected("huggingface", { timeout: 5000 }, authContext)) {
         return {
           status: "success",
           already_logged_in: true,
@@ -273,7 +273,7 @@ class HF {
     const startedAt = Date.now()
     while (Date.now() - startedAt < timeout) {
       const keys = await connect.keys("huggingface", authContext)
-      if (keys && keys.access_token) {
+      if (keys && keys.access_token && await connect.connected("huggingface", { timeout: 5000 }, authContext)) {
         if (useModal) {
           await this.closeLoginModal(req, ondata, kernel)
         }

package/kernel/api/index.js

@@ -31,6 +31,7 @@ class Api {
     this.listeners = {}
     this.counter = 0;
     this.running = {}
+    this.running_paths = {}
     this.done = {}
     this.waiter = {}
     this.proxies = {}
@@ -419,12 +420,19 @@ class Api {
       delete this.resolved_actions[id]
     }
   }
+  clearRunning(requestOrId) {
+    const id = typeof requestOrId === "string" ? requestOrId : this.requestId(requestOrId)
+    if (!id) return
+    delete this.running[id]
+    delete this.running_paths[id]
+  }
   setRunning(request, done) {
     const id = this.requestId(request)
     if (!id) {
       return
     }
     this.running[id] = true
+    this.running_paths[id] = request.path || null
     this.done[id] = done
     if (this.kernel && typeof this.kernel.markAppLaunchStarted === "function") {
       this.kernel.markAppLaunchStarted(request.path)
@@ -634,7 +642,7 @@ class Api {
     if (req.params.id) stoppedKeys.add(req.params.id)
     stoppedKeys.add(requestPath)
     for (const key of stoppedKeys) {
-      delete this.running[key]
+      this.clearRunning(key)
       delete this.kernel.memory.local[key]
     }
     if (this.kernel && typeof this.kernel.markAppLaunchStopped === "function") {
@@ -1994,7 +2002,7 @@ class Api {
               })
             } catch (e) {
               this.clearResolvedAction(request)
-              delete this.running[this.requestId(request)]
+              this.clearRunning(request)
               if (this.kernel && typeof this.kernel.markAppLaunchFailed === "function") {
                 this.kernel.markAppLaunchFailed(request.path, e)
               }
@@ -2008,7 +2016,7 @@ class Api {
 
             if (!Array.isArray(steps) || steps.length === 0) {
               this.clearResolvedAction(request)
-              delete this.running[this.requestId(request)]
+              this.clearRunning(request)
               if (this.kernel && typeof this.kernel.markAppLaunchFailed === "function") {
                 this.kernel.markAppLaunchFailed(request.path, new Error(`missing or invalid attribute: ${actionKey}`))
               }
@@ -2070,6 +2078,7 @@ class Api {
       let id = request.id ? request.id : request.method 
       if (request.id) {
         this.running[request.id] = true
+        delete this.running_paths[request.id]
       }
       try {
         resolved.dirname = resolved.dirname

package/kernel/connect/index.js

@@ -42,9 +42,9 @@ class Connect {
     let res = await this.clients[provider].keys(context)
     return res
   }
-  async connected(provider, options) {
+  async connected(provider, options, context) {
     if (this.clients[provider] && this.clients[provider].connected) {
-      return await this.clients[provider].connected(options)
+      return await this.clients[provider].connected(options, context)
     }
     return false
   }

package/kernel/connect/providers/huggingface/index.js

@@ -32,10 +32,11 @@ class Huggingface {
     try {
       const stat = await fs.promises.lstat(tokenPath)
       if (stat.isDirectory()) {
-        await fs.promises.rm(tokenPath, { recursive: true, force: true })
+        await fs.promises.rmdir(tokenPath)
       }
     } catch (error) {
-      if (!error || error.code !== "ENOENT") {
+      if (error.code !== "ENOENT") {
+        error.tokenPath = tokenPath
         throw error
       }
     }
@@ -45,6 +46,7 @@ class Huggingface {
     let env
     if (context.parentPath) {
       env = await Environment.get2(context.parentPath, this.kernel)
+      if (env.HF_TOKEN_PATH === this.kernel.path("cache", "HF_AUTH", "anonymous")) env.HF_TOKEN_PATH = this.defaultTokenPath()
     } else {
       let systemEnv = {}
       try {
@@ -96,6 +98,7 @@ class Huggingface {
         maxBuffer: 1024 * 1024,
       }, (error, stdout, stderr) => {
         if (error) {
+          error.tokenPath = env.HF_TOKEN_PATH
           error.stdout = stdout
           error.stderr = stderr
           reject(error)
@@ -244,11 +247,17 @@ class Huggingface {
       return null
     }
   }
-  async connected(options = {}) {
+  async connected(options = {}, context = {}) {
     try {
-      await this.runHf(["auth", "whoami", "--format", "quiet"], options)
+      const result = await this.runHf(["auth", "whoami", "--format", "quiet"], options, context)
+      if (this.kernel.envs && result.env.HF_TOKEN_PATH === this.defaultTokenPath()) {
+        this.kernel.envs.HF_TOKEN_PATH = result.env.HF_TOKEN_PATH
+      }
       return true
-    } catch (_) {
+    } catch (error) {
+      if (this.kernel.envs && error.tokenPath === this.defaultTokenPath()) {
+        this.kernel.envs.HF_TOKEN_PATH = this.kernel.path("cache", "HF_AUTH", "anonymous")
+      }
       return false
     }
   }

package/kernel/index.js

@@ -1221,10 +1221,22 @@ class Kernel {
       // discovery walk (spec/requirements/shared-model-store.md, trigger 5).
       this.vault = new Vault(this)
       if (this.homedir) {
-        this.vault.init().then((result) => {
-          if (result && result.enabled) return this.vault.verify()
+        this.vault.ready = this.vault.init({ existingOnly: true }).then(async (result) => {
+          if (result && result.enabled && this.vault.initialized) {
+            this.vault.verificationPending = true
+            await this.vault.runExclusive(async () => {
+              await this.vault.verify()
+              await this.vault.registry.flush()
+            })
+            this.vault.verificationPending = false
+          }
+          return result
         }).catch((err) => {
+          // Keep an enabled Vault retryable after transient registry or
+          // filesystem errors. A genuinely disabled Vault already has
+          // enabled=false from its environment check.
           console.warn("Vault init error:", err && err.message ? err.message : err)
+          return { enabled: false, error: err && err.message ? err.message : String(err) }
         })
       }
 

package/kernel/shell.js

@@ -291,7 +291,7 @@ class Shell {
     }
 
     setDefaultEnvValue(this.env, "HF_HUB_DISABLE_UPDATE_CHECK", "1")
-    setDefaultEnvValue(this.env, "HF_TOKEN_PATH", path.resolve(this.kernel.homedir, "cache", "HF_AUTH", "token"))
+    setDefaultEnvValue(this.env, "HF_TOKEN_PATH", this.kernel.envs?.HF_TOKEN_PATH || path.resolve(this.kernel.homedir, "cache", "HF_AUTH", "token"))
 
     if (this.platform === "win32") {
       const managedSslDir = path.resolve(this.kernel.homedir, "bin", "miniforge", "Library", "ssl")
@@ -1618,7 +1618,8 @@ class Shell {
 
 
     if (this.kernel.api.running[this.id]) {
-      delete this.kernel.api.running[this.id]
+      if (typeof this.kernel.api.clearRunning === "function") this.kernel.api.clearRunning(this.id)
+      else delete this.kernel.api.running[this.id]
     }
     if (this.kernel.memory.local[this.id]) {
       delete this.kernel.memory.local[this.id]

package/kernel/shells.js

@@ -47,6 +47,9 @@ class Shells {
   }
 */
   async init() {
+    const hfDisconnected = !this.kernel.envs?.HF_TOKEN_PATH &&
+      !(await this.kernel.connect.connected("huggingface", { timeout: 5000 }))
+
     // iterate through all the envs
     let sh = new Shell(this.kernel)
     let m
@@ -58,6 +61,9 @@ class Shells {
     await sh.init_env({
       env: this.kernel.bin.envs({}),
     })
+    if (hfDisconnected && sh.env.HF_TOKEN_PATH === this.kernel.path("cache", "HF_AUTH", "token")) {
+      sh.env.HF_TOKEN_PATH = this.kernel.path("cache", "HF_AUTH", "anonymous")
+    }
 
     await this.ensureBracketedPasteSupport(sh.shell)
     if (this.kernel.bracketedPasteSupport) {

package/kernel/vault/constants.js

@@ -0,0 +1,13 @@
+const MB = 1000 * 1000
+const CANDIDATE_SIZE_OPTIONS = [10, 50, 100, 500, 1000].map((value) => value * MB)
+
+module.exports = {
+  SIZE_THRESHOLD: 100 * MB,
+  CANDIDATE_SIZE_OPTIONS,
+  TMP_SUFFIX: ".pinokio-dedup-tmp",
+  SHA256_RE: /^[0-9a-f]{64}$/,
+  ENTRY_BATCH_SIZE: 256,
+  DIR_CONCURRENCY: 8,
+  STAT_CONCURRENCY: 32,
+  HASH_QUEUE_LIMIT: 256
+}

package/kernel/vault/hash_worker.js

@@ -10,13 +10,20 @@ const HASH_READ_SIZE = 1024 * 1024
 parentPort.on('message', ({ id, filePath }) => {
   const hash = crypto.createHash('sha256')
   let size = 0
-  const stream = fs.createReadStream(filePath, { highWaterMark: HASH_READ_SIZE })
+  const flags = fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW || 0)
+  const stream = fs.createReadStream(filePath, { flags, highWaterMark: HASH_READ_SIZE })
   stream.on('data', (chunk) => {
     size += chunk.length
     hash.update(chunk)
   })
   stream.on('error', (error) => {
-    parentPort.postMessage({ id, error: error.message })
+    parentPort.postMessage({
+      id,
+      error: {
+        message: error && error.message ? error.message : String(error),
+        code: error && error.code ? error.code : null
+      }
+    })
   })
   stream.on('end', () => {
     parentPort.postMessage({ id, hash: hash.digest('hex'), size })