pinokiod 8.0.38 → 8.0.40

package/kernel/api/hf/index.js

@@ -214,7 +214,7 @@ class HF {
 
     if (!force) {
       const existing = await connect.keys("huggingface", authContext)
-      if (existing && existing.access_token) {
+      if (existing && existing.access_token && await connect.connected("huggingface", { timeout: 5000 }, authContext)) {
         return {
           status: "success",
           already_logged_in: true,
@@ -273,7 +273,7 @@ class HF {
     const startedAt = Date.now()
     while (Date.now() - startedAt < timeout) {
       const keys = await connect.keys("huggingface", authContext)
-      if (keys && keys.access_token) {
+      if (keys && keys.access_token && await connect.connected("huggingface", { timeout: 5000 }, authContext)) {
         if (useModal) {
           await this.closeLoginModal(req, ondata, kernel)
         }

package/kernel/connect/index.js

@@ -42,9 +42,9 @@ class Connect {
     let res = await this.clients[provider].keys(context)
     return res
   }
-  async connected(provider, options) {
+  async connected(provider, options, context) {
     if (this.clients[provider] && this.clients[provider].connected) {
-      return await this.clients[provider].connected(options)
+      return await this.clients[provider].connected(options, context)
     }
     return false
   }

package/kernel/connect/providers/huggingface/index.js

@@ -32,10 +32,11 @@ class Huggingface {
     try {
       const stat = await fs.promises.lstat(tokenPath)
       if (stat.isDirectory()) {
-        await fs.promises.rm(tokenPath, { recursive: true, force: true })
+        await fs.promises.rmdir(tokenPath)
       }
     } catch (error) {
-      if (!error || error.code !== "ENOENT") {
+      if (error.code !== "ENOENT") {
+        error.tokenPath = tokenPath
         throw error
       }
     }
@@ -45,6 +46,7 @@ class Huggingface {
     let env
     if (context.parentPath) {
       env = await Environment.get2(context.parentPath, this.kernel)
+      if (env.HF_TOKEN_PATH === this.kernel.path("cache", "HF_AUTH", "anonymous")) env.HF_TOKEN_PATH = this.defaultTokenPath()
     } else {
       let systemEnv = {}
       try {
@@ -96,6 +98,7 @@ class Huggingface {
         maxBuffer: 1024 * 1024,
       }, (error, stdout, stderr) => {
         if (error) {
+          error.tokenPath = env.HF_TOKEN_PATH
           error.stdout = stdout
           error.stderr = stderr
           reject(error)
@@ -244,11 +247,17 @@ class Huggingface {
       return null
     }
   }
-  async connected(options = {}) {
+  async connected(options = {}, context = {}) {
     try {
-      await this.runHf(["auth", "whoami", "--format", "quiet"], options)
+      const result = await this.runHf(["auth", "whoami", "--format", "quiet"], options, context)
+      if (this.kernel.envs && result.env.HF_TOKEN_PATH === this.defaultTokenPath()) {
+        this.kernel.envs.HF_TOKEN_PATH = result.env.HF_TOKEN_PATH
+      }
       return true
-    } catch (_) {
+    } catch (error) {
+      if (this.kernel.envs && error.tokenPath === this.defaultTokenPath()) {
+        this.kernel.envs.HF_TOKEN_PATH = this.kernel.path("cache", "HF_AUTH", "anonymous")
+      }
       return false
     }
   }

package/kernel/index.js

@@ -37,7 +37,6 @@ const PinokioDomainRouter = require("./router/pinokio_domain_router")
 const Procs = require('./procs')
 const Peer = require('./peer')
 const Git = require('./git')
-const Vault = require('./vault')
 const Connect = require('./connect')
 const Favicon = require('./favicon')
 const AppLauncher = require('./app_launcher')
@@ -1217,17 +1216,6 @@ class Kernel {
       await this.git.loadCheckpoints()
       console.timeEnd("git.loadCheckpoints")
 
-      // Shared model store: registry verification only at startup — never a
-      // discovery walk (spec/requirements/shared-model-store.md, trigger 5).
-      this.vault = new Vault(this)
-      if (this.homedir) {
-        this.vault.init().then((result) => {
-          if (result && result.enabled) return this.vault.verify()
-        }).catch((err) => {
-          console.warn("Vault init error:", err && err.message ? err.message : err)
-        })
-      }
-
 //      let contents = await fs.promises.readdir(this.homedir)
       //await this.bin.init()
       let ts = Date.now()

package/kernel/shell.js

@@ -291,7 +291,7 @@ class Shell {
     }
 
     setDefaultEnvValue(this.env, "HF_HUB_DISABLE_UPDATE_CHECK", "1")
-    setDefaultEnvValue(this.env, "HF_TOKEN_PATH", path.resolve(this.kernel.homedir, "cache", "HF_AUTH", "token"))
+    setDefaultEnvValue(this.env, "HF_TOKEN_PATH", this.kernel.envs?.HF_TOKEN_PATH || path.resolve(this.kernel.homedir, "cache", "HF_AUTH", "token"))
 
     if (this.platform === "win32") {
       const managedSslDir = path.resolve(this.kernel.homedir, "bin", "miniforge", "Library", "ssl")

package/kernel/shells.js

@@ -47,6 +47,9 @@ class Shells {
   }
 */
   async init() {
+    const hfDisconnected = !this.kernel.envs?.HF_TOKEN_PATH &&
+      !(await this.kernel.connect.connected("huggingface", { timeout: 5000 }))
+
     // iterate through all the envs
     let sh = new Shell(this.kernel)
     let m
@@ -58,6 +61,9 @@ class Shells {
     await sh.init_env({
       env: this.kernel.bin.envs({}),
     })
+    if (hfDisconnected && sh.env.HF_TOKEN_PATH === this.kernel.path("cache", "HF_AUTH", "token")) {
+      sh.env.HF_TOKEN_PATH = this.kernel.path("cache", "HF_AUTH", "anonymous")
+    }
 
     await this.ensureBracketedPasteSupport(sh.shell)
     if (this.kernel.bracketedPasteSupport) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pinokiod",
-  "version": "8.0.38",
+  "version": "8.0.40",
   "description": "",
   "main": "index.js",
   "scripts": {

package/server/index.js

@@ -15732,119 +15732,6 @@ class Server {
 //    }))
 
 
-    // Vault dashboard data: everything from memory, never a walk. The scan
-    // itself runs ONLY via the explicit scan action below.
-    this.app.get("/info/dedup", ex(async (req, res) => {
-      const vault = this.kernel.vault
-      if (!vault || !vault.enabled) {
-        res.json({ enabled: false })
-        return
-      }
-      if (req.query && req.query.progress === "1") {
-        res.json(vault.progressStatus())
-        return
-      }
-      res.json(await vault.status())
-    }))
-    this.app.get("/vault", ex(async (req, res) => {
-      res.render("vault", { theme: this.theme, agent: req.agent })
-    }))
-    this.app.post("/vault/action", ex(async (req, res) => {
-      const vault = this.kernel.vault
-      if (!vault || !vault.enabled) {
-        res.json({ error: "vault is not enabled" })
-        return
-      }
-      const body = req.body || {}
-      try {
-        switch (body.action) {
-          case "add_source": {
-            const result = await vault.addExternalSource(body.path)
-            res.json({
-              created: result.created,
-              source: {
-                id: result.source.id,
-                label: result.source.label,
-                target_path: result.source.root,
-                shareable: !!result.source.shareable
-              }
-            })
-            return
-          }
-          case "deduplicate": {
-            const scopeId = typeof body.scope_id === "string" ? body.scope_id : null
-            if (scopeId) {
-              const source = vault.sources().find((item) => item.id === scopeId && item.kind !== "virtual")
-              if (!source) {
-                res.json({ error: "That location is no longer available. Scan again to refresh it." })
-                return
-              }
-              if (!source.shareable) {
-                res.json({ error: "This location is on a disk that cannot share space with this vault." })
-                return
-              }
-              if (source.kind === "app") {
-                const appRoot = this.kernel.path("api", source.app) + path.sep
-                const running = this.kernel.api && this.kernel.api.running ? this.kernel.api.running : {}
-                const busy = Object.keys(running).some((k) => running[k] && k.startsWith(appRoot))
-                if (busy) {
-                  res.json({ error: "This app has a running script. Stop it first, then deduplicate." })
-                  return
-                }
-              }
-              res.json(await vault.sweeper.convertPending(undefined, {
-                scope_id: scopeId, batch_id: `batch-${Date.now()}`
-              }))
-              return
-            }
-            const app = Object.prototype.hasOwnProperty.call(body, "app") ? (body.app || null) : undefined
-            if (app) {
-              const appRoot = this.kernel.path("api", app) + path.sep
-              const running = this.kernel.api && this.kernel.api.running ? this.kernel.api.running : {}
-              const busy = Object.keys(running).some((k) => running[k] && k.startsWith(appRoot))
-              if (busy) {
-                res.json({ error: "This app has a running script. Stop it first, then deduplicate." })
-                return
-              }
-            }
-            res.json(await vault.sweeper.convertPending(app, { batch_id: `batch-${Date.now()}` }))
-            return
-          }
-          case "reclaim":
-            res.json(await vault.reclaim(body.hash))
-            return
-          case "reclaim_all":
-            res.json(await vault.reclaimAll())
-            return
-          case "scan":
-            vault.sweeper.scan().catch(() => {})
-            res.json({ started: true })
-            return
-          case "repair":
-          case "rebuild":
-            if (vault.sweeper && vault.sweeper.state.active) {
-              res.json({ error: "Wait for the current scan to finish before repairing the index." })
-              return
-            }
-            await vault.rebuild()
-            res.json({ done: true })
-            return
-          case "undo":
-            res.json(await vault.undoBatch(body.batch_id))
-            return
-          case "detach":
-            res.json(await vault.detach(body.path))
-            return
-          case "reshare":
-            res.json(await vault.reshare(body.path))
-            return
-          default:
-            res.json({ error: "unknown action" })
-        }
-      } catch (e) {
-        res.json({ error: e && e.message ? e.message : String(e) })
-      }
-    }))
     this.app.get("/info/scripts", ex(async (req, res) => {
     /*
       returns something like this by using the this.kernel.memory.local variable, extracting the api name and adding all running scripts in each associated array, setting the uri as the script path, and the local variables as the local attribute

package/server/views/connect/huggingface.ejs

@@ -168,9 +168,9 @@
   <script>
     const LOGIN_URL = "/connect/huggingface/login"
     const LOGOUT_URL = "/connect/huggingface/logout"
-    const KEYS_URL = "/connect/huggingface/keys"
+    const STATUS_URL = "/connect/status/huggingface"
     const PROFILE_URL = "/connect/huggingface/profile"
-    const POLL_INTERVAL = 2000
+    const POLL_INTERVAL = 5000
     const POLL_TIMEOUT = 120000
 
     const badgeElement = document.getElementById("hf-badge")
@@ -261,9 +261,9 @@
       return res.json()
     }
 
-    async function readToken() {
-      const json = await postJSON(KEYS_URL)
-      return json && json.access_token && !json.error ? json.access_token : null
+    async function isConnected() {
+      const json = await fetch(STATUS_URL, { cache: "no-store" }).then((res) => res.json())
+      return Boolean(json && json.connected)
     }
 
     async function showConnected() {
@@ -326,8 +326,7 @@
     }
 
     async function pollLoginStatus() {
-      const token = await readToken()
-      if (token) {
+      if (await isConnected()) {
         await showConnected()
       }
     }
@@ -415,8 +414,7 @@
 
     window.addEventListener("load", async () => {
       try {
-        const token = await readToken()
-        if (token) {
+        if (await isConnected()) {
           await showConnected()
           return
         }

package/server/views/partials/main_sidebar.ejs

@@ -26,7 +26,6 @@
     <div class='main-sidebar-section' aria-label="Manage">
       <div class="main-sidebar-section-title">Manage</div>
       <a class="tab <%= sidebarSelected === 'checkpoints' ? 'selected' : '' %>" href="/checkpoints"><i class="fa-solid fa-clock-rotate-left"></i><div class='caption'>Checkpoints</div></a>
-      <a class="tab <%= sidebarSelected === 'vault' ? 'selected' : '' %>" href="/vault"><i class="fa-solid fa-hard-drive"></i><div class='caption'>Vault</div></a>
       <a class="tab <%= sidebarSelected === 'tools' ? 'selected' : '' %>" href="/tools"><i class="fa-solid fa-toolbox"></i><div class='caption'>Tools</div></a>
       <a class="tab <%= sidebarSelected === 'plugins' ? 'selected' : '' %>" href="/plugins"><i class="fa-solid fa-plug-circle-bolt"></i><div class='caption'>Plugins</div></a>
       <a class="tab <%= sidebarSelected === 'tasks' ? 'selected' : '' %>" href="/tasks"><i class="fa-solid fa-list-check"></i><div class='caption'>Tasks</div></a>

package/test/hf-api.test.js

@@ -41,7 +41,8 @@ test('hf.login shows a modal, waits for the user to open Hugging Face, then clos
           login,
           token_path: '/tmp/pinokio/hf-token'
         }
-      }
+      },
+      connected: async () => true
     },
     api: {
       async wait(key) {
@@ -161,7 +162,8 @@ test('hf.login returns success without opening anything when already logged in',
           access_token: 'hf_secret',
           token_path: '/tmp/pinokio/hf-token'
         }
-      }
+      },
+      connected: async () => true
     }
   }
 

package/test/huggingface-connect.test.js

@@ -240,7 +240,7 @@ test('Hugging Face connect leaves existing token and refresh files unchanged', a
   }
 })
 
-test('Hugging Face connect removes a non-empty token directory', async () => {
+test('Hugging Face connect preserves a non-empty token directory', async () => {
   const root = await fs.mkdtemp(path.join(os.tmpdir(), 'pinokio-hf-connect-invalid-dir-'))
   const authDir = path.join(root, 'cache', 'HF_AUTH')
   const tokenPath = path.join(authDir, 'token')
@@ -248,16 +248,12 @@ test('Hugging Face connect removes a non-empty token directory', async () => {
   await fs.writeFile(path.join(tokenPath, 'leftover.txt'), 'leftover')
   await fs.writeFile(path.join(authDir, 'stored_tokens'), 'refresh state')
   try {
-    const provider = new HuggingfaceConnect(createKernel(root), {})
-    await Promise.all([provider.authEnv(), provider.authEnv()])
-    provider.hfPath = () => process.execPath
-    const { env } = await provider.runHf([
-      '-e',
-      'require("node:fs").writeFileSync(process.env.HF_TOKEN_PATH, "hf_replacement_token")'
-    ])
-
-    assert.equal(env.HF_TOKEN_PATH, tokenPath)
-    assert.equal(await fs.readFile(tokenPath, 'utf8'), 'hf_replacement_token')
+    const kernel = createKernel(root)
+    kernel.envs = { HF_TOKEN_PATH: tokenPath }
+    const provider = new HuggingfaceConnect(kernel, {})
+    assert.equal(await provider.connected(), false)
+    assert.equal(kernel.envs.HF_TOKEN_PATH, path.join(authDir, 'anonymous'))
+    assert.equal(await fs.readFile(path.join(tokenPath, 'leftover.txt'), 'utf8'), 'leftover')
     assert.equal(await fs.readFile(path.join(authDir, 'stored_tokens'), 'utf8'), 'refresh state')
   } finally {
     await fs.rm(root, { recursive: true, force: true })

package/test/huggingface-token-validity.test.js

@@ -0,0 +1,135 @@
+const assert = require('node:assert/strict')
+const fs = require('node:fs/promises')
+const os = require('node:os')
+const path = require('node:path')
+const test = require('node:test')
+
+const HF = require('../kernel/api/hf')
+const HuggingfaceConnect = require('../kernel/connect/providers/huggingface')
+const Shell = require('../kernel/shell')
+const Shells = require('../kernel/shells')
+
+function createKernel(root) {
+  return {
+    homedir: root,
+    platform: process.platform,
+    path: (...parts) => path.join(root, ...parts),
+    exists: async (target) => fs.access(target).then(() => true, () => false)
+  }
+}
+
+test('hf.login does not accept an invalid saved token', async () => {
+  const context = {
+    parentPath: '/pinokio/api/test/login.js',
+    cwd: '/pinokio/api/test',
+    env: { HF_TOKEN_PATH: './token' }
+  }
+  const kernel = {
+    connect: {
+      keys: async () => ({ access_token: 'invalid' }),
+      async connected(provider, options, receivedContext) {
+        assert.equal(provider, 'huggingface')
+        assert.deepEqual(options, { timeout: 5000 })
+        assert.deepEqual(receivedContext, context)
+        return false
+      },
+      login: async () => ({ status: 'error', error: 'login started' })
+    }
+  }
+
+  const result = await new HF().login({
+    parent: { path: context.parentPath },
+    cwd: context.cwd,
+    params: { env: context.env }
+  }, () => {}, kernel)
+
+  assert.deepEqual(result, { status: 'error', error: 'login started' })
+})
+
+test('whoami updates only the managed shared token path', async () => {
+  const root = path.join(os.tmpdir(), 'pinokio-hf-state')
+  const tokenPath = path.join(root, 'cache', 'HF_AUTH', 'token')
+  const anonymousPath = path.join(root, 'cache', 'HF_AUTH', 'anonymous')
+  const kernel = createKernel(root)
+  kernel.envs = { HF_TOKEN_PATH: anonymousPath }
+  const provider = new HuggingfaceConnect(kernel, {})
+  const customContext = { env: { HF_TOKEN_PATH: '/tmp/custom-hf-token' } }
+
+  provider.runHf = async (_args, _options, receivedContext) => {
+    assert.equal(receivedContext, customContext)
+    return { env: { HF_TOKEN_PATH: customContext.env.HF_TOKEN_PATH } }
+  }
+  assert.equal(await provider.connected({}, customContext), true)
+  assert.equal(kernel.envs.HF_TOKEN_PATH, anonymousPath)
+
+  provider.runHf = async () => ({ env: { HF_TOKEN_PATH: tokenPath } })
+  assert.equal(await provider.connected(), true)
+  assert.equal(kernel.envs.HF_TOKEN_PATH, tokenPath)
+
+  provider.runHf = async () => {
+    const error = new Error('invalid token')
+    error.tokenPath = tokenPath
+    throw error
+  }
+  assert.equal(await provider.connected(), false)
+  assert.equal(kernel.envs.HF_TOKEN_PATH, anonymousPath)
+})
+
+test('provider login ignores only Pinokio\'s inherited anonymous path', async () => {
+  const root = await fs.mkdtemp(path.join(os.tmpdir(), 'pinokio-hf-context-'))
+  const appDir = path.join(root, 'api', 'demo')
+  const scriptPath = path.join(appDir, 'start.js')
+  await fs.mkdir(appDir, { recursive: true })
+  await fs.writeFile(path.join(root, 'ENVIRONMENT'), '')
+  await fs.writeFile(path.join(appDir, 'ENVIRONMENT'), '')
+  await fs.writeFile(scriptPath, '')
+  try {
+    const kernel = createKernel(root)
+    kernel.envs = { HF_TOKEN_PATH: path.join(root, 'cache', 'HF_AUTH', 'anonymous') }
+    const provider = new HuggingfaceConnect(kernel, {})
+    const context = { parentPath: scriptPath, cwd: appDir }
+
+    assert.equal((await provider.authEnv(context)).HF_TOKEN_PATH, provider.defaultTokenPath())
+    assert.equal(
+      (await provider.authEnv({ ...context, env: { HF_TOKEN_PATH: './custom-token' } })).HF_TOKEN_PATH,
+      path.join(appDir, 'custom-token')
+    )
+  } finally {
+    await fs.rm(root, { recursive: true, force: true })
+  }
+})
+
+test('invalid managed tokens are replaced once without overriding apps', async () => {
+  const root = await fs.mkdtemp(path.join(os.tmpdir(), 'pinokio-hf-shells-'))
+  await fs.writeFile(path.join(root, 'ENVIRONMENT'), '')
+  let checks = 0
+  const kernel = createKernel(root)
+  kernel.bin = { envs: () => ({}) }
+  kernel.bracketedPasteSupport = {}
+  kernel.which = () => null
+  kernel.connect = {
+    connected: async () => {
+      checks++
+      return false
+    }
+  }
+  const shells = new Shells(kernel)
+  shells.ensureBracketedPasteSupport = async () => {}
+  try {
+    await shells.init()
+    const anonymousPath = path.join(root, 'cache', 'HF_AUTH', 'anonymous')
+    assert.equal(kernel.envs.HF_TOKEN_PATH, anonymousPath)
+    await shells.init()
+    assert.equal(checks, 1)
+
+    const inheritedShell = new Shell(kernel)
+    await inheritedShell.init_env({ env: {} })
+    assert.equal(inheritedShell.env.HF_TOKEN_PATH, anonymousPath)
+
+    const appShell = new Shell(kernel)
+    await appShell.init_env({ env: { HF_TOKEN_PATH: '/tmp/app-hf-token' } })
+    assert.equal(appShell.env.HF_TOKEN_PATH, '/tmp/app-hf-token')
+  } finally {
+    await fs.rm(root, { recursive: true, force: true })
+  }
+})

package/kernel/vault/hash_worker.js

@@ -1,19 +0,0 @@
-const { parentPort } = require('worker_threads')
-const crypto = require('crypto')
-const fs = require('fs')
-
-parentPort.on('message', ({ id, filePath }) => {
-  const hash = crypto.createHash('sha256')
-  let size = 0
-  const stream = fs.createReadStream(filePath)
-  stream.on('data', (chunk) => {
-    size += chunk.length
-    hash.update(chunk)
-  })
-  stream.on('error', (error) => {
-    parentPort.postMessage({ id, error: error.message })
-  })
-  stream.on('end', () => {
-    parentPort.postMessage({ id, hash: hash.digest('hex'), size })
-  })
-})